diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c11f1066 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,18 @@ +.git +__pycache__ +*.pyc +data +uploads +config +.env +dist +build + +# Keep the runtime image context minimal +app/ +scripts/ +installer/ +tests/ +examples/ +.github/ +*.md diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..cfb94589 --- /dev/null +++ b/.env.example @@ -0,0 +1,12 @@ +# Host process identity and timezone. The ONLY service settings that live +# here; every other setting comes from config/config.yaml (copy the +# config.example.yaml template). Do not commit real values. +PUID=1000 +PGID=1000 +TZ=Europe/Istanbul + +# Host-side bind ports for the loopback port mapping. The service itself +# binds 0.0.0.0 inside the container; keep 127.0.0.1 unless a reverse +# proxy or private network explicitly publishes the ports. +TELEDRIVE_WEB_PORT=8080 +TELEDRIVE_S3_PORT=9000 diff --git a/.github/ci/fault-injection/Dockerfile b/.github/ci/fault-injection/Dockerfile new file mode 100644 index 00000000..99612944 --- /dev/null +++ b/.github/ci/fault-injection/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.12.14-slim@sha256:eeb8088e67610b37583880c7627e3931f087cba55a35810819e34a398f624a47 +ENV PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 PYTHONPATH=/app +WORKDIR /app +COPY requirements.txt /app/requirements-server.txt +RUN pip install --no-cache-dir -r /app/requirements-server.txt +COPY teledrive /app/teledrive +COPY scripts /app/scripts +COPY tests /app/tests +USER 10001:10001 diff --git a/.github/ci/fault-injection/Dockerfile.dockerignore b/.github/ci/fault-injection/Dockerfile.dockerignore new file mode 100644 index 00000000..7d21eaca --- /dev/null +++ b/.github/ci/fault-injection/Dockerfile.dockerignore @@ -0,0 +1,20 @@ +.git +__pycache__/ +*.pyc +data/ +uploads/ +config/ +.env +*.session +*.session-journal +*.log +dist/ +build/ +.venv/ +website/ +app/ +installer/ +examples/ +.github/ +*.md +.docker-fault-evidence/ diff --git a/.github/ci/fault-injection/compose.yml b/.github/ci/fault-injection/compose.yml new file mode 100644 index 00000000..f2cfc49d --- /dev/null +++ b/.github/ci/fault-injection/compose.yml @@ -0,0 +1,87 @@ +# Compose file is under .github/ci/fault-injection; ../../.. is the repo root. +# Isolated Docker fault-injection and acceptance test ONLY. No host ports, no production credentials, +# no real Telegram messages, unique Compose project/ephemeral workspace. +x-web: &web + build: + context: ../../.. + dockerfile: Dockerfile + user: "${TELEDRIVE_FAULT_UID:?}:${TELEDRIVE_FAULT_GID:?}" + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + environment: &web-env + TELEDRIVE_CONFIG: /config/config.yaml + # All other settings live in /config/config.yaml generated by the runner. + depends_on: + postgres: + condition: service_healthy + networks: [dev-isolated] + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request;urllib.request.urlopen('http://127.0.0.1:8080/api/health',timeout=4)"] + interval: 5s + timeout: 5s + retries: 24 + start_period: 15s + +services: + postgres: + image: postgres:17-alpine + environment: + POSTGRES_USER: teledrive + POSTGRES_DB: teledrive_dev + POSTGRES_PASSWORD: ${TELEDRIVE_FAULT_DB_PASSWORD:?} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U teledrive -d teledrive_dev"] + interval: 3s + timeout: 3s + retries: 30 + networks: [dev-isolated] + volumes: + - db-state:/var/lib/postgresql/data + security_opt: ["no-new-privileges:true"] + # No ports: never expose the test database on the host. + web-a: + <<: *web + volumes: + - ${TELEDRIVE_FAULT_WORKSPACE:?}/config:/config + - ${TELEDRIVE_FAULT_WORKSPACE:?}/web-a:/data + - ${TELEDRIVE_FAULT_WORKSPACE:?}/uploads:/uploads:ro + web-b: + <<: *web + volumes: + - ${TELEDRIVE_FAULT_WORKSPACE:?}/config:/config + - ${TELEDRIVE_FAULT_WORKSPACE:?}/web-b:/data + - ${TELEDRIVE_FAULT_WORKSPACE:?}/uploads:/uploads:ro + scenarios: + build: + context: ../../.. + dockerfile: .github/ci/fault-injection/Dockerfile + user: "${TELEDRIVE_FAULT_UID:?}:${TELEDRIVE_FAULT_GID:?}" + depends_on: + postgres: + condition: service_healthy + web-a: + condition: service_healthy + web-b: + condition: service_healthy + environment: + TELEDRIVE_FAULT_DB_URL: postgresql+psycopg://teledrive:${TELEDRIVE_FAULT_DB_PASSWORD:?}@postgres:5432/teledrive_dev + TELEDRIVE_FAULT_RESTORED_DB_URL: postgresql+psycopg://teledrive:${TELEDRIVE_FAULT_DB_PASSWORD:?}@postgres:5432/teledrive_restore + TELEDRIVE_FAULT_WEB_PASSWORD: ${TELEDRIVE_FAULT_WEB_PASSWORD:?} + TELEDRIVE_FAULT_REMOTE_DIR: /work-data/remote + TELEDRIVE_FAULT_EVIDENCE_DIR: /work-data/evidence + TELEDRIVE_FAULT_SOURCES: /work-data/sources + volumes: + - ${TELEDRIVE_FAULT_WORKSPACE:?}/scenario:/work-data + networks: [dev-isolated] + security_opt: ["no-new-privileges:true"] + cap_drop: [ALL] + read_only: true + tmpfs: + - /tmp:rw,nosuid,nodev,size=128m + entrypoint: ["python", "scripts/dev/docker_fault_scenarios.py"] +networks: + dev-isolated: + internal: true +volumes: + db-state: diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..5d2dc623 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + open-pull-requests-limit: 5 diff --git a/.github/workflows/delivery-release.yml b/.github/workflows/delivery-release.yml new file mode 100644 index 00000000..7fff7a2a --- /dev/null +++ b/.github/workflows/delivery-release.yml @@ -0,0 +1,213 @@ +name: 05 · Delivery | Release Gate & Packages + +on: + push: + tags: + - "v*.*.*" + # On these branch updates validate the release gate without publishing. + branches: + - main + +permissions: + contents: read + +jobs: + verify: + name: Release gate · regression and security + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out source and history + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + fetch-depth: 0 + persist-credentials: false + - name: Require release tags to point into main + if: github.ref_type == 'tag' + shell: bash + run: | + set -euo pipefail + git show-ref --verify --quiet refs/remotes/origin/main + if ! git merge-base --is-ancestor HEAD refs/remotes/origin/main; then + echo "::error::Release tag does not point to a commit reachable from main" + exit 1 + fi + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: requirements.txt + - name: Install test runtime + run: | + python -m pip install -r requirements.txt + python -m pip install pytest httpx bandit pip-audit boto3 + python -m pip check + - name: Compile application + run: python -m compileall -q teledrive web tests + - name: Offline storage, API, recovery and security regression tests + run: | + python -m pytest -q \ + tests/web/test_security.py tests/web/test_openapi.py \ + tests/storage/test_storage_engine.py \ + tests/storage/test_storage_recovery.py tests/storage/test_storage_rebuild.py tests/storage/test_storage_pending_admin.py tests/storage/test_storage_jobs.py tests/storage/test_storage_versions.py tests/storage/test_storage_ops_health.py tests/meta/test_host_soak.py tests/meta/test_acceptance_cli.py tests/storage/test_storage_gc_retention.py tests/s3/test_s3_version_reads.py tests/storage/test_storage_catalog.py \ + tests/storage/test_storage_user_mode.py tests/storage/test_mtproto_rate_guard.py tests/storage/test_mtproto_multi_credentials.py tests/storage/test_telegram_dr_rehearsal.py tests/meta/test_production_bundle.py tests/meta/test_capacity_probe.py tests/storage/test_legacy_cutover_acceptance.py tests/storage/test_telegram_target_resolution.py tests/meta/test_ci_workflows.py tests/meta/test_release_matrix.py \ + tests/web/test_api_keys.py tests/web/test_rest_api.py tests/web/test_rest_transfer.py tests/web/test_unified_service.py \ + tests/s3/test_s3_fail_closed.py tests/s3/test_s3_contract.py tests/storage/test_metadata_backup.py tests/storage/test_storage_sigkill.py tests/storage/test_metadata_snapshot_rehearsal.py tests/s3/test_s3_multipart_preconditions.py + - name: Security checks + run: | + bandit -q -r teledrive web -ll + pip-audit -r requirements.txt + + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 05 · Delivery | Release Gate & Packages' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Mainline release gate; no publishing on branch push |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" + + cleanup-merged-branches: + name: Housekeeping · delete merged PR branches + needs: verify + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: write + pull-requests: read + steps: + - name: Checkout reviewed cleanup script + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Remove only closed and merged same-head PR branches + env: + GH_TOKEN: ${{ github.token }} + run: python scripts/ops/cleanup_merged_branches.py --execute + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 05 · Housekeeping | Merged PR branches' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Deleted only verified merged, unchanged and unprotected branches |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" + + build: + name: Package · single-container Docker image + needs: verify + if: github.ref_type == 'tag' || (github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[publish-release]')) + runs-on: ubuntu-latest + steps: + - name: Check out source + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + + - name: Build the single teledrive service image + shell: bash + run: | + mkdir -p dist + docker build -t teledrive:service . + docker save -o dist/teledrive-service-image.tar teledrive:service + + - name: Upload package artifact + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: TeleDrive-docker-image + path: dist/teledrive-service-image.tar + if-no-files-found: error + + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 05 · Delivery | Release Gate & Packages' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Single-container Docker image package |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" + + publish: + name: Publish · GitHub Release + needs: build + if: github.event_name == 'push' && (github.ref_type == 'tag' || (github.ref == 'refs/heads/main' && contains(github.event.head_commit.message, '[publish-release]'))) + runs-on: ubuntu-latest + timeout-minutes: 25 + permissions: + contents: write + actions: read + steps: + - name: Checkout approved release source + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + + - name: Verify complete same-SHA P0/P1 matrix + if: github.ref_type != 'tag' + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + ready=0 + for attempt in $(seq 1 30); do + gh api "repos/${GITHUB_REPOSITORY}/actions/runs?head_sha=${GITHUB_SHA}&per_page=100" > "$RUNNER_TEMP/release-runs.json" + set +e + python scripts/ops/check_release_matrix.py --input "$RUNNER_TEMP/release-runs.json" --sha "$GITHUB_SHA" + result=$? + set -e + if [[ "$result" -eq 0 ]]; then ready=1; break; fi + if [[ "$result" -eq 1 ]]; then exit 1; fi + sleep 20 + done + if [[ "$ready" -ne 1 ]]; then echo "::error::Release blocked: incomplete same-SHA acceptance"; exit 1; fi + + - name: Download built packages + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 + with: + path: release-assets + merge-multiple: true + + - name: Publish release and attach packages + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + version="$(python -c 'import tomllib; print(tomllib.load(open("pyproject.toml","rb"))["project"]["version"])')" + tag="v$version" + if [[ "$GITHUB_REF_TYPE" == "tag" && "$GITHUB_REF_NAME" != "$tag" ]]; then + echo "::error::Tag/version mismatch"; exit 1 + fi + if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + if [[ "$GITHUB_REF_TYPE" != "tag" ]]; then + echo "::error::Existing release cannot be overwritten"; exit 1 + fi + gh release upload "$tag" release-assets/* --repo "$GITHUB_REPOSITORY" --clobber + else + gh release create "$tag" release-assets/* \ + --repo "$GITHUB_REPOSITORY" --target "$GITHUB_SHA" \ + --generate-notes \ + --title "TeleDrive $tag" + fi + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 05 · Delivery | Release Gate & Packages' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Version-tagged GitHub release assets |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/endurance-capacity.yml b/.github/workflows/endurance-capacity.yml new file mode 100644 index 00000000..92e1e363 --- /dev/null +++ b/.github/workflows/endurance-capacity.yml @@ -0,0 +1,60 @@ +name: 06 · Endurance | Isolated Capacity Soak + +on: + workflow_dispatch: + inputs: + duration_minutes: + description: "Isolated endurance duration (minimum 60 minutes)" + required: true + default: "60" + type: string + +permissions: + contents: read + +jobs: + isolated-endurance: + name: 60m+ SQL / object-engine endurance + runs-on: ubuntu-latest + timeout-minutes: 125 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + - name: Install application + run: | + python -m pip install -e . + python -m pip check + - name: Verify one-hour minimum and run isolated capacity test + env: + REQUEST_MINUTES: ${{ inputs.duration_minutes }} + run: | + set -euo pipefail + if ! [[ "$REQUEST_MINUTES" =~ ^[0-9]{1,3}$ ]]; then exit 2; fi + if (( 10#$REQUEST_MINUTES < 60 || 10#$REQUEST_MINUTES > 110 )); then exit 2; fi + python scripts/ops/capacity_probe.py \ + --duration-seconds "$((10#$REQUEST_MINUTES * 60))" \ + --object-bytes 65536 \ + --output "$RUNNER_TEMP/capacity-evidence.json" + - name: Preserve isolated, non-production evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: capacity-evidence + path: ${{ runner.temp }}/capacity-evidence.json + if-no-files-found: warn + retention-days: 14 + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 06 · Endurance | Isolated Capacity Soak' >> "$GITHUB_STEP_SUMMARY" + echo '| Field | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Type | Synthetic Telegram, real SQLite; NOT live production |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/integration-databases.yml b/.github/workflows/integration-databases.yml new file mode 100644 index 00000000..ad4550eb --- /dev/null +++ b/.github/workflows/integration-databases.yml @@ -0,0 +1,102 @@ +name: 02 · Integration | Databases + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +jobs: + database: + name: Data · PostgreSQL / MySQL / restore + runs-on: ubuntu-latest + timeout-minutes: 25 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: teledrive + POSTGRES_PASSWORD: test-password + POSTGRES_DB: teledrive + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U teledrive -d teledrive" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + mysql: + image: mysql:8.4 + env: + MYSQL_ROOT_PASSWORD: root-test-password + MYSQL_DATABASE: teledrive + MYSQL_USER: teledrive + MYSQL_PASSWORD: test-password + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h localhost -u root -proot-test-password" + --health-interval 5s + --health-timeout 5s + --health-retries 30 + env: + DB_TEST_POSTGRES_URL: postgresql+psycopg://teledrive:test-password@127.0.0.1:5432/teledrive + DB_TEST_MYSQL_URL: mysql+pymysql://teledrive:test-password@127.0.0.1:3306/teledrive + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + - name: Install application and DB drivers + run: | + python -m pip install -e . + python -m pip install pytest "psycopg[binary]>=3.2,<4" "PyMySQL>=1.1,<2" "defusedxml==0.7.1" "cryptography>=50,<51" + python -m pip check + - name: Run live PostgreSQL/MySQL integration tests + run: python -m pytest -vv tests/e2e/test_database_live.py tests/storage/test_storage_engine.py tests/storage/test_storage_recovery.py + - name: Seed realistic metadata before native database backup + run: | + python scripts/ops/rehearse_native_database_restore.py seed --url-env DB_TEST_POSTGRES_URL --receipt "$RUNNER_TEMP/pg-dr.json" --source "$RUNNER_TEMP/pg-source.bin" + python scripts/ops/rehearse_native_database_restore.py seed --url-env DB_TEST_MYSQL_URL --receipt "$RUNNER_TEMP/mysql-dr.json" --source "$RUNNER_TEMP/mysql-source.bin" + - name: Native PostgreSQL 17 pg_dump and isolated pg_restore + run: | + set -euo pipefail + docker run --rm --network host -e PGPASSWORD=test-password -v "$RUNNER_TEMP:/snap" postgres:17-alpine \ + pg_dump --host=127.0.0.1 --username=teledrive --dbname=teledrive --format=custom --file=/snap/pg-ci.backup --no-owner + docker run --rm --network host -e PGPASSWORD=test-password postgres:17-alpine \ + psql --host=127.0.0.1 --username=teledrive --dbname=teledrive --set=ON_ERROR_STOP=1 \ + --command="CREATE DATABASE teledrive_dr_ci" + docker run --rm --network host -e PGPASSWORD=test-password -v "$RUNNER_TEMP:/snap" postgres:17-alpine \ + pg_restore --host=127.0.0.1 --username=teledrive --dbname=teledrive_dr_ci --no-owner --no-privileges --exit-on-error /snap/pg-ci.backup + python scripts/ops/rehearse_native_database_restore.py verify --url-env DB_TEST_POSTGRES_URL --restore-database teledrive_dr_ci --receipt "$RUNNER_TEMP/pg-dr.json" + - name: Native MySQL 8.4 mysqldump and isolated restore + run: | + set -euo pipefail + docker run --rm --network host -e MYSQL_PWD=test-password mysql:8.4 \ + mysqldump --host=127.0.0.1 --user=teledrive --single-transaction --quick --no-tablespaces --set-gtid-purged=OFF teledrive > "$RUNNER_TEMP/mysql-ci.sql" + docker run --rm --network host -e MYSQL_PWD=root-test-password mysql:8.4 \ + mysql --host=127.0.0.1 --user=root --execute="CREATE DATABASE teledrive_dr_ci CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; GRANT ALL PRIVILEGES ON teledrive_dr_ci.* TO 'teledrive'@'%';" + docker run --rm --interactive --network host -e MYSQL_PWD=test-password mysql:8.4 \ + mysql --host=127.0.0.1 --user=teledrive --database=teledrive_dr_ci < "$RUNNER_TEMP/mysql-ci.sql" + python scripts/ops/rehearse_native_database_restore.py verify --url-env DB_TEST_MYSQL_URL --restore-database teledrive_dr_ci --receipt "$RUNNER_TEMP/mysql-dr.json" + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 02 · Integration | Databases' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Live PostgreSQL/MySQL and native restore rehearsals |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/integration-storage.yml b/.github/workflows/integration-storage.yml new file mode 100644 index 00000000..c835f894 --- /dev/null +++ b/.github/workflows/integration-storage.yml @@ -0,0 +1,120 @@ +name: 02 · Integration | REST & S3 + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: storage-api-${{ github.ref }} + cancel-in-progress: true + +jobs: + storage-api: + name: Storage · REST / S3 / AWS CLI / rclone + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + - name: Install API and test runtime + run: | + python -m pip install -r requirements.txt + python -m pip install pytest httpx boto3 + python -m pip check + - name: Install actual AWS CLI and rclone clients + run: | + python -m pip install awscli + sudo apt-get update -qq + sudo apt-get install -y --no-install-recommends rclone + aws --version + rclone version + - name: Compile API and storage + run: python -m compileall -q teledrive web tests + - name: Check browser API management scripts + run: node --check web/static/app.js + - name: Actual AWS CLI and rclone process integration + env: + TELEDRIVE_REQUIRE_CLI_CLIENTS: "1" + run: python -m pytest -vv tests/s3/test_s3_cli_compat.py + - name: Real SQL and HTTP contract tests with fake Telegram I/O + run: | + python -m pytest -q \ + tests/web/test_api_keys.py \ + tests/web/test_rest_api.py \ + tests/web/test_rest_transfer.py tests/web/test_unified_service.py \ + tests/storage/test_storage_engine.py \ + tests/storage/test_storage_user_mode.py \ + tests/storage/test_storage_recovery.py tests/storage/test_storage_rebuild.py tests/storage/test_telegram_inventory_reconcile.py \ + tests/storage/test_storage_pending_admin.py \ + tests/storage/test_storage_jobs.py \ + tests/storage/test_legacy_bridge.py \ + tests/storage/test_legacy_import_admin.py \ + tests/storage/test_metadata_backup.py \ + tests/storage/test_storage_sigkill.py \ + tests/storage/test_storage_multiworker.py \ + tests/storage/test_metadata_snapshot_rehearsal.py \ + tests/storage/test_storage_versions.py \ + tests/storage/test_storage_ops_health.py tests/meta/test_host_soak.py tests/meta/test_acceptance_cli.py \ + tests/storage/test_storage_gc_retention.py tests/storage/test_gc_reference_planner.py \ + tests/s3/test_s3_version_reads.py tests/s3/test_s3_bucket_versioning.py tests/s3/test_s3_public_access_block.py tests/s3/test_s3_lifecycle.py \ + tests/s3/test_s3_delete_markers.py \ + tests/s3/test_s3_version_listing.py \ + tests/s3/test_s3_version_tagging.py \ + tests/s3/test_s3_version_copy.py \ + tests/s3/test_s3_vhost.py \ + tests/s3/test_s3_additional_checksums.py \ + tests/s3/test_s3_presigned_complete.py \ + tests/s3/test_s3_post_policy.py \ + tests/storage/test_storage_catalog.py \ + tests/storage/test_storage_user_mode.py \ + tests/storage/test_mtproto_rate_guard.py tests/storage/test_mtproto_multi_credentials.py \ + tests/storage/test_telegram_dr_rehearsal.py \ + tests/storage/test_legacy_cutover_acceptance.py \ + tests/s3/test_s3_gateway.py tests/s3/test_s3_bucket_listing.py \ + tests/s3/test_s3_credentials.py \ + tests/s3/test_s3_admin.py \ + tests/s3/test_s3_multipart.py \ + tests/s3/test_s3_multipart_preconditions.py \ + tests/s3/test_s3_recovery.py \ + tests/s3/test_s3_presigned.py \ + tests/s3/test_s3_copy.py \ + tests/s3/test_s3_tagging.py \ + tests/s3/test_s3_batch_delete.py \ + tests/s3/test_s3_metadata.py \ + tests/s3/test_s3_bucket_location.py \ + tests/s3/test_s3_listing_v1.py \ + tests/s3/test_s3_conditions.py \ + tests/s3/test_s3_conditional_writes.py \ + tests/s3/test_s3_cors.py \ + tests/s3/test_s3_object_attributes.py \ + tests/s3/test_s3_sigv4_defense.py \ + tests/s3/test_s3_fail_closed.py tests/s3/test_s3_route_guard.py tests/s3/test_s3_policy_status.py \ + tests/s3/test_s3_contract.py \ + tests/s3/test_s3_permissions.py \ + tests/storage/test_storage_gc_audit.py \ + tests/storage/test_storage_gc_admin.py + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 02 · Integration | REST & S3' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | REST and S3 protocol; real AWS CLI and rclone processes |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/live-telegram-multiworker.yml b/.github/workflows/live-telegram-multiworker.yml new file mode 100644 index 00000000..d9fb26c5 --- /dev/null +++ b/.github/workflows/live-telegram-multiworker.yml @@ -0,0 +1,91 @@ +name: 04 · Live Telegram | Multiworker PostgreSQL E2E + +on: + workflow_dispatch: + push: + branches: + - main + +permissions: + contents: read + +# P1 acceptance includes real multiworker verification on every main push. +# The USER MTProto live queues stay isolated from each other; GitHub cancels +# the older pending job when three runs share a concurrency group, even with +# cancel-in-progress: false. Both suites still use limited test traffic. +jobs: + three-process-telegram: + name: LIVE · 3 workers, shared SQL and Telegram + concurrency: + group: teledrive-live-multiworker-${{ github.repository }} + cancel-in-progress: false + runs-on: ubuntu-latest + timeout-minutes: 25 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: teledrive + POSTGRES_PASSWORD: test-password + POSTGRES_DB: teledrive + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U teledrive -d teledrive" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + DB_TEST_POSTGRES_URL: postgresql+psycopg://teledrive:test-password@127.0.0.1:5432/teledrive + TELEDRIVE_E2E_SESSION: ${{ secrets.TELEDRIVE_E2E_SESSION }} + TELEDRIVE_E2E_API_ID: ${{ secrets.TELEDRIVE_E2E_API_ID }} + TELEDRIVE_E2E_API_HASH: ${{ secrets.TELEDRIVE_E2E_API_HASH }} + TELEDRIVE_E2E_TARGET: ${{ secrets.TELEDRIVE_E2E_TARGET }} + TELEDRIVE_E2E_CHANNEL_NAME: ${{ secrets.TELEDRIVE_E2E_CHANNEL_NAME }} + TELEDRIVE_E2E_MULTIWORKER: "1" + steps: + # The channel ID must come from an Actions SECRET, never a repository + # variable: vars can be printed unredacted in a step's initial env dump. + # Install masks before checkout, dependency installation, or pytest. + - name: Mask dedicated Telegram test target + shell: bash + run: | + if [[ -n "$TELEDRIVE_E2E_TARGET" ]]; then + echo "::add-mask::$TELEDRIVE_E2E_TARGET" + fi + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + - name: Configure dedicated live USER credentials + run: | + if [[ -z "$TELEDRIVE_E2E_SESSION" || -z "$TELEDRIVE_E2E_API_ID" || -z "$TELEDRIVE_E2E_API_HASH" ]]; then + echo "::error::Dedicated test USER MTProto credentials are required." + exit 1 + fi + if [[ -z "$TELEDRIVE_E2E_TARGET" && -z "$TELEDRIVE_E2E_CHANNEL_NAME" ]]; then + echo "::error::Configure TELEDRIVE_E2E_TARGET or TELEDRIVE_E2E_CHANNEL_NAME as a repository Actions secret." + exit 1 + fi + - name: Install application, DB driver and test runtime + run: | + python -m pip install -e . + python -m pip install pytest "psycopg[binary]>=3.2,<4" + python -m pip check + - name: Three independent SQL workers and real Telegram byte restoration + run: python -m pytest -vv tests/e2e/test_telegram_multiworker_e2e.py + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 04 · Live Telegram | Multiworker PostgreSQL E2E' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Three SQL worker processes and real Telegram test channel |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/live-telegram-user.yml b/.github/workflows/live-telegram-user.yml new file mode 100644 index 00000000..0b128ecf --- /dev/null +++ b/.github/workflows/live-telegram-user.yml @@ -0,0 +1,86 @@ +name: 04 · Live Telegram | User MTProto E2E + +on: + workflow_dispatch: + inputs: + run_stress: + description: "Run 5/16 MiB real Telethon chunk upload and recovery" + required: false + default: false + type: boolean + push: + branches: + - main + +permissions: + contents: read + +concurrency: + group: teledrive-live-user-${{ github.repository }} + # In-progress Telegram writes must finish their cleanup. + cancel-in-progress: false + +jobs: + telegram-roundtrip: + name: LIVE · Telethon upload and range read + runs-on: ubuntu-latest + timeout-minutes: 15 + env: + TELEDRIVE_E2E_API_ID: ${{ secrets.TELEDRIVE_E2E_API_ID }} + TELEDRIVE_E2E_API_HASH: ${{ secrets.TELEDRIVE_E2E_API_HASH }} + TELEDRIVE_E2E_SESSION: ${{ secrets.TELEDRIVE_E2E_SESSION }} + TELEDRIVE_E2E_TARGET: ${{ secrets.TELEDRIVE_E2E_TARGET }} + TELEDRIVE_E2E_CHANNEL_NAME: ${{ secrets.TELEDRIVE_E2E_CHANNEL_NAME }} + TELEDRIVE_E2E_STRESS: ${{ ((github.event_name == 'workflow_dispatch' && inputs.run_stress) || (github.event_name == 'push' && contains(github.event.head_commit.message, '[p1-live-size]'))) && '1' || '0' }} + steps: + # Numeric Telegram peers must be masked before Python or any + # diagnostic command runs, never sourced from visible repo vars. + - name: Mask optional Telegram user target + shell: bash + run: | + if [[ -n "$TELEDRIVE_E2E_TARGET" ]]; then + echo "::add-mask::$TELEDRIVE_E2E_TARGET" + fi + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + - name: Check user E2E session configuration + shell: bash + id: configured + run: | + if [[ -z "$TELEDRIVE_E2E_API_ID" || -z "$TELEDRIVE_E2E_API_HASH" || -z "$TELEDRIVE_E2E_SESSION" ]]; then + echo "::error::User Telegram E2E not configured; run the local setup script once." + exit 1 + elif [[ -z "$TELEDRIVE_E2E_CHANNEL_NAME" && -z "$TELEDRIVE_E2E_TARGET" ]]; then + echo "::error::User Telegram E2E target not configured." + exit 1 + else + echo "enabled=true" >> "$GITHUB_OUTPUT" + fi + - name: Install storage dependencies + if: steps.configured.outputs.enabled == 'true' + run: | + python -m pip install -e . + python -m pip install pytest + - name: Real Telegram upload, retrieval, checksum and Range + if: steps.configured.outputs.enabled == 'true' + run: python -m pytest -q tests/e2e/test_telegram_storage_e2e.py + - name: Live Telegram marker inventory with owned-fixture cleanup + if: steps.configured.outputs.enabled == 'true' + run: python -m pytest -q tests/e2e/test_telegram_history_live_e2e.py + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 04 · Live Telegram | User MTProto E2E' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Dedicated Telegram user session; byte readback and Range |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/quality-security.yml b/.github/workflows/quality-security.yml new file mode 100644 index 00000000..5f6a5578 --- /dev/null +++ b/.github/workflows/quality-security.yml @@ -0,0 +1,144 @@ +name: 01 · Quality | Security & Regression + +on: + workflow_dispatch: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: security-${{ github.ref }} + cancel-in-progress: true + +jobs: + security: + name: Security · regression, dependencies and image scan + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + - name: Install + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install pytest httpx bandit pip-audit boto3 + - name: Compile + run: python -m compileall -q teledrive web tests + - name: Packaged import and entrypoint + run: | + python -m pip install --no-deps -e . + python -c "import teledrive.db, teledrive.uploader; print('package-import-ok')" + + - name: Unit tests + run: python -m pytest -q tests/web/test_security.py tests/web/test_openapi.py tests/storage/test_storage_engine.py tests/storage/test_storage_user_mode.py tests/storage/test_storage_recovery.py tests/storage/test_storage_rebuild.py tests/storage/test_telegram_history_inventory.py tests/storage/test_telegram_inventory_reconcile.py tests/storage/test_storage_pending_admin.py tests/storage/test_storage_jobs.py tests/storage/test_legacy_bridge.py tests/storage/test_legacy_import_admin.py tests/storage/test_storage_versions.py tests/storage/test_storage_ops_health.py tests/meta/test_host_soak.py tests/meta/test_acceptance_cli.py tests/storage/test_storage_gc_retention.py tests/storage/test_gc_reference_planner.py tests/s3/test_s3_version_reads.py tests/s3/test_s3_bucket_versioning.py tests/s3/test_s3_public_access_block.py tests/s3/test_s3_lifecycle.py tests/s3/test_s3_tagging.py tests/s3/test_s3_bucket_listing.py tests/storage/test_storage_user_mode.py tests/storage/test_mtproto_rate_guard.py tests/storage/test_mtproto_multi_credentials.py tests/storage/test_telegram_dr_rehearsal.py tests/meta/test_production_bundle.py tests/meta/test_capacity_probe.py tests/storage/test_legacy_cutover_acceptance.py tests/storage/test_telegram_target_resolution.py tests/meta/test_ci_workflows.py tests/meta/test_release_matrix.py tests/storage/test_storage_catalog.py tests/web/test_api_keys.py tests/web/test_rest_api.py tests/web/test_unified_service.py tests/web/test_rest_transfer.py tests/s3/test_s3_sigv4_defense.py tests/s3/test_s3_fail_closed.py tests/s3/test_s3_route_guard.py tests/s3/test_s3_policy_status.py tests/s3/test_s3_contract.py tests/storage/test_metadata_backup.py tests/storage/test_storage_sigkill.py tests/storage/test_metadata_snapshot_rehearsal.py tests/s3/test_s3_multipart_preconditions.py + - name: Dependency consistency + run: python -m pip check + - name: Bandit + run: bandit -q -r teledrive web -ll + - name: Dependency audit + run: pip-audit -r requirements.txt + - name: Docker Compose config + env: + TELEDRIVE_WEB_PASSWORD: ci-only-not-a-real-secret + run: | + docker compose config --quiet + + - name: Docker build + run: docker build --pull -t teledrive-security-ci . + + - name: Single-container Web UI and S3 gateway smoke + shell: bash + run: | + set -euo pipefail + set -euo pipefail + mkdir -p /tmp/ci-config /tmp/ci-data + chmod 700 /tmp/ci-config /tmp/ci-data + cat > /tmp/ci-config/config.yaml <<'EOF' + web: + user: ci + password: ci-only-not-a-real-secret + allowed_hosts: "localhost,127.0.0.1,[::1]" + storage: + channel: "" + apis: [] + state: + database_url: "sqlite:////data/state.db" + EOF + chmod 600 /tmp/ci-config/config.yaml + cid="$(docker run -d --rm \ + --user "$(id -u):$(id -g)" \ + -v /tmp/ci-config:/config \ + -v /tmp/ci-data:/data \ + -p 127.0.0.1:18080:8080 \ + -p 127.0.0.1:19000:9000 \ + teledrive-security-ci)" + trap 'docker rm -f "$cid" >/dev/null 2>&1 || true' EXIT + ready=0 + for i in $(seq 1 45); do + if curl -fsS -u ci:ci-only-not-a-real-secret \ + http://127.0.0.1:18080/api/dashboard >/tmp/dashboard.json \ + && curl -fsS http://127.0.0.1:19000/__health >/tmp/s3-health.json; then + ready=1 + break + fi + sleep 1 + done + if [ "$ready" -ne 1 ]; then docker logs "$cid" | tail -60; exit 1; fi + python - <<'PY' + import json + data = json.load(open("/tmp/dashboard.json", encoding="utf-8")) + assert "queue" in data and "accounts" in data and "database_url" in data + assert json.load(open("/tmp/s3-health.json", encoding="utf-8")) == {"ok": True} + PY + curl -fsS -u ci:ci-only-not-a-real-secret \ + http://127.0.0.1:18080/openapi.json >/tmp/openapi.json + python - <<'PY' + import json + schema=json.load(open("/tmp/openapi.json",encoding="utf-8")) + assert "/api/auth" in schema["paths"] + PY + if curl -fsS http://127.0.0.1:18080/openapi.json >/dev/null; then + echo "Anonymous Web API unexpectedly available" >&2 + exit 1 + fi + if curl -fsS http://127.0.0.1:18080/__health >/dev/null; then + echo "S3 listener accidentally accessible through Web port" >&2 + exit 1 + fi + if curl -fsS http://127.0.0.1:19000/api/health >/dev/null; then + echo "Web listener accidentally accessible through S3 port" >&2 + exit 1 + fi + - name: Trivy image scan + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 + with: + image-ref: teledrive-security-ci + format: table + exit-code: "1" + ignore-unfixed: true + vuln-type: os,library + severity: CRITICAL,HIGH + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 01 · Quality | Security & Regression' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Offline regression; dependency and container security tests |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index fef569f3..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Build and publish release - -on: - push: - tags: - - "v*.*.*" - -permissions: - contents: write - -jobs: - build: - name: Build on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - include: - - os: ubuntu-latest - artifact_name: TeleDrive-linux-x86_64 - asset_path: dist-installer/TeleDrive-linux-x86_64.tar.gz - - os: windows-latest - artifact_name: TeleDrive-windows - asset_path: dist-installer/*.exe - - steps: - - name: Check out source - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.13" - - - name: Build Linux package - if: runner.os == 'Linux' - shell: bash - run: bash build-linux.sh - - - name: Build Windows installer - if: runner.os == 'Windows' - shell: pwsh - run: | - $version = $env:GITHUB_REF_NAME.TrimStart('v') - $iss = Get-Content installer\TelegramAutoUpload.iss -Raw - $iss = [regex]::Replace($iss, '#define MyAppVersion ".*?"', ('#define MyAppVersion "' + $version + '"')) - Set-Content installer\TelegramAutoUpload.iss $iss -NoNewline - .\build.ps1 - .\build-installer.ps1 - - - name: Upload package artifact - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.artifact_name }} - path: ${{ matrix.asset_path }} - if-no-files-found: error - - publish: - name: Publish GitHub release - needs: build - runs-on: ubuntu-latest - steps: - - name: Download built packages - uses: actions/download-artifact@v4 - with: - path: release-assets - merge-multiple: true - - - name: Publish release and attach packages - env: - GH_TOKEN: ${{ github.token }} - run: | - if gh release view "$GITHUB_REF_NAME" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - gh release upload "$GITHUB_REF_NAME" release-assets/* \ - --repo "$GITHUB_REPOSITORY" \ - --clobber - else - gh release create "$GITHUB_REF_NAME" release-assets/* \ - --repo "$GITHUB_REPOSITORY" \ - --generate-notes \ - --title "TeleDrive $GITHUB_REF_NAME" - fi diff --git a/.github/workflows/resilience-docker.yml b/.github/workflows/resilience-docker.yml new file mode 100644 index 00000000..e578dffd --- /dev/null +++ b/.github/workflows/resilience-docker.yml @@ -0,0 +1,51 @@ +name: 03 · Resilience | Docker Fault Injection +on: + workflow_dispatch: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +concurrency: + group: teledrive-docker-fault-${{ github.repository }}-${{ github.ref }} + cancel-in-progress: false + +jobs: + disposable-docker-fault-injection: + name: Docker · 13 fault-injection checks + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - name: Checkout + uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 + with: + persist-credentials: false + - name: Verify Docker Compose + run: | + docker version + docker compose version + - name: Install two clean TeleDrive apps and inject recoverable faults + env: + TELEDRIVE_FAULT_REPORT_DIR: ${{ runner.temp }}/docker-fault-evidence + run: bash scripts/dev/run_docker_fault_injection.sh + - name: Publish sanitized acceptance report + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: docker-fault-injection-report + path: ${{ runner.temp }}/docker-fault-evidence/*.json + if-no-files-found: warn + retention-days: 14 + - name: Publish CI run summary + if: always() + shell: bash + run: | + echo '### 03 · Resilience | Docker Fault Injection' >> "$GITHUB_STEP_SUMMARY" + echo '| Detail | Value |' >> "$GITHUB_STEP_SUMMARY" + echo '| --- | --- |' >> "$GITHUB_STEP_SUMMARY" + echo '| Scope | Isolated 2-node API; PostgreSQL; crash, restore and teardown |' >> "$GITHUB_STEP_SUMMARY" + echo '| Result | ${{ job.status }} |' >> "$GITHUB_STEP_SUMMARY" + printf '| Ref | %s |\n' "$GITHUB_REF_NAME" >> "$GITHUB_STEP_SUMMARY" + printf '| Commit | %s |\n' "$GITHUB_SHA" >> "$GITHUB_STEP_SUMMARY" diff --git a/.gitignore b/.gitignore index f772f8d0..f8cd7763 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,13 @@ config.yaml *.session-journal *.log -# Website build and dependencies -website/node_modules/ -website/dist/ -website/.astro/ +.env +config/ +uploads/ +.opencode/ +.DS_Store +.idea/ +.vscode/ +# Reusable Docker fault-injection local reports (redacted local reports) +.docker-fault-evidence/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9278402d..4a6978c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,16 +1,43 @@ # Contributing to TeleDrive -Thanks for helping improve TeleDrive. +> **Audience:** maintainers and external contributors. This fork includes a single-container WebUI/REST/S3 service, a Telegram uploader and an Astro marketing site. -## Before opening a pull request +## 1. Before changing code -- Explain the problem and the proposed change. -- Keep changes focused and user-facing behavior clear. -- Do not include API credentials, Telegram sessions, personal paths, logs, or state databases. -- Run `python -m py_compile app\window.py` and test the affected workflow. -- Update the README when installation or behavior changes. +- Open a focused issue or describe the intended change in a pull request. +- Preserve the shared SQL/Telegram storage contract: REST, S3 and durable local-file jobs must not diverge into independent object catalogs. +- Keep Telegram physical deletion, legacy migration and production acceptance behind explicit, reviewed safety gates. +- Never commit `.env`, Telethon sessions, API hashes, Bot tokens, login codes, 2FA passwords, SQL credentials, original channel data or logs containing secrets. -## Pull requests +## 2. Local development and validation -Use a descriptive title and include verification steps. Small, focused pull requests -are easier to review and merge. +~~~bash +python -m pip install -r requirements.txt pytest +python -m pytest -q +python -m compileall -q teledrive web app tests +docker compose config --quiet +~~~ + +Run the dedicated [Docker fault-injection acceptance](docs/CI_WORKFLOWS.md) when changing upload durability, SQL state, scheduler behavior, recovery or container wiring: + +~~~bash +bash scripts/dev/run_docker_fault_injection.sh +~~~ + +For the command-line uploader, use the existing test suite and the [Docker fault-injection acceptance](docs/CI_WORKFLOWS.md) above. + +Live Telegram tests require the private test channel and secrets described in the [CI and Test Guide](docs/CI_WORKFLOWS.md); never point the test workflow at production media. + +## 3. Documentation standard + +- Write all public Markdown documentation in **English**. +- Use one `#` title per file, a short purpose statement, and an ordered set of sections. +- Put user installation and examples in [README](README.md); test details in [CI and Test Guide](docs/CI_WORKFLOWS.md); status/scope/production approval in [Project Status](docs/IMPLEMENTATION_STATUS.md); backup procedures in [Backup and Restore](docs/BACKUP_RESTORE.md). +- Keep [TODO.md](TODO.md) to actionable checkboxes; record release notes on [GitHub Releases](https://github.com/babico/TeleDrive/releases), not in duplicate versioned Markdown files. +- Update references when renaming files. Ensure relative Markdown links resolve and do not assert that a CI skip equals a passing live test. + +## 4. Review and security + +Explain what changed, how it was tested, and any unsupported or unverified behavior. Keep new tests named for the **behavior** they cover, not roadmap phases such as P0/P1/P2. + +For suspected vulnerabilities, use [Security Policy](docs/SECURITY.md) and disclose privately. Do not post exploit details or secrets in public issues. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fc08bbce --- /dev/null +++ b/Dockerfile @@ -0,0 +1,32 @@ +FROM python:3.12.14-slim@sha256:eeb8088e67610b37583880c7627e3931f087cba55a35810819e34a398f624a47 + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + TELEDRIVE_CONFIG=/config/config.yaml \ + TELEDRIVE_AUTH_DIR=/data/auth + +RUN groupadd --gid 10001 teledrive && useradd --uid 10001 --gid 10001 --no-create-home --shell /usr/sbin/nologin teledrive + +WORKDIR /app + +# Apply base-image security updates available from Debian before installing Python deps. +RUN apt-get update \ + && apt-get upgrade -y --no-install-recommends \ + && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /app/requirements-server.txt +ARG TELEDRIVE_DB_EXTRA_PACKAGES="" +RUN pip install --no-cache-dir -r /app/requirements-server.txt \ + && if [ -n "$TELEDRIVE_DB_EXTRA_PACKAGES" ]; then pip install --no-cache-dir $TELEDRIVE_DB_EXTRA_PACKAGES; fi + +COPY teledrive /app/teledrive +COPY web /app/web +RUN mkdir -p /config /data/sessions /data/auth /data/tmp /uploads \ + && chown -R 10001:10001 /config /data /uploads /app + +USER 10001:10001 + +EXPOSE 8080 9000 +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD python -c "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/api/health', timeout=3); urllib.request.urlopen('http://127.0.0.1:9000/__health', timeout=3)" || exit 1 + +CMD ["python","-m","web.unified","--host","0.0.0.0","--web-port","8080","--s3-port","9000"] diff --git a/README.md b/README.md index 1ebaa3f6..e5898b7f 100644 --- a/README.md +++ b/README.md @@ -1,348 +1,299 @@ -# TeleDrive (Telegram Auto Upload) +# TeleDrive -

- Buy Me A Coffee - License: MIT - Platform -

+**Telegram-backed personal file storage and automation** with a single-container Web UI/REST/S3 service and a command-line uploader. -Safe, slow, resumable Telegram file uploader for Windows and Linux. +TeleDrive stores object payloads in Telegram and keeps object metadata, upload state, and references in a local or external SQL database. This fork supports multiple Telegram accounts for the original folder uploader and a unified USER MTProto-backed object catalog for REST, S3, and durable local-file jobs. -TeleDrive turns Telegram into a personal cloud uploader. It scans a local folder, -queues files in SQLite, and uploads them gradually with retry and flood-wait handling. +> **Security:** ordinary Telegram cloud chats and Saved Messages are **not end-to-end encrypted**. Encrypt sensitive content before uploading if it must be confidential from Telegram. +> +> **Release boundary:** [v1.0.7](https://github.com/babico/TeleDrive/releases/tag/v1.0.7) is the published milestone. Newer `main` features may not be included in those binaries. Refer to [Project Status](docs/IMPLEMENTATION_STATUS.md) before treating unreleased features as production-ready. -## Desktop application +## Documentation -The production UI is a PySide6 desktop app. It stores configuration, the Telegram -session, logs, and SQLite state under: +| Guide | Use it for | +| --- | --- | +| **[README](README.md)** | Installation, configuration, primary workflows and supported interfaces | +| [CI and Test Guide](docs/CI_WORKFLOWS.md) | GitHub Actions, local tests, Docker fault injection and development acceptance | +| [Project Status and Acceptance](docs/IMPLEMENTATION_STATUS.md) | P0/P1/P2 capabilities, release boundaries, original-data production acceptance and recovery risks | +| [Backup and Restore](docs/BACKUP_RESTORE.md) | Database backup commands, isolation and restore checks | +| [Security Policy](docs/SECURITY.md) | Responsible disclosure, trusted components, credentials and network isolation | +| [Contributing](CONTRIBUTING.md) | Development workflow and documentation rules | +| [S3 Compatibility Matrix](docs/S3_SPECS.md) | 116-operation status matrix, supported limits and P2 advanced implementation roadmap | +| [Local Agent Acceptance Mission](docs/LOCAL_AGENT_ACCEPTANCE.md) | Docker test installation, real bytes, independent recovery and GC safety acceptance, with owner-only gates | +| [Working Checklist](TODO.md) | Short, actionable outstanding tasks | -```text -Windows: `%LOCALAPPDATA%\Telegram Auto Upload` -Linux: `~/.local/share/Telegram Auto Upload` -``` +**Contents:** [Quick start](#1-quick-start-docker) · [Architecture](#2-storage-architecture) · [REST API](#3-rest-api) · [S3 gateway](#4-s3-compatible-api) · [Local jobs](#5-local-file-upload-jobs) · [CLI](#6-command-line-usage) · [Databases](#7-database-configuration) · [Security](#8-security-and-operational-limits) · [Development](#9-development-and-testing). -Run from source: +## 1. Quick start: Docker -```powershell -python -m pip install -r requirements.txt -python -m app.main -``` +Prerequisites: Docker Engine and Docker Compose v2. For modern REST/S3/local-job storage, use an authorized Telethon USER session with posting rights in a dedicated private broadcast channel. There is no Bot API transport; the unified engine is Telethon USER MTProto only and never uses Saved Messages. -## Screenshots +~~~bash +git clone https://github.com/babico/TeleDrive.git +cd TeleDrive +mkdir -p config data uploads +cp .env.example .env +cp config.example.yaml config/config.yaml # optional tidy template +chmod 600 .env config/config.yaml +~~~ -### Dashboard +Set a unique `web.password` of at least 16 characters in `config/config.yaml` (the file keeps **all** service settings; `.env` only holds `PUID`/`PGID`/`TZ`/port mappings). Adjust `PUID` and `PGID` to match the owner of the mounted directories. The service creates `config/config.yaml` with safe defaults on first start (or copy the tidy [`config.example.yaml`](config.example.yaml) template); add your authorized USER MTProto credentials there under the **`storage:`** section as an `apis:` list (see [Telethon USER Storage](docs/TELETHON_STORAGE.md)) — one or more `api_id`/`api_hash`/`session_file` sets sharing the same dedicated channel, with `round_robin` (default) or `failover` rotation. Create the private session with the instructions in [Section 4](#4-s3-compatible-api) **before** starting Docker. Without an authorized session, initial health/admin endpoints may run, but binary uploads fail closed. -![TeleDrive dashboard](images/dashboard.png) +~~~bash +docker compose up -d --build +docker compose ps +docker compose logs -f teledrive +~~~ -### Upload queue +| Endpoint | Host-local address | Purpose | +| --- | --- | --- | +| Web UI, admin REST API, Swagger | [`http://127.0.0.1:8080`](http://127.0.0.1:8080) | Account management, jobs, REST and API keys | +| SigV4 S3 gateway | `http://127.0.0.1:9000` | Path-style S3 endpoint for SDKs and CLI clients | -![TeleDrive upload queue](images/upload-queue.png) +**One service, one container, one Python process.** Both listeners use the **same SQL catalog and Telegram object engine**. You do not need `docker compose --profile s3` and there is no second S3 container. -### Settings +The service binds `0.0.0.0` inside the container; the Compose host mapping stays loopback by default. For remote access, use an SSH tunnel, private VPN, or an HTTPS reverse proxy (see `scripts/ops/generate_ssl.py` for TLS); never expose Basic credentials or S3 secrets over untrusted plaintext HTTP. -![TeleDrive settings](images/settings.png) +| Host folder | Container path | Purpose | +| --- | --- | --- | +| `./config` | `/config` | Settings and S3 master key | +| `./data` | `/data` | Sessions, metadata and internal temporary state | +| `./uploads` | `/uploads:ro` | Read-only source files for explicitly queued jobs | -## Build on Windows +To use a different source directory, change the host side of the `/uploads` mount, retaining the container-side path and read-only mode. The image has no GUI/X11/PySide6 dependency. -Create the portable Windows application folder: +**Configure S3 before writing:** the gateway listener starts automatically, but writing S3 objects requires configured authenticated USER MTProto credentials, a persistent encryption key and scoped S3 credentials. See [Section 4](#4-s3-compatible-api). -```powershell -.\build.ps1 -``` +## 2. Storage architecture -Output: `dist\TeleDrive.exe` +### Shared object catalog -This is a standalone executable. You can copy only this `.exe` to another Windows -machine; no Python installation is required. +| Entry point | How it writes | Can REST/S3 read it? | +| --- | --- | --- | +| REST `PUT /api/v1/buckets/{bucket}/content` | USER MTProto → `TelegramObjectStorage` → `BucketCatalog` | **Yes** | +| Signed S3 `PutObject` / multipart upload | Same Telegram chunk engine and SQL catalog | **Yes** | +| Authenticated local-file upload job | Explicit queued file → same shared object catalog | **Yes**, after the job finishes | +| Original Telethon folder scan/batch | User-session uploader and its legacy workflow | **Not automatically**; use the shared-job path for REST/S3 visibility | -To create the installer, install Inno Setup 6 and run: +The SQL catalog stores object heads, generations, checksums, version references, job state and encrypted S3 credential metadata. Object payloads remain in Telegram; the temporary upload scratch directory is **not** a second primary storage backend. Completed transfers have SHA-256 checks and byte-range reads. Failed/unknown Telegram acknowledgments remain protected from unsafe physical deletion. -## Desktop application +### Telegram USER account and transport -The production UI is a PySide6 desktop app. It stores configuration, the Telegram -session, logs, and SQLite state under: +The original uploader accepts Telegram user accounts through MTProto/Telethon; a user account can use Saved Messages (`me`), channel usernames, or supported numeric target IDs. Each account has its own session and optional API/target overrides. -```text -Windows: `%LOCALAPPDATA%\Telegram Auto Upload` -Linux: `~/.local/share/Telegram Auto Upload` -``` +| Strategy | Behavior | +| --- | --- | +| `single` | Use the first enabled account | +| `round_robin` | Rotate preferred sending accounts between files | +| `failover` | Try configured accounts in priority order | +| `parallel` | Independent account-affine workers with fallback | -Run from source: +**Unified storage never uses Saved Messages or Bot API**: select a private broadcast channel that the authorized user can post to. The old standalone uploader still has historical `me` settings; they are not enabled in REST/S3. Respect Telegram rate limits, and never reuse a writable session among independent uploader processes. -```powershell -python -m pip install -r requirements.txt -python -m app.main -``` +The **REST/S3/local-job engine requires USER MTProto**. `TELEDRIVE_STORAGE_TRANSPORT=telethon` is the default and the only accepted value; anything else fails closed. No Bot credentials exist in the runtime. The same SQL catalog and bounded chunk engine are retained. Existing `api-bot` objects are readable using the authorized USER session only when the original **same channel** is accessible and the immutable SQL manifest retains real message IDs; full checksums are still verified. Old objects in inaccessible channels require a separately verified migration and backup. No existing media is automatically deleted. -## Screenshots +## 3. REST API -### Dashboard +The Web listener publishes an authenticated interface at: -![TeleDrive dashboard](images/dashboard.png) +| Endpoint | Purpose | +| --- | --- | +| `http://127.0.0.1:8080/docs` | Swagger UI | +| `http://127.0.0.1:8080/redoc` | ReDoc | +| `http://127.0.0.1:8080/openapi.json` | Machine-readable OpenAPI schema | -### Upload queue +~~~bash +curl -u "admin:YOUR_PASSWORD" http://127.0.0.1:8080/openapi.json +~~~ -![TeleDrive upload queue](images/upload-queue.png) +Use HTTP Basic authentication for administrative API key issuance/rotation/revocation and include `X-TeleDrive-CSRF: 1` on administrative mutations. REST object calls use separately issued, scoped Bearer API keys; those are **not** S3 Access Key IDs. -### Settings +Key operations include bucket management, object listing, logical unlink, full object upload/download, metadata-only HEAD and checksum-verified Range GET. Administrative keys are shown once; only digests are persisted. -![TeleDrive settings](images/settings.png) +~~~bash +# First create the bucket and issue a scoped REST key in API Management. +curl -f -X PUT \ + 'http://127.0.0.1:8080/api/v1/buckets/backups/content?key=folder%2Ffile.bin' \ + -H "Authorization: Bearer $TELEDRIVE_API_KEY" \ + -H 'Content-Type: application/octet-stream' \ + --data-binary '@file.bin' -## Build on Windows +curl -f -H "Authorization: Bearer $TELEDRIVE_API_KEY" \ + 'http://127.0.0.1:8080/api/v1/buckets/backups/content?key=folder%2Ffile.bin' \ + -o restored.bin +~~~ -Create the portable Windows application folder: +The default REST upload ceiling is 2 GiB, configurable with `web.rest_max_upload_bytes` in `config/config.yaml`. Uploads require the configured authorized Telethon USER session with access to a dedicated private channel. Physical Telegram message deletion is **not** implied by an HTTP object delete. -```powershell -.\build.ps1 -``` +## 4. S3-compatible API -Output: `dist\TeleDrive.exe` +For the **116-operation AWS S3 API compatibility matrix**, parameter-level limitations, unsupported-operation groups, and the P2 Advanced S3 work plan, see [S3 Specifications](docs/S3_SPECS.md). -This is a standalone executable. You can copy only this `.exe` to another Windows -machine; no Python installation is required. +The S3 endpoint is **part of the default container**, not a separate Compose profile. It implements a tested **subset** of S3 over the shared Telegram-backed catalog. It is **not AWS S3 feature parity**. -To create the installer, install Inno Setup 6 and run: +A connected user-MTProto server also exposes **admin-only, read-only** `GET /api/v1/admin/storage/channel-history?limit=100&include_external_markers=true`. Use `before_message_id=` to enumerate older pages. Other authors' marker captions are reported only as **unverified clues**: do not use them to restore a bucket/key or authorize physical deletion without independent manifests and checksum evidence. Only an authorized USER session can run the unified service or enumerate channel history. -```powershell -.\build-installer.ps1 -``` +### Provision the S3 master key and credentials -The installer is per-user and does not require administrator privileges. +An account holder creates the user StringSession **once** on a trusted interactive host (no credentials are written to Git). The server then reconnects automatically under that real user authorization: -The installer output is `dist-installer\TeleDrive-Setup-1.0.6.exe`. +~~~bash +# Set the Telegram API credentials privately in the shell; the program +# prompts for sign-in once and writes an authorized session file. +python scripts/dev/create_storage_user_session.py --output config/mtproto-user.session +chmod 600 config/mtproto-user.session +~~~ -### Quick install on Windows +Run this interactively on a trusted host with Python dependencies installed, before the first Docker startup. Ensure the resulting `0600` file is readable by the configured container `PUID`/`PGID`. The API ID/hash are from [my.telegram.org](https://my.telegram.org/apps). The script prompts for the real user's Telegram sign-in code and optional 2FA and refuses to overwrite an existing session. Its MTProto application/device metadata identifies the TeleDrive client accurately; it does **not** masquerade as an official Telegram Desktop/Android app. Choose a channel username or a numeric channel ID present in this user's dialogs. If selecting a numeric private channel, it must be visible among the user's first 5,000 dialogs. No user account => no user-only history access. -Open Command Prompt and paste: +1. Save one or more authorized user StringSessions under `./config/` (mode 0600) using the interactive command below, then list them with their API credentials and the shared channel in the `storage:` section of `config/config.yaml` ([Telethon USER Storage](docs/TELETHON_STORAGE.md)). The user needs posting rights in that private **broadcast channel**, not Saved Messages. Existing installations from the pre-switch era must configure a user session before further uploads; **do not delete original media or SQL backups**. +2. Generate a persistent master encryption key once, on the trusted host: -```bat -mkdir "%TEMP%\TeleDrive" 2>nul & curl.exe -fL --progress-bar -o "%TEMP%\TeleDrive\TeleDrive-Setup-1.0.6.exe" "https://github.com/amjadlle/TeleDrive/releases/download/v1.0.6/TeleDrive-Setup-1.0.6.exe" & if errorlevel 1 (echo Download failed. & exit /b 1) else if not exist "%TEMP%\TeleDrive\TeleDrive-Setup-1.0.6.exe" (echo Installer was not downloaded. & exit /b 1) else (powershell -NoProfile -Command "Unblock-File -LiteralPath $env:TEMP\TeleDrive\TeleDrive-Setup-1.0.6.exe; Start-Process -FilePath $env:TEMP\TeleDrive\TeleDrive-Setup-1.0.6.exe" & echo TeleDrive setup launched.) -``` +~~~bash +python - <<'PY' +from pathlib import Path +from cryptography.fernet import Fernet +target = Path("config/s3-master.key") +if target.exists(): + raise SystemExit("Refusing to replace the existing S3 master key") +with target.open("xb") as stream: + stream.write(Fernet.generate_key() + b"\n") +target.chmod(0o600) +PY +~~~ -This downloads the Windows installer with visible progress and opens the setup -wizard. +3. Install the `cryptography` package on the host if necessary. Ensure the container's `PUID`/`PGID` can **read** this file. Back up the key separately: losing it makes stored S3 secrets undecryptable. +4. Start the default Compose service and issue a **scoped** S3 access key and secret in **WebUI → API Management → S3-compatible gateway** . The secret appears once and is encrypted in SQL. -### Quick install on Linux +### SDK example -Close TeleDrive first, then open a terminal and paste: +~~~python +import boto3 +from botocore.config import Config -```bash -pkill -x TeleDrive 2>/dev/null || true; curl -fL --progress-bar -o /tmp/TeleDrive.tar.gz "https://github.com/amjadlle/TeleDrive/releases/download/v1.0.6/TeleDrive-linux-x86_64.tar.gz" && tar -xzf /tmp/TeleDrive.tar.gz -C "$HOME" && "$HOME/TeleDrive/TeleDrive" -``` +client = boto3.client( + "s3", + endpoint_url="http://127.0.0.1:9000", + aws_access_key_id="YOUR_ACCESS_KEY_ID", + aws_secret_access_key="YOUR_SECRET_ACCESS_KEY", + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + ), +) +client.create_bucket(Bucket="backup-bucket") +client.put_object(Bucket="backup-bucket", Key="hello.txt", Body=b"Hello Telegram") +assert client.get_object(Bucket="backup-bucket", Key="hello.txt")["Body"].read() == b"Hello Telegram" +~~~ -This works for both a fresh installation and an update. It closes a running -TeleDrive process, downloads the Linux package, extracts it to `~/TeleDrive`, -and launches the new version. Stop any active upload before running it. Your -settings, login, queue, and history remain in `~/.local/share/Telegram Auto Upload`. +### Implemented features and limits -These commands are platform-specific: Windows uses the `.exe` installer, while -Linux uses the `.tar.gz` archive. +| Feature group | Available behavior | +| --- | --- | +| Authentication | Scoped encrypted S3 keys, SigV4 request checks, bounded signed/presigned operations | +| Buckets and listing | Create/Head/DeleteBucket; V1/V2 object listing, prefixes, delimiters, marker/continuation and URL encoding | +| Objects | Put/Get/Head/Delete (logical unlink), Range, CopyObject, conditional GET/HEAD and signed conditional writes/deletes | +| Multipart | UploadPart, Complete/Abort and guarded stale-completion recovery through an admin-only API | +| Metadata and versions | Selected published VersionId reads/listing, always-Enabled versioning, fixed private-only Public Access Block and incomplete-MPU lifecycle abort rules, object/bucket tags, metadata, CopySource, CORS and GetObjectAttributes (`ETag`, `ObjectSize`, full-object SHA-256 `Checksum`, `STANDARD`, historical `VersionId`, immutable multipart `ObjectParts` with bounded pagination) | +| Checksums | SHA-256, SHA-1, Content-MD5, CRC32, CRC32C and CRC64NVME validation where explicitly implemented | +| Known gaps | Object lifecycle expiration/transitions, bucket policies, SSE, full AWS delete-marker semantics, every multipart POST variant and unverified clients | -## Build on Linux +**Safety:** `UNSIGNED-PAYLOAD` writes are accepted only when an appropriately **signed, verifiable** checksum binds the actual received bytes. Public ACLs and unsupported archival tiers must not be inferred from AWS-style parameters. Failed conditional publications may retain protected, unpublished Telegram manifests rather than deleting uncertain remote media. -The same source code runs on Linux with Python, PySide6, Telethon, and PyYAML: +S3 client integrations tested in CI use **real** AWS CLI and rclone executables against the actual gateway and SQL with **synthetic Telegram transport**; real Telegram byte transfer is tested in separate live E2E workflows. For rclone use `type=s3`, `provider=Other`, `force_path_style=true` and your issued S3 key pair. Create buckets first if `no_check_bucket=true`. -```bash -python3 -m venv .venv -source .venv/bin/activate -python -m pip install -r requirements.txt -python -m app.main -``` +### Non-destructive storage and multipart maintenance -Create a Linux executable folder with PyInstaller: +Before considering stale multipart recovery or garbage collection, inspect the admin-only endpoints and **dry-run** responses. The admin API provides `/api/v1/admin/s3-multipart/stale`, `/api/v1/admin/s3-multipart/{upload_id}/recover` (with `"execute": false` for preview), and `/api/v1/admin/storage/gc-candidates`. A stale completion may be explicitly returned to `active` under expiry/fencing conditions; it is not equivalent to deleting Telegram messages. -```bash -python -m pip install pyinstaller -python -m PyInstaller --noconfirm --clean --windowed \ - --name "Telegram Auto Upload" --hidden-import uploader desktop.py -``` +**Physical Telegram garbage collection remains disabled** until independently verified owner/alias/version/retention/backup/fencing controls and approval exist. The [P2 safety section](docs/IMPLEMENTATION_STATUS.md#41-physical-telegram-garbage-collection-disabled) describes the incomplete safeguards. -The Linux output is created under `dist/Telegram Auto Upload/`. The Windows `.exe` -and Inno Setup installer are Windows-specific; they are not used on Linux. +## 5. Local-file upload jobs -The repository also includes `build-linux.sh`, which automates the Linux build and -creates `dist-installer/TeleDrive-linux-x86_64.tar.gz`: +The Docker mount `./uploads:/uploads:ro` gives the service read-only access to approved source files. Use the authenticated **durable job API** to publish files into the **same REST/S3 catalog**. -```bash -bash build-linux.sh -``` +~~~bash +# Ensure the bucket exists. Then enqueue an existing local file: +curl -fsS -u "admin:$WEB_PASSWORD" \ + -H "X-TeleDrive-CSRF: 1" \ + -H "Content-Type: application/json" \ + -d '{"bucket":"backups","key":"local/example.bin","source_path":"/uploads/example.bin"}' \ + http://127.0.0.1:8080/api/v1/admin/storage/jobs +~~~ -After extracting the archive, launch the app with: +With `jobs.auto_run: true` in `config/config.yaml` (the default) the service processes **explicitly queued** jobs automatically. Set it to `false` to disable background processing while retaining queue entries. **A copy into `/uploads` does not automatically trigger a scan or upload.** -```bash -./TeleDrive/TeleDrive -``` +Jobs are persisted in SQL and protect against source mutation, unexpected generation changes and interrupted uploads. Keep the source file until the job reaches `done`. The admin API also supports job inspection/retry, stale-pending previews, history restore and safe health checks. Failed/uncertain remote sends do not authorize physical cleanup. -To package the Linux build for sharing: +## 6. Command-line usage -```bash -tar -czf TeleDrive-linux-x86_64.tar.gz -C dist "Telegram Auto Upload" -``` +TeleDrive is a single WebUI + REST + S3 service; there is no separate headless uploader CLI. All operation is through the WebUI, REST API, or S3-compatible API. -Linux users need to build on Linux (or inside WSL); PyInstaller does not cross-compile -a Linux executable from Windows. +WebUI automation is configured through **Settings → Automation**. Do not launch multiple client processes against the same writable Telethon session file. Ingestion of an existing folder is done through the admin `import-legacy` endpoint, which queues files as durable jobs in the shared SQL catalog using the configured Telethon transport; it never starts a separate direct-send path. -### Large files on free Telegram accounts +### Telegram account setup -TeleDrive automatically splits files at 1.9 GB before uploading them. The parts are -created as lossless byte-for-byte pieces, so no video re-encoding is performed and -the original file is never changed. Split parts are uploaded as documents and are -removed from the source drive after every part uploads successfully. To reconstruct -an original file after downloading all parts, join them in order with a binary file -concatenation tool. +1. Get a Telegram `api_id` and `api_hash` from [my.telegram.org](https://my.telegram.org/apps). +2. Choose `me`, a permitted channel username, or an appropriate numeric chat ID as the user-account uploader destination. +3. Authenticate with the Telegram code and, if enabled, 2FA password. +4. Test a small scan/upload and confirm readback before enabling scheduling. -## Telegram setup and destinations +A BotFather token is not used by the unified storage runtime. Historical Bot messages can be read through a USER session only when the original channel and exact message IDs remain accessible. Never commit user sessions, API hashes or other secrets. -TeleDrive signs in as the user's Telegram account through the official Telegram API -(MTProto). It needs the user's `api_id` and `api_hash`, not a BotFather token. +## 7. Database configuration -### Create Telegram API credentials +Configure `state.database_url` in the YAML configuration or Web UI. SQLite needs no external server; the server image includes drivers for the tested PostgreSQL and MySQL/MariaDB integrations. Other synchronous SQLAlchemy dialects may work with separately reviewed drivers but are **not certified merely by accepting their URLs**. -1. Create or use a normal Telegram account in the official Telegram app. -2. Open [my.telegram.org](https://my.telegram.org) and sign in with that account's phone number. -3. Open **API development tools**. -4. Create an application and copy the displayed `api_id` and `api_hash` into TeleDrive Settings. -5. Keep the `api_hash` private. During the first upload, TeleDrive asks for the Telegram login - code and, if enabled, the two-step verification password. +~~~yaml +state: + database_url: "sqlite:////data/state.db" +~~~ -Do not create a bot or paste a BotFather token into TeleDrive: Bot API tokens are for bot-based -programs, while TeleDrive uploads as the logged-in user account. +Representative alternatives: -### Choose where files are uploaded +~~~text +postgresql+psycopg://user:password@db:5432/teledrive +mysql+pymysql://user:password@db:3306/teledrive +~~~ -The **Target channel / Saved Messages** setting accepts: +For additional reviewed packages, set `TELEDRIVE_DB_EXTRA_PACKAGES` in `.env` and rebuild the Docker image. Some database drivers require OS libraries in a derived Docker image. Enable DB TLS remotely and use least-privilege credentials. Back up the DB together with Telegram access and the S3 master key; see [Backup and Restore](docs/BACKUP_RESTORE.md). -- `me` — your personal Saved Messages; recommended for the first test. -- A public username such as `@my_backup_channel` — the logged-in account must be a member - and have permission to post. -- A numeric chat ID such as `-1001234567890` — useful for private channels and supergroups. +## 8. Security and operational limits -For a private channel or group, first join it with the same Telegram account used by TeleDrive. -For a channel, make that account an administrator with permission to post. Keep the destination -private if it contains personal backups. +- Keep WebUI/REST/S3 on loopback unless protected by an HTTPS reverse proxy or VPN. +- Treat session files, `.env`, historical Bot tokens, `config.yaml`, the S3 master key and SQL backups as secrets. +- Docker defaults include non-root execution, a read-only filesystem, capability drops and a read-only source mount. +- Never assume a successful database-only restore is proof of original Telegram media recovery. +- Never enable automatic physical Telegram deletion or bypass unresolved ACK/reference fences. +- Use [Security Policy](docs/SECURITY.md) to report vulnerabilities privately and [Project Status](docs/IMPLEMENTATION_STATUS.md) to distinguish development CI from production acceptance. -### Find a private channel or group ID +## 9. Development and testing -The easiest option is to use the destination's public username when one exists. For a private -channel or supergroup, open a message from it in Telegram Desktop and choose **Copy Message Link**. -A link like `https://t.me/c/1234567890/123` contains the internal ID `1234567890`; enter it in -TeleDrive as `-1001234567890`. +Repository layout: -If the chat is a basic group or the message-link method is unavailable, an ID-reporting bot such -as `@userinfobot` or `@RawDataBot` may show the chat ID after you send a message or add the bot. -Only use a third-party bot if you trust it, and do not send private files or credentials to it. +| Directory | Purpose | +| --- | --- | +| `teledrive/` | Telegram uploader, object engine, SQL state, recovery, API/transport adapters | +| `web/` | Unified WebUI/REST/S3 ASGI service and admin interface | +| `scripts/` | Setup, maintenance and CI tooling | +| `tests/` | Offline regression, integration and live test contracts | +| `.github/workflows/` | Categorized GitHub Actions pipelines | +| `.github/ci/` | Workflow-specific supporting Docker assets | -### Bot API note +~~~bash +python -m pip install -r requirements.txt pytest +python -m pytest -q +python -m compileall -q teledrive web app tests +docker compose config --quiet +# Isolated fault-injection acceptance: +bash scripts/dev/run_docker_fault_injection.sh +~~~ -Creating a bot with [@BotFather](https://t.me/BotFather) and obtaining a bot token is not required -for the current TeleDrive application. A future bot-based mode would require separate code and -different permissions; the instructions above are for the current user-account mode. +Seven mandatory GitHub Actions workflows validate a given commit. The delivery workflow also includes a guarded housekeeping job that deletes only merged, unchanged, unprotected branches with no open PR, after its release gate passes. Real User MTProto and multiworker tests use private **test-only** Telegram credentials. The test secrets can be configured using `python scripts/dev/setup_telegram_e2e.py --channel "TeleDrive E2E Storage" --create-if-missing` on a trusted local machine. Never upload live credentials to GitHub issues or the repository. See [CI and Test Guide](docs/CI_WORKFLOWS.md) for workflow names, command details, evidence and acceptance boundaries. -## First use +## Credits and license -1. Launch the desktop application. -2. Open Settings. -3. Enter the Telegram API ID and API hash from `my.telegram.org`. -4. Set the target to `me` for a safe Saved Messages test. -5. Choose a small source folder. -6. Save settings and scan the folder. -7. Run one upload batch. - -## Recommended settings - -For a steady, conservative personal backup, start with: - -| Setting | Recommended value | Meaning | -| --- | ---: | --- | -| Files per run | `20` | Maximum files uploaded in one batch. | -| Files per day | `500` | Daily upload cap across all batches. | -| Delay between uploads (minimum) | `20 sec` | Shortest wait after a successful upload. | -| Delay between uploads (maximum) | `60 sec` | Longest wait after a successful upload; TeleDrive chooses a random wait in this range. | -| Delay between automatic runs | `60 min` | Wait between batches when automatic loop mode is enabled. | -| Retry attempts per file | `3` | Total attempts for a file after temporary errors, including the first attempt. | -| Retry backoff base | `30 sec` | Starting retry delay; later retries increase to 60, 120 seconds, and so on. | -| Flood-wait buffer | `30 sec` | Extra time added when Telegram tells TeleDrive to wait. | -| Send files as | `Document` | Keeps files as file attachments rather than converting media. | - -These values are a conservative starting point for a private personal archive. Telegram -limits are dynamic, so no configuration can guarantee that an account will never receive -a flood wait or other restriction. If Telegram reports a `FLOOD_WAIT`, let TeleDrive -finish the server-requested wait instead of restarting repeatedly or running multiple -copies of the app. - -### How the timing settings work - -- **Delay between uploads:** After each successful file, TeleDrive waits a random time - between the minimum and maximum values before continuing. -- **Delay between automatic runs:** A batch starts immediately when the loop is enabled; - this value controls the wait before the next batch begins. -- **Retry attempts per file:** Temporary network or Telegram errors trigger retries. A - file is not uploaded repeatedly when it succeeds. - -- **Retry backoff base:** Failed retries wait progressively longer: with a 30-second base, - the delays are 30, 60, and 120 seconds. -- **Flood-wait buffer:** If Telegram returns `FLOOD_WAIT_60` and the buffer is 30 seconds, - TeleDrive waits 90 seconds before continuing. - -For the safest operation, upload only to your own private channel or Saved Messages, -keep one TeleDrive process running, and avoid using the same account for bulk messaging -while a backup is active. - -The first login requests the Telegram code and, if enabled, the two-step verification -password. The session is stored locally and reused on later runs. - -## Backend CLI - -The uploader engine remains available for automation and diagnostics: - -```powershell -python uploader.py --config path\to\config.yaml --scan-only -python uploader.py --config path\to\config.yaml --run-once -``` - -## Safety behavior - -- Recursive source-folder scanning -- SQLite queue with resumable state -- One file uploaded at a time -- Configurable delays and daily/run limits -- Retry and flood-wait handling -- Changed or missing files are skipped safely -- Single-process lock prevents overlapping runs - -Keep the target private while testing, avoid modifying files during an active upload, -and back up the local state database and Telegram session when migrating machines. - -## 👨‍💻 Author - -**Amjad P A** — Full-Stack AI Engineer & Solo Builder - -- 🌐 **Portfolio**: [amjad.mapki.in](https://amjad.mapki.in) -- ☕ **Buy Me a Coffee**: [buymeacoffee.com/amjadlle](https://buymeacoffee.com/amjadlle) -- 🐙 **GitHub**: [@amjadlle](https://github.com/amjadlle) -- 💼 **LinkedIn**: [linkedin.com/in/amjadlle](https://linkedin.com/in/amjadlle) -- 𝕏 **X (Twitter)**: [@amjadlle](https://x.com/amjadlle) -- ✉️ **Email**: [hire.amjad@gmail.com](mailto:hire.amjad@gmail.com) - ---- - -## ☕ Support TeleDrive - -If TeleDrive helps you backup your files safely, manage your storage, or simplifies your Telegram workflow, consider buying me a coffee to support maintenance and future updates! - -

- - Buy Me A Coffee - -

- ---- - -## License - -This project is licensed under the MIT License. See [LICENSE](LICENSE). - -## Contributing - -Bug reports, documentation improvements, and code contributions are welcome. Please -read [CONTRIBUTING.md](CONTRIBUTING.md) before opening a pull request. +Originally created by [Amjad P A](https://github.com/amjadlle/TeleDrive). Maintained here as the [babico/TeleDrive fork](https://github.com/babico/TeleDrive), with Docker, multi-account, REST/S3 and recovery enhancements. Licensed under [MIT](LICENSE). diff --git a/SECURITY.md b/SECURITY.md deleted file mode 100644 index 69373814..00000000 --- a/SECURITY.md +++ /dev/null @@ -1,12 +0,0 @@ -# Security Policy - -## Reporting a vulnerability - -Please do not publish credentials, Telegram session files, or exploitable details in -a public issue. Report security concerns privately through the repository's GitHub -security contact or a private maintainer message. - -When reporting an issue, include the affected version, operating system, steps to -reproduce, and the potential impact. Remove personal data and secrets from logs. - -Never share your Telegram API hash, login code, two-step password, or session file. diff --git a/TODO.md b/TODO.md new file mode 100644 index 00000000..e39bf5f1 --- /dev/null +++ b/TODO.md @@ -0,0 +1,107 @@ +# Engineering Checklist + +> **Purpose:** short, actionable development and release tasks. For decisions and evidence limits, see [Project Status](docs/IMPLEMENTATION_STATUS.md). For commands and CI contracts, see [CI and Test Guide](docs/CI_WORKFLOWS.md). + +## 0. Telethon-only purge — complete (merged PR #15) + +> Owner: Bot work is discarded completely; every live transport becomes Telethon USER MTProto only. CI was green on the exact PR head `56813a8`. + +- [x] Bot adapter/receipt modules, Bot tests and the Bot E2E workflow deleted. +- [x] GC inventory/reference graph and physical-cleanup fences: MTProto send intents/receipts + active durable jobs only. +- [x] Web/S3 services: `storage_transport` factory, Telethon `RECOMMENDED_CHUNK_SIZE`, MTProto health alerts, `mtproto-unresolved` admin endpoint. +- [x] `rehearse_telegram_dr.py`: USER-only read path, reusable `open_read_only_user`, no Bot fallback. +- [x] MTProto rate guard: shared send slots, flood-wait cooldowns, durable flood observations; wired into `UnifiedUserTransport`. +- [x] Fault-injection scenarios and all converted test fakes run USER MTProto. +- [x] Multiworker PostgreSQL E2E converted to USER MTProto; channel resolution by name or numeric target. +- [x] All scripts (acceptance, rehearse, setup, cutover), workflows (7-workflow matrix), `test_ci_workflows.py` and docs converted. + +## 0.1 In progress — config-only settings overhaul (owner directive) + +> Owner: everything Telethon-only and config-driven. Except TZ and the config-file path, NO env settings; the UI must edit the config; strategies must cover single/round_robin/failover/parallel; SSL script; compose healthcheck/logging; DB extra packages via config; all tracked here. + +- [x] `storage:` config section with `apis:` list + channel/strategy (merged PR #20); root `config.example.yaml`; examples/ folder removed. +- [x] Central settings loader (`teledrive/settings.py`): all service settings from `config/config.yaml` only (`web:`, `jobs:`, `state:`, `storage:`, `paths:` sections); no operator env except TZ + `TELEDRIVE_CONFIG`. +- [x] Removed `TELEDRIVE_STORAGE_TRANSPORT` (only Telethon exists). +- [x] Strategies `single | round_robin | failover | parallel` supported in `storage.strategy`; `parallel` uploads chunks concurrently across the apis sets (engine batches via `parallel_degree`). +- [x] Legacy investigation: `state.database_url` and `upload.source_dir` are what the unified service/bridge use; accounts/phone dropped from the example config (sessions are pre-authorized; phone/2FA used once interactively by `scripts/dev/create_storage_user_session.py`). +- [x] UI: Settings page gained a Storage card (channel, strategy, apis: list editor) writing back to config.yaml via the existing /api/config PUT. +- [x] SSL: `scripts/ops/generate_ssl.py` (openssl self-signed cert+key, config snippet, `--test` HTTPS verification) + `web.ssl_certfile/keyfile` + unified.py TLS support. +- [x] docker-compose: env stripped to TELEDRIVE_CONFIG + TZ; added healthcheck and json-file log rotation; service binds 0.0.0.0 (host mapping stays loopback). +- [x] `state.db_extra_packages:` installs extra DB drivers into /data/site-packages at startup (oracledb example in config.example.yaml); no env. +- [ ] Local skipped tests: install aws CLI so `tests/s3/test_s3_cli_compat.py` runs locally; document why live e2e skips stay credential-gated (they run green in CI). + +## 1. Owner directives + + +- [x] **Desktop app cancelled completely** (merged PR #16): removed `app/`, `installer/`, desktop requirements/example config, build scripts, desktop modules/tests and the desktop CI workflow. Shared config/state helpers kept. +- [x] **Single requirements + single config** (merged PR #16): one `requirements.txt`; the service generates `config/config.yaml` with defaults (no example files needed). +- [x] **Website not maintained here** (merged PR #18): `website/` and every reference removed. +- [x] **Multiple api_id/api_hash pairs** (config-driven): `storage:` section in `config/config.yaml` holds the dedicated channel, `round_robin`/`failover` strategy and an `apis:` list of `{api_id, api_hash, session_file, account}` sets; root `config.example.yaml` is the tidy template; env comma-lists remain a legacy fallback; adversarial tests. +- [x] **Regroup `scripts/` and `tests/`** (PR #19): tests split into `s3/ storage/ web/ e2e/ meta/`, scripts into `ops/ dev/`; every import, CI path and doc link updated. Release packaging now builds the single-container Docker image. + +## 2. Verified milestones + +- [x] Publish [v1.0.7](https://github.com/babico/TeleDrive/releases/tag/v1.0.7) from `747747652cd1ca36878453d49daf627d8c0a60ad` with Linux and Windows release assets. +- [x] Validate the bounded REST/S3 storage feature set and 13-gate disposable Docker fault-injection acceptance (the desktop application was later removed). +- [x] Exercise real Bot API, user MTProto and multiworker PostgreSQL transfer scenarios against dedicated test Telegram targets. +- [x] Provide shared REST/S3/local-job object catalog and a single WebUI/REST/S3 Docker container. +- [x] Implement guarded metadata recovery, owner-verified isolated Telegram rebuild (atomic bounded multi-object batches), and the offline two-person production-evidence verifier. +- [x] Verify all nine standard GitHub Actions workflows successfully on development commit `c2b76880b4ada3d61d47a7926ea819244047fa33`, including User MTProto, isolated historical Bot, and PostgreSQL multiworker workflows. This does **not** change the contents of v1.0.7. + +## 3. Engineering backlog + +- [x] Implement and CI-validate an isolated, read-only **Telethon USER MTProto original-media restore rehearsal** using an independently sealed SQL snapshot, original channel/message references and whole-body/range SHA-256 verification. Owner-authorized production-equivalent execution remains an operator acceptance task. +- [x] Require Telethon USER MTProto for every unified REST/S3/local-job upload; reject Bot mode and do not read stale Bot secrets, while retaining hash-verified read access to original Bot objects from the same channel. +- [x] Add mainline release-gated cleanup of only merged, unchanged, unprotected PR branches; keep open/unmerged/advanced branches untouched. +- [x] Add bounded, admin-authenticated, read-only history enumeration for the live Telethon storage session, with exclusive cursor pagination and visibly unverified other-sender marker candidates; prohibit guessed bucket/key mapping and Telegram mutations. +- [x] Implement an authenticated USER-MTProto transport for the unified REST/S3/local-job SQL catalog, private-channel-only operation, durable uncertain-send evidence, signed S3 synthetic regression and backward-compatible Bot object read fallback. +- [x] Write a complete [owner-local Docker, independent media backup, restore and GC acceptance mission](docs/LOCAL_AGENT_ACCEPTANCE.md). This checks off the **runbook delivery only**, not operator execution or production authorization. +- [x] Complete bounded ListParts and ListMultipartUploads with SQL-keyset pagination, genuine metadata fields, delimiter/URL encoding, ownership isolation, boto3 and adversarial tests; promote both documented private S3 contracts after CI. +- [x] Test non-destructive GC SQL graph guards (including removed versions, writer fences and active durable jobs) and real test-channel history inventory automatically in CI. +- [x] Implement the full GC **non-destructive machinery**: StorageGCInventory.inspect (complete reference graph: HEADs, published/removed versions, delete markers, transitive copy aliases, multipart parts, MTProto intents/receipts, active durable jobs, writer fences), preview_failed_retention (bounded dry-run candidates) and plan()/verify_plan() (integrity-bound, stale-plan-detecting dry-run plan with explicit owner-gate checklist). Physical deletion remains disabled by default. +- [ ] **Authorize physical GC** only after sealed independent payload backup, ownership/alias/retention checks, an actual restore drill, writer barrier, two-person approval and an owner-authorized isolated single-message trial. `deletion_permitted` stays False; no Telegram media has been deleted. +- [x] Add a bounded, read-only marker inventory for one owner-authorized Telegram channel, with private output and synthetic regression coverage; candidate hints are not recovery proofs. +- [x] Compare history candidates against independently supplied bucket/key, full SHA and message identity; real CI performs isolated byte-verified restore of its own Telegram test fixture. +- [ ] Extend trustworthy recovery of unknown original Telegram history beyond candidate markers without inventing bucket/key, full SHA-256 or missing version metadata. +- [x] Align S3 documentation with CRC64NVME integration-test coverage; retain explicit limits for other checksum modes. +- [x] Count unresolved MTProto send intents without journal/receipt evidence in non-destructive GC audits. +- [x] Implement signed Get/PutBucketVersioning with strictly Enabled semantics, scoped permissions and real boto3 compatibility tests. +- [x] Add signed Get/PutPublicAccessBlock for the enforced all-private configuration; reject any weakening attempts. +- [x] Support signed lifecycle XML GET/PUT/DELETE and hourly bounded incomplete-MPU abort without physical Telegram deletion. +- [x] Add 116-operation machine-readable S3 status manifest plus signature-independent, fail-closed selector validation and adversarial REST/S3 CI coverage. +- [x] Implement scoped read-only GetBucketPolicyStatus as an honest private-only response with boto3 integration and negative policy tests. +- [x] Complete AWS-compatible PutBucketTagging with 50 UTF-16-aware bucket tags, idempotent replacement, MD5/CRC validation and boto3 regression coverage. +- [x] Complete ListBuckets: bounded SQL keyset pagination, Prefix/MaxBuckets/BucketRegion, tamper-proof scoped cursor, actual bucket creation times and boto3 clients. +- [x] Extend GetObjectAttributes with published VersionId reads, immutable SHA-256 Checksum and STANDARD storage class; verify signed requests and boto3 compatibility. +- [x] Complete bounded GetObjectAttributes for private general-purpose buckets: immutable committed multipart ObjectParts snapshots, keyset part pagination, copy/history preservation, pre-upgrade journal fallback and fail-closed tamper handling. Validate with real boto3/AWS CLI/rclone and SQL/HTTP regression CI; keep AWS-only tiers/SSE and directory buckets out of scope. +- [ ] Deliver [P2 Advanced S3](docs/S3_SPECS.md#5-p2-advanced-s3-implementation-plan) from the [116-operation S3 compatibility matrix](docs/S3_SPECS.md): fail-closed selectors, advanced multipart/checksums, policies, encryption, lifecycle and independent client tests. Keep partial capabilities explicitly partial until verified. +- [x] Expose opt-in lease-fenced legacy folder ingestion in default single-container Compose and verify cross-interface REST/S3 readback from the shared catalog. +- [ ] Establish longer-duration real-environment capacity benchmarks on owner-controlled infrastructure. + +## 4. Operator-only production sign-off + +**Assigned to on-device agent:** [Owner-local acceptance mission](docs/LOCAL_AGENT_ACCEPTANCE.md). Its safe synthetic and test-channel work can run on one Docker host. Original-media recovery, physically distinct hosts, authorized soak, real legacy cutover and independent approvals must remain unchecked until documented original-owner evidence exists. +- [ ] Collect independently verified identities from physically distinct deployment hosts sharing the intended SQL database. +- [ ] Rehearse original-media recovery using the owner's sealed SQL archive, Telegram credentials and real channel without overwriting production data. +- [ ] Conduct the authorized physical soak; capture genuine telemetry without deliberately flooding Telegram. +- [ ] Perform legacy cutover only when a real legacy source exists, otherwise obtain the explicit signed no-legacy exemption. +- [ ] Validate independent owner/operator approvals with `scripts/ops/verify_production_bundle.py`; never infer sign-off from a synthetic CI fixture. + +## 5. Release decision rules + +- [ ] For any **new** release candidate, require all mandatory checks to pass on **that exact SHA**; previous green commits cannot be reused as proof. +- [ ] Review binaries, tag, release metadata and supported-feature limits before publication; do not silently publish unreleased `main` features as v1.0.7. +- [ ] Keep automatic physical Telegram deletion disabled until its independent safety conditions are verified. + +**Reference:** [Project Status and Acceptance](docs/IMPLEMENTATION_STATUS.md) · [CI and Test Guide](docs/CI_WORKFLOWS.md) · [Backup and Restore](docs/BACKUP_RESTORE.md). + +## 2. Legacy direct-send uploader removal — complete (merged cf98e1a) + +- [x] Legacy Telethon folder scanner/uploader CLI (`teledrive.uploader` main/async_main/parse_args) removed. +- [x] Legacy direct-send scheduling, account management, `files`/`account_status` DB helpers removed from `uploader.py` (config/state/scan primitives retained). +- [x] WebUI legacy endpoints removed: `/api/dashboard`, `/api/files`, `/api/process/*`, `/api/auth`, `/api/logs`; `scheduler_loop`, `start_uploader`, `stop_uploader`, `process_snapshot` removed. +- [x] Legacy auto-bucket scheduler (`jobs.legacy_auto_bucket`) removed; folder ingestion now only via admin `import-legacy` endpoint. +- [x] `pyproject.toml` CLI entry removed; README §6 updated. +- [x] `build_caption` retained (used by `validate_config`); tests updated (test_openapi, test_security, test_legacy_bridge, test_legacy_import_admin). +- [x] Full local suite: 1085 passed / 61 skipped / 0 failed on `cf98e1a`. +- [x] PR opened: https://github.com/amjadlle/TeleDrive/pull/2 (CI pending maintainer approval of fork workflows). diff --git a/TeleDrive Logo.png b/TeleDrive Logo.png deleted file mode 100644 index 62a90a5b..00000000 Binary files a/TeleDrive Logo.png and /dev/null differ diff --git a/app/__init__.py b/app/__init__.py deleted file mode 100644 index e66aa81d..00000000 --- a/app/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Telegram Auto Upload desktop application.""" diff --git a/app/main.py b/app/main.py deleted file mode 100644 index b87a425f..00000000 --- a/app/main.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys -from PySide6.QtWidgets import QApplication -from PySide6.QtGui import QPalette, QColor -from .window import MainWindow - -def main() -> int: - if "--cli" in sys.argv: - sys.argv.remove("--cli") - from uploader import main as uploader_main - return uploader_main() - app = QApplication(sys.argv) - app.setApplicationName("TeleDrive") - app.setApplicationDisplayName("TeleDrive") - app.setOrganizationName("TeleDrive") - app.setStyle("Fusion") - palette = app.palette() - palette.setColor(QPalette.ColorRole.Window, QColor("#f7f6f2")) - palette.setColor(QPalette.ColorRole.Base, QColor("#ffffff")) - palette.setColor(QPalette.ColorRole.AlternateBase, QColor("#f3f1eb")) - palette.setColor(QPalette.ColorRole.Text, QColor("#202126")) - palette.setColor(QPalette.ColorRole.WindowText, QColor("#202126")) - palette.setColor(QPalette.ColorRole.ButtonText, QColor("#202126")) - palette.setColor(QPalette.ColorRole.HighlightedText, QColor("#202126")) - app.setPalette(palette) - window = MainWindow() - window.show() - return app.exec() - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/app/paths.py b/app/paths.py deleted file mode 100644 index b7d3ff79..00000000 --- a/app/paths.py +++ /dev/null @@ -1,55 +0,0 @@ -from pathlib import Path -import os - -APP_NAME = "Telegram Auto Upload" - -def app_data_dir() -> Path: - if os.name == "nt": - root = Path(os.environ.get("LOCALAPPDATA", Path.home() / "AppData" / "Local")) - else: - root = Path(os.environ.get("XDG_DATA_HOME", Path.home() / ".local" / "share")) - path = root / APP_NAME - path.mkdir(parents=True, exist_ok=True) - return path - -def config_path() -> Path: - path = app_data_dir() / "config.yaml" - if not path.exists(): - path.write_text(DEFAULT_CONFIG, encoding="utf-8") - return path - -def state_path() -> Path: - path = app_data_dir() / "data" / "state.db" - path.parent.mkdir(parents=True, exist_ok=True) - return path - -def log_path() -> Path: - path = app_data_dir() / "logs" / "uploader.log" - path.parent.mkdir(parents=True, exist_ok=True) - return path - -DEFAULT_CONFIG = """telegram: - api_id: 0 - api_hash: "" - phone: "" - target: "me" - session_path: "data/telethon_user" -upload: - source_dir: "" - allowed_extensions: [] - max_file_size_mb: 0 - sleep_min_seconds: 60 - sleep_max_seconds: 90 - max_files_per_run: 3 - max_files_per_day: 100 - retry_attempts: 5 - backoff_base_seconds: 10 - floodwait_buffer_seconds: 5 - caption_template: "{name}" - send_mode: "document" -state: - db_path: "data/state.db" -logging: - log_path: "logs/uploader.log" - level: "INFO" -""" diff --git a/app/window.py b/app/window.py deleted file mode 100644 index b835c719..00000000 --- a/app/window.py +++ /dev/null @@ -1,621 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -import json -import sqlite3 -import sys -from urllib.request import Request, urlopen - -import yaml -from PySide6.QtCore import QProcess, QTimer, Qt, QPoint, QThread, Signal, QUrl -from PySide6.QtGui import QFont, QColor, QBrush -from PySide6.QtWidgets import ( - QAbstractItemView, QFileDialog, QFormLayout, QFrame, QHBoxLayout, QLabel, - QLineEdit, QMainWindow, QMessageBox, QPlainTextEdit, QProgressBar, - QPushButton, QSpinBox, QStackedWidget, QTableWidget, QTableWidgetItem, - QVBoxLayout, QWidget, QInputDialog, QComboBox, QScrollArea, QHeaderView, QGridLayout, -) -from PySide6.QtGui import QDesktopServices - -from .paths import app_data_dir, config_path, state_path - -STYLE = """ -QMainWindow { background: transparent; } -QWidget { background: #eef3f8; color: #172235; } -#appShell { background: #eef3f8; border-radius: 22px; } -QLabel { color: #202126; background: transparent; } -QFrame#sidebar, QFrame#card { background: #f9fbfe; border: 1px solid #d9e3ef; border-radius: 22px; } -QLabel#brand { color: #202126; font-size: 22px; font-weight: 700; } -QLabel#eyebrow { color: #77736d; font-size: 12px; font-weight: 600; } -QLabel#sectionTitle { color: #41688f; font-size: 13px; font-weight: 700; padding-top: 10px; padding-bottom: 2px; } -QLabel#fieldHint { color: #77736d; font-size: 11px; } -QLabel#statusBadge { color: #327454; background: #e7f4ec; border: 1px solid #cbe7d5; border-radius: 10px; padding: 5px 9px; font-size: 11px; font-weight: 700; } -QPushButton { background: #22252b; color: white; border: 0; border-radius: 9px; padding: 10px 16px; font-weight: 600; } -QPushButton:hover { background: #3a3f48; } -QPushButton#secondary { background: #e7eef6; color: #20344d; border: 1px solid #d5e0ec; } -QPushButton#secondary:hover { background: #dce8f4; } -QPushButton#coffeeButton { background: #ffdd00; color: #172235; border: 1px solid #e5c700; font-weight: 700; border-radius: 9px; padding: 8px 12px; } -QPushButton#coffeeButton:hover { background: #ffe633; border-color: #ffd000; } -QPushButton#danger { background: #b63838; } -QPushButton#danger:hover { background: #982d2d; } -QLabel#legend { color: #77736d; background: #f4f7fb; border: 1px solid #d9e3ef; border-radius: 10px; padding: 9px 12px; font-size: 11px; } -QLineEdit, QSpinBox, QComboBox { background: #ffffff; color: #202126; selection-background-color: #a9d8ff; selection-color: #172235; border: 1px solid #d2deeb; border-radius: 10px; padding: 8px 10px; min-height: 18px; } -QLineEdit:focus, QSpinBox:focus, QComboBox:focus { border: 1px solid #6ca9d7; background: #fbfdff; } -QSpinBox { padding-right: 32px; } -QSpinBox::up-button, QSpinBox::down-button { subcontrol-origin: border; width: 24px; border: 0; border-left: 1px solid #c4d3e3; background: #e8f0f8; } -QSpinBox::up-button { subcontrol-position: top right; border-top-right-radius: 9px; } -QSpinBox::down-button { subcontrol-position: bottom right; border-bottom-right-radius: 9px; } -QSpinBox::up-button:hover, QSpinBox::down-button:hover { background: #d5e5f3; } -QComboBox QAbstractItemView { background: white; color: #202126; border: 1px solid #dcd8d0; selection-background-color: #eeeae2; } -QScrollArea { border: 0; background: transparent; } -QScrollBar:vertical { background: transparent; width: 10px; margin: 4px 0; } -QScrollBar::handle:vertical { background: #d6d1c8; border-radius: 5px; min-height: 30px; } -QScrollBar::add-line:vertical, QScrollBar::sub-line:vertical { height: 0; } -QTableWidget, QTableWidget QTableCornerButton::section { background: white; color: #202126; border: 1px solid #e8e5de; border-radius: 14px; gridline-color: #efede7; } -QTableWidget { alternate-background-color: #fbfaf7; selection-background-color: #f1e4d0; selection-color: #202126; } -QTableWidget::item { color: #202126; padding: 3px 8px; border-bottom: 1px solid #f0eee9; } -QHeaderView::section { background: #f3f1eb; color: #514d46; border: 0; padding: 10px 8px; font-size: 12px; font-weight: 700; } -QProgressBar { background: #ede9e1; border: 0; border-radius: 6px; height: 12px; text-align: center; } -QProgressBar { color: #202126; } -QProgressBar::chunk { background: #ffad43; border-radius: 6px; } -QPlainTextEdit { background: #171a1f; color: #d9e3ee; border: 0; border-radius: 12px; padding: 12px; font-family: Consolas; } -QPlainTextEdit#activity { padding: 18px; } -QFrame#titleBar { background: #ffffff; border: 1px solid #e8e5de; border-radius: 16px; } -QPushButton#windowButton, QPushButton#maximizeButton, QPushButton#closeButton { background: transparent; color: #25252a; border: 0; border-radius: 8px; padding: 2px 0 0 0; min-width: 36px; max-width: 36px; min-height: 36px; max-height: 36px; font-family: "Segoe UI"; font-size: 18px; font-weight: 400; } -QPushButton#closeButton { font-size: 21px; } -QPushButton#windowButton:hover, QPushButton#maximizeButton:hover { background: #f0eee9; color: #25252a; } -QPushButton#closeButton:hover { background: #f4dede; color: #a52d2d; } -QPushButton#menuButton, QPushButton#maximizeButton, QPushButton#closeButton { background: transparent; color: #25252a; border: 0; } -QPushButton#menuButton { width: 42px; font-size: 22px; font-weight: 500; } -QPushButton#menuButton:hover { background: transparent; color: #25252a; } -""" - -TABLE_ROW_LIMIT = 100 -APP_VERSION = "1.0.6" -RELEASES_API_URL = "https://api.github.com/repos/amjadlle/TeleDrive/releases/latest" -UPDATE_CHECK_INTERVAL_MS = 24 * 60 * 60 * 1000 - - -class UpdateCheckWorker(QThread): - update_available = Signal(str, str) - - def run(self) -> None: - try: - request = Request(RELEASES_API_URL, headers={"Accept": "application/vnd.github+json", "User-Agent": "TeleDrive"}) - with urlopen(request, timeout=5) as response: - release = json.load(response) - latest = str(release.get("tag_name", "")).strip().lstrip("v") - release_url = str(release.get("html_url", "")).strip() - if latest and release_url and self._version_key(latest) > self._version_key(APP_VERSION): - self.update_available.emit(latest, release_url) - except Exception: - # Update checks are optional and must never affect app startup. - return - - @staticmethod - def _version_key(version: str) -> tuple[int, ...]: - try: - return tuple(int(part) for part in version.split(".")) - except ValueError: - return () - - -class MainWindow(QMainWindow): - def __init__(self) -> None: - super().__init__() - self.setWindowFlags(Qt.WindowType.FramelessWindowHint | Qt.WindowType.Window) - self.setAttribute(Qt.WidgetAttribute.WA_TranslucentBackground) - self.setWindowTitle(f"TeleDrive v{APP_VERSION}") - self.resize(1220, 780) - self.setMinimumSize(980, 650) - self.setStyleSheet(STYLE) - self.process = QProcess(self) - self.process.setProcessChannelMode(QProcess.ProcessChannelMode.MergedChannels) - self.process.readyReadStandardOutput.connect(self._read_process_output) - self.process.finished.connect(self._process_finished) - self.loop_timer = QTimer(self) - self.loop_timer.timeout.connect(lambda: self._start_uploader("--run-once")) - self.awaiting_login_prompt = False - self.pending_login_mode: str | None = None - self.login_code: str | None = None - self.auth_dir = app_data_dir() / "auth" - self.auth_dir.mkdir(parents=True, exist_ok=True) - self._build_ui() - self._load_settings() - self._refresh_all() - self._update_worker: UpdateCheckWorker | None = None - QTimer.singleShot(2000, self._check_for_updates) - self.refresh_timer = QTimer(self) - self.refresh_timer.timeout.connect(self._refresh_all) - self.refresh_timer.start(3000) - self.update_timer = QTimer(self) - self.update_timer.timeout.connect(self._check_for_updates) - self.update_timer.start(UPDATE_CHECK_INTERVAL_MS) - - def _build_ui(self) -> None: - root = QWidget(objectName="appShell") - outer = QVBoxLayout(root); outer.setContentsMargins(12, 12, 12, 12); outer.setSpacing(10) - self._outer_layout = outer - outer.addWidget(self._build_title_bar()) - layout = QHBoxLayout(); layout.setContentsMargins(0, 0, 0, 0); layout.setSpacing(18) - self._sidebar = self._build_sidebar(); layout.addWidget(self._sidebar) - self.pages = QStackedWidget() - self.pages.addWidget(self._build_dashboard()) - self.pages.addWidget(self._build_queue_page()) - self.pages.addWidget(self._build_history_page()) - self.pages.addWidget(self._build_settings_page()) - self.pages.addWidget(self._build_logs_page()) - layout.addWidget(self.pages, 1); outer.addLayout(layout, 1) - self.setCentralWidget(root) - - def _build_title_bar(self) -> QFrame: - bar = QFrame(objectName="titleBar"); bar.setFixedHeight(52) - layout = QHBoxLayout(bar); layout.setContentsMargins(12, 8, 8, 8); layout.setSpacing(0) - menu = QPushButton("≡", objectName="menuButton"); menu.setToolTip("Toggle sidebar") - menu.clicked.connect(lambda: self._toggle_sidebar()) - layout.addWidget(menu) - title = QLabel(f"TeleDrive v{APP_VERSION}"); title.setStyleSheet("font-weight: 700; color: #25252a;") - layout.addStretch(); layout.addWidget(title); layout.addStretch() - minimize = QPushButton("-", objectName="windowButton"); minimize.clicked.connect(self.showMinimized); layout.addWidget(minimize) - maximize = QPushButton("□", objectName="maximizeButton"); maximize.clicked.connect(self._toggle_maximized); layout.addWidget(maximize) - close = QPushButton("×", objectName="windowButton"); close.setObjectName("closeButton"); close.clicked.connect(self.close); layout.addWidget(close) - for button in (menu, minimize, maximize, close): - button.setCursor(Qt.CursorShape.PointingHandCursor) - bar.mousePressEvent = self._title_press; bar.mouseMoveEvent = self._title_move - self._title_bar = bar; self._drag_position = QPoint() - return bar - - def _toggle_sidebar(self) -> None: - self._sidebar.setVisible(not self._sidebar.isVisible()) - - def _toggle_maximized(self) -> None: - if self.isMaximized(): - self.showNormal() - self._set_floating_shell(True) - else: - self.showMaximized() - self._set_floating_shell(False) - - def _set_floating_shell(self, floating: bool) -> None: - self._outer_layout.setContentsMargins(12, 12, 12, 12) if floating else self._outer_layout.setContentsMargins(0, 0, 0, 0) - self._outer_layout.setSpacing(10 if floating else 0) - self.centralWidget().setStyleSheet("#appShell { border-radius: 22px; }" if floating else "#appShell { border-radius: 0px; }") - self._title_bar.setStyleSheet("QFrame#titleBar { background: #ffffff; border: 1px solid #e8e5de; border-radius: 16px; }" if floating else "QFrame#titleBar { background: #ffffff; border: 1px solid #e8e5de; border-radius: 0px; }") - - def _title_press(self, event) -> None: - if event.button() == Qt.MouseButton.LeftButton: - self._drag_position = event.globalPosition().toPoint() - self.frameGeometry().topLeft() - - def _title_move(self, event) -> None: - if event.buttons() & Qt.MouseButton.LeftButton and not self.isMaximized(): - self.move(event.globalPosition().toPoint() - self._drag_position) - - def _build_sidebar(self) -> QFrame: - sidebar = QFrame(objectName="sidebar") - sidebar.setFixedWidth(224) - layout = QVBoxLayout(sidebar) - layout.setContentsMargins(18, 20, 18, 18) - brand = QLabel("TeleDrive") - brand.setObjectName("brand") - layout.addWidget(brand) - layout.addWidget(QLabel("PERSONAL TELEGRAM CLOUD", objectName="eyebrow")) - layout.addSpacing(18) - self.nav_buttons: list[QPushButton] = [] - for index, label in enumerate(("Dashboard", "Upload Queue", "History", "Settings", "Activity Log")): - button = QPushButton(label) - button.setObjectName("secondary") - button.clicked.connect(lambda _=False, i=index: self._show_page(i)) - layout.addWidget(button) - self.nav_buttons.append(button) - layout.addStretch() - coffee_btn = QPushButton("☕ Buy Me a Coffee") - coffee_btn.setObjectName("coffeeButton") - coffee_btn.setCursor(Qt.CursorShape.PointingHandCursor) - coffee_btn.setToolTip("Support TeleDrive development") - coffee_btn.clicked.connect(lambda: QDesktopServices.openUrl(QUrl("https://buymeacoffee.com/amjadlle"))) - layout.addWidget(coffee_btn) - layout.addSpacing(6) - self.status_badge = QLabel("Ready", objectName="statusBadge") - layout.addWidget(self.status_badge) - layout.addWidget(QLabel("Private · Resumable · Local", objectName="eyebrow")) - return sidebar - - def _title(self, title: str, subtitle: str) -> QVBoxLayout: - layout = QVBoxLayout() - heading = QLabel(title) - heading.setFont(QFont("Segoe UI", 25, QFont.Weight.Bold)) - layout.addWidget(heading) - layout.addWidget(QLabel(subtitle, objectName="eyebrow")) - return layout - - def _card(self) -> tuple[QFrame, QVBoxLayout]: - card = QFrame(objectName="card") - box = QVBoxLayout(card) - box.setContentsMargins(20, 18, 20, 18) - return card, box - - def _build_dashboard(self) -> QWidget: - page = QWidget(); layout = QVBoxLayout(page); layout.setSpacing(16) - layout.addLayout(self._title("Welcome back", "Turn Telegram into your personal unlimited cloud storage.")) - card, box = self._card() - self.stats_label = QLabel("No uploads yet") - self.stats_label.setFont(QFont("Segoe UI", 18, QFont.Weight.DemiBold)) - self.stats_detail = QLabel("Set up your account and source folder to get started.", objectName="eyebrow") - self.progress = QProgressBar() - box.addWidget(self.stats_label); box.addWidget(self.stats_detail); box.addSpacing(6); box.addWidget(self.progress) - layout.addWidget(card) - actions = QHBoxLayout() - self.run_button = QPushButton("Run upload batch") - self.run_button.clicked.connect(lambda: self._start_uploader("--run-once")) - scan = QPushButton("Scan folder"); scan.setObjectName("secondary"); scan.clicked.connect(lambda: self._start_uploader("--scan-only")) - loop = QPushButton("Start automatic loop"); loop.setObjectName("secondary"); loop.clicked.connect(self._start_loop) - self.stop_loop_button = QPushButton("Stop loop"); self.stop_loop_button.setObjectName("secondary"); self.stop_loop_button.clicked.connect(self._stop_loop) - self.stop_button = QPushButton("Stop"); self.stop_button.setObjectName("danger"); self.stop_button.clicked.connect(self._stop_uploader) - self.stop_loop_button.setEnabled(False) - self.stop_button.setEnabled(False) - clear = QPushButton("Clear activity"); clear.setObjectName("secondary"); clear.clicked.connect(self._clear_activity) - reset = QPushButton("Reset all"); reset.setObjectName("danger"); reset.clicked.connect(self._reset_all) - for button, tip in ((self.run_button, "Upload queued files now."), (scan, "Scan the source folder and update the queue."), (loop, "Run uploads automatically at the selected interval."), (self.stop_loop_button, "Turn off the automatic loop."), (self.stop_button, "Stop the current upload process."), (clear, "Clear only this activity panel."), (reset, "Delete all tracked file history and start fresh.")): - button.setToolTip(tip) - actions.addWidget(self.run_button); actions.addWidget(scan); actions.addWidget(loop); actions.addWidget(self.stop_loop_button); actions.addWidget(self.stop_button); actions.addWidget(clear); actions.addWidget(reset); actions.addStretch() - layout.addLayout(actions) - layout.addWidget(QLabel("Live activity", objectName="eyebrow")) - self.activity = QPlainTextEdit(); self.activity.setObjectName("activity"); self.activity.setReadOnly(True) - self.activity.setPlaceholderText("No activity yet\n\nRun a batch or scan your source folder to see live progress here.") - self.activity.setCenterOnScroll(False) - layout.addWidget(self.activity, 1) - layout.addWidget(QLabel("Legend: Run batch = upload queued files • Scan = find new files • Loop = repeat automatically • Stop = cancel current work • Clear activity = clear messages • Reset all = remove tracked history", objectName="legend")) - return page - - def _table_page(self, title: str, subtitle: str) -> tuple[QWidget, QTableWidget]: - page = QWidget(); layout = QVBoxLayout(page); layout.setSpacing(16) - layout.addLayout(self._title(title, subtitle)) - table = QTableWidget(0, 4) - table.setHorizontalHeaderLabels(("File", "Status", "Attempts", "Updated")) - table.setEditTriggers(QAbstractItemView.EditTrigger.NoEditTriggers) - table.setSelectionBehavior(QAbstractItemView.SelectionBehavior.SelectRows) - table.setAlternatingRowColors(True) - table.setShowGrid(False) - table.setWordWrap(False) - table.verticalHeader().setVisible(False) - table.verticalHeader().setDefaultSectionSize(42) - header = table.horizontalHeader() - header.setSectionResizeMode(0, QHeaderView.ResizeMode.Stretch) - header.setSectionResizeMode(1, QHeaderView.ResizeMode.ResizeToContents) - header.setSectionResizeMode(2, QHeaderView.ResizeMode.ResizeToContents) - header.setSectionResizeMode(3, QHeaderView.ResizeMode.ResizeToContents) - header.setMinimumSectionSize(90) - layout.addWidget(table, 1) - return page, table - - def _build_queue_page(self) -> QWidget: - page, self.queue_table = self._table_page("Upload Queue", "Pending and retryable items") - return page - - def _build_history_page(self) -> QWidget: - page, self.history_table = self._table_page("History", "Uploaded files and items that need attention") - return page - - def _build_settings_page(self) -> QWidget: - page = QWidget(); layout = QVBoxLayout(page); layout.setSpacing(16) - layout.addLayout(self._title("Settings", "Stored privately in your Windows app-data folder")) - scroll = QScrollArea(); scroll.setWidgetResizable(True); scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarPolicy.ScrollBarAlwaysOff) - content = QWidget(); content_layout = QVBoxLayout(content); content_layout.setContentsMargins(0, 0, 4, 0) - card, form_box = self._card(); form = QFormLayout(); form.setHorizontalSpacing(18); form.setVerticalSpacing(10) - form.setLabelAlignment(Qt.AlignmentFlag.AlignLeft | Qt.AlignmentFlag.AlignVCenter) - form.setFieldGrowthPolicy(QFormLayout.FieldGrowthPolicy.AllNonFixedFieldsGrow) - form.setRowWrapPolicy(QFormLayout.RowWrapPolicy.DontWrapRows) - self.api_id = QLineEdit(); self.api_hash = QLineEdit(); self.api_hash.setEchoMode(QLineEdit.EchoMode.Password) - self.phone = QLineEdit(); self.phone.setPlaceholderText("+country code and number") - self.target = QLineEdit(); self.target.setPlaceholderText("me, @channel, or channel ID") - self.source = QLineEdit(); self.source.setPlaceholderText("Select the folder containing files to upload") - for text_field in (self.api_id, self.api_hash, self.phone, self.target): - text_field.setFixedWidth(420) - self.source.setFixedWidth(300) - choose = QPushButton("Choose folder"); choose.setObjectName("secondary"); choose.setFixedWidth(112); choose.clicked.connect(self._choose_folder) - source_row = QHBoxLayout(); source_row.setSpacing(8); source_row.addWidget(self.source); source_row.addWidget(choose); source_row.addStretch() - self.per_run = QSpinBox(); self.per_run.setRange(1, 10000) - self.per_day = QSpinBox(); self.per_day.setRange(1, 50000) - self.sleep_min = QSpinBox(); self.sleep_min.setRange(1, 86400); self.sleep_min.setSuffix(" sec") - self.sleep_max = QSpinBox(); self.sleep_max.setRange(1, 86400); self.sleep_max.setSuffix(" sec") - self.run_interval = QSpinBox(); self.run_interval.setRange(1, 10080); self.run_interval.setSuffix(" min") - self.retry_attempts = QSpinBox(); self.retry_attempts.setRange(1, 50) - self.backoff = QSpinBox(); self.backoff.setRange(0, 3600); self.backoff.setSuffix(" sec") - self.flood_buffer = QSpinBox(); self.flood_buffer.setRange(0, 3600); self.flood_buffer.setSuffix(" sec") - for numeric in (self.per_run, self.per_day, self.sleep_min, self.sleep_max, self.run_interval, self.retry_attempts, self.backoff, self.flood_buffer): - numeric.setFixedWidth(150) - self.send_mode = QComboBox(); self.send_mode.setFixedWidth(300); self.send_mode.addItem("Document (file attachment)", "document"); self.send_mode.addItem("Media (photo/video)", "media"); self.send_mode.addItem("Auto (Telegram default)", "auto") - form.addRow(QLabel("Telegram connection", objectName="sectionTitle")) - form.addRow("Telegram API ID", self.api_id); form.addRow("Telegram API hash", self.api_hash) - form.addRow("Phone number", self.phone); form.addRow("Target channel / Saved Messages", self.target) - form.addRow(QLabel("Upload source", objectName="sectionTitle")) - form.addRow("Source folder", source_row) - upload_limits = QGridLayout(); upload_limits.setHorizontalSpacing(18); upload_limits.setVerticalSpacing(10) - upload_limits.setColumnStretch(1, 1); upload_limits.setColumnStretch(3, 1) - upload_limits.addWidget(QLabel("Files per run"), 0, 0); upload_limits.addWidget(self.per_run, 0, 1) - upload_limits.addWidget(QLabel("Files per day"), 0, 2); upload_limits.addWidget(self.per_day, 0, 3) - for column in (0, 2): - upload_limits.itemAtPosition(0, column).widget().setMinimumWidth(0) - form.addRow(upload_limits) - form.addRow(QLabel("Timing & reliability", objectName="sectionTitle")) - timing = QGridLayout(); timing.setHorizontalSpacing(18); timing.setVerticalSpacing(10) - timing.setColumnStretch(1, 1); timing.setColumnStretch(3, 1) - timing_rows = (("Delay between uploads (minimum)", self.sleep_min, "Delay between uploads (maximum)", self.sleep_max), ("Delay between automatic runs", self.run_interval, "Retry attempts per file", self.retry_attempts), ("Retry backoff base", self.backoff, "Flood-wait buffer", self.flood_buffer)) - for row, (left_label, left_widget, right_label, right_widget) in enumerate(timing_rows): - left = QLabel(left_label); left.setWordWrap(True); left.setMaximumWidth(210) - right = QLabel(right_label); right.setWordWrap(True); right.setMaximumWidth(210) - timing.addWidget(left, row, 0); timing.addWidget(left_widget, row, 1) - timing.addWidget(right, row, 2); timing.addWidget(right_widget, row, 3) - for column in (0, 2): - timing.setColumnMinimumWidth(column, 120) - form.addRow(timing) - form.addRow(QLabel("File handling", objectName="sectionTitle")) - form.addRow("Send files as", self.send_mode) - form_box.addLayout(form) - quick_help = QLabel("Quick help: API ID/hash identify your Telegram account. Source folder is scanned for files. Files per run/day control limits. Timing values control delays. Retry settings control recovery after failures. Save changes before starting an upload.", objectName="legend") - quick_help.setMaximumWidth(420); quick_help.setWordWrap(True) - form_box.addWidget(quick_help) - save = QPushButton("Save settings"); save.setFixedWidth(420); save.setMinimumHeight(42); save.clicked.connect(self._save_settings); form_box.addWidget(save) - for widget, tip in ((self.api_id, "Telegram API ID from my.telegram.org."), (self.api_hash, "Telegram API hash from my.telegram.org."), (self.source, "Folder that TeleDrive scans for files."), (self.run_interval, "How often the automatic loop starts a new run."), (self.retry_attempts, "How many times a failed upload is retried."), (self.send_mode, "Choose whether files are sent as documents or media.")): - widget.setToolTip(tip) - content_layout.addWidget(card) - config_note = QLabel(f"Config location: {config_path()}", objectName="fieldHint"); config_note.setMaximumWidth(420) - content_layout.addWidget(config_note) - help_note = QLabel("Help is also shown here so it is always visible, even when a tooltip disappears.", objectName="legend") - help_note.setMaximumWidth(420); help_note.setWordWrap(True) - content_layout.addWidget(help_note) - content_layout.addStretch() - scroll.setWidget(content) - layout.addWidget(scroll, 1) - return page - - def _build_logs_page(self) -> QWidget: - page = QWidget(); layout = QVBoxLayout(page); layout.setSpacing(16) - layout.addLayout(self._title("Activity Log", "Recent uploader output")) - self.log_view = QPlainTextEdit(); self.log_view.setReadOnly(True); layout.addWidget(self.log_view, 1) - return page - - def _load_settings(self) -> None: - raw = yaml.safe_load(config_path().read_text(encoding="utf-8")) or {} - telegram, upload = raw.get("telegram", {}), raw.get("upload", {}) - api_id = int(telegram.get("api_id", 0) or 0) - self.api_id.setText(str(api_id) if api_id else "") - self.api_hash.setText(str(telegram.get("api_hash", ""))) - self.phone.setText(str(telegram.get("phone", ""))) - self.target.setText(str(telegram.get("target", "me"))) - self.source.setText(str(upload.get("source_dir", ""))) - self.per_run.setValue(int(upload.get("max_files_per_run", 3))) - self.per_day.setValue(int(upload.get("max_files_per_day", 100))) - self.sleep_min.setValue(int(upload.get("sleep_min_seconds", 60))) - self.sleep_max.setValue(int(upload.get("sleep_max_seconds", 90))) - self.run_interval.setValue(int(raw.get("app", {}).get("run_interval_minutes", 60))) - self.retry_attempts.setValue(int(upload.get("retry_attempts", 5))) - self.backoff.setValue(int(upload.get("backoff_base_seconds", 10))) - self.flood_buffer.setValue(int(upload.get("floodwait_buffer_seconds", 5))) - mode = str(upload.get("send_mode", "document")).lower() - self.send_mode.setCurrentIndex(max(0, self.send_mode.findData(mode))) - - def _save_settings(self) -> None: - try: - api_id = int(self.api_id.text().strip()) - if api_id <= 0 or not self.api_hash.text().strip(): - raise ValueError("Enter your valid Telegram API ID and API hash.") - source = Path(self.source.text().strip()) - if not source.is_dir(): - raise ValueError("Choose an existing source folder.") - if self.sleep_min.value() > self.sleep_max.value(): - raise ValueError("Minimum upload delay cannot exceed maximum delay.") - except ValueError as exc: - QMessageBox.warning(self, "Settings", str(exc)); return - raw = yaml.safe_load(config_path().read_text(encoding="utf-8")) or {} - raw.setdefault("telegram", {}).update({"api_id": api_id, "api_hash": self.api_hash.text().strip(), "phone": self.phone.text().strip(), "target": self.target.text().strip() or "me"}) - raw.setdefault("upload", {}).update({"source_dir": str(source), "max_files_per_run": self.per_run.value(), "max_files_per_day": self.per_day.value(), "sleep_min_seconds": self.sleep_min.value(), "sleep_max_seconds": self.sleep_max.value(), "retry_attempts": self.retry_attempts.value(), "backoff_base_seconds": self.backoff.value(), "floodwait_buffer_seconds": self.flood_buffer.value(), "send_mode": self.send_mode.currentData()}) - raw.setdefault("app", {})["run_interval_minutes"] = self.run_interval.value() - temp = config_path().with_suffix(".yaml.tmp") - temp.write_text(yaml.safe_dump(raw, sort_keys=False), encoding="utf-8") - temp.replace(config_path()) - self.activity.appendPlainText("Settings saved.") - QMessageBox.information(self, "Settings", "Settings saved. On your first upload, Telegram will ask for a login code in the terminal setup flow.") - - def _choose_folder(self) -> None: - folder = QFileDialog.getExistingDirectory(self, "Choose source folder", self.source.text()) - if folder: self.source.setText(folder) - - def _show_page(self, index: int) -> None: - self.pages.setCurrentIndex(index) - if index == 1: - self._refresh_table(self.queue_table, "status IN ('pending', 'failed')") - elif index == 2: - self._refresh_table(self.history_table, "status = 'uploaded'") - - def _check_for_updates(self) -> None: - if self._update_worker is not None and self._update_worker.isRunning(): - return - self._update_worker = UpdateCheckWorker(self) - self._update_worker.update_available.connect(self._show_update_available) - self._update_worker.finished.connect(self._update_worker.deleteLater) - self._update_worker.start() - - def _show_update_available(self, version: str, release_url: str) -> None: - answer = QMessageBox.information( - self, - "TeleDrive update available", - f"TeleDrive v{version} is available. You are using v{APP_VERSION}.", - QMessageBox.StandardButton.Open | QMessageBox.StandardButton.Cancel, - QMessageBox.StandardButton.Open, - ) - if answer == QMessageBox.StandardButton.Open: - QDesktopServices.openUrl(QUrl(release_url)) - - def closeEvent(self, event) -> None: - if self._update_worker is not None and self._update_worker.isRunning(): - self._update_worker.requestInterruption() - self._update_worker.wait(6000) - super().closeEvent(event) - - def _command(self, mode: str) -> list[str]: - if getattr(sys, "frozen", False): - return [sys.executable, "--cli", "--config", str(config_path()), "--auth-dir", str(self.auth_dir), mode] - return [sys.executable, str(Path(__file__).parents[1] / "uploader.py"), "--config", str(config_path()), "--auth-dir", str(self.auth_dir), mode] - - def _start_uploader(self, mode: str) -> None: - if self.process.state() != QProcess.ProcessState.NotRunning: - QMessageBox.information(self, "Uploader", "An upload process is already running."); return - if not self.source.text().strip() or not self.api_hash.text().strip(): - self.pages.setCurrentIndex(3) - QMessageBox.information(self, "Setup required", "Complete and save the Settings page before running uploads."); return - command = self._command(mode) - for auth_file in (self.auth_dir / "code.txt", self.auth_dir / "password.txt"): - auth_file.unlink(missing_ok=True) - self.activity.appendPlainText("$ " + " ".join(command)) - self.status_badge.setText("Working") - self.run_button.setEnabled(False) - self.stop_button.setEnabled(True) - self.process.start(command[0], command[1:]) - - def _start_loop(self) -> None: - if self.run_interval.value() < 1: - return - self._start_uploader("--run-once") - self.loop_timer.start(self.run_interval.value() * 60 * 1000) - self.status_badge.setText("Automatic loop enabled") - self.stop_loop_button.setEnabled(True) - - def _stop_loop(self) -> None: - self.loop_timer.stop() - self.status_badge.setText("Ready") - self.activity.appendPlainText("Automatic loop stopped.") - self.stop_loop_button.setEnabled(False) - - def _stop_uploader(self) -> None: - if self.process.state() != QProcess.ProcessState.NotRunning: - self.process.terminate() - if not self.process.waitForFinished(5000): self.process.kill() - self.activity.appendPlainText("Upload process stopped by user.") - self.status_badge.setText("Stopped") - self.stop_button.setEnabled(False) - else: - self.activity.appendPlainText("No upload process is running.") - - def _clear_activity(self) -> None: - self.activity.clear() - self.activity.appendPlainText("Activity cleared.") - - def _reset_all(self) -> None: - if self.process.state() != QProcess.ProcessState.NotRunning or self.loop_timer.isActive(): - QMessageBox.information(self, "Stop first", "Stop the upload process and automatic loop before resetting.") - return - answer = QMessageBox.warning(self, "Reset all tracked files", "This removes all queue and upload history from TeleDrive. Your files and settings will not be deleted. Continue?", QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, QMessageBox.StandardButton.No) - if answer != QMessageBox.StandardButton.Yes: - return - conn = self._connection() - if conn is not None: - try: - conn.execute("DELETE FROM files") - conn.commit() - finally: - conn.close() - self.activity.appendPlainText("All tracked files and history were reset.") - self.status_badge.setText("Ready") - self._refresh_all() - - def _read_process_output(self) -> None: - text = bytes(self.process.readAllStandardOutput()).decode("utf-8", errors="replace").rstrip() - if text: - self.activity.appendPlainText(text) - self.log_view.appendPlainText(text) - self._answer_login_prompt(text) - - def _answer_login_prompt(self, text: str) -> None: - if self.awaiting_login_prompt: - return - prompt = None - secret = False - if "Please enter the code" in text: - prompt = "Enter the Telegram code sent to your phone." - self.pending_login_mode = "code" - elif "Please enter your password" in text or "Please enter the password" in text: - prompt = "Enter your Telegram two-step verification password." - secret = True - self.pending_login_mode = "password" - if prompt is None: - return - self.awaiting_login_prompt = True - answer, ok = QInputDialog.getText(self, "Telegram sign in", prompt, QLineEdit.EchoMode.Password if secret else QLineEdit.EchoMode.Normal) - if ok and answer.strip() and self.pending_login_mode: - value = answer.strip() - if self.pending_login_mode == "code": - self.login_code = value - response = self.auth_dir / f"{self.pending_login_mode}.txt" - response.write_text(value, encoding="utf-8") - else: - self._stop_uploader() - self.awaiting_login_prompt = False - - - def _process_finished(self, exit_code: int, _status: QProcess.ExitStatus) -> None: - self.run_button.setEnabled(True) - self.stop_button.setEnabled(False) - self.awaiting_login_prompt = False - self.status_badge.setText("Ready" if exit_code == 0 else "Needs attention") - self.activity.appendPlainText(f"Uploader finished with exit code {exit_code}.") - self._refresh_all() - - def _refresh_all(self) -> None: - self._refresh_stats() - current_page = self.pages.currentIndex() - if current_page == 1: - self._refresh_table(self.queue_table, "status IN ('pending', 'failed')") - elif current_page == 2: - self._refresh_table(self.history_table, "status = 'uploaded'") - - def _connection(self) -> sqlite3.Connection | None: - if not state_path().exists(): return None - conn = sqlite3.connect(state_path(), timeout=2); conn.execute("PRAGMA busy_timeout=2000"); return conn - - def _refresh_stats(self) -> None: - conn = self._connection() - if conn is None: - self.stats_label.setText("No uploads yet"); self.stats_detail.setText("Save settings, then scan the source folder."); self.progress.setValue(0); return - try: - total = conn.execute("SELECT COUNT(*) FROM files").fetchone()[0] - uploaded = conn.execute("SELECT COUNT(*) FROM files WHERE status='uploaded'").fetchone()[0] - failed = conn.execute("SELECT COUNT(*) FROM files WHERE status='failed'").fetchone()[0] - except sqlite3.Error: - return - finally: - conn.close() - self.stats_label.setText(f"{uploaded:,} uploaded • {max(total-uploaded, 0):,} remaining") - self.stats_detail.setText(f"{total:,} files tracked • {failed:,} need attention") - self.progress.setValue(round(uploaded * 100 / total) if total else 0) - - def _refresh_table(self, table: QTableWidget, condition: str) -> None: - conn = self._connection() - if conn is None: table.setRowCount(0); return - try: - rows = conn.execute(f"SELECT path, status, attempts, datetime(last_update_ts, 'unixepoch', 'localtime') FROM files WHERE {condition} ORDER BY last_update_ts DESC LIMIT ?", (TABLE_ROW_LIMIT,)).fetchall() - except sqlite3.Error: - return - finally: - conn.close() - table.setUpdatesEnabled(False) - table.blockSignals(True) - try: - table.clearContents() - table.setRowCount(len(rows)) - status_colors = {"pending": "#9a661d", "failed": "#b63838", "uploaded": "#327454"} - for row_index, row in enumerate(rows): - for column, value in enumerate(row): - text = str(value or "") - item = QTableWidgetItem(text) - if column == 0: - item.setToolTip(text) - elif column == 1: - item.setTextAlignment(Qt.AlignmentFlag.AlignCenter) - color = status_colors.get(text.lower()) - if color: - item.setForeground(QBrush(QColor(color))) - table.setItem(row_index, column, item) - finally: - table.blockSignals(False) - table.setUpdatesEnabled(True) diff --git a/build-installer.ps1 b/build-installer.ps1 deleted file mode 100644 index c014c00d..00000000 --- a/build-installer.ps1 +++ /dev/null @@ -1,28 +0,0 @@ -$ErrorActionPreference = 'Stop' -$root = Split-Path -Parent $MyInvocation.MyCommand.Path -Set-Location $root -$compilerPath = (Get-Command ISCC -ErrorAction SilentlyContinue).Source -if (-not $compilerPath) { - $fallback = Join-Path ${env:ProgramFiles(x86)} 'Inno Setup 6\ISCC.exe' - if (Test-Path $fallback) { $compilerPath = $fallback } -} -if (-not $compilerPath) { - $fallbacks = @( - (Join-Path ${env:ProgramFiles} 'Inno Setup 7\ISCC.exe'), - (Join-Path ${env:ProgramFiles(x86)} 'Inno Setup 7\ISCC.exe'), - (Join-Path ${env:LOCALAPPDATA} 'Programs\Antigravity IDE\resources\app\node_modules\innosetup\bin\ISCC.exe') - ) - $found = $fallbacks | Where-Object { Test-Path $_ } | Select-Object -First 1 - if ($found) { $compilerPath = $found } -} -if (-not $compilerPath) { - throw "Inno Setup 6 is required. Install it from https://jrsoftware.org/isinfo.php, then run this script again." -} -$appExe = Join-Path $root 'dist\TeleDrive.exe' -if (-not (Test-Path $appExe)) { - throw "Application executable not found at '$appExe'. Run build.ps1 first." -} -& $compilerPath (Join-Path $root 'installer\TelegramAutoUpload.iss') -$installer = Get-ChildItem (Join-Path $root 'dist-installer') -Filter '*.exe' | Sort-Object LastWriteTime -Descending | Select-Object -First 1 -if (-not $installer) { throw 'Inno Setup finished without producing an installer.' } -Write-Host "Installer completed: $($installer.FullName)" diff --git a/build-linux.sh b/build-linux.sh deleted file mode 100644 index 609fa26e..00000000 --- a/build-linux.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "$ROOT_DIR" - -python3 -m venv .venv -.venv/bin/python -m pip install --upgrade pip -.venv/bin/python -m pip install -r requirements.txt pyinstaller - -rm -rf build dist -.venv/bin/python -m PyInstaller --noconfirm --clean --windowed \ - --name "TeleDrive" --hidden-import uploader desktop.py - -mkdir -p dist-installer -tar -czf dist-installer/TeleDrive-linux-x86_64.tar.gz -C dist TeleDrive - -echo "Linux package created: dist-installer/TeleDrive-linux-x86_64.tar.gz" diff --git a/build.ps1 b/build.ps1 deleted file mode 100644 index 21147faf..00000000 --- a/build.ps1 +++ /dev/null @@ -1,7 +0,0 @@ -$ErrorActionPreference = 'Stop' -python -m pip install --upgrade pip -python -m pip install -r requirements.txt -python -m pip install pyinstaller -$icon = if (Test-Path 'installer\TeleDrive.ico') { '--icon', 'installer\TeleDrive.ico' } else { @() } -pyinstaller --noconfirm --clean --onefile --windowed --name "TeleDrive" @icon --hidden-import uploader desktop.py -Write-Host "Build completed: dist\TeleDrive.exe" diff --git a/config.example.yaml b/config.example.yaml index 88a336a3..38ce63fc 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -1,31 +1,95 @@ -telegram: - api_id: 123456 - api_hash: "YOUR_API_HASH" - # phone is optional; Telethon may still ask interactively on first login - phone: "" - # Temporary safe target for first test: your Saved Messages. - # Later change this to your private channel username or numeric channel ID. - target: "me" - session_path: "data/telethon_user" - -upload: - source_dir: "./uploads" - allowed_extensions: [] # empty means all files - max_file_size_mb: 0 # 0 means no size limit - sleep_min_seconds: 60 - sleep_max_seconds: 90 - max_files_per_run: 3 - max_files_per_day: 100 - retry_attempts: 5 - backoff_base_seconds: 10 - floodwait_buffer_seconds: 5 - caption_template: "{name}" - # document = file attachment, media = Telegram photo/video, auto = Telethon default - send_mode: "document" +# TeleDrive configuration example (copy to config/config.yaml and edit). +# +# The unified WebUI/REST/S3/local-job service reads this file as its ONLY +# settings source. Except TZ, PUID/PGID and TELEDRIVE_CONFIG, nothing is +# configured through environment variables. Secrets belong ONLY in this +# file and the session files; never commit real values. +# --------------------------------------------------------------------------- +# Web service: authentication, listeners and request limits. +# --------------------------------------------------------------------------- +web: + user: "admin" + password: "CHANGE_ME_AT_LEAST_16_CHARACTERS" + allowed_hosts: "localhost,127.0.0.1,[::1]" + allow_insecure: false # true only for explicitly trusted local setups + bind_address: "0.0.0.0" # in-container bind; host mapping stays loopback + port: 8080 + s3_port: 9000 + rest_max_upload_bytes: 2147483648 # 2 GiB + auth_failure_limit: 5 + auth_failure_window_seconds: 300 + # Optional HTTPS: generate certs with scripts/ops/generate_ssl.py and set: + # ssl_certfile: "/config/tls/server.crt" + # ssl_keyfile: "/config/tls/server.key" + +# --------------------------------------------------------------------------- +# Unified storage: Telethon USER MTProto (the only live transport). +# Sessions are pre-authorized files; phone numbers and 2FA codes are never +# stored here (they are used once, interactively, by +# scripts/dev/create_storage_user_session.py). +# --------------------------------------------------------------------------- +storage: + # Dedicated private broadcast channel the authorized user can post to. + # Numeric private channel IDs must be visible in the user's dialogs. + channel: "YOUR_PRIVATE_CHANNEL_ID_OR_USERNAME" + + # Credential rotation across the apis: list below: + # single use only the first set + # round_robin distribute sends across the sets in turn (default) + # failover keep one set until a definite flood-wait rejection, + # then switch to the next with a fresh send intent + # parallel upload chunks concurrently across all sets + strategy: round_robin + + scratch_dir: "/data/tmp" + + # Persistent S3 credential-encryption master key file. + s3_master_key_file: "/config/s3-master.key" + + # One or more Telegram API credential sets. Every set must resolve the + # SAME dedicated channel above. account labels are optional and default + # to mtproto-user, mtproto-user-2, ... + apis: + - api_id: 111111 + api_hash: "YOUR_TELEGRAM_API_HASH_1" + session_file: "/config/mtproto-user.session" + account: mtproto-user + + - api_id: 222222 + api_hash: "YOUR_TELEGRAM_API_HASH_2" + session_file: "/config/mtproto-user-2.session" + account: mtproto-user-2 + +# --------------------------------------------------------------------------- +# Durable local-file jobs and the opt-in lease-fenced legacy ingestion. +# --------------------------------------------------------------------------- +jobs: + source_root: "/uploads" + auto_run: true # process explicitly queued jobs + +# --------------------------------------------------------------------------- +# SQL state: the shared object catalog database. +# --------------------------------------------------------------------------- state: - db_path: "data/state.db" + database_url: "sqlite:////data/state.db" + # Or any synchronous SQLAlchemy URL whose driver/dialect is installed: + # database_url: "postgresql+psycopg://user:pass@db/teledrive" + # database_url: "mysql+pymysql://user:pass@db/teledrive" + + # Optional DB drivers NOT bundled with the image (e.g. oracledb) are pip + # installed into /data/site-packages at startup and added to sys.path: + # db_extra_packages: ["oracledb"] + + allowed_sqlite_roots: "/data" -logging: - log_path: "logs/uploader.log" - level: "INFO" +# --- +# Internal path allowlists for the legacy uploader validation. +# --------------------------------------------------------------------------- +paths: + source_roots: "/uploads" + session_roots: "/data/sessions" + log_roots: "/data" + split_dir: "/data/tmp" + split_min_free_bytes: 536870912 + lock_dir: "/data/tmp" diff --git a/desktop.py b/desktop.py deleted file mode 100644 index cb3d2767..00000000 --- a/desktop.py +++ /dev/null @@ -1,5 +0,0 @@ -"""PyInstaller entry point for the desktop application.""" -from app.main import main - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..12ac8264 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,43 @@ +# One process/container serves WebUI 8080 and S3 9000; same SQL/Telegram metadata. +# All service settings come from config/config.yaml (see config.example.yaml). +# The only environment variables are TZ, PUID/PGID and TELEDRIVE_CONFIG. +services: + teledrive: + build: + context: . + args: + TELEDRIVE_DB_EXTRA_PACKAGES: ${TELEDRIVE_DB_EXTRA_PACKAGES:-} + container_name: teledrive + image: teledrive:local + restart: unless-stopped + user: "${PUID:-1000}:${PGID:-1000}" + ports: + # The service binds 0.0.0.0 inside the container; keep the host side + # loopback unless a reverse proxy/VPN explicitly publishes it. + - "127.0.0.1:${TELEDRIVE_WEB_PORT:-8080}:8080" + - "127.0.0.1:${TELEDRIVE_S3_PORT:-9000}:9000" + environment: + TELEDRIVE_CONFIG: /config/config.yaml + TZ: ${TZ:-Europe/Istanbul} + init: true + read_only: true + cap_drop: [ALL] + security_opt: ["no-new-privileges:true"] + pids_limit: 256 + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,size=64m + volumes: + - ./config:/config + - ./data:/data + - ./uploads:/uploads:ro + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/api/health', timeout=5); urllib.request.urlopen('http://127.0.0.1:9000/__health', timeout=5)"] + interval: 30s + timeout: 8s + start_period: 20s + retries: 5 + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" diff --git a/docs/BACKUP_RESTORE.md b/docs/BACKUP_RESTORE.md new file mode 100644 index 00000000..bea8397f --- /dev/null +++ b/docs/BACKUP_RESTORE.md @@ -0,0 +1,95 @@ +# Backup and Restore Operations + +> **Audience:** deployment operators and release maintainers. +> **Purpose:** create and verify protected SQL metadata backups and rehearse recovery **without modifying a live installation**. +> **Related:** [Project Status and production sign-off](IMPLEMENTATION_STATUS.md#3-production-acceptance-not-yet-signed-off) · [Security Policy](SECURITY.md). + +## 1. What a backup contains + +TeleDrive stores **payload bytes in Telegram** and **object references/versions, job state, MTProto message coordinates, and encrypted S3 credential metadata in SQL**. An SQL backup is **not** an independent copy of the Telegram objects. + +Protect these as separate, coordinated recovery inputs: + +| Asset | Why it matters | Handling | +| --- | --- | --- | +| Metadata database | Object heads, history, chunk identities, durable jobs | Encrypted backup; recorded independent SHA-256 | +| Telegram USER StringSession / API credentials | Required for current REST/S3/local-job access and same-channel message history | Protected `0600` private file/secret manager, independent of SQL archive | +| Historical Bot credentials (if any) | No longer usable by the Telethon-only runtime; retain only as a migration audit trail | Separately protected archive | +| S3 encryption master key | Decrypts issued S3 access secrets | Separate secret backup; never silently rotate/replace | +| Deployment configuration | DB URL, channel target, source mounts and allowlists | Protected copy; no credentials in source control | + +Never store these assets in Git, GitHub Actions logs or public test artifacts. Record the SHA-256 and restore instructions in the operator's private change record. + +## 2. Back up SQLite metadata + +Run on the TeleDrive host with a destination on encrypted, access-controlled storage: + +~~~bash +python scripts/ops/backup_metadata.py \ + --database /data/teledrive.db \ + --output /secure-backups/teledrive-YYYY-MM-DD.db +~~~ + +The command uses SQLite's online backup API, checks `PRAGMA integrity_check`, restricts output to `0600` and refuses to overwrite an existing backup. Save the printed SHA-256 **independently**. + +Validate the saved copy without touching the live database: + +~~~bash +python scripts/ops/verify_metadata_backup.py \ + --database /secure-backups/teledrive-YYYY-MM-DD.db \ + --sha256 EXPECTED_ARCHIVE_SHA256 +~~~ + +The verifier checks SQLite consistency, foreign keys, current HEAD hashes and published/removed version bookkeeping. It rejects orphan version removal entries and a revoked current HEAD. It does **not** contact Telegram, restore unavailable remote media, rotate secrets or authorize garbage collection. + +### SQLite restore rehearsal + +1. Create a **separate disposable installation** with workers initially stopped. +2. Copy the verified snapshot to the staging installation's database path with owner-only permissions. +3. Verify schema, SQL HEAD pointers, version history, references, MTProto send intents and credential decryptability with the staged secrets. +4. Reconnect only to approved **test/original owner-authorized** Telegram media, verify full SHA-256 and at least one cross-chunk range. +5. Record the results privately. **Never overwrite a running production database.** + +## 3. PostgreSQL and MySQL/MariaDB + +### PostgreSQL + +Use a consistent native logical backup, for example `pg_dump -Fc`, under a dedicated and appropriately scoped backup role. Retrieve credentials through a secret store; **do not put a password in a shell command or committed script**. + +Rehearse a native `pg_restore` into a **new, empty, isolated** database. Compare schema, object heads and generations, published/deleted versions, chunk references and SHA-256 hashes, MTProto send receipts, durable jobs and S3 credential decryptability. + +### MySQL / MariaDB + +Choose an engine-aware, transactionally consistent backup method such as `mysqldump --single-transaction` where applicable. Validate storage-engine behavior, schema, triggers/routines and required migrations. Restore into a **separately provisioned staging database**; never replace the production database as a test. + +## 4. Automated restore evidence + +### Disposable database integration CI + +The [Database Integration workflow](../.github/workflows/integration-databases.yml) exercises PostgreSQL 17 `pg_dump -Fc`/`pg_restore` and MySQL 8.4 `mysqldump --single-transaction`/restore using **new disposable databases**. + +It seeds multiple published versions, chunk-boundary SHA metadata, delete markers/version removal, tags, MTProto send intent/receipt records, queued jobs and independently encrypted **test** S3 credentials. Fresh SQL connections check exact heads/generations, chunk identities, retention semantics, job state and decryptability. + +**Limit:** this checks actual SQL backup and restore machinery, **not** original Telegram media or a production secret-recovery drill. + +### Dedicated USER-MTProto recovery CI + +`scripts/ops/rehearse_telegram_dr.py` verifies original-media readback through an authorized **read-only USER MTProto** session (`TELEDRIVE_STORAGE_MTPROTO_API_ID` / `TELEDRIVE_STORAGE_MTPROTO_API_HASH` / `TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE`, plus `--target` for the original channel). Old `api-bot` chunks in the **same** authorized channel remain readable by message ID without any Bot token. The drill checks archive identity, published SQL metadata, original Telegram bytes, object SHA/length and cross-chunk ranges from a sealed snapshot; it is a single named-object check, **not** proof of a complete independent payload backup or full archive recovery. + +The [User MTProto E2E workflow](../.github/workflows/live-telegram-user.yml) exercises a real Telethon user session with checksum and range readback, and the [Multiworker PostgreSQL E2E workflow](../.github/workflows/live-telegram-multiworker.yml) runs three independent SQL workers against a real test channel. Cleanup is scoped to messages created by each test run. + +**Limit:** these tests exercise the live USER writer and isolated recovery, not the owner's original production channel or media. + +## 5. Production disaster-recovery acceptance + +For actual owner-controlled original Telegram data, follow [Project Status: original-media DR](IMPLEMENTATION_STATUS.md#32-recovery-of-original-telegram-media). The evidence must link a sealed SQL archive and its SHA-256 to the **original** channel/account and verify real Telegram payloads and byte ranges without deleting or reuploading them. + +A successful SQL restore does **not** mean production recovery passed. Owner credentials, independently backed-up encryption keys, source-root mounts, versioned GET/HEAD, MTProto send receipts, credential revocation and queued-job behavior must also be checked. + +## 6. Failure modes and prohibited actions + +- A process crash after Telegram accepted media can leave an unindexed message. Do **not** delete such media or assume a retry is safe when acknowledgment is ambiguous. +- Concurrent processes must never write using the **same** Telethon session file. +- Retained historical versions, aliases, multipart parts and failed upload intents can still reference Telegram messages. +- Physical Telegram deletion remains disabled until independently proven retention-aware GC, verified media backups and explicit owner authorization exist. +- Only promote a staging restore after independent owner acceptance and controlled change management; this runbook itself performs **no production cutover**. diff --git a/docs/CI_WORKFLOWS.md b/docs/CI_WORKFLOWS.md new file mode 100644 index 00000000..8453a04b --- /dev/null +++ b/docs/CI_WORKFLOWS.md @@ -0,0 +1,139 @@ +# Continuous Integration and Test Guide + +> **Audience:** contributors, maintainers, and CI operators. +> **Scope:** automated workflows, local validation, fault-injection acceptance, and evidence interpretation. +> **Not covered:** production approval, original-data disaster recovery, and physical Telegram deletion. See [Project Status](IMPLEMENTATION_STATUS.md). + +## 1. Workflow matrix + +GitHub Actions does not support workflow folders in its sidebar. The numeric prefixes in display names group related checks; workflow files live under [`.github/workflows/`](../.github/workflows/). + +| Group | Workflow | Validates | +| --- | --- | --- | +| 01 · Quality | [Security & Regression](../.github/workflows/quality-security.yml) | Python regression, security checks, dependencies, Docker image and listener smoke tests | +| 02 · Integration | [Databases](../.github/workflows/integration-databases.yml) | PostgreSQL/MySQL, native backups and isolated restores | +| 02 · Integration | [REST & S3](../.github/workflows/integration-storage.yml) | API contracts, signed always-Enabled versioning and all-private Public Access Block (real boto3), SigV4, AWS CLI, rclone and shared storage catalog | +| 03 · Resilience | [Docker Fault Injection](../.github/workflows/resilience-docker.yml) | Disposable PostgreSQL, two Web instances, crash/restart, corruption and recovery | +| 04 · Live Telegram | [User MTProto E2E](../.github/workflows/live-telegram-user.yml) | Real Telethon user session, checksum and range recovery | +| 04 · Live Telegram | [Multiworker PostgreSQL E2E](../.github/workflows/live-telegram-multiworker.yml) | Three independent workers sharing PostgreSQL and a real Telegram test target | +| 05 · Delivery | [Release Gate & Packages](../.github/workflows/delivery-release.yml) | Release regression, gated Linux/Windows packages and publishing | +| 06 · Endurance | [Isolated Capacity Soak](../.github/workflows/endurance-capacity.yml) | Manually dispatched 60–110-minute synthetic-media SQLite durability test | + +The **first seven workflows** form the normal `main` validation matrix. The endurance workflow is deliberately manual (`workflow_dispatch`). The live Telegram workflows require dedicated **test** credentials; synthetic Docker validation does not. + +### Interpreting CI results + +- **Success:** the specified checks passed at the workflow's exact commit SHA. It is not evidence for a later SHA or for production operation. +- **Failure:** inspect the failing step and logs, fix the cause, and run checks on the updated SHA. +- **Cancelled, queued, or skipped:** not a successful acceptance result. An optional test case skipped inside a successful workflow is also not evidence that the optional scenario ran. +- **Release gate vs. published release:** the gate can succeed without publishing binaries. Packaging and GitHub Release creation follow their explicit tag/`[publish-release]` rules and release preconditions. +- **Synthetic vs. real Telegram:** AWS CLI/rclone compatibility and Docker fault injection use fake remote Telegram I/O with real SQL. Separate live Telegram workflows verify actual remote bytes. +- **Production sign-off:** requires a different, owner-authorized evidence process documented in [Project Status](IMPLEMENTATION_STATUS.md#3-production-acceptance-not-yet-signed-off). + +## 2. Run checks locally + +Python 3.11+, the appropriate dependencies, and optional Docker Engine/Compose v2 are required. + +~~~bash +python -m pip install -r requirements.txt pytest +python -m pytest -q +python -m compileall -q teledrive web app tests +docker compose config --quiet +~~~ + +Docker integration jobs install the tools and database services they need inside disposable runners; they do not require a pre-existing TeleDrive deployment. + +### One-command Docker fault-injection acceptance + +~~~bash +bash scripts/dev/run_docker_fault_injection.sh +# Optional interactive operator wrapper: +python scripts/ops/acceptance.py +~~~ + +**Purpose:** development acceptance, not a simulated production cutover. The runner creates a randomly named, isolated Compose project with **PostgreSQL 17, two TeleDrive Web containers, and a fault-scenario container** using actual SQLAlchemy and `TelegramObjectStorage`. The Telegram side is persistent, disk-backed synthetic media. Test credentials, a deployed production instance, an original Telegram account and a legacy folder are **not required**. + +The **normal production Compose** has only **one** `teledrive` container. The integration suite also tests optional lease-fenced legacy source ingestion through that same WebUI/REST/S3 catalog with no direct Telethon duplicate send. Its single Python process exposes WebUI/REST on port 8080 and S3 on port 9000. Two separate Web containers in **this fault test** deliberately exercise shared-state behavior and crashes; they are not the deployment layout. + +## 3. Fault-injection acceptance gates + +All 13 gates and the teardown must pass. The identifiers are evidence labels, not development feature names. + +| Gate | Fault or operation | Acceptance requirement | +| --- | --- | --- | +| DEV-01 | Fresh PostgreSQL + two Web instances | Both authenticated health checks work; anonymous access fails | +| DEV-02 | 0 B, 1 B, 4,097 B, 65,537 B, and 5 MiB + 7 B | Durable SQL heads/manifests match the disk-backed chunks | +| DEV-03 | Remove local source after upload | Full SHA-256 and cross-chunk range readback match | +| DEV-04 | Lost Telegram ACK / SQL crash | Persisted send intent prevents unsafe duplicate sends | +| DEV-05 | Aged failed object / GC guard | Unknown ACK remains protected; physical deletion is not enabled | +| DEV-06 | Synthetic Telegram HTTP 429 | SQL send slot reserved first; cooldown/metrics shared across nodes | +| DEV-07 | Real `pg_dump` and isolated `pg_restore` | Object IDs, chunk offsets, SHA-256 and range reads survive | +| DEV-08 | Corrupt SQL archive | Native restore validation rejects the damaged dump | +| DEV-09 | Corrupt remote bytes | Hash mismatch never publishes partial output; repair permits readback | +| DEV-10 | Isolated project teardown | Only the randomly named Compose project and its volumes are removed | +| DEV-11 | Web `SIGKILL` and PostgreSQL restart | Head, send fence, cooldown and remote bytes remain recoverable | +| DEV-12 | Missing chunk / second-chunk timeout | Existing destination remains intact; repair restores checksums | +| DEV-13 | Overall acceptance and cleanup | Four verified stages, all 13 gates and successful scoped cleanup | + +### Execution flow + +1. Start fresh database and Web instances; exercise real application configuration parsing and authenticated endpoints. +2. Upload size-boundary fixtures, persist immutable manifests, remove sources, and verify checksums and byte ranges. +3. Inject an uncertain upload acknowledgment and a rate limit after reserving a SQL send slot. +4. Kill `web-a` and restart PostgreSQL; assert shared persistent state from reconnected workers. +5. Dump and restore into an **independent** database; reject a deliberately corrupted dump. +6. Inject remote-media corruption, a missing chunk and an interrupted second chunk; confirm fail-closed output behavior. +7. Collect `seed.json`, `restarted.json`, `restored.json` and `acceptance.json` and complete narrowly scoped teardown. + +### Artifacts and evidence + +Reports are written under `.docker-fault-evidence/docker-fault-.json` by default; `TELEDRIVE_FAULT_REPORT_DIR` can override the directory. Evidence files use permission mode `0600` and must not contain passwords, Bot tokens, Telegram sessions or database credentials. The runner uses only its own project-scoped `docker compose down -v`—never `docker system prune` or global volume cleanup. + +The following is a **schema illustration, not an actual run result**: + +~~~json +{ + "mode": "disposable-docker-development", + "result": "PASS", + "docker_project_removed": true, + "production_data_used": false, + "real_telegram_tested": false, + "legacy_cutover_required": false, + "production_acceptance_signed_off": false +} +~~~ + +An actual PASS requires the complete four-stage evidence and `stages.acceptance.checks_count == 13` with `all_checks_verified == true`. Cleanup failure or a missing gate is a failure. + +## 4. Regression invariants + +- The USER MTProto shared flood-wait cooldown must be preceded by `reserve_send_slot`; tests cannot bypass send fencing to manufacture a passing flood observation. +- The Web `/api/health` endpoint alone does not validate full configuration; fault tests use the actual `load_config` parser, including `sleep_min_seconds`, `sleep_max_seconds`, `max_files_per_run` and `max_files_per_day`. +- `tests/web/test_unified_service.py` verifies REST-to-S3, S3-to-REST, local-job-to-S3 data visibility and the shared generation/catalog. The Security workflow checks both listeners **inside one container**. +- Live User and Multiworker E2E are distinct from synthetic fault tests. A generated flood observation is not a measured production rate limit. +- Normal Security and Storage CI run adversarial GC reference-graph tests. The live User MTProto E2E job creates one owned Telegram marker fixture, validates bounded read-only history inventory, reconciles independently known marker coordinates, creates a fresh isolated catalog after full byte/SHA verification, then cleans up only its own fixture. This does not exercise production garbage collection. +- Offline marker inventory and independent-plan reconciliation tests cover bounded Telegram history reads, forged/non-media messages, scan truncation, duplicate ordinals and private non-overwriting reports; discovery is not a recovery plan or full historic inventory. +- Offline capacity soak tests real SQLite operations, durable synthetic media, SQL restarts and historical readback. It does not exercise original Telegram media, independent physical hosts or real-world long-term Telegram behavior. + +## 5. Source layout and related guides + +| Path | Role | +| --- | --- | +| [`.github/workflows/`](../.github/workflows/) | GitHub Actions YAML definitions | +| [`.github/ci/fault-injection/`](../.github/ci/fault-injection/) | Test-only Dockerfile, ignore rules and Compose project | +| [`docker-compose.yml`](../docker-compose.yml) | **One-container** default WebUI/REST/S3 deployment | +| [`scripts/dev/run_docker_fault_injection.sh`](../scripts/dev/run_docker_fault_injection.sh) | Fault test entrypoint | +| [`scripts/dev/docker_fault_scenarios.py`](../scripts/dev/docker_fault_scenarios.py) | Synthetic fault scenarios and evidence | +| [`requirements.txt`](../requirements.txt) | API/runtime dependencies | + +For feature status and production release boundaries, see [Project Status](IMPLEMENTATION_STATUS.md). For actual database backups and restore procedures, see [Backup and Restore](BACKUP_RESTORE.md). For security reporting, see [Security Policy](SECURITY.md). + +**S3 P2-0 selector gate:** `tests/s3/test_s3_route_guard.py` is mandatory in both Quality and REST/S3 workflow regressions. It verifies 116/116 operation statuses in the machine-readable inventory against `docs/S3_SPECS.md` and rejects unrecognized signed bucket/object selectors without exposing or mutating data. + +**Unified transport policy:** Live REST/S3/local-job I/O is USER MTProto only; there is no Bot API transport. The delivery workflow automatically prunes proven merged, unchanged and unprotected PR branches. + +**User MTProto unified transport:** `tests/storage/test_storage_user_mode.py` checks signed S3 user-MTProto write/readback, same-channel legacy Bot-media read without any Bot fallback, unknown-ACK SQL fencing, owner isolation and fail-closed transport selection. The ASGI user session has its own encrypted secret handling outside the repository; separate real user E2E tests exercise account/chunk behavior but do not prove live REST/S3 user-mode deployment. Only an authorized USER session supports channel-history enumeration. + +**Multipart listing parity:** `tests/s3/test_s3_multipart.py` verifies scoped SQL-bounded ListParts/ListMultipartUploads cursors, sparse/replaced part metadata, timestamps, grouping, URL encoding, invalid requests and actual boto3 over live HTTP. These are developer/test-channel gates; physical GC, recovery and owner sign-off use [Local Agent Acceptance](LOCAL_AGENT_ACCEPTANCE.md). + +**ListBuckets pagination contract:** `tests/s3/test_s3_bucket_listing.py` exercises SigV4/boto3 end-to-end, SQL creation dates, 1–10,000 page bounds, prefix and region filters, cursor integrity and credential scoping. Quality and REST/S3 workflows both require it. diff --git a/docs/IMPLEMENTATION_STATUS.md b/docs/IMPLEMENTATION_STATUS.md new file mode 100644 index 00000000..f6e3932d --- /dev/null +++ b/docs/IMPLEMENTATION_STATUS.md @@ -0,0 +1,200 @@ +# Project Status, Acceptance, and Recovery Roadmap + +> **Audience:** contributors, release maintainers, and deployment owners. +> **Scope:** supported features, development evidence, production sign-off, recovery limits, and remaining engineering work. +> **Source of test details:** [CI and Test Guide](CI_WORKFLOWS.md). **Source of executable backup procedures:** [Backup and Restore](BACKUP_RESTORE.md). +> +> **Do not treat a successful CI run as approval to operate on production data or physically delete Telegram messages.** + +## 1. Current release and supported scope + +| Area | State | Evidence and limitations | +| --- | --- | --- | +| P0 — basic upload and supported REST/S3 subset | **Development-validated** | Transfer, authentication, integrity and tested client subset; not complete AWS S3 parity. The desktop application and the Bot API layer have been removed; only the Telethon USER MTProto WebUI service remains. | +| P1 — disposable Docker acceptance | **Development-validated** | Two Web processes, real PostgreSQL, 13 resilience and cleanup gates; Telegram I/O is synthetic | +| P1 — real test-channel transfers | **Development-validated** | Telethon user and three-worker PostgreSQL integration (historical Bot exercises retired with the Bot layer); not the owner's original production media | +| P1 — physical production operation | **Not signed off** | Independent host identity, owner data recovery and authorized change control not established | +| P2 — advanced recovery, GC and compatibility | **In development** | Rebuild and offline verification components exist; production safeguards and further protocol work remain | + +**Published release:** [v1.0.7](https://github.com/babico/TeleDrive/releases/tag/v1.0.7), source SHA `747747652cd1ca36878453d49daf627d8c0a60ad`. Published release contents are not automatically updated when new development commits reach `main`. Release notes and binaries belong on [GitHub Releases](https://github.com/babico/TeleDrive/releases), not in a separate per-version Markdown file. + +**Verified newer development baseline:** `c2b76880b4ada3d61d47a7926ea819244047fa33`; all nine normal CI workflows completed successfully on that exact SHA. Its guarded delivery job also removed seven merged PR branches; only `main` remained. Since then the matrix has become **seven workflows** (the Bot E2E and Desktop workflows were removed with their features). For any *later* source SHA, read its own [Actions runs](https://github.com/babico/TeleDrive/actions) before making a CI claim. A cancelled, pending or fully skipped required workflow is not evidence of successful live acceptance. + +### P0: implementation boundaries + +- **State and durability:** SQLAlchemy with SQLite/PostgreSQL/MySQL integration testing, immutable object/chunk manifests, SHA-256 verification, ranged reads, fencing and persistent job history. +- **API and credentials:** separately scoped REST Bearer credentials and S3 SigV4 credentials, signed request checks, conditional writes, restricted administrative operations and fail-closed unsupported inputs. +- **Supported S3 subset:** object/bucket operations, a bounded multipart implementation, conditional operations, selected versioning (including safe always-Enabled controls), fixed all-private Public Access Block, tags, copies, CORS and presigned requests. Tested clients include boto3, AWS CLI and rclone. +- **Not complete S3:** arbitrary AWS lifecycle and bucket policy behavior, SSE, every checksum mode, full delete-marker semantics and untested client combinations are not warranted. +- **Telegram transports:** dedicated test-channel Telethon USER MTProto scenarios plus a shared PostgreSQL multiworker scenario. Credentials and private test channel are isolated from production; the Bot API layer has been removed. +- **Backups:** native PostgreSQL/MySQL backups and separate SQL restore rehearsals, plus disposable Docker fault injection. These exercises do not certify recovery of the owner's original Telegram account or channel. + +### P1: development acceptance + +The development acceptance criteria consist of the 13 Docker fault-injection gates, four evidence stages and scoped cleanup, plus the distinct real test-channel User/Multiworker workflows. Run the documented [fault-injection command and checklist](CI_WORKFLOWS.md#3-fault-injection-acceptance-gates); do not maintain a second copy of the 13 gates here. + +A production installation, original legacy files, signed operator approval, an hour of real production rate limits, and restoration of original production media are **not** prerequisites for declaring disposable *development* acceptance. They are separate owner/operator responsibilities. + +## 2. Architecture and unified storage contract + +### A single service deployment + +The default [`docker-compose.yml`](../docker-compose.yml) creates **one `teledrive` container** and one Python process with two local listeners. There is no desktop application, no Bot API transport and no separate S3 container. One or more Telegram API credential sets are configured as an **`apis:` list** in `config/config.yaml` (`{api_id, api_hash, session_file, account}` entries sharing the same dedicated channel) with `round_robin` (default) or `failover` flood-rejection rotation; env comma-lists remain a legacy fallback. + +| Interface | Local endpoint | Auth model | Storage | +| --- | --- | --- | --- | +| WebUI and REST v1 | `http://127.0.0.1:8080` | Basic admin + scoped REST Bearer tokens | Shared SQL catalog and Telegram object engine | +| S3-compatible API | `http://127.0.0.1:9000` | Scoped S3 credentials with SigV4 | **Same** SQL catalog and Telegram object engine | +| Local-folder durable jobs | Admin job API on 8080 | Basic admin + CSRF | **Same** catalog; files under allowed read-only mount are explicitly queued | + +**Read-only channel recovery visibility:** The admin-only `GET /api/v1/admin/storage/channel-history` uses the connected user MTProto session, with bounded `limit` (1–1,000), exclusive `before_message_id` pagination, and optional `include_external_markers=true` for explicitly unverified other-sender marker hints. It neither downloads media nor updates SQL, never infers bucket/key/whole SHA, and requires the connected USER session. An original mapping still needs independently verified metadata and full payload restoration before publication. + +**Unified runtime transport:** Telethon USER MTProto is the only transport for REST/S3/local-file jobs (no selector exists anymore). A preauthorized USER StringSession and a dedicated private broadcast channel are required to write; both are configured in the `storage:` section of `config/config.yaml`. There is no Bot upload mode; the entire Bot API layer has been removed. The web application's single ASGI lifespan holds the user session and is shared with the S3 listener. Historical SQL heads are retained; old Bot chunks in the exact same channel can be read through the authorized USER session by message ID. Telethon channel history provides candidate markers but not independent key/whole-hash proof. MTProto unknown-ACK send intents and receipts are durable SQL GC blockers. Do not use Saved Messages. + +An S3 request and a REST request can address the same bucket/key and observe the same published object and generation. A local file is **not** automatically imported merely by copying it into `/uploads`. Create an authenticated job for that path; `jobs.auto_run: true` in `config/config.yaml` (the default) processes explicitly queued jobs. This is distinct from the original user-account Telethon folder scanner/batch uploader, which has been removed. Folder ingestion is now done explicitly through the admin `import-legacy` endpoint, which queues files as durable jobs in the shared SQL catalog using the configured Telethon transport. + +For REST/S3 payload I/O, configure an authorized **Telethon USER** (MTProto) session with posting rights in a dedicated private channel; all interfaces use the same session and SQL catalog. Legacy `api-bot` chunks from the pre-switch era can also be read directly by the authorized user from the same channel via their immutable SQL message IDs; for inaccessible/different channels preserve original SQL, media and independent backup until a separately verified migration. The S3 master key/credentials are separate. Launching the listeners alone does not provision those secrets. See the [README](../README.md) for installation and examples and [Security Policy](SECURITY.md) for network exposure rules. + +### Metadata and physical payloads + +SQL stores heads, version records, chunk identities/hashes, copy aliases, jobs and encrypted S3 key metadata. Actual object payload chunks reside in Telegram. SQL backups alone do **not** restore missing Telegram media. Both endpoints must enforce checksum verification and fail closed when an immutable chunk cannot be trusted. + +## 3. Production acceptance (not yet signed off) + +**Delegated local execution:** [Local Agent Acceptance](LOCAL_AGENT_ACCEPTANCE.md) specifies Docker deployment, exact byte fixtures, independent payload+SQL backups, isolated disaster-recovery drills, GC race tests and evidence files. Providing this mission does not prove that owner-only acceptance gates have run. In particular, multiple Docker containers on a single host are not physically separate machines. +The following gates apply **only after** an owner has provided actual hosts, a suitable private Telegram target, sealed independent backups, and permission to conduct production-equivalent drills. Never upload sessions, tokens, connection URLs or original media identifiers to this public repository. + +### 3.1 Independent physical hosts and rate-limit soak + +Provide at least two **physically independent** application hosts sharing an explicitly identified SQL database. Record independently observed SHA-256 machine identities and a common DB identity. Two URLs terminating on the same host do not satisfy the requirement. + +Configure `TELEDRIVE_P1_HEALTH_HOSTS` with two to eight unique `label`/`url` HTTPS entries, alongside `TELEDRIVE_P1_BASIC_USERNAME` and `TELEDRIVE_P1_BASIC_PASSWORD` in a protected environment. Then: + +~~~bash +python scripts/ops/monitor_host_soak.py \ + --output /secure/acceptance/soak.json \ + --duration-seconds 3600 \ + --interval-seconds 60 +~~~ + +The evidence must demonstrate a run of at least one hour, `telemetry_gate_passed=true`, real observed Telegram 429s as required by the gate, shared counter coherence, no unresolved send acknowledgment and drained queues. It is still **telemetry evidence**, not independent proof of physical host separation. Do not intentionally flood Telegram to force 429s. + +### 3.2 Recovery of original Telegram media + +Keep the original SQL snapshot, expected archive SHA-256, Telegram account credentials and destination/channel identifiers in separate protected storage. Restore into an **isolated staging database**, never over the live production DB. + +SQLite backup validation is described in [Backup and Restore](BACKUP_RESTORE.md). **Current script scope:** `scripts/ops/rehearse_telegram_dr.py` is a read-only USER-MTProto verifier. It authenticates an authorized user session, resolves the exact original broadcast channel, reads the sealed snapshot from a temporary clone and verifies whole-body SHA-256, size and a cross-chunk range for one named object. It never uploads or deletes Telegram media and never modifies the source database. Set `TELEDRIVE_STORAGE_MTPROTO_API_ID`, `TELEDRIVE_STORAGE_MTPROTO_API_HASH` and (optionally) `TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE` in the protected staging environment: + +~~~bash +python scripts/ops/rehearse_telegram_dr.py \ + --snapshot /secure/backups/sealed.db \ + --sha256 EXPECTED_ARCHIVE_SHA256 \ + --bucket ORIGINAL_BUCKET \ + --key ORIGINAL_KEY \ + --account ORIGINAL_ACCOUNT \ + --chat-id ORIGINAL_CHAT_ID \ + --target ORIGINAL_CHANNEL_ID_OR_USERNAME +~~~ + +Old `api-bot` chunks in the same authorized channel are read through the USER session without any Bot token. This drill checks archive identity, published SQL metadata, original Telegram bytes, object SHA/length and cross-chunk ranges without reuploading or deleting media, but it proves **one named object**, not a complete independent payload backup or full archive recovery. PostgreSQL/MySQL installations must use their appropriate native backups and independent restore databases. A consistent SQL dump **alone** is insufficient original-data disaster-recovery proof. + +### 3.3 Legacy cutover only if original legacy sources exist + +If real legacy files exist, stop the legacy uploader, make its source directory read-only, fence concurrent writes, constrain paths to approved roots and verify original Telegram bytes and the current SQL heads. This is a **controlled operator action**, not an automated development acceptance step. + +~~~bash +python scripts/ops/verify_legacy_cutover.py \ + --database /secure/isolated.db \ + --source-root /approved/legacy \ + --allowed-root /approved \ + --scratch-dir /secure/scratch \ + --bucket ORIGINAL_BUCKET \ + --account ORIGINAL_ACCOUNT \ + --chat-id ORIGINAL_CHAT_ID \ + --target ORIGINAL_CHANNEL_ID_OR_USERNAME +~~~ + +Require `verified=true`, a complete scan, zero inconsistent/queued items, and exact source inode/revision/hash correspondence with the published head and original media. If **no legacy installation exists**, obtain an independent owner attestation (`no_legacy_source_attested=true`) and record `{"not_applicable":true}` rather than inventing a test. No live traffic cutover occurs without change approval. + +### 3.4 Two-person evidence review + +The offline verifier `scripts/ops/verify_production_bundle.py` consumes `release_sha`, `topology`, `soak`, `dr`, `cutover`, `original_telegram_data_attested` and two independent approvals. Each host's topology entry includes `label`, `machine_id_sha256` and `shared_database_sha256`, with `independent_hosts_owner_attested=true`. + +The same canonical JSON (excluding `approvals`) must be signed by **two different people** using separate Ed25519 private keys. Provide independently obtained 32-byte public keys: + +~~~bash +python scripts/ops/verify_production_bundle.py \ + --bundle /secure/acceptance/production-bundle.json \ + --release-sha REVIEWED_40_HEX_COMMIT_SHA \ + --owner-public-key /secure/keys/owner.pub \ + --operator-public-key /secure/keys/operator.pub +~~~ + +An incorrect SHA, incomplete drill, duplicate host identity, invalid media proof or missing approval fails closed. Even a verified bundle is **not** an automatic deployment action or authorization to delete Telegram messages. Explicit owner change control remains necessary. + +## 4. P2 recovery and safety roadmap + +### 4.1 Physical Telegram garbage collection: disabled + +**Implemented safeguards:** The mandatory USER MTProto transport uses a durable SQL send journal to persist pre-send ambiguous intents and acknowledged remote message coordinates, which block non-destructive GC candidate approval; actual independent media backup and owner approval are still required. `StorageGCInventory` performs retention/reference previews with a fail-closed, fingerprinted SQL graph covering HEADs, published/removed versions, transitive copy aliases, multipart parts, writer fences, active durable upload jobs (by object identity, expected head and bucket/key) and MTProto intents/receipts; `RecoveryManager.cleanup` checks SQL heads, published and removed history, copy aliases, multipart references, MTProto send intents and active durable jobs. The shared MTProto rate guard records flood-wait observations and cooldowns that fence every worker. The Telethon ownership probe also checks account/channel/marker identity. + +**Implemented now:** transactional, version-aware, writer-fenced GC reference graphs and dry-run planning are complete and race-tested (aliases, versions, multipart parts, MTProto intents/receipts, active durable jobs, stale leases, concurrent writers, recycled plans, tampered fingerprints). `StorageGCInventory.inspect`, `preview_failed_retention` and `plan()/verify_plan()` produce integrity-bound plans with an explicit owner-gate checklist; `deletion_permitted` is always False. `RecoveryManager.cleanup` performs ownership-verified physical deletion only when explicitly invoked with dry_run=False and only after every fence passes. + +**Not yet authorized/complete:** independently verified media backups, a successful isolated restore drill, complete ownership audit, explicit two-person deletion approval and an owner-authorized single-message trial. Deleted Telegram **message IDs cannot simply be restored from SQL**. Automatic physical GC in WebUI/API remains disabled and no Telegram media has been deleted. + +Do not enable physical deletion merely because offline integrity tests pass. Require a signed dry-run plan, complete remote inventory, retention/reference validation, independent payload snapshot plus successful restore drill, writer fencing and recorded approval. + +### 4.2 Rebuild metadata from verified original Telegram messages + +`teledrive.storage.rebuild.restore_isolated_catalog_batch` and `scripts/ops/rebuild_from_telegram.py` verify original Telethon message identifiers, stable UUID/ordinal/SHA-prefix markers, account/channel/message identity, full per-chunk checksums and an **independently supplied** whole-object SHA-256. They create a **new isolated SQLite database**, not a replacement for the live database. + +Each plan requires `object_id`, `bucket`, `key`, `sha256`, `size` and `chunks`. Each chunk requires `ordinal`, `account`, `chat_id`, `message_id`, `size` and `sha256`. Multiple plans use `{"objects":[...]}`. The bounded batch accepts 1–128 objects, up to 4,096 chunks and 1 GiB total; the staging database is privately validated (`PRAGMA integrity_check`) and atomically published with mode `0600` only after **all** source bytes verify. A failed object never exposes a partial destination catalog. + +~~~bash +# Supply TELEDRIVE_REBUILD_API_ID, TELEDRIVE_REBUILD_API_HASH, +# and TELEDRIVE_REBUILD_SESSION through a secure environment. +python scripts/ops/rebuild_from_telegram.py \ + --plan /secure/verified-plan.json \ + --new-database /secure/isolated-rebuilt.db \ + --account original-user \ + --target ORIGINAL_TELEGRAM_CHANNEL +~~~ + +**Bounded history candidate inventory:** `scripts/ops/inventory_telegram_history.py` scans at most 10,000 readable messages from **one explicitly authorized original Telegram channel**, checks canonical TeleDrive UUID/ordinal/SHA-prefix markers and outgoing/channel/media hints, and creates a new private JSON report (mode `0600`). It downloads no media and writes no SQL. Missing and repeated ordinals are flagged; the report never claims that the channel scan represents all historical sources. + +~~~bash +# Use the same protected TELEDRIVE_REBUILD_API_ID, _API_HASH and _SESSION +# environment variables as the isolated rebuild command. +python scripts/ops/inventory_telegram_history.py \ + --account original-user \ + --target ORIGINAL_TELEGRAM_CHANNEL \ + --limit 1000 \ + --output /secure/isolated-history-candidates.json +~~~ + +**Independent manifest reconciliation:** `teledrive.storage.reconcile_inventory.reconcile_inventory` compares bounded marker candidates with separately provided, already independently verified recovery plans. Wrong object IDs, ordinals, Telegram account/chat/message coordinates and SHA-prefixes fail closed. `scripts/ops/rebuild_from_telegram.py --inventory /secure/candidates.json` optionally makes this check a prerequisite to full payload byte verification. CI also tests a real channel-marker upload, read-only discovery, independent manifest reconciliation and a new isolated SQLite restore with original Telegram bytes. A match is **not** by itself a verified restore. + +**Limits:** inventory markers are **unverified candidates**, not an importable rebuild plan. They do not prove the original bucket/key mapping, complete chunk SHA-256 or sizes, whole-object hash, historical versions, or the existence of messages outside the scanned window/channel. Actual isolated reconstruction still requires independent metadata and byte-level verification. Automated discovery without trustworthy independent metadata is still open work. Inaccessible original channels require a separately verified migration and preserved backups. This reconstruction does **not** upload or delete Telegram media. + +### 4.3 Capacity and recovery endurance + +The manually triggered [Capacity Soak workflow](../.github/workflows/endurance-capacity.yml) runs the real SQLite/object engine with persistent **synthetic** Telegram files, periodic SQL restarts, historical SHA readback and p50/p95 timings. It does not prove actual Telegram bandwidth, natural 429 behavior, physically separate hosts or an hour of production operation. Those require a separate owner-authorized environment and measured evidence. + +### 4.4 Advanced S3 protocol parity + +Use the [S3 Specifications and Compatibility Matrix](S3_SPECS.md) as the operation-level P2 Advanced S3 backlog. It distinguishes verified bounded routes from partial AWS semantics and deliberately unsupported APIs; ListParts and ListMultipartUploads now implement owner-scoped SQL-bounded keyset paging, immutable part metadata and validated prefix/delimiter/URL-encoded listings in the documented TeleDrive private S3 contract; this status page remains the source for production acceptance boundaries. GetObjectAttributes is now **supported for the documented private, single-region, general-purpose TeleDrive contract**: signed selected fields, immutable full-object SHA-256, fixed STANDARD class, published VersionIds and recorded multipart ObjectParts with bounded marker pagination. Completion atomically snapshots the selected parts, CopyObject preserves the snapshot, and historic uploads use a strictly validated read-only legacy journal fallback. SQL inconsistency fails closed; Telegram chunks are never mistaken for S3 part boundaries. This is **not** support for AWS managed storage tiers, SSE-C, payer billing, directory buckets or alternate region semantics. See its regression and boto3 tests; validate all mandatory checks on the exact release candidate SHA before making a release claim. + +Only SQL-only incomplete-multipart lifecycle abort is now supported, applied by the hourly one-service scheduler; no Telegram messages or completed versions are deleted. Full object lifecycle expiration/transitions, SSE, advanced checksum/SDK modes, bucket policies, comprehensive delete-marker semantics and unverified third-party clients (including Cyberduck) remain unsupported or unverified until independently tested. Do not substitute an expected AWS behavior for TeleDrive's documented supported subset. + +## 5. Outstanding work and release decision rules + +- [x] Publish bounded v1.0.7 development milestone with separate release artifacts and evidence. +- [x] Implement disposable 13-gate Docker validation and distinct live test-channel User/Multiworker workflows (historical Bot exercises retired with the Bot layer). +- [x] Implement fail-closed recovery guards, owner-proofed isolated one/multi-object rebuild and an offline dual-approval evidence verifier. +- [x] Verify nine normal workflows on the newer `aa44bd6` development baseline (not the published v1.0.7 SHA). The current matrix has **seven workflows**: Security, REST & S3, Databases, Docker Fault Injection, User MTProto E2E, Multiworker PostgreSQL E2E and the Release Gate. Release packaging now ships the single-container Docker image instead of desktop installers. +- [ ] Verify **every required workflow on the exact SHA** of any subsequent release candidate. +- [ ] Obtain owner-supplied physical-host identities and original-media disaster-recovery evidence. (Legacy direct-send uploader code removed in cf98e1a; no-legacy attestation is now code-level, not a runtime gate.) +- [ ] Implement and independently rehearse the complete physical GC safety/rollback alternative before authorizing deletion. +- [ ] Complete unknown-history discovery, additional S3 parity tests and long-duration real-environment capacity profiling. + +Use [TODO.md](../TODO.md) for the short working checklist. Update this file when the verified *behavior or acceptance boundary* changes; do not paste every transient CI run log here. Separate per-version Markdown release notes are intentionally not maintained; use [GitHub Releases](https://github.com/babico/TeleDrive/releases). diff --git a/docs/LOCAL_AGENT_ACCEPTANCE.md b/docs/LOCAL_AGENT_ACCEPTANCE.md new file mode 100644 index 00000000..ab3e4c3a --- /dev/null +++ b/docs/LOCAL_AGENT_ACCEPTANCE.md @@ -0,0 +1,129 @@ +# Local Agent Mission — Real-Data Docker, Recovery, GC Safety and Owner Acceptance + +> **Give this document to the agent running on the owner's machine.** Repository: `babico/TeleDrive`. This is an executable engineering assignment, **not** permission to operate on production or to delete Telegram history. All written evidence, source code, and documentation must be in English. Perform the work autonomously on this host wherever possible, test it, and report objective results; ask for owner secrets/authorization only when essential. + +## 0. Non-negotiable boundaries + +1. Work on a **fresh dedicated clone and dedicated disposable environment**, never on an existing production `data/`, `config/`, database, Telegram channel or `~/.ssh` keys. Name all Docker Compose projects with a unique prefix; never run `docker system prune`, global image/volume deletion, `git reset --hard`, or blanket Telegram message deletion. Clean up only fixtures explicitly created and recorded by this run. +2. **No automatic physical Telegram GC** and no mutating original-media recovery. Unknown acknowledgments, orphan-looking messages, removed versions, aliases, historical copies, multipart parts, active write leases, and MTProto send intents/receipts with uncertain origin are all **protected**. If a complete owner-approved independent payload backup and restore proof is absent, stop at a signed dry-run plan. +3. Use a **new private test channel** with an authorized USER Telethon session and posting rights. Verify Bot mode is rejected, old same-channel Bot objects are still readable by MTProto without Bot credentials, and test the required USER transport: create a private `0600` session via `scripts/dev/create_storage_user_session.py`, verify channel history after a forced lost-send ACK, restore readback across restart, and preserve old Bot chunks if supplied. Do not run two active clients on one Telethon session. Bot-only installs cannot enumerate arbitrary history. If the owner authorizes a *read-only* inspection of an original channel, separate it entirely from test fixtures. For original-media verification use the read-only USER-MTProto `scripts/ops/rehearse_telegram_dr.py` against a sealed isolated catalog and complete SHA evidence; it checks one named object and is **not** proof of a complete independent payload backup or full archive recovery. An inaccessible old Bot channel is a migration blocker, not evidence of successful recovery or permission to delete anything. No test is allowed to manufacture or substitute an original `bucket`, `key`, `versionId`, full SHA-256, machine identity, original-media claim, Telegram HTTP 429, or approval signature. +4. Keep credentials in protected `0600` private files or a secret manager; do not expose tokens, sessions, Bot file IDs, SQL URLs, chat IDs or actual media filenames in commit messages, logs, CI artifacts or GitHub issues. Use redacted, integrity-checked evidence only. +5. A **single Docker host, even with several containers, is not two physically independent hosts**. A fabricated 429 and fake Telegram transport are not rate-limit/real-media evidence. Two owner/operator approvals require two real independent people. On unmet gates, report `BLOCKED_OWNER_ACTION` — never mark TODO items passed. + +## 1. Discovery and isolated installation + +From the trusted owner-controlled host: + +```bash +git clone https://github.com/babico/TeleDrive.git TeleDrive-local-audit +cd TeleDrive-local-audit +git fetch origin +git checkout main +git rev-parse HEAD | tee /tmp/teledrive-reviewed-sha.txt +docker version +docker compose version +python3 --version +umask 077 +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements.txt pytest boto3 +python -m compileall -q teledrive web app tests +python -m pytest -q +docker compose config --quiet +bash scripts/dev/run_docker_fault_injection.sh +``` + +Use Docker only on the explicitly approved test instance. Run the normal single-service `docker-compose.yml` **separately** from the two-Web fault-injection fixture: normal WebUI/REST port 8080 and S3 port 9000 must live inside **one `teledrive` container**, sharing one SQL catalog and Telegram object store. + +Provision test-only configuration from `README.md` section 1 and 4. **Check existing `docker ps -a --format '{{.Names}}'` and port 8080/9000 owners first:** normal Compose uses global `container_name: teledrive`. Do not reuse, replace, stop or rename any existing installation to make this test fit. If there is a collision, use a genuinely separate VM/host or stop with a request for owner direction. The fault runner uses isolated Compose project resources: + +```bash +cp -n .env.example .env +chmod 600 .env +mkdir -p config data uploads +# Set a unique long TELEDRIVE_WEB_PASSWORD, PUID/PGID, and ONLY +# disposable TELEDRIVE_STORAGE_BOT_TOKEN / TELEDRIVE_STORAGE_BOT_CHAT_ID. +# Create config/config.yaml and config/s3-master.key according to README. +docker compose up -d --build +docker compose ps +docker compose logs --tail=120 teledrive +``` + +The agent must verify loopback-only bindings, secret-file permissions, startup readiness, admin authentication, denied unauthenticated access and a unique catalog identity. **Do not guess or overwrite missing secret material.** If Telegram test credentials are missing, finish SQL/Docker synthetic work, mark live-media steps blocked, and request the minimum owner input. + +Create a private evidence directory *outside* the Git checkout: `$HOME/teledrive-audit-evidence//`, permissions `0700`, receipts `0600`. Save OS/container versions, reviewed SHA, GitHub Actions URL, sanitized `docker compose ps`, CI and local test results and clean worktree diff. + +## 2. Seed real bytes, not just metadata + +Issue a test-only scoped S3 key via **WebUI → API Management → S3-compatible gateway**. Configure boto3 with the explicit `http://127.0.0.1:9000` path-style endpoint and SigV4; never record its secret. Implement a repeatable `scripts/ops/local_owner_storage_matrix.py` (or tests), keeping random seed + SHA-256 manifest private. Create a unique bucket and deterministic test file paths. At minimum: + +- Upload **0 B, 1 B, 4,097 B, 65,537 B, 5 MiB + 7 B, 16 MiB + 101 B** and at least one multi-chunk 32 MiB+ object if the user's dedicated test channel limits allow. Mix incompressible random, repetitive and Unicode-named payloads. Avoid unbounded uploads or rate-limit floods. +- For each: PUT via S3; GET and HEAD via both S3 and REST; verify exact bytes, full SHA-256, stored object size, ETag consistency where applicable and cross-chunk `Range` (first/middle/boundary/last). +- Test two-version overwrite and explicit `VersionId` reads, delete marker, removed historical version, server-side CopyObject preserving source after logical deletion, and REST/S3/local-job visibility for one explicitly queued source file under the approved read-only `uploads/` mount. +- Exercise multipart `CreateMultipartUpload`, 5 MiB first part + final remainder, reordered/sparse/replaced parts, `ListParts` pages, `ListMultipartUploads` prefix/delimiter/URL encoding pages, `Complete`, `Abort`, `GetObjectAttributes/ObjectParts` and published-version reads using **real boto3**. Assert aborted, uncommitted and wrong-owner data remain unavailable. +- Repeat GET/HEAD/list pagination after container restart and an isolated SQL restore. Run storage under bounded concurrency with both same-key competing writers and distinct buckets. Compare the shared SQL HEAD/generation journal to actual bytes. +- Run negative cases: changed payload with old checksum, wrong SigV4, revoked credential, cursor tampering, unknown S3 controls, truncated Telegram transfer, missing remote chunk, stale multipart writer and ambiguous USER MTProto send outcome. **No failed case may publish an unverified head or delete remote media.** + +All byte-level production-equivalent scenarios must actually contact the owner's **dedicated test Telegram USER/channel**. The separate CI fake transport results should be labeled `synthetic`. + +## 3. Independent backups and isolated disaster recovery + +Review `docs/BACKUP_RESTORE.md` and `docs/IMPLEMENTATION_STATUS.md` sections 3–4 before executing. First generate and seal a consistent SQL snapshot and receipt: + +```bash +python scripts/ops/backup_metadata.py --database data/teledrive.db \ + --output "$EVIDENCE_DIR/sealed-test-sqlite.db" +sha256sum "$EVIDENCE_DIR/sealed-test-sqlite.db" > "$EVIDENCE_DIR/sealed-sqlite.sha256" +python scripts/ops/verify_metadata_backup.py \ + --database "$EVIDENCE_DIR/sealed-test-sqlite.db" \ + --sha256 "$(cut -d' ' -f1 "$EVIDENCE_DIR/sealed-sqlite.sha256")" +``` + +For PostgreSQL, use `pg_dump -Fc` and `pg_restore` into a **new database**; for MySQL use transactionally consistent dump and independent restore. Execute `scripts/ops/rehearse_native_database_restore.py` where applicable. Metadata backup **alone is not an independent media backup**. + +Before any GC experiment, produce a **separate encrypted payload archive**, independent of both SQL and Telegram, with all bytes needed for *all* candidate media and referenced aliases/versions/parts. Manifest the full SHA-256/size and exact source relationships in private storage; verify archive integrity from an independent location and a separate read-only restoration process. Rehydrate archive bytes into a **new disposable test channel**, then prove full-body and range hash matches using a **new isolated SQL catalog**. Do not claim that Telegram message IDs can be restored after deletion; only a complete separately maintained plaintext archive can rebuild the logical content with new Telegram references. + +For an owner-authorized **original** dataset, run the read-only original-media check separately, with the original *sealed* archive and its independent hash: + +```bash +python scripts/ops/rehearse_telegram_dr.py \ + --snapshot "$ORIGINAL_SEALED_DB" --sha256 "$ORIGINAL_SEALED_SHA256" \ + --bucket "$ORIGINAL_BUCKET" --key "$ORIGINAL_KEY" \ + --account "$ORIGINAL_ACCOUNT" --chat-id "$ORIGINAL_CHAT_ID" +``` + +Do not substitute synthetic fixtures for this command or reuse production DB paths. Record `BLOCKED_OWNER_ACTION` if original inputs or permissions do not exist. + +## 4. History recovery and GC implementation acceptance + +Read `teledrive/storage/gc.py`, `gc_safety.py`, `history_inventory.py`, `rebuild.py`, `docs/BACKUP_RESTORE.md`, and the existing tests. Extend actual implementation and tests rather than checking TODO boxes by inspection. + +- Use the admin-only `/api/v1/admin/storage/channel-history` on the isolated USER transport with bounded `limit` and continuation `before_message_id`; `include_external_markers=true` may reveal other-sender/admin posts but must label them unverified. Compare administrative read-only reports with the CLI report, and never convert marker hints into object ownership or deletion authority. +- Inventory one explicitly authorized channel **read-only** via `scripts/ops/inventory_telegram_history.py --account TEST_ACCOUNT --target TEST_CHANNEL --limit 1000 --output "$EVIDENCE_DIR/history-private.json"`. Distinguish verified original message identity from weak filename/UUID markers. An unknown message without independent key, whole SHA, version or owner stays an **unresolved candidate**, never auto-published. Compare only against owner-supplied signed metadata and verified bytes; recover only into isolated catalog via `scripts/ops/rebuild_from_telegram.py --help` and a validated plan. +- Construct a version-aware GC graph covering live HEADs, published/removed histories, delete markers, CopyObject aliases (including transitive), MPU active/completed/aborted parts, unresolved MTProto send intents/receipts, inflight writer intents, active durable jobs, leases, retention/hold rules, shared chat IDs and owner scope. Assert no planned physical message deletion if **any** potential reference/ownership doubt exists. +- Build a **deterministic dry-run plan** with message identity + expected media checksum, ref graph fingerprint, SQL generation, backup-set identity, prospective action count and no-op outcome. Test races (new alias/version/writer after planning), plan replay/reuse, stale lease, DB reconnect, interrupted deletion, unknown Telegram response, absent original identity, and duplicate/retried upload. Any race must stop/refresh the plan; avoid TOCTOU between final writer fence and the deletion. +- On this assignment's **isolated test-only fixture** and only after a verified external payload archive plus a successful isolated restore, you may implement and test a **separately gated one-message manual delete** of a message created by this run, if the owner explicitly authorizes this experimental action. Refuse all wildcard IDs, production targets, untracked candidates and any 2-person-approval bypass. Keep the production GC feature disabled unless a genuine reviewed authorization permits it. +- Rehearse rollback as **restoration of logical objects from independent payload backup**, *not* pretending deleted Telegram message IDs are resurrected. Recheck REST/S3 and historical versions with fresh persisted references. Unknown send acknowledgment => zero deletion. +- Run stress/endurance with rate budgets and backoff on a disposable test channel. Capture genuine Telegram responses and counters; **never synthesize HTTP 429 as real evidence and never induce abuse to force rate limiting**. + +## 5. Owner-only acceptance — mandatory unresolved gates + +These cannot be completed by a single local agent merely by starting containers: + +| Gate | What proves it | If unavailable | +| --- | --- | --- | +| Physically independent hosts | Two different physical machines, independently obtained host identity hashes, same external SQL DB identity | `BLOCKED_OWNER_ACTION` | +| Authentic one-hour soak | `scripts/ops/monitor_host_soak.py --output "$EVIDENCE_DIR/soak.json" --duration-seconds 3600 --interval-seconds 60` with genuine telemetry and required invariants | `BLOCKED_OWNER_ACTION` | +| Original-media DR | Original sealed archive, independent checksum, correct credentials/channel, real original bytes and range checks in isolated restore | `BLOCKED_OWNER_ACTION` | +| Legacy cutover/exemption | Real legacy source + verified owner-controlled cutover via `scripts/ops/verify_legacy_cutover.py`, **or** independent signed no-legacy attestation | `BLOCKED_OWNER_ACTION` | +| Two-person bundle | Independent owner + operator Ed25519 signatures, matching reviewed SHA; verify with `scripts/ops/verify_production_bundle.py --help` | `BLOCKED_OWNER_ACTION` | +| GC physical deletion | Independent plaintext-media backup, verified restore, exact target authorization, fenced dry-run, no ambiguous refs, two-person change approval | **DISABLED** | + +Do not fabricate a release tag or publish binaries. For a new release candidate require the seven mandatory GitHub Actions workflows **green on its exact commit SHA**, review the Docker image package and supported protocol limits, then request release owner authorization. Development `main` does not retroactively update the v1.0.7 release. + +## 6. Required deliverables and report + +Create an English `docs/LOCAL_AGENT_ACCEPTANCE_RESULTS.md` **template** in the working branch (redacted, never real secrets) and a private `$EVIDENCE_DIR/report.json` with: +`release_sha`, `host_identity_redacted`, `compose_topology`, `fixture_seed`, `fixtures[{size,sha256,version_id_redacted}]`, `test_command`, `tests_passed`, `tests_failed`, `failure_reasons`, `sql_backup_sha256`, `payload_backup_sha256`, `restoration_verified`, `gates`, `blocked_owner_actions` and `garbage_collection_enabled:false`. + +For each item in `TODO.md` **outside Advanced S3**, add evidence links and change `[ ]` to `[x]` **only after the exact acceptance criterion genuinely passes**. If evidence is missing, preserve the checkbox and append a specific blocker and safest next action. Submit reviewed code/tests/docs through one PR; do not merge failing CI or secretly change production policy. End with a table for every original TODO item: `PASS`, `FAIL`, `BLOCKED_OWNER_ACTION` or `NOT_APPLICABLE_ATTESTED`, evidence path and exact SHA. diff --git a/docs/S3_SPECS.md b/docs/S3_SPECS.md new file mode 100644 index 00000000..c98ee374 --- /dev/null +++ b/docs/S3_SPECS.md @@ -0,0 +1,250 @@ +# S3 API Specifications and Compatibility Matrix + +> **Scope:** TeleDrive's S3-compatible listener over the shared SQL catalog and USER MTProto engine with historical same-channel Bot-media reads only, **not** AWS S3. **Last reviewed:** 2026-10-15 against the development default branch. **Target:** P2 Advanced S3 planning; this document does not assert production certification or update the released v1.0.7 binary. +> +> **Canonical operation inventory:** [AWS S3 REST API, 2006-03-01, operation list](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html). This matrix enumerates **all 116 operations** on that AWS S3 service-reference page as reviewed. AWS S3 Control, S3 Tables, S3 on Outposts, S3 Files and future additions are **separate services**, not silently included in this list. This is a protocol/API coverage inventory, not a claim to implement every possible request parameter, header, mode or AWS service. + +## 1. Read the status correctly + +| Status | Meaning | +| --- | --- | +| ✅ Supported | Implemented for the explicitly documented TeleDrive contract and backed by focused regression coverage; **not** a claim of complete AWS parity. | +| 🟡 Partial | Route/behavior exists, but significant AWS options, response fields or state transitions are deliberately restricted. Treat unsupported variants as unsupported. | +| ❌ Unsupported | No supported TeleDrive contract; do not rely on that AWS method. It may be rejected as NotImplemented (501) **or** require additional fail-closed routing tests; do not assume every selector currently returns the same error. | +| Not verified | Client, deployment configuration or optional parameter is untested; never upgrade an operation's status merely because another client happens to succeed. | + +**Inventory totals:** 11 supported, 27 partial, 78 unsupported; **116 AWS S3 operations**. Count operations, not request permutations. Partial operations are **not** fully supported. These statuses describe development code and tests, not a fresh CI run or independently validated production setup. + +**Machine-readable inventory:** [116-operation JSON manifest](../teledrive/s3/operation_manifest.json) is checked against every row of this matrix in [fail-closed route tests](../tests/s3/test_s3_route_guard.py). Supported AWS subresources are explicitly allowlisted before the signer, SQL and Telegram transports can mutate data; unknown selectors cannot silently become normal bucket/object operations. This addresses a P2-0 subtask, **not** the entire protocol parity gate. + +**Implementation evidence:** [gateway router](../web/s3.py), [S3 protocol modules](../teledrive/s3/), [S3 tests](../tests/), [CI guide](CI_WORKFLOWS.md), [implementation/production status](IMPLEMENTATION_STATUS.md). On a new code SHA, rerun its own CI rather than recycling a historical green status. No new test execution was performed solely to prepare this inventory. + +## 2. Platform contract and parameter-level specifications + +| Specification area | TeleDrive behavior / boundary | Evidence or next step | +| --- | --- | --- | +| Deployment and object identity | Web/REST port 8080 and S3 port 9000 in **one default container**; S3, REST and explicitly queued local-file jobs share the catalog, SQL heads, versions and Telegram chunk payloads. | [Unified architecture](IMPLEMENTATION_STATUS.md#2-architecture-and-unified-storage-contract), [integration tests](../tests/web/test_unified_service.py) | +| Authentication | Scoped persisted S3 keys and **AWS Signature V4** header signing; access-key secrets encrypted at rest in SQL via an external master key. REST Bearer credentials are **not** S3 credentials. | [SigV4](../teledrive/s3/sigv4.py), [credentials](../teledrive/s3/credentials.py), [auth tests](../tests/s3/test_s3_sigv4_defense.py) | +| Signature variants | Query presigns with expiry **1–604,800 seconds**, limited methods and stricter checksum-bound PUT/multipart POST rules. SigV2, STS/session tokens and aws-chunked streaming are not implemented. | [SigV4 code](../teledrive/s3/sigv4.py), [presigned tests](../tests/s3/test_s3_presigned.py), [multipart presigns](../tests/s3/test_s3_presigned_complete.py) | +| Browser POST | Restricted SigV4 policy forms; checksum-pinned, maximum **20 MiB form** and policy expiry within **15 minutes**; not arbitrary AWS POST policy clauses. | [POST policy](../teledrive/s3/post_policy.py), [POST tests](../tests/s3/test_s3_post_policy.py) | +| Addressing/regions | Path-style endpoint is the documented client default; limited configured virtual-host addressing exists. Bucket location reports us-east-1-compatible empty constraint. ListBuckets supports paginated bucket inventory scoped to this private us-east-1 region, with immutable SQL CreationDate. No AWS multi-region, Access Point ARN or directory-bucket routing. | [gateway](../web/s3.py), [virtual-host tests](../tests/s3/test_s3_vhost.py), [location tests](../tests/s3/test_s3_bucket_location.py) | +| Object payloads | PUT/GET/HEAD with immutable manifest, Telegram USER MTProto chunks (historic Bot chunks can be read by original message ID in the same user-authorized channel), SHA-256 integrity and byte-range GET. Default REST/S3 single-transfer upload ceiling is **2 GiB**, configurable via TELEDRIVE_REST_MAX_UPLOAD_BYTES; actual deploy/Telegram limits apply. | [gateway](../web/s3.py), [storage tests](../tests/storage/test_storage_engine.py) | +| ETag | TeleDrive single PUT returns a SHA-256-derived ETag; multipart has its own ETag representation. **Never assume AWS single-part MD5 ETag equality.** | [gateway](../web/s3.py), [multipart](../teledrive/s3/multipart.py), [contract tests](../tests/s3/test_s3_contract.py) | +| Checksums on upload | Integrity validation for signed SHA-256, SHA-1, Content-MD5, CRC32, CRC32C and CRC64NVME on implemented PUT/UploadPart paths. When `x-amz-checksum-mode: ENABLED` is supplied for an object GET/HEAD, return the actual immutable full-object SHA-256 checksum; GetObjectAttributes can expose the same digest as `ChecksumSHA256` with `FULL_OBJECT` semantics. **Not** a guarantee of all SDK trailer/checksum algorithms or AWS multipart composite checksums. | [checksum helper](../teledrive/s3/checksums.py), [additional checksum tests](../tests/s3/test_s3_additional_checksums.py) | +| Read preconditions | Selected If-Match, If-None-Match, If-Modified-Since and If-Unmodified-Since for GET/HEAD; byte ranges supported. | [conditions](../teledrive/s3/conditions.py), [tests](../tests/s3/test_s3_conditions.py) | +| Write preconditions | Signed If-Match/If-None-Match on bounded simple PUT/DELETE and multipart completion; date-based write preconditions and conditional explicit-version deletes are not implemented. | [write conditions](../teledrive/s3/write_conditions.py), [tests](../tests/s3/test_s3_conditional_writes.py) | +| Metadata | Selected signed HTTP metadata and x-amz-meta-* persisted per version, limited to **2 KiB serialized**; metadata and copy directives restricted. | [object metadata](../teledrive/s3/object_metadata.py), [metadata tests](../tests/s3/test_s3_metadata.py) | +| Versioning and delete markers | Each publish retains a version; GET/HEAD and tags may select a published VersionId, list versions and delete markers. Only **Enabled** accepted for Get/PutBucketVersioning; no Suspended/null-version/MFADelete semantics. DeleteObjects cannot select individual VersionIds. | [versioning](../teledrive/s3/versioning.py), [version/read/delete tests](../tests/s3/test_s3_bucket_versioning.py), [marker tests](../tests/s3/test_s3_delete_markers.py) | +| Bucket/object tags | Bucket tags support the AWS general-purpose bucket 50-tag limit with case-sensitive UTF-16-length-validated keys/values, atomic replacement, scoped permissions, verified request integrity and get/delete; object tags remain limited to 10 per immutable version. | [tagging](../teledrive/s3/tagging.py), [tag tests](../tests/s3/test_s3_tagging.py) | +| Bucket CORS | Restricted origin/method/header CORS rules, max **100** rules, **64 KiB** config. CORS is not authentication. | [CORS](../teledrive/s3/cors.py), [tests](../tests/s3/test_s3_cors.py) | +| Public access | **Private only.** Get/PutPublicAccessBlock expose/accept all four values as true. GetBucketPolicyStatus is read-only and always IsPublic=false because public grants/policies cannot be configured. Non-private ACLs and public bucket policies are **not** supported. | [public access module](../teledrive/s3/public_access.py), [tests](../tests/s3/test_s3_public_access_block.py) | +| Multipart | Durable staged parts, up to **10,000** part numbers, minimum **5 MiB** for non-final completed parts, size/part limits, guarded completion and abort. Completion atomically records an immutable object-version part snapshot (number/size/SHA-256/ETag); historical and copied multipart attributes have bounded 1–1,000 part pages and a read-only pre-upgrade SQL fallback. Failed/aborted/unused Telegram chunks remain protected until approved GC. | [multipart](../teledrive/s3/multipart.py), [tests](../tests/s3/test_s3_multipart.py), [recovery tests](../tests/s3/test_s3_recovery.py) | +| Copy | Metadata-level local bucket/key and selected source-VersionId copy; signed source, permissions, metadata/tag directives and ETag source preconditions. No UploadPartCopy, presigned CopyObject or ranged source copy. | [gateway copy route](../web/s3.py), [tests](../tests/s3/test_s3_copy.py), [version copy](../tests/s3/test_s3_version_copy.py) | +| Unsupported write semantics | SSE-S3/SSE-KMS/SSE-C, retention/Object Lock, public ACLs, non-STANDARD tiers, request-payer, aws-chunked/trailer semantics and select copy conditions **must not be silently acknowledged**. | [fail-closed write checks](../web/s3.py), [tests](../tests/s3/test_s3_fail_closed.py) | +| Clients and operational proof | Boto3, AWS CLI and rclone scenarios exist, with **synthetic Telegram transport** for most storage integration tests; dedicated live Telegram E2E tests validate other actual payload paths. **Cyberduck, other SDKs and every SDK checksum mode remain unverified.** | [client tests](../tests/s3/test_s3_cli_compat.py), [CI guide](CI_WORKFLOWS.md) | +| Physical delete and lifecycle | Object deletion removes catalog visibility or version references; it **does not authorize deleting Telegram messages**. Only SQL-only AbortIncompleteMultipartUpload lifecycle rules are implemented; an hourly task marks old unfinished upload sessions aborted while retaining Telegram chunks. Object expiration/transitions and physical GC are not enabled. | [Lifecycle](../teledrive/s3/lifecycle.py), [lifecycle tests](../tests/s3/test_s3_lifecycle.py), [GC safeguards](IMPLEMENTATION_STATUS.md#41-physical-telegram-garbage-collection-disabled) | + +## 3. Implemented and partially implemented AWS S3 actions + +The following operations have a routed implementation. Every qualification in the right column is part of the contract. + +| AWS S3 API operation | Status | Implemented scope / explicit gap | +| --- | --- | --- | +| [CreateBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucket.html) | ✅ Supported | Creates a bucket in the shared catalog. | +| [HeadBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadBucket.html) | ✅ Supported | Authenticated existence check. | +| [DeleteBucket](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucket.html) | ✅ Supported | Requires an empty/deletable catalog bucket. | +| [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html) | ✅ Supported | Private single-region us-east-1 contract: signed GET, creation date, owner ID, 1–10,000 max-buckets, prefix/region filters, HMAC-scoped continuation token and keyset pagination. AWS account quota APIs, multi-region and directory buckets do not exist in TeleDrive. | +| [GetBucketLocation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLocation.html) | 🟡 Partial | Only the us-east-1-compatible empty LocationConstraint. | +| [ListObjects](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html) | 🟡 Partial | V1 prefix, delimiter, marker and bounded max-keys. | +| [ListObjectsV2](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html) | 🟡 Partial | V2 continuation, start-after, prefix, delimiter, encoding-type=url; bounded max-keys. | +| [ListObjectVersions](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html) | 🟡 Partial | Published versions/delete markers; restricted selectors and pagination. | +| [PutObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObject.html) | 🟡 Partial | Size-bounded, checksum-verified upload with metadata/tags and version creation; no SSE, Object Lock or aws-chunked. | +| [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) | 🟡 Partial | Full, byte-range and selected VersionId reads, ETag/date conditions; no archive or SSE behavior. | +| [HeadObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html) | 🟡 Partial | Basic object metadata and selected VersionId/read preconditions; not all AWS headers. | +| [DeleteObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObject.html) | 🟡 Partial | Creates logical delete marker or deletes an explicit published VersionId; does not delete Telegram messages. | +| [DeleteObjects](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html) | 🟡 Partial | Up to 1,000 key-only entries; no per-entry VersionId, MFA or governance bypass. | +| [CopyObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CopyObject.html) | 🟡 Partial | Catalog-local source/version copy, directives and ETag conditions; no presigned/range/date-based copy. | +| [GetObjectAttributes](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAttributes.html) | ✅ Supported | Documented private single-region general-purpose contract: signed ETag, ObjectSize, SHA-256 FULL_OBJECT Checksum, fixed STANDARD StorageClass, published historical VersionId, LastModified header, and immutable completed-multipart ObjectParts (selected part number/size/SHA-256). Supports signed MaxParts 1–1,000, PartNumberMarker 0–10,000, IsTruncated/NextPartNumberMarker and copied version snapshots. Non-MPU objects never fabricate parts; old SQL completion journals are read only and strictly checked. Fails closed on invalid/unsigned inputs and inconsistent history. AWS SSE-C, RequestPayer, directory buckets and provider-managed tiers are outside the TeleDrive contract. | +| [CreateMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateMultipartUpload.html) | 🟡 Partial | Signed initiation and selected metadata; no SSE/retention/storage-tier variants. | +| [UploadPart](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPart.html) | 🟡 Partial | 1–10,000 part numbers, configured part-size bound and selected integrity checks. | +| [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html) | ✅ Supported | TeleDrive private general-purpose contract: scoped active uploads, SQL-bounded 1–1,000 part pages, part-number marker 0–10,000, sparse/replaced parts, accurate ETag, Size, LastModified, STANDARD StorageClass, IsTruncated/NextPartNumberMarker. Invalid/over-limit cursors and wrong-owner/inactive IDs fail closed without Telegram reads. AWS Owner/Initiator account identities, multipart composite checksums, other tiers/directory buckets remain outside the contract. | +| [ListMultipartUploads](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListMultipartUploads.html) | ✅ Supported | TeleDrive private general-purpose contract: scoped active sessions, bounded SQL keyset pages 1–1,000, prefix, delimiter/CommonPrefixes, UTF-8 encoding-type=url, same-key UploadId cursors, Initiated/StorageClass and NextKeyMarker/NextUploadIdMarker. Completed, aborted and foreign-owner sessions excluded. AWS billing/requester pays, account ownership identities and directory buckets remain outside scope. | +| [CompleteMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html) | 🟡 Partial | Fenced, zero-reupload staged-part publication; selected SHA-bound requests only. | +| [AbortMultipartUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/API_AbortMultipartUpload.html) | 🟡 Partial | Logical abort; remote orphan media remains protected pending approved GC. | +| [GetBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketTagging.html) | ✅ Supported | Reads bucket tags; missing tag set reports NoSuchTagSet. | +| [PutBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketTagging.html) | ✅ Supported | Full general-purpose private-bucket tagging contract: 0–50 unique tags, 128/256 UTF-16-unit limits, signed checksum-verified XML, atomic replacement and correct HTTP 200; AWS billing/ABAC services not emulated. | +| [DeleteBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketTagging.html) | ✅ Supported | Removes stored bucket tag set. | +| [GetObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTagging.html) | 🟡 Partial | Current/published VersionId tags with scoped access. | +| [PutObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectTagging.html) | 🟡 Partial | Current/published VersionId tags with ownership checks. | +| [DeleteObjectTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectTagging.html) | 🟡 Partial | Current/published VersionId tags with ownership checks. | +| [GetBucketCors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketCors.html) | 🟡 Partial | Reads restricted CORS rules (maximum 100 rules). | +| [PutBucketCors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketCors.html) | 🟡 Partial | Limited 64 KiB XML grammar, origins and allowed methods. | +| [DeleteBucketCors](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketCors.html) | ✅ Supported | Removes CORS configuration. | +| [GetBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketVersioning.html) | 🟡 Partial | Always Enabled; not AWS unversioned/suspended states. | +| [PutBucketVersioning](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketVersioning.html) | 🟡 Partial | Idempotent Enabled only; rejects Suspended and MFADelete. | +| [GetPublicAccessBlock](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetPublicAccessBlock.html) | 🟡 Partial | Fixed all-four-fields true, no public mode. | +| [GetBucketPolicyStatus](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicyStatus.html) | 🟡 Partial | Read-only IsPublic=false for the enforced private-only gateway; no bucket policy engine or public ACLs. | +| [PutPublicAccessBlock](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutPublicAccessBlock.html) | 🟡 Partial | Accepts only all-four-fields true; cannot weaken privacy. | +| [GetBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycleConfiguration.html) | 🟡 Partial | Read a private bucket's bounded, prefix-scoped AbortIncompleteMultipartUpload-only rules; no AWS expiration/transition support. | +| [PutBucketLifecycleConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html) | 🟡 Partial | Signed strict XML; only Disabled/Enabled incomplete multipart abort rules with DaysAfterInitiation 1..3650 and Prefix. | +| [DeleteBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketLifecycle.html) | 🟡 Partial | Removes SQL configuration; it never deletes an object or Telegram media. | + +## 4. Unsupported AWS S3 actions grouped for the roadmap + +**All operations below are unsupported even when a similarly named REST admin endpoint or internal SQL function exists.** The identifiers specify planned workstreams, **not** commitments that every AWS feature will ship in P2. Security- or cloud-specific functionality may remain intentionally unsupported. Grouping is exhaustive for the reference inventory above. + +### P2-C — Access control, ACLs, policies and ownership + +Keep private-by-default security. Define either a tested private-only compatibility response or a real policy/ownership engine before advertising parity. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [DeletePublicAccessBlock](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeletePublicAccessBlock.html) | ❌ Unsupported | P2-C | +| [GetBucketAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAcl.html) | ❌ Unsupported | P2-C | +| [PutBucketAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAcl.html) | ❌ Unsupported | P2-C | +| [GetObjectAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAcl.html) | ❌ Unsupported | P2-C | +| [PutObjectAcl](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAcl.html) | ❌ Unsupported | P2-C | +| [GetBucketOwnershipControls](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketOwnershipControls.html) | ❌ Unsupported | P2-C | +| [PutBucketOwnershipControls](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketOwnershipControls.html) | ❌ Unsupported | P2-C | +| [DeleteBucketOwnershipControls](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketOwnershipControls.html) | ❌ Unsupported | P2-C | +| [GetBucketPolicy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicy.html) | ❌ Unsupported | P2-C | +| [PutBucketPolicy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketPolicy.html) | ❌ Unsupported | P2-C | +| [DeleteBucketPolicy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketPolicy.html) | ❌ Unsupported | P2-C | + +### P2-B — Encryption and immutable retention + +Implement actual encryption/key lifecycle and enforce per-version retention and legal-hold semantics across REST/S3/local jobs, copy, version delete and eventual physical GC. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [GetBucketEncryption](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html) | ❌ Unsupported | P2-B | +| [PutBucketEncryption](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html) | ❌ Unsupported | P2-B | +| [DeleteBucketEncryption](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketEncryption.html) | ❌ Unsupported | P2-B | +| [GetObjectLockConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectLockConfiguration.html) | ❌ Unsupported | P2-B | +| [PutObjectLockConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLockConfiguration.html) | ❌ Unsupported | P2-B | +| [GetObjectRetention](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectRetention.html) | ❌ Unsupported | P2-B | +| [PutObjectRetention](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html) | ❌ Unsupported | P2-B | +| [GetObjectLegalHold](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectLegalHold.html) | ❌ Unsupported | P2-B | +| [PutObjectLegalHold](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html) | ❌ Unsupported | P2-B | + +### P2-D — Lifecycle, expiration, transitions, replication and restore + +Model rules and durable workers without promising AWS transitions that Telegram cannot provide. Physical expiration/deletion requires independent backup, writer fences and two-person authorization. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [GetBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLifecycle.html) | ❌ Unsupported | P2-D | +| [PutBucketLifecycle](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycle.html) | ❌ Unsupported | P2-D | +| [GetBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketReplication.html) | ❌ Unsupported | P2-D | +| [PutBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html) | ❌ Unsupported | P2-D | +| [DeleteBucketReplication](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketReplication.html) | ❌ Unsupported | P2-D | +| [RestoreObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) | ❌ Unsupported | P2-D | + +### P2-E — Advanced data-plane protocol compatibility + +Support copy-part source ranges, signing and ownership, checksum composition and crash/retry semantics; also address parameter/header gaps for already partial operations. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [UploadPartCopy](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UploadPartCopy.html) | ❌ Unsupported | P2-E | + +### P2-F — Website, notifications and access logging + +Specify whether private-only hosting permits website mode; use transactional outbox for events, durable retries, scoped destinations and secret-safe log delivery. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [GetBucketWebsite](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketWebsite.html) | ❌ Unsupported | P2-F | +| [PutBucketWebsite](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketWebsite.html) | ❌ Unsupported | P2-F | +| [DeleteBucketWebsite](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketWebsite.html) | ❌ Unsupported | P2-F | +| [GetBucketNotification](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketNotification.html) | ❌ Unsupported | P2-F | +| [GetBucketNotificationConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketNotificationConfiguration.html) | ❌ Unsupported | P2-F | +| [PutBucketNotification](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketNotification.html) | ❌ Unsupported | P2-F | +| [PutBucketNotificationConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketNotificationConfiguration.html) | ❌ Unsupported | P2-F | +| [GetBucketLogging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketLogging.html) | ❌ Unsupported | P2-F | +| [PutBucketLogging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLogging.html) | ❌ Unsupported | P2-F | + +### P3-G — Analytics, inventory, metrics and tier policies + +Requires schedulers/report delivery and storage-tier accounting; out of P2 core. Telegram storage has no AWS Glacier/Intelligent-Tiering equivalence. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [GetBucketAnalyticsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAnalyticsConfiguration.html) | ❌ Unsupported | P3-G | +| [PutBucketAnalyticsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAnalyticsConfiguration.html) | ❌ Unsupported | P3-G | +| [DeleteBucketAnalyticsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketAnalyticsConfiguration.html) | ❌ Unsupported | P3-G | +| [ListBucketAnalyticsConfigurations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBucketAnalyticsConfigurations.html) | ❌ Unsupported | P3-G | +| [GetBucketInventoryConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketInventoryConfiguration.html) | ❌ Unsupported | P3-G | +| [PutBucketInventoryConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketInventoryConfiguration.html) | ❌ Unsupported | P3-G | +| [DeleteBucketInventoryConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketInventoryConfiguration.html) | ❌ Unsupported | P3-G | +| [ListBucketInventoryConfigurations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBucketInventoryConfigurations.html) | ❌ Unsupported | P3-G | +| [GetBucketMetricsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketMetricsConfiguration.html) | ❌ Unsupported | P3-G | +| [PutBucketMetricsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketMetricsConfiguration.html) | ❌ Unsupported | P3-G | +| [DeleteBucketMetricsConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketMetricsConfiguration.html) | ❌ Unsupported | P3-G | +| [ListBucketMetricsConfigurations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBucketMetricsConfigurations.html) | ❌ Unsupported | P3-G | +| [GetBucketIntelligentTieringConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketIntelligentTieringConfiguration.html) | ❌ Unsupported | P3-G | +| [PutBucketIntelligentTieringConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketIntelligentTieringConfiguration.html) | ❌ Unsupported | P3-G | +| [DeleteBucketIntelligentTieringConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketIntelligentTieringConfiguration.html) | ❌ Unsupported | P3-G | +| [ListBucketIntelligentTieringConfigurations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBucketIntelligentTieringConfigurations.html) | ❌ Unsupported | P3-G | + +### P3-H — AWS-managed products, directory buckets, annotations and provider services + +Mostly depends on AWS account services (Express sessions, directory buckets, S3 Metadata, Object Lambda, acceleration, billing), not generic S3-compatible object storage. + +| AWS S3 API operation | Status | Target | +| --- | --- | --- | +| [CreateBucketMetadataConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucketMetadataConfiguration.html) | ❌ Unsupported | P3-H | +| [CreateBucketMetadataTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateBucketMetadataTableConfiguration.html) | ❌ Unsupported | P3-H | +| [GetBucketMetadataConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketMetadataConfiguration.html) | ❌ Unsupported | P3-H | +| [GetBucketMetadataTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketMetadataTableConfiguration.html) | ❌ Unsupported | P3-H | +| [DeleteBucketMetadataConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketMetadataConfiguration.html) | ❌ Unsupported | P3-H | +| [DeleteBucketMetadataTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteBucketMetadataTableConfiguration.html) | ❌ Unsupported | P3-H | +| [UpdateBucketMetadataAnnotationTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UpdateBucketMetadataAnnotationTableConfiguration.html) | ❌ Unsupported | P3-H | +| [UpdateBucketMetadataInventoryTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UpdateBucketMetadataInventoryTableConfiguration.html) | ❌ Unsupported | P3-H | +| [UpdateBucketMetadataJournalTableConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UpdateBucketMetadataJournalTableConfiguration.html) | ❌ Unsupported | P3-H | +| [GetObjectAnnotation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectAnnotation.html) | ❌ Unsupported | P3-H | +| [PutObjectAnnotation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectAnnotation.html) | ❌ Unsupported | P3-H | +| [DeleteObjectAnnotation](https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjectAnnotation.html) | ❌ Unsupported | P3-H | +| [ListObjectAnnotations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectAnnotations.html) | ❌ Unsupported | P3-H | +| [GetBucketAbac](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAbac.html) | ❌ Unsupported | P3-H | +| [PutBucketAbac](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAbac.html) | ❌ Unsupported | P3-H | +| [GetBucketRequestPayment](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketRequestPayment.html) | ❌ Unsupported | P3-H | +| [PutBucketRequestPayment](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketRequestPayment.html) | ❌ Unsupported | P3-H | +| [GetBucketAccelerateConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketAccelerateConfiguration.html) | ❌ Unsupported | P3-H | +| [PutBucketAccelerateConfiguration](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketAccelerateConfiguration.html) | ❌ Unsupported | P3-H | +| [GetObjectTorrent](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObjectTorrent.html) | ❌ Unsupported | P3-H | +| [CreateSession](https://docs.aws.amazon.com/AmazonS3/latest/API/API_CreateSession.html) | ❌ Unsupported | P3-H | +| [ListDirectoryBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListDirectoryBuckets.html) | ❌ Unsupported | P3-H | +| [RenameObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RenameObject.html) | ❌ Unsupported | P3-H | +| [SelectObjectContent](https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html) | ❌ Unsupported | P3-H | +| [UpdateObjectEncryption](https://docs.aws.amazon.com/AmazonS3/latest/API/API_UpdateObjectEncryption.html) | ❌ Unsupported | P3-H | +| [WriteGetObjectResponse](https://docs.aws.amazon.com/AmazonS3/latest/API/API_WriteGetObjectResponse.html) | ❌ Unsupported | P3-H | + +## 5. P2 Advanced S3 implementation plan + +| Priority | Workstream | Implementation deliverable | Acceptance gate | +| --- | --- | --- | --- | +| P2-0 | Compatibility contract and fail-closed audit | Every known AWS selector, query parameter and unsupported header routed explicitly; central operation registry and machine-readable support manifest generated from tests. | Negative tests prove unsupported requests cannot fall through as successful list/get/put/delete; all documented statuses match behavior. | +| P2-1 | P2-E: multipart, checksums and SDK parity | UploadPartCopy or explicit permanent rejection; checksum negotiation, trailers, multipart composition, VersionId/copy attributes, SDK request-mode compatibility. | boto3 + AWS CLI + rclone matrix; corruption/mismatch rejection, retries, presigns, concurrent writers, cross-client REST/S3 readback. | +| P2-2 | P2-C: policies and ownership | Decide immutable private-only model versus scoped policy engine; implement safe Get/Put/Delete semantics only if authorization is real. | Cross-tenant permissions, deny-by-default, anonymous/public denial, authorization fuzz tests. | +| P2-3 | P2-B: encryption and retention | Design managed keys, SSE at-rest semantics, key rotation and version-aware Object Lock; preserve immutable historical reads. | Restart/backup/restore and recovery proofs, byte-level validation, enforced retention on *every* delete path. | +| P2-4 | P2-D: lifecycle and replication | Durable lifecycle state machine, dry-run expiration, target/failure handling, version-aware safety fences; replication only with genuinely independent destination. | SQL crash/replay/race tests, leaked/orphan protection, no Telegram physical delete without separately approved safety gates. | +| P2-5 | P2-F: notifications/logging/website | Event outbox and subscriber retries first; hosting only if consistent with private-only policy. | Exactly-once-visible event IDs or explicit dedupe policy, backpressure, no credential leakage, restart recovery. | +| After P2 | P3-G/H: reports and AWS-specific services | Independently evaluate ROI; avoid fake AWS-specific semantics (billing, S3 Express, Object Lambda, managed metadata). | Separate design/RFC and client contracts before implementing. | + +### Engineering checklist / definition of done + +- [x] Commit a machine-readable 116-operation manifest synchronized by CI with the reviewed specification matrix, and add a strict S3 request-parameter allowlist. Upstream AWS additions still require explicit periodic re-audits. +- [ ] For every supported operation, test normal, missing/auth-failed, malformed, over-limit, retry, multiworker race, signature-tampered and crash/restart cases; assert no partial SQL publication. +- [ ] For unsupported operations, add adversarial selector/HTTP method tests: **no accidental HTTP 200 or mutation**; unsupported-sensitive headers must fail before Telegram I/O. +- [ ] Verify all S3 mutation paths are reflected in REST/local-job heads, version records, tombstones, tags and copy aliases; invariants run against both SQLite and PostgreSQL. +- [ ] Add boto3, AWS CLI and rclone version-pinned compatibility matrix plus optional client tests; distinguish simulated Telegram I/O from real test-channel Telegram E2E. +- [ ] Define atomic migration/rollback and API backward compatibility for every metadata table or new header field; no irreversible rewrite of historic versions. +- [ ] Respect GC blocking prerequisites: independent payload backup and restore rehearsal, owner/alias/retention graph, writer fencing and explicit independent approval **before** physical Telegram deletion. +- [ ] Update this matrix, [README](../README.md), [Project Status](IMPLEMENTATION_STATUS.md) and [CI Guide](CI_WORKFLOWS.md) **only when implementation and evidence change**; keep unsupported rows marked unsupported until validated on the new SHA. + +## 6. Release and evidence boundaries + +A successfully parsed or authenticated AWS request is **not** automatically a supported S3 specification. Successful synthetic tests prove only their scenario, not general AWS service parity, multi-region behavior, original-media recovery or an operator-authorized production cutover. P2 work should preserve the default **one-container** interface topology, bounded resource use, shared REST/S3/local-job catalog, and fail-closed safety guarantees documented in [Project Status](IMPLEMENTATION_STATUS.md). + +**Upstream reference:** [Amazon S3 REST API operations](https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html). **Repository evidence:** [S3 route](../web/s3.py), [S3 modules](../teledrive/s3/), [S3 tests](../tests/), [TODO](../TODO.md). The AWS operation inventory can grow; re-audit it before every advanced-compatibility milestone. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 00000000..374ea1f6 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,58 @@ +# Security Policy + +> **Scope:** vulnerability reporting, trust boundaries, network exposure, secret handling, and security validation for the TeleDrive fork. +> **Do not report vulnerabilities or post credentials in public issues.** + +## 1. Report a vulnerability privately + +Use the repository's [GitHub Security tab](https://github.com/babico/TeleDrive/security) and private vulnerability reporting when available. Include the affected release or commit, operating system, minimal reproduction steps, expected/observed impact, and carefully redacted logs. + +Do **not** submit active credentials, session files, database connection URLs, Telegram messages or exploitable details in a public issue. If private reporting is unavailable, contact the repository maintainer through an appropriately private channel rather than exposing an unpatched exploit. + +## 2. Threat model and trusted components + +TeleDrive is primarily a **single-administrator** application. The WebUI can manage Telegram accounts, target channels, local source paths, credential issuance and metadata state. Treat this authority as sensitive. + +The host OS administrator, Docker daemon, host user managing mounted files, SQL server administrator, reviewed Python/database drivers and Telegram itself are **trusted boundaries**. TeleDrive does not sandbox a malicious host administrator who can modify mounts or binaries while the service runs. + +Telegram Saved Messages and ordinary cloud channels are **not end-to-end encrypted**. Encrypt sensitive payloads separately if Telegram must not see their contents. + +## 3. Secrets and credentials + +Treat all of the following as secrets: + +| Secret | Consequence if exposed | +| --- | --- | +| Telethon `*.session`, Telegram API ID/hash, login codes and 2FA | User-account and channel access | +| `config.yaml`, `.env`, Web password | Administrative control and deployment information | +| SQL URL and SQL backups | Object manifests, references, jobs and potentially encrypted credential metadata | +| `config/s3-master.key` and S3 access keys | S3 key decryption and object API access | + +Store secrets in protected volumes or a secrets manager with restrictive file permissions. Never commit them, print them to CI logs, or paste them into bug reports. When a secret is compromised, rotate/revoke it using the applicable Telegram, WebUI or S3 credential controls; review the dependent backups and sessions. + +## 4. Network exposure + +The default Docker Compose deployment uses **one container with two loopback-bound listeners**: + +| Interface | Default host binding | Authentication | +| --- | --- | --- | +| WebUI / REST | `127.0.0.1:8080` | Basic admin and scoped REST Bearer tokens | +| S3 | `127.0.0.1:9000` | Scoped S3 SigV4 access/secret keys | + +Do not expose either endpoint over untrusted plaintext HTTP. For remote access, use an HTTPS reverse proxy or VPN, set `TELEDRIVE_ALLOWED_HOSTS` appropriately, use a unique Web password of at least 16 characters, and restrict firewall access. + +The unified server dispatches by its actual socket-local listener port, **not** by client-supplied Host or forwarding headers. REST Bearer tokens are not interchangeable with S3 SigV4 keys. S3 master-key material must not be embedded in URLs or requests. + +## 5. Database, files and runtime defenses + +`state.database_url` accepts synchronous SQLAlchemy dialects without a hardcoded vendor whitelist. Review and pin additional driver packages, use database TLS where available, and scope the SQL principal to the intended schema/database. Initial schema creation/migrations may need DDL privileges; reduce privileges after migrations when operationally feasible. + +The Docker default includes non-root execution, dropped Linux capabilities, `no-new-privileges`, a read-only root filesystem, a read-only `/uploads` mount, protected secret files, source/session/log/SQL path allowlists, non-regular and symlink checks, upload leases and atomic claims, rate limiting, Host validation, CSRF protection, security headers and credential redaction. + +A SQL snapshot alone cannot restore lost Telegram payloads or S3 master keys. Independent originals and rehearsed recovery are required. Physical Telegram garbage collection remains disabled pending the [production recovery and safety controls](IMPLEMENTATION_STATUS.md#41-physical-telegram-garbage-collection-disabled). + +## 6. Security testing and release evidence + +The [Security and Regression workflow](../.github/workflows/quality-security.yml) covers Python compilation/regression, `pip check`, Bandit, `pip-audit`, Docker image build, Trivy and application listener smoke tests. GitHub Actions are pinned to immutable revisions; Dependabot monitors Python, Docker and Actions dependencies. + +A successful scanner run is **not** permission to expose the administration UI publicly or a substitute for owner-approved production penetration testing. Consult the [CI and Test Guide](CI_WORKFLOWS.md) and distinguish test-channel E2E from actual production data. diff --git a/docs/TELETHON_STORAGE.md b/docs/TELETHON_STORAGE.md new file mode 100644 index 00000000..68e2486a --- /dev/null +++ b/docs/TELETHON_STORAGE.md @@ -0,0 +1,181 @@ +# Telethon USER Storage — Installation, Recovery and Migration + +> **Status:** development `main`, not the published v1.0.7 binaries. +> **Scope:** the single-container WebUI/REST/S3/local-job object engine. The +> separate historic folder uploader and archived Bot fixtures are not new +> storage backends. For owner-only production approval see +> [Project Status](IMPLEMENTATION_STATUS.md) and [Local Acceptance](LOCAL_AGENT_ACCEPTANCE.md). + +## 1. One storage architecture + +- The only live unified payload transport is **an authorized Telegram USER + account over Telethon MTProto**, not a Telegram Bot and not Saved Messages. +- REST, S3 and explicitly queued local-file jobs publish into **the same SQL + heads, object/version manifests and Telegram channel**. The Docker default + uses one `teledrive` container with local ports 8080 and 9000. +- Choose a **dedicated private broadcast channel** where the authorized user + has posting rights. The channel must be in the user's accessible dialogs; + for numeric private IDs the runtime resolves within the first 5,000 dialogs. +- Telethon USER MTProto is the only transport; no transport selector + exists. Old `TELEDRIVE_STORAGE_BOT_*` values are ignored by the unified + server. +- Telethon's `device_model` and `app_version` identify **TeleDrive Storage + Client**. They are not an HTTP User-Agent and do not claim to be an official + Telegram client. Respect Telegram flood waits and account usage limits. + +## 2. Provision one or more authorized user sessions + +Work on the owner-controlled host; the same identity must be able to post to +the dedicated channel. Never put phone codes, 2FA passwords or session bytes +in a Git repository, shell history, issue, CI log or the Docker environment. + +```bash +git clone https://github.com/babico/TeleDrive.git +cd TeleDrive +mkdir -p config data uploads +cp .env.example .env +cp ../config.example.yaml config/config.yaml # optional tidy template +chmod 600 .env config/config.yaml +python3 -m venv .venv +. .venv/bin/activate +python -m pip install -r requirements.txt + +# The script asks for the API credentials interactively (never commit them); +# it prompts for the Telegram sign-in code once and writes the session. +python scripts/dev/create_storage_user_session.py --output config/mtproto-user.session +chmod 600 config/mtproto-user.session +``` + +Obtain API credentials from [my.telegram.org/apps](https://my.telegram.org/apps). +Generate the session **before** starting a configured TeleDrive container; +the script refuses to overwrite an existing session. The service creates +`config/config.yaml` with safe defaults on first start; add your credentials +under the **`storage:`** section as an `apis:` list there (do not commit it): + +```yaml +# config/config.yaml — storage.apis is the authoritative credential source. +storage: + channel: "YOUR_PRIVATE_CHANNEL_ID_OR_USERNAME" + strategy: round_robin # round_robin (default) | failover + apis: + - api_id: 111111 + api_hash: "YOUR_TELEGRAM_API_HASH_1" + session_file: "/config/mtproto-user.session" + account: mtproto-user # optional label + - api_id: 222222 + api_hash: "YOUR_TELEGRAM_API_HASH_2" + session_file: "/config/mtproto-user-2.session" + account: mtproto-user-2 # optional label +``` + +Every set must resolve the **same** dedicated `storage.channel`. +`storage.strategy` selects `round_robin` (default) or `failover`: round_robin +distributes sends across the sets; failover keeps one set until a definite +flood-wait rejection, then switches to the next with a fresh send intent. +Ambiguous send outcomes never trigger cross-set retries. Account labels +default to `mtproto-user`, `mtproto-user-2`, … when omitted. +There are no service settings in environment variables: the +config section is the only source. + +The service never falls back to environment credentials; a missing or +empty `storage.apis:` section fails closed at startup. + +`docker-compose.yml` mounts `./config` at `/config`; the session paths above +must live under that mount. Ensure owner/group and mode permit the configured +non-root `PUID:PGID` to read them. **Never** copy the StringSession into +`.env`. Set a unique 16+-character Web password and generate an independently +backed-up S3 master key as documented in [README](../README.md#4-s3-compatible-api). + +```bash +docker compose config --quiet +docker compose up -d --build +docker compose ps +docker compose logs --tail=120 teledrive +``` + +The listeners may be reachable before a new installation is provisioned, but +**uploads remain unavailable until the live authenticated session connects**. +Any partially configured identity may prevent startup; correct the credentials +or restore the sealed session rather than falling back to a Bot. + +## 3. What stays readable after retiring Bot uploads + +| Historical payload | Can the USER-only server read it? | Safe action | +| --- | --- | --- | +| New USER channel object with intact SQL | **Yes**, if the original account/channel/session is accessible | Read and SHA-256-verify | +| Old `api-bot` chunk in the **same** accessible channel with original SQL `chat_id` and `message_id` | **Yes**, via Telethon message lookup without any Bot token | Keep original SQL; verify every chunk/full hash | +| Old Bot chunk in a different/inaccessible channel | **No automatic fallback** | Preserve media, credentials and SQL; independently authorize a read-only migration | +| Missing SQL or unknown channel marker | **Unverified candidate only** | Inventory read-only; require independent bucket/key/size/full-hash/version evidence | + +The server never rewrites historic object manifests to invent ownership, +transfers payloads from inaccessible channels, or enables physical deletion. +Removing Bot configuration **does not prove** every existing object is +recoverable. Back up first and test an isolated restore. + +## 4. Read-only history and original-media disaster recovery + +With a connected owner USER session, the authenticated admin can inspect +bounded history pages: + +```bash +curl -u admin \ + 'http://127.0.0.1:8080/api/v1/admin/storage/channel-history?limit=100&include_external_markers=true' +``` + +Use `next_before_message_id` from the result as +`before_message_id` for older pages. Other-sender/Bot markers are **not** +ownership, bucket/key, full SHA-256 or version proofs. No media is downloaded, +deleted or indexed by this endpoint. + +For an **original owner-provided sealed SQLite metadata snapshot**, use the +read-only user-MTProto DR command in an **isolated environment**, not with a +production writer active on the same session: + +```bash +# In the staging shell, set TELEDRIVE_STORAGE_MTPROTO_API_ID and +# TELEDRIVE_STORAGE_MTPROTO_API_HASH, and point the following variable to +# a protected owner-authorized copy of the original USER StringSession. +export TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE=/secure/staging/owner-user.session +python scripts/ops/rehearse_telegram_dr.py \ + --snapshot /secure/staging/sealed-original.db \ + --sha256 EXPECTED_ARCHIVE_SHA256 \ + --bucket ORIGINAL_BUCKET --key ORIGINAL_KEY \ + --account ORIGINAL_ACCOUNT --chat-id ORIGINAL_CHAT_ID \ + --target ORIGINAL_CHANNEL_ID_OR_USERNAME +``` + +The tool checks the archive fingerprint, opens a **temporary clone**, validates +the original channel identity, reads via user MTProto and verifies whole-body +SHA-256, exact size and a byte range. No upload, remote delete or production +SQL mutation occurs. An old `api-bot` object is readable if its immutable SQL +reference points to **this same authorized channel**. This is a sample-object +DR check—not proof of a complete independent payload backup or full archive +recovery. PostgreSQL/MySQL owners must rehearse their native SQL restores +separately. + +For objects with no reliable SQL map, use +`scripts/ops/inventory_telegram_history.py` (private candidate inventory) +and `scripts/ops/rebuild_from_telegram.py` (separately proven plan, new isolated +catalog); see [Backup and Restore](BACKUP_RESTORE.md). + +## 5. Status, CI and unfinished safety gates + +The exact-source 9-workflow baseline +[`a430212`](https://github.com/babico/TeleDrive/commit/a4302122ff80d0f5d05bab8a362e1983a5910235) +passed Security, REST/S3, Databases, Docker fault injection, live +USER MTProto, live Bot **historical-compatibility** E2E, live Multiworker +PostgreSQL and release gate. Check [Actions](https://github.com/babico/TeleDrive/actions) +for any newer commit; PR tests alone do not certify production recovery. + +**Still open:** independently archived media bytes and real restore; +unknown-history reconciliation without invented metadata; complete +writer/version/alias-aware GC fencing; physically separate host acceptance; +owner-approved long-duration capacity/429 telemetry; genuine legacy cutover +or signed no-legacy exemption; two-person production approval; and +release-candidate checks on the new exact SHA. The advanced S3 feature +matrix is tracked separately in [S3 Specs](S3_SPECS.md). + +**Physical Telegram deletion remains disabled.** Original Telegram message +IDs cannot be resurrected from a SQL snapshot. Follow the +[owner-local mission](LOCAL_AGENT_ACCEPTANCE.md); never mark these gates +passed based on synthetic fixtures or a green unit-test suite. diff --git a/images/dashboard.png b/images/dashboard.png deleted file mode 100644 index 1487b95c..00000000 Binary files a/images/dashboard.png and /dev/null differ diff --git a/images/settings.png b/images/settings.png deleted file mode 100644 index 915c14cc..00000000 Binary files a/images/settings.png and /dev/null differ diff --git a/images/upload-queue.png b/images/upload-queue.png deleted file mode 100644 index c75cb0ff..00000000 Binary files a/images/upload-queue.png and /dev/null differ diff --git a/installer/TeleDrive.ico b/installer/TeleDrive.ico deleted file mode 100644 index 28157eab..00000000 Binary files a/installer/TeleDrive.ico and /dev/null differ diff --git a/installer/TelegramAutoUpload.iss b/installer/TelegramAutoUpload.iss deleted file mode 100644 index 3c3a60be..00000000 --- a/installer/TelegramAutoUpload.iss +++ /dev/null @@ -1,33 +0,0 @@ -#define MyAppName "TeleDrive" -#define MyAppVersion "1.0.6" -#define MyAppPublisher "TeleDrive" -#define MyAppExeName "TeleDrive.exe" -#define MyAppDescription "Turn Telegram into your personal unlimited cloud storage." - -[Setup] -AppId={{8F15A45F-47A7-4955-944B-E9A60FEB566A} -AppName={#MyAppName} -AppVersion={#MyAppVersion} -AppPublisher={#MyAppPublisher} -DefaultDirName={localappdata}\Programs\TeleDrive -DefaultGroupName={#MyAppName} -OutputDir=..\dist-installer -OutputBaseFilename=TeleDrive-Setup-{#MyAppVersion} -Compression=lzma -SolidCompression=yes -WizardStyle=modern -PrivilegesRequired=lowest -SetupIconFile=..\installer\TeleDrive.ico - -[Files] -Source: "..\dist\TeleDrive.exe"; DestDir: "{app}"; Flags: ignoreversion - -[Icons] -Name: "{autoprograms}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}" -Name: "{autodesktop}\{#MyAppName}"; Filename: "{app}\{#MyAppExeName}"; Tasks: desktopicon - -[Tasks] -Name: "desktopicon"; Description: "Create a desktop shortcut"; GroupDescription: "Additional shortcuts:" - -[Run] -Filename: "{app}\{#MyAppExeName}"; Description: "Launch {#MyAppName}"; Flags: nowait postinstall skipifsilent diff --git a/pyproject.toml b/pyproject.toml index ff3dcb67..cabbf3ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,18 +4,17 @@ build-backend = "setuptools.build_meta" [project] name = "telegram-auto-upload" -version = "1.0.6" +version = "1.0.7" description = "Safe, resumable Telegram file uploader for Windows and Linux" requires-python = ">=3.11" dependencies = [ - "PySide6>=6.7,<7", "Telethon>=1.36,<2", "PyYAML>=6.0,<7", + "SQLAlchemy>=2.0,<3", ] -[project.scripts] -telegram-auto-upload = "app.main:main" -telegram-auto-upload-cli = "uploader:main" - [tool.setuptools.packages.find] -include = ["app*"] +include = ["teledrive*"] + +[tool.pytest.ini_options] +addopts = "-p no:cacheprovider" diff --git a/requirements.txt b/requirements.txt index 39eca723..6e0c6c1c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,10 @@ -telethon>=1.36.0,<2 -PyYAML>=6.0 -PySide6>=6.7,<7 +# Headless Web/API and isolated Docker integration tests (no PySide6). +fastapi==0.142.2 +uvicorn[standard]==0.54.0 +PyYAML==6.0.3 +Telethon==1.45.0 +SQLAlchemy==2.1.2 +psycopg[binary]==3.3.6 +PyMySQL==1.2.3 +cryptography==50.0.2 +defusedxml==0.7.1 diff --git a/scripts/dev/create_storage_user_session.py b/scripts/dev/create_storage_user_session.py new file mode 100644 index 00000000..d98695cc --- /dev/null +++ b/scripts/dev/create_storage_user_session.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Create a private Telethon USER StringSession for REST/S3 storage. + +Run interactively on an owner-controlled machine. The session grants account +access; do not put it in Git, CI logs or Docker environment variables. +""" +from __future__ import annotations + +import argparse +import asyncio +import os +import platform +from pathlib import Path + + +async def authorize(output: Path) -> None: + from telethon import TelegramClient + from telethon.sessions import StringSession + + api_id = int(os.environ["TELEDRIVE_STORAGE_MTPROTO_API_ID"]) + api_hash = os.environ["TELEDRIVE_STORAGE_MTPROTO_API_HASH"].strip() + if api_id <= 0 or len(api_hash) != 32: + raise ValueError("Invalid Telegram application API credentials") + if not output.parent.is_dir() or output.exists() or output.is_symlink(): + raise FileExistsError("Session parent must exist; refuse overwrite") + client = TelegramClient( + StringSession(), api_id, api_hash, + device_model="TeleDrive Storage Client", + system_version=platform.system(), app_version="TeleDrive 2.1", + receive_updates=False, flood_sleep_threshold=0, + ) + try: + await client.start() # Interactive user phone, OTP and optional 2FA. + identity = await client.get_me() + if identity is None or getattr(identity, "bot", False): + raise RuntimeError("Use a real Telegram USER session, not a bot token") + contents = client.session.save() + if not contents: + raise RuntimeError("Telegram returned an empty authorized user session") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(str(output), flags, 0o600) + try: + with os.fdopen(fd, "w", encoding="ascii") as stream: + stream.write(contents + "\n") + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + output.unlink(missing_ok=True) + raise + finally: + await client.disconnect() + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--output", type=Path, default=Path("config/mtproto-user.session")) + args = p.parse_args() + os.umask(0o077) + asyncio.run(authorize(args.output)) + print("Authorized user session saved with mode 0600; keep it private.") + + +if __name__ == "__main__": + main() diff --git a/scripts/dev/docker_fault_scenarios.py b/scripts/dev/docker_fault_scenarios.py new file mode 100644 index 00000000..4f678c7a --- /dev/null +++ b/scripts/dev/docker_fault_scenarios.py @@ -0,0 +1,470 @@ +#!/usr/bin/env python3 +"""Disposable Docker DOCKER FAULT scenarios. Never connects to real Telegram. + +A disk-backed transport models Telegram media, while PostgreSQL, the +production storage engine and two real Web API processes are not mocked. +All injected corruption is limited to the compose project's scratch volume. +""" +from __future__ import annotations +import asyncio +import base64 +import hashlib +import json +import os +from pathlib import Path +import shutil +import sys +import time +import urllib.error +import urllib.request +from uuid import uuid4 + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard +from teledrive.storage.user_mode import UserSendJournal +from teledrive.storage.engine import IntegrityError, TelegramObjectStorage +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore + +SOURCE = Path(os.environ["TELEDRIVE_FAULT_SOURCES"]) +REMOTE = Path(os.environ["TELEDRIVE_FAULT_REMOTE_DIR"]) +EVIDENCE = Path(os.environ["TELEDRIVE_FAULT_EVIDENCE_DIR"]) +DB_URL = os.environ["TELEDRIVE_FAULT_DB_URL"] +RESTORED_URL = os.environ["TELEDRIVE_FAULT_RESTORED_DB_URL"] +BUCKET = "docker-fault-only" +CHAT = -100700800900123 +ACCOUNT = "fake-dev-telegram" # NO Telegram network credentials, ever. + + +def sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def document_path(message_id: int) -> Path: + return REMOTE / (str(message_id) + ".bin") + + +class DiskTelegram: + """Real byte I/O and durable fake message IDs across Docker process restarts.""" + async def upload(self, source: Path, *, object_id: str, + ordinal: int, checksum: str): + body = source.read_bytes() + if sha(body) != checksum: + raise IntegrityError("Original source checksum changed") + # Per-upload unique BIGINT fits PostgreSQL signed integer; + # no central in-memory counter or fake process-local reuse. + msg = uuid4().int & ((1 << 62) - 1) + path = document_path(msg) + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "wb") as dest: + dest.write(body) + return ACCOUNT, CHAT, msg + + async def download(self, ref, dest: Path): + source = document_path(ref.message_id) + with source.open("rb") as original, dest.open("wb") as output: + shutil.copyfileobj(original, output) + + +async def read_range(engine, identifier: str, a: int, b: int) -> bytes: + return b"".join([piece async for piece in engine.iter_bytes( + identifier, start=a, end=b, block_size=1021, + )]) + + +def health(host: str, *, authorized=True): + credential = base64.b64encode( + ("dev-operator:" + os.environ["TELEDRIVE_FAULT_WEB_PASSWORD"]).encode() + ).decode("ascii") + headers = {"Authorization": "Basic " + credential} if authorized else {} + req = urllib.request.Request( + "http://" + host + ":8080/api/v1/admin/storage/health", headers=headers, + ) + try: + with urllib.request.urlopen(req, timeout=15) as response: + payload = json.load(response) + assert response.status == 200 + return payload + except urllib.error.HTTPError as exc: + if not authorized and exc.code in (401, 403): + return None + raise + + +def check_two_web_nodes(): + assert health("web-a", authorized=False) is None + assert health("web-b", authorized=False) is None + one, two = health("web-a"), health("web-b") + for name in ("physical_deletion_enabled",): + assert one[name] is False and two[name] is False + # Both independent Uvicorn processes must share SQL flood-wait state. + assert one["mtproto_explicit_flood_total"] == two["mtproto_explicit_flood_total"] + return one + + +def write_receipt(stage: str, details: dict): + EVIDENCE.mkdir(parents=True, exist_ok=True) + record = { + "environment": "disposable-docker-development", + "stage": stage, + "verified": True, + "production_acceptance_signed_off": False, + "real_telegram_tested": False, + "physical_telegram_deletions": 0, + **details, + } + target = EVIDENCE / (stage + ".json") + with target.open("x", encoding="utf-8") as output: + json.dump(record, output, indent=2, sort_keys=True) + output.write("\n") + print("PASS", stage, json.dumps(details, sort_keys=True)) + + +def seed(): + for folder in (SOURCE, REMOTE, EVIDENCE): + folder.mkdir(parents=True, exist_ok=True) + assert not (EVIDENCE / "seed.json").exists(), "Disposable workspace must be empty" + db = connect_database(DB_URL) + remote = DiskTelegram() + now = int(time.time()) + manifest = ManifestStore(db) + catalog = BucketCatalog(manifest) + try: + catalog.create_bucket(BUCKET, now) + saved = [] + for i, size in enumerate([0, 1, 4097, 65537, 5 * 1024 * 1024 + 7]): + # Fully synthetic. Covers 0 B, 1 B, short, chunk boundary and 5MiB+. + content = bytes([i + 37]) * size + path = SOURCE / ("seed-" + str(i) + ".bin") + path.write_bytes(content) + item = asyncio.run(TelegramObjectStorage( + manifest, remote, chunk_size=128 * 1024, scratch_directory=SOURCE, + ).put_file(BUCKET, "objects/" + path.name, path)) + catalog.publish(item.id, now=now) + assert item.sha256 == sha(content) + parts = manifest.parts(item.id) + assert sum(p.size for p in parts) == size + # Delete source to prove restored payload comes from stored chunks. + path.unlink() + out = SOURCE / ("readback-" + str(i)) + asyncio.run(TelegramObjectStorage( + manifest, remote, chunk_size=128 * 1024, scratch_directory=SOURCE, + ).download_to_path(item.id, out)) + assert out.read_bytes() == content + out.unlink() + saved.append({"id": item.id, "key": item.key, + "size": size, "sha256": item.sha256, "parts": len(parts)}) + + # A remote send DID happen, but SQL receipt/chunk append never happened: + # simulated process loss immediately after Telegram accepted bytes. + missing = manifest.begin(BUCKET, "crash/unknown-ack.bin", 9, now - 10 * 86400) + manifest.fail(missing, "simulated worker crash before ACK journal", now - 10 * 86400) + journal = UserSendJournal(db) + payload = b"orphaned" + journal.start(ACCOUNT, CHAT, missing, 0, sha(payload), len(payload)) + orphan_source = SOURCE / "unindexed.bin" + orphan_source.write_bytes(payload) + asyncio.run(remote.upload(orphan_source, object_id=missing, ordinal=0, + checksum=sha(payload))) + orphan_source.unlink() + assert journal.unresolved_count() == 1 + try: + journal.start(ACCOUNT, CHAT, missing, 0, sha(payload), len(payload)) + except RuntimeError: + pass + else: + raise AssertionError("Unsafe duplicate send accepted") + candidate = next(x for x in StorageGCInventory(manifest).preview_failed_retention( + now=now, + ) if x["object_id"] == missing) + assert candidate["reference_blocked"] is True + assert candidate["approved_for_deletion"] is False + # Synthetic flood-wait: a real sender ALWAYS reserves its shared + # account/channel SQL slot before it can receive a definitive + # flood-wait. The fault fixture must preserve this ordering, rather + # than weakening the production fail-closed rate-limit contract. + guard = MtprotoRateGuard(db) + assert guard.reserve_send_slot(ACCOUNT, CHAT, 0, now_ms=1000) == 0 + guard.impose_shared_cooldown(ACCOUNT, CHAT, 7, now_ms=1000) + assert guard.shared_cooldown_wait(ACCOUNT, CHAT, now_ms=1000) == 7 + db.close() + reopened = connect_database(DB_URL) + try: + again = UserSendJournal(reopened) + rate = MtprotoRateGuard(reopened) + assert again.unresolved_count() == 1 + assert rate.total_explicit_flood_observations() == 1 + assert rate.shared_cooldown_wait(ACCOUNT, CHAT, now_ms=1000) == 7 + assert rate.reserve_send_slot(ACCOUNT, CHAT, 0, now_ms=1000) == 7 + finally: + reopened.close() + nodes = check_two_web_nodes() + assert nodes["unresolved_mtproto_send_count"] == 1 + assert nodes["mtproto_explicit_flood_total"] == 1 + assert nodes["physical_deletion_enabled"] is False + (EVIDENCE / "objects.json").write_text(json.dumps(saved), encoding="utf-8") + write_receipt("seed", { + "objects": len(saved), "bytes": sum(x["size"] for x in saved), + "simulated_unknown_ack_blocked": True, + "synthetic_429_observations": 1, + "web_nodes": 2, "corruption_scope": "test-only-docker-scratch", + }) + finally: + db.close() + + +def verify_process_restart(): + """Real Web SIGKILL + PostgreSQL service restart; SQL/byte history persists. + + Called only by the disposable Compose runner after it kills web-a, + restarts postgres and brings both Web nodes back to healthy. + """ + saved = json.loads((EVIDENCE / "objects.json").read_text(encoding="utf-8")) + nodes = check_two_web_nodes() + assert nodes["unresolved_mtproto_send_count"] == 1 + assert nodes["mtproto_explicit_flood_total"] == 1 + assert nodes["unjournaled_mtproto_receipt_count"] == 0 + remote = DiskTelegram() + before_media = len(list(REMOTE.glob("*.bin"))) + assert before_media == sum(x["parts"] for x in saved) + 1 + db = connect_database(DB_URL) + try: + manifest = ManifestStore(db) + catalog = BucketCatalog(manifest) + idx = UserSendJournal(db) + rate = MtprotoRateGuard(db) + assert idx.unresolved_count() == 1 + assert rate.total_explicit_flood_observations() == 1 + assert rate.shared_cooldown_wait(ACCOUNT, CHAT, now_ms=1000) == 7 + for original in saved: + head = catalog.get(BUCKET, original["key"]) + assert head.item.id == original["id"] + assert head.item.sha256 == original["sha256"] + assert head.item.size == original["size"] + # Read on a fresh SQL connection after the real container kill + # and PostgreSQL restart, without accessing any local source. + chosen = [saved[0], saved[-1]] + engine = TelegramObjectStorage( + manifest, remote, chunk_size=128 * 1024, + scratch_directory=SOURCE, + ) + for original in chosen: + out = SOURCE / "post-sigkill-readback" + asyncio.run(engine.download_to_path(original["id"], out)) + assert sha(out.read_bytes()) == original["sha256"] + assert out.stat().st_size == original["size"] + out.unlink() + assert before_media == len(list(REMOTE.glob("*.bin"))) + assert not any(SOURCE.glob("seed-*.bin")) + write_receipt("restarted", { + "actual_web_container_sigkill_recovered": True, + "postgresql_service_restart_recovered": True, + "web_nodes_reconnected_to_shared_sql": 2, + "manifest_head_identities_preserved": len(saved), + "post_restart_telegram_sha_readbacks": len(chosen), + "lost_ack_retry_remains_fenced": True, + "synthetic_429_count_preserved": True, + "remote_media_count_unchanged": True, + }) + finally: + db.close() + + +def verify_restored(): + saved = json.loads((EVIDENCE / "objects.json").read_text(encoding="utf-8")) + remote = DiskTelegram() + db = connect_database(RESTORED_URL) + try: + manifest = ManifestStore(db) + catalog = BucketCatalog(manifest) + engine = TelegramObjectStorage(manifest, remote, chunk_size=128 * 1024, + scratch_directory=SOURCE) + for item in saved: + head = catalog.get(BUCKET, item["key"]) + assert head.item.id == item["id"] + assert head.item.sha256 == item["sha256"] + assert len(manifest.parts(item["id"])) == item["parts"] + dest = SOURCE / "isolated-restore" + asyncio.run(engine.download_to_path(item["id"], dest)) + assert dest.stat().st_size == item["size"] + assert sha(dest.read_bytes()) == item["sha256"] + if item["size"]: + start = max(0, item["size"] // 2 - 73) + end = min(item["size"], start + 145) + with dest.open("rb") as stream: + stream.seek(start) + assert asyncio.run(read_range(engine, item["id"], start, end)) == ( + stream.read(end - start) + ) + dest.unlink() + idx = UserSendJournal(db) + rate = MtprotoRateGuard(db) + assert idx.unresolved_count() == 1 + assert rate.total_explicit_flood_observations() == 1 + # Deliberately corrupt an externally stored chunk and verify that + # the production download code refuses to publish partial bytes. + candidate = next(x for x in saved if x["parts"] >= 2) + chunk = manifest.parts(candidate["id"])[0] + media = document_path(chunk.message_id) + original = media.read_bytes() + output = SOURCE / "corrupted-download" + try: + media.write_bytes(b"CORRUPTED" + original[9:]) + try: + asyncio.run(engine.download_to_path(candidate["id"], output)) + except (IntegrityError, ValueError, RuntimeError): + pass + else: + raise AssertionError("Remote-byte corruption escaped checksum check") + assert not output.exists(), "Corrupt download published partial data" + finally: + media.write_bytes(original) + output.unlink(missing_ok=True) + # Destroy a test-only media reference temporarily. A missing + # Telegram message must never replace an already-good destination. + missing = SOURCE / "temporarily-missing-remote-media" + previous = SOURCE / "missing-download" + previous.write_bytes(b"PREVIOUS-GOOD-DATA") + media.replace(missing) + try: + try: + asyncio.run(engine.download_to_path(candidate["id"], previous)) + except FileNotFoundError: + pass + else: + raise AssertionError("Missing remote Telegram message was accepted") + assert previous.read_bytes() == b"PREVIOUS-GOOD-DATA" + finally: + missing.replace(media) + previous.unlink() + + class InterruptedDownload(DiskTelegram): + def __init__(self): + self.calls = 0 + + async def download(self, ref, dest): + self.calls += 1 + if self.calls == 2: + raise TimeoutError("injected transport disconnect") + await super().download(ref, dest) + + interrupted = InterruptedDownload() + interrupted_engine = TelegramObjectStorage( + manifest, interrupted, chunk_size=128 * 1024, + scratch_directory=SOURCE, + ) + previous = SOURCE / "interrupted-download" + previous.write_bytes(b"PREVIOUS-GOOD-DATA") + try: + try: + asyncio.run(interrupted_engine.download_to_path( + candidate["id"], previous, + )) + except TimeoutError: + pass + else: + raise AssertionError("Broken Telegram stream published bytes") + assert interrupted.calls == 2 + assert previous.read_bytes() == b"PREVIOUS-GOOD-DATA" + finally: + previous.unlink() + + good = SOURCE / "recovered-after-fault" + asyncio.run(engine.download_to_path(candidate["id"], good)) + assert sha(good.read_bytes()) == candidate["sha256"] + good.unlink() + # A restored but incomplete manifest cannot authorize a deletion. + assert all(not x["approved_for_deletion"] for x in + StorageGCInventory(manifest).preview_failed_retention( + now=int(time.time()))) + write_receipt("restored", { + "native_postgresql_restore": True, + "full_sha_and_range_verified": len(saved), + "injected_remote_corruption_rejected": True, + "missing_remote_media_fails_closed": True, + "interrupted_remote_download_preserves_destination": True, + "original_media_restored_after_fault": True, + "orphan_send_fence_survived_restore": True, + "synthetic_429_counter_survived_restore": True, + "physical_deletion_enabled": False, + }) + finally: + db.close() + + +def development_acceptance(): + """Assert the complete disposable P1 DEV gate, never production signoff.""" + required = ("seed", "restarted", "restored") + receipts = {} + for stage in required: + receipt = json.loads( + (EVIDENCE / (stage + ".json")).read_text(encoding="utf-8") + ) + assert receipt["verified"] is True + assert receipt["environment"] == "disposable-docker-development" + assert receipt["real_telegram_tested"] is False + assert receipt["production_acceptance_signed_off"] is False + receipts[stage] = receipt + + seed_receipt = receipts["seed"] + restart_receipt = receipts["restarted"] + restored_receipt = receipts["restored"] + assert seed_receipt["objects"] == 5 + assert seed_receipt["simulated_unknown_ack_blocked"] is True + assert seed_receipt["synthetic_429_observations"] == 1 + assert restart_receipt["actual_web_container_sigkill_recovered"] is True + assert restart_receipt["postgresql_service_restart_recovered"] is True + assert restart_receipt["remote_media_count_unchanged"] is True + assert restored_receipt["native_postgresql_restore"] is True + assert restored_receipt["full_sha_and_range_verified"] == 5 + assert restored_receipt["injected_remote_corruption_rejected"] is True + assert restored_receipt["missing_remote_media_fails_closed"] is True + assert restored_receipt["interrupted_remote_download_preserves_destination"] is True + assert restored_receipt["orphan_send_fence_survived_restore"] is True + assert restored_receipt["synthetic_429_counter_survived_restore"] is True + assert restored_receipt["physical_deletion_enabled"] is False + assert check_two_web_nodes()["physical_deletion_enabled"] is False + checklist = { + "DEV-01": "clean_postgresql_two_real_web_nodes", + "DEV-02": "mixed_size_upload_and_persistent_manifest", + "DEV-03": "source_deleted_full_sha_and_range", + "DEV-04": "lost_ack_fenced_no_double_send", + "DEV-05": "unreferenced_media_gc_fails_closed", + "DEV-06": "synthetic_429_shared_sql_durability", + "DEV-07": "native_postgresql_isolated_restore", + "DEV-08": "tampered_native_backup_rejected", + "DEV-09": "corrupt_media_rejected_and_recovered", + "DEV-10": "unique_compose_cleanup_external", + "DEV-11": "real_web_sigkill_and_postgres_restart", + "DEV-12": "missing_media_and_broken_download_fails_closed", + "DEV-13": "combined_machine_checked_dev_acceptance", + } + write_receipt("acceptance", { + "checks_passed": sorted(checklist), + "checks_count": len(checklist), + "all_checks_verified": True, + "real_telegram_tested": False, + "production_acceptance_signed_off": False, + "legacy_cutover_required": False, + "telegram_remote_deletions": 0, + }) + + +def main(): + if len(sys.argv) != 2 or sys.argv[1] not in ( + "seed", "restarted", "restored", "acceptance", + ): + raise SystemExit( + "Usage: docker_fault_scenarios.py seed|restarted|restored|acceptance" + ) + { + "seed": seed, + "restarted": verify_process_restart, + "restored": verify_restored, + "acceptance": development_acceptance, + }[sys.argv[1]]() + + +if __name__ == "__main__": + main() diff --git a/scripts/dev/run_docker_fault_injection.sh b/scripts/dev/run_docker_fault_injection.sh new file mode 100644 index 00000000..30d3ef25 --- /dev/null +++ b/scripts/dev/run_docker_fault_injection.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +umask 077 +REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)" +cd "$REPO" +command -v docker >/dev/null +docker compose version >/dev/null +command -v python3 >/dev/null +if [[ -n "${TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE:-}" || -n "${TELEDRIVE_STORAGE_MTPROTO_API_HASH:-}" ]]; then + echo "Üretim MTProto session içeren shell'de DEV fault injection çalıştırma." >&2; exit 2 +fi +ID="$(python3 -c 'import secrets;print(secrets.token_hex(6))')" +PROJECT="tdfault$ID" +WORKSPACE="$(mktemp -d "${TMPDIR:-/tmp}/teledrive-fault-disposable.XXXXXXXX")" +export TELEDRIVE_FAULT_WORKSPACE="$WORKSPACE" TELEDRIVE_FAULT_UID="$(id -u)" TELEDRIVE_FAULT_GID="$(id -g)" +export TELEDRIVE_FAULT_DB_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))')" +export TELEDRIVE_FAULT_WEB_PASSWORD="$(python3 -c 'import secrets;print(secrets.token_urlsafe(32))')" +COMPOSE=(docker compose -f "$REPO/.github/ci/fault-injection/compose.yml" -p "$PROJECT") +REPORT_DIR="${TELEDRIVE_FAULT_REPORT_DIR:-$REPO/.docker-fault-evidence}" +if [[ -L "$REPORT_DIR" ]]; then echo "Symlink rapor dizini reddedildi" >&2; exit 2; fi +mkdir -p "$REPORT_DIR"; chmod 700 "$REPORT_DIR" +REPORT="$REPORT_DIR/docker-fault-$ID.json" +cleanup() { + rc=$? + trap - EXIT + if [[ "$rc" -ne 0 ]]; then + echo "DOCKER FAULT hata tanısı: yalnızca rastgele izole container logları:" >&2 + "${COMPOSE[@]}" logs --no-color --tail=35 web-a web-b postgres scenarios >&2 || : + fi + local docker_removed="false" + if "${COMPOSE[@]}" down -v --remove-orphans >/dev/null 2>&1; then + docker_removed="true" + else + rc=1 + echo "İzole Compose projesi tam temizlenemedi; manuel kontrol gerekli." >&2 + fi + local report_rc=0 + python3 - "$WORKSPACE/scenario/evidence" "$REPORT" "$rc" "$docker_removed" <<'PY' || report_rc=$? +import json, os, pathlib, sys +evidence, target, rc = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]), int(sys.argv[3]) +docker_removed = sys.argv[4] == "true" +receipts = {} +required = {"seed", "restarted", "restored", "acceptance"} +for stage in sorted(required): + p = evidence / (stage + ".json") + if p.is_file(): + receipts[stage] = json.loads(p.read_text(encoding="utf-8")) +complete = ( + rc == 0 + and docker_removed + and set(receipts) == required + and all(r.get("verified") is True for r in receipts.values()) + and receipts["acceptance"].get("all_checks_verified") is True + and receipts["acceptance"].get("checks_count") == 13 +) +payload = {"mode": "disposable-docker-development", + "result": "PASS" if complete else "FAIL", + "stages": receipts, "production_acceptance_signed_off": False, + "real_telegram_tested": False, "legacy_cutover_required": False, + "docker_project_removed": docker_removed, "production_data_used": False} +fd = os.open(target, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) +with os.fdopen(fd, "w") as output: + json.dump(payload, output, indent=2, sort_keys=True) + output.write("\n") +print("DOCKER FAULT:", payload["result"], "| rapor:", target) +sys.exit(0 if complete else 1) +PY + if [[ "$docker_removed" == "true" && "$WORKSPACE" == "${TMPDIR:-/tmp}"/teledrive-fault-disposable.* ]]; then + rm -rf -- "$WORKSPACE" + else + echo "İzole test workspace saklandı: Docker teardown doğrulanmadı." >&2 + fi + if [[ "$report_rc" -ne 0 ]]; then + rc=1 + fi + exit "$rc" +} +trap cleanup EXIT +trap 'exit 130' INT +trap 'exit 143' TERM +mkdir -p "$WORKSPACE"/{config,web-a,web-b,uploads,scenario/{evidence,remote,sources}} +chmod 700 "$WORKSPACE"/{config,web-a,web-b,uploads,scenario,scenario/evidence,scenario/remote,scenario/sources} +python3 - "$WORKSPACE/config/config.yaml" <<'PY' +from pathlib import Path +import os,sys +from urllib.parse import quote +p=quote(os.environ["TELEDRIVE_FAULT_DB_PASSWORD"],safe="") +w=os.environ["TELEDRIVE_FAULT_WEB_PASSWORD"] +with Path(sys.argv[1]).open("x", encoding="utf-8") as out: + out.write('web:\n user: "dev-operator"\n' + ' password: "%s"\n' + ' allowed_hosts: "web-a,web-b,localhost,127.0.0.1,[::1]"\n' + ' allow_insecure: false\n' + 'storage:\n channel: ""\n strategy: "round_robin"\n' + ' scratch_dir: "/data/tmp"\n apis: []\n' + 'jobs:\n source_root: "/uploads"\n auto_run: false\n' + ' legacy_auto_bucket: ""\n' + 'telegram:\n strategy: single\n api_id: 12345\n' + ' api_hash: "DISPOSABLE_OFFLINE_NO_SESSION"\n' + ' target: "me"\n accounts: []\n' + 'upload:\n source_dir: "/uploads"\n' + ' allowed_extensions: []\n max_file_size_mb: 0\n' + ' sleep_min_seconds: 60\n sleep_max_seconds: 90\n' + ' max_files_per_run: 12\n max_files_per_day: 100\n' + ' retry_attempts: 5\n backoff_base_seconds: 10\n' + ' floodwait_buffer_seconds: 5\n' + ' caption_template: "{name}"\n send_mode: "document"\n' + 'state:\n database_url: "postgresql+psycopg://teledrive:%s@postgres:5432/teledrive_dev"\n' + ' allowed_sqlite_roots: "/data"\n' + 'logging:\n log_path: "/data/uploader.log"\n level: "WARNING"\n' + 'app:\n auto_run: false\n run_interval_minutes: 60\n' % (w, p)) +PY +echo "DOCKER FAULT: PostgreSQL 17 + iki sıfır TeleDrive Web container kuruluyor." +"${COMPOSE[@]}" config -q +"${COMPOSE[@]}" up --build --wait -d postgres web-a web-b +# A Uvicorn liveness response alone does not exercise config parsing. +# Validate the actual production loader on BOTH running containers before +# the expensive SQL + Telegram-byte fault scenario is seeded. +for node in web-a web-b; do + "${COMPOSE[@]}" exec -T "$node" python -c 'from teledrive.uploader import load_config; c = load_config("/config/config.yaml"); assert c.db_path.startswith("postgresql+psycopg://")' +done +echo "DOCKER FAULT: 5MiB+ karışık boyut, sentetik 429/kayıp ACK, gerçek SQL/HTTP." +"${COMPOSE[@]}" run -T --rm scenarios seed + +# Kill a real Web process and restart the shared PostgreSQL container. +# No production service or global Docker volume is touched. +echo "DOCKER FAULT: web-a SIGKILL + ortak PostgreSQL restart uygulanıyor." +"${COMPOSE[@]}" kill -s SIGKILL web-a +"${COMPOSE[@]}" restart postgres +"${COMPOSE[@]}" up -d --wait postgres web-a web-b +"${COMPOSE[@]}" run -T --rm scenarios restarted + +BACKUP="$WORKSPACE/scenario/evidence/native-postgres.dump" +"${COMPOSE[@]}" exec -T postgres pg_dump -U teledrive -d teledrive_dev -Fc > "$BACKUP" +test -s "$BACKUP" +BAD="$WORKSPACE/scenario/evidence/tampered-postgres.dump" +cp "$BACKUP" "$BAD" +python3 - "$BAD" <<'PY' +from pathlib import Path +import sys +with Path(sys.argv[1]).open("r+b") as f: f.write(b"FAIL!") +PY +if "${COMPOSE[@]}" exec -T postgres pg_restore --list < "$BAD" >/dev/null 2>&1; then + echo "Bozuk native SQL yedeği reddedilmedi" >&2; exit 1 +fi +"${COMPOSE[@]}" exec -T postgres createdb -U teledrive teledrive_restore +"${COMPOSE[@]}" exec -T postgres pg_restore -U teledrive -d teledrive_restore --exit-on-error < "$BACKUP" +echo "DOCKER FAULT: bağımsız restore, SHA/Range ve medya bozulma enjeksiyonu." +"${COMPOSE[@]}" run -T --rm scenarios restored +echo "DOCKER FAULT: DEV-01..13 acceptance doğrulanıyor." +"${COMPOSE[@]}" run -T --rm scenarios acceptance +echo "DOCKER FAULT: PASS; yalnızca izole Compose projesi temizleniyor." diff --git a/scripts/dev/setup_telegram_e2e.py b/scripts/dev/setup_telegram_e2e.py new file mode 100644 index 00000000..0e9290ed --- /dev/null +++ b/scripts/dev/setup_telegram_e2e.py @@ -0,0 +1,248 @@ +#!/usr/bin/env python3 +"""Secure, one-time bootstrap of TeleDrive's real Telegram E2E test secrets. + +Prerequisites: + python -m pip install "Telethon>=1.36,<2" + gh auth login + python scripts/setup_telegram_e2e.py --channel "TeleDrive E2E Storage" --create-if-missing + +Telethon asks for a login code on first sign-in, and for 2FA only if enabled. +The StringSession and test channel ID are never printed or written to disk. +Use --secure-user-target to move the existing unmasked Actions variable +into a repository secret without disclosing the value or logging in to Telegram. +There is no Bot API configuration; the live E2E transport is Telethon USER only. +""" + +from __future__ import annotations + +import argparse +import asyncio +import getpass +import json +import shutil +import subprocess +import sys + +from typing import TYPE_CHECKING + +# Secret migration must remain usable on a bare operator Python install. +# Telegram SDK imports are needed only for actual Telegram login/setup. +if TYPE_CHECKING: + from telethon import TelegramClient + + +def require_github_cli(repo: str) -> None: + if not shutil.which("gh"): + raise RuntimeError( + "GitHub CLI (gh) is not installed. Install it from https://cli.github.com/" + ) + checks = ( + (["gh", "auth", "status"], "Sign in with 'gh auth login' first."), + (["gh", "repo", "view", repo, "--json", "nameWithOwner"], + f"GitHub repository {repo!r} is unavailable to this GitHub login."), + ) + for command, message in checks: + result = subprocess.run(command, capture_output=True, check=False) + if result.returncode: + raise RuntimeError(message) + + +def store_secret(repo: str, name: str, value: str) -> None: + """Pass the secret on stdin, never via command-line arguments or files.""" + subprocess.run( + ["gh", "secret", "set", name, "--repo", repo], + input=value, + text=True, + check=True, + stdout=subprocess.DEVNULL, + ) + + +def legacy_user_target_variable_names(repo: str) -> set[str]: + """Read only Actions variable names; never expose their stored values.""" + # gh variable list supports names-only JSON, but has no --limit option. + command = ["gh", "variable", "list", "--repo", repo, "--json", "name"] + listed = subprocess.run(command, capture_output=True, check=False) + if listed.returncode: + raise RuntimeError("Cannot inspect legacy Actions variable names") + try: + rows = json.loads(listed.stdout) + if not isinstance(rows, list) or any( + not isinstance(entry, dict) or not isinstance(entry.get("name"), str) + for entry in rows + ): + raise ValueError("Malformed variable list") + except (ValueError, TypeError) as exc: + raise RuntimeError("Cannot verify legacy Actions variable list") from exc + return {entry["name"] for entry in rows} + + +def remove_public_user_target_variable(repo: str) -> None: + """Delete the unmasked legacy variable and re-list to verify absence.""" + name = "TELEDRIVE_E2E_TARGET" + if name in legacy_user_target_variable_names(repo): + command = ["gh", "variable", "delete", name, "--repo", repo] + removed = subprocess.run(command, capture_output=True, check=False) + if removed.returncode: + raise RuntimeError( + "Legacy unmasked Actions variable could not be deleted" + ) + # A successful CLI exit does not prove absence (a racing write can + # re-create the variable). Fail closed rather than making that claim. + if name in legacy_user_target_variable_names(repo): + raise RuntimeError("Legacy unmasked Actions variable is still present") + + +def secure_existing_user_target(repo: str) -> None: + """One-time non-echoing migration from Actions variable to Secret. + + Read the existing variable only into Python's captured subprocess + stdout, never the operator terminal. If the variable is already gone, + prompt for the peer ID using getpass rather than a CLI argument. + """ + import re + + require_github_cli(repo) + fetched = subprocess.run( + ["gh", "variable", "get", "TELEDRIVE_E2E_TARGET", "--repo", repo], + capture_output=True, + check=False, + ) + if fetched.returncode == 0: + try: + chat_id = fetched.stdout.decode("utf-8").strip() + except UnicodeDecodeError as exc: + raise ValueError("Invalid stored private channel ID") from exc + else: + # A failed 'get' can mean insufficient permissions or network errors, + # not just an absent variable. A names-only query must confirm absence. + if "TELEDRIVE_E2E_TARGET" in legacy_user_target_variable_names(repo): + raise RuntimeError( + "Legacy Actions variable exists but cannot be securely read" + ) + chat_id = getpass.getpass( + "Telegram test channel ID (hidden, old variable unavailable): " + ).strip() + if not re.fullmatch(r"-100[0-9]{5,16}", chat_id): + raise ValueError("User target must be a numeric private channel ID") + store_secret(repo, "TELEDRIVE_E2E_TARGET", chat_id) + remove_public_user_target_variable(repo) + print("Saved GitHub Actions secret: TELEDRIVE_E2E_TARGET") + print("Verified that the previous unmasked variable is absent.") + + +async def find_channel(client: TelegramClient, title: str) -> list: + matches = [] + async for dialog in client.iter_dialogs(): + if (dialog.is_channel and not dialog.is_group + and not getattr(dialog.entity, "username", None) + and dialog.name.casefold() == title.casefold()): + matches.append(dialog) + return matches + + +async def setup(repo: str, title: str, create_if_missing: bool, + configure_bot: bool = False) -> None: + require_github_cli(repo) + from telethon import TelegramClient + from telethon.sessions import StringSession + from telethon.tl.functions.channels import CreateChannelRequest + + api_id = int(input("Telegram test account API ID: ").strip()) + api_hash = getpass.getpass("Telegram test account API hash: ").strip() + phone = input("Telegram TEST account phone (+country code): ").strip() + if not api_hash or not phone: + raise ValueError("API hash and test account phone must not be empty.") + + client = TelegramClient(StringSession(), api_id, api_hash) + try: + await client.start( + phone=phone, + code_callback=lambda: input("Telegram login code: ").strip(), + password=lambda: getpass.getpass("Telegram 2FA password (only if enabled): "), + ) + + user = await client.get_me() + if not user or user.bot: + raise RuntimeError("A dedicated Telegram USER account is required.") + + matches = await find_channel(client, title) + if not matches and create_if_missing: + print(f"Creating a private broadcast test channel named {title!r}...") + await client(CreateChannelRequest( + title=title, + about="Private TeleDrive automated storage integration tests", + broadcast=True, + megagroup=False, + )) + matches = await find_channel(client, title) + + if len(matches) != 1: + if not matches: + raise RuntimeError( + f"No private channel named {title!r} is visible to this test account. " + "Create the channel first, or rerun with --create-if-missing." + ) + raise RuntimeError( + f"{len(matches)} channels are named {title!r}; use a unique channel title." + ) + + if not matches[0].entity.broadcast: + raise RuntimeError("The target is not a broadcast channel.") + + session = client.session.save() + if not session: + raise RuntimeError("No authorized StringSession was created.") + + values = { + "TELEDRIVE_E2E_API_ID": str(api_id), + "TELEDRIVE_E2E_API_HASH": api_hash, + "TELEDRIVE_E2E_SESSION": session, + "TELEDRIVE_E2E_CHANNEL_NAME": title, + } + for secret_name, secret_value in values.items(): + store_secret(repo, secret_name, secret_value) + print(f"Saved GitHub Actions secret: {secret_name}") + + # The user session resolves the private channel ID during this + # one-time setup, so no manual ID lookup is needed. + channel_id = int(matches[0].id) + store_secret(repo, "TELEDRIVE_E2E_TARGET", str(channel_id)) + print("Saved GitHub Actions secret: TELEDRIVE_E2E_TARGET") + remove_public_user_target_variable(repo) + + print( + "\nE2E credentials configured. No session or API hash was displayed.\n" + "Run the manual Telegram Storage E2E workflow after its YAML file " + "is available on the repository default branch." + ) + finally: + await client.disconnect() + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", default="babico/TeleDrive") + parser.add_argument("--channel", default="TeleDrive E2E Storage", + help="Unique private Telegram broadcast channel title") + parser.add_argument("--create-if-missing", action="store_true", + help="Create the private broadcast channel on the test account") + parser.add_argument("--secure-user-target", action="store_true", + help="Move an existing test channel ID into a masked GitHub Secret") + args = parser.parse_args() + if not args.channel.strip(): + parser.error("--channel cannot be empty") + try: + if args.secure_user_target: + secure_existing_user_target(args.repo) + else: + asyncio.run(setup( + args.repo, args.channel.strip(), args.create_if_missing, + )) + except (ValueError, RuntimeError, subprocess.CalledProcessError) as error: + print(f"Setup failed: {error}", file=sys.stderr) + raise SystemExit(1) from None + + +if __name__ == "__main__": + main() diff --git a/scripts/ops/acceptance.py b/scripts/ops/acceptance.py new file mode 100644 index 00000000..92adf0c5 --- /dev/null +++ b/scripts/ops/acceptance.py @@ -0,0 +1,692 @@ +#!/usr/bin/env python3 +"""Interactive, fail-closed TeleDrive acceptance assistant. + +Run as a file with "python scripts/acceptance.py". Do not pipe the +source to "python -": that consumes stdin and makes input() raise EOFError. + +Produces a private, sanitized report. Never approves production, deletes +Telegram messages, changes routing, or applies unresolved MTProto receipts. +""" +from __future__ import annotations + +import argparse +import asyncio +import datetime as dt +import getpass +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import tempfile +from types import SimpleNamespace + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +REPO = "babico/TeleDrive" +WORKFLOWS = { + "user": "04 · Live Telegram | User MTProto E2E", + "multiworker": "04 · Live Telegram | Multiworker PostgreSQL E2E", +} + + +def ask(prompt: str, default: str = "") -> str: + suffix = (" [" + default + "]") if default else "" + answer = input(prompt + suffix + ": ").strip() + return answer or default + + +def secret(prompt: str) -> str: + return getpass.getpass(prompt + " (gizli): ").strip() + + +def yes(prompt: str, default: bool = False) -> bool: + value = ask(prompt + " (e/h)", "e" if default else "h").lower() + if value not in ("e", "evet", "h", "hayir", "hayır"): + raise ValueError("Yalnızca e/h kullanın") + return value in ("e", "evet") + + +def safe_run(args, *, env=None, stdin=None, timeout=1800): + # Captures outputs so DB URLs, Telegram ids and sensitive CLI diagnostics + # are never echoed or copied verbatim to the evidence report. + return subprocess.run( + args, cwd=ROOT, env=env, stdin=stdin, + capture_output=True, text=stdin is None, timeout=timeout, check=False, + ) + + +def file_digest(path: Path) -> str: + hash_ = hashlib.sha256() + with path.open("rb") as source: + while block := source.read(1024 * 1024): + hash_.update(block) + return hash_.hexdigest() + + +def output_file(directory: Path, prefix: str, suffix: str) -> Path: + from uuid import uuid4 + return directory / (prefix + "-" + dt.datetime.now(dt.timezone.utc).strftime( + "%Y%m%dT%H%M%SZ" + ) + "-" + uuid4().hex[:8] + suffix) + + +def report_status(result: dict, stage: str, state: str, reason: str): + assert state in ("PASS", "BLOCKED", "FAIL", "NOT_APPLICABLE") + result["stages"][stage] = {"status": state, "reason": reason} + print(" [" + state + "] " + stage + ": " + reason) + + +async def session_login(session_path: Path) -> bool: + from telethon import TelegramClient + from telethon.sessions import StringSession + print("\n[Telegram kullanıcı StringSession]") + api_id = int(ask("API ID (my.telegram.org/apps)")) + api_hash = secret("API Hash") + phone = ask("Kullanıcı telefon numarası (+90...)") + existing = session_path.read_text(encoding="utf-8").strip() if session_path.exists() else "" + client = TelegramClient(StringSession(existing), api_id, api_hash) + try: + await client.start( + phone=phone, + code_callback=lambda: ask("Telegram giriş kodu"), + password=lambda: secret("Telegram 2FA şifresi"), + ) + identity = await client.get_me() + if not identity or getattr(identity, "bot", False): + raise ValueError("Bot değil Telegram kullanıcı hesabı gerekli") + saved = client.session.save() + if not saved: + raise ValueError("Oturum kaydı oluşturulamadı") + if not existing: + session_path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + fd = os.open(session_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as destination: + destination.write(saved) + # Never print session, phone, Telegram ID, hash or auth codes. + print(" Kullanıcı girişi doğrulandı; session gizli dosyada saklanıyor.") + return True + finally: + await client.disconnect() + + +def github_stage(result: dict): + print("\n[1/6 GitHub E2E kanıt kontrolü]") + if shutil.which("gh") is None: + report_status(result, "github_e2e", "BLOCKED", "GitHub CLI yok") + return + output = safe_run([ + "gh", "run", "list", "--repo", REPO, "--limit", "100", + "--json", "databaseId,headSha,workflowName,status,conclusion,url", + ], timeout=45) + if output.returncode: + report_status(result, "github_e2e", "BLOCKED", "GitHub login/API listesi alınamadı") + return + try: + records = json.loads(output.stdout) + except (TypeError, ValueError): + report_status(result, "github_e2e", "FAIL", "Geçersiz GitHub JSON cevabı") + return + passed = {} + for key, name in WORKFLOWS.items(): + found = next((x for x in records if x.get("workflowName") == name + and x.get("conclusion") == "success"), None) + if found: + passed[key] = {"run_id": int(found["databaseId"]), + "sha": str(found["headSha"])[:12]} + result["github"] = passed + if len(passed) == len(WORKFLOWS): + report_status(result, "github_e2e", "PASS", "Üç ayrı başarılı E2E koşumu bulundu") + else: + missing = ", ".join(sorted(set(WORKFLOWS) - set(passed))) + report_status(result, "github_e2e", "BLOCKED", "Geçmiş başarılı koşum yok: " + missing) + + +def session_stage(result: dict, session_path: Path): + print("\n[2/6 Telegram kullanıcı oturumu]") + if session_path.exists(): + if session_path.is_symlink() or not session_path.is_file(): + raise ValueError("Session dosyası güvenli bir regular file değil") + if session_path.stat().st_mode & 0o077: + raise ValueError("Session dosyası izni fazla geniş; chmod 600 uygulayın") + print(" Kayıtlı session dosyası mevcut. İçeriği gösterilmeyecek.") + if not yes("Telegram kullanıcı oturumunu şimdi doğrula/oluştur?"): + report_status(result, "telegram_session", "NOT_APPLICABLE", + "Oturum kontrolü operatör tarafından atlandı") + return + try: + asyncio.run(session_login(session_path)) + report_status(result, "telegram_session", "PASS", + "Kullanıcı oturumu doğrulandı, gizli session ekrana yazılmadı") + except (EOFError, KeyboardInterrupt): + raise + except Exception as exc: + report_status(result, "telegram_session", "BLOCKED", + "Telegram giriş kontrolü başarısız: " + type(exc).__name__) + + +def soak_stage(result: dict, evidence: Path): + print("\n[3/6 Gerçek iki host / minimum bir saat / doğal 429]") + if not yes("İki ayrı fiziksel VM/host ve TLS health URL'leri hazır mı?"): + report_status(result, "multi_host_soak", "BLOCKED", + "İki ayrı canlı host yok; CI bu kabulü ikame edemez") + return + first = ask("Host A HTTPS origin (443, path yok)") + second = ask("Host B HTTPS origin (443, path yok)") + from scripts.ops.monitor_host_soak import parse_hosts + hosts = [{"label": "host-a", "url": first}, + {"label": "host-b", "url": second}] + try: + parsed = parse_hosts(json.dumps(hosts)) + except (ValueError, TypeError): + report_status(result, "multi_host_soak", "FAIL", "Host URL/doğrulama hatası") + return + username = ask("Her iki host için Basic Auth kullanıcı adı") + password = secret("Basic Auth parolası") + if not username or not password: + report_status(result, "multi_host_soak", "FAIL", "Yönetici kimlik doğrulaması eksik") + return + if not yes("Doğal iş yükü hazır mı? En az bir saat gözlem başlatılsın mı?"): + report_status(result, "multi_host_soak", "BLOCKED", "1 saatlik gözlem başlatılmadı") + return + env = dict(os.environ) + env.update({ + "TELEDRIVE_P1_HEALTH_HOSTS": json.dumps(hosts), + "TELEDRIVE_P1_BASIC_USERNAME": username, + "TELEDRIVE_P1_BASIC_PASSWORD": password, + }) + target = output_file(evidence, "p1-soak", ".json") + # Host names / passwords are never recorded in our master report. + run = safe_run([ + sys.executable, "scripts/monitor_host_soak.py", + "--duration-seconds", "3600", "--interval-seconds", "60", + "--output", str(target), + ], env=env, timeout=3900) + if target.exists(): + try: + content = json.loads(target.read_text(encoding="utf-8")) + result["soak"] = { + "sample_count": content.get("sample_count"), + "duration_seconds": content.get("duration_seconds"), + "telemetry_gate_passed": content.get("telemetry_gate_passed"), + "final_shared_counter_agreement": + content.get("final_shared_counter_agreement"), + "explicit_429_deltas_by_host": + content.get("explicit_429_deltas_by_host"), + } + except (OSError, ValueError): + pass + report_status(result, "multi_host_soak", + "PASS" if run.returncode == 0 else "BLOCKED", + "1 saatlik telemetry geçti; fiziksel host onayı ayrıca gerekli" + if run.returncode == 0 else + "429/backlog/TLS/host izleme kapısı tamamlanmadı") + if parsed: + print(" Host adresleri ve parolalar rapora yazılmadı.") + + +def ask_user_identity(): + print(" DR: ÖZGÜN Telethon USER session ve özgün mesajların kanalı gerekir.") + session_file = Path(secret("Üretim USER StringSession dosya yolu")).expanduser() + api_id = secret("Telegram API ID") + api_hash = secret("Telegram API hash") + account = ask("TeleDrive USER account alias (kullanıcı adı değil)") + chat_text = secret("Orijinal Telegram kanal ID") + if not re.fullmatch(r"-100[0-9]{5,16}", chat_text): + raise ValueError("Telegram kanalı -100... biçiminde olmalı") + chat_id = int(chat_text) + target = secret("Kanal ID veya username (target)") + bucket = ask("Mevcut S3 bucket") + key = ask("Mevcut S3 object key") + if not all((session_file, api_id, api_hash, account, chat_id, target, + bucket, key)): + raise ValueError("DR kimliği eksik") + return session_file, int(api_id), api_hash, account, chat_id, target, bucket, key + + +async def native_telegram_readback(db_url, *, session_file, api_id, api_hash, + target, account, chat_id, bucket, key, + scratch: Path): + """Read-only Telegram payload validation against restored native SQL.""" + from teledrive.db import connect_database + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.engine import TelegramObjectStorage + from teledrive.storage.manifest import ManifestStore + from scripts.ops.rehearse_telegram_dr import ( + ReadOnlyUserChannelTransport, open_read_only_user, + ) + db = connect_database(db_url) + try: + manifests = ManifestStore(db) + obj = BucketCatalog(manifests).get(bucket, key).item + if obj.size < 1 or obj.size > 64 * 1024 * 1024: + raise ValueError("DR örneği 1B-64MiB aralığında olmalı") + client, resolved = await open_read_only_user( + session_file, api_id, api_hash, target, chat_id, + ) + try: + transport = ReadOnlyUserChannelTransport( + client, resolved, account=account, chat_id=chat_id, + ) + storage = TelegramObjectStorage(manifests, transport, + scratch_directory=scratch) + destination = scratch / "restored-telegram-object" + await storage.download_to_path(obj.id, destination) + if destination.stat().st_size != obj.size or file_digest(destination) != obj.sha256: + raise ValueError("Telegram restore SHA/size uyuşmuyor") + start = max(0, obj.size // 2 - 17) + end = min(obj.size, start + 37) + seen = b"".join([part async for part in storage.iter_bytes( + obj.id, start=start, end=end, block_size=7, + )]) + with destination.open("rb") as handle: + handle.seek(start) + if handle.read(end - start) != seen: + raise ValueError("Telegram restore Range uyuşmuyor") + return {"verified": True, "size": obj.size, "sha256_match": True, + "range_match": True, "telegram_uploads": 0, + "telegram_deletions": 0} + finally: + await client.disconnect() + finally: + db.close() + + +def db_connection_parts(kind: str, name: str): + print(" " + name + " DB hedefi; parolayı terminale yazdırmıyoruz.") + host = ask(name + " hostname") + port = int(ask(name + " port", "5432" if kind == "postgresql" else "3306")) + dbname = ask(name + " DB adı") + username = ask(name + " kullanıcı") + password = secret(name + " DB parolası") + if not host or not dbname or not username or not password or not 1 <= port <= 65535: + raise ValueError("DB ayarları eksik") + if any("\n" in v or "\r" in v for v in (host, dbname, username, password)): + raise ValueError("DB ayarlarında satır sonu yasak") + return dict(host=host, port=port, dbname=dbname, user=username, + password=password) + + +def native_restore(kind: str, source: dict, target: dict, + backup: Path, workspace: Path) -> str: + if (source["host"].lower(), source["port"]) == ( + target["host"].lower(), target["port"] + ): + raise ValueError("Aynı DB sunucusuna geri yükleme yasak") + if source["dbname"] == target["dbname"] and source["host"].lower() == target["host"].lower(): + raise ValueError("Kaynak DB hedefi staging'den farklı olmalı") + if backup.exists(): + raise ValueError("Yedek dosyası zaten mevcut") + if kind == "postgresql": + for executable in ("pg_dump", "pg_restore", "psql"): + if not shutil.which(executable): + raise RuntimeError("Gerekli PostgreSQL aracı yok: " + executable) + def env_for(item): + env = dict(os.environ) + env.update({ + "PGHOST": item["host"], "PGPORT": str(item["port"]), + "PGUSER": item["user"], "PGDATABASE": item["dbname"], + "PGPASSWORD": item["password"], + }) + return env + source_env, target_env = env_for(source), env_for(target) + check = safe_run(["psql", "-At", "-c", + "SELECT count(*) FROM pg_tables WHERE schemaname NOT IN ('pg_catalog','information_schema')"], + env=target_env, timeout=30) + if check.returncode or check.stdout.strip() != "0": + raise RuntimeError("Staging PostgreSQL DB boş değil veya ulaşılamıyor") + command = ["pg_dump", "-Fc", "--no-owner", "--no-acl", "-f", str(backup)] + if safe_run(command, env=source_env, timeout=1800).returncode: + raise RuntimeError("pg_dump başarısız") + if safe_run(["pg_restore", "--no-owner", "--no-acl", "--exit-on-error", + "-d", target["dbname"], str(backup)], + env=target_env, timeout=1800).returncode: + raise RuntimeError("İzole pg_restore başarısız") + from sqlalchemy.engine import URL + return URL.create( + "postgresql+psycopg", username=target["user"], + password=target["password"], host=target["host"], + port=target["port"], database=target["dbname"], + ).render_as_string(hide_password=False) + for executable in ("mysqldump", "mysql"): + if not shutil.which(executable): + raise RuntimeError("Gerekli MySQL aracı yok: " + executable) + def option_file(item): + fd, name = tempfile.mkstemp(prefix="mysql-login-", suffix=".cnf", dir=workspace) + with os.fdopen(fd, "w", encoding="utf-8") as out: + out.write("[client]\n") + for key, val in ( + ("host", item["host"]), ("port", str(item["port"])), + ("user", item["user"]), ("password", item["password"]), + ): + if '"' in val or "\\" in val: + raise ValueError("MySQL kimlik bilgisi güvenli option-file biçiminde değil") + out.write(key + '="' + val + '"\n') + return Path(name) + original_opt = restored_opt = None + try: + original_opt = option_file(source) + restored_opt = option_file(target) + probe = safe_run([ + "mysql", "--defaults-extra-file=" + str(restored_opt), + "--batch", "--skip-column-names", "--database=" + target["dbname"], + "--execute=SHOW TABLES", + ], timeout=30) + if probe.returncode or probe.stdout.strip(): + raise RuntimeError("Staging MySQL DB boş değil veya ulaşılamıyor") + dumped = safe_run([ + "mysqldump", "--defaults-extra-file=" + str(original_opt), + "--single-transaction", "--routines", "--triggers", "--no-tablespaces", + "--result-file=" + str(backup), source["dbname"], + ], timeout=1800) + if dumped.returncode: + raise RuntimeError("mysqldump başarısız") + with backup.open("rb") as stream: + loaded = subprocess.run([ + "mysql", "--defaults-extra-file=" + str(restored_opt), + "--database=" + target["dbname"], + ], cwd=ROOT, stdin=stream, stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, timeout=1800, check=False) + if loaded.returncode: + raise RuntimeError("İzole MySQL restore başarısız") + from sqlalchemy.engine import URL + return URL.create( + "mysql+pymysql", username=target["user"], + password=target["password"], host=target["host"], + port=target["port"], database=target["dbname"], + ).render_as_string(hide_password=False) + finally: + for path in (original_opt, restored_opt): + if path is not None: + path.unlink(missing_ok=True) + + +def dr_stage(result: dict, evidence: Path): + print("\n[4/6 DR: özgün SQL yedeği + gerçek Telegram dosyası]") + if not yes("Bu hostta orijinal DB'ye yetkili backup erişimin var mı?"): + report_status(result, "disaster_recovery", "BLOCKED", "Üretim yedekleme yetkisi yok") + return + kind = ask("DB tipi (sqlite/postgresql/mysql)", "sqlite").lower() + if kind not in ("sqlite", "postgresql", "mysql"): + report_status(result, "disaster_recovery", "BLOCKED", "DB tipi desteklenmiyor") + return + backup = output_file(evidence, "p1-" + kind + "-backup", ".db" if kind == "sqlite" else ".dump") + try: + if kind == "sqlite": + source = Path(ask("Orijinal SQLite DB mutlak dosya yolu")).expanduser() + if not source.is_absolute() or not source.is_file(): + raise ValueError("Mevcut mutlak SQLite dosya yolu gerekli") + if source.is_symlink(): + raise ValueError("SQLite kaynak symlink olmamalı") + backup_result = safe_run([ + sys.executable, "scripts/backup_metadata.py", + "--database", str(source), "--output", str(backup), + ], timeout=300) + if backup_result.returncode or not backup.is_file(): + raise RuntimeError("Online SQLite backup başarısız") + checksum = file_digest(backup) + check = safe_run([ + sys.executable, "scripts/verify_metadata_backup.py", + "--database", str(backup), "--sha256", checksum, + ], timeout=120) + if check.returncode: + raise RuntimeError("SQLite backup bütünlüğü doğrulanamadı") + if not yes("Orijinal Telegram dosyasıyla full SHA ve Range testi yapılsın mı?"): + report_status(result, "disaster_recovery", "BLOCKED", + "SQL backup doğrulandı fakat gerçek Telegram readback atlandı") + return + (session_file, api_id, api_hash, account, chat_id, target, + bucket, key) = ask_user_identity() + from scripts.ops.rehearse_telegram_dr import verify_original_user_channel + result_obj = asyncio.run(verify_original_user_channel( + backup, checksum, bucket=bucket, key=key, account=account, + chat_id=chat_id, target=target, session_file=session_file, + api_id=api_id, api_hash=api_hash, + )) + if (result_obj.get("verified") is not True + or result_obj.get("telegram_uploads") != 0 + or result_obj.get("telegram_deletions") != 0): + raise ValueError("DR doğrulama sözleşmesi geçmedi") + else: + print(" İzole, boş staging DB aynı DB sunucusunda olmamalıdır.") + src = db_connection_parts(kind, "ÜRETİM") + dst = db_connection_parts(kind, "STAGING") + if not yes("Staging DB'nin boş ve tamamen disposable olduğunu ONAYLIYOR musun?"): + raise ValueError("Staging restore yetkisi verilmedi") + with tempfile.TemporaryDirectory(prefix="p1-restore-conf-") as workspace: + staging_db_url = native_restore( + kind, src, dst, backup, Path(workspace), + ) + if not yes("Staging DB üzerinden orijinal Telegram dosyası doğrulansın mı?"): + report_status(result, "disaster_recovery", "BLOCKED", + "Native SQL restore tamamlandı; Telegram okunmadı") + return + (session_file, api_id, api_hash, account, chat_id, target, + bucket, key) = ask_user_identity() + with tempfile.TemporaryDirectory(prefix="p1-telegram-readback-") as temp: + result_obj = asyncio.run(native_telegram_readback( + staging_db_url, session_file=session_file, + api_id=api_id, api_hash=api_hash, target=target, + account=account, chat_id=chat_id, bucket=bucket, + key=key, scratch=Path(temp), + )) + result["dr"] = {"db_type": kind, "backup_checksum_matches": True, + "original_telegram_bytes_verified": True, + "range_verified": True, + "telegram_uploads": 0, "telegram_deletions": 0} + report_status(result, "disaster_recovery", "PASS", + "İzole SQL backup/restore + gerçek Telegram SHA ve Range geçti") + except (KeyboardInterrupt, EOFError): + raise + except Exception as exc: + report_status(result, "disaster_recovery", "FAIL", + "Backup/restore/readback tamamlanamadı: " + type(exc).__name__) + + +def legacy_stage(result: dict): + print("\n[5/6 Legacy cutover salt-okunur SHA/HEAD kontrolü]") + if not yes("Eski uploader verisi yeni TeleDrive'a taşınıyor mu?"): + report_status(result, "legacy_cutover", "NOT_APPLICABLE", + "Gerçek legacy migration hedefi yok") + return + if not yes("Eski uploader KAPALI, dosyalar değişmiyor, tüm jobs done ve lease serbest mi?"): + report_status(result, "legacy_cutover", "BLOCKED", + "Eski işçi/source yazımı veya kuyruk hâlâ aktif") + return + try: + db_value = secret("Üretim/staging SQL URL veya SQLite yolu") + source = Path(ask("Legacy source-root (mutlak)")).expanduser() + allowed = Path(ask("İzin verilen üst kök (mutlak)")).expanduser() + scratch = Path(ask("Scratch dizini (mutlak)")).expanduser() + (session_file, api_id, api_hash, account, chat_id, target, + bucket, _) = ask_user_identity() + prefix = ask("Hedef key prefix", "archived") + max_files = int(ask("Maksimum dosya sayısı (1-1000)", "1000")) + max_bytes = int(ask("Tek dosya maksimum byte", "67108864")) + extensions = ask("Uygun uzantılar, virgülle; boş=tümü", "") + if not (source.is_absolute() and allowed.is_absolute() + and scratch.is_absolute() and scratch.is_dir()): + raise ValueError("Güvenli mutlak kaynak/scratch yolları gerekli") + from scripts.ops.verify_legacy_cutover import run + original_env = {} + for key, value in ( + ("TELEDRIVE_STORAGE_MTPROTO_API_ID", str(api_id)), + ("TELEDRIVE_STORAGE_MTPROTO_API_HASH", api_hash), + ("TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE", str(session_file)), + ): + original_env[key] = os.environ.get(key) + os.environ[key] = value + try: + args = SimpleNamespace( + database=db_value, source_root=source, + allowed_root=allowed, scratch_dir=scratch, bucket=bucket, + prefix=prefix, account=account, chat_id=chat_id, + target=target, max_files=max_files, max_bytes=max_bytes, + extensions=extensions, max_file_size_mb=0, + ) + report = asyncio.run(run(args)) + finally: + for key, value in original_env.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + result["legacy"] = { + field: report.get(field) for field in ( + "verified", "eligible_files", "verified_files", + "not_yet_published", "inconsistent", "excluded", "truncated", + "telegram_uploads", "telegram_deletions", + "source_modifications", + ) + } + report_status(result, "legacy_cutover", "PASS" if report["verified"] else "BLOCKED", + "Tam kaynak SHA ve Telegram HEAD doğrulaması geçti; read switch onayı AYRI" + if report["verified"] else "Cutover kaynağı eksik/uyumsuz") + except (KeyboardInterrupt, EOFError): + raise + except Exception as exc: + report_status(result, "legacy_cutover", "FAIL", + "Legacy ön kabul doğrulanamadı: " + type(exc).__name__) + + +def ack_stage(result: dict, session_path: Path): + print("\n[6/6 MTProto lost-send / bağımsız kanal kanıtı]") + if not yes("Sağlık API'sinde çözümlenmemiş MTProto gönderim intent'i var mı?"): + report_status(result, "mtproto_ack", "NOT_APPLICABLE", + "Operatör belirsiz MTProto gönderimi bulunmadığını beyan etti") + return + if (not session_path.is_file() or session_path.is_symlink() + or session_path.stat().st_mode & 0o077): + report_status(result, "mtproto_ack", "BLOCKED", + "Güvenli, mode 0600 Telegram kullanıcı session dosyası yok") + return + if not yes("Orijinal message_id VE tam SHA-256 kanıtı mevcut mu?"): + report_status(result, "mtproto_ack", "BLOCKED", + "Kanıt olmadan kör tekrar gönderim/ACK uygulanamaz") + return + try: + from scripts.ops.reconcile_mtproto_receipt import run + args = SimpleNamespace( + database=secret("Salt-okunur DR staging veritabanı yolu/URL"), + account=ask("Orijinal USER account alias"), + chat_id=int(secret("Orijinal özel kanal ID")), + target=secret("Kanal ID veya username (target)"), + object_id=ask("SQL pre-send intent object UUID"), + ordinal=int(ask("Parça ordinal", "0")), + message_id=int(secret("Orijinal Telegram message ID")), + sha256=secret("Orijinal parça SHA-256"), + size=int(secret("Orijinal parça byte boyutu")), + apply=False, + ) + print(" Sadece dry-run çalışır; --apply yok.") + original = {} + for key, value in ( + ("TELEDRIVE_RECONCILE_API_ID", ask("Telegram API ID")), + ("TELEDRIVE_RECONCILE_API_HASH", secret("Telegram API Hash")), + ("TELEDRIVE_RECONCILE_SESSION", + session_path.read_text(encoding="utf-8").strip()), + ): + original[key] = os.environ.get(key) + os.environ[key] = value + try: + verified = asyncio.run(run(args)) + finally: + for key, value in original.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + if not verified.get("verified") or verified.get("applied"): + raise ValueError("Bağımsız MTProto kanıt doğrulaması geçmedi") + result["ack"] = {"independent_channel_proof": True, + "applied": False, "telegram_uploads": 0, + "telegram_deletions": 0} + report_status(result, "mtproto_ack", "PASS", + "USER kanal + tam SHA kanıtı doğrulandı; apply yapılmadı") + except (EOFError, KeyboardInterrupt): + raise + except Exception as exc: + report_status(result, "mtproto_ack", "FAIL", + "Özgün MTProto kanıtı doğrulanamadı: " + type(exc).__name__) + + +def save_report(folder: Path, result: dict) -> Path: + target = output_file(folder, "p1-acceptance", ".json") + result["production_acceptance_signed_off"] = False + result["operator_review_required"] = True + result["physical_telegram_deletions"] = 0 + fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as file: + json.dump(result, file, sort_keys=True, indent=2, ensure_ascii=False) + file.write("\n") + return target + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--evidence-dir", type=Path, + default=Path.home() / "teledrive-p1-evidence") + options = parser.parse_args() + if not sys.stdin.isatty(): + parser.error("Interaktif terminal gerekli. 'python - < str: + """Take an online SQLite backup and return its SHA-256 checksum. + + sqlite3.Connection.backup() obtains a transactionally consistent image. + The destination is committed with an atomic same-directory hard link so + pre-existing files and concurrent backup results are never overwritten. + """ + source = Path(database).expanduser() + target = Path(destination).expanduser() + if source.is_symlink() or not source.is_file(): + raise BackupError("Source must be an existing regular database file") + resolved = source.resolve(strict=True) + if resolved == target.resolve(): + raise BackupError("Backup destination cannot be the live database") + if not target.parent.is_dir(): + raise BackupError("Backup destination directory does not exist") + if target.exists() or target.is_symlink(): + raise FileExistsError("Backup destination already exists") + fd, filename = tempfile.mkstemp(prefix=".teledrive-snapshot-", dir=target.parent) + temp = Path(filename) + try: + os.fchmod(fd, 0o600) + os.close(fd) + fd = -1 + connection = sqlite3.connect( + "file:" + quote(str(resolved), safe="/") + "?mode=ro", + uri=True, timeout=30, + ) + try: + backup = sqlite3.connect(temp) + try: + connection.backup(backup, pages=256, sleep=0.1) + check = backup.execute("PRAGMA integrity_check").fetchone() + if not check or check[0] != "ok": + raise BackupError("SQLite snapshot integrity check failed") + backup.commit() + finally: + backup.close() + finally: + connection.close() + with temp.open("rb") as stream: + digest = hashlib.sha256() + while block := stream.read(1024 * 1024): + digest.update(block) + with temp.open("rb") as stream: + os.fsync(stream.fileno()) + # Hard-link create is atomic and refuses existing destination, whereas + # os.replace() could destroy a previously verified operator backup. + os.link(temp, target, follow_symlinks=False) + return digest.hexdigest() + except sqlite3.DatabaseError as exc: + raise BackupError("SQLite backup failed or source is corrupt") from exc + finally: + if fd != -1: + os.close(fd) + temp.unlink(missing_ok=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Consistent TeleDrive SQLite metadata snapshot") + parser.add_argument("--database", required=True, type=Path, + help="Existing SQLite database file, not an SQL URL") + parser.add_argument("--output", required=True, type=Path, + help="New backup file: never overwrites existing files") + args = parser.parse_args() + try: + checksum = backup_sqlite(args.database, args.output) + except (BackupError, OSError) as exc: + parser.exit(1, "Backup refused: " + str(exc) + "\n") + print("Backup complete. SHA-256: " + checksum) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/capacity_probe.py b/scripts/ops/capacity_probe.py new file mode 100644 index 00000000..03498e37 --- /dev/null +++ b/scripts/ops/capacity_probe.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python3 +"""Isolated long-duration SQLite + persistent synthetic Telegram endurance. + +Real storage engine and SQL transactions, disk-backed immutable remote chunks, +periodic SQL restarts and retrospective byte-level readback. No live Telegram +credentials, no physical multi-host load and no remote message deletion. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import math +import os +import shutil +import tempfile +import time +from pathlib import Path + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.manifest import ManifestStore + + +class IsolatedMedia: + """Persist simulated Telegram message bytes separately from SQL metadata.""" + + def __init__(self, root: Path): + self.root = Path(root) + self.root.mkdir(mode=0o700) + self.next_id = 0 + self.download_count = 0 + + def path(self, message_id: int) -> Path: + return self.root / f"{message_id:012d}.blob" + + async def upload(self, path, *, object_id, ordinal, checksum): + payload = Path(path).read_bytes() + if hashlib.sha256(payload).hexdigest() != checksum: + raise ValueError("Uploader checksum mismatch") + self.next_id += 1 + with self.path(self.next_id).open("xb") as target: + target.write(payload) + return ("synthetic", 123456, self.next_id) + + async def download(self, reference, destination): + if reference.account != "synthetic" or reference.chat_id != 123456: + raise FileNotFoundError("Synthetic remote identity mismatch") + source = self.path(reference.message_id) + if not source.is_file() or source.is_symlink(): + raise FileNotFoundError("Synthetic remote message missing") + self.download_count += 1 + shutil.copyfile(source, destination) + + +def percentile(values: list[float], fraction: float) -> float: + if not values or not 0 <= fraction <= 1: + raise ValueError("Invalid percentile input") + ordered = sorted(values) + return round(ordered[min(len(ordered) - 1, math.ceil(len(ordered) * fraction) - 1)], 3) + + +def _readback(catalog: BucketCatalog, media: IsolatedMedia, object_id: str, + expected_sha: str, size: int, destination: Path, + scratch: Path) -> None: + engine = TelegramObjectStorage( + catalog.manifests, media, chunk_size=128 * 1024, + scratch_directory=scratch, + ) + asyncio.run(engine.download_to_path(object_id, destination)) + digest = hashlib.sha256() + observed = 0 + with destination.open("rb") as reader: + while block := reader.read(1024 * 1024): + observed += len(block) + digest.update(block) + destination.unlink() + if observed != size or digest.hexdigest() != expected_sha: + raise ValueError("Historical Telegram readback failed integrity proof") + + +def run(*, duration_seconds: int, object_bytes: int, interval_seconds: float = 1.0, + workspace: Path | None = None, clock=time.monotonic, sleep=time.sleep) -> dict: + if (type(duration_seconds) is not int or not 1 <= duration_seconds <= 21600 + or type(object_bytes) is not int or not 1 <= object_bytes <= 4 * 1024 * 1024 + or type(interval_seconds) not in (int, float) + or not 0 <= interval_seconds <= 60): + raise ValueError("Invalid bounded endurance parameters") + if workspace is None: + with tempfile.TemporaryDirectory(prefix="teledrive-capacity-") as dirname: + return run(duration_seconds=duration_seconds, object_bytes=object_bytes, + interval_seconds=interval_seconds, workspace=Path(dirname), + clock=clock, sleep=sleep) + workspace = Path(workspace) + workspace.mkdir(parents=True, exist_ok=True) + if any(workspace.iterdir()): + raise FileExistsError("Capacity workspace must be empty") + + dbpath = workspace / "isolated-sqlite.db" + media = IsolatedMedia(workspace / "telegram") + samples: list[float] = [] + history: list[tuple[str, str, int]] = [] + historic_rereads = 0 + restarts = 0 + start = clock() + db = connect_database(str(dbpath)) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("capacity-test", int(time.time())) + while True: + original = workspace / "original.bin" + recovered = workspace / "recovered.bin" + payload = os.urandom(object_bytes) + digest = hashlib.sha256(payload).hexdigest() + original.write_bytes(payload) + begin = clock() + item = asyncio.run(TelegramObjectStorage( + catalog.manifests, media, chunk_size=128 * 1024, + scratch_directory=workspace, + ).put_file("capacity-test", f"items/{len(history):08d}", original)) + catalog.publish(item.id, now=int(time.time())) + original.unlink() + _readback(catalog, media, item.id, digest, object_bytes, recovered, workspace) + history.append((item.id, digest, object_bytes)) + samples.append(max(0.0, (clock() - begin) * 1000)) + + # Reopen the actual catalog periodically, verify old remote bytes + # without source files, and ensure the synthetic transport retained + # its original immutable messages rather than discarding them. + if len(history) % 8 == 0: + db.close() + db = connect_database(str(dbpath)) + catalog = BucketCatalog(ManifestStore(db)) + restarts += 1 + index = len(history) // 2 - 1 + old_id, old_sha, old_size = history[index] + _readback(catalog, media, old_id, old_sha, old_size, + recovered, workspace) + historic_rereads += 1 + + if clock() - start >= duration_seconds: + break + if interval_seconds: + sleep(interval_seconds) + # A final fresh-connection readback proves that SQL and Telegram + # survived end-to-end retention for the full duration. + db.close() + db = connect_database(str(dbpath)) + catalog = BucketCatalog(ManifestStore(db)) + restarts += 1 + for index in {0, len(history) // 2, len(history) - 1}: + ident, sha, length = history[index] + _readback(catalog, media, ident, sha, length, + workspace / "recovered.bin", workspace) + historic_rereads += 1 + finally: + db.close() + elapsed = clock() - start + files = tuple((workspace / "telegram").glob("*.blob")) + total_disk_bytes = sum(p.stat().st_size for p in files) + if total_disk_bytes != object_bytes * len(history): + raise ValueError("Retained synthetic Telegram payloads are incomplete") + return { + "mode": "isolated-sqlite-persistent-disk-telegram", + "duration_seconds": round(elapsed, 2), + "object_bytes": object_bytes, + "verified_roundtrips": len(history), + "verified_bytes": object_bytes * len(history), + "p50_roundtrip_ms": percentile(samples, .5), + "p95_roundtrip_ms": percentile(samples, .95), + "max_roundtrip_ms": round(max(samples), 3), + "sql_restarts": restarts, + "historical_rereads": historic_rereads, + "retained_telegram_chunks": len(files), + "retained_telegram_bytes": total_disk_bytes, + "real_telegram_tested": False, + "physical_hosts_tested": False, + "production_acceptance_signed_off": False, + "telegram_deletions": 0, + } + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--duration-seconds", type=int, default=3600) + p.add_argument("--object-bytes", type=int, default=65536) + p.add_argument("--interval-seconds", type=float, default=1.0) + p.add_argument("--output", type=Path, required=True) + args = p.parse_args() + try: + result = run(duration_seconds=args.duration_seconds, + object_bytes=args.object_bytes, interval_seconds=args.interval_seconds) + args.output.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as out: + json.dump(result, out, sort_keys=True, indent=2) + out.write("\n") + print(json.dumps(result, sort_keys=True)) + return 0 if result["duration_seconds"] >= args.duration_seconds else 2 + except Exception as exc: + p.exit(1, "Capacity soak refused: " + type(exc).__name__ + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/check_release_matrix.py b/scripts/ops/check_release_matrix.py new file mode 100644 index 00000000..0ab8dd60 --- /dev/null +++ b/scripts/ops/check_release_matrix.py @@ -0,0 +1,50 @@ +"""Fail-closed external workflow proof for mainline release requests.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path + +REQUIRED = { + "01 · Quality | Security & Regression", + "02 · Integration | Databases", + "02 · Integration | REST & S3", + "03 · Resilience | Docker Fault Injection", + "04 · Live Telegram | User MTProto E2E", + "04 · Live Telegram | Multiworker PostgreSQL E2E", +} + +def check(payload: dict, sha: str) -> tuple[int, list[str]]: + runs = payload.get("workflow_runs") + if not isinstance(runs, list): + return 1, ["Malformed Actions result"] + selected = {} + for run in sorted(runs, key=lambda x: (x.get("created_at", ""), x.get("id", 0)), reverse=True): + if run.get("head_sha") == sha and run.get("name") in REQUIRED: + selected.setdefault(run["name"], run) + reasons, pending, failed = [], False, False + for name in sorted(REQUIRED): + item = selected.get(name) + if item is None: + reasons.append(name + ": missing") + pending = True + elif item.get("status") != "completed": + reasons.append(name + ": " + str(item.get("status"))) + pending = True + elif item.get("conclusion") != "success": + reasons.append(name + ": " + str(item.get("conclusion"))) + failed = True + return (1 if failed else 2 if pending else 0), reasons + +def main() -> int: + p = argparse.ArgumentParser() + p.add_argument("--input", required=True, type=Path) + p.add_argument("--sha", required=True) + a = p.parse_args() + result, reasons = check(json.loads(a.input.read_text("utf-8")), a.sha) + for reason in reasons: + print(reason) + print("P0/P1 same-SHA release gate:", ("PASS", "FAIL", "WAIT")[result]) + return result + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/cleanup_merged_branches.py b/scripts/ops/cleanup_merged_branches.py new file mode 100644 index 00000000..27bb2360 --- /dev/null +++ b/scripts/ops/cleanup_merged_branches.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Safely delete only same-repository, merged-PR branches still at their PR head. + +Designed for GitHub Actions with a short-lived GITHUB_TOKEN with contents:write. +Dry-run is the default; --execute is required. No tokens appear in output. +Never delete main, protected, foreign, unmerged, reopened or advanced branches. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from urllib.parse import quote + +ELIGIBLE_NAME = re.compile(r"(?:p[0-9]+|fix|feature|feat|docs|chore|test)/[a-zA-Z0-9._/-]+\Z") + + +def api(route: str, *, delete: bool = False): + command = ["gh", "api", f"repos/{os.environ['GITHUB_REPOSITORY']}/{route}"] + if delete: + command += ["--method", "DELETE"] + result = subprocess.run(command, capture_output=True, text=True, check=False) + if result.returncode: + # Do not echo token, GitHub JSON body, or sensitive endpoint values. + raise RuntimeError(f"GitHub API {'DELETE' if delete else 'GET'} failed for {route}") + return None if delete else json.loads(result.stdout) + + +def pages(resource: str, *, params: str = ""): + for page in range(1, 101): + result = api(f"{resource}?per_page=100&page={page}{('&' + params) if params else ''}") + if not isinstance(result, list): + raise RuntimeError(f"Unexpected GitHub response type for {resource}") + yield from result + if len(result) < 100: + return + raise RuntimeError("Too many GitHub pages, refusing incomplete inspection") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--execute", action="store_true", help="Actually delete verified merged refs") + args = parser.parse_args() + if os.environ.get("GITHUB_REPOSITORY") != "babico/TeleDrive": + raise RuntimeError("Repository identity mismatch; explicit review required") + if not os.environ.get("GH_TOKEN"): + raise RuntimeError("GitHub Actions token not configured") + closed = list(pages("pulls", params="state=closed&sort=updated&direction=desc")) + merged = [pr for pr in closed if pr.get("merged_at") and + pr.get("base", {}).get("ref") == "main" and + (pr.get("head", {}).get("repo") or {}).get("full_name") == + os.environ["GITHUB_REPOSITORY"]] + open_prs = list(pages("pulls", params="state=open")) + active = {pr.get("head", {}).get("ref") for pr in open_prs + if (pr.get("head", {}).get("repo") or {}).get("full_name") == + os.environ["GITHUB_REPOSITORY"]} + current = list(pages("branches")) + eligible = [] + for candidate in current: + name, sha = candidate.get("name", ""), candidate.get("commit", {}).get("sha") + if (not ELIGIBLE_NAME.fullmatch(name) or name == "main" or + candidate.get("protected", True) or name in active or not sha): + continue + if not any(pr["head"]["ref"] == name and pr["head"]["sha"] == sha + for pr in merged): + continue + eligible.append((name, sha)) + for name, sha in sorted(eligible): + # Double-check the branch has not moved between initial listing and + # deletion. (No CAS is exposed by REST DELETE; use a dedicated + # maintenance window for the final deletion.) + ref = api("git/ref/heads/" + quote(name, safe="/")) + if ref.get("object", {}).get("sha") != sha: + print(f"SKIP: head advanced for {name}") + continue + if args.execute: + api("git/refs/heads/" + quote(name, safe="/"), delete=True) + print(f"DELETED merged branch: {name}") + else: + print(f"DRY RUN eligible merged branch: {name}") + print(f"Inspected {len(current)} refs, {len(merged)} merged main PRs; " + f"eligible={len(eligible)}, execute={args.execute}") + + +if __name__ == "__main__": + try: + main() + except Exception as exc: + print(f"BRANCH_CLEANUP_ERROR: {type(exc).__name__}: {exc}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/ops/gc_dry_run.py b/scripts/ops/gc_dry_run.py new file mode 100644 index 00000000..39622b20 --- /dev/null +++ b/scripts/ops/gc_dry_run.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Read-only GC dry-run plan producer. + +Enumerates every SQL reference for one failed object, fingerprints the graph +and records the owner gates that remain unmet. This command NEVER deletes +Telegram messages, SQL rows or metadata. Physical deletion stays disabled +until independent payload backup, verified restore, writer fencing and +two-person approval are proven. +""" +from __future__ import annotations + +import argparse +import json +import os +import time +from pathlib import Path + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore + + +def run(args) -> dict: + db = connect_database(args.database) + try: + catalog = BucketCatalog(ManifestStore(db)) + plan = StorageGCInventory(catalog.manifests).plan( + args.object_id, now=args.now, backup_set_id=args.backup_set_id, + writer_fence_token=args.writer_fence_token, + min_age_seconds=args.min_age_seconds, + ) + # Re-verify the plan is internally consistent before writing it. + StorageGCInventory(catalog.manifests).verify_plan(plan) + return plan + finally: + db.close() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database", required=True) + parser.add_argument("--object-id", required=True) + parser.add_argument("--min-age-seconds", type=int, default=7 * 86400) + parser.add_argument("--backup-set-id", default="") + parser.add_argument("--writer-fence-token", default="") + parser.add_argument("--now", type=int, default=0) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + plan = run(args) + except Exception as exc: + parser.exit(1, "GC dry-run plan refused: " + type(exc).__name__ + "\n") + # Private evidence file with mode 0600; no opaque Telegram file IDs. + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as stream: + json.dump(plan, stream, sort_keys=True, indent=2) + stream.write("\n") + print(json.dumps({ + "plan_written": str(args.output), + "object_id": plan["object_id"], + "reference_blocked": plan["reference_blocked"], + "reference_blockers": plan["reference_blockers"], + "deletion_permitted": plan["deletion_permitted"], + "dry_run": plan["dry_run"], + "remote_history_scan_required": plan["remote_history_scan_required"], + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/generate_ssl.py b/scripts/ops/generate_ssl.py new file mode 100644 index 00000000..e2b03de1 --- /dev/null +++ b/scripts/ops/generate_ssl.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Generate a self-signed TLS certificate for the TeleDrive service. + +The unified service serves HTTPS when config/config.yaml sets +web.ssl_certfile and web.ssl_keyfile. This script creates those files with +openssl (or Python cryptography as fallback) and can verify a running +instance over HTTPS with the generated CA certificate. + +Examples: + python scripts/ops/generate_ssl.py --out /config/tls + python scripts/ops/generate_ssl.py --out /config/tls \ + --test https://127.0.0.1:8443 --insecure --days 365 +""" +from __future__ import annotations + +import argparse +import shutil +import ssl +import subprocess +import sys +import urllib.request +from pathlib import Path + + +def generate(target: Path, *, days: int, hostname: str) -> dict: + target.mkdir(parents=True, exist_ok=True) + cert = target / "server.crt" + key = target / "server.key" + if cert.exists() or key.exists(): + raise FileExistsError("Refusing to overwrite existing TLS material") + if shutil.which("openssl"): + command = [ + "openssl", "req", "-x509", "-nodes", "-newkey", "rsa:2048", + "-keyout", str(key), "-out", str(cert), "-days", str(days), + "-subj", f"/CN={hostname}", + "-addext", f"subjectAltName=DNS:{hostname},IP:127.0.0.1", + ] + completed = subprocess.run(command, capture_output=True, text=True) + if completed.returncode != 0: + raise RuntimeError("openssl failed: " + completed.stderr[-800:]) + else: + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + import datetime + + private = rsa.generate_private_key(public_exponent=65537, key_size=2048) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, hostname)]) + certificate = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(private.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.datetime.now(datetime.timezone.utc)) + .not_valid_after( + datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(days=days) + ) + .add_extension( + x509.SubjectAlternativeName( + [x509.DNSName(hostname), x509.IPAddress( + __import__("ipaddress").ip_address("127.0.0.1"))] + ), + critical=False, + ) + .sign(private, hashes.SHA256()) + ) + with key.open("wb") as stream: + stream.write(private.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + )) + with cert.open("wb") as stream: + stream.write(certificate.public_bytes(serialization.Encoding.PEM)) + key.chmod(0o600) + cert.chmod(0o644) + return { + "certfile": str(cert), + "keyfile": str(key), + "config_snippet": ( + "web:\n" + f' ssl_certfile: "{cert}"\n' + f' ssl_keyfile: "{key}"\n' + ), + } + + +def test_https(url: str, cert: Path, *, insecure: bool) -> None: + context = ssl.create_default_context(cafile=str(cert)) + if insecure: + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with urllib.request.urlopen(url, context=context, timeout=10) as response: + body = response.read(256) + print(f"HTTPS OK: {url} responded with HTTP {response.status}") + if not body: + raise RuntimeError("Empty HTTPS response") + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--out", type=Path, required=True, + help="Directory for server.crt and server.key") + parser.add_argument("--days", type=int, default=365) + parser.add_argument("--hostname", default="localhost") + parser.add_argument("--test", default="", + help="Verify a running instance, e.g. https://127.0.0.1:8443") + parser.add_argument("--insecure", action="store_true", + help="Accept the self-signed chain without hostname checks") + args = parser.parse_args() + try: + if args.test: + cert = args.out / "server.crt" + if not cert.is_file(): + parser.error(f"{cert} missing; generate it first") + test_https(args.test, cert, insecure=args.insecure) + return 0 + result = generate(args.out, days=args.days, hostname=args.hostname) + print("TLS material generated:") + print(" cert:", result["certfile"]) + print(" key: ", result["keyfile"]) + print("Add to config/config.yaml:") + print(result["config_snippet"]) + print("Then set web.port/web.s3_port as needed and restart the service.") + return 0 + except Exception as exc: + print("SSL tool failed: " + type(exc).__name__ + ": " + str(exc), + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/inventory_telegram_history.py b/scripts/ops/inventory_telegram_history.py new file mode 100644 index 00000000..56c59223 --- /dev/null +++ b/scripts/ops/inventory_telegram_history.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Inventory original Telegram marker candidates without touching stored media. + +Supply TELEDRIVE_REBUILD_API_ID, TELEDRIVE_REBUILD_API_HASH and +TELEDRIVE_REBUILD_SESSION from a protected operator environment. +The 0600 JSON report includes original Telegram message coordinates and must +never be committed, uploaded to CI, or treated as a restoration manifest. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +from teledrive.storage.history_inventory import ( + inventory_channel, write_private_report, +) + + +async def execute(*, account: str, target: str, limit: int, + api_id: int, api_hash: str, session: str) -> dict: + from telethon import TelegramClient + from telethon.sessions import StringSession + + async with TelegramClient(StringSession(session), api_id, api_hash) as client: + entity = await client.get_entity(target) + return await inventory_channel( + client, entity, account=account, scan_limit=limit, + ) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--account", required=True, help="Original user-account label") + parser.add_argument("--target", required=True, help="Authorized original channel") + parser.add_argument("--limit", type=int, default=1000, + help="Maximum messages inspected (1..10000)") + parser.add_argument("--output", required=True, type=Path, + help="New private JSON path; existing files are never replaced") + args = parser.parse_args() + try: + # Validate arguments and report destination before authenticating. + if not 1 <= args.limit <= 10_000: + raise ValueError("Invalid scan limit") + if args.output.exists() or args.output.is_symlink(): + raise FileExistsError("Refuse to replace an existing inventory") + if not args.output.parent.is_dir(): + raise FileNotFoundError("Inventory output directory missing") + api_id = int(os.environ["TELEDRIVE_REBUILD_API_ID"]) + api_hash = os.environ["TELEDRIVE_REBUILD_API_HASH"] + session = os.environ["TELEDRIVE_REBUILD_SESSION"] + if not api_hash or not session: + raise ValueError("Missing original Telegram credentials") + report = asyncio.run(execute( + account=args.account, target=args.target, limit=args.limit, + api_id=api_id, api_hash=api_hash, session=session, + )) + write_private_report(args.output, report) + except Exception as exc: + parser.exit(1, "Read-only Telegram inventory refused: " + + type(exc).__name__ + "\n") + + # Avoid leaking channel IDs and individual Telegram message IDs to logs. + print(json.dumps({ + "report_created": True, + "scanned_messages": report["scanned_messages"], + "truncated": report["truncated"], + "candidate_object_count": report["candidate_object_count"], + "candidate_chunk_count": report["candidate_chunk_count"], + "payload_verified": False, + "production_database_modified": False, + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/monitor_host_soak.py b/scripts/ops/monitor_host_soak.py new file mode 100644 index 00000000..875598b6 --- /dev/null +++ b/scripts/ops/monitor_host_soak.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Read-only multi-host P1 production-soak telemetry receipt. + +This command NEVER uploads/deletes Telegram messages or modifies SQL. +It cannot attest physical host independence, original-media DR, or legacy +cutover; those require separate signed operator evidence. +""" +from __future__ import annotations + +import argparse +import base64 +import json +import os +import re +import time +import urllib.parse +import urllib.request +from pathlib import Path + + +HEALTH = "/api/v1/admin/storage/health" + + +def parse_hosts(raw: str) -> list[dict[str, str]]: + hosts = json.loads(raw) + if not isinstance(hosts, list) or not 2 <= len(hosts) <= 8: + raise ValueError("Specify two to eight separate HTTPS health endpoints") + result = [] + names, peers = set(), set() + for entry in hosts: + if not isinstance(entry, dict) or set(entry) != {"label", "url"}: + raise ValueError("Invalid host entry") + label, url = entry["label"], entry["url"] + if (not isinstance(label, str) + or not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", label) + or not isinstance(url, str)): + raise ValueError("Invalid host label or URL") + parsed = urllib.parse.urlsplit(url) + if (parsed.scheme != "https" or not parsed.hostname + or parsed.username is not None or parsed.password is not None + or parsed.path not in ("", "/") or parsed.query or parsed.fragment + or parsed.port not in (None, 443)): + raise ValueError("Soak requires clean HTTPS host origins") + origin = "https://" + parsed.hostname.lower() + if label in names or origin in peers: + raise ValueError("Host labels and network endpoints must be distinct") + names.add(label) + peers.add(origin) + result.append({"label": label, "origin": origin}) + return result + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("Health endpoint redirected; credentials withheld") + + +def health_probe(origin: str, username: str, password: str) -> dict: + # Never follow even a same-origin redirect while passing credentials. + secret = base64.b64encode( + (username + ":" + password).encode("utf-8") + ).decode("ascii") + req = urllib.request.Request( + origin + HEALTH, + headers={"Authorization": "Basic " + secret, "Accept": "application/json"}, + ) + with urllib.request.build_opener(NoRedirect()).open(req, timeout=15) as response: + if response.status != 200: + raise RuntimeError("Health endpoint refused the observation") + raw = response.read(65537) + if len(raw) > 65536: + raise ValueError("Oversized health response") + payload = json.loads(raw) + if not isinstance(payload, dict): + raise ValueError("Malformed health response") + return payload + + +def selected_metrics(payload: dict) -> dict: + fields = ( + "bot_explicit_429_total", "unresolved_bot_send_count", + "unjournaled_bot_ack_count", "stale_pending_sample_count", + "stale_multipart_sample_count", + ) + result = {} + for key in fields: + value = payload.get(key) + if type(value) is not int or value < 0: + raise ValueError("Invalid health telemetry counter") + result[key] = value + states = payload.get("job_states") + if not isinstance(states, dict): + raise ValueError("Missing upload backlog state") + for name in ("queued", "running", "failed"): + value = states.get(name, 0) + if type(value) is not int or value < 0: + raise ValueError("Invalid upload backlog metric") + result[name + "_jobs"] = value + if (payload.get("samples_truncated") is not False + or payload.get("physical_deletion_enabled") is not False): + raise ValueError("Unsafe or truncated storage-health inventory") + return result + + +def summarize(samples: list[dict], *, duration: int, interval: int) -> dict: + if len(samples) < 2 or samples[-1]["elapsed"] < duration: + raise ValueError("Soak did not cover the requested duration") + if any(b["elapsed"] - a["elapsed"] > interval * 2 + for a, b in zip(samples, samples[1:])): + raise ValueError("Unobserved gap in production soak") + labels = list(samples[0]["hosts"]) + if len(labels) < 2 or any(set(s["hosts"]) != set(labels) for s in samples): + raise ValueError("Host topology changed during acceptance") + deltas = {} + safe = True + for label in labels: + observations = [s["hosts"][label] for s in samples] + counts = [x["bot_explicit_429_total"] for x in observations] + if counts != sorted(counts): + safe = False # SQL changed/rolled back; cannot certify a counter. + deltas[label] = counts[-1] - counts[0] + last = observations[-1] + if any(last[x] for x in ( + "unresolved_bot_send_count", "unjournaled_bot_ack_count", + "stale_pending_sample_count", "stale_multipart_sample_count", + "queued_jobs", "running_jobs", "failed_jobs", + )): + safe = False + # Shared counter agreement is useful evidence but cannot by itself + # prove independent physical hosts or that traffic was unmodified. + agree = len({ + samples[-1]["hosts"][x]["bot_explicit_429_total"] for x in labels + }) == 1 + return { + "duration_seconds": samples[-1]["elapsed"], + "sample_count": len(samples), + "host_labels": labels, + "explicit_429_deltas_by_host": deltas, + "final_shared_counter_agreement": agree, + "telemetry_gate_passed": safe and agree and all(v > 0 for v in deltas.values()), + "physical_host_independence_verified": False, + "original_media_restore_verified": False, + "legacy_cutover_approved": False, + "production_acceptance_signed_off": False, + "telegram_mutations_performed": 0, + } + + +def collect(hosts: list[dict], probe, *, duration: int, interval: int, + clock=time.monotonic, sleep=time.sleep) -> dict: + beginning = clock() + samples = [] + while True: + elapsed = int(clock() - beginning) + node_metrics = { + h["label"]: selected_metrics(probe(h["origin"])) + for h in hosts + } + samples.append({"elapsed": elapsed, "hosts": node_metrics}) + if elapsed >= duration: + break + deadline = beginning + min( + duration, ((len(samples)) * interval), + ) + sleep(max(0, deadline - clock())) + return summarize(samples, duration=duration, interval=interval) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--duration-seconds", type=int, default=3600) + parser.add_argument("--interval-seconds", type=int, default=60) + args = parser.parse_args() + if args.duration_seconds < 3600 or not 30 <= args.interval_seconds <= 600: + parser.error("Production soak requires >=3600 seconds, 30-600s sampling") + try: + hosts = parse_hosts(os.environ.get("TELEDRIVE_P1_HEALTH_HOSTS", "")) + user = os.environ["TELEDRIVE_P1_BASIC_USERNAME"] + password = os.environ["TELEDRIVE_P1_BASIC_PASSWORD"] + if not user or not password: + raise ValueError("Missing protected health credentials") + report = collect( + hosts, lambda origin: health_probe(origin, user, password), + duration=args.duration_seconds, interval=args.interval_seconds, + ) + # Fail if the destination already exists. Never overwrite existing + # audit evidence or print secrets and HTTP exception text. + args.output.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(args.output, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as out: + json.dump(report, out, indent=2, sort_keys=True) + out.write("\n") + print(json.dumps({ + "telemetry_gate_passed": report["telemetry_gate_passed"], + "report_written": True, + "production_acceptance_signed_off": False, + })) + return 0 if report["telemetry_gate_passed"] else 2 + except Exception as exc: + parser.exit(1, "Soak verification refused: " + type(exc).__name__ + "\n") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/rebuild_from_telegram.py b/scripts/ops/rebuild_from_telegram.py new file mode 100644 index 00000000..b56576fd --- /dev/null +++ b/scripts/ops/rebuild_from_telegram.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Recover original Telegram bytes into a NEW isolated SQLite catalog. + +Requires an independently verified recovery plan with bucket/key, object UUID, +size, full file SHA-256, all Telegram chunk hashes and message coordinates. +Original authorized Telegram user session is required; never touches the +existing production database, sends messages or deletes Telegram media. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path + +from teledrive.storage.rebuild import ( + restore_isolated_catalog, restore_isolated_catalog_batch, validate_batch, +) +from teledrive.storage.telethon_adapter import TelethonChunkTransport + + +async def execute(plan: dict, destination: Path, *, account: str, target: str, + api_id: int, api_hash: str, session: str) -> dict: + from telethon import TelegramClient + from telethon.sessions import StringSession + + is_batch = isinstance(plan, dict) and set(plan) == {"objects"} + plans = plan["objects"] if is_batch else [plan] + validated = validate_batch(plans) + refs = [ref for _, entries in validated for ref in entries] + if not refs: + raise ValueError("Live source ownership requires at least one Telegram message") + if any(ref.account != account for ref in refs): + raise ValueError("One rebuild must use the same independently authorized account") + async with TelegramClient(StringSession(session), api_id, api_hash) as client: + entity = await client.get_entity(target) + original_chat = int(await client.get_peer_id(entity)) + if any(ref.chat_id != original_chat for ref in refs): + raise ValueError("Original Telegram channel does not match the recovery plan") + remote = TelethonChunkTransport({account: (client, entity)}) + if is_batch: + return await restore_isolated_catalog_batch(plans, remote, destination) + return await restore_isolated_catalog(plan, remote, destination) + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--plan", type=Path, required=True) + p.add_argument("--inventory", type=Path, + help="Optional private marker inventory; must exactly match independent manifest") + p.add_argument("--new-database", type=Path, required=True) + p.add_argument("--account", required=True) + p.add_argument("--target", required=True, help="Original Telegram target") + args = p.parse_args() + try: + api_id = int(os.environ["TELEDRIVE_REBUILD_API_ID"]) + api_hash = os.environ["TELEDRIVE_REBUILD_API_HASH"] + session = os.environ["TELEDRIVE_REBUILD_SESSION"] + plan = json.loads(args.plan.read_text(encoding="utf-8")) + if args.inventory is not None: + from teledrive.storage.reconcile_inventory import reconcile_inventory + plans = plan["objects"] if ( + isinstance(plan, dict) and set(plan) == {"objects"} + ) else [plan] + comparison = reconcile_inventory( + json.loads(args.inventory.read_text(encoding="utf-8")), plans + ) + if not comparison["all_plans_match"]: + raise ValueError("Telegram inventory differs from independent plan") + if not api_hash or not session: + raise ValueError("Missing original Telegram account credentials") + result = asyncio.run(execute( + plan, args.new_database, account=args.account, + target=args.target, api_id=api_id, api_hash=api_hash, + session=session, + )) + except Exception as exc: + p.exit(1, "Original Telegram reconstruction refused: " + + type(exc).__name__ + "\n") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/reconcile_mtproto_receipt.py b/scripts/ops/reconcile_mtproto_receipt.py new file mode 100644 index 00000000..b1bbb28f --- /dev/null +++ b/scripts/ops/reconcile_mtproto_receipt.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Operator-gated USER-MTProto receipt reconciliation with channel proof. + +A lost send intent is resolved ONLY by the authorized USER session reading +the exact original channel message: caption marker identity, full SHA-256 +and size are verified before any SQL receipt is recorded. Nothing is +deleted or resent; --apply records a receipt only after all checks pass. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import os +import tempfile +from pathlib import Path + + +async def run(args) -> dict: + from telethon import TelegramClient + from telethon.sessions import StringSession + from teledrive.db import connect_database + from teledrive.storage.user_mode import UserSendJournal + + api_id = int(os.environ["TELEDRIVE_RECONCILE_API_ID"]) + api_hash = os.environ["TELEDRIVE_RECONCILE_API_HASH"] + session = os.environ["TELEDRIVE_RECONCILE_SESSION"].strip() + if not api_id or not api_hash or not session: + raise ValueError("Missing reconciliation MTProto configuration") + if (not args.account or not args.object_id or args.ordinal < 0 + or args.message_id <= 0 + or not re_full_sha(args.sha256)): + raise ValueError("Invalid reconciliation identity") + + client = TelegramClient( + StringSession(session), api_id, api_hash, + device_model="TeleDrive Storage Client", + app_version="TeleDrive 2.1", + receive_updates=False, flood_sleep_threshold=0, + ) + await client.connect() + try: + target = args.target.strip() + if target.lstrip("-").isdigit(): + entity = None + async for dialog in client.iter_dialogs(limit=5000): + if int(dialog.id) == int(target): + entity = dialog.entity + break + if entity is None: + raise FileNotFoundError("Original channel not in user dialogs") + else: + entity = await client.get_entity(target) + if int(await client.get_peer_id(entity)) != args.chat_id: + raise ValueError("Original Telegram channel identity mismatch") + message = await client.get_messages(entity, ids=args.message_id) + if message is None or not getattr(message, "media", None): + raise ValueError("Original Telegram message/media is missing") + expected_caption = ( + f"teledrive:{args.object_id}:{args.ordinal}:{args.sha256[:16]}" + ) + if getattr(message, "message", "") != expected_caption: + raise ValueError("Original message caption does not match the intent") + with tempfile.TemporaryDirectory(prefix="teledrive-reconcile-") as name: + destination = Path(name) / "original.bin" + result = await client.download_media(message, file=str(destination)) + if result is None: + raise ValueError("Original Telegram media download failed") + digest = hashlib.sha256() + size = 0 + with destination.open("rb") as media: + while block := media.read(1024 * 1024): + digest.update(block) + size += len(block) + if digest.hexdigest() != args.sha256 or size != args.size: + raise ValueError("Original Telegram bytes do not match the intent") + applied = False + if args.apply: + db = connect_database(args.database) + try: + UserSendJournal(db).acknowledged( + args.account, args.chat_id, args.object_id, args.ordinal, + args.message_id, + ) + applied = True + finally: + db.close() + return { + "verified": True, + "applied": applied, + "message_id": args.message_id, + "sha256": args.sha256, + "size": args.size, + "telegram_uploads": 0, + "telegram_deletions": 0, + } + finally: + await client.disconnect() + + +def re_full_sha(value: str) -> bool: + import re + return bool(re.fullmatch(r"[0-9a-f]{64}", value)) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--database", required=True) + parser.add_argument("--account", required=True) + parser.add_argument("--chat-id", type=int, required=True) + parser.add_argument("--target", required=True, + help="Exact authorized original channel") + parser.add_argument("--object-id", required=True) + parser.add_argument("--ordinal", type=int, required=True) + parser.add_argument("--message-id", type=int, required=True) + parser.add_argument("--sha256", required=True) + parser.add_argument("--size", type=int, required=True) + parser.add_argument("--apply", action="store_true", + help="Record the verified receipt into the SQL journal") + args = parser.parse_args() + try: + result = asyncio.run(run(args)) + except Exception as exc: + parser.exit(1, "MTProto receipt verification failed: " + + type(exc).__name__ + "\n") + import json + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/rehearse_native_database_restore.py b/scripts/ops/rehearse_native_database_restore.py new file mode 100644 index 00000000..f175e63e --- /dev/null +++ b/scripts/ops/rehearse_native_database_restore.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +"""Disposable native PostgreSQL/MySQL dump/restore acceptance fixture. + +CI only: a separate restored database is REQUIRED; no production restore, +mutation of existing data, or Telegram message deletion is performed. +The external pg_dump/pg_restore/mysqldump commands run in the CI workflow. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +import time +from pathlib import Path +from uuid import uuid4 + +from cryptography.fernet import Fernet +from sqlalchemy import and_, func, select + +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.tagging import ObjectTagStore +from teledrive.storage.catalog import BucketCatalog, removed_versions +from teledrive.storage.manifest import chunks +from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard, _mtproto_flood_observations +from teledrive.storage.user_mode import UserSendJournal, mtproto_send_intents +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ChunkReference, ManifestStore + +# Test-only Fernet key, never a production secret. +TEST_MASTER_KEY = base64.urlsafe_b64encode( + hashlib.sha256(b"TeleDrive disposable DR CI master key only").digest() +) +PAYLOAD = bytes(range(256)) * 32 + b"native-restore-tail" +S3_KEY = "dr/ünicode/multi-part.bin" + + +def fingerprint(url: str, receipt: dict) -> dict: + """Read data through the actual production stores in a fresh DB session.""" + db = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(db)) + journal = UserSendJournal(db) + owner = receipt["bucket"] + head = catalog.get(owner, S3_KEY) + versions, truncated = catalog.list_bucket_versions(owner, limit=10) + first = receipt["first"] + pending = receipt["pending"] + parts = catalog.manifests.parts(first) + return { + "head": head.item.id, + "generation": head.generation, + "head_sha": head.item.sha256, + "versions": [[v["version_id"], v["is_current"], v["is_delete_marker"]] + for v in versions], + "truncated": truncated, + "source_visible": catalog.is_visible_version(owner, S3_KEY, first), + "deleted_version_hidden": not catalog.is_visible_version( + owner, S3_KEY, receipt["second"]), + "deleted_marker_hidden": not catalog.is_visible_version( + owner, S3_KEY, receipt["marker"]), + "removed_count": db.connection.execute( + select(func.count()).select_from(removed_versions) + .where(removed_versions.c.object_id.in_( + [receipt["marker"], receipt["second"]] + )) + ).scalar_one(), + "parts": [[p.ordinal, p.offset, p.size, p.sha256, + p.account, p.chat_id, p.message_id] for p in parts], + "tags": [list(row) for row in ObjectTagStore(catalog).get(first)], + "pending_ack": db.connection.execute( + select(mtproto_send_intents.c.message_id).where( + mtproto_send_intents.c.object_id == pending, + mtproto_send_intents.c.ordinal == 0, + ) + ).scalar_one(), + "unjournaled": db.connection.execute( + select(func.count()).select_from( + mtproto_send_intents.outerjoin(chunks, and_( + chunks.c.account == mtproto_send_intents.c.account, + chunks.c.chat_id == mtproto_send_intents.c.chat_id, + chunks.c.message_id == mtproto_send_intents.c.message_id, + )) + ).where( + mtproto_send_intents.c.message_id.is_not(None), + chunks.c.object_id.is_(None), + ) + ).scalar_one(), + "mtproto_flood_evidence": list(db.connection.execute(select( + _mtproto_flood_observations.c.observations, + _mtproto_flood_observations.c.last_observed_ms, + _mtproto_flood_observations.c.last_retry_after_seconds, + ).where( + _mtproto_flood_observations.c.account == receipt["rate_account"], + _mtproto_flood_observations.c.chat_id == receipt["rate_chat"], + )).one()), + "job": next( + {"id": j["id"], "state": j["state"], "attempts": j["attempts"]} + for j in StorageUploadJobs(catalog).list_jobs(limit=1000) + if j["id"] == receipt["job"] + ), + "s3_credential_decrypts": ( + S3CredentialStore(db, TEST_MASTER_KEY).resolve( + receipt["access_key"], "s3:GetObject" + ) is not None + ), + } + finally: + db.close() + + +def seed(url: str, receipt_path: Path, source: Path): + """Write realistic version history, chunk journal, ACK and queued job.""" + db = connect_database(url) + bucket = "ci-dr-" + uuid4().hex[:19] + message = int(uuid4().hex[:8], 16) % 1_800_000_000 + 1000 + now = int(time.time()) + receipt = {"bucket": bucket} + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket(bucket, now) + first = catalog.manifests.begin(bucket, S3_KEY, len(PAYLOAD), now) + for index, part in enumerate((PAYLOAD[:8192], PAYLOAD[8192:])): + catalog.manifests.append( + first, ChunkReference( + ordinal=index, offset=index * 8192, size=len(part), + sha256=hashlib.sha256(part).hexdigest(), + account="ci-dr-user", chat_id=-100900901, + message_id=message + index, + ), now, + ) + catalog.manifests.finish(first, hashlib.sha256(PAYLOAD).hexdigest(), now) + catalog.publish(first, now=now) + second = catalog.manifests.begin(bucket, S3_KEY, 0, now) + catalog.manifests.finish(second, hashlib.sha256(b"").hexdigest(), now) + catalog.publish(second, now=now) + marker = catalog.delete_marker(bucket, S3_KEY) + assert catalog.delete_version(bucket, S3_KEY, marker) + assert catalog.delete_version(bucket, S3_KEY, second) + ObjectTagStore(catalog).replace_current( + bucket, S3_KEY, [("dr", "restored"), ("lang", "Türkçe")] + ) + pending = catalog.manifests.begin(bucket, "ack-orphan", 1, now) + journal = UserSendJournal(db) + journal.start("ci-dr-user", -100900901, pending, 0, + hashlib.sha256(b"Q").hexdigest(), 1) + journal.acknowledged("ci-dr-user", -100900901, pending, 0, message + 3) + # Validate native PostgreSQL/MySQL restore preserves a definitive + # rejected-send observation, not merely upload manifests/ACKs. + # This is a *synthetic* flood-wait: never label it production evidence. + rate_account = "ci-dr-rate-" + uuid4().hex[:12] + rate_chat = -100900902 + rate_index = MtprotoRateGuard(db) + rate_index.reserve_send_slot(rate_account, rate_chat, 0, now_ms=1000) + rate_index.impose_shared_cooldown( + rate_account, rate_chat, 9, now_ms=1000, + ) + source.write_bytes(b"durable-queued-source" * 13) + job = StorageUploadJobs(catalog).enqueue( + bucket, "queue/pending.bin", source, + allowed_root=source.parent, + ) + credential = S3CredentialStore(db, TEST_MASTER_KEY).create( + "disposable-dr", ["s3:GetObject"] + ) + receipt.update({ + "first": first, "second": second, "marker": marker, + "pending": pending, "job": job, "access_key": credential.access_key, + "rate_account": rate_account, "rate_chat": rate_chat, + }) + finally: + db.close() + receipt["fingerprint"] = fingerprint(url, receipt) + assert receipt["fingerprint"]["head"] == receipt["first"] + assert receipt["fingerprint"]["head_sha"] == hashlib.sha256(PAYLOAD).hexdigest() + assert receipt["fingerprint"]["removed_count"] == 2 + assert len(receipt["fingerprint"]["parts"]) == 2 + assert receipt["fingerprint"]["unjournaled"] == 1 + assert receipt["fingerprint"]["mtproto_flood_evidence"] == [1, 1000, 9] + assert receipt["fingerprint"]["s3_credential_decrypts"] + # Durable test receipt contains only synthetic CI IDs and no secret keys. + receipt_path.write_text(json.dumps(receipt, ensure_ascii=False), encoding="utf-8") + + +def verify(url: str, receipt_path: Path): + receipt = json.loads(receipt_path.read_text(encoding="utf-8")) + restored = fingerprint(url, receipt) + # JSON receipts normalize tuple rows to arrays; compare typed JSON views. + restored = json.loads(json.dumps(restored, ensure_ascii=False)) + assert restored == receipt["fingerprint"], { + "expected": receipt["fingerprint"], "restored": restored, + } + print(json.dumps({ + "restore_valid": True, + "backend": url.split(":", 1)[0], + "chunk_refs": len(restored["parts"]), + "removed_versions": restored["removed_count"], + "unjournaled_receipts": restored["unjournaled"], + "synthetic_mtproto_flood_observations": restored["mtproto_flood_evidence"][0], + "queue_state": restored["job"]["state"], + "s3_credential_decrypts": restored["s3_credential_decrypts"], + "physical_telegram_delete": False, + }, sort_keys=True)) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("phase", choices=("seed", "verify")) + parser.add_argument("--url-env", required=True, + choices=("DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL")) + parser.add_argument("--receipt", type=Path, required=True) + parser.add_argument("--source", type=Path) + parser.add_argument("--restore-database") + args = parser.parse_args() + url = os.environ[args.url_env] + if args.phase == "seed": + if args.source is None: + parser.error("seed requires --source") + seed(url, args.receipt, args.source) + else: + if not args.restore_database or args.restore_database == "teledrive": + parser.error("verify must target a separate, nonproduction database") + from sqlalchemy.engine.url import make_url + url = make_url(url).set(database=args.restore_database).render_as_string( + hide_password=False, + ) + verify(url, args.receipt) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/rehearse_telegram_dr.py b/scripts/ops/rehearse_telegram_dr.py new file mode 100644 index 00000000..b4038d3f --- /dev/null +++ b/scripts/ops/rehearse_telegram_dr.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +"""Staging DR: verify SQL snapshot, then read the original Telegram bytes. + +No live database is modified and no Telegram uploads or deletes are issued. +The verified backup is copied into a disposable scratch directory before +initializing ORM metadata, preserving the exact SHA-256 receipt of the +operator's archive. The CLI authenticates a read-only USER MTProto session +against the ORIGINAL channel; there is no Bot API transport. +""" +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import shutil +import tempfile +from dataclasses import replace +from pathlib import Path + +from scripts.ops.verify_metadata_backup import verify_snapshot +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.telethon_adapter import TelethonChunkTransport + + +class ReadOnlyUserChannelTransport: + """Permit only an exact authorized channel and original SQL account label. + + Historical api-bot messages in the *same* channel may be read by a real + Telegram user. No mutation method is exposed; no cross-channel fallback. + Chunk hash and full-object integrity remain checked by the storage engine. + """ + + def __init__(self, client, channel, *, account: str, chat_id: int): + self.account = account + self.chat_id = chat_id + self.reader = TelethonChunkTransport({account: (client, channel)}) + + async def download(self, reference, destination): + if (reference.chat_id != self.chat_id or + reference.account not in (self.account, "api-bot")): + raise FileNotFoundError("Original account/channel scope does not match") + return await self.reader.download( + replace(reference, account=self.account), destination, + ) + + +async def verify_restored_telegram_object( + snapshot: Path, receipt_sha256: str, *, + bucket: str, key: str, account: str, chat_id: int, + max_bytes: int = 64 * 1024 * 1024, + transport, +) -> dict: + if type(max_bytes) is not int or not 1 <= max_bytes <= 1024**3: + raise ValueError("Invalid staging DR maximum object size") + if transport is None: + raise ValueError("No authorized Telegram read transport configured") + original_report = verify_snapshot(Path(snapshot), receipt_sha256) + with tempfile.TemporaryDirectory(prefix="teledrive-staging-dr-") as dirname: + root = Path(dirname) + clone = root / "restored-metadata.db" + shutil.copyfile(snapshot, clone) + # Verify the clone's identical bytes before any ORM may add tables. + if hashlib.sha256(clone.read_bytes()).hexdigest() != receipt_sha256.lower(): + raise ValueError("Staging clone differs from authenticated SQL snapshot") + db = connect_database(str(clone)) + try: + manifests = ManifestStore(db) + catalog = BucketCatalog(manifests) + record = catalog.get(bucket, key) + obj = record.item + if obj.size > max_bytes: + raise ValueError("Staging DR sample exceeds explicit size budget") + engine = TelegramObjectStorage( + manifests, transport, scratch_directory=root, + ) + target = root / "verified-telegram-payload" + await engine.download_to_path(obj.id, target) + with target.open("rb") as media: + digest = hashlib.sha256() + read_size = 0 + while block := media.read(1024 * 1024): + digest.update(block) + read_size += len(block) + if digest.hexdigest() != obj.sha256 or read_size != obj.size: + raise ValueError("Staging DR Telegram payload checksum mismatch") + # Independently cross-check a half-open byte window across a + # retrieved chunk boundary, not only the whole file SHA. + start = max(0, obj.size // 2 - 17) + end = min(obj.size, start + 37) + observed = b"".join([ + piece async for piece in engine.iter_bytes( + obj.id, start=start, end=end, block_size=7, + ) + ]) + with target.open("rb") as file: + file.seek(start) + expected = file.read(end - start) + if observed != expected: + raise ValueError("Staging DR Telegram Range read differs from verified file") + return { + "verified": True, "archive_sha256": original_report["sha256"], + "object_id": obj.id, "size": obj.size, + "sha256": obj.sha256, "sample_range": [start, end], + "readback_bytes": read_size, "telegram_uploads": 0, + "telegram_deletions": 0, "source_database_modified": False, + } + finally: + db.close() + + +async def open_read_only_user(session_file: Path, api_id: int, + api_hash: str, target: str, chat_id: int): + """Connect an authorized USER session and resolve the exact channel.""" + from telethon import TelegramClient + from telethon.sessions import StringSession + from telethon.tl.types import Channel + + if api_id <= 0 or not api_hash or not target: + raise ValueError("Missing authorized user MTProto configuration") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(session_file, flags) + with os.fdopen(descriptor, "r", encoding="ascii") as stream: + session = stream.read(8193).strip() + if not session or len(session) > 8192: + raise ValueError("Invalid private user StringSession file") + client = TelegramClient( + StringSession(session), api_id, api_hash, + device_model="TeleDrive Storage Client", + app_version="TeleDrive 2.1", + receive_updates=False, flood_sleep_threshold=0, + ) + try: + await client.connect() + identity = await client.get_me() + if identity is None or getattr(identity, "bot", False): + raise ValueError("Staging original-media DR requires a USER session") + if target.lstrip("-").isdigit(): + resolved = None + async for dialog in client.iter_dialogs(limit=5000): + if int(dialog.id) == int(target): + resolved = dialog.entity + break + if resolved is None: + raise FileNotFoundError("Original Telegram channel not in user dialogs") + else: + resolved = await client.get_entity(target) + if not isinstance(resolved, Channel) or not getattr(resolved, "broadcast", False): + raise ValueError("Staging original-media DR requires a broadcast channel") + actual_chat_id = int(await client.get_peer_id(resolved)) + if actual_chat_id != chat_id: + raise ValueError("Original Telegram channel identity mismatch") + return client, resolved + except Exception: + await client.disconnect() + raise + + +async def verify_original_user_channel( + snapshot: Path, receipt_sha256: str, *, bucket: str, key: str, + account: str, chat_id: int, target: str, + session_file: Path, api_id: int, api_hash: str, + max_bytes: int = 64 * 1024 * 1024, +) -> dict: + """Read owner-authorized original Telegram messages without mutations.""" + client, resolved = await open_read_only_user( + session_file, api_id, api_hash, target, chat_id, + ) + try: + reader = ReadOnlyUserChannelTransport( + client, resolved, account=account, chat_id=chat_id, + ) + return await verify_restored_telegram_object( + snapshot, receipt_sha256, bucket=bucket, key=key, + account=account, chat_id=chat_id, max_bytes=max_bytes, + transport=reader, + ) + finally: + await client.disconnect() + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument("--sha256", required=True, help="SHA256 from online backup receipt") + parser.add_argument("--bucket", required=True) + parser.add_argument("--key", required=True) + parser.add_argument("--account", required=True) + parser.add_argument("--chat-id", type=int, required=True) + parser.add_argument("--target", required=True, help="Exact authorized original channel") + parser.add_argument("--max-bytes", type=int, default=64 * 1024 * 1024) + args = parser.parse_args() + try: + api_id = int(os.environ["TELEDRIVE_STORAGE_MTPROTO_API_ID"]) + api_hash = os.environ["TELEDRIVE_STORAGE_MTPROTO_API_HASH"] + session_file = Path(os.environ.get( + "TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE", "config/mtproto-user.session", + )) + result = asyncio.run(verify_original_user_channel( + args.snapshot, args.sha256, bucket=args.bucket, key=args.key, + account=args.account, chat_id=args.chat_id, target=args.target, + session_file=session_file, api_id=api_id, api_hash=api_hash, + max_bytes=args.max_bytes, + )) + except Exception as exc: + parser.exit(1, "User MTProto DR verification failed: " + type(exc).__name__ + "\n") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/s3_large_roundtrip.py b/scripts/ops/s3_large_roundtrip.py new file mode 100644 index 00000000..32e03024 --- /dev/null +++ b/scripts/ops/s3_large_roundtrip.py @@ -0,0 +1,337 @@ +#!/usr/bin/env python3 +"""Large-file S3 round-trip acceptance test. + +Generates ten random files with sizes uniformly chosen between 100 MiB and +4 GiB, records their SHA-256 digests, uploads them through the TeleDrive S3 +gateway, removes the originals, downloads them again and verifies every byte +against the sealed digest. No production data, tokens or media filenames are +recorded; evidence is redacted and written under 0600 permissions. + +Usage (against a running TeleDrive instance with issued S3 credentials): + export TELEDRIVE_S3_ENDPOINT=http://127.0.0.1:9000 + export TELEDRIVE_S3_ACCESS_KEY=... + export TELEDRIVE_S3_SECRET_KEY=... + export TELEDRIVE_LARGE_ROUNDTRIP_DIR=/secure/scratch + python scripts/ops/s3_large_roundtrip.py + +Optional: + --seed N deterministic random sizes/content (default: OS entropy) + --bucket NAME target bucket (default: ci-large-roundtrip-) + --count N number of files (default: 10) + --min-mib N minimum file size in MiB (default: 100) + --max-mib N maximum file size in MiB (default: 4096) + --report PATH redacted evidence JSON (default: /report.json) + --keep keep downloaded files after verification +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import random +import sys +import tempfile +import time +from datetime import datetime, timezone +from pathlib import Path + +CHUNK = 64 * 1024 * 1024 # 64 MiB staging blocks + + +def env(name: str, default: str = "") -> str: + return os.environ.get(name, default).strip() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for block in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def generate_file(path: Path, size: int, rng: random.Random) -> str: + """Write `size` random bytes and return the SHA-256 digest.""" + digest = hashlib.sha256() + remaining = size + with path.open("wb") as handle: + while remaining: + block_size = min(CHUNK, remaining) + block = rng.randbytes(block_size) + handle.write(block) + digest.update(block) + remaining -= block_size + handle.flush() + return digest.hexdigest() + + +def make_client(endpoint: str, access_key: str, secret_key: str): + import boto3 + from botocore.config import Config + + return boto3.client( + "s3", + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 3}, + ), + ) + + +def upload_file(client, bucket: str, key: str, path: Path) -> int: + """Multipart upload for large objects; returns total bytes.""" + import os as _os + + size = path.stat().st_size + if size <= 100 * 1024 * 1024: + with path.open("rb") as handle: + client.put_object(Bucket=bucket, Key=key, Body=handle) + return size + part_size = 16 * 1024 * 1024 # 16 MiB parts + parts = [] + multipart = client.create_multipart_upload(Bucket=bucket, Key=key) + upload_id = multipart["UploadId"] + try: + with path.open("rb") as handle: + part_number = 1 + while True: + data = handle.read(part_size) + if not data: + break + response = client.upload_part( + Bucket=bucket, Key=key, UploadId=upload_id, + PartNumber=part_number, Body=data, + ) + parts.append({"PartNumber": part_number, "ETag": response["ETag"]}) + part_number += 1 + client.complete_multipart_upload( + Bucket=bucket, Key=key, UploadId=upload_id, + MultipartUpload={"Parts": parts}, + ) + except Exception: + client.abort_multipart_upload( + Bucket=bucket, Key=key, UploadId=upload_id, + ) + raise + return size + + +def download_file(client, bucket: str, key: str, destination: Path) -> int: + response = client.get_object(Bucket=bucket, Key=key) + size = 0 + with destination.open("wb") as handle: + for block in iter(lambda: response["Body"].read(1024 * 1024), b""): + handle.write(block) + size += len(block) + return size + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, default=None, + help="Deterministic random seed") + parser.add_argument("--count", type=int, default=10, + help="Number of files (default: 10)") + parser.add_argument("--min-mib", type=int, default=100, + help="Minimum size in MiB") + parser.add_argument("--max-mib", type=int, default=4096, + help="Maximum size in MiB") + parser.add_argument("--bucket", default="", + help="Target bucket (default: derived)") + parser.add_argument("--report", default="", + help="Redacted evidence JSON path") + parser.add_argument("--keep", action="store_true", + help="Keep downloaded files after verification") + args = parser.parse_args() + + endpoint = env("TELEDRIVE_S3_ENDPOINT", "http://127.0.0.1:9000") + access_key = env("TELEDRIVE_S3_ACCESS_KEY") + secret_key = env("TELEDRIVE_S3_SECRET_KEY") + scratch = Path(env("TELEDRIVE_LARGE_ROUNDTRIP_DIR", "/tmp/teledrive-large-roundtrip")) + if not access_key or not secret_key: + print("Missing TELEDRIVE_S3_ACCESS_KEY / TELEDRIVE_S3_SECRET_KEY", + file=sys.stderr) + return 2 + if args.count < 1 or args.count > 100: + print("count must be 1..100", file=sys.stderr) + return 2 + if args.min_mib < 1 or args.max_mib < args.min_mib: + print("invalid min/max MiB", file=sys.stderr) + return 2 + + rng = random.Random(args.seed) + seed_value = args.seed if args.seed is not None else rng.randrange(1 << 32) + rng = random.Random(seed_value) + scratch.mkdir(parents=True, exist_ok=True) + if os.name != "nt": + os.chmod(scratch, 0o700) + + bucket = args.bucket or f"ci-large-roundtrip-{seed_value % 100000}" + report_path = Path(args.report) if args.report else (scratch / "report.json") + originals = scratch / "originals" + restored = scratch / "restored" + originals.mkdir(parents=True, exist_ok=True) + restored.mkdir(parents=True, exist_ok=True) + + print(f"Endpoint: {endpoint}") + print(f"Bucket: {bucket}") + print(f"Seed: {seed_value}") + print(f"Scratch: {scratch}") + + client = make_client(endpoint, access_key, secret_key) + try: + client.create_bucket(Bucket=bucket) + except Exception as exc: + print(f"create_bucket: {type(exc).__name__} (continuing if it exists)") + + sizes = [rng.randint(args.min_mib * 1024 * 1024, + args.max_mib * 1024 * 1024) + for _ in range(args.count)] + + records = [] + total = 0 + for index, size in enumerate(sizes, start=1): + filename = f"large-{index:02d}.bin" + original = originals / filename + digest = generate_file(original, size, rng) + records.append({ + "index": index, + "key": f"roundtrip/{filename}", + "size": size, + "sha256": digest, + "original_path": str(original), + }) + total += size + print(f"[{index:2d}/{args.count}] generated {size / (1024**3):.3f} GiB " + f"sha256={digest[:12]}...", flush=True) + + evidence = { + "run_utc": datetime.now(timezone.utc).isoformat(), + "seed": seed_value, + "endpoint": endpoint, + "bucket": bucket, + "count": args.count, + "total_bytes": total, + "files": [], + "uploads": 0, + "downloads": 0, + "verified": 0, + "failures": [], + "garbage_collection_enabled": False, + } + + # Upload phase. + print("\n=== UPLOAD ===") + for record in records: + key = record["key"] + original = Path(record["original_path"]) + started = time.time() + try: + upload_file(client, bucket, key, original) + record["uploaded"] = True + evidence["uploads"] += 1 + print(f" [{record['index']:2d}] uploaded {key} " + f"({time.time() - started:.1f}s)", flush=True) + except Exception as exc: + record["uploaded"] = False + evidence["failures"].append({ + "phase": "upload", "key": key, + "error": type(exc).__name__, + }) + print(f" [{record['index']:2d}] UPLOAD FAILED: {type(exc).__name__}", + file=sys.stderr, flush=True) + + # Delete originals. + print("\n=== DELETE ORIGINALS ===") + for record in records: + if record.get("uploaded"): + Path(record["original_path"]).unlink(missing_ok=True) + print(" originals removed from local disk") + + # Download + verify phase. + print("\n=== DOWNLOAD + VERIFY ===") + for record in records: + key = record["key"] + if not record.get("uploaded"): + evidence["files"].append({ + "index": record["index"], "size": record["size"], + "uploaded": False, "verified": False, + }) + continue + destination = restored / key.split("/")[-1] + destination.parent.mkdir(parents=True, exist_ok=True) + started = time.time() + try: + size = download_file(client, bucket, key, destination) + actual = sha256_file(destination) + ok = (size == record["size"] and actual == record["sha256"]) + record["downloaded"] = True + record["downloaded_size"] = size + record["downloaded_sha256"] = actual + record["verified"] = ok + if ok: + evidence["verified"] += 1 + evidence["downloads"] += 1 + print(f" [{record['index']:2d}] verified {key} " + f"({time.time() - started:.1f}s)", flush=True) + else: + evidence["failures"].append({ + "phase": "verify", "key": key, + "expected_size": record["size"], "actual_size": size, + "expected_sha256": record["sha256"], + "actual_sha256": actual, + }) + print(f" [{record['index']:2d}] MISMATCH {key}", + file=sys.stderr, flush=True) + if not args.keep: + destination.unlink(missing_ok=True) + except Exception as exc: + record["downloaded"] = False + evidence["failures"].append({ + "phase": "download", "key": key, + "error": type(exc).__name__, + }) + print(f" [{record['index']:2d}] DOWNLOAD FAILED: {type(exc).__name__}", + file=sys.stderr, flush=True) + + for record in records: + evidence["files"].append({ + "index": record["index"], + "size": record["size"], + "sha256": record["sha256"], + "uploaded": bool(record.get("uploaded")), + "downloaded": bool(record.get("downloaded")), + "verified": bool(record.get("verified")), + }) + + with open(report_path, "w", encoding="utf-8") as handle: + json.dump(evidence, handle, indent=2, sort_keys=True) + handle.write("\n") + if os.name != "nt": + os.chmod(report_path, 0o600) + + print("\n=== SUMMARY ===") + print(f" files: {args.count}") + print(f" uploads: {evidence['uploads']}") + print(f" downloads: {evidence['downloads']}") + print(f" verified: {evidence['verified']}") + print(f" failures: {len(evidence['failures'])}") + print(f" total bytes: {total}") + print(f" report: {report_path}") + print(f" garbage_collection_enabled: false") + + if evidence["failures"] or evidence["verified"] != args.count: + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/verify_legacy_cutover.py b/scripts/ops/verify_legacy_cutover.py new file mode 100644 index 00000000..852709be --- /dev/null +++ b/scripts/ops/verify_legacy_cutover.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Verify every eligible legacy source against finished SQL jobs and Telegram. + +This command neither queues nor uploads. It is for an operator who has +already disabled the old uploader, quiesced the source tree and enabled +an approved durable cutover. Exits nonzero if ANY eligible file is missing, +unpublished, changed or not independently readable from Telegram. +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import os +from pathlib import Path +from types import SimpleNamespace + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.cutover_acceptance import verify_legacy_cutover +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore + + +async def run(args) -> dict: + from scripts.ops.rehearse_telegram_dr import ( + ReadOnlyUserChannelTransport, open_read_only_user, + ) + api_id = int(os.environ["TELEDRIVE_STORAGE_MTPROTO_API_ID"]) + api_hash = os.environ["TELEDRIVE_STORAGE_MTPROTO_API_HASH"] + session_file = Path(os.environ.get( + "TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE", "config/mtproto-user.session", + )) + db = connect_database(args.database) + try: + cfg = SimpleNamespace( + source_dir=str(args.source_root), + allowed_extensions=[s.lower() for s in args.extensions.split(",") if s], + max_file_size_mb=args.max_file_size_mb, + ) + queue = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + client, resolved = await open_read_only_user( + session_file, api_id, api_hash, args.target, args.chat_id, + ) + try: + transport = ReadOnlyUserChannelTransport( + client, resolved, account=args.account, chat_id=args.chat_id, + ) + return await verify_legacy_cutover( + cfg, queue, transport, bucket=args.bucket, prefix=args.prefix, + approved_root=args.allowed_root, max_files=args.max_files, + max_bytes=args.max_bytes, scratch_directory=args.scratch_dir, + ) + finally: + await client.disconnect() + finally: + db.close() + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--database", required=True) + p.add_argument("--source-root", type=Path, required=True) + p.add_argument("--allowed-root", type=Path, required=True) + p.add_argument("--scratch-dir", type=Path, required=True) + p.add_argument("--bucket", required=True) + p.add_argument("--prefix", default="") + p.add_argument("--account", required=True) + p.add_argument("--chat-id", type=int, required=True) + p.add_argument("--target", required=True, + help="Exact authorized original channel") + p.add_argument("--extensions", default="") + p.add_argument("--max-file-size-mb", type=int, default=0) + p.add_argument("--max-files", type=int, default=1000) + p.add_argument("--max-bytes", type=int, default=64 * 1024 * 1024) + args = p.parse_args() + try: + result = asyncio.run(run(args)) + except Exception as exc: + # Do not leak local source paths, SQL credentials or Telegram URLs. + p.exit(1, "Cutover verification refused: " + type(exc).__name__ + "\n") + print(json.dumps(result, sort_keys=True)) + return 0 if result["verified"] else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/verify_metadata_backup.py b/scripts/ops/verify_metadata_backup.py new file mode 100644 index 00000000..cb24983d --- /dev/null +++ b/scripts/ops/verify_metadata_backup.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +"""Read-only check of SQLite metadata snapshots before a staging restore.""" +from __future__ import annotations + +import argparse +import hashlib +import hmac +import json +import re +import sqlite3 +from pathlib import Path +from urllib.parse import quote + + +class InvalidSnapshot(RuntimeError): + pass + + +REQUIRED = ("storage_objects", "storage_chunks", "storage_object_heads", + "storage_object_aliases", "storage_published_versions", + "storage_removed_versions") + + +def verify_snapshot(path: Path, expected_sha256: str | None = None) -> dict: + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise InvalidSnapshot("Snapshot must be an existing regular file") + digest = hashlib.sha256() + with path.open("rb") as stream: + while block := stream.read(1024 * 1024): + digest.update(block) + sha = digest.hexdigest() + if expected_sha256 is not None: + if not re.fullmatch(r"[0-9a-fA-F]{64}", expected_sha256): + raise InvalidSnapshot("Expected SHA-256 must be 64 hex digits") + if not hmac.compare_digest(sha, expected_sha256.lower()): + raise InvalidSnapshot("Snapshot SHA-256 receipt mismatch") + try: + uri = "file:" + quote(str(path.resolve(strict=True)), safe="/") + "?mode=ro" + with sqlite3.connect(uri, uri=True, timeout=30) as db: + if db.execute("PRAGMA integrity_check").fetchone() != ("ok",): + raise InvalidSnapshot("SQLite integrity_check failed") + if db.execute("PRAGMA foreign_key_check").fetchone(): + raise InvalidSnapshot("Snapshot has dangling foreign keys") + tables = {r[0] for r in db.execute( + "SELECT name FROM sqlite_master WHERE type='table'")} + if set(REQUIRED) - tables: + raise InvalidSnapshot("TeleDrive storage schema is incomplete") + for key_hash, key, bucket, original_bucket, status in db.execute( + "SELECT h.key_hash,o.object_key,h.bucket,o.bucket,o.status " + "FROM storage_object_heads h LEFT JOIN storage_objects o " + "ON o.id=h.object_id" + ): + if (status not in ("ready", "delete-marker") or original_bucket != bucket + or hashlib.sha256(key.encode()).hexdigest() != key_hash): + raise InvalidSnapshot("Current HEAD manifest is inconsistent") + for key_hash, key, bucket, original_bucket, status in db.execute( + "SELECT v.key_hash,o.object_key,v.bucket,o.bucket,o.status " + "FROM storage_published_versions v LEFT JOIN storage_objects o " + "ON o.id=v.object_id" + ): + if (status not in ("ready", "delete-marker") or original_bucket != bucket + or hashlib.sha256(key.encode()).hexdigest() != key_hash): + raise InvalidSnapshot("Published version metadata is inconsistent") + if db.execute( + "SELECT 1 FROM storage_removed_versions r " + "LEFT JOIN storage_published_versions p ON p.object_id=r.object_id " + "WHERE p.object_id IS NULL LIMIT 1" + ).fetchone(): + raise InvalidSnapshot("Removed VersionId lacks published history") + if db.execute( + "SELECT 1 FROM storage_object_heads h " + "JOIN storage_removed_versions r ON r.object_id=h.object_id LIMIT 1" + ).fetchone(): + raise InvalidSnapshot("Current HEAD points to a revoked version") + def count(table): + return int(db.execute("SELECT COUNT(*) FROM " + table).fetchone()[0]) + summary = { + "valid": True, "sha256": sha, + "objects": count("storage_objects"), + "heads": count("storage_object_heads"), + "versions": count("storage_published_versions"), + "removed_versions": count("storage_removed_versions"), + "telegram_chunk_refs": count("storage_chunks"), + "physical_deletion_performed": False, + } + return summary + except sqlite3.DatabaseError as exc: + raise InvalidSnapshot("Unreadable or corrupt SQLite image") from exc + + +def main() -> int: + parser = argparse.ArgumentParser(description="Verify a staging SQLite snapshot") + parser.add_argument("--database", type=Path, required=True) + parser.add_argument("--sha256", help="Expected SHA-256 from backup receipt") + args = parser.parse_args() + try: + print(json.dumps(verify_snapshot(args.database, args.sha256), sort_keys=True)) + except (InvalidSnapshot, OSError) as exc: + parser.exit(1, "Invalid snapshot: " + str(exc) + "\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ops/verify_production_bundle.py b/scripts/ops/verify_production_bundle.py new file mode 100644 index 00000000..91f306e8 --- /dev/null +++ b/scripts/ops/verify_production_bundle.py @@ -0,0 +1,160 @@ +"""Offline, dual-attested production evidence bundle verifier. + +This validates receipts and operator signatures; it does NOT poll production +hosts or assert an installation exists. Do not sign off a deployment without +independently verifying original sources and machine ownership. +""" +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import re +from pathlib import Path + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + +_SHA256 = re.compile(r"[0-9a-f]{64}\Z") +_COMMIT = re.compile(r"[0-9a-f]{40}\Z") + + +def canonical_payload(bundle: dict) -> bytes: + evidence = dict(bundle) + evidence.pop("approvals", None) + return json.dumps(evidence, sort_keys=True, separators=(",", ":"), + ensure_ascii=True, allow_nan=False).encode("ascii") + + +def verify_bundle(bundle: dict, *, expected_sha: str, + owner_public_key: bytes, operator_public_key: bytes) -> dict: + if not isinstance(bundle, dict) or not _COMMIT.fullmatch(expected_sha): + raise ValueError("Invalid production release identity") + if bundle.get("release_sha") != expected_sha: + raise ValueError("Receipts do not match the reviewed release source") + if owner_public_key == operator_public_key or len(owner_public_key) != 32 or len(operator_public_key) != 32: + raise ValueError("Independent owner and operator keys are required") + if not {"soak", "dr", "cutover", "topology", "approvals"}.issubset(bundle): + raise ValueError("Missing production evidence family") + + topology = bundle["topology"] + if not isinstance(topology, dict) or not isinstance(topology.get("hosts"), list): + raise ValueError("Physical deployment inventory absent") + hosts = topology["hosts"] + if not 2 <= len(hosts) <= 8: + raise ValueError("At least two independently attested hosts required") + labels, machines, dbs = set(), set(), set() + for host in hosts: + if not isinstance(host, dict) or set(host) != { + "label", "machine_id_sha256", "shared_database_sha256", + }: + raise ValueError("Invalid independent host evidence") + if (not isinstance(host["label"], str) + or not re.fullmatch(r"[A-Za-z0-9_-]{1,32}", host["label"]) + or any(not isinstance(host[k], str) or not _SHA256.fullmatch(host[k]) + for k in ("machine_id_sha256", "shared_database_sha256"))): + raise ValueError("Malformed physical deployment identifiers") + labels.add(host["label"]) + machines.add(host["machine_id_sha256"]) + dbs.add(host["shared_database_sha256"]) + if len(labels) != len(hosts) or len(machines) != len(hosts) or len(dbs) != 1: + raise ValueError("Physical hosts must differ and share one SQL database") + if topology.get("independent_hosts_owner_attested") is not True: + raise ValueError("Original host ownership has not been attested") + + soak = bundle["soak"] + if (not isinstance(soak, dict) or soak.get("telemetry_gate_passed") is not True + or type(soak.get("duration_seconds")) is not int + or soak["duration_seconds"] < 3600 + or soak.get("final_shared_counter_agreement") is not True + or soak.get("telegram_mutations_performed") != 0 + or set(soak.get("host_labels", [])) != labels + or not isinstance(soak.get("explicit_429_deltas_by_host"), dict) + or set(soak["explicit_429_deltas_by_host"]) != labels + or any(type(delta) is not int or delta <= 0 + for delta in soak["explicit_429_deltas_by_host"].values())): + raise ValueError("Missing independent one-hour real 429 soak evidence") + + dr = bundle["dr"] + if (not isinstance(dr, dict) or dr.get("verified") is not True + or dr.get("source_database_modified") is not False + or dr.get("telegram_uploads") != 0 or dr.get("telegram_deletions") != 0 + or type(dr.get("size")) is not int or dr["size"] < 0 + or dr.get("readback_bytes") != dr["size"] + or not isinstance(dr.get("sha256"), str) + or not _SHA256.fullmatch(dr["sha256"]) + or not isinstance(dr.get("archive_sha256"), str) + or not _SHA256.fullmatch(dr["archive_sha256"]) + or bundle.get("original_telegram_data_attested") is not True): + raise ValueError("Missing verified original-media DR receipt") + + cutover = bundle["cutover"] + if not isinstance(cutover, dict): + raise ValueError("Cutover evidence missing") + if cutover.get("not_applicable") is True: + if bundle.get("no_legacy_source_attested") is not True: + raise ValueError("No-legacy exemption must be independently attested") + elif not ( + cutover.get("verified") is True + and type(cutover.get("eligible_files")) is int + and cutover["eligible_files"] > 0 + and cutover.get("verified_files") == cutover["eligible_files"] + and cutover.get("not_yet_published") == 0 + and cutover.get("inconsistent") == 0 + and cutover.get("truncated") is False + and cutover.get("telegram_uploads") == 0 + and cutover.get("telegram_deletions") == 0 + ): + raise ValueError("Cutover is incomplete or ownership not approved") + + approval = bundle["approvals"] + if not isinstance(approval, dict) or set(approval) != {"owner", "operator"}: + raise ValueError("Independent production approvals missing") + signed_message = canonical_payload(bundle) + for role, key_bytes in (("owner", owner_public_key), ("operator", operator_public_key)): + try: + encoded = approval[role] + if not isinstance(encoded, str): + raise ValueError("Invalid approval type") + signature = base64.b64decode(encoded, validate=True) + Ed25519PublicKey.from_public_bytes(key_bytes).verify(signature, signed_message) + except (ValueError, TypeError, Exception) as exc: + # Deliberately suppress key contents and signing details. + raise ValueError("Invalid " + role + " evidence signature") from exc + + return { + "technical_evidence_verified": True, + "two_independent_attestations_verified": True, + "release_sha": expected_sha, + "bundle_sha256": hashlib.sha256(signed_message).hexdigest(), + # Real deployment remains an owner-controlled operational decision, + # never silently changed by a machine-readable proof. + "production_acceptance_signed_off": False, + "physical_telegram_deletion_enabled": False, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bundle", required=True, type=Path) + parser.add_argument("--release-sha", required=True) + parser.add_argument("--owner-public-key", required=True, type=Path, + help="Raw 32-byte Ed25519 public key") + parser.add_argument("--operator-public-key", required=True, type=Path) + args = parser.parse_args() + try: + result = verify_bundle( + json.loads(args.bundle.read_text("utf-8")), + expected_sha=args.release_sha, + owner_public_key=args.owner_public_key.read_bytes(), + operator_public_key=args.operator_public_key.read_bytes(), + ) + except Exception as exc: + parser.exit(1, "Production evidence verification refused: " + + type(exc).__name__ + "\\n") + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/teledrive/__init__.py b/teledrive/__init__.py new file mode 100644 index 00000000..afc50f81 --- /dev/null +++ b/teledrive/__init__.py @@ -0,0 +1 @@ +"""TeleDrive core: state, Telegram uploads and desktop launcher.""" diff --git a/teledrive/api_keys.py b/teledrive/api_keys.py new file mode 100644 index 00000000..9e2df928 --- /dev/null +++ b/teledrive/api_keys.py @@ -0,0 +1,151 @@ +"""High-entropy REST API keys with SQL-backed scopes and atomic revocation. + +No plaintext secrets are stored. The token appears exactly once on creation +or rotation; only its SHA-256 digest and non-secret prefix are persisted. +""" +from __future__ import annotations + +import hashlib +import json +import secrets +import time +from dataclasses import dataclass + +from sqlalchemy import BigInteger, Boolean, Column, MetaData, String, Table, Text, delete, insert, select, update +from sqlalchemy.exc import IntegrityError + +from teledrive.db import Database + +_metadata = MetaData() +api_keys = Table( + "api_credentials", _metadata, + Column("id", String(36), primary_key=True), + Column("digest", String(64), nullable=False, unique=True), + Column("prefix", String(24), nullable=False), + Column("name", String(120), nullable=False), + Column("scopes", Text, nullable=False), + Column("created_at", BigInteger, nullable=False), + Column("expires_at", BigInteger), + Column("last_used_at", BigInteger), + Column("revoked", Boolean, nullable=False), +) +ALLOWED_SCOPES = frozenset({ + "objects:read", "objects:write", "objects:delete", + "buckets:read", "buckets:manage", + "jobs:read", "jobs:manage", "accounts:read", + "accounts:manage", "settings:read", "settings:manage", + "keys:manage", "admin:*", +}) + + +@dataclass(frozen=True) +class IssuedKey: + id: str + value: str + prefix: str + scopes: tuple[str, ...] + + +class ApiKeyStore: + def __init__(self, db: Database): + self.db = db + _metadata.create_all(db.engine) + + @staticmethod + def _check_scopes(scopes: list[str]) -> tuple[str, ...]: + if not isinstance(scopes, list) or not scopes or any( + not isinstance(s, str) or s not in ALLOWED_SCOPES for s in scopes + ): + raise ValueError("Unknown or empty permission scope") + normalized = tuple(sorted(set(scopes))) + if "admin:*" in normalized and len(normalized) != 1: + raise ValueError("admin:* cannot be combined with other scopes") + return normalized + + @staticmethod + def _digest(token: str) -> str: + return hashlib.sha256(token.encode("utf-8")).hexdigest() + + def create(self, name: str, scopes: list[str], *, expires_at: int | None = None, + now: int | None = None) -> IssuedKey: + now = int(time.time()) if now is None else now + if not isinstance(name, str) or not (1 <= len(name.strip()) <= 120): + raise ValueError("Invalid API key name") + permissions = self._check_scopes(scopes) + if expires_at is not None and (expires_at <= now or expires_at > now + 10 * 365 * 86400): + raise ValueError("API key expiration must be in the future (max 10 years)") + identifier = secrets.token_hex(16) + token = "td_live_" + secrets.token_urlsafe(32) + prefix = token[:16] + self.db.connection.execute(insert(api_keys).values( + id=identifier, digest=self._digest(token), prefix=prefix, + name=name.strip(), scopes=json.dumps(permissions), + created_at=now, expires_at=expires_at, last_used_at=None, revoked=False, + )) + self.db.commit() + return IssuedKey(identifier, token, prefix, permissions) + + def list(self) -> list[dict]: + rows = self.db.connection.execute( + select(api_keys.c.id, api_keys.c.prefix, api_keys.c.name, + api_keys.c.scopes, api_keys.c.created_at, api_keys.c.expires_at, + api_keys.c.last_used_at, api_keys.c.revoked) + .order_by(api_keys.c.revoked.asc(), api_keys.c.created_at.desc(), api_keys.c.id) + ).mappings().all() + return [{**dict(row), "scopes": json.loads(row["scopes"])} for row in rows] + + def authorize(self, token: str, scope: str, *, now: int | None = None) -> bool: + now = int(time.time()) if now is None else now + if (not isinstance(token, str) or len(token) > 200 + or not token.startswith("td_live_")): + return False + result = self.db.connection.execute( + select(api_keys.c.id, api_keys.c.scopes, api_keys.c.revoked, api_keys.c.expires_at) + .where(api_keys.c.digest == self._digest(token)) + ).first() + if not result or result[2] or (result[3] is not None and result[3] <= now): + return False + scopes = json.loads(result[1]) + if scope not in scopes and "admin:*" not in scopes: + return False + self.db.connection.execute( + update(api_keys).where(api_keys.c.id == result[0]).values(last_used_at=now) + ) + self.db.commit() + return True + + def revoke(self, identifier: str) -> bool: + changed = self.db.connection.execute( + update(api_keys).where(api_keys.c.id == identifier, api_keys.c.revoked.is_(False)) + .values(revoked=True) + ) + self.db.commit() + return changed.rowcount == 1 + + def rotate(self, identifier: str, *, now: int | None = None) -> IssuedKey: + now = int(time.time()) if now is None else now + old = self.db.connection.execute( + select(api_keys.c.name, api_keys.c.scopes, api_keys.c.expires_at) + .where(api_keys.c.id == identifier, api_keys.c.revoked.is_(False)) + ).first() + if old is None: + raise FileNotFoundError("API key not found or revoked") + if old[2] is not None and old[2] <= now: + raise ValueError("Cannot rotate expired key") + permissions = json.loads(old[1]) + # Keep creation+revoke in one transaction, so no partial rotation. + token = "td_live_" + secrets.token_urlsafe(32) + new_id = secrets.token_hex(16) + changed = self.db.connection.execute(update(api_keys).where( + api_keys.c.id == identifier, api_keys.c.revoked.is_(False) + ).values(revoked=True)) + if changed.rowcount != 1: + self.db.connection.rollback() + raise ValueError("Concurrent key revocation") + self.db.connection.execute(insert(api_keys).values( + id=new_id, digest=self._digest(token), prefix=token[:16], + name=old[0], scopes=old[1], created_at=now, + expires_at=old[2], last_used_at=None, revoked=False, + )) + self.db.commit() + return IssuedKey(new_id, token, token[:16], tuple(permissions)) diff --git a/teledrive/db.py b/teledrive/db.py new file mode 100644 index 00000000..b6925b73 --- /dev/null +++ b/teledrive/db.py @@ -0,0 +1,373 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +from sqlalchemy import ( + BigInteger, Column, Float, Index, Integer, MetaData, String, Table, Text, + and_, create_engine, delete, func, insert, inspect, or_, select, update, +) +from sqlalchemy.engine import Connection, Engine, Row +from sqlalchemy.engine.url import make_url +from sqlalchemy.exc import IntegrityError, NoSuchModuleError +from sqlalchemy.schema import CreateColumn + +metadata = MetaData() + +files = Table( + "files", metadata, + Column("path", String(700), primary_key=True), + Column("size", BigInteger, nullable=False), + Column("mtime", Float, nullable=False), + Column("status", String(32), nullable=False, server_default="pending"), + Column("tg_message_id", BigInteger), + Column("tg_account", String(255)), + Column("attempts", Integer, nullable=False, server_default="0"), + Column("last_error", Text), + Column("first_seen_ts", BigInteger, nullable=False), + Column("last_update_ts", BigInteger, nullable=False), + Column("uploaded_ts", BigInteger), + Column("claim_token", String(255)), + Column("claim_ts", BigInteger), +) +Index("idx_files_status", files.c.status) +Index("idx_files_tg_account", files.c.tg_account) + +account_status = Table( + "account_status", metadata, + Column("name", String(255), primary_key=True), + Column("state", String(32), nullable=False, server_default="unknown"), + Column("last_error", Text), + Column("cooldown_until", BigInteger), + Column("last_success_ts", BigInteger), + Column("last_update_ts", BigInteger, nullable=False), +) + +leases = Table( + "leases", metadata, + Column("name", String(128), primary_key=True), + Column("holder", String(255), nullable=False), + Column("lease_until", BigInteger, nullable=False), + Column("last_update_ts", BigInteger, nullable=False), +) + + +class DBRow: + def __init__(self, row: Row[Any]): + self._mapping = dict(row._mapping) + self._values = tuple(self._mapping.values()) + + def __getitem__(self, key): + return self._values[key] if isinstance(key, int) else self._mapping[key] + + def as_dict(self) -> dict[str, Any]: + return dict(self._mapping) + + +def normalize_database_url(value: str, *, config_dir: Path | None = None) -> str: + """Accept any synchronous SQLAlchemy URL; local paths become SQLite URLs.""" + raw = str(value or "").strip() + if not raw: + raise ValueError("Database URL/path must not be empty") + + if "://" not in raw: + path = Path(raw).expanduser() + if not path.is_absolute() and config_dir is not None: + path = config_dir / path + return f"sqlite:///{path.resolve().as_posix()}" + + # Convenience aliases only. Explicit dialect+driver URLs pass through unchanged. + if raw.startswith("postgres://"): + raw = "postgresql+psycopg://" + raw[len("postgres://"):] + elif raw.startswith("postgresql://"): + raw = "postgresql+psycopg://" + raw[len("postgresql://"):] + elif raw.startswith("mysql://"): + raw = "mysql+pymysql://" + raw[len("mysql://"):] + + if raw.startswith("sqlite:///") and not raw.startswith("sqlite:////") and raw != "sqlite:///:memory:" and config_dir: + rel = raw[len("sqlite:///"):] + return f"sqlite:///{(config_dir / rel).expanduser().resolve().as_posix()}" + return raw + + +def sqlite_database_path(url: str) -> str | None: + """Return the filesystem path for a SQLite URL, otherwise None.""" + try: + parsed = make_url(normalize_database_url(url)) + except Exception: + return None + if parsed.get_backend_name() != "sqlite": + return None + database = parsed.database + if not database or database == ":memory:": + return None + return str(Path(database).expanduser().resolve()) + + +def redact_database_url(url: str) -> str: + """Render a URL without userinfo passwords or sensitive query values.""" + try: + rendered = make_url(normalize_database_url(url)).render_as_string(hide_password=True) + parts = urlsplit(rendered) + sensitive = {"password", "passwd", "pwd", "token", "access_token", "secret", "api_key", "apikey"} + query = [ + (key, "***" if key.lower() in sensitive else value) + for key, value in parse_qsl(parts.query, keep_blank_values=True) + ] + return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(query), parts.fragment)) + except Exception: + return "" + + +@dataclass +class Database: + url: str + engine: Engine + connection: Connection + + def close(self) -> None: + try: + self.connection.close() + finally: + self.engine.dispose() + + def commit(self) -> None: + self.connection.commit() + + def file_signature(self, path: str) -> DBRow | None: + row = self.connection.execute( + select(files.c.size, files.c.mtime, files.c.status).where(files.c.path == path) + ).first() + return DBRow(row) if row else None + + def insert_pending(self, path: str, size: int, mtime: float, now: int) -> None: + self.connection.execute(insert(files).values( + path=path, size=size, mtime=mtime, status="pending", + first_seen_ts=now, last_update_ts=now, + )) + + def requeue_changed(self, path: str, size: int, mtime: float, now: int) -> None: + self.connection.execute(update(files).where(files.c.path == path).values( + size=size, mtime=mtime, status="pending", tg_message_id=None, + tg_account=None, uploaded_ts=None, last_error=None, + claim_token=None, claim_ts=None, last_update_ts=now, + )) + + def uploaded_between(self, start_ts: int, end_ts: int) -> int: + return int(self.connection.execute( + select(func.count()).select_from(files).where( + files.c.status == "uploaded", + files.c.uploaded_ts >= start_ts, + files.c.uploaded_ts < end_ts, + ) + ).scalar_one() or 0) + + def pending(self, limit: int, now: int, stale_after_seconds: int) -> list[DBRow]: + stale_before = now - stale_after_seconds + rows = self.connection.execute( + select(files.c.path, files.c.size, files.c.mtime, files.c.attempts) + .where( + or_( + files.c.status.in_(["pending", "failed"]), + and_( + files.c.status == "uploading", + or_(files.c.claim_ts.is_(None), files.c.claim_ts < stale_before), + ), + ) + ) + .order_by(files.c.first_seen_ts.asc()).limit(limit) + ).all() + return [DBRow(row) for row in rows] + + def claim_file(self, path: str, claim_token: str, now: int, stale_after_seconds: int) -> bool: + stale_before = now - stale_after_seconds + self.connection.execute( + update(files) + .where( + files.c.path == path, + or_( + files.c.status.in_(["pending", "failed"]), + and_( + files.c.status == "uploading", + or_(files.c.claim_ts.is_(None), files.c.claim_ts < stale_before), + ), + ), + ) + .values( + status="uploading", + claim_token=claim_token, + claim_ts=now, + last_update_ts=now, + ) + ) + self.connection.commit() + row = self.connection.execute( + select(files.c.status, files.c.claim_token).where(files.c.path == path) + ).first() + return bool(row and row[0] == "uploading" and row[1] == claim_token) + + def mark_failed(self, path: str, error: str, attempts_inc: int, now: int) -> None: + self.connection.execute(update(files).where(files.c.path == path).values( + status="failed", attempts=files.c.attempts + attempts_inc, + last_error=error[:1000], claim_token=None, claim_ts=None, last_update_ts=now, + )) + + def mark_uploaded(self, path: str, message_id: int | None, account_name: str, now: int) -> None: + self.connection.execute(update(files).where(files.c.path == path).values( + status="uploaded", tg_message_id=message_id, tg_account=account_name, + attempts=files.c.attempts + 1, last_error=None, + claim_token=None, claim_ts=None, uploaded_ts=now, last_update_ts=now, + )) + + def account_state(self, name: str) -> DBRow | None: + row = self.connection.execute( + select(account_status).where(account_status.c.name == name) + ).first() + return DBRow(row) if row else None + + def set_account_state(self, name: str, state: str, *, last_error: str | None, + cooldown_until: int | None, last_success_ts: int | None, now: int) -> None: + values = dict( + state=state, last_error=last_error, cooldown_until=cooldown_until, + last_success_ts=last_success_ts, last_update_ts=now, + ) + result = self.connection.execute( + update(account_status).where(account_status.c.name == name).values(**values) + ) + if result.rowcount: + return + try: + self.connection.execute(insert(account_status).values(name=name, **values)) + except IntegrityError: + self.connection.rollback() + self.connection.execute( + update(account_status).where(account_status.c.name == name).values(**values) + ) + + def acquire_lease(self, name: str, holder: str, now: int, ttl_seconds: int) -> bool: + lease_until = now + ttl_seconds + result = self.connection.execute( + update(leases) + .where( + leases.c.name == name, + (leases.c.holder == holder) | (leases.c.lease_until < now), + ) + .values(holder=holder, lease_until=lease_until, last_update_ts=now) + ) + if result.rowcount: + self.connection.commit() + row = self.connection.execute( + select(leases.c.holder, leases.c.lease_until).where(leases.c.name == name) + ).first() + owned = bool(row and row[0] == holder and int(row[1]) >= now) + # Close read snapshot before another SQL connection writes jobs. + self.connection.commit() + return owned + try: + self.connection.execute(insert(leases).values( + name=name, holder=holder, lease_until=lease_until, last_update_ts=now, + )) + self.connection.commit() + return True + except IntegrityError: + self.connection.rollback() + return False + + def renew_lease(self, name: str, holder: str, now: int, ttl_seconds: int) -> bool: + result = self.connection.execute( + update(leases) + .where(leases.c.name == name, leases.c.holder == holder, leases.c.lease_until >= now) + .values(lease_until=now + ttl_seconds, last_update_ts=now) + ) + self.connection.commit() + if not result.rowcount: + self.connection.rollback() + return False + row = self.connection.execute( + select(leases.c.holder, leases.c.lease_until).where(leases.c.name == name) + ).first() + owned = bool(row and row[0] == holder and int(row[1]) >= now) + self.connection.commit() + return owned + + def release_lease(self, name: str, holder: str) -> None: + self.connection.execute( + delete(leases).where(leases.c.name == name, leases.c.holder == holder) + ) + self.connection.commit() + + def dashboard_counts(self) -> dict[str, int]: + rows = self.connection.execute( + select(files.c.status, func.count()).group_by(files.c.status) + ).all() + return {str(row[0]): int(row[1]) for row in rows} + + def all_account_states(self) -> dict[str, dict[str, Any]]: + rows = self.connection.execute(select(account_status).order_by(account_status.c.name)).all() + return {str(row._mapping["name"]): dict(row._mapping) for row in rows} + + def uploaded_counts_by_account(self) -> dict[str, int]: + label = func.coalesce(files.c.tg_account, "unknown").label("account") + rows = self.connection.execute( + select(label, func.count()).where(files.c.status == "uploaded").group_by(label) + ).all() + return {str(row[0]): int(row[1]) for row in rows} + + def list_files(self, kind: str, limit: int) -> list[dict[str, Any]]: + condition = files.c.status.in_(["pending", "failed", "uploading"]) if kind == "queue" else files.c.status == "uploaded" + rows = self.connection.execute( + select( + files.c.path, files.c.size, files.c.status, files.c.attempts, + files.c.last_error, files.c.tg_account, files.c.tg_message_id, + files.c.first_seen_ts, files.c.last_update_ts, files.c.uploaded_ts, + ).where(condition).order_by(files.c.last_update_ts.desc()).limit(limit) + ).all() + return [dict(row._mapping) for row in rows] + + def clear_files(self) -> None: + self.connection.execute(delete(files)) + + +def connect_database(url_or_path: str, *, config_dir: Path | None = None) -> Database: + url = normalize_database_url(url_or_path, config_dir=config_dir) + connect_args: dict[str, Any] = {} + + if url.startswith("sqlite:"): + connect_args["timeout"] = 30 + if url != "sqlite:///:memory:": + db_path = url[len("sqlite:///"):] + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + + try: + engine = create_engine(url, pool_pre_ping=True, connect_args=connect_args) + except (NoSuchModuleError, ModuleNotFoundError) as exc: + raise RuntimeError( + "Database dialect/driver is not installed. Install the SQLAlchemy dialect/DBAPI " + "required by the configured database_url." + ) from exc + + metadata.create_all(engine) + + # Dialect-aware migrations for older TeleDrive databases. + columns = {c["name"] for c in inspect(engine).get_columns("files")} + migrations = [ + Column("tg_account", String(255)), + Column("claim_token", String(255)), + Column("claim_ts", BigInteger), + ] + missing = [column for column in migrations if column.name not in columns] + if missing: + preparer = engine.dialect.identifier_preparer + table_name = preparer.quote("files") + with engine.begin() as conn: + for column in missing: + rendered = str(CreateColumn(column).compile(dialect=engine.dialect)) + conn.exec_driver_sql(f"ALTER TABLE {table_name} ADD {rendered}") + + connection = engine.connect() + if engine.url.get_backend_name() == "sqlite": + connection.exec_driver_sql("PRAGMA busy_timeout=30000") + connection.exec_driver_sql("PRAGMA journal_mode=WAL") + return Database(url=url, engine=engine, connection=connection) diff --git a/teledrive/s3/__init__.py b/teledrive/s3/__init__.py new file mode 100644 index 00000000..fcc2dd26 --- /dev/null +++ b/teledrive/s3/__init__.py @@ -0,0 +1 @@ +"""Telegram-backed S3 compatibility gateway.""" diff --git a/teledrive/s3/batch_delete.py b/teledrive/s3/batch_delete.py new file mode 100644 index 00000000..d66c0f19 --- /dev/null +++ b/teledrive/s3/batch_delete.py @@ -0,0 +1,119 @@ +"""Strict S3 DeleteObjects request validation. + +No Telegram message is deleted by this parser. A valid batch produces a +bounded list of logical keys for the catalog to unlink one at a time. +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import re +import zlib +from dataclasses import dataclass +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException +from xml.etree.ElementTree import ParseError + + +MAX_BATCH_OBJECTS = 1000 +MAX_XML_BYTES = 1024 * 1024 + + +class BatchDeleteError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + self.code = code + self.status = status + super().__init__(message) + + +@dataclass(frozen=True) +class BatchDeleteRequest: + keys: tuple[str, ...] + quiet: bool + + +def verify_body(body: bytes, headers: dict | object) -> None: + """Reject corrupted or unsigned batch manifests before any unlink. + + S3 requires Content-MD5 or one of the modern checksum headers; at least + one must match. The SigV4-bound SHA-256 must match independently. + """ + signature_sha = headers.get("x-amz-content-sha256", "") + if not isinstance(signature_sha, str) or not hmac.compare_digest( + signature_sha, hashlib.sha256(body).hexdigest() + ): + raise BatchDeleteError("XAmzContentSHA256Mismatch", + "DeleteObjects signed SHA-256 does not match") + content_md5 = headers.get("content-md5") + crc32 = headers.get("x-amz-checksum-crc32") + sha256 = headers.get("x-amz-checksum-sha256") + if not any((content_md5, crc32, sha256)): + raise BatchDeleteError("InvalidRequest", "DeleteObjects requires a body checksum") + expected = { + "content-md5": hashlib.md5(body, usedforsecurity=False).digest(), + "x-amz-checksum-crc32": (zlib.crc32(body) & 0xFFFFFFFF).to_bytes(4, "big"), + "x-amz-checksum-sha256": hashlib.sha256(body).digest(), + } + for name, supplied in (("content-md5", content_md5), + ("x-amz-checksum-crc32", crc32), + ("x-amz-checksum-sha256", sha256)): + if supplied is None: + continue + try: + provided = base64.b64decode(supplied, validate=True) + except (ValueError, binascii.Error) as exc: + raise BatchDeleteError("InvalidDigest", f"Invalid {name} base64") from exc + if not hmac.compare_digest(provided, expected[name]): + raise BatchDeleteError("BadDigest", f"Incorrect {name}") + + +def parse_batch(body: bytes) -> BatchDeleteRequest: + if not body or len(body) > MAX_XML_BYTES: + raise BatchDeleteError("MalformedXML", "Invalid DeleteObjects XML length") + # XML entities, DTDs and external resources must never be resolved. + try: + root = SafeET.fromstring(body) + except (ParseError, DefusedXmlException) as exc: + raise BatchDeleteError("MalformedXML", "Unsafe or malformed XML") from exc + + def local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + if local_name(root.tag) != "Delete" or root.attrib: + raise BatchDeleteError("MalformedXML", "Invalid DeleteObjects root") + + keys: list[str] = [] + quiet = False + quiet_seen = False + for entry in root: + if entry.attrib: + raise BatchDeleteError("MalformedXML", "Unexpected XML attributes") + name = local_name(entry.tag) + if name == "Quiet": + if quiet_seen or list(entry) or (entry.text or "").strip().lower() not in ( + "true", "false" + ): + raise BatchDeleteError("MalformedXML", "Invalid Quiet field") + quiet = (entry.text or "").strip().lower() == "true" + quiet_seen = True + elif name == "Object": + if len(keys) >= MAX_BATCH_OBJECTS: + raise BatchDeleteError("MalformedXML", "Too many DeleteObjects entries") + fields = list(entry) + if (len(fields) != 1 or local_name(fields[0].tag) != "Key" + or fields[0].attrib or list(fields[0])): + # VersionId is intentionally refused until real S3 version + # addressing/retention can be honored. + raise BatchDeleteError("NotImplemented", + "DeleteObjects VersionId or extension unsupported", 501) + key = fields[0].text or "" + if not (1 <= len(key.encode("utf-8")) <= 1024): + raise BatchDeleteError("InvalidObjectName", "Invalid S3 key") + keys.append(key) + else: + raise BatchDeleteError("MalformedXML", "Unexpected DeleteObjects field") + if not keys: + raise BatchDeleteError("MalformedXML", "No objects in DeleteObjects request") + return BatchDeleteRequest(tuple(keys), quiet) diff --git a/teledrive/s3/bucket_listing.py b/teledrive/s3/bucket_listing.py new file mode 100644 index 00000000..eac5362d --- /dev/null +++ b/teledrive/s3/bucket_listing.py @@ -0,0 +1,139 @@ +"""AWS-compatible ListBuckets pagination for TeleDrive's single private region. + +No remote Telegram I/O: listing uses SQL bucket records and the shared catalog. +Cursor signing binds a lexicographic high-water mark to the S3 access key, +prefix and fixed region; invalid/tampered tokens cannot change the filter. +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import time +from xml.etree import ElementTree as ET + +from teledrive.storage.catalog import BucketCatalog + +NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/" +BUCKET_REGION = "us-east-1" +MAX_BUCKETS = 10000 +MAX_CURSOR_LENGTH = 1024 +MAX_PREFIX_LENGTH = 255 + + +class BucketListingError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code, self.status = code, status + + +def _error(message: str) -> BucketListingError: + return BucketListingError("InvalidArgument", message) + + +def _signed_cursor(master: bytes, owner: str, prefix: str, after: str) -> str: + payload = json.dumps( + {"v": 1, "owner": owner, "prefix": prefix, "after": after, + "region": BUCKET_REGION}, + separators=(",", ":"), sort_keys=True, + ).encode("utf-8") + encoded = base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + mac = hmac.new(master, b"teledrive:list-buckets:v1:" + encoded.encode(), + hashlib.sha256).hexdigest() + token = encoded + "." + mac + if len(token) > MAX_CURSOR_LENGTH: + raise _error("Bucket cursor exceeds maximum length") + return token + + +def _read_cursor(token: str, master: bytes, owner: str, prefix: str) -> str: + if not token or len(token) > MAX_CURSOR_LENGTH: + raise _error("Invalid ListBuckets continuation token") + try: + encoded, mac = token.split(".") + if len(mac) != 64: + raise ValueError("Bad token MAC") + computed = hmac.new( + master, b"teledrive:list-buckets:v1:" + encoded.encode("ascii"), + hashlib.sha256, + ).hexdigest() + if not hmac.compare_digest(computed, mac): + raise ValueError("Bad token signature") + payload = base64.b64decode( + encoded + "=" * (-len(encoded) % 4), altchars=b"-_", validate=True, + ) + decoded = json.loads(payload) + if not isinstance(decoded, dict) or set(decoded) != { + "v", "owner", "prefix", "after", "region", + }: + raise ValueError("Invalid token structure") + if (type(decoded["v"]) is not int or decoded["v"] != 1 + or decoded["owner"] != owner or decoded["prefix"] != prefix + or decoded["region"] != BUCKET_REGION + or not isinstance(decoded["after"], str) + or not decoded["after"] or len(decoded["after"]) > 63): + raise ValueError("Token does not match S3 request") + return decoded["after"] + except (ValueError, TypeError, UnicodeError, KeyError, binascii.Error) as exc: + raise _error("Invalid or mismatched ListBuckets continuation token") from exc + + +def list_buckets_xml(catalog: BucketCatalog, params, master: bytes, + owner: str) -> bytes: + """Produce SDK-parsable XML with a stable, bounded SQL keyset cursor. + + Page boundaries are lexicographic, not offsets: a bucket inserted before + the cursor cannot cause duplication when listing continues. Like S3, + this is not a snapshot across concurrent bucket creations/deletions. + """ + prefix = params.get("prefix", "") + if len(prefix) > MAX_PREFIX_LENGTH: + raise _error("Bucket prefix is too long") + requested_region = params.get("bucket-region") + if requested_region is not None and requested_region != BUCKET_REGION: + raise BucketListingError( + "InvalidRequest", "ListBuckets region does not match the endpoint", + ) + raw_max = params.get("max-buckets") + if raw_max is None: + maximum = MAX_BUCKETS + else: + if not raw_max.isdecimal() or not 1 <= len(raw_max) <= 5: + raise _error("max-buckets must be an integer from 1 to 10000") + maximum = int(raw_max) + if maximum < 1 or maximum > MAX_BUCKETS: + raise _error("max-buckets must be from 1 to 10000") + + resume = params.get("continuation-token") + after = _read_cursor(resume, master, owner, prefix) if resume is not None else "" + page, more = catalog.list_bucket_records( + prefix=prefix, after=after, limit=maximum, + ) + root = ET.Element("ListAllMyBucketsResult", xmlns=NAMESPACE) + owner_tag = ET.SubElement(root, "Owner") + ET.SubElement(owner_tag, "ID").text = hmac.new( + master, b"teledrive:s3:deployment-owner:v1", hashlib.sha256, + ).hexdigest() + buckets = ET.SubElement(root, "Buckets") + show_region = any( + key in params for key in ( + "max-buckets", "bucket-region", "continuation-token", "prefix", + ) + ) + for name, created_at in page: + entry = ET.SubElement(buckets, "Bucket") + ET.SubElement(entry, "Name").text = name + ET.SubElement(entry, "CreationDate").text = time.strftime( + "%Y-%m-%dT%H:%M:%S.000Z", time.gmtime(created_at), + ) + if show_region: + ET.SubElement(entry, "BucketRegion").text = BUCKET_REGION + if "prefix" in params: + ET.SubElement(root, "Prefix").text = prefix + if more and page: + ET.SubElement(root, "ContinuationToken").text = _signed_cursor( + master, owner, prefix, page[-1][0], + ) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) diff --git a/teledrive/s3/checksums.py b/teledrive/s3/checksums.py new file mode 100644 index 00000000..ff854ac8 --- /dev/null +++ b/teledrive/s3/checksums.py @@ -0,0 +1,55 @@ +"""Streaming CRC32C (Castagnoli) for S3 upload integrity checks. + +Portable pure-Python implementation: reflected polynomial 0x82F63B78. +This checksum detects accidental corruption; it does not authenticate data. +""" +from __future__ import annotations + + +def _table() -> tuple[int, ...]: + values = [] + for index in range(256): + word = index + for _ in range(8): + word = (word >> 1) ^ (0x82F63B78 if word & 1 else 0) + values.append(word & 0xFFFFFFFF) + return tuple(values) + + +_TABLE = _table() + + +def crc32c(data: bytes, prior: int = 0) -> int: + """Return the finalized CRC32C, allowing incremental block updates.""" + if not 0 <= prior <= 0xFFFFFFFF: + raise ValueError("Invalid CRC32C accumulator") + state = prior ^ 0xFFFFFFFF + for byte in data: + state = _TABLE[(state ^ byte) & 255] ^ (state >> 8) + return state ^ 0xFFFFFFFF + + +# CRC-64/NVME (reflected polynomial 0x9A6C9329AC4BC9B5). +# Initial value and final XOR are both all ones (CRC-64/NVME check: +# b"123456789" -> 0xAE8B14860A799888). +def _crc64nvme_table() -> tuple[int, ...]: + entries = [] + for index in range(256): + word = index + for _ in range(8): + word = (word >> 1) ^ (0x9A6C9329AC4BC9B5 if word & 1 else 0) + entries.append(word & 0xFFFFFFFFFFFFFFFF) + return tuple(entries) + + +_CRC64NVME_TABLE = _crc64nvme_table() + + +def crc64nvme(data: bytes, prior: int = 0) -> int: + """Return finalized CRC-64/NVME; pass the previous result for streaming.""" + if not 0 <= prior <= 0xFFFFFFFFFFFFFFFF: + raise ValueError("Invalid CRC64NVME accumulator") + state = prior ^ 0xFFFFFFFFFFFFFFFF + for byte in data: + state = _CRC64NVME_TABLE[(state ^ byte) & 255] ^ (state >> 8) + return state ^ 0xFFFFFFFFFFFFFFFF diff --git a/teledrive/s3/conditions.py b/teledrive/s3/conditions.py new file mode 100644 index 00000000..db51f0df --- /dev/null +++ b/teledrive/s3/conditions.py @@ -0,0 +1,61 @@ +"""Read preconditions for S3 GetObject / HeadObject. + +Matches use the object ETag (multipart or standard), not necessarily SHA256. +Only the immutable manifest timestamp is used for HTTP date comparisons. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from email.utils import formatdate, parsedate_to_datetime +from starlette.requests import Request + + +def http_last_modified(created_at: int) -> str: + return formatdate(int(created_at), usegmt=True) + + +def _matches(raw: str, current_etag: str) -> bool: + # S3 condition headers use quoted ETags; '*' follows HTTP semantics. + expected = '"' + current_etag + '"' + return any(v.strip() in (expected, "*") for v in raw.split(",")) + + +def _date(raw: str | None) -> datetime | None: + if not raw: + return None + try: + parsed = parsedate_to_datetime(raw) + if parsed.tzinfo is None: + return None + return parsed.astimezone(timezone.utc) + except (TypeError, ValueError, IndexError, OverflowError): + return None + + +def read_precondition(request: Request, etag: str, created_at: int) -> int | None: + """Return 304, 412, or None without initiating a Telegram transfer. + + ETag conditions take precedence over date conditions of the same class. + An unmatched If-Match is 412; matched If-None-Match is 304. + S3-specific combinations of conditions beyond these standard HTTP + outcomes will be expanded when tested against more client SDKs. + """ + if_match = request.headers.get("if-match") + if_none_match = request.headers.get("if-none-match") + if if_match is not None and not _matches(if_match, etag): + return 412 + if if_none_match is not None and _matches(if_none_match, etag): + return 304 + + created = datetime.fromtimestamp(created_at, tz=timezone.utc) + created = created.replace(microsecond=0) + + # RFC 9110: If-Unmodified-Since is ignored if If-Match is present; + # If-Modified-Since is ignored if If-None-Match is present. + unmodified_since = _date(request.headers.get("if-unmodified-since")) + if if_match is None and unmodified_since is not None and created > unmodified_since: + return 412 + modified_since = _date(request.headers.get("if-modified-since")) + if if_none_match is None and modified_since is not None and created <= modified_since: + return 304 + return None diff --git a/teledrive/s3/cors.py b/teledrive/s3/cors.py new file mode 100644 index 00000000..4b334d51 --- /dev/null +++ b/teledrive/s3/cors.py @@ -0,0 +1,229 @@ +"""Bounded S3 bucket CORS policy storage and conservative origin matching. + +CORS is browser policy metadata, *never* S3 authentication or authorization. +Only a real signature can authorize an object operation. OPTIONS preflights +are public and disclose only permitted cross-origin request headers. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from urllib.parse import urlsplit +from xml.etree import ElementTree as ET + +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException +from sqlalchemy import Column, ForeignKey, String, Table, Text, delete, insert, select, update + +from teledrive.storage.catalog import BucketCatalog, buckets, catalog_metadata + +MAX_CORS_XML_BYTES = 64 * 1024 +MAX_CORS_RULES = 100 +METHODS = frozenset({"GET", "PUT", "HEAD", "POST", "DELETE"}) +cors_config = Table( + "storage_bucket_cors", catalog_metadata, + Column("bucket", String(63), ForeignKey("storage_buckets.name"), primary_key=True), + Column("rules", Text, nullable=False), +) + + +class CORSError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code, self.status = code, status + + +def _name(node) -> str: + return node.tag.rsplit("}", 1)[-1] + + +def _origin_pattern(value: str) -> str: + if value == "*": + return value + if not isinstance(value, str) or len(value) > 256: + raise CORSError("InvalidRequest", "CORS origin is invalid") + parsed = urlsplit(value) + if (parsed.scheme not in {"http", "https"} or not parsed.hostname + or parsed.username or parsed.password or parsed.path or parsed.query + or parsed.fragment or parsed.netloc.endswith(":")): + raise CORSError("InvalidRequest", "CORS origins must be HTTP(S) origins") + host = parsed.hostname + if "*" in host and (host.count("*") != 1 or not host.startswith("*.")): + raise CORSError("InvalidRequest", "Unsupported wildcard origin pattern") + if re.search(r"[^a-zA-Z0-9.*-]", host) and ":" not in host: + raise CORSError("InvalidRequest", "Invalid CORS host") + if parsed.port is not None and not (1 <= parsed.port <= 65535): + raise CORSError("InvalidRequest", "Invalid CORS port") + return value + + +def _header_rule(value: str) -> str: + if value == "*": + return value + if not re.fullmatch(r"[A-Za-z0-9!#$%&'*+.^_`|~-]{1,128}", value): + raise CORSError("InvalidRequest", "Invalid CORS header name") + return value + + +def parse_cors_xml(payload: bytes) -> list[dict]: + if len(payload) > MAX_CORS_XML_BYTES: + raise CORSError("MalformedXML", "CORS configuration exceeds 64 KiB") + try: + root = SafeET.fromstring(payload) + except (ET.ParseError, DefusedXmlException) as exc: + raise CORSError("MalformedXML", "Invalid CORS XML") from exc + if _name(root) != "CORSConfiguration" or root.attrib: + raise CORSError("MalformedXML", "Unexpected CORS document") + if not 1 <= len(root) <= MAX_CORS_RULES: + raise CORSError("InvalidRequest", "CORS requires 1..100 rules") + result = [] + for node in root: + if _name(node) != "CORSRule" or node.attrib: + raise CORSError("MalformedXML", "Unexpected CORS rule element") + fields = {"AllowedOrigin": [], "AllowedMethod": [], + "AllowedHeader": [], "ExposeHeader": [], + "MaxAgeSeconds": [], "ID": []} + for element in node: + label = _name(element) + if label not in fields or element.attrib or len(element): + raise CORSError("MalformedXML", "Unsupported CORS rule field") + fields[label].append(element.text or "") + origins = fields["AllowedOrigin"] + methods = fields["AllowedMethod"] + if not origins or not methods: + raise CORSError("InvalidRequest", "CORS rule requires origins and methods") + if len(fields["MaxAgeSeconds"]) > 1 or len(fields["ID"]) > 1: + raise CORSError("InvalidRequest", "Repeated CORS rule field") + if any(method not in METHODS for method in methods): + raise CORSError("InvalidRequest", "Unsupported CORS method") + if len(set(methods)) != len(methods) or len(set(origins)) != len(origins): + raise CORSError("InvalidRequest", "Duplicate CORS method or origin") + for raw in origins: + _origin_pattern(raw) + for raw in fields["AllowedHeader"] + fields["ExposeHeader"]: + _header_rule(raw) + try: + maximum = int(fields["MaxAgeSeconds"][0]) if fields["MaxAgeSeconds"] else 0 + except ValueError as exc: + raise CORSError("InvalidRequest", "Invalid MaxAgeSeconds") from exc + if not 0 <= maximum <= 86400: + raise CORSError("InvalidRequest", "MaxAgeSeconds exceeds 24 hours") + rule_id = fields["ID"][0] if fields["ID"] else None + if rule_id is not None and len(rule_id.encode("utf-8")) > 255: + raise CORSError("InvalidRequest", "CORS rule ID too long") + result.append({ + "origins": origins, + "methods": methods, + "allowed_headers": fields["AllowedHeader"], + "expose_headers": fields["ExposeHeader"], + "max_age": maximum, + "id": rule_id, + }) + return result + + +def cors_xml(rules: list[dict]) -> bytes: + root = ET.Element("CORSConfiguration", xmlns="http://s3.amazonaws.com/doc/2006-03-01/") + for rule in rules: + child = ET.SubElement(root, "CORSRule") + if rule["id"] is not None: + ET.SubElement(child, "ID").text = rule["id"] + for field, values in ( + ("AllowedOrigin", rule["origins"]), ("AllowedMethod", rule["methods"]), + ("AllowedHeader", rule["allowed_headers"]), + ("ExposeHeader", rule["expose_headers"]), + ): + for value in values: + ET.SubElement(child, field).text = value + if rule["max_age"]: + ET.SubElement(child, "MaxAgeSeconds").text = str(rule["max_age"]) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def _origin_matches(pattern: str, origin: str) -> bool: + if not origin or len(origin) > 256 or any(ord(c) < 33 for c in origin): + return False + parsed = urlsplit(origin) + if (parsed.scheme not in {"http", "https"} or not parsed.hostname or + parsed.path or parsed.query or parsed.fragment or parsed.username or parsed.password): + return False + if pattern == "*": + return True + if "*" not in pattern: + return origin == pattern + # Match a single label at *.example.org; never echo untrusted wildcard + # patterns as literal Access-Control-Allow-Origin values. + regex = "^" + re.escape(pattern).replace(r"\*", r"[^./:]+") + "$" + return bool(re.fullmatch(regex, origin)) + + +def matching_rule(rules: list[dict], origin: str, method: str, + requested_headers: list[str] | None = None) -> dict | None: + requested_headers = requested_headers or [] + for rule in rules: + if method not in rule["methods"] or not any( + _origin_matches(pattern, origin) for pattern in rule["origins"] + ): + continue + allowed = [header.lower() for header in rule["allowed_headers"]] + if not all(any( + re.fullmatch("^" + re.escape(pattern).replace(r"\*", ".*") + "$", name.lower()) + for pattern in allowed + ) for name in requested_headers): + continue + return rule + return None + + +def response_cors_headers(rule: dict, origin: str, *, preflight: bool, + requested_headers: list[str] | None = None) -> dict[str, str]: + result = { + "Access-Control-Allow-Origin": origin, + "Vary": "Origin, Access-Control-Request-Method, Access-Control-Request-Headers", + } + if preflight: + result["Access-Control-Allow-Methods"] = ", ".join(rule["methods"]) + if requested_headers: + result["Access-Control-Allow-Headers"] = ", ".join(requested_headers) + result["Access-Control-Max-Age"] = str(rule["max_age"]) + elif rule["expose_headers"]: + result["Access-Control-Expose-Headers"] = ", ".join(rule["expose_headers"]) + return result + + +class BucketCORSStore: + def __init__(self, catalog: BucketCatalog): + self.catalog, self.db = catalog, catalog.db + catalog_metadata.create_all(self.db.engine, tables=[cors_config]) + + def _bucket_exists(self, name: str) -> None: + if self.db.connection.execute( + select(buckets.c.name).where(buckets.c.name == name) + ).first() is None: + raise FileNotFoundError("Bucket not found") + + def get(self, name: str) -> list[dict] | None: + self._bucket_exists(name) + data = self.db.connection.execute(select(cors_config.c.rules).where( + cors_config.c.bucket == name + )).scalar_one_or_none() + return json.loads(data) if data is not None else None + + def replace(self, name: str, rules: list[dict]) -> None: + """Fence concurrent bucket deletion with a conditional SQL UPDATE.""" + try: + guard = self.db.connection.execute(update(buckets).where( + buckets.c.name == name + ).values(created_at=buckets.c.created_at)) + if guard.rowcount != 1: + raise FileNotFoundError("Bucket not found") + self.db.connection.execute(delete(cors_config).where(cors_config.c.bucket == name)) + if rules: + self.db.connection.execute(insert(cors_config).values( + bucket=name, rules=json.dumps(rules, ensure_ascii=False, sort_keys=True) + )) + self.db.commit() + except BaseException: + self.db.connection.rollback() + raise diff --git a/teledrive/s3/credentials.py b/teledrive/s3/credentials.py new file mode 100644 index 00000000..5f7f3ded --- /dev/null +++ b/teledrive/s3/credentials.py @@ -0,0 +1,130 @@ +"""Encrypted, independently revocable S3 access credentials. + +A dedicated Fernet master key is mandatory. DB backups alone never reveal +AWS secret keys. Losing the master key makes existing keys unusable. +""" +from __future__ import annotations + +import json +import os +import secrets +import time +from dataclasses import dataclass + +from cryptography.fernet import Fernet, InvalidToken +from sqlalchemy import BigInteger, Boolean, Column, MetaData, String, Table, Text, insert, select, update + +from teledrive.db import Database + +_meta = MetaData() +credentials = Table( + "storage_s3_credentials", _meta, + Column("id", String(36), primary_key=True), + Column("name", String(120), nullable=False), + Column("access_key", String(64), nullable=False, unique=True), + Column("encrypted_secret", Text, nullable=False), + Column("permissions", Text, nullable=False), + Column("created_at", BigInteger, nullable=False), + Column("expires_at", BigInteger), + Column("revoked", Boolean, nullable=False), +) +PERMISSIONS = frozenset({ + "s3:ListAllMyBuckets", "s3:CreateBucket", "s3:DeleteBucket", + "s3:ListBucket", "s3:ListBucketVersions", "s3:GetBucketLocation", "s3:GetObject", "s3:GetObjectAttributes", "s3:PutObject", "s3:DeleteObject", + "s3:GetObjectTagging", "s3:PutObjectTagging", "s3:DeleteObjectTagging", + "s3:GetBucketTagging", "s3:PutBucketTagging", "s3:DeleteBucketTagging", + "s3:GetBucketCORS", "s3:PutBucketCORS", + "s3:GetBucketVersioning", "s3:PutBucketVersioning", + "s3:GetBucketPublicAccessBlock", "s3:PutBucketPublicAccessBlock", + "s3:GetLifecycleConfiguration", "s3:PutLifecycleConfiguration", + "s3:GetBucketPolicyStatus", +}) +DEFAULT_PERMISSIONS = tuple(sorted(PERMISSIONS)) + + +def load_master_key() -> bytes: + secret_file = os.environ.get("TELEDRIVE_S3_MASTER_KEY_FILE", "").strip() + if secret_file: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(secret_file, flags) + with os.fdopen(fd, "rb") as handle: + data = handle.read(256).strip() + else: + data = os.environ.get("TELEDRIVE_S3_MASTER_KEY", "").encode("ascii").strip() + try: + Fernet(data) + except (ValueError, TypeError) as exc: + raise RuntimeError("Set a valid 32-byte Fernet TELEDRIVE_S3_MASTER_KEY") from exc + return data + + +@dataclass(frozen=True) +class IssuedS3Credential: + id: str + access_key: str + secret_key: str + permissions: tuple[str, ...] + + +class S3CredentialStore: + def __init__(self, database: Database, master_key: bytes): + self.db = database + self.cipher = Fernet(master_key) + _meta.create_all(database.engine) + + def create(self, name: str, permissions: list[str], *, + expires_at: int | None = None, now: int | None = None) -> IssuedS3Credential: + now = int(time.time()) if now is None else now + if not isinstance(name, str) or not (1 <= len(name.strip()) <= 120): + raise ValueError("S3 credential requires a name (1–120 characters)") + if not isinstance(permissions, list) or not permissions or any( + p not in PERMISSIONS for p in permissions): + raise ValueError("Unknown or empty S3 permission") + if expires_at is not None and (expires_at <= now or expires_at > now + 10 * 365 * 86400): + raise ValueError("Expiration must be in the future (up to 10 years)") + from uuid import uuid4 + identifier = str(uuid4()) + access = "TD" + secrets.token_hex(12).upper() + secret = secrets.token_urlsafe(40) + perms = tuple(sorted(set(permissions))) + self.db.connection.execute(insert(credentials).values( + id=identifier, name=name.strip(), access_key=access, + encrypted_secret=self.cipher.encrypt(secret.encode()).decode(), + permissions=json.dumps(perms), created_at=now, expires_at=expires_at, + revoked=False, + )) + self.db.commit() + return IssuedS3Credential(identifier, access, secret, perms) + + def list(self) -> list[dict]: + rows = self.db.connection.execute(select( + credentials.c.id, credentials.c.name, credentials.c.access_key, + credentials.c.permissions, credentials.c.created_at, + credentials.c.expires_at, credentials.c.revoked, + ).order_by(credentials.c.created_at.desc(), credentials.c.id)).mappings().all() + return [{**dict(row), "permissions": json.loads(row["permissions"])} for row in rows] + + def resolve(self, access_key: str, permission: str, *, + now: int | None = None) -> str | None: + now = int(time.time()) if now is None else now + if not isinstance(access_key, str) or len(access_key) > 64 or permission not in PERMISSIONS: + return None + row = self.db.connection.execute(select( + credentials.c.encrypted_secret, credentials.c.permissions, + credentials.c.revoked, credentials.c.expires_at, + ).where(credentials.c.access_key == access_key)).first() + if not row or row[2] or (row[3] is not None and row[3] <= now): + return None + if permission not in json.loads(row[1]): + return None + try: + return self.cipher.decrypt(row[0].encode()).decode() + except (InvalidToken, ValueError, UnicodeError): + return None + + def revoke(self, identifier: str) -> bool: + changed = self.db.connection.execute(update(credentials).where( + credentials.c.id == identifier, credentials.c.revoked.is_(False) + ).values(revoked=True)) + self.db.commit() + return changed.rowcount == 1 diff --git a/teledrive/s3/lifecycle.py b/teledrive/s3/lifecycle.py new file mode 100644 index 00000000..5e6b966c --- /dev/null +++ b/teledrive/s3/lifecycle.py @@ -0,0 +1,189 @@ +"""S3 lifecycle subset: abort expired incomplete multipart sessions in SQL only. + +Never deletes Telegram messages, object manifests, or any completed version. +""" +from __future__ import annotations +import json +from xml.etree import ElementTree as ET +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException +from sqlalchemy import Column, ForeignKey, String, Table, Text, select, insert, update, delete +from sqlalchemy.exc import IntegrityError +from teledrive.storage.catalog import BucketCatalog, catalog_metadata + +NS = "http://s3.amazonaws.com/doc/2006-03-01/" +MAX_XML = 32768 +rules_table = Table("storage_bucket_lifecycle", catalog_metadata, + Column("bucket", String(63), ForeignKey("storage_buckets.name"), primary_key=True), + Column("rules_json", Text, nullable=False)) + + +class LifecycleError(ValueError): + def __init__(self, code, message, status=400): + super().__init__(message) + self.code, self.status = code, status + + +def fields(node, names): + result = {} + for el in node: + k = el.tag.rsplit("}", 1)[-1] + if (el.tag not in {k, f"{{{NS}}}{k}"} or el.attrib + or k not in names or k in result): + raise LifecycleError("NotImplemented", "Unsupported lifecycle rule", 501) + result[k] = el + return result + + +def scalar(el): + if el is None or len(el): + raise LifecycleError("MalformedXML", "Invalid lifecycle element") + return el.text or "" + + +def parse_lifecycle_xml(body: bytes): + if len(body) > MAX_XML: + raise LifecycleError("MalformedXML", "Lifecycle XML exceeds 32 KiB") + try: + root = SafeET.fromstring(body) + except (ET.ParseError, DefusedXmlException) as exc: + raise LifecycleError("MalformedXML", "Malformed lifecycle XML") from exc + if (root.tag not in {"LifecycleConfiguration", f"{{{NS}}}LifecycleConfiguration"} + or root.attrib or not 1 <= len(root) <= 50): + raise LifecycleError("MalformedXML", "Requires 1..50 lifecycle rules") + result, ids = [], set() + for el in root: + if el.tag not in {"Rule", f"{{{NS}}}Rule"} or el.attrib: + raise LifecycleError("NotImplemented", "Unsupported lifecycle node", 501) + rule = fields(el, {"ID", "Status", "Filter", "AbortIncompleteMultipartUpload"}) + if "Status" not in rule or "AbortIncompleteMultipartUpload" not in rule: + raise LifecycleError("InvalidArgument", "Missing lifecycle abort action") + status = scalar(rule["Status"]) + if status not in {"Enabled", "Disabled"}: + raise LifecycleError("InvalidArgument", "Invalid rule status") + ident = scalar(rule["ID"]) if "ID" in rule else str(len(result) + 1) + if not 1 <= len(ident.encode("utf-8")) <= 255 or ident in ids: + raise LifecycleError("InvalidArgument", "Invalid duplicate rule ID") + ids.add(ident) + prefix = "" + if "Filter" in rule: + fil = fields(rule["Filter"], {"Prefix"}) + if set(fil) != {"Prefix"}: + raise LifecycleError("NotImplemented", "Only Prefix filter supported", 501) + prefix = scalar(fil["Prefix"]) + if len(prefix.encode("utf-8")) > 1024: + raise LifecycleError("InvalidArgument", "Lifecycle prefix too long") + abort = fields(rule["AbortIncompleteMultipartUpload"], {"DaysAfterInitiation"}) + if set(abort) != {"DaysAfterInitiation"}: + raise LifecycleError("InvalidArgument", "Missing abort age") + value = scalar(abort["DaysAfterInitiation"]) + if not value.isascii() or not value.isdigit() or value.startswith("0") or len(value) > 4: + raise LifecycleError("InvalidArgument", "Invalid abort age") + days = int(value) + if not 1 <= days <= 3650: + raise LifecycleError("InvalidArgument", "Abort age outside 1..3650 days") + result.append({"id": ident, "status": status, "prefix": prefix, "days": days}) + return result + + +def lifecycle_xml(rules): + root = ET.Element("LifecycleConfiguration", xmlns=NS) + for rule in rules: + el = ET.SubElement(root, "Rule") + ET.SubElement(el, "ID").text = rule["id"] + ET.SubElement(el, "Status").text = rule["status"] + fil = ET.SubElement(el, "Filter") + ET.SubElement(fil, "Prefix").text = rule["prefix"] + abort = ET.SubElement(el, "AbortIncompleteMultipartUpload") + ET.SubElement(abort, "DaysAfterInitiation").text = str(rule["days"]) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +class BucketLifecycleStore: + def __init__(self, catalog: BucketCatalog): + self.catalog, self.db = catalog, catalog.db + catalog_metadata.create_all(self.db.engine, tables=[rules_table]) + + def get(self, bucket): + if bucket not in self.catalog.list_buckets(): + raise FileNotFoundError("No such bucket") + raw = self.db.connection.execute(select(rules_table.c.rules_json).where( + rules_table.c.bucket == bucket)).scalar_one_or_none() + if raw is None: + return None + try: + return json.loads(raw) + except (TypeError, ValueError) as exc: + raise RuntimeError("Corrupted lifecycle policy") from exc + + def replace(self, bucket, rules): + if bucket not in self.catalog.list_buckets(): + raise FileNotFoundError("No such bucket") + try: + if rules is None: + self.db.connection.execute(delete(rules_table).where( + rules_table.c.bucket == bucket)) + else: + data = json.dumps(rules, separators=(",", ":"), sort_keys=True) + changed = self.db.connection.execute(update(rules_table).where( + rules_table.c.bucket == bucket).values(rules_json=data)) + if changed.rowcount == 0: + self.db.connection.execute(insert(rules_table).values( + bucket=bucket, rules_json=data)) + self.db.commit() + except IntegrityError as exc: + self.db.connection.rollback() + raise LifecycleError("OperationAborted", "Concurrent policy edit", 409) from exc + except BaseException: + self.db.connection.rollback() + raise + + def apply_due(self, *, now: int, limit=100): + """Bounded active-state CAS; no remote or SQL journal deletion.""" + if type(now) is not int or now < 0 or type(limit) is not int or not 1 <= limit <= 1000: + raise ValueError("Invalid lifecycle run bounds") + from teledrive.s3.multipart import uploads, _meta + _meta.create_all(self.db.engine, tables=[uploads]) + entries = self.db.connection.execute(select( + rules_table.c.bucket, rules_table.c.rules_json)).all() + count = 0 + try: + for bucket, data in entries: + try: + rules = json.loads(data) + if not isinstance(rules, list): + raise ValueError() + except (TypeError, ValueError) as exc: + raise RuntimeError("Corrupted lifecycle policy") from exc + for rule in rules: + if (not isinstance(rule, dict) or set(rule) != {"id", "status", "prefix", "days"} + or rule["status"] not in {"Enabled", "Disabled"} + or not isinstance(rule["prefix"], str) + or type(rule["days"]) is not int or not 1 <= rule["days"] <= 3650): + raise RuntimeError("Corrupted lifecycle policy") + if rule["status"] != "Enabled": + continue + remaining = limit - count + if remaining <= 0: + self.db.commit() + return count + select_ids = select(uploads.c.id).where( + uploads.c.bucket == bucket, uploads.c.state == "active", + uploads.c.created_at <= now - rule["days"] * 86400, + ) + if rule["prefix"]: + select_ids = select_ids.where(uploads.c.object_key.startswith( + rule["prefix"], autoescape=True)) + ids = self.db.connection.execute( + select_ids.order_by(uploads.c.created_at, uploads.c.id) + .limit(remaining)).scalars() + for ident in list(ids): + changed = self.db.connection.execute(update(uploads).where( + uploads.c.id == ident, uploads.c.state == "active" + ).values(state="aborted")) + count += int(changed.rowcount) + self.db.commit() + return count + except BaseException: + self.db.connection.rollback() + raise diff --git a/teledrive/s3/multipart.py b/teledrive/s3/multipart.py new file mode 100644 index 00000000..df9e2e37 --- /dev/null +++ b/teledrive/s3/multipart.py @@ -0,0 +1,764 @@ +"""Crash-conscious S3 multipart staging, ownership, and zero-reupload completion. + +Part bytes live in Telegram's immutable message storage. SQL holds staged +object IDs; Complete verifies every part and re-parents existing chunk refs +inside one DB transaction. No payload bytes persist in SQL. + +Aborted/replaced/unselected parts are retained as *unpublished* objects until +an independently verified garbage collector supports them. +""" +from __future__ import annotations + +import asyncio +import hashlib +import re +import time +from dataclasses import dataclass +from uuid import uuid4 + +from sqlalchemy import ( + BigInteger, Column, ForeignKey, Integer, MetaData, String, Table, Text, + delete, insert, select, update, and_, +) +from sqlalchemy.exc import IntegrityError + +from teledrive.storage.catalog import BucketCatalog, buckets, heads +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.s3.object_metadata import ObjectMetadataStore +from teledrive.storage.manifest import chunks, objects + +MIN_PART_SIZE = 5 * 1024 * 1024 +MAX_PARTS = 10000 +_meta = MetaData() +uploads = Table( + "storage_multipart_uploads", _meta, + Column("id", String(36), primary_key=True), + Column("bucket", String(63), nullable=False), + Column("object_key", Text, nullable=False), + Column("owner_access_key", String(64), nullable=False), + Column("state", String(16), nullable=False), + Column("created_at", BigInteger, nullable=False), + Column("completed_object_id", String(36)), + Column("completed_etag", String(128)), +) +parts = Table( + "storage_multipart_parts", _meta, + Column("upload_id", String(36), ForeignKey("storage_multipart_uploads.id"), + primary_key=True), + Column("number", Integer, primary_key=True), + Column("object_id", String(36), nullable=False), + Column("etag", String(64), nullable=False), + Column("size", BigInteger, nullable=False), + Column("updated_at", BigInteger, nullable=False), +) +completion_parts = Table( + "storage_multipart_completion_parts", _meta, + Column("upload_id", String(36), ForeignKey("storage_multipart_uploads.id"), primary_key=True), + Column("position", Integer, primary_key=True), + Column("number", Integer, nullable=False), + Column("etag", String(64), nullable=False), +) +completion_leases = Table( + "storage_multipart_completion_leases", _meta, + Column("upload_id", String(36), ForeignKey("storage_multipart_uploads.id"), primary_key=True), + Column("token", String(80), nullable=False), + Column("lease_until", BigInteger, nullable=False), + Column("heartbeat_at", BigInteger, nullable=False), +) +multipart_etags = Table( + "storage_multipart_etags", _meta, + Column("object_id", String(36), primary_key=True), + Column("etag", String(128), nullable=False), +) +# Additive immutable snapshot. Completion publishes this in the SAME SQL +# transaction as the final manifest, multipart ETag, and catalog head. +# Never derive logical S3 part numbers from physical Telegram chunks. +multipart_object_parts = Table( + "storage_multipart_object_parts", _meta, + Column("object_id", String(36), primary_key=True), + Column("number", Integer, primary_key=True), + Column("size", BigInteger, nullable=False), + Column("sha256", String(64), nullable=False), + Column("etag", String(64), nullable=False), +) + + +class MultipartError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code = code + self.status = status + + +@dataclass(frozen=True) +class CompletedUpload: + object_id: str + etag: str + size: int + + +class MultipartStore: + def __init__(self, catalog: BucketCatalog): + self.catalog = catalog + self.db = catalog.db + self.manifests = catalog.manifests + _meta.create_all(self.db.engine) + + def initiate(self, bucket: str, key: str, owner: str, now: int | None = None, + *, http_metadata: dict[str, str] | None = None) -> str: + if bucket not in self.catalog.list_buckets(): + raise FileNotFoundError("Bucket does not exist") + self.catalog._keyhash(key) + if not owner: + raise ValueError("Missing multipart owner") + identifier = str(uuid4()) + self.db.connection.execute(insert(uploads).values( + id=identifier, bucket=bucket, object_key=key, owner_access_key=owner, + state="active", created_at=now or int(time.time()), + completed_object_id=None, completed_etag=None, + )) + if http_metadata: + ObjectMetadataStore(self.catalog).stage_upload_uncommitted( + identifier, http_metadata, + ) + self.db.commit() + return identifier + + def require(self, identifier: str, bucket: str, key: str, owner: str, + *, allow_completed: bool = False): + row = self.db.connection.execute( + select(uploads).where( + uploads.c.id == identifier, uploads.c.bucket == bucket, + uploads.c.object_key == key, uploads.c.owner_access_key == owner, + ) + ).mappings().first() + if not row or row["state"] not in (("active", "completed") if allow_completed else ("active",)): + raise MultipartError("NoSuchUpload", "Multipart upload not found or inactive", 404) + return row + + def list_uploads(self, bucket: str, owner: str, *, + prefix: str = "", after_key: str = "", + after_id: str = "", limit: int = 1000): + """Return a bounded, owner-scoped page and whether another page exists.""" + if not 1 <= limit <= 1000 or len(prefix.encode("utf-8")) > 1024: + raise MultipartError("InvalidArgument", "Invalid multipart list pagination") + if after_id and not after_key: + raise MultipartError("InvalidArgument", "Upload-id marker requires key marker") + stmt = select(uploads.c.id, uploads.c.object_key, uploads.c.created_at).where( + uploads.c.bucket == bucket, + uploads.c.owner_access_key == owner, + uploads.c.state == "active", + ) + if prefix: + stmt = stmt.where(uploads.c.object_key.startswith(prefix, autoescape=True)) + if after_key: + if after_id: + stmt = stmt.where( + (uploads.c.object_key > after_key) + | ((uploads.c.object_key == after_key) & (uploads.c.id > after_id)) + ) + else: + stmt = stmt.where(uploads.c.object_key > after_key) + rows = self.db.connection.execute( + stmt.order_by(uploads.c.object_key, uploads.c.id).limit(limit + 1) + ).all() + return rows[:limit], len(rows) > limit + + def list_parts(self, identifier: str, bucket: str, key: str, owner: str): + """Legacy internal unpaged result for recovery/metadata callers.""" + self.require(identifier, bucket, key, owner) + return self.db.connection.execute( + select(parts.c.number, parts.c.size, parts.c.etag).where( + parts.c.upload_id == identifier + ).order_by(parts.c.number) + ).all() + + def list_parts_page(self, identifier: str, bucket: str, key: str, owner: str, + *, marker: int = 0, limit: int = 1000): + """Bounded S3 ListParts page selected in SQL, never Python slicing.""" + if not 0 <= marker <= MAX_PARTS or not 1 <= limit <= 1000: + raise MultipartError("InvalidArgument", "Invalid ListParts pagination") + self.require(identifier, bucket, key, owner) + rows = self.db.connection.execute( + select( + parts.c.number, parts.c.size, parts.c.etag, parts.c.updated_at, + ).where( + parts.c.upload_id == identifier, + parts.c.number > marker, + ).order_by(parts.c.number).limit(limit + 1) + ).all() + return rows[:limit], len(rows) > limit + + def list_uploads_grouped( + self, bucket: str, owner: str, *, prefix: str = "", + delimiter: str, after_key: str = "", after_id: str = "", + limit: int = 1000, + ): + """Return ordered Upload/CommonPrefixes entries using bounded SQL batches. + + Query in stable (key, upload ID) order, reduce groups by their first + delimiter after prefix, and page over *returned entries*. Cursors for + common prefixes carry the prefix and an empty upload ID, so the next + request suppresses that already-delivered group. + """ + if (not 1 <= limit <= 1000 or len(prefix.encode("utf-8")) > 1024 + or not delimiter or len(delimiter.encode("utf-8")) > 16 + or (after_id and not after_key)): + raise MultipartError("InvalidArgument", "Invalid multipart list pagination") + self.catalog._keyhash(after_key) if after_key else None + base = select( + uploads.c.id, uploads.c.object_key, uploads.c.created_at, + ).where( + uploads.c.bucket == bucket, + uploads.c.owner_access_key == owner, + uploads.c.state == "active", + ) + if prefix: + base = base.where(uploads.c.object_key.startswith(prefix, autoescape=True)) + cursor_key, cursor_id = after_key, after_id + results = [] + last_cursor = ("", "") + # One extra *emittable* result detects truncation. Never pull all rows + # into Python; also avoid repeatedly materializing large common-prefix + # groups in a single unbounded SELECT. + while len(results) <= limit: + statement = base + if cursor_key: + statement = statement.where( + (uploads.c.object_key > cursor_key) + | ((uploads.c.object_key == cursor_key) & + (uploads.c.id > cursor_id)) + ) + batch = self.db.connection.execute( + statement.order_by(uploads.c.object_key, uploads.c.id).limit(256) + ).all() + if not batch: + break + for record in batch: + cursor_key, cursor_id = record.object_key, record.id + suffix = record.object_key[len(prefix):] + before, found, _ = suffix.partition(delimiter) + if found: + group = prefix + before + delimiter + if group <= after_key: + continue + if results and results[-1][0] == "prefix" and results[-1][1] == group: + continue + if any(kind == "prefix" and value == group for kind, value in results): + continue + results.append(("prefix", group)) + last_cursor = (group, "") + else: + results.append(("upload", record)) + last_cursor = (record.object_key, record.id) + if len(results) > limit: + break + if len(results) > limit: + break + if len(batch) < 256: + break + truncated = len(results) > limit + page = results[:limit] + if page: + kind, value = page[-1] + last_cursor = (value, "") if kind == "prefix" else (value.object_key, value.id) + return page, truncated, last_cursor + + def record_part(self, identifier: str, bucket: str, key: str, owner: str, + *, number: int, object_id: str, etag: str) -> None: + if number < 1 or number > MAX_PARTS or not re.fullmatch(r"[0-9a-f]{32}", etag): + raise MultipartError("InvalidArgument", "Invalid part number or MD5 digest") + obj = self.manifests.get(object_id) + if obj.bucket != bucket or obj.key != key: + raise MultipartError("InvalidPart", "Part object ownership mismatch") + # Conditional state check blocks publishing a part after Abort/Complete. + self.require(identifier, bucket, key, owner) + old = self.db.connection.execute(select(parts.c.object_id).where( + parts.c.upload_id == identifier, parts.c.number == number + )).scalar_one_or_none() + now = int(time.time()) + if old is None: + try: + with self.db.connection.begin_nested(): + self.db.connection.execute(insert(parts).values( + upload_id=identifier, number=number, object_id=object_id, + etag=etag, size=obj.size, updated_at=now, + )) + except IntegrityError as exc: + raise MultipartError("OperationAborted", "Concurrent part update", 409) from exc + else: + self.db.connection.execute(update(parts).where( + parts.c.upload_id == identifier, parts.c.number == number, + parts.c.object_id == old, + ).values(object_id=object_id, etag=etag, size=obj.size, updated_at=now)) + # Compare-and-update the upload row, rejecting races with completion. + live = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "active" + ).values(state="active")) + if live.rowcount != 1: + self.db.connection.rollback() + raise MultipartError("NoSuchUpload", "Multipart upload changed state", 404) + self.db.commit() + + def abort(self, identifier: str, bucket: str, key: str, owner: str) -> None: + self.require(identifier, bucket, key, owner) + changed = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "active" + ).values(state="aborted")) + self.db.commit() + if changed.rowcount != 1: + raise MultipartError("OperationAborted", "Concurrent multipart mutation", 409) + + def _claim_completion(self, identifier: str, now: int, ttl: int) -> str: + """The state change and ownership token commit together.""" + token = uuid4().hex + try: + changed = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "active" + ).values(state="completing")) + if changed.rowcount != 1: + self.db.connection.rollback() + raise MultipartError("OperationAborted", "Upload is already completing", 409) + self.db.connection.execute(insert(completion_leases).values( + upload_id=identifier, token=token, + lease_until=now + ttl, heartbeat_at=now, + )) + self.db.commit() + except BaseException: + self.db.connection.rollback() + raise + return token + + def _renew_completion(self, identifier: str, token: str, now: int, ttl: int, + *, commit: bool = True) -> bool: + changed = self.db.connection.execute(update(completion_leases).where( + completion_leases.c.upload_id == identifier, + completion_leases.c.token == token, + completion_leases.c.lease_until >= now, + ).values(lease_until=now + ttl, heartbeat_at=now)) + if commit: + self.db.commit() + return changed.rowcount == 1 + + async def _heartbeat_completion(self, identifier: str, token: str, ttl: int, + stop: asyncio.Event, lost: asyncio.Event) -> None: + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=max(1, ttl // 3)) + return + except asyncio.TimeoutError: + pass + try: + if not self._renew_completion(identifier, token, int(time.time()), ttl): + lost.set() + return + except Exception: + self.db.connection.rollback() + lost.set() + return + + def _reset_owned_completion(self, identifier: str, token: str) -> bool: + """Never reset a lease that another worker has recovered or taken.""" + now = int(time.time()) + try: + owned = self.db.connection.execute(update(completion_leases).where( + completion_leases.c.upload_id == identifier, + completion_leases.c.token == token, + completion_leases.c.lease_until >= now, + ).values(heartbeat_at=now)) + if owned.rowcount != 1: + self.db.connection.rollback() + return False + changed = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "completing" + ).values(state="active")) + if changed.rowcount != 1: + self.db.connection.rollback() + return False + self.db.connection.execute(delete(completion_leases).where( + completion_leases.c.upload_id == identifier, + completion_leases.c.token == token, + )) + self.db.commit() + return True + except BaseException: + self.db.connection.rollback() + raise + + def stale_completions(self, *, now: int | None = None, + min_age_seconds: int = 3600, limit: int = 100) -> list[dict]: + """Read-only operator inventory; never infer ownership of legacy unfenced rows.""" + if min_age_seconds < 60 or not 1 <= limit <= 1000: + raise ValueError("Unsafe stale completion limits") + now = int(time.time()) if now is None else now + records = self.db.connection.execute( + select(uploads.c.id, uploads.c.bucket, uploads.c.object_key, + completion_leases.c.lease_until) + .join(completion_leases, uploads.c.id == completion_leases.c.upload_id) + .where(uploads.c.state == "completing", + completion_leases.c.lease_until < now - min_age_seconds) + .order_by(completion_leases.c.lease_until, uploads.c.id) + .limit(limit) + ).mappings().all() + return [dict(row) for row in records] + + def recover_stale_completion(self, identifier: str, *, dry_run: bool = True, + now: int | None = None, + min_age_seconds: int = 3600) -> bool: + """Fence an expired writer and put uncommitted multipart data back in ACTIVE. + + An upload with no recorded lease (older pre-fencing installations) + is deliberately not recovered by this method. No Telegram message + is removed. Completed writes never become active again. + """ + if min_age_seconds < 60: + raise ValueError("Stale completion minimum age must be >= 60 seconds") + now = int(time.time()) if now is None else now + try: + found = self.db.connection.execute(select( + uploads.c.state, completion_leases.c.token, + completion_leases.c.lease_until, + ).select_from( + uploads.join(completion_leases, + uploads.c.id == completion_leases.c.upload_id) + ).where(uploads.c.id == identifier)).first() + if (not found or found[0] != "completing" + or int(found[2]) >= now - min_age_seconds): + self.db.connection.rollback() + return False + if dry_run: + self.db.connection.rollback() + return True + # Take ownership by atomically invalidating the prior writer. + # The final SQL publication renews this same row in its transaction. + fenced = self.db.connection.execute(update(completion_leases).where( + completion_leases.c.upload_id == identifier, + completion_leases.c.token == found[1], + completion_leases.c.lease_until < now - min_age_seconds, + ).values(token="recovered:" + uuid4().hex, + lease_until=now, heartbeat_at=now)) + if fenced.rowcount != 1: + self.db.connection.rollback() + return False + result = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "completing" + ).values(state="active")) + if result.rowcount != 1: + self.db.connection.rollback() + return False + self.db.connection.execute(delete(completion_leases).where( + completion_leases.c.upload_id == identifier, + )) + self.db.commit() + return True + except BaseException: + self.db.connection.rollback() + raise + + @staticmethod + def _validate_selection(selected: list[tuple[int, str]]) -> None: + if not selected or len(selected) > MAX_PARTS: + raise MultipartError("InvalidPart", "A multipart completion must select 1..10000 parts") + previous = 0 + for number, etag in selected: + if number <= previous: + raise MultipartError("InvalidPartOrder", "Parts must be strictly increasing") + if number < 1 or number > MAX_PARTS or not re.fullmatch(r"[0-9a-f]{32}", etag): + raise MultipartError("InvalidPart", "Invalid part number or ETag") + previous = number + + async def complete(self, identifier: str, bucket: str, key: str, owner: str, + selected: list[tuple[int, str]], + engine: TelegramObjectStorage, *, + lease_seconds: int = 120, + expected_generation: int | None = None, + expected_object_id: str | None = None) -> CompletedUpload: + self._validate_selection(selected) + row = self.require(identifier, bucket, key, owner, allow_completed=True) + if row["state"] == "completed": + # A retry can return the committed result only for exactly the + # same part selection. Another completion body is not equivalent. + original = self.db.connection.execute(select( + completion_parts.c.number, completion_parts.c.etag + ).where(completion_parts.c.upload_id == identifier) + .order_by(completion_parts.c.position)).all() + if [(p.number, p.etag) for p in original] != selected: + raise MultipartError("InvalidPart", "Completion part list differs from committed object") + final = self.manifests.get(row["completed_object_id"]) + return CompletedUpload(final.id, row["completed_etag"], final.size) + if lease_seconds < 5: + raise ValueError("Completion lease must last at least 5 seconds") + token = self._claim_completion(identifier, int(time.time()), lease_seconds) + stop = asyncio.Event() + lost = asyncio.Event() + renewal = asyncio.create_task( + self._heartbeat_completion(identifier, token, lease_seconds, stop, lost) + ) + try: + available = {int(p.number): p for p in self.db.connection.execute( + select(parts).where(parts.c.upload_id == identifier) + ).all()} + chosen = [] + chosen_hashes: list[str] = [] + total_size = 0 + full_sha = hashlib.sha256() + etag_md5 = hashlib.md5(usedforsecurity=False) + for index, (number, etag) in enumerate(selected): + part = available.get(number) + if part is None or part.etag != etag: + raise MultipartError("InvalidPart", "Missing part or mismatched ETag") + if index < len(selected) - 1 and part.size < MIN_PART_SIZE: + raise MultipartError("EntityTooSmall", "All parts except the last must be at least 5 MiB") + # Read real Telegram bytes, checking each immutable part digest + # as well as the final concatenated plaintext SHA-256. + source_manifest = self.manifests.get(part.object_id) + measured = 0 + part_sha = hashlib.sha256() + async for block in engine.iter_bytes(part.object_id): + measured += len(block) + part_sha.update(block) + full_sha.update(block) + if (measured != part.size or source_manifest.size != part.size + or part_sha.hexdigest() != source_manifest.sha256): + raise MultipartError("InvalidPart", "Part size or SHA-256 changed") + total_size += part.size + etag_md5.update(bytes.fromhex(etag)) + chosen.append(part) + chosen_hashes.append(source_manifest.sha256) + # Stop background DB writes before the publication transaction. + stop.set() + await renewal + if lost.is_set(): + raise MultipartError("OperationAborted", "Completion writer lease lost", 409) + final_etag = f"{etag_md5.hexdigest()}-{len(selected)}" + object_id = str(uuid4()) + digest = full_sha.hexdigest() + now = int(time.time()) + + # One SQL transaction: existing Telegram chunk messages move from + # immutable temporary part manifests to the final object. + try: + # SQL fencing point: recovery cannot invalidate this token + # while the committed object/chunk/head transaction is running. + if not self._renew_completion( + identifier, token, int(time.time()), lease_seconds, commit=False + ): + raise MultipartError("OperationAborted", "Completion lease was lost", 409) + self.db.connection.execute(insert(objects).values( + id=object_id, bucket=bucket, object_key=key, + status="pending", size=total_size, + sha256=None, created_at=now, updated_at=now, error=None, + )) + offset, ordinal = 0, 0 + for part in chosen: + refs = self.manifests.parts(part.object_id) + part_bytes = 0 + for ref in refs: + moved = self.db.connection.execute( + update(chunks).where( + chunks.c.object_id == part.object_id, + chunks.c.ordinal == ref.ordinal, + ).values( + object_id=object_id, ordinal=ordinal, + offset=offset + ref.offset, + ) + ) + if moved.rowcount != 1: + raise MultipartError("InvalidPart", "Telegram chunk ownership changed") + ordinal += 1 + part_bytes += ref.size + if part_bytes != part.size: + raise MultipartError("InvalidPart", "Part manifest is incomplete") + offset += part.size + self.db.connection.execute(update(objects).where( + objects.c.id == part.object_id, objects.c.status == "ready" + ).values(status="consumed", updated_at=now)) + if offset != total_size: + raise MultipartError("InvalidPart", "Incomplete multipart assembly") + self.db.connection.execute(update(objects).where( + objects.c.id == object_id, objects.c.status == "pending" + ).values(status="ready", sha256=digest)) + self.db.connection.execute(insert(multipart_etags).values( + object_id=object_id, etag=final_etag, + )) + # Publish the selected logical multipart layout atomically. + # A discarded/replaced upload part is never exposed by this + # object's attributes or historical VersionId. + for (number, etag), part, part_sha in zip( + selected, chosen, chosen_hashes, strict=True + ): + self.db.connection.execute(insert(multipart_object_parts).values( + object_id=object_id, number=number, size=part.size, + sha256=part_sha, etag=etag, + )) + # Metadata set at CreateMultipartUpload becomes immutable + # metadata on the newly published version, not on parts. + ObjectMetadataStore(self.catalog).attach_upload_uncommitted( + identifier, object_id + ) + for position, (number, etag) in enumerate(selected): + self.db.connection.execute(insert(completion_parts).values( + upload_id=identifier, position=position, + number=number, etag=etag, + )) + + keyhash = self.catalog._keyhash(key) + current = self.db.connection.execute(select( + heads.c.object_id, heads.c.generation + ).where(heads.c.bucket == bucket, heads.c.key_hash == keyhash)).first() + if current is None: + if expected_object_id is not None or expected_generation not in (None, 0): + raise MultipartError( + "PreconditionFailed", "Multipart destination changed", 412 + ) + self.db.connection.execute(insert(heads).values( + bucket=bucket, key_hash=keyhash, object_id=object_id, + generation=1, updated_at=now, + )) + else: + if expected_generation is not None and ( + expected_generation != int(current[1]) + or expected_object_id != current[0] + ): + raise MultipartError( + "PreconditionFailed", "Multipart destination changed", 412 + ) + live = self.manifests.get(current[0]) + if live.bucket != bucket or live.key != key: + raise MultipartError("InvalidPart", "Object key collision") + moved = self.db.connection.execute(update(heads).where( + heads.c.bucket == bucket, heads.c.key_hash == keyhash, + heads.c.object_id == current[0], heads.c.generation == current[1] + ).values( + object_id=object_id, generation=current[1] + 1, + updated_at=now, + )) + if moved.rowcount != 1: + raise MultipartError("OperationAborted", "Concurrent object replacement", 409) + + self.catalog._record_published_uncommitted( + self.manifests.get(object_id) + ) + completed = self.db.connection.execute(update(uploads).where( + uploads.c.id == identifier, uploads.c.state == "completing" + ).values(state="completed", completed_object_id=object_id, + completed_etag=final_etag)) + if completed.rowcount != 1: + raise MultipartError("OperationAborted", "Multipart upload lost ownership", 409) + self.db.connection.execute(delete(completion_leases).where( + completion_leases.c.upload_id == identifier, + completion_leases.c.token == token, + )) + self.db.commit() + except BaseException: + self.db.connection.rollback() + raise + return CompletedUpload(object_id, final_etag, total_size) + except BaseException: + # An old worker must never reopen an upload once its lease expired, + # was fenced by recovery, or was committed by another worker. + self.db.connection.rollback() + self._reset_owned_completion(identifier, token) + raise + finally: + stop.set() + await renewal + + def object_parts_for_version( + self, object_id: str, object_size: int + ) -> list[tuple[int, int, str, str]] | None: + """Return the immutable S3 layout, or None for a non-multipart object. + + Old completed uploads are read through their original SQL completion + journal; no destructive backfill and no Telegram I/O are needed. + Aliased server-side copies resolve their verified original payload. + Missing/inconsistent records fail closed rather than emitting a + plausible but false ObjectParts success response. + """ + tag = self.db.connection.execute(select(multipart_etags.c.etag).where( + multipart_etags.c.object_id == object_id + )).scalar_one_or_none() + if tag is None: + return None + match = re.fullmatch(r"[0-9a-f]{32}-([0-9]+)", tag) + if match is None: + raise MultipartError("InvalidObjectState", "Invalid multipart ETag record", 409) + count = int(match.group(1)) + if count < 1 or count > MAX_PARTS: + raise MultipartError("InvalidObjectState", "Invalid multipart part count", 409) + + content_id = self.manifests.resolve_content_id(object_id) + content = self.manifests.get(content_id) + target = self.manifests.get(object_id) + if (content.size != target.size or content.sha256 != target.sha256 + or target.size != object_size): + raise MultipartError("InvalidObjectState", "Copied multipart manifest mismatch", 409) + + def saved(identifier: str) -> list[tuple[int, int, str, str]]: + rows = self.db.connection.execute(select( + multipart_object_parts.c.number, + multipart_object_parts.c.size, + multipart_object_parts.c.sha256, + multipart_object_parts.c.etag, + ).where( + multipart_object_parts.c.object_id == identifier + ).order_by(multipart_object_parts.c.number)).all() + return [(int(row[0]), int(row[1]), str(row[2]), str(row[3])) + for row in rows] + + layout = saved(object_id) + if not layout and content_id != object_id: + layout = saved(content_id) + if not layout: + # Compatibility for multipart versions committed before the + # additive object-parts snapshot table was deployed. A completed + # journal binds this source object ID, not the current HEAD. + rows = self.db.connection.execute( + select( + completion_parts.c.number, parts.c.size, + objects.c.sha256, completion_parts.c.etag, parts.c.etag, + ).select_from( + completion_parts.join( + uploads, uploads.c.id == completion_parts.c.upload_id, + ).join( + parts, and_( + parts.c.upload_id == completion_parts.c.upload_id, + parts.c.number == completion_parts.c.number, + ), + ).join(objects, objects.c.id == parts.c.object_id) + ).where( + uploads.c.completed_object_id == content_id, + uploads.c.state == "completed", + ).order_by(completion_parts.c.position) + ).all() + if any(row[3] != row[4] for row in rows): + raise MultipartError("InvalidObjectState", "Legacy multipart ETag mismatch", 409) + layout = [(int(r[0]), int(r[1]), str(r[2]), str(r[3])) for r in rows] + + if len(layout) != count or sum(p[1] for p in layout) != object_size: + raise MultipartError("InvalidObjectState", "Multipart snapshot is incomplete", 409) + previous = 0 + for number, size, digest, etag in layout: + if (number <= previous or number > MAX_PARTS or size < 0 + or not re.fullmatch(r"[0-9a-f]{64}", digest) + or not re.fullmatch(r"[0-9a-f]{32}", etag)): + raise MultipartError("InvalidObjectState", "Invalid immutable multipart part", 409) + previous = number + return layout + + def snapshot_copy_parts( + self, source_id: str, destination_id: str, source_size: int + ) -> None: + """Clone logical part metadata, never Telegram messages, in COPY's txn.""" + layout = self.object_parts_for_version(source_id, source_size) + for number, size, digest, etag in layout or (): + self.db.connection.execute(insert(multipart_object_parts).values( + object_id=destination_id, number=number, size=size, + sha256=digest, etag=etag, + )) + + def object_etag(self, object_id: str, default_sha256: str) -> str: + etag = self.db.connection.execute(select(multipart_etags.c.etag).where( + multipart_etags.c.object_id == object_id + )).scalar_one_or_none() + return etag or default_sha256 diff --git a/teledrive/s3/object_metadata.py b/teledrive/s3/object_metadata.py new file mode 100644 index 00000000..8d5cf149 --- /dev/null +++ b/teledrive/s3/object_metadata.py @@ -0,0 +1,130 @@ +"""Immutable S3 HTTP headers and user metadata per Telegram-backed object. + +Only small metadata lives in SQL. Bodies and all chunks remain in Telegram. +Writes can join the caller's head-pointer transaction; multipart initiations +journal metadata separately and copy it when Complete commits the new object. +""" +from __future__ import annotations + +import json +import re + +from sqlalchemy import Column, ForeignKey, String, Table, Text, insert, select, delete +from teledrive.storage.manifest import metadata as manifest_metadata +from teledrive.storage.catalog import BucketCatalog + +object_http_metadata = Table( + "storage_object_http_metadata", manifest_metadata, + Column("object_id", String(36), ForeignKey("storage_objects.id"), primary_key=True), + Column("properties", Text, nullable=False), +) +upload_http_metadata = Table( + "storage_multipart_http_metadata", manifest_metadata, + Column("upload_id", String(36), primary_key=True), + Column("properties", Text, nullable=False), +) + +STANDARD_HEADERS = frozenset({ + "content-type", "content-encoding", "content-language", + "content-disposition", "cache-control", "expires", +}) +_METADATA_KEY = re.compile(r"[a-z0-9][a-z0-9_-]{0,127}\Z") +MAX_METADATA_BYTES = 2048 + + +class ObjectMetadataError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code, self.status = code, status + + +def parse_object_metadata(headers, signed_headers: set[str]) -> dict[str, str]: + """Only bind supported, signed headers; reject hidden unsigned mutations. + + Starlette's Headers supports multi_items(); an ordinary mapping supports + items(). Repeated user metadata/header names are ambiguous and rejected. + Unsupported S3 directives must never silently claim success. + """ + pairs = headers.multi_items() if hasattr(headers, "multi_items") else headers.items() + result: dict[str, str] = {} + for name, value in pairs: + key = name.lower() + if key not in STANDARD_HEADERS and not key.startswith("x-amz-meta-"): + continue + if key in result: + raise ObjectMetadataError("InvalidArgument", "Repeated object metadata field") + if key not in signed_headers: + raise ObjectMetadataError("AccessDenied", "Object metadata must be signed", 403) + if key.startswith("x-amz-meta-") and not _METADATA_KEY.fullmatch(key[11:]): + raise ObjectMetadataError("InvalidArgument", "Invalid user metadata name") + if not isinstance(value, str) or any(ord(c) < 32 or ord(c) == 127 for c in value): + raise ObjectMetadataError("InvalidArgument", "Invalid object metadata value") + # Starlette emits HTTP headers through latin-1. A Unicode value + # outside that range otherwise fails late during GET/HEAD after + # a successful remote upload, making the published object unreadable. + try: + value.encode("latin-1") + except UnicodeEncodeError as exc: + raise ObjectMetadataError( + "InvalidArgument", "Object metadata cannot be encoded as HTTP header" + ) from exc + if len(value.encode("utf-8")) > 1024: + raise ObjectMetadataError("InvalidArgument", "Object metadata field too large") + result[key] = value + if len(json.dumps(result, ensure_ascii=False).encode("utf-8")) > MAX_METADATA_BYTES: + raise ObjectMetadataError("MetadataTooLarge", "Object metadata exceeds 2 KiB") + return result + + +def signed_header_names(request) -> set[str]: + if "X-Amz-Algorithm" in request.query_params: + return set(request.query_params.get("X-Amz-SignedHeaders", "").split(";")) + auth = request.headers.get("authorization", "") + return set(auth.partition("SignedHeaders=")[2].split(",", 1)[0].split(";")) + + +class ObjectMetadataStore: + def __init__(self, catalog: BucketCatalog): + self.db = catalog.db + manifest_metadata.create_all( + self.db.engine, tables=[object_http_metadata, upload_http_metadata], + ) + + def get(self, object_id: str) -> dict[str, str]: + data = self.db.connection.execute( + select(object_http_metadata.c.properties).where( + object_http_metadata.c.object_id == object_id + ) + ).scalar_one_or_none() + return json.loads(data) if data is not None else {} + + def attach_uncommitted(self, object_id: str, values: dict[str, str]) -> None: + if values: + self.db.connection.execute(insert(object_http_metadata).values( + object_id=object_id, + properties=json.dumps(values, ensure_ascii=False, sort_keys=True), + )) + + def copy_uncommitted(self, source_id: str, destination_id: str) -> None: + self.attach_uncommitted(destination_id, self.get(source_id)) + + def stage_upload_uncommitted(self, upload_id: str, values: dict[str, str]) -> None: + if values: + self.db.connection.execute(insert(upload_http_metadata).values( + upload_id=upload_id, + properties=json.dumps(values, ensure_ascii=False, sort_keys=True), + )) + + def attach_upload_uncommitted(self, upload_id: str, object_id: str) -> None: + raw = self.db.connection.execute(select( + upload_http_metadata.c.properties + ).where(upload_http_metadata.c.upload_id == upload_id)).scalar_one_or_none() + if raw: + self.db.connection.execute(insert(object_http_metadata).values( + object_id=object_id, properties=raw, + )) + + def delete_upload_uncommitted(self, upload_id: str) -> None: + self.db.connection.execute(delete(upload_http_metadata).where( + upload_http_metadata.c.upload_id == upload_id + )) diff --git a/teledrive/s3/operation_manifest.json b/teledrive/s3/operation_manifest.json new file mode 100644 index 00000000..100f8b5c --- /dev/null +++ b/teledrive/s3/operation_manifest.json @@ -0,0 +1,588 @@ +{ + "schema_version": 1, + "source": "docs/S3_SPECS.md", + "reference": "https://docs.aws.amazon.com/AmazonS3/latest/API/API_Operations_Amazon_Simple_Storage_Service.html", + "note": "This inventory describes only the documented TeleDrive subset; partial is never full AWS parity.", + "operations": [ + { + "name": "CreateBucket", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "HeadBucket", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "DeleteBucket", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "ListBuckets", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "GetBucketLocation", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "ListObjects", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "ListObjectsV2", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "ListObjectVersions", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutObject", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetObject", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "HeadObject", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeleteObject", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeleteObjects", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "CopyObject", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetObjectAttributes", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "CreateMultipartUpload", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "UploadPart", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "ListParts", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "ListMultipartUploads", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "CompleteMultipartUpload", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "AbortMultipartUpload", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetBucketTagging", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "PutBucketTagging", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "DeleteBucketTagging", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "GetObjectTagging", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutObjectTagging", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeleteObjectTagging", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetBucketCors", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutBucketCors", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeleteBucketCors", + "status": "supported", + "workstream": "implemented" + }, + { + "name": "GetBucketVersioning", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutBucketVersioning", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetPublicAccessBlock", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutPublicAccessBlock", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetBucketLifecycleConfiguration", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "PutBucketLifecycleConfiguration", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeleteBucketLifecycle", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "DeletePublicAccessBlock", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "GetBucketAcl", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "PutBucketAcl", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "GetObjectAcl", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "PutObjectAcl", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "GetBucketOwnershipControls", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "PutBucketOwnershipControls", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "DeleteBucketOwnershipControls", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "GetBucketPolicy", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "PutBucketPolicy", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "DeleteBucketPolicy", + "status": "unsupported", + "workstream": "P2-C" + }, + { + "name": "GetBucketPolicyStatus", + "status": "partial", + "workstream": "implemented" + }, + { + "name": "GetBucketEncryption", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "PutBucketEncryption", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "DeleteBucketEncryption", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "GetObjectLockConfiguration", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "PutObjectLockConfiguration", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "GetObjectRetention", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "PutObjectRetention", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "GetObjectLegalHold", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "PutObjectLegalHold", + "status": "unsupported", + "workstream": "P2-B" + }, + { + "name": "GetBucketLifecycle", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "PutBucketLifecycle", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "GetBucketReplication", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "PutBucketReplication", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "DeleteBucketReplication", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "RestoreObject", + "status": "unsupported", + "workstream": "P2-D" + }, + { + "name": "UploadPartCopy", + "status": "unsupported", + "workstream": "P2-E" + }, + { + "name": "GetBucketWebsite", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "PutBucketWebsite", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "DeleteBucketWebsite", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "GetBucketNotification", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "GetBucketNotificationConfiguration", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "PutBucketNotification", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "PutBucketNotificationConfiguration", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "GetBucketLogging", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "PutBucketLogging", + "status": "unsupported", + "workstream": "P2-F" + }, + { + "name": "GetBucketAnalyticsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "PutBucketAnalyticsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "DeleteBucketAnalyticsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "ListBucketAnalyticsConfigurations", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "GetBucketInventoryConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "PutBucketInventoryConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "DeleteBucketInventoryConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "ListBucketInventoryConfigurations", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "GetBucketMetricsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "PutBucketMetricsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "DeleteBucketMetricsConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "ListBucketMetricsConfigurations", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "GetBucketIntelligentTieringConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "PutBucketIntelligentTieringConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "DeleteBucketIntelligentTieringConfiguration", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "ListBucketIntelligentTieringConfigurations", + "status": "unsupported", + "workstream": "P3-G" + }, + { + "name": "CreateBucketMetadataConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "CreateBucketMetadataTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetBucketMetadataConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetBucketMetadataTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "DeleteBucketMetadataConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "DeleteBucketMetadataTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "UpdateBucketMetadataAnnotationTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "UpdateBucketMetadataInventoryTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "UpdateBucketMetadataJournalTableConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetObjectAnnotation", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "PutObjectAnnotation", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "DeleteObjectAnnotation", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "ListObjectAnnotations", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetBucketAbac", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "PutBucketAbac", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetBucketRequestPayment", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "PutBucketRequestPayment", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetBucketAccelerateConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "PutBucketAccelerateConfiguration", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "GetObjectTorrent", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "CreateSession", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "ListDirectoryBuckets", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "RenameObject", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "SelectObjectContent", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "UpdateObjectEncryption", + "status": "unsupported", + "workstream": "P3-H" + }, + { + "name": "WriteGetObjectResponse", + "status": "unsupported", + "workstream": "P3-H" + } + ] +} diff --git a/teledrive/s3/post_policy.py b/teledrive/s3/post_policy.py new file mode 100644 index 00000000..ec531a64 --- /dev/null +++ b/teledrive/s3/post_policy.py @@ -0,0 +1,210 @@ +"""Strict, checksum-pinned SigV4 browser POST policy verifier.""" +import base64 +import binascii +import hashlib +import hmac +import json +import re +from datetime import datetime, timedelta, timezone +from email import policy as mime_policy +from email.parser import BytesParser +from urllib.parse import urlsplit + +from .sigv4 import SigV4Error, _hmac + +FORM_LIMIT = 20 * 1024 * 1024 +REQUIRED_FIELDS = frozenset(("key", "policy", "x-amz-algorithm", + "x-amz-credential", "x-amz-date", + "x-amz-signature", "x-amz-checksum-sha256")) +OPTIONAL_FIELDS = frozenset(( + "success_action_status", "success_action_redirect", "Content-Type", + "Cache-Control", "Content-Disposition", "Content-Encoding", + "Content-Language", "Expires", "x-amz-tagging", +)) +FIELDS = REQUIRED_FIELDS | OPTIONAL_FIELDS +CONDITIONS = (REQUIRED_FIELDS - {"policy", "x-amz-signature"}) | {"bucket"} +MAX_CUSTOM_METADATA_FIELDS = 12 +MAX_FORM_PARTS = len(FIELDS) + MAX_CUSTOM_METADATA_FIELDS + 1 +_METADATA_FIELD = re.compile(r"x-amz-meta-[a-z0-9][a-z0-9_-]{0,127}$") + + +def _allowed_field(name): + return isinstance(name, str) and ( + name in FIELDS or _METADATA_FIELD.fullmatch(name) is not None + ) + + +def parse_form(content_type: str, data: bytes): + """Parse one file, mandatory signing fields and a bounded response status.""" + if len(data) > FORM_LIMIT: + raise SigV4Error("EntityTooLarge", "Browser form is too large") + if (len(content_type) > 256 or not content_type.lower().startswith("multipart/form-data;") + or "\r" in content_type or "\n" in content_type): + raise SigV4Error("InvalidRequest", "Expected multipart/form-data") + try: + mime = ("Content-Type: " + content_type + + "\r\nMIME-Version: 1.0\r\n\r\n").encode("ascii") + data + msg = BytesParser(policy=mime_policy.default).parsebytes(mime) + except (ValueError, UnicodeError) as exc: + raise SigV4Error("InvalidRequest", "Malformed upload form") from exc + if not msg.is_multipart() or msg.defects: + raise SigV4Error("InvalidRequest", "Malformed multipart upload") + items = list(msg.iter_parts()) + if len(items) > MAX_FORM_PARTS: + raise SigV4Error("InvalidRequest", "Too many browser form parts") + fields = {} + file_bytes = None + for part in items: + name = part.get_param("name", header="content-disposition") + if (part.is_multipart() or part.defects or + part.get_content_disposition() != "form-data" or + (name != "file" and not _allowed_field(name)) or + part.get("Content-Transfer-Encoding", "binary").lower() + not in ("binary", "7bit", "8bit")): + raise SigV4Error("InvalidRequest", "Unsupported form field") + content = part.get_payload(decode=True) + if not isinstance(content, bytes): + raise SigV4Error("InvalidRequest", "Malformed browser form part") + if name == "file": + if file_bytes is not None: + raise SigV4Error("InvalidRequest", "Duplicate form file") + file_bytes = content + else: + if name in fields or len(content) > 12000: + raise SigV4Error("InvalidRequest", "Duplicate or oversized form field") + try: + fields[name] = content.decode("utf-8") + except UnicodeError as exc: + raise SigV4Error("InvalidRequest", "Invalid field encoding") from exc + if (file_bytes is None or not REQUIRED_FIELDS.issubset(fields) + or sum(name.startswith("x-amz-meta-") for name in fields) + > MAX_CUSTOM_METADATA_FIELDS + or any(not _allowed_field(name) for name in fields)): + raise SigV4Error("InvalidRequest", "Missing or unsupported signed browser form fields") + return fields, file_bytes + + +def verify_post_policy(fields, body, bucket, lookup_secret, *, max_bytes, now=None): + if len(body) > max_bytes: + raise SigV4Error("EntityTooLarge", "Browser payload too large") + scope = re.fullmatch( + r"([^/\s]{1,64})/(\d{8})/([a-z0-9-]{1,32})/s3/aws4_request", + fields["x-amz-credential"], + ) + if scope is None or fields["x-amz-algorithm"] != "AWS4-HMAC-SHA256": + raise SigV4Error("AccessDenied", "Unsupported browser POST scope") + access, day, region = scope.groups() + stamp = fields["x-amz-date"] + if not re.fullmatch(r"\d{8}T\d{6}Z", stamp) or stamp[:8] != day: + raise SigV4Error("AccessDenied", "Invalid policy date") + try: + signed_at = datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise SigV4Error("AccessDenied", "Invalid policy signing date") from exc + now = now or datetime.now(timezone.utc) + if not -300 <= (now - signed_at).total_seconds() <= 900: + raise SigV4Error("AccessDenied", "Policy signing date expired") + status = fields.get("success_action_status") + redirect = fields.get("success_action_redirect") + content_type = fields.get("Content-Type") + extra = set(fields) - REQUIRED_FIELDS + if (len(extra) > len(OPTIONAL_FIELDS) + MAX_CUSTOM_METADATA_FIELDS + or any(not _allowed_field(name) for name in fields)): + raise SigV4Error("InvalidArgument", "Unsupported POST metadata fields") + if status is not None and status not in ("200", "201", "204"): + raise SigV4Error("InvalidArgument", "Unsupported POST success response status") + # Redirects are only browser instructions (the server never fetches + # their target). Require an exact signed, HTTPS-only destination and + # forbid credentials, fragments and ambiguous URL representations. + if redirect is not None: + if (status is not None or len(redirect) > 2048 + or any(ord(char) < 32 or char == "\\" for char in redirect)): + raise SigV4Error("InvalidArgument", "Unsupported POST success redirect") + try: + target = urlsplit(redirect) + valid = (target.scheme == "https" and target.netloc + and target.hostname and target.username is None + and target.password is None and target.fragment == "" + and (target.port is None or target.port > 0)) + except ValueError: + valid = False + if not valid: + raise SigV4Error("InvalidArgument", "Unsafe POST success redirect") + key = fields["key"] + if not key or len(key.encode()) > 1024 or "$" + "{" in key: + raise SigV4Error("InvalidArgument", "Exact non-template object key required") + try: + expected_sha = base64.b64decode(fields["x-amz-checksum-sha256"], validate=True) + raw = base64.b64decode(fields["policy"], validate=True) + except (binascii.Error, ValueError) as exc: + raise SigV4Error("InvalidArgument", "Invalid policy base64") from exc + if len(expected_sha) != 32 or len(raw) > 8192 or len(raw) == 0: + raise SigV4Error("InvalidArgument", "Invalid policy or checksum size") + if not hmac.compare_digest(expected_sha, hashlib.sha256(body).digest()): + raise SigV4Error("BadDigest", "Browser form checksum mismatch") + try: + doc = json.loads(raw) + expiration = datetime.strptime(doc["expiration"], "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc) + clauses = doc["conditions"] + except (ValueError, TypeError, KeyError, UnicodeError) as exc: + raise SigV4Error("InvalidArgument", "Invalid POST policy JSON") from exc + if (not isinstance(doc, dict) or set(doc) != {"conditions", "expiration"} + or not isinstance(clauses, list) + or len(clauses) != len(CONDITIONS) + 1 + len(extra) + or not now < expiration <= signed_at + timedelta(minutes=15)): + raise SigV4Error("AccessDenied", "Policy must expire within fifteen minutes") + bound = {name: fields[name] for name in CONDITIONS if name != "bucket"} + bound["bucket"] = bucket + for name in extra: + bound[name] = fields[name] + expected_conditions = set(bound) + seen = set() + has_range = False + for entry in clauses: + if (isinstance(entry, list) and len(entry) == 3 and + entry[0] == "content-length-range"): + if (has_range or any(type(value) is not int for value in entry[1:]) + or not 0 <= entry[1] <= len(body) <= entry[2] <= max_bytes): + raise SigV4Error("AccessDenied", "Invalid signed size range") + has_range = True + continue + # Prefix policies may authorize the key, MIME type or a + # specifically named user-metadata field. This never permits + # prefix wildcards over checksum, signer scope, bucket, date, + # redirect or HTTP cache/retention semantics. + if (isinstance(entry, list) and len(entry) == 3 + and entry[0] == "starts-with" + and isinstance(entry[1], str) and entry[1].startswith("$")): + name = entry[1][1:] + if (name not in ("key", "Content-Type") + and not _METADATA_FIELD.fullmatch(name)): + raise SigV4Error("AccessDenied", "Unsupported POST wildcard field") + if (name in seen or name not in bound + or not isinstance(entry[2], str) + or not bound[name].startswith(entry[2])): + raise SigV4Error("AccessDenied", "POST field is outside signed prefix") + seen.add(name) + continue + if isinstance(entry, dict) and len(entry) == 1: + name, value = next(iter(entry.items())) + elif (isinstance(entry, list) and len(entry) == 3 + and entry[0] == "eq" and isinstance(entry[1], str) + and entry[1].startswith("$")): + name, value = entry[1][1:], entry[2] + else: + raise SigV4Error("AccessDenied", "Unsupported POST policy condition") + if name not in expected_conditions or name in seen or value != bound[name]: + raise SigV4Error("AccessDenied", "POST policy does not pin every form field") + seen.add(name) + if seen != expected_conditions or not has_range: + raise SigV4Error("AccessDenied", "Incomplete POST upload constraints") + secret = lookup_secret(access, "s3:PutObject") + if secret is None: + raise SigV4Error("AccessDenied", "Revoked or unauthorized POST credential") + key_bytes = _hmac(b"AWS4" + secret.encode(), day) + for element in (region, "s3", "aws4_request"): + key_bytes = _hmac(key_bytes, element) + signature = hmac.new(key_bytes, fields["policy"].encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(signature, fields["x-amz-signature"]): + raise SigV4Error("SignatureDoesNotMatch", "Browser form signature mismatch") + return key diff --git a/teledrive/s3/public_access.py b/teledrive/s3/public_access.py new file mode 100644 index 00000000..586ea013 --- /dev/null +++ b/teledrive/s3/public_access.py @@ -0,0 +1,60 @@ +"""Private-only Public Access Block S3 controls. + +TeleDrive denies anonymous/public object access and public ACLs globally. +The only truthful configuration is all four fields enabled. It is enforced +even before a bucket exists and cannot be relaxed with an AWS-style API call. +""" +from __future__ import annotations + +from xml.etree import ElementTree as ET +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException + +MAX_PUBLIC_ACCESS_XML_BYTES = 16 * 1024 +NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/" +FIELDS = ( + "BlockPublicAcls", "IgnorePublicAcls", + "BlockPublicPolicy", "RestrictPublicBuckets", +) + + +class PublicAccessError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code, self.status = code, status + + +def parse_public_access_xml(payload: bytes) -> None: + if len(payload) > MAX_PUBLIC_ACCESS_XML_BYTES: + raise PublicAccessError("MalformedXML", "Public access XML exceeds 16 KiB") + try: + root = SafeET.fromstring(payload) + except (ET.ParseError, DefusedXmlException) as exc: + raise PublicAccessError("MalformedXML", "Invalid public access XML") from exc + if (root.tag not in {"PublicAccessBlockConfiguration", + f"{{{NAMESPACE}}}PublicAccessBlockConfiguration"} + or root.attrib or len(root) != 4): + raise PublicAccessError("MalformedXML", "Unexpected public access document") + seen = set() + for element in root: + field = element.tag.rsplit("}", 1)[-1] + if (field not in FIELDS or field in seen + or element.tag not in {field, f"{{{NAMESPACE}}}{field}"} + or element.attrib or len(element)): + raise PublicAccessError("MalformedXML", "Unknown or duplicate public access field") + seen.add(field) + if element.text != "true": + if element.text == "false": + raise PublicAccessError( + "NotImplemented", "Disabling public access protection is unsupported", 501, + ) + raise PublicAccessError("InvalidArgument", "Public access fields must be true") + if seen != set(FIELDS): + raise PublicAccessError("MalformedXML", "Incomplete public access document") + + +def public_access_xml() -> bytes: + root = ET.Element("PublicAccessBlockConfiguration", xmlns=NAMESPACE) + for field in FIELDS: + ET.SubElement(root, field).text = "true" + return ET.tostring(root, encoding="utf-8", xml_declaration=True) diff --git a/teledrive/s3/route_guard.py b/teledrive/s3/route_guard.py new file mode 100644 index 00000000..f4d7ca04 --- /dev/null +++ b/teledrive/s3/route_guard.py @@ -0,0 +1,156 @@ +"""Fail-closed S3 query routing contract. + +The actual S3 handler still authenticates every operation with SigV4. This +module prevents *unrecognized* AWS control-plane subresources and ambiguous +query combinations from degrading into a normal bucket/object request. +This is a parameter-level guard, not a claim that all 116 S3 APIs exist. +""" +from __future__ import annotations + +from collections.abc import Mapping + +# Explicit allowlist of implemented selectors and their concrete parameters. +# SDK ?x-id=OperationName is a non-authoritative hint, not a selector; the +# signed HTTP method and URL are always authoritative for routing/auth. +_ROOT_BUCKET_LIST = frozenset({"max-buckets", "prefix", "bucket-region", "continuation-token"}) +_BUCKET_GET_V1 = frozenset({ + "prefix", "delimiter", "marker", "max-keys", "encoding-type", +}) +_BUCKET_GET_V2 = frozenset({ + "list-type", "prefix", "delimiter", "continuation-token", + "start-after", "max-keys", "encoding-type", +}) +_BUCKET_GET_VERSIONS = frozenset({ + "versions", "prefix", "max-keys", "key-marker", "version-id-marker", +}) +_BUCKET_GET_MULTIPART = frozenset({ + "uploads", "prefix", "delimiter", "encoding-type", + "key-marker", "upload-id-marker", "max-uploads", +}) +_BUCKET_CONTROL = frozenset({ + "location", "cors", "tagging", "versioning", "publicAccessBlock", "lifecycle", + "policyStatus", +}) +_OBJECT_CONTROL = frozenset({"tagging", "attributes", "uploads", "uploadId", "versionId"}) +_EMPTY_FLAG = frozenset({ + "location", "cors", "tagging", "versioning", "publicAccessBlock", + "lifecycle", "policyStatus", "uploads", "versions", "delete", "attributes", +}) +# No upper/lower-case folding for S3 query fields: AWS subresources are +# case-sensitive. A misspelled control may not be silently ignored. + + +class S3QueryError(ValueError): + def __init__(self, code: str, message: str, status: int): + super().__init__(message) + self.code, self.status = code, status + + +def validate_s3_query(method: str, bucket: str, object_path: bool, params) -> None: + """Reject unknown controls before SigV4 resolver/DB/media side effects. + + The only exceptions to the enumerated S3 params are AWS SigV4 X-Amz-* + query fields, verified downstream, and the SDK's non-authoritative x-id. + """ + keys = set(params) + if len(keys) > 64: + raise S3QueryError("InvalidArgument", "Too many query parameters", 400) + real = {key for key in keys if not key.startswith("X-Amz-") and key != "x-id"} + if "x-id" in keys: + hint = params.getlist("x-id") + if len(hint) != 1 or len(hint[0]) > 128: + raise S3QueryError("InvalidArgument", "Invalid SDK operation hint", 400) + + if not bucket: + allowed = _ROOT_BUCKET_LIST if method == "GET" else frozenset() + elif not object_path: + control = real & _BUCKET_CONTROL + if control: + if len(control) != 1: + raise S3QueryError("InvalidArgument", "Conflicting bucket controls", 400) + selector = next(iter(control)) + methods = { + "location": {"GET"}, "cors": {"GET", "PUT", "DELETE"}, + "tagging": {"GET", "PUT", "DELETE"}, + "versioning": {"GET", "PUT"}, + "publicAccessBlock": {"GET", "PUT"}, + "lifecycle": {"GET", "PUT", "DELETE"}, + "policyStatus": {"GET"}, + } + allowed = frozenset({selector}) if method in methods[selector] else frozenset() + elif method == "GET": + if "versions" in real: + allowed = _BUCKET_GET_VERSIONS + elif "uploads" in real: + allowed = _BUCKET_GET_MULTIPART + else: + allowed = _BUCKET_GET_V2 if "list-type" in real else _BUCKET_GET_V1 + elif method == "POST": + allowed = frozenset({"delete"}) if "delete" in real else frozenset() + else: + # HEAD/PUT/DELETE cannot treat ?policy/?acl/?prefix as regular + # bucket mutations. An unadorned request remains supported. + allowed = frozenset() + else: + if "uploadId" in real: + allowed = { + "PUT": frozenset({"uploadId", "partNumber"}), + "GET": frozenset({"uploadId", "part-number-marker", "max-parts"}), + "DELETE": frozenset({"uploadId"}), + "POST": frozenset({"uploadId"}), + }.get(method, frozenset()) + elif "uploads" in real: + allowed = frozenset({"uploads"}) if method == "POST" else frozenset() + elif "tagging" in real: + allowed = (frozenset({"tagging", "versionId"}) + if method in {"GET", "PUT", "DELETE"} else frozenset()) + elif "attributes" in real: + # GetObjectAttributes may address a published historical VersionId. + # Other object controls still fail closed, even if correctly signed. + allowed = (frozenset({"attributes", "versionId"}) + if method == "GET" else frozenset()) + elif "versionId" in real: + allowed = (frozenset({"versionId"}) + if method in {"GET", "HEAD", "DELETE"} else frozenset()) + else: + allowed = frozenset() + + # A recognized ListObjects cursor in the wrong API dialect is an + # InvalidArgument (400), not an unsupported AWS extension (501). + # This preserves AWS CLI/boto3 V1/V2 cursor error contracts. + if (bucket and not object_path and method == "GET" + and not (real & (_BUCKET_CONTROL | {"versions", "uploads"}))): + if ("list-type" not in real + and real & {"continuation-token", "start-after"}): + raise S3QueryError( + "InvalidArgument", "V2 cursor requires list-type=2", 400, + ) + if "list-type" in real and "marker" in real: + raise S3QueryError( + "InvalidArgument", "V1 marker cannot be used with V2 listing", 400, + ) + + forbidden = real - allowed + if forbidden: + # Fail closed: no unexpected list, create, overwrite, delete or + # anonymous body staging even for a valid signed request. + raise S3QueryError( + "NotImplemented", "Unsupported S3 query parameter or selector", 501, + ) + + for name in real: + values = params.getlist(name) + if len(values) != 1: + raise S3QueryError("InvalidArgument", "Duplicate S3 query parameter", 400) + if name in _EMPTY_FLAG and values[0] != "": + raise S3QueryError("InvalidArgument", "Unexpected selector value", 400) + + if "uploadId" in real and not params["uploadId"]: + raise S3QueryError("InvalidArgument", "Missing multipart UploadId", 400) + if "list-type" in real and params["list-type"] != "2": + raise S3QueryError("InvalidArgument", "Unsupported bucket listing mode", 400) + + # Not all whitelisted parameter combinations are valid: multipart UploadPart + # needs BOTH parameters; list cursors may not mix V1 and V2. + if object_path and method == "PUT" and "uploadId" in real and "partNumber" not in real: + raise S3QueryError("NotImplemented", "Multipart partNumber required", 501) diff --git a/teledrive/s3/sigv4.py b/teledrive/s3/sigv4.py new file mode 100644 index 00000000..5b98b37e --- /dev/null +++ b/teledrive/s3/sigv4.py @@ -0,0 +1,278 @@ +"""AWS Signature V4 authorization for path-style S3 requests. + +Supports signed headers and short-lived query presigns. A presigned PUT must +bind a SHA-256 checksum in its signed headers; anonymous mutable payload +presigns, AWS chunked streaming and SigV2 are deliberately refused. +""" +from __future__ import annotations + +import hashlib +import hmac +import re +from datetime import datetime, timezone +from urllib.parse import parse_qsl, quote, urlsplit + +from starlette.requests import Request + + +class SigV4Error(ValueError): + def __init__(self, code: str, message: str): + self.code, self.message = code, message + super().__init__(message) + + +def _sha(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _hmac(key: bytes, data: str) -> bytes: + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() + + +MAX_QUERY_BYTES = 16 * 1024 +MAX_QUERY_FIELDS = 100 + + +def _query_pairs(raw: bytes) -> list[tuple[str, str]]: + """Reject malformed or oversized wire queries before canonicalizing them. + + urllib.parse defaults to replacement characters on invalid UTF-8; a + signature verifier must not authenticate normalized substitute bytes. + """ + if len(raw) > MAX_QUERY_BYTES: + raise SigV4Error("InvalidArgument", "S3 query string exceeds 16 KiB") + try: + return parse_qsl( + raw.decode("ascii"), keep_blank_values=True, + encoding="utf-8", errors="strict", max_num_fields=MAX_QUERY_FIELDS, + ) + except (UnicodeError, ValueError) as exc: + raise SigV4Error("InvalidArgument", "Invalid S3 query string") from exc + + +def _canonical_query(raw: bytes) -> str: + pairs = _query_pairs(raw) + return "&".join( + f"{quote(key, safe='-_.~')}={quote(value, safe='-_.~')}" + for key, value in sorted(pairs, key=lambda x: (quote(x[0], safe='-_.~'), quote(x[1], safe='-_.~'))) + ) + + +def _canonical_path(request: Request) -> str: + # S3SigV4Auth (unlike generic SigV4Auth) deliberately does NOT re-encode + # or normalize the already URL-encoded request path. Preserve percent + # escapes, double slashes and dot segments byte-for-byte. + raw = request.scope.get("raw_path") or request.url.path.encode("utf-8") + try: + path = raw.decode("ascii") + except UnicodeDecodeError as exc: + raise SigV4Error("SignatureDoesNotMatch", "Invalid encoded URL path") from exc + if not path.startswith("/"): + raise SigV4Error("SignatureDoesNotMatch", "Invalid absolute URL path") + return path + + +def _presigned_signature(request: Request, lookup_secret, permission: str, + now: datetime | None) -> str: + if request.method not in {"GET", "HEAD", "PUT", "DELETE", "POST"}: + raise SigV4Error("NotImplemented", "Presigned S3 method unsupported") + if request.headers.get("authorization"): + raise SigV4Error("AccessDenied", "Ambiguous Authorization and query signature") + pairs = _query_pairs(request.scope.get("query_string", b"")) + required = { + "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", + "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Signature", + } + values = {} + for key, value in pairs: + if key in values: + raise SigV4Error("SignatureDoesNotMatch", "Repeated presigned query parameter") + values[key] = value + if not required.issubset(values): + raise SigV4Error("AccessDenied", "Incomplete presigned S3 URL") + if values["X-Amz-Algorithm"] != "AWS4-HMAC-SHA256": + raise SigV4Error("AccessDenied", "Only AWS Signature V4 is supported") + if "X-Amz-Security-Token" in values: + raise SigV4Error("InvalidToken", "Temporary AWS session tokens are unsupported") + credential = re.fullmatch( + r"([^/\s]{1,64})/(\d{8})/([a-z0-9-]{1,32})/s3/aws4_request", + values["X-Amz-Credential"], + ) + if credential is None: + raise SigV4Error("AccessDenied", "Invalid presigned credential scope") + access_key, date, region = credential.groups() + signed_string = values["X-Amz-SignedHeaders"] + signed = signed_string.split(";") + if (not signed or signed != sorted(set(signed)) or "host" not in signed + or any(not re.fullmatch(r"[a-z0-9-]+", x) for x in signed)): + raise SigV4Error("SignatureDoesNotMatch", "Invalid presigned SignedHeaders") + headers = {} + for name in signed: + matching = [v.decode("latin-1") for k, v in request.scope["headers"] + if k.decode("ascii").lower() == name] + if len(matching) != 1: + raise SigV4Error("SignatureDoesNotMatch", "Ambiguous or missing signed header") + headers[name] = " ".join(matching[0].strip().split()) + if request.method == "POST": + # Permit only checksum-bound Complete or empty-body Initiate. + # Both selectors are authenticated in the canonical query, and + # arbitrary POSTs (including browser form uploads) remain refused. + completing = ( + bool(values.get("uploadId")) + and len(values["uploadId"]) <= 80 + and all(name in required | {"uploadId"} for name in values) + ) + initiating = ( + "uploads" in values and values["uploads"] == "" + and all(name in required | {"uploads"} for name in values) + ) + if not (completing or initiating): + raise SigV4Error("NotImplemented", "Only bound multipart POST supported") + if ("x-amz-content-sha256" not in signed or not re.fullmatch( + r"[0-9a-f]{64}", headers.get("x-amz-content-sha256", "") + )): + raise SigV4Error("AccessDenied", "Multipart POST requires signed SHA-256") + if initiating and headers["x-amz-content-sha256"] != _sha(b""): + raise SigV4Error( + "XAmzContentSHA256Mismatch", + "Presigned multipart initiation must bind an empty request body", + ) + if request.method == "PUT": + # Never expose an S3 write URL which permits arbitrary replacement + # with a different payload by anyone who obtains the link. + if not {"x-amz-checksum-sha256", "x-amz-content-sha256"}.intersection(signed): + raise SigV4Error("AccessDenied", "Presigned PUT requires signed SHA-256 checksum") + if "x-amz-content-sha256" in signed and not re.fullmatch( + r"[0-9a-f]{64}", headers["x-amz-content-sha256"] + ): + raise SigV4Error("XAmzContentSHA256Mismatch", "Invalid signed upload checksum") + if "x-amz-checksum-sha256" in signed: + import base64 + import binascii + try: + checksum = base64.b64decode(headers["x-amz-checksum-sha256"], validate=True) + except (ValueError, binascii.Error) as exc: + raise SigV4Error("XAmzContentSHA256Mismatch", "Invalid signed upload checksum") from exc + if len(checksum) != 32: + raise SigV4Error("XAmzContentSHA256Mismatch", "Invalid signed upload checksum") + stamp = values["X-Amz-Date"] + if not re.fullmatch(r"\d{8}T\d{6}Z", stamp) or stamp[:8] != date: + raise SigV4Error("AccessDenied", "Invalid presigned date") + try: + instant = datetime.strptime(stamp, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) + expires = int(values["X-Amz-Expires"]) + except ValueError as exc: + raise SigV4Error("AccessDenied", "Invalid presigned expiration") from exc + if not (1 <= expires <= 604800): + raise SigV4Error("AccessDenied", "Presigned URL expiry must be 1..604800 seconds") + clock = now or datetime.now(timezone.utc) + if (clock - instant).total_seconds() > expires: + raise SigV4Error("AccessDenied", "Presigned URL expired") + if (instant - clock).total_seconds() > 900: + raise SigV4Error("RequestTimeTooSkewed", "Presigned timestamp too far in future") + signature = values["X-Amz-Signature"] + if not re.fullmatch(r"[0-9a-f]{64}", signature): + raise SigV4Error("SignatureDoesNotMatch", "Invalid signature format") + canonical_pairs = [(k, v) for k, v in pairs if k != "X-Amz-Signature"] + canonical_query = "&".join( + f"{quote(k, safe='-_.~')}={quote(v, safe='-_.~')}" + for k, v in sorted(canonical_pairs, key=lambda p: + (quote(p[0], safe='-_.~'), quote(p[1], safe='-_.~'))) + ) + # botocore's SigV4 canonical request uses the declared payload hash + # when x-amz-content-sha256 is present, including presigned POST. + # The signed header above pins the digest, and the multipart handler + # independently rehashes the received XML before committing it. + canonical_payload = ( + headers["x-amz-content-sha256"] + if "x-amz-content-sha256" in signed else "UNSIGNED-PAYLOAD" + ) + canonical = ( + f"{request.method}\n{_canonical_path(request)}\n{canonical_query}\n" + + "".join(f"{name}:{headers[name]}\n" for name in signed) + + f"\n{signed_string}\n{canonical_payload}" + ) + scope = f"{date}/{region}/s3/aws4_request" + string_to_sign = ( + f"AWS4-HMAC-SHA256\n{stamp}\n{scope}\n{_sha(canonical.encode())}" + ) + secret = lookup_secret(access_key, permission) + if secret is None: + raise SigV4Error("AccessDenied", "Unknown, expired, revoked or unauthorized access key") + key = _hmac(b"AWS4" + secret.encode(), date) + for word in (region, "s3", "aws4_request"): + key = _hmac(key, word) + expected = hmac.new(key, string_to_sign.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(expected, signature): + raise SigV4Error("SignatureDoesNotMatch", "Presigned signature mismatch") + return access_key + + +def verify_header_signature(request: Request, lookup_secret, permission: str, *, + now: datetime | None = None) -> str: + """Return authenticated access-key ID; never reveal the secret on failure. + + lookup_secret(access_key, permission) must atomically honor revocation and + scope. The body digest is separately checked by the upload handler. + """ + if "X-Amz-Algorithm" in request.query_params or "X-Amz-Signature" in request.query_params: + return _presigned_signature(request, lookup_secret, permission, now) + auth = request.headers.get("authorization", "") + match = re.fullmatch( + r"AWS4-HMAC-SHA256 Credential=([^\s,/]{1,64})/(\d{8})/([a-z0-9-]{1,32})/s3/aws4_request," + r"\s*SignedHeaders=([a-z0-9;\-]+),\s*Signature=([0-9a-f]{64})", + auth, + ) + if match is None: + raise SigV4Error("AccessDenied", "AWS Signature V4 Authorization required") + access_key, date, region, headers_part, actual_signature = match.groups() + signed = headers_part.split(";") + if signed != sorted(set(signed)) or "host" not in signed or "x-amz-date" not in signed: + raise SigV4Error("SignatureDoesNotMatch", "Invalid SignedHeaders") + headers: dict[str, str] = {} + for name in signed: + # Ambiguous repeated signed headers are rejected rather than combined. + values = [v.decode("latin-1") for k, v in request.scope["headers"] + if k.decode("ascii").lower() == name] + if len(values) != 1: + raise SigV4Error("SignatureDoesNotMatch", "Missing or ambiguous signed header") + headers[name] = " ".join(values[0].strip().split()) + request_date = request.headers.get("x-amz-date", "") + if not re.fullmatch(r"\d{8}T\d{6}Z", request_date) or request_date[:8] != date: + raise SigV4Error("RequestTimeTooSkewed", "Invalid signed timestamp") + try: + instant = datetime.strptime(request_date, "%Y%m%dT%H%M%SZ").replace(tzinfo=timezone.utc) + except ValueError as exc: + raise SigV4Error("RequestTimeTooSkewed", "Invalid request timestamp") from exc + clock = now or datetime.now(timezone.utc) + if abs((clock - instant).total_seconds()) > 900: + raise SigV4Error("RequestTimeTooSkewed", "Request timestamp outside 15-minute window") + payload = request.headers.get("x-amz-content-sha256", "") + if not re.fullmatch(r"[0-9a-f]{64}", payload): + if payload != "UNSIGNED-PAYLOAD": + raise SigV4Error("XAmzContentSHA256Mismatch", "Signed SHA-256 payload required") + if request.method in {"PUT", "POST"} and not ( + "content-md5" in signed or "x-amz-checksum-sha256" in signed + ): + raise SigV4Error( + "XAmzContentSHA256Mismatch", + "Unsigned upload requires a signed Content-MD5 or SHA-256 checksum", + ) + canonical = ( + f"{request.method}\n{_canonical_path(request)}\n" + f"{_canonical_query(request.scope.get('query_string', b''))}\n" + + "".join(f"{key}:{headers[key]}\n" for key in signed) + + f"\n{headers_part}\n{payload}" + ) + scope = f"{date}/{region}/s3/aws4_request" + to_sign = f"AWS4-HMAC-SHA256\n{request_date}\n{scope}\n{_sha(canonical.encode())}" + secret = lookup_secret(access_key, permission) + if secret is None: + raise SigV4Error("AccessDenied", "Unknown, expired, revoked or unauthorized access key") + key = _hmac(b"AWS4" + secret.encode(), date) + for word in (region, "s3", "aws4_request"): + key = _hmac(key, word) + expected = hmac.new(key, to_sign.encode(), hashlib.sha256).hexdigest() + if not hmac.compare_digest(expected, actual_signature): + raise SigV4Error("SignatureDoesNotMatch", "Request signature mismatch") + return access_key diff --git a/teledrive/s3/tagging.py b/teledrive/s3/tagging.py new file mode 100644 index 00000000..94a69109 --- /dev/null +++ b/teledrive/s3/tagging.py @@ -0,0 +1,213 @@ +"""S3 object tags stored per immutable Telegram object version. + +Object body bytes always live in Telegram. Tags are small SQL metadata only. +Writing tags on one version never mutates an older version or a CopyObject +source. An object head is locked before current-version tag mutation. +""" +from __future__ import annotations + +import re +import time + +from sqlalchemy import ( + Column, ForeignKey, String, Table, delete, insert, select, update, +) +from sqlalchemy.exc import IntegrityError + +from teledrive.storage.catalog import ( + BucketCatalog, heads, removed_versions, catalog_metadata, +) +from teledrive.storage.manifest import metadata as _meta, objects +object_tags = Table( + "storage_object_tags", _meta, + Column("object_id", String(36), ForeignKey("storage_objects.id"), primary_key=True), + Column("tag_key", String(512), primary_key=True), + Column("tag_value", String(1024), nullable=False), +) + + +bucket_tags = Table( + "storage_bucket_tags", catalog_metadata, + Column("bucket", String(63), ForeignKey("storage_buckets.name"), primary_key=True), + Column("tag_key", String(512), primary_key=True), + Column("tag_value", String(1024), nullable=False), +) + + +class TaggingError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code = code + self.status = status + + +def validate_tags(tags: list[tuple[str, str]], *, max_tags: int = 10) -> list[tuple[str, str]]: + if not isinstance(tags, list) or len(tags) > max_tags: + raise TaggingError("InvalidTag", f"Resource supports at most {max_tags} tags") + seen = set() + result = [] + for key, value in tags: + if not isinstance(key, str) or not isinstance(value, str): + raise TaggingError("InvalidTag", "Tag keys and values must be strings") + # AWS counts tag lengths in UTF-16 code units, not UTF-8 bytes. + try: + key_units = len(key.encode("utf-16-le")) // 2 + val_units = len(value.encode("utf-16-le")) // 2 + except UnicodeError as exc: + raise TaggingError("InvalidTag", "Invalid Unicode in tag") from exc + if (not 1 <= key_units <= 128 or val_units > 256 + or key.lower().startswith("aws:")): + raise TaggingError("InvalidTag", "Invalid key/value length or reserved aws: prefix") + if key in seen: + raise TaggingError("InvalidTag", "Duplicate tag key") + seen.add(key) + result.append((key, value)) + return result + + +class ObjectTagStore: + def __init__(self, catalog: BucketCatalog): + self.catalog = catalog + self.db = catalog.db + _meta.create_all(self.db.engine, tables=[object_tags]) + + def get(self, object_id: str) -> list[tuple[str, str]]: + return list(self.db.connection.execute( + select(object_tags.c.tag_key, object_tags.c.tag_value) + .where(object_tags.c.object_id == object_id) + .order_by(object_tags.c.tag_key) + ).all()) + + def replace_current(self, bucket: str, key: str, + tags: list[tuple[str, str]]) -> str: + tags = validate_tags(tags) + # Lock the current head row while replacing its version-scoped tags. + # On PostgreSQL/MySQL this UPDATE serializes concurrent head changes. + digest = self.catalog._keyhash(key) + try: + current = self.catalog.get(bucket, key) + guard = self.db.connection.execute(update(heads).where( + heads.c.bucket == bucket, heads.c.key_hash == digest, + heads.c.object_id == current.item.id, + heads.c.generation == current.generation, + ).values(updated_at=int(time.time()))) + if guard.rowcount != 1: + raise TaggingError("OperationAborted", "Object replaced during tagging", 409) + self.db.connection.execute(delete(object_tags).where( + object_tags.c.object_id == current.item.id + )) + for tag_key, tag_value in tags: + self.db.connection.execute(insert(object_tags).values( + object_id=current.item.id, tag_key=tag_key, tag_value=tag_value + )) + self.db.commit() + return current.item.id + except BaseException: + self.db.connection.rollback() + raise + + def replace_published(self, bucket: str, key: str, identifier: str, + tags: list[tuple[str, str]]) -> None: + """Mutate tags on only a proven published immutable version. + + Unknown/unpublished READY staging is never writable through VersionId. + SQL row lock fences concurrent physical metadata modifications while + the actual Telegram payload remains immutable. + """ + tags = validate_tags(tags) + try: + # Acquire the writer lock FIRST, before any consistent SELECT. + # MySQL REPEATABLE READ would otherwise pin a stale snapshot + # while waiting on a concurrent VersionId DELETE's object lock. + locked = self.db.connection.execute( + update(objects).where( + objects.c.id == identifier, + objects.c.bucket == bucket, + objects.c.object_key == key, + objects.c.status == "ready", + ).values(updated_at=objects.c.updated_at) + ) + if locked.rowcount != 1: + raise TaggingError("NoSuchVersion", "Object version unavailable", 404) + revoked = self.db.connection.execute( + select(removed_versions.c.object_id).where( + removed_versions.c.object_id == identifier + ).with_for_update() + ).scalar_one_or_none() + if revoked is not None or not self.catalog.is_visible_version( + bucket, key, identifier + ): + raise TaggingError("NoSuchVersion", "Object version revoked or unpublished", 404) + self.db.connection.execute( + delete(object_tags).where(object_tags.c.object_id == identifier) + ) + for tag_key, tag_value in tags: + self.db.connection.execute(insert(object_tags).values( + object_id=identifier, tag_key=tag_key, tag_value=tag_value, + )) + self.db.commit() + except BaseException: + self.db.connection.rollback() + raise + + def attach_uncommitted(self, object_id: str, + tags: list[tuple[str, str]]) -> None: + """Set initial tags before publishing head; caller commits both atomically.""" + for tag_key, tag_value in validate_tags(tags): + self.db.connection.execute(insert(object_tags).values( + object_id=object_id, tag_key=tag_key, tag_value=tag_value + )) + + def copy_tags(self, source_id: str, destination_id: str) -> None: + """Copy in the caller's transaction; never issue a separate commit.""" + for tag_key, tag_value in self.get(source_id): + self.db.connection.execute(insert(object_tags).values( + object_id=destination_id, tag_key=tag_key, tag_value=tag_value + )) + + +class BucketTagStore: + """Bucket tags are distinct metadata from object version tags.""" + + def __init__(self, catalog: BucketCatalog): + self.catalog = catalog + self.db = catalog.db + catalog_metadata.create_all(self.db.engine, tables=[bucket_tags]) + + def _ensure_bucket(self, bucket: str) -> None: + from teledrive.storage.catalog import buckets + present = self.db.connection.execute( + select(buckets.c.name).where(buckets.c.name == bucket) + ).first() + if present is None: + raise FileNotFoundError("Bucket not found") + + def get(self, bucket: str) -> list[tuple[str, str]]: + self._ensure_bucket(bucket) + return list(self.db.connection.execute( + select(bucket_tags.c.tag_key, bucket_tags.c.tag_value) + .where(bucket_tags.c.bucket == bucket) + .order_by(bucket_tags.c.tag_key) + ).all()) + + def replace(self, bucket: str, tags: list[tuple[str, str]]) -> None: + from teledrive.storage.catalog import buckets + tags = validate_tags(tags, max_tags=50) + try: + # Serialize concurrent bucket deletion/tag mutation. + guarded = self.db.connection.execute(update(buckets).where( + buckets.c.name == bucket + ).values(created_at=buckets.c.created_at)) + if guarded.rowcount != 1: + raise FileNotFoundError("Bucket not found") + self.db.connection.execute(delete(bucket_tags).where( + bucket_tags.c.bucket == bucket + )) + for key, value in tags: + self.db.connection.execute(insert(bucket_tags).values( + bucket=bucket, tag_key=key, tag_value=value + )) + self.db.commit() + except BaseException: + self.db.connection.rollback() + raise diff --git a/teledrive/s3/versioning.py b/teledrive/s3/versioning.py new file mode 100644 index 00000000..02cd698c --- /dev/null +++ b/teledrive/s3/versioning.py @@ -0,0 +1,61 @@ +"""Safe, immutable S3 bucket-versioning compatibility. + +TeleDrive versions are always-on: every published object keeps its original +VersionId and prior data. "Suspended" is NOT supported because acknowledging +it without implementing null-version semantics would risk client data loss. +""" +from __future__ import annotations + +from xml.etree import ElementTree as ET + +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException + +MAX_VERSIONING_XML_BYTES = 16 * 1024 +NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/" + + +class VersioningError(ValueError): + def __init__(self, code: str, message: str, status: int = 400): + super().__init__(message) + self.code, self.status = code, status + + +def _tag(element) -> str: + return element.tag.rsplit("}", 1)[-1] + + +def parse_versioning_xml(payload: bytes) -> None: + """Validate a supported idempotent Enabled request without side effects.""" + if len(payload) > MAX_VERSIONING_XML_BYTES: + raise VersioningError("MalformedXML", "Bucket versioning XML exceeds 16 KiB") + try: + root = SafeET.fromstring(payload) + except (ET.ParseError, DefusedXmlException) as exc: + raise VersioningError("MalformedXML", "Invalid bucket versioning XML") from exc + if (_tag(root) != "VersioningConfiguration" or root.attrib + or root.tag not in {"VersioningConfiguration", + f"{{{NAMESPACE}}}VersioningConfiguration"}): + raise VersioningError("MalformedXML", "Unexpected versioning document") + values = {} + for element in root: + field = _tag(element) + if (field not in {"Status", "MFADelete"} or element.attrib + or len(element) or field in values + or element.tag not in {field, f"{{{NAMESPACE}}}{field}"}): + raise VersioningError("MalformedXML", "Unexpected versioning field") + values[field] = element.text or "" + if values.get("MFADelete") is not None: + raise VersioningError("NotImplemented", "MFADelete is not implemented", 501) + if values.get("Status") == "Suspended": + raise VersioningError( + "NotImplemented", "Versioning suspension is unsupported", 501, + ) + if values != {"Status": "Enabled"}: + raise VersioningError("InvalidArgument", "Only Enabled bucket versioning is supported") + + +def versioning_xml() -> bytes: + root = ET.Element("VersioningConfiguration", xmlns=NAMESPACE) + ET.SubElement(root, "Status").text = "Enabled" + return ET.tostring(root, encoding="utf-8", xml_declaration=True) diff --git a/teledrive/s3/write_conditions.py b/teledrive/s3/write_conditions.py new file mode 100644 index 00000000..d010f880 --- /dev/null +++ b/teledrive/s3/write_conditions.py @@ -0,0 +1,82 @@ +"""Atomic S3 write/delete precondition evaluation. + +The condition is checked against the current object before uploading Telegram +bytes, then the exact generation AND immutable object ID are compare-and-set +at publication. Object-ID comparison prevents delete/recreate ABA races. +""" +from __future__ import annotations + +from dataclasses import dataclass + +from teledrive.s3.object_metadata import signed_header_names +from teledrive.s3.multipart import MultipartStore +from teledrive.storage.catalog import BucketCatalog + + +class WritePreconditionError(ValueError): + def __init__(self, code: str, message: str, status: int): + super().__init__(message) + self.code = code + self.status = status + + +@dataclass(frozen=True) +class ExpectedHead: + generation: int + object_id: str | None + + +def check_write_precondition(request, catalog: BucketCatalog, bucket: str, + key: str, *, delete: bool = False) -> ExpectedHead | None: + """Return a CAS snapshot or None if no write condition was requested. + + Conditions MUST be included in the SigV4 signed headers, otherwise an + intermediary could add/strip If-Match while reusing a valid signature. + """ + if_match = request.headers.get("if-match") + if_none_match = request.headers.get("if-none-match") + if if_match is None and if_none_match is None: + return None + if if_match is not None and if_none_match is not None: + raise WritePreconditionError("InvalidRequest", "Conflicting write preconditions", 400) + if delete and if_none_match is not None: + raise WritePreconditionError("NotImplemented", "Conditional DELETE If-None-Match is unsupported", 501) + + name = "if-match" if if_match is not None else "if-none-match" + if name not in signed_header_names(request): + raise WritePreconditionError("AccessDenied", "Write precondition must be signed", 403) + + try: + head = catalog.get(bucket, key) + except FileNotFoundError: + head = None + + if if_none_match is not None: + if if_none_match.strip() != "*": + raise WritePreconditionError("NotImplemented", "PUT If-None-Match supports only *", 501) + if head is not None: + raise WritePreconditionError("PreconditionFailed", "Object already exists", 412) + # A tombstone is absent for If-None-Match:* but its SQL HEAD + # still carries a CAS generation, preventing delete/recreate races. + if head is None: + from teledrive.storage.catalog import heads + from sqlalchemy import select + digest = catalog._keyhash(key) + marker = catalog.current_marker(bucket, key) + if marker is not None: + generation = catalog.db.connection.execute(select( + heads.c.generation + ).where(heads.c.bucket == bucket, heads.c.key_hash == digest)).scalar_one() + return ExpectedHead(generation=int(generation), object_id=marker) + return ExpectedHead(generation=0, object_id=None) + + if not if_match or (if_match != "*" and + not (len(if_match) >= 3 and if_match.startswith('"') + and if_match.endswith('"') and '"' not in if_match[1:-1])): + raise WritePreconditionError("InvalidArgument", "Invalid If-Match ETag", 400) + if head is None: + raise WritePreconditionError("PreconditionFailed", "Object does not exist", 412) + current_etag = MultipartStore(catalog).object_etag(head.item.id, head.item.sha256) + if if_match not in ("*", '"' + current_etag + '"'): + raise WritePreconditionError("PreconditionFailed", "ETag no longer matches", 412) + return ExpectedHead(generation=head.generation, object_id=head.item.id) diff --git a/teledrive/settings.py b/teledrive/settings.py new file mode 100644 index 00000000..6667cb5d --- /dev/null +++ b/teledrive/settings.py @@ -0,0 +1,207 @@ +"""Central service settings, loaded from config/config.yaml only. + +The owner policy: everything Telethon-only and config-driven. No service +settings come from environment variables except TZ and the config-file +path (TELEDRIVE_CONFIG). The WebUI admin config endpoints read and write +the same file, so UI changes persist to disk. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +import os +from pathlib import Path +from typing import Any + + +# Owner policy: the service binds every in-container interface; +# the host-side port mapping stays loopback. # nosec B104 +DEFAULT_BIND_ADDRESS = "0.0.0.0" # nosec B104 + + +@dataclass +class ServiceSettings: + web_user: str = "admin" + web_password: str = "" + web_password_file: str = "" + allowed_hosts: tuple[str, ...] = ("localhost", "127.0.0.1", "[::1]") + allow_insecure: bool = False + web_port: int = 8080 + s3_port: int = 9000 + bind_address: str = DEFAULT_BIND_ADDRESS + rest_max_upload_bytes: int = 2 * 1024**3 + max_request_bytes: int = 32 * 1024**2 + storage_scratch_dir: str = "/data/tmp" + job_source_root: str = "/uploads" + job_auto_run: bool = True + allowed_sqlite_roots: tuple[str, ...] = ("/data",) + auth_failure_limit: int = 5 + auth_failure_window_seconds: int = 300 + ssl_certfile: str = "" + ssl_keyfile: str = "" + db_extra_packages: tuple[str, ...] = () + auth_dir: str = "/data/auth" + config_path: str = "config/config.yaml" + s3_master_key_file: str = "/config/s3-master.key" + allowed_source_roots: str = "/uploads" + allowed_session_roots: str = "/data/sessions" + allowed_log_roots: str = "/data" + split_dir: str = "/data/tmp" + split_min_free_bytes: int = 512 * 1024 * 1024 + lock_dir: str = "/data/tmp" + + +def _section(cfg: Any, name: str) -> dict: + section = getattr(cfg, name, None) + if section is None: + return {} + if not isinstance(section, dict): + raise ValueError(f"Invalid {name} configuration section") + return section + + +def _hosts(value: Any, default: tuple[str, ...]) -> tuple[str, ...]: + if not value: + return default + if isinstance(value, str): + entries = [part.strip() for part in value.split(",") if part.strip()] + elif isinstance(value, list): + entries = [str(part).strip() for part in value if str(part).strip()] + else: + raise ValueError("Invalid allowed-hosts configuration") + if not entries: + return default + return tuple(entries) + + +def _int(value: Any, default: int, name: str, *, minimum: int = 0) -> int: + if value is None or value == "": + return default + parsed = int(value) + if parsed < minimum: + raise ValueError(f"Invalid {name} configuration") + return parsed + + +def load_settings(cfg: Any, config_path: str) -> ServiceSettings: + """Build service settings from the parsed deployment config. + + `cfg` is the AppConfig produced by teledrive.uploader.load_config; its + extra web/jobs/storage sections hold the service configuration. + """ + web = _section(cfg, "web") + jobs = _section(cfg, "jobs") + storage = _section(cfg, "storage") + state = _section(cfg, "state") + + paths = _section(cfg, "paths") + extra_packages = state.get("db_extra_packages", ()) + if not extra_packages: + packages = () + elif isinstance(extra_packages, str): + packages = tuple( + part.strip() for part in extra_packages.split(",") if part.strip() + ) + elif isinstance(extra_packages, (list, tuple)): + packages = tuple(str(part).strip() for part in extra_packages + if str(part).strip()) + else: + raise ValueError("Invalid db_extra_packages configuration") + + return ServiceSettings( + web_user=str(_cfg_or_env("user", "TELEDRIVE_WEB_USER", "admin")), + web_password=str(_cfg_or_env("password", "TELEDRIVE_WEB_PASSWORD", "")), + web_password_file=str(_cfg_or_env("password_file", "TELEDRIVE_WEB_PASSWORD_FILE", "")), + allowed_hosts=_hosts(_cfg_or_env("allowed_hosts", "TELEDRIVE_ALLOWED_HOSTS", "localhost,127.0.0.1,[::1]"), + ("localhost", "127.0.0.1", "[::1]")), + allow_insecure=bool( + _cfg_or_env("allow_insecure", "TELEDRIVE_ALLOW_INSECURE", False) + in (True, "1", "true", "yes", "True") + ), + web_port=_int(_cfg_or_env("port", "TELEDRIVE_WEB_PORT", 8080), 8080, + "web port", minimum=1), + s3_port=_int(_cfg_or_env("s3_port", "TELEDRIVE_S3_PORT", 9000), 9000, + "s3 port", minimum=1), + bind_address=str(_cfg_or_env("bind_address", "TELEDRIVE_BIND_ADDRESS", + DEFAULT_BIND_ADDRESS)), + rest_max_upload_bytes=_int( + _cfg_or_env("rest_max_upload_bytes", "TELEDRIVE_REST_MAX_UPLOAD_BYTES", 2 * 1024**3), + 2 * 1024**3, "rest_max_upload_bytes", minimum=1), + max_request_bytes=_int( + _cfg_or_env("max_request_bytes", "TELEDRIVE_MAX_REQUEST_BYTES", 32 * 1024**2), + 32 * 1024**2, "max_request_bytes", minimum=1), + storage_scratch_dir=str( + storage.get("scratch_dir") + or os.environ.get("TELEDRIVE_STORAGE_SCRATCH_DIR", "/data/tmp")), + job_source_root=str( + jobs.get("source_root") + or os.environ.get("TELEDRIVE_JOB_SOURCE_ROOT", "/uploads")), + job_auto_run=bool(jobs.get("auto_run", + os.environ.get("TELEDRIVE_JOB_AUTO_RUN", "1") == "1")), + allowed_sqlite_roots=_hosts( + state.get("allowed_sqlite_roots") + or os.environ.get("TELEDRIVE_ALLOWED_SQLITE_ROOTS", "/data"), + ("/data",)), + auth_failure_limit=_int( + _cfg_or_env("auth_failure_limit", "TELEDRIVE_AUTH_FAILURE_LIMIT", 5), + 5, "auth_failure_limit", minimum=1), + auth_failure_window_seconds=_int( + _cfg_or_env("auth_failure_window_seconds", + "TELEDRIVE_AUTH_FAILURE_WINDOW_SECONDS", 300), + 300, "auth_failure_window_seconds", minimum=1), + ssl_certfile=str(_cfg_or_env("ssl_certfile", "TELEDRIVE_SSL_CERTFILE", "")), + ssl_keyfile=str(_cfg_or_env("ssl_keyfile", "TELEDRIVE_SSL_KEYFILE", "")), + db_extra_packages=packages, + auth_dir=str(_cfg_or_env("auth_dir", "TELEDRIVE_AUTH_DIR", "/data/auth")), + config_path=config_path, + s3_master_key_file=str( + storage.get("s3_master_key_file") + or os.environ.get("TELEDRIVE_S3_MASTER_KEY_FILE", "/config/s3-master.key")), + allowed_source_roots=str( + paths.get("source_roots") + or os.environ.get("TELEDRIVE_ALLOWED_SOURCE_ROOTS", "/uploads")), + allowed_session_roots=str( + paths.get("session_roots") + or os.environ.get("TELEDRIVE_ALLOWED_SESSION_ROOTS", "/data/sessions")), + allowed_log_roots=str( + paths.get("log_roots") + or os.environ.get("TELEDRIVE_ALLOWED_LOG_ROOTS", "/data")), + split_dir=str( + paths.get("split_dir") + or os.environ.get("TELEDRIVE_SPLIT_DIR", "/data/tmp")), + split_min_free_bytes=_int( + paths.get("split_min_free_bytes") + or os.environ.get("TELEDRIVE_SPLIT_MIN_FREE_BYTES", 512 * 1024 * 1024), + 512 * 1024 * 1024, "split_min_free_bytes", minimum=1), + lock_dir=str( + paths.get("lock_dir") + or os.environ.get("TELEDRIVE_LOCK_DIR", "/data/tmp")), + ) + + +def install_db_extra_packages(settings: ServiceSettings, + target_dir: str) -> None: + """Install configured extra DB drivers into a writable runtime dir. + + The container root filesystem is read-only, so drivers go under the + data volume and are prepended to sys.path before any database URL is + opened. Failures are loud: a configured driver must import. + """ + if not settings.db_extra_packages: + return + target = Path(target_dir) + target.mkdir(parents=True, exist_ok=True) + import subprocess + import sys + + command = [ + sys.executable, "-m", "pip", "install", "--no-cache-dir", + "--target", str(target), + *settings.db_extra_packages, + ] + completed = subprocess.run(command, capture_output=True, text=True) + if completed.returncode != 0: + raise RuntimeError( + "Configured DB extra packages could not be installed: " + + (completed.stderr or completed.stdout)[-800:] + ) + sys.path.insert(0, str(target)) diff --git a/teledrive/storage/__init__.py b/teledrive/storage/__init__.py new file mode 100644 index 00000000..66b5aa77 --- /dev/null +++ b/teledrive/storage/__init__.py @@ -0,0 +1,11 @@ +"""Telegram-only object storage primitives. + +The metadata database stores manifests only; object bytes live in Telegram. +This is an early storage-engine foundation, not yet an S3 gateway. +""" + +from .engine import TelegramObjectStorage +from .manifest import ManifestStore, ChunkReference, StoredObject +from .recovery import RecoveryManager, CleanupReport, UnsafeCleanup + +__all__ = ["TelegramObjectStorage", "ManifestStore", "ChunkReference", "StoredObject", "RecoveryManager", "CleanupReport", "UnsafeCleanup"] diff --git a/teledrive/storage/catalog.py b/teledrive/storage/catalog.py new file mode 100644 index 00000000..7c1f9f8d --- /dev/null +++ b/teledrive/storage/catalog.py @@ -0,0 +1,658 @@ +"""Bucket and current-object catalog shared by future REST and S3 gateways. + +Telegram is the only payload backend. This module stores names, pointers and +immutable object IDs, never object body bytes. Previous object manifests +remain available after replacement/removal until explicit safe GC. +""" +from __future__ import annotations + +import hashlib +import re +import time +from dataclasses import dataclass +from uuid import uuid4 + +from sqlalchemy import ( + BigInteger, Column, ForeignKey, MetaData, String, Table, Index, + and_, or_, delete, insert, select, update, inspect, text, +) +from sqlalchemy.exc import IntegrityError + +from .manifest import ManifestStore, StoredObject, objects + +catalog_metadata = MetaData() +buckets = Table( + "storage_buckets", catalog_metadata, + Column("name", String(63), primary_key=True), + Column("created_at", BigInteger, nullable=False), +) +heads = Table( + "storage_object_heads", catalog_metadata, + Column("bucket", String(63), ForeignKey("storage_buckets.name"), primary_key=True), + Column("key_hash", String(64), primary_key=True), + Column("object_id", String(36), nullable=False), + Column("generation", BigInteger, nullable=False), + Column("updated_at", BigInteger, nullable=False), +) +published_versions = Table( + "storage_published_versions", catalog_metadata, + Column("object_id", String(36), primary_key=True), + Column("bucket", String(63), nullable=False), + Column("key_hash", String(64), nullable=False), + Column("committed_ns", BigInteger, nullable=False), +) +# Logical S3 VersionId deletion revokes visibility but NEVER removes Telegram +# media, manifests or aliases. Existing deployments receive an additive table. +removed_versions = Table( + "storage_removed_versions", catalog_metadata, + Column("object_id", String(36), primary_key=True), + Column("removed_ns", BigInteger, nullable=False), +) +Index("ix_storage_published_versions_key", + published_versions.c.bucket, published_versions.c.key_hash, + published_versions.c.committed_ns) + +_BUCKET_RE = re.compile(r"[a-z0-9](?:[a-z0-9.-]{1,61}[a-z0-9])?\Z") + + +@dataclass(frozen=True) +class ObjectHead: + item: StoredObject + generation: int + + +class BucketCatalog: + """Manage bucket names and a single current pointer per logical object key. + + The underlying manifest remains immutable. A newer version simply moves + the head pointer atomically; old Telegram chunks are not deleted. + """ + + def __init__(self, manifests: ManifestStore): + self.manifests = manifests + self.db = manifests.db + catalog_metadata.create_all(self.db.engine) + + @staticmethod + def validate_bucket(name: str) -> None: + if (not isinstance(name, str) or len(name) < 3 or len(name) > 63 + or not _BUCKET_RE.fullmatch(name) or ".." in name + or ".-" in name or "-." in name + or all(part.isdigit() for part in name.split("."))): + raise ValueError("Invalid S3-compatible bucket name") + + @staticmethod + def _keyhash(key: str) -> str: + if not isinstance(key, str) or not (1 <= len(key.encode("utf-8")) <= 1024): + raise ValueError("Object key must contain 1..1024 UTF-8 bytes") + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + def create_bucket(self, name: str, now: int) -> None: + self.validate_bucket(name) + try: + with self.db.connection.begin_nested(): + self.db.connection.execute(insert(buckets).values(name=name, created_at=now)) + except IntegrityError as exc: + raise ValueError("Bucket already exists") from exc + self.db.commit() + + def list_buckets(self) -> list[str]: + return list(self.db.connection.execute(select(buckets.c.name).order_by(buckets.c.name)).scalars()) + + def list_bucket_records(self, *, prefix: str = "", after: str = "", + limit: int = 10000) -> tuple[list[tuple[str, int]], bool]: + """SQL-only keyset page; return creation dates and a has-next marker. + + LIKE patterns are escaped to preserve literal S3 prefixes. SQL never + materializes all matching buckets to serve a single bounded page. + """ + if not 1 <= limit <= 10000: + raise ValueError("Invalid bucket page limit") + # Bucket names are strictly lowercase ASCII. SQLite LIKE can compare + # case-insensitively, so uppercase prefixes can never match. + if prefix != prefix.lower(): + return [], False + statement = select(buckets.c.name, buckets.c.created_at) + if prefix: + statement = statement.where(buckets.c.name.startswith(prefix, autoescape=True)) + if after: + statement = statement.where(buckets.c.name > after) + rows = self.db.connection.execute( + statement.order_by(buckets.c.name).limit(limit + 1), + ).all() + return [(str(r[0]), int(r[1])) for r in rows[:limit]], len(rows) > limit + + def get(self, bucket: str, key: str) -> ObjectHead: + digest = self._keyhash(key) + row = self.db.connection.execute( + select(heads.c.object_id, heads.c.generation) + .where(heads.c.bucket == bucket, heads.c.key_hash == digest) + ).first() + if row is None: + raise FileNotFoundError("Object key does not exist") + item = self.manifests.get(row[0]) + if item.bucket != bucket or item.key != key: + raise RuntimeError("Object key digest collision; refusing unsafe resolution") + return ObjectHead(item, int(row[1])) + + def is_visible_version(self, bucket: str, key: str, version_id: str) -> bool: + """Require exact published key proof AND absence of a removal record.""" + row = self.db.connection.execute( + select(published_versions.c.object_id) + .select_from(published_versions.join( + objects, objects.c.id == published_versions.c.object_id, + ).outerjoin( + removed_versions, + removed_versions.c.object_id == published_versions.c.object_id, + )).where( + published_versions.c.object_id == version_id, + published_versions.c.bucket == bucket, + published_versions.c.key_hash == self._keyhash(key), + objects.c.bucket == bucket, + objects.c.object_key == key, + objects.c.status.in_(("ready", "delete-marker")), + removed_versions.c.object_id.is_(None), + ) + ).scalar_one_or_none() + return row is not None + + def delete_marker(self, bucket: str, key: str, *, + expected_generation: int | None = None, + expected_object_id: str | None = None) -> str: + """Publish a zero-byte logical tombstone with one atomic head CAS. + + The marker is NOT a ready payload manifest; Telegram is never touched. + A concurrent write must not be overwritten by a stale DELETE. + """ + if bucket not in self.list_buckets(): + raise FileNotFoundError("Bucket does not exist") + digest = self._keyhash(key) + row = self.db.connection.execute(select( + heads.c.object_id, heads.c.generation, + ).where(heads.c.bucket == bucket, heads.c.key_hash == digest)).first() + if expected_object_id is not None and (row is None or row[0] != expected_object_id): + raise ValueError("Delete precondition object ID changed") + if expected_generation is not None and ( + (int(row[1]) if row else 0) != expected_generation + ): + raise ValueError("Delete precondition generation changed") + marker = str(uuid4()) + now = int(time.time()) + try: + self.db.connection.execute(insert(objects).values( + id=marker, bucket=bucket, object_key=key, + status="delete-marker", size=0, sha256=None, + created_at=now, updated_at=now, error=None, + )) + if row is None: + try: + with self.db.connection.begin_nested(): + self.db.connection.execute(insert(heads).values( + bucket=bucket, key_hash=digest, object_id=marker, + generation=1, updated_at=now, + )) + except IntegrityError as exc: + raise ValueError("Concurrent object creation") from exc + else: + changed = self.db.connection.execute(update(heads).where( + heads.c.bucket == bucket, heads.c.key_hash == digest, + heads.c.object_id == row[0], + heads.c.generation == row[1], + ).values(object_id=marker, generation=int(row[1]) + 1, updated_at=now)) + if changed.rowcount != 1: + raise ValueError("Concurrent object deletion/write") + self.db.connection.execute(insert(published_versions).values( + object_id=marker, bucket=bucket, key_hash=digest, + committed_ns=time.time_ns(), + )) + self.db.commit() + return marker + except BaseException: + self.db.connection.rollback() + raise + + def delete_version(self, bucket: str, key: str, version_id: str) -> bool: + """Hide an exact published VersionId, retaining immutable remote data. + + If the selected version owns HEAD, advance to the latest remaining + version (including a prior marker); this is a compare-and-swap in the + same SQL transaction as the version revocation. + """ + digest = self._keyhash(key) + try: + # Serialize revocation with version-specific tag edits on the + # underlying object row, even for historical (non-HEAD) versions. + # A second DELETE waits for this write lock and then observes the + # committed logical removal rather than leaking a unique-key error. + lock = self.db.connection.execute( + update(objects).where( + objects.c.id == version_id, + objects.c.status.in_(("ready", "delete-marker")), + ).values(updated_at=objects.c.updated_at) + ) + if lock.rowcount != 1: + self.db.connection.rollback() + return False + # A plain SELECT can read an obsolete REPEATABLE READ snapshot on + # MySQL, established earlier by the S3 credential lookup. This + # locking current-read observes another session's committed + # revocation even if our original snapshot predates it. + revoked = self.db.connection.execute( + select(removed_versions.c.object_id).where( + removed_versions.c.object_id == version_id, + ).with_for_update() + ).scalar_one_or_none() + if revoked is not None or not self.is_visible_version( + bucket, key, version_id + ): + self.db.connection.rollback() + return False + head = self.db.connection.execute(select( + heads.c.object_id, heads.c.generation, + ).where(heads.c.bucket == bucket, heads.c.key_hash == digest)).first() + self.db.connection.execute(insert(removed_versions).values( + object_id=version_id, removed_ns=time.time_ns(), + )) + if head is not None and head[0] == version_id: + previous = self.db.connection.execute( + select(published_versions.c.object_id) + .select_from(published_versions.join( + objects, objects.c.id == published_versions.c.object_id, + ).outerjoin(removed_versions, + removed_versions.c.object_id == published_versions.c.object_id, + )).where( + published_versions.c.bucket == bucket, + published_versions.c.key_hash == digest, + objects.c.object_key == key, + objects.c.status.in_(("ready", "delete-marker")), + removed_versions.c.object_id.is_(None), + ).order_by( + published_versions.c.committed_ns.desc(), + published_versions.c.object_id.desc(), + ).limit(1) + ).scalar_one_or_none() + selector = ( + heads.c.bucket == bucket, heads.c.key_hash == digest, + heads.c.object_id == version_id, + heads.c.generation == head[1], + ) + if previous is None: + changed = self.db.connection.execute(delete(heads).where(*selector)) + else: + changed = self.db.connection.execute(update(heads).where( + *selector + ).values(object_id=previous, generation=int(head[1]) + 1, + updated_at=int(time.time()))) + if changed.rowcount != 1: + raise ValueError("Version HEAD changed concurrently") + self.db.commit() + return True + except IntegrityError: + # An idempotent duplicate DELETE is not a server error. Keep the + # fail-closed behavior if a different constraint is violated. + self.db.connection.rollback() + if not self.is_visible_version(bucket, key, version_id): + return False + raise + except BaseException: + self.db.connection.rollback() + raise + + def current_marker(self, bucket: str, key: str) -> str | None: + digest = self._keyhash(key) + return self.db.connection.execute( + select(objects.c.id).select_from(heads.join( + objects, heads.c.object_id == objects.c.id, + )).where( + heads.c.bucket == bucket, heads.c.key_hash == digest, + objects.c.bucket == bucket, objects.c.object_key == key, + objects.c.status == "delete-marker", + ) + ).scalar_one_or_none() + + def _record_published_uncommitted(self, item: StoredObject) -> None: + """Append a proven head-version record in its publication transaction. + + Records never represent an incomplete upload, overwritten staged part, + or a failed conditional write. A historical HEAD predating this index + can be registered explicitly when an operator requests its versions. + """ + self.db.connection.execute(insert(published_versions).values( + object_id=item.id, bucket=item.bucket, + key_hash=self._keyhash(item.key), + committed_ns=time.time_ns(), + )) + + def index_legacy_current_heads(self, *, dry_run: bool = True, + limit: int = 100) -> list[str]: + """Backfill only proven current HEADs; unknown old versions stay hidden. + + An old READY manifest can be a never-published upload and is NOT safe + to promote. This backfill only trusts pointers already in HEADs. + """ + if not 1 <= limit <= 1000: + raise ValueError("Invalid history index limit") + missing = self.db.connection.execute( + select(heads.c.object_id, objects.c.bucket, objects.c.object_key) + .select_from( + heads.join(objects, heads.c.object_id == objects.c.id) + .outerjoin( + published_versions, + heads.c.object_id == published_versions.c.object_id, + ) + ) + .where( + published_versions.c.object_id.is_(None), + objects.c.status == "ready", + ) + .order_by(heads.c.bucket, heads.c.key_hash).limit(limit) + ).all() + ids = [] + for identifier, bucket, key in missing: + if self._keyhash(key) != self.db.connection.execute( + select(heads.c.key_hash).where( + heads.c.bucket == bucket, heads.c.object_id == identifier, + ) + ).scalar_one_or_none(): + raise RuntimeError("Legacy HEAD references inconsistent key metadata") + ids.append(identifier) + if not dry_run: + self._record_published_uncommitted( + self.manifests.get(identifier) + ) + if not dry_run and ids: + self.db.commit() + return ids + + def list_versions(self, bucket: str, key: str, *, + limit: int = 100) -> list[dict]: + """Read versions proven published, not all READY/staged manifests.""" + if not 1 <= limit <= 1000: + raise ValueError("Invalid version limit") + digest = self._keyhash(key) + try: + current = self.get(bucket, key).item.id + except FileNotFoundError: + current = None + rows = self.db.connection.execute( + select(published_versions.c.object_id, + published_versions.c.committed_ns, + objects.c.size, objects.c.sha256) + .select_from(published_versions.join( + objects, published_versions.c.object_id == objects.c.id + ).outerjoin( + removed_versions, + removed_versions.c.object_id == published_versions.c.object_id, + )) + .where( + removed_versions.c.object_id.is_(None), + published_versions.c.bucket == bucket, + published_versions.c.key_hash == digest, + objects.c.status == "ready", + objects.c.bucket == bucket, + objects.c.object_key == key, + ).order_by( + published_versions.c.committed_ns.desc(), + published_versions.c.object_id.desc(), + ).limit(limit) + ).mappings().all() + return [ + { + "version_id": row["object_id"], + "size": int(row["size"]), + "sha256": row["sha256"], + "committed_ns": int(row["committed_ns"]), + "is_current": row["object_id"] == current, + } + for row in rows + ] + + def list_bucket_versions( + self, bucket: str, *, prefix: str = "", key_marker: str = "", + version_marker: str = "", limit: int = 1000, + ) -> tuple[list[dict], bool]: + """Bounded, deterministic S3 listing of proven published history. + + Never include READY manifests that did not win HEAD publication. + The pagination marker must correspond to a published version of + exactly the supplied bucket/key; dangling or forged markers fail. + Delete markers are modeled as SQL-only published tombstones. + """ + if not 1 <= limit <= 1000 or len(prefix.encode("utf-8")) > 1024: + raise ValueError("Invalid version-list pagination") + if version_marker and not key_marker: + raise ValueError("Version-id marker requires key marker") + where = [ + published_versions.c.bucket == bucket, + objects.c.bucket == bucket, + objects.c.status.in_(("ready", "delete-marker")), + removed_versions.c.object_id.is_(None), + ] + if prefix: + where.append(objects.c.object_key.startswith(prefix, autoescape=True)) + if key_marker: + self._keyhash(key_marker) + if version_marker: + selected = self.db.connection.execute( + select(published_versions.c.committed_ns) + .select_from(published_versions.join( + objects, published_versions.c.object_id == objects.c.id + ).outerjoin( + removed_versions, + removed_versions.c.object_id == published_versions.c.object_id, + )).where( + removed_versions.c.object_id.is_(None), + published_versions.c.bucket == bucket, + objects.c.object_key == key_marker, + published_versions.c.object_id == version_marker, + objects.c.status.in_(("ready", "delete-marker")), + ) + ).scalar_one_or_none() + if selected is None: + raise ValueError("Unknown version-id marker") + where.append(or_( + objects.c.object_key > key_marker, + and_( + objects.c.object_key == key_marker, + or_( + published_versions.c.committed_ns < selected, + and_( + published_versions.c.committed_ns == selected, + published_versions.c.object_id < version_marker, + ), + ), + ), + )) + else: + where.append(objects.c.object_key > key_marker) + rows = self.db.connection.execute( + select( + published_versions.c.object_id, + published_versions.c.committed_ns, + objects.c.object_key, objects.c.size, objects.c.sha256, + objects.c.status.label("object_status"), + heads.c.object_id.label("current_id"), + ) + .select_from( + published_versions.join( + objects, published_versions.c.object_id == objects.c.id + ).outerjoin( + removed_versions, + removed_versions.c.object_id == published_versions.c.object_id, + ).outerjoin( + heads, + and_( + heads.c.bucket == published_versions.c.bucket, + heads.c.key_hash == published_versions.c.key_hash, + heads.c.object_id == published_versions.c.object_id, + ), + ) + ) + .where(*where) + .order_by( + objects.c.object_key, + published_versions.c.committed_ns.desc(), + published_versions.c.object_id.desc(), + ).limit(limit + 1) + ).mappings().all() + return [ + { + "key": row["object_key"], + "version_id": row["object_id"], + "committed_ns": int(row["committed_ns"]), + "size": int(row["size"]), + "sha256": row["sha256"], + "is_delete_marker": row["object_status"] == "delete-marker", + "is_current": row["current_id"] is not None, + } + for row in rows[:limit] + ], len(rows) > limit + + def publish(self, identifier: str, *, now: int, + expected_generation: int | None = None, + expected_object_id: str | None = None) -> ObjectHead: + """Atomically update the current pointer after Telegram upload succeeds. + + expected_generation=0 is create-if-absent, and a positive value is + optimistic concurrency control. None permits last-writer-wins. + """ + item = self.manifests.get(identifier) + digest = self._keyhash(item.key) + if item.bucket not in self.list_buckets(): + raise ValueError("Bucket does not exist") + existing = self.db.connection.execute( + select(heads.c.object_id, heads.c.generation) + .where(heads.c.bucket == item.bucket, heads.c.key_hash == digest) + ).first() + if existing is None: + if expected_object_id is not None: + raise ValueError("Object changed; object ID precondition failed") + if expected_generation not in (None, 0): + raise ValueError("Object changed; generation precondition failed") + try: + with self.db.connection.begin_nested(): + self.db.connection.execute(insert(heads).values( + bucket=item.bucket, key_hash=digest, object_id=item.id, + generation=1, updated_at=now, + )) + except IntegrityError as exc: + raise ValueError("Concurrent object creation; retry with generation") from exc + self._record_published_uncommitted(item) + self.db.commit() + return ObjectHead(item, 1) + + old = self.db.connection.execute(select( + objects.c.bucket, objects.c.object_key, objects.c.status, + ).where(objects.c.id == existing[0])).one_or_none() + if old is None or old[0] != item.bucket or old[1] != item.key: + raise RuntimeError("Object hash collision detected") + if old[2] not in ("ready", "delete-marker"): + raise RuntimeError("Current head refers to a non-published object") + generation = int(existing[1]) + if expected_object_id is not None and expected_object_id != existing[0]: + raise ValueError("Object changed; object ID precondition failed") + if expected_generation is not None and expected_generation != generation: + raise ValueError("Object changed; generation precondition failed") + changed = self.db.connection.execute( + update(heads) + .where( + heads.c.bucket == item.bucket, heads.c.key_hash == digest, + heads.c.object_id == existing[0], + heads.c.generation == generation, + ) + .values(object_id=item.id, generation=generation + 1, updated_at=now) + ) + if changed.rowcount != 1: + self.db.connection.rollback() + raise ValueError("Concurrent object modification") + self._record_published_uncommitted(item) + self.db.commit() + return ObjectHead(item, generation + 1) + + def list_objects(self, bucket: str, *, prefix: str = "", + after: str | None = None, limit: int = 100) -> list[ObjectHead]: + if not 1 <= limit <= 1000: + raise ValueError("Invalid page limit") + stmt = ( + select(heads.c.generation, objects) + .select_from(heads.join(objects, heads.c.object_id == objects.c.id)) + .where(heads.c.bucket == bucket, objects.c.status == "ready") + ) + if prefix: + stmt = stmt.where(objects.c.object_key.startswith(prefix, autoescape=True)) + if after is not None: + stmt = stmt.where(objects.c.object_key > after) + rows = self.db.connection.execute( + stmt.order_by(objects.c.object_key, objects.c.id).limit(limit) + ).mappings().all() + return [ObjectHead( + StoredObject( + id=row["id"], bucket=row["bucket"], key=row["object_key"], + size=int(row["size"]), sha256=row["sha256"], + created_at=int(row["created_at"]), + ), int(row["generation"]), + ) for row in rows] + + def remove_current(self, bucket: str, key: str, *, + expected_generation: int | None = None, + expected_object_id: str | None = None) -> bool: + """Unlink a head only; Telegram data stays until separately verified GC.""" + digest = self._keyhash(key) + try: + current = self.get(bucket, key) + except FileNotFoundError: + return False + if expected_generation is not None and current.generation != expected_generation: + raise ValueError("Object changed; generation precondition failed") + if expected_object_id is not None and current.item.id != expected_object_id: + raise ValueError("Object changed; object ID precondition failed") + condition = [ + heads.c.bucket == bucket, heads.c.key_hash == digest, + heads.c.object_id == current.item.id, + heads.c.generation == current.generation, + ] + changed = self.db.connection.execute(delete(heads).where(and_(*condition))) + self.db.commit() + return changed.rowcount == 1 + + def delete_bucket(self, name: str) -> None: + """Refuse buckets with any current or historical manifests.""" + existing = self.db.connection.execute( + select(buckets.c.name).where(buckets.c.name == name) + ).first() + if existing is None: + raise FileNotFoundError("Bucket does not exist") + history = self.db.connection.execute( + select(objects.c.id).where(objects.c.bucket == name).limit(1) + ).first() + # A bucket with initiated multipart uploads must not disappear, even + # before the first part is uploaded. Retain all abort/completion + # history until a verified lifecycle GC can safely remove it. + if inspect(self.db.engine).has_table("storage_multipart_uploads"): + mpu_history = self.db.connection.execute( + text("SELECT id FROM storage_multipart_uploads WHERE bucket = :bucket LIMIT 1"), + {"bucket": name}, + ).first() + if mpu_history is not None: + raise ValueError("Bucket has multipart history; safe GC required") + if history is not None: + raise ValueError("Bucket has object history; safe physical GC required") + # A physically empty bucket can be deleted even if its tag set is + # nonempty; do not leave dangling metadata across SQL backends. + if inspect(self.db.engine).has_table("storage_bucket_tags"): + self.db.connection.execute( + text("DELETE FROM storage_bucket_tags WHERE bucket = :bucket"), + {"bucket": name}, + ) + if inspect(self.db.engine).has_table("storage_bucket_lifecycle"): + self.db.connection.execute( + text("DELETE FROM storage_bucket_lifecycle WHERE bucket = :bucket"), + {"bucket": name}, + ) + if inspect(self.db.engine).has_table("storage_bucket_cors"): + self.db.connection.execute( + text("DELETE FROM storage_bucket_cors WHERE bucket = :bucket"), + {"bucket": name}, + ) + self.db.connection.execute(delete(buckets).where(buckets.c.name == name)) + self.db.commit() diff --git a/teledrive/storage/copy.py b/teledrive/storage/copy.py new file mode 100644 index 00000000..84364b02 --- /dev/null +++ b/teledrive/storage/copy.py @@ -0,0 +1,154 @@ +"""Atomic server-side S3 object copy without uploading Telegram media again. + +The new object owns a distinct SQL manifest and head pointer, but refers to +the immutable original chunk journal. This is deliberately NOT a copy of +Telegram messages: future GC must preserve media while aliases reference it. +""" +from __future__ import annotations + +import time +from dataclasses import dataclass +from uuid import uuid4 + +from sqlalchemy import insert, select, update +from sqlalchemy.exc import IntegrityError + +from teledrive.storage.catalog import BucketCatalog, heads +from teledrive.storage.manifest import object_aliases, objects +from teledrive.s3.multipart import MultipartStore, multipart_etags +from teledrive.s3.tagging import ObjectTagStore +from teledrive.s3.object_metadata import ObjectMetadataStore + + +class CopyConflict(ValueError): + """Concurrent destination object creation or replacement.""" + + +@dataclass(frozen=True) +class CopiedObject: + object_id: str + etag: str + size: int + source_object_id: str + + +def copy_object( + catalog: BucketCatalog, source_bucket: str, source_key: str, + destination_bucket: str, destination_key: str, *, now: int | None = None, + expected_source_id: str | None = None, + replacement_tags: list[tuple[str, str]] | None = None, + replacement_metadata: dict[str, str] | None = None, + source_version_id: str | None = None, + expected_destination_generation: int | None = None, + expected_destination_object_id: str | None = None, +) -> CopiedObject: + """Publish a snapshot copy with one SQL transaction. + + A copy of a copy points directly to the original content manifest, so + chains cannot grow indefinitely. It works across buckets using the same + Telegram storage, and preserves S3 multipart ETags where applicable. + """ + db = catalog.db + manifests = catalog.manifests + now = int(time.time()) if now is None else now + if source_version_id is not None: + # The caller may restore only a version previously published to this + # exact bucket/key, never arbitrary READY objects or staged MPU parts. + if not catalog.is_visible_version(source_bucket, source_key, source_version_id): + raise FileNotFoundError("Published object version not found") + source = manifests.get(source_version_id) + if source.bucket != source_bucket or source.key != source_key: + raise ValueError("Version source metadata mismatch") + else: + source = catalog.get(source_bucket, source_key).item + if expected_source_id is not None and source.id != expected_source_id: + raise CopyConflict("Copy source object changed during request") + base_id = manifests.resolve_content_id(source.id) + base = manifests.get(base_id) + if base.size != source.size or base.sha256 != source.sha256: + raise ValueError("Copy source manifest integrity mismatch") + catalog._keyhash(destination_key) + if destination_bucket not in catalog.list_buckets(): + raise FileNotFoundError("Destination bucket not found") + + multipart = MultipartStore(catalog) + tagging = ObjectTagStore(catalog) + original_tag = multipart.object_etag(source.id, source.sha256) + identifier = str(uuid4()) + key_hash = catalog._keyhash(destination_key) + + try: + db.connection.execute(insert(objects).values( + id=identifier, bucket=destination_bucket, object_key=destination_key, + status="ready", size=source.size, sha256=source.sha256, + created_at=now, updated_at=now, error=None, + )) + db.connection.execute(insert(object_aliases).values( + object_id=identifier, source_object_id=base_id, + )) + # COPY snapshots the source's tag set; REPLACE attaches only the + # caller's validated tags. Both happen before the head-pointer commit. + if replacement_tags is None: + tagging.copy_tags(source.id, identifier) + else: + tagging.attach_uncommitted(identifier, replacement_tags) + http = ObjectMetadataStore(catalog) + if replacement_metadata is None: + http.copy_uncommitted(source.id, identifier) + else: + http.attach_uncommitted(identifier, replacement_metadata) + if original_tag != source.sha256: + db.connection.execute(insert(multipart_etags).values( + object_id=identifier, etag=original_tag, + )) + # Preserve actual selected part boundaries as immutable metadata. + # Without this, a copied historical multipart ETag could claim a + # part count while GetObjectAttributes cannot prove its contents. + multipart.snapshot_copy_parts(source.id, identifier, source.size) + + current = db.connection.execute(select( + heads.c.object_id, heads.c.generation + ).where( + heads.c.bucket == destination_bucket, heads.c.key_hash == key_hash + )).first() + if current is None: + if expected_destination_object_id is not None or ( + expected_destination_generation not in (None, 0) + ): + raise CopyConflict("Destination generation changed") + db.connection.execute(insert(heads).values( + bucket=destination_bucket, key_hash=key_hash, + object_id=identifier, generation=1, updated_at=now, + )) + else: + if expected_destination_generation is not None and ( + expected_destination_generation != int(current[1]) + or expected_destination_object_id != current[0] + ): + raise CopyConflict("Destination generation changed") + old = db.connection.execute(select( + objects.c.bucket, objects.c.object_key, objects.c.status, + ).where(objects.c.id == current[0])).one_or_none() + if (old is None or old[0] != destination_bucket or old[1] != destination_key + or old[2] not in ("ready", "delete-marker")): + raise ValueError("Destination hash collision") + updated = db.connection.execute(update(heads).where( + heads.c.bucket == destination_bucket, + heads.c.key_hash == key_hash, + heads.c.object_id == current[0], + heads.c.generation == current[1], + ).values( + object_id=identifier, generation=current[1] + 1, + updated_at=now, + )) + if updated.rowcount != 1: + raise CopyConflict("Destination was changed concurrently") + catalog._record_published_uncommitted(manifests.get(identifier)) + db.commit() + return CopiedObject(identifier, original_tag, source.size, source.id) + except IntegrityError as exc: + db.connection.rollback() + raise CopyConflict("Concurrent copy or destination mutation") from exc + except BaseException: + db.connection.rollback() + raise diff --git a/teledrive/storage/cutover_acceptance.py b/teledrive/storage/cutover_acceptance.py new file mode 100644 index 00000000..c4833284 --- /dev/null +++ b/teledrive/storage/cutover_acceptance.py @@ -0,0 +1,206 @@ +"""Read-only production cutover preflight: local source -> SQL HEAD -> Telegram. + +This is an acceptance gate, not an uploader. It cannot switch the live +legacy worker, mutate SQL, or delete Telegram media. A release operator +must separately configure durable cutover after a complete verified report. +""" +from __future__ import annotations + +import hashlib +import os +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +from sqlalchemy import select + +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage, _regular_source +from teledrive.storage.jobs import ( + StorageUploadJobs, UploadJobError, _source_identity, jobs, +) + +def _revision(info: os.stat_result) -> tuple[int, int, int, int]: + return ( + int(info.st_dev), int(info.st_ino), + int(info.st_size), int(info.st_mtime_ns), + ) + + +def _hash_same_source_revision( + source: Path, root: Path, expected: os.stat_result, +) -> str: + """Hash the exact regular-file inode without ever following final symlinks. + + A concurrent rename/symlink replacement between path validation and open + must not cause verification to read arbitrary files outside source root. + """ + resolved, initial = _source_identity(source, root) + if resolved != source or _revision(initial) != _revision(expected): + raise UploadJobError("Legacy source revision changed before checksum") + with _regular_source(source) as handle: + if _revision(os.fstat(handle.fileno())) != _revision(expected): + raise UploadJobError("Legacy source revision changed during open") + digest = hashlib.sha256() + while block := handle.read(1024 * 1024): + digest.update(block) + if _revision(os.fstat(handle.fileno())) != _revision(expected): + raise UploadJobError("Legacy source revision changed while hashing") + resolved, final = _source_identity(source, root) + if resolved != source or _revision(final) != _revision(expected): + raise UploadJobError("Legacy source revision changed after checksum") + return digest.hexdigest() + + +async def verify_legacy_cutover( + cfg: Any, queue: StorageUploadJobs, transport: Any, *, + bucket: str, prefix: str = "", approved_root: Path, + max_files: int = 1000, max_bytes: int = 64 * 1024 * 1024, + scratch_directory: Path | None = None, +) -> dict: + """Only report complete when EVERY eligible local file is remotely read back. + + The caller must pause the legacy uploader and hold the global cutover + lease externally before collecting a production acceptance report. + A source revision with no successful queue job never qualifies. + """ + from teledrive.uploader import iter_source_files, file_allowed + + if (type(max_files) is not int or not 1 <= max_files <= 1000 + or type(max_bytes) is not int or not 1 <= max_bytes <= 1024**3): + raise ValueError("Invalid cutover verification bounds") + root = Path(cfg.source_dir).resolve(strict=True) + permitted = Path(approved_root).resolve(strict=True) + if (not root.is_dir() or not root.is_relative_to(permitted) + or bucket not in queue.catalog.list_buckets() + or prefix.startswith("/") or "\\" in prefix + or any(part in ("", ".", "..") for part in prefix.split("/") if prefix)): + raise UploadJobError("Unsafe or unavailable cutover source/bucket/prefix") + manifests = queue.catalog.manifests + engine = TelegramObjectStorage( + manifests, transport, scratch_directory=scratch_directory, + ) + total = excluded = verified = queued = inconsistent = 0 + truncated = False + initial_paths = [] + verified_revisions: dict[ + str, tuple[os.stat_result, str, str, str, int, str] + ] = {} + # Materialize at most max_files + 1 eligible entries. The original + # sorted(os.walk(...)) scanned an unbounded number of paths into RAM + # before noticing its max_files cap. + for raw_path in iter_source_files(str(root)): + if not file_allowed(raw_path, cfg): + excluded += 1 + continue + initial_paths.append(raw_path) + if len(initial_paths) > max_files: + truncated = True + break + baseline = set(initial_paths) + if not truncated: + for raw_path in sorted(initial_paths): + if not file_allowed(raw_path, cfg): + excluded += 1 + continue + total += 1 + if total > max_files: + truncated = True + break + try: + source, before = _source_identity(Path(raw_path), root) + if before.st_size > max_bytes: + inconsistent += 1 + continue + relative = source.relative_to(root).as_posix() + key = f"{prefix}/{relative}" if prefix else relative + job_id = queue.scan_id(bucket, key, source, before) + task = queue.db.connection.execute( + select(jobs).where(jobs.c.id == job_id) + ).mappings().one_or_none() + if task is None or task["state"] != "done" or not task["object_id"]: + queued += 1 + continue + if task["bucket"] != bucket or task["object_key"] != key: + inconsistent += 1 + continue + live_record = queue.catalog.get(bucket, key) + live_head = live_record.item + if live_head.id != task["object_id"] or live_head.size != before.st_size: + inconsistent += 1 + continue + source_sha = _hash_same_source_revision(source, root, before) + if live_head.sha256 != source_sha: + inconsistent += 1 + continue + remote_hash = hashlib.sha256() + received = 0 + async for chunk in engine.iter_bytes(live_head.id): + remote_hash.update(chunk) + received += len(chunk) + final_stat = _source_identity(source, root)[1] + if (received != before.st_size + or remote_hash.hexdigest() != source_sha + or _revision(before) != _revision(final_stat)): + inconsistent += 1 + continue + verified += 1 + verified_revisions[raw_path] = ( + before, source_sha, key, live_head.id, + live_record.generation, job_id, + ) + except (OSError, ValueError, FileNotFoundError): + inconsistent += 1 + if not truncated: + # Files added/removed while Telegram was being read must block the + # switchover, even if every file seen at the start passed its SHA. + after_paths = set() + for raw_path in iter_source_files(str(root)): + if file_allowed(raw_path, cfg): + after_paths.add(raw_path) + if len(after_paths) > max_files: + truncated = True + break + if not truncated and after_paths != baseline: + inconsistent += len(after_paths.symmetric_difference(baseline)) + if not truncated: + # Paths staying constant is NOT sufficient. An earlier verified + # source may be modified while a later file is downloaded, then + # have its original mtime restored. Independently re-hash EVERY + # verified source at the end, when all Telegram reads are done. + for raw_path, ( + original_stat, source_sha, key, head_id, + generation, job_id, + ) in verified_revisions.items(): + try: + source = Path(raw_path) + if _hash_same_source_revision(source, root, original_stat) != source_sha: + inconsistent += 1 + continue + # Publication may change during another object's + # Telegram read. A previously valid source SHA is not + # enough if the serving HEAD/generation has moved. + final_head = queue.catalog.get(bucket, key) + final_job = queue.db.connection.execute( + select(jobs.c.state, jobs.c.object_id) + .where(jobs.c.id == job_id) + ).one_or_none() + if (final_head.item.id != head_id + or final_head.generation != generation + or final_job is None + or tuple(final_job) != ("done", head_id)): + inconsistent += 1 + except (OSError, ValueError, FileNotFoundError): + inconsistent += 1 + + return { + "verified": total > 0 and not truncated and total == verified + and queued == 0 and inconsistent == 0, + "eligible_files": min(total, max_files), + "verified_files": verified, + "not_yet_published": queued, + "inconsistent": inconsistent, + "excluded": excluded, "truncated": truncated, + "telegram_uploads": 0, "telegram_deletions": 0, + "source_modifications": 0, + } diff --git a/teledrive/storage/engine.py b/teledrive/storage/engine.py new file mode 100644 index 00000000..1e8bc14a --- /dev/null +++ b/teledrive/storage/engine.py @@ -0,0 +1,410 @@ +"""Telegram-backed immutable object write/read engine. + +Each successful write stores all bytes in Telegram and atomically publishes a +SQL manifest. Staging is temporary; it is NEVER a durable S3 primary backend. +A failed/interrupted upload remains invisible until explicitly reconciled. + +This module intentionally does not implement S3's HTTP semantics yet. +""" +from __future__ import annotations + +import asyncio +import hashlib +import secrets +import os +import stat +import tempfile +import time +from pathlib import Path +from typing import AsyncIterator, Protocol + +from sqlalchemy import select + +from .manifest import ChunkReference, ManifestStore, StoredObject, ManifestError, objects + + +class IntegrityError(RuntimeError): + """Source or Telegram chunk is missing, truncated or checksum-mismatched.""" + + +class TelegramTransport(Protocol): + async def upload(self, path: Path, *, object_id: str, ordinal: int, + checksum: str) -> tuple[str, int, int]: + """Upload bytes and return (account_name, Telegram chat_id, message_id).""" + ... + + async def download(self, reference: ChunkReference, destination: Path) -> None: + """Download exactly one Telegram message body to a local regular file.""" + ... + + +def _regular_source(path: Path): + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(path, flags) + handle = os.fdopen(fd, "rb") + if not stat.S_ISREG(os.fstat(handle.fileno()).st_mode): + handle.close() + raise ValueError("Source must be a regular file") + return handle + + +def _digest_file(path: Path) -> tuple[int, str]: + sha = hashlib.sha256() + total = 0 + with _regular_source(path) as f: + while block := f.read(1024 * 1024): + sha.update(block) + total += len(block) + return total, sha.hexdigest() + + +class TelegramObjectStorage: + def __init__(self, manifests: ManifestStore, transport: TelegramTransport, + *, chunk_size: int = 64 * 1024 * 1024, + scratch_directory: Path | None = None, + writer_ttl_seconds: int = 120): + if chunk_size <= 0 or chunk_size > 1024 * 1024 * 1024: + raise ValueError("chunk_size must be between 1 byte and 1 GiB") + self.manifests = manifests + self.transport = transport + self.chunk_size = chunk_size + self.scratch_directory = scratch_directory + if writer_ttl_seconds < 5: + raise ValueError("writer_ttl_seconds must be at least 5") + self.writer_ttl_seconds = writer_ttl_seconds + + async def _keep_writer_alive(self, identifier: str, token: str, + stop: asyncio.Event, lost: asyncio.Event) -> None: + interval = max(1, self.writer_ttl_seconds // 3) + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=interval) + return + except asyncio.TimeoutError: + pass + try: + if not self.manifests.heartbeat_writer( + identifier, token, int(time.time()), self.writer_ttl_seconds + ): + lost.set() + return + except Exception: + lost.set() + return + + async def put_file(self, bucket: str, key: str, source: Path, *, + on_begin=None) -> StoredObject: + source = Path(source) + with _regular_source(source) as handle: + before = os.fstat(handle.fileno()) + identifier = self.manifests.begin(bucket, key, before.st_size, int(time.time())) + token = secrets.token_hex(24) + self.manifests.start_writer(identifier, token, int(time.time()), self.writer_ttl_seconds) + stop = asyncio.Event() + lost = asyncio.Event() + renewal = asyncio.create_task(self._keep_writer_alive(identifier, token, stop, lost)) + full_sha = hashlib.sha256() + offset = 0 + ordinal = 0 + try: + if on_begin is not None: + on_begin(identifier) + degree_fn = getattr(self.transport, "parallel_degree", None) + concurrency = degree_fn() if callable(degree_fn) else 1 + with tempfile.TemporaryDirectory( + prefix="teledrive-chunks-", dir=self.scratch_directory + ) as scratch: + root = Path(scratch) + staged: list[tuple[int, int, int, str, Path]] = [] + while data := handle.read(self.chunk_size): + full_sha.update(data) + digest = hashlib.sha256(data).hexdigest() + part = root / f"part-{ordinal:08d}" + with part.open("xb") as f: + f.write(data) + part.chmod(0o600) + staged.append( + (ordinal, offset, len(data), digest, part) + ) + offset += len(data) + ordinal += 1 + # parallel strategy uploads up to one chunk per + # credential set concurrently; every other transport + # stays strictly sequential. + for index in range(0, len(staged), max(concurrency, 1)): + batch = staged[index:index + max(concurrency, 1)] + results = await asyncio.gather(*[ + self.transport.upload( + part_path, object_id=identifier, + ordinal=ordinal_number, checksum=digest, + ) + for (ordinal_number, _, _, digest, + part_path) in batch + ]) + for (ordinal_number, part_offset, part_size, digest, + part_path), (account, chat_id, message_id) in zip( + batch, results): + # The remote message may already exist if our SQL + # lease expired: reject publication and leave the + # marker for GC. + if lost.is_set() or not self.manifests.heartbeat_writer( + identifier, token, int(time.time()), + self.writer_ttl_seconds, + ): + raise ManifestError( + "Writer ownership lost before chunk journal" + ) + self.manifests.append(identifier, ChunkReference( + ordinal=ordinal_number, offset=part_offset, + size=part_size, sha256=digest, + account=account, chat_id=int(chat_id), + message_id=int(message_id), + ), int(time.time()), writer_token=token) + part_path.unlink() + after = os.fstat(handle.fileno()) + if (offset != before.st_size or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_mtime_ns != after.st_mtime_ns + or before.st_size != after.st_size): + raise IntegrityError("Source changed during upload") + if lost.is_set() or not self.manifests.heartbeat_writer( + identifier, token, int(time.time()), self.writer_ttl_seconds + ): + raise ManifestError("Writer ownership lost before publication") + return self.manifests.finish( + identifier, full_sha.hexdigest(), int(time.time()), writer_token=token + ) + except BaseException as exc: + # Do not delete recorded Telegram chunks here. They form a + # recovery journal and may be needed by a later reconciler. + try: + self.manifests.fail(identifier, type(exc).__name__, int(time.time()), + writer_token=token) + except Exception: + pass + raise + finally: + stop.set() + await renewal + + async def resume_file(self, bucket: str, key: str, source: Path, + identifier: str) -> StoredObject: + """Resume journaled chunks after reclaiming an expired SQL writer. + + All saved Telegram messages and the corresponding source ranges are + independently read/verified before uploading any missing tail bytes. + A Bot API ACK committed to its SQL receipt index (but not to the + chunk journal) can be recovered after independent byte-for-byte + verification. Unknown ACKs still require manual reconciliation. + No Telegram message is ever deleted by resume. + """ + source = Path(source) + with _regular_source(source) as handle: + before = os.fstat(handle.fileno()) + row = self.manifests.db.connection.execute( + select(objects.c.bucket, objects.c.object_key, objects.c.size, + objects.c.status).where(objects.c.id == identifier) + ).one_or_none() + if (row is None or row[0] != bucket or row[1] != key + or int(row[2]) != before.st_size or row[3] != "pending"): + raise ManifestError("Resumable manifest identity is invalid") + token = secrets.token_hex(24) + if not self.manifests.reclaim_writer( + identifier, token, int(time.time()), self.writer_ttl_seconds + ): + raise ManifestError("Cannot reclaim an active or non-pending writer") + stop, lost = asyncio.Event(), asyncio.Event() + renewal = asyncio.create_task( + self._keep_writer_alive(identifier, token, stop, lost) + ) + full_sha = hashlib.sha256() + offset, ordinal = 0, 0 + try: + with tempfile.TemporaryDirectory( + prefix="teledrive-resume-", dir=self.scratch_directory + ) as scratch: + root = Path(scratch) + for ref in self.manifests.recovery_parts(identifier): + if (ref.ordinal != ordinal or ref.offset != offset + or ref.size <= 0 or offset + ref.size > before.st_size): + raise ManifestError("Noncontiguous resumable chunk journal") + existing = root / f"saved-{ordinal:08d}" + await self.transport.download(ref, existing) + if not existing.is_file() or existing.stat().st_size != ref.size: + raise IntegrityError("Saved Telegram chunk missing or truncated") + section_sha = hashlib.sha256() + with existing.open("rb") as remote: + remaining = ref.size + while remaining: + amount = min(1024 * 1024, remaining) + local = handle.read(amount) + saved = remote.read(amount) + if len(local) != amount or saved != local: + raise IntegrityError("Resumable Telegram bytes differ from source") + section_sha.update(local) + full_sha.update(local) + remaining -= amount + if section_sha.hexdigest() != ref.sha256: + raise IntegrityError("Resumable journal checksum mismatch") + existing.unlink() + offset += ref.size + ordinal += 1 + while data := handle.read(self.chunk_size): + checksum = hashlib.sha256(data).hexdigest() + full_sha.update(data) + part = root / f"part-{ordinal:08d}" + with part.open("xb") as output: + output.write(data) + part.chmod(0o600) + find_receipt = getattr(self.transport, "recover_receipt", None) + receipt = ( + find_receipt(identifier, ordinal, checksum, len(data)) + if callable(find_receipt) else None + ) + if receipt is not None: + # SQL ACK receipt is NOT proof of Telegram content. + # Reject missing/changed/foreign data instead of + # silently retransmitting or adopting wrong bytes. + if (receipt.ordinal != ordinal or receipt.size != len(data) + or receipt.sha256 != checksum): + raise IntegrityError("Recovered ACK chunk metadata mismatch") + recovered = root / f"receipt-{ordinal:08d}" + await self.transport.download(receipt, recovered) + if (not recovered.is_file() or recovered.is_symlink() + or recovered.stat().st_size != len(data) + or recovered.read_bytes() != data): + raise IntegrityError("Recovered ACK bytes differ from source") + recovered.unlink() + account, chat_id, message_id = ( + receipt.account, receipt.chat_id, receipt.message_id + ) + else: + account, chat_id, message_id = await self.transport.upload( + part, object_id=identifier, ordinal=ordinal, + checksum=checksum, + ) + if lost.is_set() or not self.manifests.heartbeat_writer( + identifier, token, int(time.time()), self.writer_ttl_seconds + ): + raise ManifestError("Resumed writer ownership lost") + self.manifests.append( + identifier, ChunkReference( + ordinal=ordinal, offset=offset, size=len(data), + sha256=checksum, account=account, + chat_id=int(chat_id), message_id=int(message_id), + ), int(time.time()), writer_token=token, + ) + part.unlink() + offset += len(data) + ordinal += 1 + after = os.fstat(handle.fileno()) + if (offset != before.st_size or before.st_dev != after.st_dev + or before.st_ino != after.st_ino + or before.st_mtime_ns != after.st_mtime_ns + or before.st_size != after.st_size): + raise IntegrityError("Resumed source changed during upload") + if lost.is_set() or not self.manifests.heartbeat_writer( + identifier, token, int(time.time()), self.writer_ttl_seconds + ): + raise ManifestError("Resumed writer ownership lost before publication") + return self.manifests.finish( + identifier, full_sha.hexdigest(), int(time.time()), + writer_token=token, + ) + except BaseException as exc: + try: + self.manifests.fail(identifier, type(exc).__name__, + int(time.time()), writer_token=token) + except Exception: + pass + raise + finally: + stop.set() + await renewal + + async def iter_bytes(self, identifier: str, *, + start: int = 0, end: int | None = None, + block_size: int = 1024 * 1024) -> AsyncIterator[bytes]: + """Stream a half-open [start,end) range, verifying each downloaded chunk.""" + obj = self.manifests.get(identifier) + if end is None: + end = obj.size + if block_size <= 0 or start < 0 or end < start or end > obj.size: + raise ValueError("Invalid range or block size") + if start == end: + return + parts = self.manifests.parts(identifier) + expected = 0 + with tempfile.TemporaryDirectory( + prefix="teledrive-read-", dir=self.scratch_directory + ) as scratch: + root = Path(scratch) + for part in parts: + if part.offset != expected: + raise IntegrityError("Non-contiguous chunk manifest") + expected += part.size + if part.offset >= end: + break + if part.offset + part.size <= start: + continue + target = root / f"read-{part.ordinal:08d}" + # Telegram's remote file endpoint can occasionally return + # incorrect bytes on an otherwise successful HTTPS response. + # A GET may safely be repeated; a send MUST NOT be. Never + # expose any bytes until the independently journaled digest + # validates. Re-read the same immutable message, boundedly, + # and fail closed if every attempt disagrees with the manifest. + for attempt in range(3): + target.unlink(missing_ok=True) + await self.transport.download(part, target) + if not target.is_file() or target.is_symlink(): + raise IntegrityError("Telegram chunk missing or unsafe") + size, sha = _digest_file(target) + if size == part.size and sha == part.sha256: + break + target.unlink(missing_ok=True) + if attempt == 2: + raise IntegrityError("Telegram chunk checksum mismatch") + await asyncio.sleep(0.25 * (attempt + 1)) + local_start = max(0, start - part.offset) + local_end = min(part.size, end - part.offset) + with _regular_source(target) as file: + file.seek(local_start) + remaining = local_end - local_start + while remaining: + block = file.read(min(remaining, block_size)) + if not block: + raise IntegrityError("Telegram chunk truncated while streaming") + remaining -= len(block) + yield block + target.unlink() + # Reaching end is only valid if manifests cover all requested bytes. + if expected < end: + raise IntegrityError("Missing part of requested range") + + async def download_to_path(self, identifier: str, destination: Path) -> StoredObject: + """Atomically publish a verified full object; never leave a partial file.""" + obj = self.manifests.get(identifier) + destination = Path(destination) + if destination.is_symlink(): + raise ValueError("Destination must not be a symlink") + destination.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=".teledrive-", dir=destination.parent) + temp = Path(name) + total = 0 + sha = hashlib.sha256() + try: + with os.fdopen(fd, "wb") as output: + async for block in self.iter_bytes(identifier): + output.write(block) + sha.update(block) + total += len(block) + output.flush() + os.fsync(output.fileno()) + if total != obj.size or sha.hexdigest() != obj.sha256: + raise IntegrityError("Object checksum mismatch") + os.chmod(temp, 0o600) + os.replace(temp, destination) + return obj + finally: + temp.unlink(missing_ok=True) diff --git a/teledrive/storage/gc.py b/teledrive/storage/gc.py new file mode 100644 index 00000000..ebb0ed10 --- /dev/null +++ b/teledrive/storage/gc.py @@ -0,0 +1,300 @@ +"""Read-only Telegram chunk reachability audit for eventual mark-and-sweep GC. + +This module NEVER deletes remote messages or local metadata. It deliberately +treats all historical manifests as retained until a separate versioning / +retention policy and verified Telegram ownership scans are implemented. +""" +from __future__ import annotations +import time + +from dataclasses import dataclass +from collections import defaultdict + +from sqlalchemy import and_, inspect, select, func, or_ + +from teledrive.storage.catalog import BucketCatalog, heads +from teledrive.storage.manifest import ManifestStore, object_aliases, objects, chunks + + +@dataclass(frozen=True) +class ChunkAudit: + owner_object_id: str + message_count: int + reference_count: int + live_heads: int + copied_aliases: int + multipart_parts: int + retained_history: bool + + +@dataclass(frozen=True) +class GCInventory: + total_manifests: int + total_telegram_messages: int + live_object_heads: int + copy_aliases: int + multipart_parts: int + protected_chunks: tuple[ChunkAudit, ...] + eligible_for_deletion: int + # Confirmed MTProto send receipts without manifest chunks still own media. + unjournaled_mtproto_receipts: int = 0 + # A pending pre-send intent MAY correspond to a Telegram message. + unresolved_mtproto_intents: int = 0 + # Queued/running durable upload jobs can still publish new heads. + active_upload_jobs: int = 0 + + +class StorageGCInventory: + """Conservative inventory: an audit does not authorize physical deletion.""" + + def __init__(self, manifests: ManifestStore): + self.manifests = manifests + self.db = manifests.db + + def preview_failed_retention(self, *, now: int, + min_age_seconds: int = 7 * 86400, + limit: int = 100) -> list[dict]: + """Read-only mark-phase report, never a Telegram deletion authorization.""" + if min_age_seconds < 86400 or not 1 <= limit <= 1000: + raise ValueError("Unsafe GC retention parameters") + from .gc_safety import GCReferenceGuard + guard = GCReferenceGuard(self.manifests) + guard.validate_schema() + db = self.db.connection + found = db.execute( + select(objects.c.id, objects.c.bucket, objects.c.object_key, + objects.c.updated_at) + .where(objects.c.status == "failed", + objects.c.updated_at <= now - min_age_seconds) + .order_by(objects.c.updated_at, objects.c.id).limit(limit) + ).all() + result = [] + for ident, bucket, key, updated_at in found: + count = db.execute(select(func.count()).select_from(chunks).where( + chunks.c.object_id == ident + )).scalar_one() + result.append({ + "object_id": ident, "bucket": bucket, "key": key, + "updated_at": int(updated_at), "journaled_chunks": int(count), + **guard.evidence(ident), + "remote_history_scan_required": True, + "approved_for_deletion": False, "dry_run": True, + }) + return result + + def plan(self, object_id: str, *, now: int | None = None, + backup_set_id: str = "", writer_fence_token: str = "", + min_age_seconds: int = 7 * 86400) -> dict: + """Build a bounded, integrity-bound dry-run GC plan. + + The plan enumerates every SQL reference for `object_id`, fingerprints + the reference graph and records the owner gates that must still pass + before any physical deletion. It NEVER authorizes deletion: the + deletion gates stay explicit and unmet by default. A plan is stale + the moment any reference changes; use verify_plan() before acting. + """ + if not isinstance(object_id, str) or not object_id: + raise ValueError("GC plan requires an object id") + if min_age_seconds < 86400: + raise ValueError("Unsafe GC retention parameters") + now = int(time.time()) if now is None else int(now) + from .gc_safety import GCReferenceGuard + guard = GCReferenceGuard(self.manifests) + guard.validate_schema() + db = self.db.connection + row = db.execute(select(objects.c.id, objects.c.bucket, + objects.c.object_key, objects.c.status, + objects.c.updated_at) + .where(objects.c.id == object_id)).first() + if row is None: + raise FileNotFoundError("GC candidate missing") + ident, bucket, key, status, updated_at = row + if status != "failed": + raise ValueError("Only failed objects may be planned for cleanup") + if int(updated_at) > now - min_age_seconds: + raise ValueError("Failure has not aged beyond the safety interval") + evidence = guard.evidence(ident) + age_seconds = now - int(updated_at) + return { + "plan_version": 1, + "dry_run": True, + "object_id": ident, + "bucket": bucket, + "key": key, + "status": status, + "updated_at": int(updated_at), + "age_seconds": age_seconds, + "min_age_seconds": min_age_seconds, + "journaled_chunks": int(db.execute( + select(func.count()).select_from(chunks).where( + chunks.c.object_id == ident + ) + ).scalar_one()), + "sql_reference_fingerprint": evidence["sql_reference_fingerprint"], + "reference_blocked": evidence["reference_blocked"], + "reference_blockers": evidence["reference_blockers"], + "copy_alias_descendants": evidence["copy_alias_descendants"], + "writer_fence_present": evidence["writer_fence_present"], + "historical_version_retained": evidence["historical_version_retained"], + "unresolved_mtproto_intents": evidence["unresolved_mtproto_intents"], + "backup_set_id": backup_set_id or None, + "writer_fence_token": writer_fence_token or None, + "remote_history_scan_required": True, + "writer_barrier_verified": False, + "independent_payload_backup_verified": False, + "restore_drill_verified": False, + "two_person_deletion_approval_verified": False, + "deletion_permitted": False, + "owner_gates": [ + "independent_payload_backup_verified", + "restore_drill_verified", + "writer_barrier_verified", + "two_person_deletion_approval_verified", + ], + "approved_for_deletion": False, + "physical_deletion_enabled": False, + "rollback_strategy": "new_media_upload_and_fresh_sql_identities", + } + + def verify_plan(self, plan: dict) -> None: + from .gc_safety import GCReferenceGuard + guard = GCReferenceGuard(self.manifests) + guard.verify_plan(plan) + + def inspect(self) -> GCInventory: + connection = self.db.connection + + # Current head pointers are only one reference source. Deleted or + # overwritten objects are still historical versions, not safe garbage. + # S3 delete markers are SQL HEAD tombstones, not live payload + # objects. They must not inflate the live-object count or the + # retained Telegram reference inventory. + head_ids = list(connection.execute( + select(heads.c.object_id).select_from(heads.join( + objects, heads.c.object_id == objects.c.id, + )).where(objects.c.status == "ready") + ).scalars()) + aliases = list(connection.execute(select( + object_aliases.c.object_id, object_aliases.c.source_object_id + )).all()) + all_objects = list(connection.execute(select( + objects.c.id, objects.c.status + )).all()) + all_chunks = list(connection.execute(select( + chunks.c.object_id, chunks.c.account, + chunks.c.chat_id, chunks.c.message_id + )).all()) + + multipart_objects: list[str] = [] + if inspect(self.db.engine).has_table("storage_multipart_parts"): + from teledrive.s3.multipart import parts + multipart_objects = list(connection.execute( + select(parts.c.object_id) + ).scalars()) + + # Detect invalid/cyclic metadata and refuse to claim a complete audit. + source_of = dict(aliases) + known_ids = {identifier for identifier, _ in all_objects} + def root(identifier: str) -> str: + visited = set() + current = identifier + while current in source_of: + if current in visited or current not in known_ids: + raise RuntimeError("Copy alias graph is corrupt") + visited.add(current) + current = source_of[current] + if current not in known_ids: + raise RuntimeError("Copy source manifest missing") + return current + + head_counts = defaultdict(int) + alias_counts = defaultdict(int) + part_counts = defaultdict(int) + for identifier in head_ids: + head_counts[root(identifier)] += 1 + for alias_id, source_id in aliases: + # Historical aliases protect their source media too, until + # immutable-history retention and physical versioning exist. + if alias_id not in known_ids: + raise RuntimeError("Copy target manifest missing") + alias_counts[root(source_id)] += 1 + for identifier in multipart_objects: + part_counts[root(identifier)] += 1 + + chunk_counts = defaultdict(int) + message_identities = set() + for identifier, account, chat_id, message_id in all_chunks: + if identifier not in known_ids: + raise RuntimeError("Telegram chunk references deleted manifest") + identity = (account, chat_id, message_id) + if identity in message_identities: + raise RuntimeError("Duplicate Telegram message ownership") + message_identities.add(identity) + chunk_counts[identifier] += 1 + + # A durable MTProto receipt may precede the manifest chunk transaction + # or survive a crash. Count that acknowledged remote message in the + # inventory even if no immutable chunk journal exists yet. + inspector = inspect(self.db.engine) + unresolved_mtproto_intents = 0 + unjournaled_mtproto_receipts = 0 + if inspector.has_table("storage_mtproto_send_intents"): + from teledrive.storage.user_mode import mtproto_send_intents + unresolved_mtproto_intents = int(connection.execute( + select(func.count()).select_from(mtproto_send_intents).where( + mtproto_send_intents.c.message_id.is_(None) + ) + ).scalar_one()) + unjournaled_mtproto_receipts = int(connection.execute( + select(func.count()).select_from( + mtproto_send_intents.outerjoin(chunks, and_( + chunks.c.account == mtproto_send_intents.c.account, + chunks.c.chat_id == mtproto_send_intents.c.chat_id, + chunks.c.message_id == mtproto_send_intents.c.message_id, + )) + ).where( + mtproto_send_intents.c.message_id.is_not(None), + chunks.c.object_id.is_(None), + ) + ).scalar_one()) + + # In-flight durable upload jobs are live references: a queued/running + # job can still publish a head over the candidate's bucket/key. + active_upload_jobs = 0 + if inspector.has_table("storage_upload_jobs"): + from teledrive.storage.gc_safety import ACTIVE_JOB_STATES + from teledrive.storage.jobs import jobs as upload_jobs + active_upload_jobs = int(connection.execute( + select(func.count()).select_from(upload_jobs).where( + upload_jobs.c.state.in_(ACTIVE_JOB_STATES) + ) + ).scalar_one()) + + # Roots with no current refs can still be historical versions; do not + # interpret this report as an approved deletion plan. + audits = tuple( + ChunkAudit( + owner_object_id=identifier, + message_count=chunk_counts[identifier], + reference_count=head_counts[identifier]+alias_counts[identifier]+part_counts[identifier], + live_heads=head_counts[identifier], + copied_aliases=alias_counts[identifier], + multipart_parts=part_counts[identifier], + retained_history=True, + ) + for identifier, _ in all_objects if chunk_counts[identifier] + ) + return GCInventory( + total_manifests=len(all_objects), + total_telegram_messages=( + len(message_identities) + unjournaled_mtproto_receipts + ), + live_object_heads=len(head_ids), + copy_aliases=len(aliases), + multipart_parts=len(multipart_objects), + protected_chunks=audits, + eligible_for_deletion=0, + unjournaled_mtproto_receipts=unjournaled_mtproto_receipts, + unresolved_mtproto_intents=unresolved_mtproto_intents, + active_upload_jobs=active_upload_jobs, + ) diff --git a/teledrive/storage/gc_safety.py b/teledrive/storage/gc_safety.py new file mode 100644 index 00000000..a8e471cc --- /dev/null +++ b/teledrive/storage/gc_safety.py @@ -0,0 +1,178 @@ +"""Fail-closed, non-destructive SQL graph checks for Telegram garbage collection. + +An SQL preview does not prove absence of remote media or stop concurrent writers. +No method in this module is allowed to delete any metadata or payload. +""" +from __future__ import annotations + +import hashlib +import json + +from sqlalchemy import func, inspect, or_, select +from teledrive.storage.catalog import heads, published_versions, removed_versions +from teledrive.storage.manifest import chunks, object_aliases, objects, writer_fences + +ACTIVE_JOB_STATES = ("queued", "running", "staged", "publishing") + + +def durable_job_blockers(db, names, linked): + """Active durable upload-job IDs that hold or will publish linked objects. + + A queued/running job can still publish the candidate head (by object_id, + expected_head_id, or the same bucket/key), so it is a live reference. + Terminal jobs are excluded: their published head is covered by the head + graph entry, and failed/cancelled jobs hold no pending publication. + """ + if "storage_upload_jobs" not in names: + return [] + from teledrive.storage.jobs import jobs as upload_jobs + + by_identity = db.execute(select(upload_jobs.c.id).where( + upload_jobs.c.state.in_(ACTIVE_JOB_STATES), + or_( + upload_jobs.c.object_id.in_(linked), + upload_jobs.c.expected_head_id.in_(linked), + ), + )).scalars() + blocked = set(by_identity) + keyed = db.execute(select( + objects.c.bucket, objects.c.object_key + ).where(objects.c.id.in_(linked))).all() + for bucket, key in {(row[0], row[1]) for row in keyed}: + blocked.update(db.execute(select(upload_jobs.c.id).where( + upload_jobs.c.state.in_(ACTIVE_JOB_STATES), + upload_jobs.c.bucket == bucket, + upload_jobs.c.object_key == key, + )).scalars()) + return sorted(blocked) + + +class GCReferenceGuard: + REQUIRED = frozenset({ + "storage_objects", "storage_chunks", "storage_object_aliases", + "storage_writer_fences", "storage_object_heads", + "storage_published_versions", "storage_removed_versions", + }) + + def __init__(self, manifests): + self.db = manifests.db + + def validate_schema(self): + names = set(inspect(self.db.engine).get_table_names()) + if not self.REQUIRED.issubset(names): + raise RuntimeError("Incomplete GC reference graph schema") + return names + + def evidence(self, identifier): + from teledrive.s3.multipart import parts + from teledrive.storage.user_mode import mtproto_send_intents + + names = self.validate_schema() + db = self.db.connection + known_ids = set(db.execute(select(objects.c.id)).scalars()) + if identifier not in known_ids: + raise RuntimeError("GC candidate missing") + edges = dict(db.execute(select( + object_aliases.c.object_id, object_aliases.c.source_object_id + )).all()) + for alias in edges: + visited = set() + current = alias + while current in edges: + if current in visited or current not in known_ids: + raise RuntimeError("Copy alias graph is corrupt") + visited.add(current) + current = edges[current] + if current not in known_ids: + raise RuntimeError("Copy alias graph is corrupt") + + linked = {identifier} + for alias in edges: + current = alias + while current in edges: + current = edges[current] + if current == identifier: + linked.add(alias) + break + + def refs(column): + return sorted(db.execute( + select(column).where(column.in_(linked)) + ).scalars()) + + graph = { + "head": refs(heads.c.object_id), + "published_version": refs(published_versions.c.object_id), + "removed_version": refs(removed_versions.c.object_id), + "copy_alias": sorted(linked - {identifier}), + "writer_fence": refs(writer_fences.c.object_id), + "multipart_part": refs(parts.c.object_id) + if "storage_multipart_parts" in names else [], + "mtproto_send_intent_or_receipt": refs(mtproto_send_intents.c.object_id) + if "storage_mtproto_send_intents" in names else [], + "durable_job": durable_job_blockers(db, names, linked), + } + chunk_journal = [ + (int(n), account, int(chat), int(mid), digest) + for n, account, chat, mid, digest in db.execute( + select(chunks.c.ordinal, chunks.c.account, chunks.c.chat_id, + chunks.c.message_id, chunks.c.sha256) + .where(chunks.c.object_id == identifier) + .order_by(chunks.c.ordinal) + ) + ] + fingerprint = hashlib.sha256(json.dumps({ + "id": identifier, "graph": graph, "journal": chunk_journal, + }, sort_keys=True, separators=(",", ":")).encode("utf-8")).hexdigest() + blockers = sorted(k for k, refs_found in graph.items() if refs_found) + # Unresolved = pre-send intents without any remote message ID yet. + # Acknowledged receipts still block via the graph entry above. + unresolved_intents = 0 + if "storage_mtproto_send_intents" in names: + unresolved_intents = int(db.execute( + select(func.count()).select_from(mtproto_send_intents).where( + mtproto_send_intents.c.object_id.in_(linked), + mtproto_send_intents.c.message_id.is_(None), + ) + ).scalar_one()) + return { + "reference_blocked": bool(blockers), + "reference_blockers": blockers, + "sql_reference_fingerprint": fingerprint, + "copy_alias_descendants": len(graph["copy_alias"]), + "writer_fence_present": bool(graph["writer_fence"]), + "historical_version_retained": bool( + graph["published_version"] or graph["removed_version"] + ), + "unresolved_mtproto_intents": unresolved_intents, + "writer_barrier_verified": False, + "independent_payload_backup_verified": False, + "restore_drill_verified": False, + "two_person_deletion_approval_verified": False, + "deletion_permitted": False, + } + + def verify_plan(self, plan: dict) -> None: + """Reject a stale or tampered dry-run plan. + + A plan is valid only when its fingerprint still matches the current + SQL reference graph. Any new reference, writer fence or alias makes + the plan stale and blocks deletion. This closes the TOCTOU window + between planning and execution. + """ + if not isinstance(plan, dict): + raise RuntimeError("GC plan must be a mapping") + identifier = plan.get("object_id") + if not isinstance(identifier, str) or not identifier: + raise RuntimeError("GC plan missing object identity") + recorded = plan.get("sql_reference_fingerprint") + if not isinstance(recorded, str) or len(recorded) != 64: + raise RuntimeError("GC plan missing fingerprint") + current = self.evidence(identifier)["sql_reference_fingerprint"] + if current != recorded: + raise RuntimeError("GC dry-run plan is stale; re-plan before execution") + if plan.get("deletion_permitted") is not False: + raise RuntimeError("GC plan must not authorize deletion") + if plan.get("dry_run") is not True: + raise RuntimeError("GC plan must remain a dry run") + diff --git a/teledrive/storage/history_inventory.py b/teledrive/storage/history_inventory.py new file mode 100644 index 00000000..082bad9c --- /dev/null +++ b/teledrive/storage/history_inventory.py @@ -0,0 +1,184 @@ +"""Bounded, read-only inventory of TeleDrive markers in one Telegram channel. + +Captions are hints, NOT payload integrity, ownership, S3 key, bucket, or +version proofs. This code deliberately cannot create a rebuild plan or delete +or download Telegram media. +""" +from __future__ import annotations + +import json +import os +import re +from pathlib import Path +from uuid import UUID + +from .telethon_adapter import TelethonChunkTransport + +MAX_SCAN_MESSAGES = 10_000 +_MARKER = re.compile( + r"teledrive:([0-9a-f-]{36}):([0-9]{1,10}):([0-9a-f]{16})\Z" +) + + +def parse_candidate_marker(caption: str | None) -> tuple[str, int, str] | None: + """Parse a canonical UUID marker with the existing transport validator.""" + if not isinstance(caption, str): + return None + match = _MARKER.fullmatch(caption) + if match is None: + return None + object_id = match.group(1) + try: + if str(UUID(object_id)) != object_id: + return None + except ValueError: + return None + part = TelethonChunkTransport._parse_marker(caption, object_id) + if part is None: + return None + return object_id, part[0], part[1] + + +async def inventory_channel(client, target, *, account: str, + scan_limit: int = 1000, + before_message_id: int | None = None, + include_external_markers: bool = False) -> dict: + """Inspect at most scan_limit messages from a single resolved target. + + The extra message is used solely to detect truncation; it is not + analyzed. Telegram iteration errors propagate: no partial report is + returned as if the scan had succeeded. + """ + if (type(scan_limit) is not int or not 1 <= scan_limit <= MAX_SCAN_MESSAGES): + raise ValueError("scan_limit must be an integer between 1 and 10000") + if (not isinstance(account, str) or not 1 <= len(account) <= 255 + or any(ord(char) < 32 or ord(char) == 127 for char in account)): + raise ValueError("Invalid authorized account label") + if (before_message_id is not None and + (type(before_message_id) is not int or before_message_id < 1)): + raise ValueError("Invalid channel-history continuation message ID") + if type(include_external_markers) is not bool: + raise ValueError("Invalid external-marker inventory flag") + chat_id = int(await client.get_peer_id(target)) + groups: dict[str, list[dict]] = {} + seen_message_ids: set[int] = set() + scanned = 0 + rejected_markers = 0 + truncated = False + external_markers = [] + last_message_id = None + iterator_kwargs = {"limit": scan_limit + 1} + if before_message_id is not None: + # Telethon max_id is EXCLUSIVE; do not re-read the last page's tail. + iterator_kwargs["max_id"] = before_message_id + async for message in client.iter_messages(target, **iterator_kwargs): + if scanned == scan_limit: + truncated = True + break + scanned += 1 + mid = getattr(message, "id", None) + if type(mid) is int and mid > 0: + last_message_id = (mid if last_message_id is None + else min(last_message_id, mid)) + marker = parse_candidate_marker(getattr(message, "message", None)) + if marker is None: + continue + remote_chat = getattr(message, "chat_id", None) + # The outgoing flag and chat match are prerequisites for a candidate, + # but even together they do NOT prove independently verified ownership. + if (not getattr(message, "out", False) + or getattr(message, "media", None) is None + or type(mid) is not int or mid <= 0 + or type(remote_chat) is not int or remote_chat != chat_id): + rejected_markers += 1 + # A user can read bot/other-admin channel posts but not prove + # authorship. Preserve only exact markers from the SAME channel + # as explicitly UNVERIFIED hints for owner-side investigation. + if (include_external_markers and not getattr(message, "out", False) + and getattr(message, "media", None) is not None + and type(mid) is int and mid > 0 + and type(remote_chat) is int and remote_chat == chat_id): + object_id, ordinal, prefix = marker + external_markers.append({ + "object_id": object_id, "ordinal": ordinal, + "checksum_prefix": prefix, "chat_id": chat_id, + "message_id": mid, "ownership_verified": False, + "payload_verified": False, + "safe_to_publish_or_delete": False, + }) + continue + if mid in seen_message_ids: + raise ValueError("Duplicate Telegram message ID in channel scan") + seen_message_ids.add(mid) + object_id, ordinal, prefix = marker + groups.setdefault(object_id, []).append({ + "ordinal": ordinal, + "checksum_prefix": prefix, + "account": account, + "chat_id": chat_id, + "message_id": mid, + }) + + if truncated and last_message_id is None: + raise ValueError("History page cannot advance safely without message IDs") + objects = [] + for object_id, parts in sorted(groups.items()): + parts.sort(key=lambda item: (item["ordinal"], item["message_id"])) + ordinals = [part["ordinal"] for part in parts] + unique = sorted(set(ordinals)) + contiguous = (len(ordinals) == len(unique) and bool(unique) + and unique[0] == 0 and all( + current == previous + 1 + for previous, current in zip(unique, unique[1:]) + )) + objects.append({ + "object_id": object_id, + "chunks": parts, + "candidate_ordinals_contiguous": contiguous, + "duplicate_ordinals": len(unique) != len(ordinals), + "bucket": None, + "key": None, + "version_id": None, + "whole_object_sha256": None, + "payload_verified": False, + "original_mapping_verified": False, + }) + return { + "schema_version": 1, + "scope": {"account": account, "chat_id": chat_id, + "scan_limit": scan_limit}, + "scanned_messages": scanned, + "truncated": truncated, + "before_message_id": before_message_id, + "next_before_message_id": last_message_id if truncated else None, + "external_marker_candidates": external_markers, + "external_marker_candidate_count": len(external_markers), + "readable_window_exhausted": not truncated, + "rejected_marker_messages": rejected_markers, + "candidate_objects": objects, + "candidate_object_count": len(objects), + "candidate_chunk_count": sum(len(obj["chunks"]) for obj in objects), + "payload_downloads": 0, + "telegram_mutations": 0, + "production_database_modified": False, + "restorable_without_independent_manifest": False, + "complete_unknown_history_discovered": False, + } + + +def write_private_report(destination: Path, report: dict) -> None: + """Create a 0600 JSON report exactly once; never replace an old report.""" + destination = Path(destination) + if not destination.parent.is_dir(): + raise FileNotFoundError("Report parent directory does not exist") + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(destination, flags, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(report, stream, indent=2, sort_keys=True) + stream.write("\n") + stream.flush() + os.fsync(stream.fileno()) + except BaseException: + destination.unlink(missing_ok=True) + raise diff --git a/teledrive/storage/jobs.py b/teledrive/storage/jobs.py new file mode 100644 index 00000000..bc24ef2f --- /dev/null +++ b/teledrive/storage/jobs.py @@ -0,0 +1,471 @@ +"""Durable, fenced upload jobs for operator-approved local sources. + +The job table is metadata-only: payload bytes stay in the permitted local +source until uploaded to Telegram. An interrupted job may retry from the +beginning, leaving any previous immutable Telegram journals for safe GC. +A staged READY manifest can be published without uploading bytes again. +""" +from __future__ import annotations + +import asyncio +import json +import os +import stat +import time +from pathlib import Path +from uuid import uuid4, uuid5, NAMESPACE_URL + +from sqlalchemy import ( + BigInteger, Column, Integer, MetaData, String, Table, Text, + and_, or_, insert, select, update, +) +from sqlalchemy.exc import IntegrityError, OperationalError + +from .catalog import BucketCatalog +from .engine import TelegramObjectStorage +from .manifest import objects + +_meta = MetaData() +jobs = Table( + "storage_upload_jobs", _meta, + Column("id", String(36), primary_key=True), + Column("bucket", String(63), nullable=False), + Column("object_key", Text, nullable=False), + Column("source_path", Text, nullable=False), + Column("source_size", BigInteger, nullable=False), + Column("source_device", BigInteger, nullable=False), + Column("source_inode", BigInteger, nullable=False), + Column("source_mtime_ns", BigInteger, nullable=False), + Column("expected_generation", BigInteger, nullable=False), + Column("expected_head_id", String(36)), + Column("state", String(16), nullable=False), + Column("object_id", String(36)), + Column("attempts", Integer, nullable=False), + Column("lease_token", String(80)), + Column("lease_until", BigInteger), + Column("created_at", BigInteger, nullable=False), + Column("updated_at", BigInteger, nullable=False), + Column("last_error", String(500)), +) + + +class UploadJobError(ValueError): + pass + + +def _source_identity(path: Path, allowed_root: Path) -> tuple[Path, os.stat_result]: + """Root-bound regular-file identity: never follow an untrusted final symlink.""" + root = allowed_root.resolve(strict=True) + if not root.is_dir(): + raise UploadJobError("Upload source root must be a directory") + if path.is_symlink(): + raise UploadJobError("Symlink upload sources are forbidden") + resolved = path.resolve(strict=True) + if not resolved.is_relative_to(root) or resolved == root: + raise UploadJobError("Upload source is outside the configured root") + fd = os.open(resolved, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + try: + stat_info = os.fstat(fd) + if not stat.S_ISREG(stat_info.st_mode): + raise UploadJobError("Upload source must be a regular file") + return resolved, stat_info + finally: + os.close(fd) + + +class StorageUploadJobs: + """One SQL connection per worker. Only lease owner may publish a job.""" + + def __init__(self, catalog: BucketCatalog): + self.catalog = catalog + self.db = catalog.db + _meta.create_all(self.db.engine) + + @staticmethod + def scan_id(bucket: str, key: str, path: Path, info: os.stat_result) -> str: + """Stable identity for one exact source revision under one bucket/key. + + A UUIDv5 primary-key collision is serialized by the database even if + two legacy folder scanners race. Manual jobs remain UUIDv4. + """ + identity = json.dumps( + ["teledrive-legacy-tree-v1", bucket, key, str(path), + int(info.st_dev), int(info.st_ino), int(info.st_size), + int(info.st_mtime_ns)], ensure_ascii=False, separators=(",", ":"), + ) + return str(uuid5(NAMESPACE_URL, identity)) + + def enqueue(self, bucket: str, key: str, source: Path, *, + allowed_root: Path, now: int | None = None, + deduplicate: bool = False) -> str: + now = int(time.time()) if now is None else now + if bucket not in self.catalog.list_buckets(): + raise FileNotFoundError("Bucket does not exist") + self.catalog._keyhash(key) + path, info = _source_identity(Path(source), allowed_root) + # A delete marker hides the payload but still owns a versioned HEAD. + # Snapshot that generation/ID for CAS instead of assuming an absent + # head and rejecting every cutover job after a logical DELETE. + from .catalog import heads + head = self.db.connection.execute(select( + heads.c.generation, heads.c.object_id, + ).where( + heads.c.bucket == bucket, + heads.c.key_hash == self.catalog._keyhash(key), + )).one_or_none() + generation, observed = (int(head[0]), head[1]) if head else (0, None) + ident = ( + self.scan_id(bucket, key, path, info) + if deduplicate else str(uuid4()) + ) + if deduplicate: + previous = self.db.connection.execute( + select(jobs).where(jobs.c.id == ident) + ).mappings().first() + if previous is not None: + return ident + try: + self.db.connection.execute(insert(jobs).values( + id=ident, bucket=bucket, object_key=key, + source_path=str(path), source_size=info.st_size, + source_device=info.st_dev, source_inode=info.st_ino, + source_mtime_ns=info.st_mtime_ns, + expected_generation=generation, expected_head_id=observed, + state="queued", object_id=None, attempts=0, + lease_token=None, lease_until=None, + created_at=now, updated_at=now, last_error=None, + )) + self.db.commit() + except IntegrityError: + self.db.connection.rollback() + if not deduplicate: + raise + # The other worker may have won the same stable primary key. + # Never swallow an unrelated constraint violation. + previous = self.db.connection.execute( + select(jobs.c.bucket, jobs.c.object_key, jobs.c.source_path, + jobs.c.source_size, jobs.c.source_device, + jobs.c.source_inode, jobs.c.source_mtime_ns) + .where(jobs.c.id == ident) + ).one_or_none() + if previous != ( + bucket, key, str(path), info.st_size, info.st_dev, + info.st_ino, info.st_mtime_ns, + ): + raise + return ident + + def list_jobs(self, *, limit: int = 100) -> list[dict]: + if not 1 <= limit <= 1000: + raise ValueError("Invalid job list limit") + rows = self.db.connection.execute( + select(jobs).order_by(jobs.c.created_at.desc(), jobs.c.id) + .limit(limit) + ).mappings().all() + # Tokens are credentials for in-flight writes; never disclose them. + return [{key: value for key, value in row.items() + if key not in {"lease_token"}} for row in rows] + + def claim_next(self, *, now: int | None = None, ttl: int = 120) -> dict | None: + if ttl < 5: + raise ValueError("Job lease TTL must be >= 5") + # Independent SQLite workers can both read the same queued row + # before one upgrades its read transaction to a writer. WAL then + # raises SQLITE_BUSY_SNAPSHOT instead of waiting for a stale write. + # Roll back and resnapshot rather than leaking the lock failure to + # the operator; PostgreSQL/MySQL CAS still choose a single owner. + for attempt in range(12): + timestamp = int(time.time()) if now is None else now + eligible = or_( + jobs.c.state == "queued", + and_(jobs.c.state.in_(("running", "staged", "publishing")), + jobs.c.lease_until < timestamp), + ) + try: + candidates = self.db.connection.execute( + select(jobs.c.id) + .where(eligible).order_by(jobs.c.created_at, jobs.c.id).limit(10) + ).scalars().all() + for ident in candidates: + token = uuid4().hex + result = self.db.connection.execute( + update(jobs).where(jobs.c.id == ident, eligible) + .values(state="running", lease_token=token, + lease_until=timestamp + ttl, updated_at=timestamp, + attempts=jobs.c.attempts + 1) + ) + self.db.commit() + if result.rowcount == 1: + row = self.db.connection.execute( + select(jobs).where(jobs.c.id == ident) + ).mappings().one() + snapshot = dict(row) + # A leftover read transaction after a successful + # claim can become SQLITE_BUSY_SNAPSHOT when a second + # worker commits before this worker begins uploading. + self.db.connection.commit() + return snapshot + self.db.connection.rollback() + return None + except OperationalError as exc: + self.db.connection.rollback() + # Retry only SQLite write-contention errors; broken database + # connections and other dialect errors are not transient + # claim races and must remain visible. + if (self.db.engine.dialect.name != "sqlite" + or "locked" not in str(exc).lower() + or attempt == 11): + raise + time.sleep(min(0.025 * (attempt + 1), 0.2)) + return None + + def _owned(self, ident: str, token: str, now: int): + return and_(jobs.c.id == ident, jobs.c.lease_token == token, + jobs.c.state.in_(("running", "staged", "publishing")), + jobs.c.lease_until >= now) + + def renew(self, ident: str, token: str, *, now: int, ttl: int = 120) -> bool: + result = self.db.connection.execute( + update(jobs).where(self._owned(ident, token, now)) + .values(lease_until=now + ttl, updated_at=now) + ) + self.db.commit() + return result.rowcount == 1 + + def mark_pending(self, ident: str, token: str, object_id: str, *, + now: int) -> bool: + """Record an immutable pending object ID before sending remote bytes.""" + row = self.db.connection.execute( + select(objects.c.bucket, objects.c.object_key, objects.c.status) + .where(objects.c.id == object_id) + ).one_or_none() + if row is None or row[2] != "pending": + raise UploadJobError("Only a PENDING manifest can be linked") + job = self.db.connection.execute( + select(jobs.c.bucket, jobs.c.object_key).where(jobs.c.id == ident) + ).one_or_none() + if job is None or tuple(job) != (row[0], row[1]): + raise UploadJobError("Pending upload job bucket/key mismatch") + changed = self.db.connection.execute( + update(jobs).where( + self._owned(ident, token, now), + jobs.c.state == "running", + jobs.c.object_id.is_(None), + ).values(object_id=object_id, updated_at=now) + ) + self.db.commit() + return changed.rowcount == 1 + + def mark_staged(self, ident: str, token: str, object_id: str, *, now: int) -> bool: + obj = self.catalog.manifests.get(object_id) + job = self.db.connection.execute( + select(jobs.c.bucket, jobs.c.object_key) + .where(jobs.c.id == ident) + ).one_or_none() + if job is None or job[0] != obj.bucket or job[1] != obj.key: + raise UploadJobError("Staged object has mismatched bucket/key") + result = self.db.connection.execute( + update(jobs).where(self._owned(ident, token, now)) + .values(object_id=object_id, state="staged", updated_at=now) + ) + self.db.commit() + return result.rowcount == 1 + + def finish(self, ident: str, token: str, *, now: int) -> bool: + row = self.db.connection.execute( + select(jobs).where(self._owned(ident, token, now)) + ).mappings().first() + if not row or not row["object_id"]: + self.db.connection.rollback() + return False + candidate = row["object_id"] + try: + current = self.catalog.get(row["bucket"], row["object_key"]).item.id + except FileNotFoundError: + current = None + if current != candidate: + fenced = self.db.connection.execute( + update(jobs).where(self._owned(ident, token, now)) + .values(state="publishing", updated_at=now) + ) + if fenced.rowcount != 1: + self.db.connection.rollback() + return False + try: + # This commit also persists state=publishing; a restarted worker + # recognizes the published HEAD without uploading again. + self.catalog.publish( + candidate, now=now, + expected_generation=int(row["expected_generation"]), + expected_object_id=row["expected_head_id"], + ) + except BaseException: + self.db.connection.rollback() + raise + finished = self.db.connection.execute( + update(jobs).where(self._owned(ident, token, now)) + .values(state="done", lease_token=None, lease_until=None, + updated_at=now, last_error=None) + ) + self.db.commit() + return finished.rowcount == 1 + + def fail(self, ident: str, token: str, error: str, *, now: int) -> bool: + result = self.db.connection.execute( + update(jobs).where(self._owned(ident, token, now)) + .values(state="failed", lease_token=None, lease_until=None, + last_error=error[:500], updated_at=now) + ) + self.db.commit() + return result.rowcount == 1 + + def cancel(self, ident: str) -> bool: + """Cancel only inactive jobs; never interrupt a live Telegram writer. + + A staged/expired job must first be claimed and reconciled by the + regular worker. Cancellation cannot discard its durable manifest. + No Telegram message is deleted by this operation. + """ + result = self.db.connection.execute( + update(jobs).where( + jobs.c.id == ident, + jobs.c.state.in_(("queued", "failed")), + ).values( + state="cancelled", lease_token=None, lease_until=None, + updated_at=int(time.time()), + ) + ) + self.db.commit() + return result.rowcount == 1 + + def retry(self, ident: str) -> bool: + row = self.db.connection.execute( + select(jobs.c.object_id).where( + jobs.c.id == ident, jobs.c.state == "failed" + ) + ).one_or_none() + if row is None: + self.db.connection.rollback() + return False + object_id = row[0] + if object_id is not None: + # A READY staged manifest is safe to reuse. FAILED or missing + # journals cannot be resumed; keep them for independent GC + # evidence, and retry with a fresh object ID. + state = self.db.connection.execute( + select(objects.c.status).where(objects.c.id == object_id) + ).scalar_one_or_none() + if state != "ready": + object_id = None + result = self.db.connection.execute( + update(jobs).where(jobs.c.id == ident, jobs.c.state == "failed") + .values(state="queued", last_error=None, lease_token=None, + lease_until=None, object_id=object_id) + ) + self.db.commit() + return result.rowcount == 1 + + async def _heartbeat(self, ident: str, token: str, ttl: int, + stop: asyncio.Event, lost: asyncio.Event) -> None: + while not stop.is_set(): + try: + await asyncio.wait_for(stop.wait(), timeout=max(1, ttl // 3)) + return + except asyncio.TimeoutError: + pass + try: + if not self.renew(ident, token, now=int(time.time()), ttl=ttl): + lost.set() + return + except Exception: + self.db.connection.rollback() + lost.set() + return + + async def run_next(self, transport, *, allowed_root: Path, + scratch_directory: Path | None = None, + chunk_size: int = 16 * 1024 * 1024, + lease_seconds: int = 120) -> dict | None: + """Process one queued or abandoned job; never physically delete media.""" + claim = self.claim_next(ttl=lease_seconds) + if claim is None: + return None + ident, token = claim["id"], claim["lease_token"] + stop, lost = asyncio.Event(), asyncio.Event() + heartbeat = asyncio.create_task( + self._heartbeat(ident, token, lease_seconds, stop, lost) + ) + try: + object_id = claim["object_id"] + pending_resume = False + if object_id is not None: + state = self.db.connection.execute( + select(objects.c.status).where(objects.c.id == object_id) + ).scalar_one_or_none() + if state == "pending": + pending_resume = True + elif state != "ready": + raise UploadJobError("Job references unavailable or failed object") + if object_id is None or pending_resume: + path, stat_info = _source_identity( + Path(claim["source_path"]), allowed_root + ) + expected = ( + claim["source_device"], claim["source_inode"], + claim["source_size"], claim["source_mtime_ns"] + ) + observed = ( + stat_info.st_dev, stat_info.st_ino, + stat_info.st_size, stat_info.st_mtime_ns + ) + if observed != expected: + raise UploadJobError("Source changed since job was queued") + verify = getattr(transport, "verify_target", None) + if callable(verify): + await verify() + engine = TelegramObjectStorage( + self.catalog.manifests, transport, + chunk_size=chunk_size, scratch_directory=scratch_directory, + ) + if pending_resume: + obj = await engine.resume_file( + claim["bucket"], claim["object_key"], path, object_id, + ) + else: + def link_pending(new_id): + if not self.mark_pending( + ident, token, new_id, now=int(time.time()) + ): + raise UploadJobError("Job lease lost before remote send") + obj = await engine.put_file( + claim["bucket"], claim["object_key"], path, + on_begin=link_pending, + ) + # Do not publish a fingerprint mismatch even if Telegram has + # already accepted chunks; they remain for future safe audit. + _, after = _source_identity(path, allowed_root) + latest = ( + after.st_dev, after.st_ino, + after.st_size, after.st_mtime_ns + ) + if latest != expected: + raise UploadJobError("Source changed before job publication") + object_id = obj.id + if lost.is_set() or not self.mark_staged( + ident, token, object_id, now=int(time.time()) + ): + raise UploadJobError("Job lease lost before staging") + stop.set() + await heartbeat + if lost.is_set() or not self.finish( + ident, token, now=int(time.time()) + ): + raise UploadJobError("Job lease lost before publication") + return {"id": ident, "state": "done", "object_id": object_id} + except BaseException as exc: + self.db.connection.rollback() + self.fail(ident, token, type(exc).__name__, now=int(time.time())) + raise + finally: + stop.set() + await heartbeat diff --git a/teledrive/storage/legacy_bridge.py b/teledrive/storage/legacy_bridge.py new file mode 100644 index 00000000..372fb977 --- /dev/null +++ b/teledrive/storage/legacy_bridge.py @@ -0,0 +1,160 @@ +"""Explicit, non-uploading bridge from legacy folder scan to durable jobs. + +This NEVER changes the legacy files table or sends Telegram data. Operators +must opt in to a destination bucket and stop legacy background uploaders +before enabling the independent durable object worker. +""" +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass +from pathlib import Path + +from sqlalchemy import select + +from teledrive.storage.jobs import StorageUploadJobs, jobs, _source_identity, UploadJobError + + +@dataclass(frozen=True) +class LegacyQueueReport: + queued: int + already_queued: int + excluded: int + job_ids: tuple[str, ...] + + +def enqueue_legacy_folder( + cfg, queue: StorageUploadJobs, *, bucket: str, prefix: str = "", + max_files: int = 1000, dry_run: bool = False, + lease_heartbeat=None, +) -> LegacyQueueReport: + """Queue eligible legacy source files as immutable S3 object-key jobs. + + Any upload is performed later by the separately configured durable worker. + Repeated scans of the same device/inode/mtime/size+destination are + idempotent even when the old file table contains uploaded entries. + """ + from teledrive.uploader import iter_source_files, file_allowed + + if not 1 <= max_files <= 1000: + raise ValueError("Legacy bridge max_files must be between 1 and 1000") + if (prefix.startswith("/") or "\\" in prefix + or any(segment in ("", ".", "..") for segment in prefix.split("/")) + and prefix != ""): + raise ValueError("Legacy object prefix must be a relative safe key path") + if bucket not in queue.catalog.list_buckets(): + raise FileNotFoundError("Object bucket must already exist") + root = Path(cfg.source_dir).resolve(strict=True) + if not root.is_dir(): + raise UploadJobError("Legacy source root must be a directory") + queued = previous = excluded = 0 + identifiers: list[str] = [] + # Sort for deterministic bounded scans across restarts. + for candidate in sorted(iter_source_files(str(root))): + if lease_heartbeat is not None: + lease_heartbeat() + if not file_allowed(candidate, cfg): + excluded += 1 + continue + try: + path, stat_info = _source_identity(Path(candidate), root) + relative = path.relative_to(root).as_posix() + key = f"{prefix}/{relative}" if prefix else relative + queue.catalog._keyhash(key) + ident = queue.scan_id(bucket, key, path, stat_info) + found = queue.db.connection.execute( + select(jobs.c.id).where(jobs.c.id == ident) + ).scalar_one_or_none() + if found is not None: + previous += 1 + continue + if not dry_run: + queue.enqueue( + bucket, key, path, allowed_root=root, + deduplicate=True, + ) + queued += 1 + identifiers.append(ident) + except (OSError, ValueError, FileNotFoundError): + # The source may have vanished between scandir and stat. A later + # scan can retry; do not enqueue an unverified/symlinked target. + excluded += 1 + if queued >= max_files: + break + return LegacyQueueReport(queued, previous, excluded, tuple(identifiers)) + + +async def run_legacy_cutover_cycle( + cfg, queue: StorageUploadJobs, transport, *, lease_db, + bucket: str, allowed_root: Path, scratch_directory: Path, + lease_name: str, lease_ttl: int, chunk_size: int, + prefix: str = "", max_files: int = 100, +): + """One fenced legacy scan + one durable job, without a legacy send path. + + The independently connected lease_db owns the legacy uploader's lease + for the *entire* Telegram operation. A renewer cancels the active send + if the lease can no longer be held. The durable job retains its journal + for a later fenced recovery; no remotely accepted bytes are deleted. + """ + if lease_ttl < 15: + raise ValueError("Cutover uploader lease TTL must be at least 15 seconds") + source_root = Path(cfg.source_dir).resolve(strict=True) + permitted = allowed_root.resolve(strict=True) + if not source_root.is_dir() or not source_root.is_relative_to(permitted): + raise UploadJobError("Legacy source must be inside the approved job root") + from teledrive.db import normalize_database_url + if (queue.db.url != lease_db.url + or normalize_database_url(cfg.db_path) != queue.db.url): + raise UploadJobError("Legacy uploader, importer and job queue must share the same database") + + from uuid import uuid4 + holder = "legacy-cutover-" + uuid4().hex + if not lease_db.acquire_lease(lease_name, holder, int(time.time()), lease_ttl): + raise UploadJobError("Legacy uploader currently holds the upload lease") + worker = None + monitor = None + try: + def heartbeat(): + if not lease_db.renew_lease( + lease_name, holder, int(time.time()), lease_ttl, + ): + raise UploadJobError("Legacy upload lease lost during cutover") + + report = enqueue_legacy_folder( + cfg, queue, bucket=bucket, prefix=prefix, + max_files=max_files, lease_heartbeat=heartbeat, + ) + heartbeat() + + async def keep_lease(): + while True: + await asyncio.sleep(max(2, lease_ttl // 4)) + try: + heartbeat() + except BaseException: + if worker is not None and not worker.done(): + worker.cancel() + raise + + monitor = asyncio.create_task(keep_lease()) + worker = asyncio.create_task(queue.run_next( + transport, allowed_root=permitted, + scratch_directory=scratch_directory, chunk_size=chunk_size, + )) + try: + result = await worker + # Detect an expired/fenced lease before returning success. + heartbeat() + if monitor.done(): + monitor.result() + return report, result + finally: + monitor.cancel() + try: + await monitor + except asyncio.CancelledError: + pass + finally: + lease_db.release_lease(lease_name, holder) diff --git a/teledrive/storage/manifest.py b/teledrive/storage/manifest.py new file mode 100644 index 00000000..8e4d01dd --- /dev/null +++ b/teledrive/storage/manifest.py @@ -0,0 +1,393 @@ +"""Portable SQLAlchemy object/chunk manifests. + +An object is readable only after status becomes READY. Upload progress is +journaled per chunk to support cleanup/recovery after interrupted uploads. +No file payload bytes are persisted in the SQL database. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any +from uuid import uuid4 + +from sqlalchemy import ( + BigInteger, CheckConstraint, Column, ForeignKey, Index, Integer, MetaData, + String, Table, Text, UniqueConstraint, and_, func, select, update, insert, delete, +) + +from teledrive.db import Database + +metadata = MetaData() +objects = Table( + "storage_objects", metadata, + Column("id", String(36), primary_key=True), + Column("bucket", String(63), nullable=False), + Column("object_key", Text, nullable=False), + Column("status", String(16), nullable=False), + Column("size", BigInteger, nullable=False), + Column("sha256", String(64)), + Column("created_at", BigInteger, nullable=False), + Column("updated_at", BigInteger, nullable=False), + Column("error", Text), + CheckConstraint("size >= 0", name="ck_storage_objects_size"), +) +chunks = Table( + "storage_chunks", metadata, + Column("object_id", String(36), ForeignKey("storage_objects.id"), primary_key=True), + Column("ordinal", Integer, primary_key=True), + Column("offset", BigInteger, nullable=False), + Column("size", BigInteger, nullable=False), + Column("sha256", String(64), nullable=False), + Column("account", String(255), nullable=False), + Column("chat_id", BigInteger, nullable=False), + Column("message_id", BigInteger, nullable=False), + UniqueConstraint("account", "chat_id", "message_id", name="uq_storage_chunk_message"), + CheckConstraint("size > 0", name="ck_storage_chunk_size"), +) +# Immutable server-side copies reference the original Telegram chunk journal. +# No duplicate (account, chat_id, message_id) rows are created. Physical GC +# MUST account for these aliases before deleting any original Telegram media. +object_aliases = Table( + "storage_object_aliases", metadata, + Column("object_id", String(36), ForeignKey("storage_objects.id"), primary_key=True), + Column("source_object_id", String(36), ForeignKey("storage_objects.id"), + nullable=False), +) + +writer_fences = Table( + "storage_writer_fences", metadata, + Column("object_id", String(36), ForeignKey("storage_objects.id"), primary_key=True), + Column("token", String(80), nullable=False), + Column("lease_until", BigInteger, nullable=False), + Column("heartbeat_at", BigInteger, nullable=False), +) + +Index("ix_storage_objects_bucket", objects.c.bucket) +Index("ix_storage_objects_status", objects.c.status) + + +@dataclass(frozen=True) +class ChunkReference: + ordinal: int + offset: int + size: int + sha256: str + account: str + chat_id: int + message_id: int + + +@dataclass(frozen=True) +class StoredObject: + id: str + bucket: str + key: str + size: int + sha256: str + created_at: int + + +class ManifestError(RuntimeError): + pass + + +class ManifestStore: + """One SQLAlchemy connection per worker; not safe to share across threads. + + The caller owns the Database instance and its lifetime. Chunk writes are + committed separately, before completion, forming a recovery journal. + """ + + def __init__(self, database: Database): + self.db = database + metadata.create_all(database.engine) + + def begin(self, bucket: str, key: str, size: int, now: int) -> str: + if not bucket or len(bucket) > 63 or not key or size < 0: + raise ValueError("Invalid bucket/key/size") + identifier = str(uuid4()) + self.db.connection.execute(insert(objects).values( + id=identifier, bucket=bucket, object_key=key, status="pending", + size=size, sha256=None, created_at=now, updated_at=now, + )) + self.db.commit() + return identifier + + def start_writer(self, identifier: str, token: str, now: int, ttl: int = 120) -> None: + """Bind an unguessable owner to a pending object before Telegram I/O.""" + if not token or len(token) > 80 or ttl < 5: + raise ValueError("Invalid writer token or TTL") + self.db.connection.execute(insert(writer_fences).values( + object_id=identifier, token=token, lease_until=now + ttl, + heartbeat_at=now, + )) + self.db.commit() + + def heartbeat_writer(self, identifier: str, token: str, now: int, ttl: int = 120) -> bool: + """Conditional SQL update is the fencing point across worker processes.""" + if ttl < 5: + raise ValueError("Invalid TTL") + changed = self.db.connection.execute( + update(writer_fences).where( + writer_fences.c.object_id == identifier, + writer_fences.c.token == token, + writer_fences.c.lease_until >= now, + ).values(lease_until=now + ttl, heartbeat_at=now) + ) + self.db.commit() + return changed.rowcount == 1 + + def _assert_writer(self, identifier: str, token: str | None, now: int) -> None: + row = self.db.connection.execute( + select(writer_fences.c.token, writer_fences.c.lease_until) + .where(writer_fences.c.object_id == identifier) + ).first() + if row is None: + if token is not None: + raise ManifestError("Writer lease disappeared") + return + if token != row[0] or int(row[1]) < now: + raise ManifestError("Writer lease lost or expired") + + def stale_pending(self, *, now: int, min_age_seconds: int = 3600, + limit: int = 100) -> list[dict[str, Any]]: + """Dry-run inventory: only expired, aged and explicitly fenced pending rows. + + Never infer that a legacy PENDING object without a lease is abandoned. + The result deliberately excludes the bearer-style writer token. + """ + if min_age_seconds < 60 or not 1 <= limit <= 1000: + raise ValueError("Invalid abandonment policy") + records = self.db.connection.execute( + select( + objects.c.id, objects.c.bucket, objects.c.object_key, + objects.c.size, objects.c.updated_at, + writer_fences.c.lease_until, writer_fences.c.heartbeat_at, + ) + .join(writer_fences, objects.c.id == writer_fences.c.object_id) + .where( + objects.c.status == "pending", + objects.c.updated_at <= now - min_age_seconds, + writer_fences.c.lease_until < now, + ) + .order_by(objects.c.updated_at, objects.c.id).limit(limit) + ).mappings().all() + result = [] + for row in records: + count = self.db.connection.execute( + select(func.count()).select_from(chunks) + .where(chunks.c.object_id == row["id"]) + ).scalar_one() + result.append({ + "id": row["id"], "bucket": row["bucket"], "key": row["object_key"], + "size": int(row["size"]), "updated_at": int(row["updated_at"]), + "lease_until": int(row["lease_until"]), + "heartbeat_at": int(row["heartbeat_at"]), + "journaled_chunks": int(count), + }) + return result + + def abandon_stale_pending(self, *, now: int, min_age_seconds: int = 3600, + limit: int = 100, object_id: str | None = None) -> list[str]: + """Fence expired writers and mark aged pending objects failed. + + Never touch unleased/legacy pending rows or a live writer. The expired + fence is consumed under the same SQL transaction as the status update. + The failed object's Telegram journal is kept for subsequent dry-run GC. + """ + if min_age_seconds < 60 or not (1 <= limit <= 1000): + raise ValueError("Invalid abandonment policy") + candidates = ( + select(objects.c.id) + .join(writer_fences, objects.c.id == writer_fences.c.object_id) + .where( + objects.c.status == "pending", + objects.c.updated_at <= now - min_age_seconds, + writer_fences.c.lease_until < now, + ) + ) + if object_id is not None: + candidates = candidates.where(objects.c.id == object_id) + rows = self.db.connection.execute( + candidates.order_by(objects.c.created_at, objects.c.id).limit(limit) + ).all() + abandoned = [] + for row in rows: + identifier = row[0] + # Atomically fence the old owner first. A concurrent heartbeat + # cannot succeed after this conditional compare-and-update. + affected = self.db.connection.execute( + update(writer_fences).where( + writer_fences.c.object_id == identifier, + writer_fences.c.lease_until < now, + ).values(token="abandoned:" + identifier, lease_until=now) + ) + if affected.rowcount != 1: + continue + changed = self.db.connection.execute( + update(objects).where( + objects.c.id == identifier, objects.c.status == "pending", + objects.c.updated_at <= now - min_age_seconds, + ).values(status="failed", error="Writer lease expired; needs reconciliation", + updated_at=now) + ) + if changed.rowcount == 1: + abandoned.append(identifier) + self.db.commit() + return abandoned + + def reclaim_writer(self, identifier: str, token: str, now: int, + ttl: int = 120) -> bool: + """Reclaim only an expired, explicitly fenced PENDING upload writer.""" + if not token or len(token) > 80 or ttl < 5: + raise ValueError("Invalid writer token or TTL") + try: + row = self.db.connection.execute( + select(objects.c.id).where( + objects.c.id == identifier, objects.c.status == "pending", + ) + ).scalar_one_or_none() + if row is None: + self.db.connection.rollback() + return False + changed = self.db.connection.execute( + update(writer_fences).where( + writer_fences.c.object_id == identifier, + writer_fences.c.lease_until < now, + ).values(token=token, heartbeat_at=now, lease_until=now + ttl) + ) + if changed.rowcount != 1: + self.db.connection.rollback() + return False + self.db.commit() + return True + except BaseException: + self.db.connection.rollback() + raise + + def append(self, identifier: str, part: ChunkReference, now: int, *, + writer_token: str | None = None) -> None: + row = self.db.connection.execute( + select(objects.c.status, objects.c.size).where(objects.c.id == identifier) + ).first() + if row is None or row[0] != "pending": + raise ManifestError("Object is not pending") + self._assert_writer(identifier, writer_token, now) + if part.ordinal < 0 or part.offset < 0 or part.size <= 0: + raise ManifestError("Invalid chunk coordinates") + if len(part.sha256) != 64 or not all(c in "0123456789abcdef" for c in part.sha256): + raise ManifestError("Invalid SHA-256 digest") + if not part.account or part.message_id <= 0 or part.chat_id == 0: + raise ManifestError("Invalid Telegram chunk reference") + count, position = self.db.connection.execute( + select(func.count(), func.coalesce(func.sum(chunks.c.size), 0)) + .where(chunks.c.object_id == identifier) + ).one() + if part.ordinal != count or part.offset != position or part.offset + part.size > row[1]: + raise ManifestError("Chunk order, offset, or size is inconsistent") + self.db.connection.execute(insert(chunks).values( + object_id=identifier, ordinal=part.ordinal, offset=part.offset, + size=part.size, sha256=part.sha256, account=part.account, + chat_id=part.chat_id, message_id=part.message_id, + )) + self.db.connection.execute( + update(objects).where(objects.c.id == identifier).values(updated_at=now) + ) + self.db.commit() + + def finish(self, identifier: str, sha256: str, now: int, *, writer_token: str | None = None) -> StoredObject: + if len(sha256) != 64 or any(c not in "0123456789abcdef" for c in sha256): + raise ManifestError("Invalid object digest") + row = self.db.connection.execute( + select(objects).where(objects.c.id == identifier) + ).mappings().first() + if row is None or row["status"] != "pending": + raise ManifestError("Object is not pending") + self._assert_writer(identifier, writer_token, now) + size, count = self.db.connection.execute( + select(func.coalesce(func.sum(chunks.c.size), 0), func.count()) + .where(chunks.c.object_id == identifier) + ).one() + if size != row["size"] or (count == 0 and size != 0): + raise ManifestError("Cannot finalize incomplete object") + self.db.connection.execute(update(objects).where( + objects.c.id == identifier, objects.c.status == "pending" + ).values(status="ready", sha256=sha256, updated_at=now, error=None)) + if writer_token is not None: + self.db.connection.execute(delete(writer_fences).where( + writer_fences.c.object_id == identifier, + writer_fences.c.token == writer_token, + )) + self.db.commit() + return self.get(identifier) + + def fail(self, identifier: str, error: str, now: int, *, writer_token: str | None = None) -> None: + self._assert_writer(identifier, writer_token, now) + self.db.connection.execute( + update(objects).where(objects.c.id == identifier, objects.c.status == "pending") + .values(status="failed", error=error[:500], updated_at=now) + ) + if writer_token is not None: + self.db.connection.execute(delete(writer_fences).where( + writer_fences.c.object_id == identifier, + writer_fences.c.token == writer_token, + )) + self.db.commit() + + def get(self, identifier: str) -> StoredObject: + row = self.db.connection.execute(select(objects).where( + objects.c.id == identifier, objects.c.status == "ready" + )).mappings().first() + if row is None: + raise FileNotFoundError("Object not found or not committed") + return StoredObject(id=row["id"], bucket=row["bucket"], key=row["object_key"], + size=int(row["size"]), sha256=row["sha256"], + created_at=int(row["created_at"])) + + def resolve_content_id(self, identifier: str) -> str: + """Locate immutable Telegram bytes for a ready object or server-side copy. + + Depth is bounded and cycles fail closed if metadata is corrupt. + This read-only lookup also validates source readiness at each hop. + """ + visited: set[str] = set() + current = identifier + for _ in range(64): + if current in visited: + raise ManifestError("Cyclic copy alias metadata") + visited.add(current) + self.get(current) + source = self.db.connection.execute(select( + object_aliases.c.source_object_id + ).where(object_aliases.c.object_id == current)).scalar_one_or_none() + if source is None: + return current + current = source + raise ManifestError("Copy alias chain is too deep") + + def parts(self, identifier: str) -> list[ChunkReference]: + current = self.resolve_content_id(identifier) + rows = self.db.connection.execute(select(chunks).where( + chunks.c.object_id == current + ).order_by(chunks.c.ordinal)).mappings().all() + return [ChunkReference(ordinal=int(r["ordinal"]), offset=int(r["offset"]), + size=int(r["size"]), sha256=r["sha256"], account=r["account"], + chat_id=int(r["chat_id"]), message_id=int(r["message_id"])) + for r in rows] + + def interrupted(self) -> list[tuple[str, str]]: + """List non-ready objects requiring reconciliation; never show to clients.""" + rows = self.db.connection.execute( + select(objects.c.id, objects.c.status).where(objects.c.status.in_(["pending", "failed"])) + ).all() + return [(str(row[0]), str(row[1])) for row in rows] + + def recovery_parts(self, identifier: str) -> list[ChunkReference]: + """Administrator-only journal access; pending/failed objects are not readable.""" + rows = self.db.connection.execute(select(chunks).where( + chunks.c.object_id == identifier + ).order_by(chunks.c.ordinal)).mappings().all() + return [ChunkReference(ordinal=int(r["ordinal"]), offset=int(r["offset"]), + size=int(r["size"]), sha256=r["sha256"], account=r["account"], + chat_id=int(r["chat_id"]), message_id=int(r["message_id"])) + for r in rows] diff --git a/teledrive/storage/mtproto_rate_guard.py b/teledrive/storage/mtproto_rate_guard.py new file mode 100644 index 00000000..2dce5aee --- /dev/null +++ b/teledrive/storage/mtproto_rate_guard.py @@ -0,0 +1,242 @@ +"""Shared MTProto send-slot, flood-wait cooldown and observation journal. + +The Telethon USER transport is the only live payload path. Independent +workers sharing one SQL database must serialize sends per account/channel +and observe definitive FloodWaitError backoffs globally, exactly like the +retired Bot adapter did for HTTP 429. No method here sends or deletes +Telegram media; the guard is pure durable SQL fencing. +""" +from __future__ import annotations + +import time +from typing import Any + +from sqlalchemy import ( + BigInteger, Column, Integer, MetaData, String, Table, case, func, + insert, select, update, +) +from sqlalchemy.exc import IntegrityError as SQLIntegrityError +from sqlalchemy.exc import OperationalError as SQLOperationalError + +_meta = MetaData() + + +# A process-local asyncio.Lock cannot coordinate independent USER workers +# sharing a Telegram channel. Reserve start slots in durable shared SQL. +# One committed UPDATE serializes competing processes on PostgreSQL/MySQL +# and SQLite. A slot is never proof that a Telegram send was accepted. +_mtproto_chat_slots = Table( + "storage_mtproto_chat_send_slots", _meta, + Column("account", String(255), primary_key=True), + Column("chat_id", BigInteger, primary_key=True), + Column("next_allowed_ms", BigInteger, nullable=False), +) + + +# Separate from the send-slot queue: a flood-wait can arrive *after* +# another worker already reserved a future slot. Pending workers must +# re-check the shared cooldown immediately before their MTProto send. +_mtproto_chat_cooldowns = Table( + "storage_mtproto_chat_cooldowns", _meta, + Column("account", String(255), primary_key=True), + Column("chat_id", BigInteger, primary_key=True), + Column("cooldown_until_ms", BigInteger, nullable=False), +) + + +# A durable, non-secret observation counter helps an operator distinguish +# a genuinely exercised production flood-wait window from purely synthetic +# tests. An observation is a rejection, never proof an upload was accepted. +_mtproto_flood_observations = Table( + "storage_mtproto_flood_observations", _meta, + Column("account", String(255), primary_key=True), + Column("chat_id", BigInteger, primary_key=True), + Column("observations", Integer, nullable=False), + Column("last_observed_ms", BigInteger, nullable=False), + Column("last_retry_after_seconds", Integer, nullable=False), +) + + +class MtprotoRateGuard: + """Durable account/channel rate fencing for the Telethon USER transport.""" + + def __init__(self, database: Any): + self.db = database + _meta.create_all(database.engine) + + def reserve_send_slot(self, account: str, chat_id: int, + interval_seconds: float, *, + now_ms: int | None = None) -> float: + """Reserve one globally serialized per-chat send start time. + + Returns a nonnegative delay in seconds. Shared SQL serialization + prevents independent workers from all starting Telegram sends in + the same second; only an actual FloodWaitError means a definite + rejection. Hosts must have synchronized wall clocks. No slot grants + exclusive ownership of a message or authorizes retransmission. + """ + if (not account or type(chat_id) is not int or not chat_id + or not 0 <= interval_seconds <= 60): + raise ValueError("Invalid MTProto channel rate interval") + gap = int(interval_seconds * 1000 + 0.5) + # Even when local pacing is disabled, allocate a SQL slot row: + # a definitive flood-wait must remain visible to every worker. + timestamp = int(time.time() * 1000) if now_ms is None else now_ms + if type(timestamp) is not int or timestamp < 0: + raise ValueError("Invalid shared MTProto time") + predicate = ( + _mtproto_chat_slots.c.account == account, + _mtproto_chat_slots.c.chat_id == chat_id, + ) + # Begin with a WRITE, not a read: concurrent SQLite workers must + # not upgrade a stale WAL snapshot to a writer. On PostgreSQL and + # MySQL the UPDATE takes the row lock for the entire reservation. + for attempt in range(12): + try: + row = self.db.connection.execute(update(_mtproto_chat_slots).where( + *predicate + ).values(next_allowed_ms=case( + (_mtproto_chat_slots.c.next_allowed_ms > timestamp, + _mtproto_chat_slots.c.next_allowed_ms + gap), + else_=timestamp + gap, + ))) + if row.rowcount == 1: + next_ms = self.db.connection.execute(select( + _mtproto_chat_slots.c.next_allowed_ms + ).where(*predicate)).scalar_one() + self.db.commit() + return max(0.0, (int(next_ms) - gap - timestamp) / 1000) + # First reservation for this pair. If another worker won + # the unique insert, rollback and retry the UPDATE. + self.db.connection.execute(insert(_mtproto_chat_slots).values( + account=account, chat_id=chat_id, + next_allowed_ms=timestamp + gap, + )) + self.db.commit() + return 0.0 + except (SQLIntegrityError, SQLOperationalError): + self.db.connection.rollback() + if attempt == 11: + raise RuntimeError( + "Shared MTProto rate reservation failed after bounded retries" + ) from None + time.sleep(0.015 * (attempt + 1)) + raise RuntimeError("Shared MTProto rate reservation could not finish") + + def impose_shared_cooldown(self, account: str, chat_id: int, + retry_after: int, *, + now_ms: int | None = None) -> None: + """Atomically publish a definitive flood-wait across all workers. + + A worker may already hold a prebooked send slot. Both the next + reservation timestamp and the independent wait-before-send fence + therefore commit in ONE transaction; neither may become visible + separately after a crash or SQL exception. + """ + if (type(retry_after) is not int or not 1 <= retry_after <= 3600 + or not account or type(chat_id) is not int or not chat_id): + raise ValueError("Invalid MTProto flood-wait cooldown") + timestamp = int(time.time() * 1000) if now_ms is None else now_ms + if type(timestamp) is not int or timestamp < 0: + raise ValueError("Invalid MTProto flood-wait timestamp") + until = timestamp + retry_after * 1000 + predicate = ( + _mtproto_chat_cooldowns.c.account == account, + _mtproto_chat_cooldowns.c.chat_id == chat_id, + ) + for attempt in range(12): + try: + row = self.db.connection.execute(update(_mtproto_chat_slots).where( + _mtproto_chat_slots.c.account == account, + _mtproto_chat_slots.c.chat_id == chat_id, + ).values(next_allowed_ms=case( + (_mtproto_chat_slots.c.next_allowed_ms < until, until), + else_=_mtproto_chat_slots.c.next_allowed_ms, + ))) + if row.rowcount != 1: + raise RuntimeError("MTProto shared cooldown row missing") + changed = self.db.connection.execute(update( + _mtproto_chat_cooldowns + ).where(*predicate).values(cooldown_until_ms=case( + (_mtproto_chat_cooldowns.c.cooldown_until_ms < until, until), + else_=_mtproto_chat_cooldowns.c.cooldown_until_ms, + ))) + if changed.rowcount != 1: + self.db.connection.execute(insert(_mtproto_chat_cooldowns).values( + account=account, chat_id=chat_id, + cooldown_until_ms=until, + )) + # The counter is an observation of a verified flood-wait, + # not proof that an upload was accepted. Serialize it with + # the same SQL writer transaction as the rate-limit fence. + observed = self.db.connection.execute( + update(_mtproto_flood_observations).where( + _mtproto_flood_observations.c.account == account, + _mtproto_flood_observations.c.chat_id == chat_id, + ).values( + observations=_mtproto_flood_observations.c.observations + 1, + last_observed_ms=case( + (_mtproto_flood_observations.c.last_observed_ms < timestamp, + timestamp), + else_=_mtproto_flood_observations.c.last_observed_ms, + ), + last_retry_after_seconds=case( + (_mtproto_flood_observations.c.last_observed_ms <= timestamp, + retry_after), + else_=_mtproto_flood_observations.c.last_retry_after_seconds, + ), + ) + ) + if observed.rowcount != 1: + self.db.connection.execute(insert(_mtproto_flood_observations).values( + account=account, chat_id=chat_id, + observations=1, last_observed_ms=timestamp, + last_retry_after_seconds=retry_after, + )) + self.db.commit() + return + except (SQLIntegrityError, SQLOperationalError): + self.db.connection.rollback() + if attempt == 11: + raise RuntimeError( + "Shared MTProto cooldown failed after bounded retries" + ) from None + time.sleep(0.015 * (attempt + 1)) + except BaseException: + self.db.connection.rollback() + raise + + def total_explicit_flood_observations(self) -> int: + """Non-sensitive SQL-wide count of definite flood-wait rejections. + + Compare readings at the beginning and end of a real-host soak. + Synthetic tests must never be presented as production flood evidence. + """ + value = self.db.connection.execute( + select(func.coalesce( + func.sum(_mtproto_flood_observations.c.observations), 0, + )) + ).scalar_one() + self.db.commit() + return int(value) + + def shared_cooldown_wait(self, account: str, chat_id: int, *, + now_ms: int | None = None) -> float: + """Observe a newer flood-wait even after reserving an older send start. + + This is read-only (no remote I/O) and releases the SQL read + transaction promptly so the next check sees concurrent changes. + """ + if not account or type(chat_id) is not int or not chat_id: + raise ValueError("Invalid MTProto shared cooldown identity") + timestamp = int(time.time() * 1000) if now_ms is None else now_ms + if type(timestamp) is not int or timestamp < 0: + raise ValueError("Invalid MTProto shared cooldown timestamp") + deadline = self.db.connection.execute(select( + _mtproto_chat_cooldowns.c.cooldown_until_ms, + ).where( + _mtproto_chat_cooldowns.c.account == account, + _mtproto_chat_cooldowns.c.chat_id == chat_id, + )).scalar_one_or_none() + self.db.commit() + return max(0.0, (int(deadline or 0) - timestamp) / 1000) diff --git a/teledrive/storage/rebuild.py b/teledrive/storage/rebuild.py new file mode 100644 index 00000000..7b457acf --- /dev/null +++ b/teledrive/storage/rebuild.py @@ -0,0 +1,240 @@ +"""Recover independently proven Telegram objects into one isolated SQL catalog. + +Strict P2 recovery: original messages + signed/out-of-band manifest mapping +are required. This module never sends or deletes Telegram media, nor modifies +the original SQL database. A batch becomes visible only after all objects +pass remote identity, per-chunk SHA-256 and whole-object SHA-256 validation. +""" +from __future__ import annotations + +import hashlib +import os +import re +import tempfile +import time +from pathlib import Path +from uuid import UUID + +from sqlalchemy import insert + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.recovery import RemoteChunk +from teledrive.storage.manifest import ( + ChunkReference, ManifestStore, chunks, objects, +) + +_SHA = re.compile(r"[0-9a-f]{64}\Z") +MAX_BATCH_OBJECTS = 128 +MAX_BATCH_CHUNKS = 4096 +MAX_BATCH_BYTES = 1024**3 + + +def validate_plan(plan: dict) -> tuple[list[ChunkReference], int]: + if not isinstance(plan, dict) or set(plan) != { + "object_id", "bucket", "key", "sha256", "size", "chunks", + }: + raise ValueError("Recovery plan must contain exact original identities") + if not isinstance(plan["object_id"], str) or str(UUID(plan["object_id"])) != plan["object_id"]: + raise ValueError("Invalid original object UUID") + BucketCatalog.validate_bucket(plan["bucket"]) + BucketCatalog._keyhash(plan["key"]) + if not isinstance(plan["sha256"], str) or not _SHA.fullmatch(plan["sha256"]): + raise ValueError("Expected independent full-file SHA-256 is required") + size = plan["size"] + if type(size) is not int or size < 0 or size > MAX_BATCH_BYTES: + raise ValueError("Recovery size must be within the one GiB verification cap") + entries = plan["chunks"] + if not isinstance(entries, list) or len(entries) > MAX_BATCH_CHUNKS: + raise ValueError("Invalid chunk inventory") + if bool(size) != bool(entries): + raise ValueError("Empty files cannot have remote chunks") + references = [] + offset = 0 + identities = set() + for position, item in enumerate(entries): + if not isinstance(item, dict) or set(item) != { + "ordinal", "account", "chat_id", "message_id", "size", "sha256", + }: + raise ValueError("Incomplete original Telegram chunk identity") + if (type(item["ordinal"]) is not int or item["ordinal"] != position + or type(item["chat_id"]) is not int + or type(item["message_id"]) is not int + or item["message_id"] <= 0 + or not isinstance(item["account"], str) + or not item["account"] or len(item["account"]) > 255 + or type(item["size"]) is not int or item["size"] <= 0 + or not isinstance(item["sha256"], str) + or not _SHA.fullmatch(item["sha256"])): + raise ValueError("Invalid Telegram chunk reference") + identity = (item["account"], item["chat_id"], item["message_id"]) + if identity in identities: + raise ValueError("Duplicate Telegram message identity") + identities.add(identity) + references.append(ChunkReference( + ordinal=position, offset=offset, size=item["size"], + sha256=item["sha256"], account=item["account"], + chat_id=item["chat_id"], message_id=item["message_id"], + )) + offset += item["size"] + if offset > size: + raise ValueError("Manifest chunks exceed verified object size") + if offset != size: + raise ValueError("Chunk offsets do not cover the complete original file") + return references, size + + +def validate_batch(plans: list[dict]) -> list[tuple[dict, list[ChunkReference]]]: + """Reject ambiguous cross-object identities before any remote IO.""" + if not isinstance(plans, list) or not 1 <= len(plans) <= MAX_BATCH_OBJECTS: + raise ValueError("Recovery batch must contain 1..128 original objects") + seen_ids = set() + seen_heads = set() + seen_messages = set() + total_bytes = 0 + total_chunks = 0 + validated = [] + for plan in plans: + refs, size = validate_plan(plan) + object_id = plan["object_id"] + head = (plan["bucket"], plan["key"]) + if object_id in seen_ids or head in seen_heads: + raise ValueError("Duplicate object ID or published bucket/key") + seen_ids.add(object_id) + seen_heads.add(head) + total_bytes += size + total_chunks += len(refs) + if total_bytes > MAX_BATCH_BYTES or total_chunks > MAX_BATCH_CHUNKS: + raise ValueError("Recovery batch exceeds verification resource bounds") + for ref in refs: + identity = (ref.account, ref.chat_id, ref.message_id) + if identity in seen_messages: + raise ValueError("Telegram message referenced by multiple objects") + seen_messages.add(identity) + validated.append((plan, refs)) + return validated + + +async def _verify_remote_object(plan: dict, refs: list[ChunkReference], + transport, scratch: Path) -> None: + if not callable(getattr(transport, "probe", None)): + raise ValueError("Original Telegram ownership probe is required") + digest = hashlib.sha256() + count = 0 + for ref in refs: + proof = await transport.probe(RemoteChunk( + ref.account, ref.chat_id, ref.message_id, ref.ordinal, + ref.sha256[:16], + ), object_id=plan["object_id"]) + if proof != "verified": + raise ValueError("Original Telegram message ownership unproven") + target = scratch / f"part-{ref.ordinal}" + await transport.download(ref, target) + if not target.is_file() or target.is_symlink(): + raise ValueError("Telegram did not produce a regular file") + part_digest = hashlib.sha256() + read = 0 + with target.open("rb") as stream: + while block := stream.read(1024 * 1024): + read += len(block) + if read > ref.size: + raise ValueError("Unexpected extra Telegram bytes") + part_digest.update(block) + digest.update(block) + if read != ref.size or part_digest.hexdigest() != ref.sha256: + raise ValueError("Telegram chunk hash/length mismatch") + count += read + target.unlink() + if count != plan["size"] or digest.hexdigest() != plan["sha256"]: + raise ValueError("Original Telegram payload does not match external SHA") + + +async def restore_isolated_catalog_batch(plans: list[dict], transport, + destination: Path) -> dict: + """Verify every remote object, then atomically publish a new SQLite file. + + An O_EXCL-style hard link makes the result appear all-or-nothing and never + overwrites an existing path. SQLite is first closed and verified while + still private. The temporary DB is on the same filesystem as destination. + """ + validated = validate_batch(plans) + destination = Path(destination) + if destination.exists() or destination.is_symlink() or not destination.parent.is_dir(): + raise FileExistsError("Refuse to replace an existing or unsafe database") + with tempfile.TemporaryDirectory( + prefix=".teledrive-rebuild-", dir=destination.parent, + ) as temp: + scratch = Path(temp) + for plan, refs in validated: + await _verify_remote_object(plan, refs, transport, scratch) + staging = scratch / "recovered.sqlite" + db = connect_database(str(staging)) + try: + manifests = ManifestStore(db) + catalog = BucketCatalog(manifests) + now = int(time.time()) + for bucket in sorted({p["bucket"] for p, _ in validated}): + catalog.create_bucket(bucket, now) + for plan, refs in validated: + db.connection.execute(insert(objects).values( + id=plan["object_id"], bucket=plan["bucket"], + object_key=plan["key"], status="ready", + size=plan["size"], sha256=plan["sha256"], + created_at=now, updated_at=now, error=None, + )) + if refs: + db.connection.execute(insert(chunks), [ + dict(object_id=plan["object_id"], ordinal=ref.ordinal, + offset=ref.offset, size=ref.size, sha256=ref.sha256, + account=ref.account, chat_id=ref.chat_id, + message_id=ref.message_id) + for ref in refs + ]) + db.commit() + catalog.publish(plan["object_id"], now=now, expected_generation=0) + # Re-check the staged SQL structure and all restored current heads + # before the file can be opened by an operator. + for plan, refs in validated: + item = catalog.get(plan["bucket"], plan["key"]).item + if (item.id != plan["object_id"] + or item.size != plan["size"] + or item.sha256 != plan["sha256"] + or catalog.manifests.parts(item.id) != refs): + raise ValueError("Staged recovery manifest differs from source proof") + finally: + db.close() + # Verify the SQLite file itself before publishing a pathname. + import sqlite3 + with sqlite3.connect(f"file:{staging}?mode=ro", uri=True) as audit: + if audit.execute("PRAGMA integrity_check").fetchone()[0] != "ok": + raise ValueError("Staging SQLite integrity check failed") + # Protect original message coordinates while staging is still private. + # chmod AFTER publication would briefly expose the recovered catalog. + os.chmod(staging, 0o600) + try: + os.link(staging, destination, follow_symlinks=False) + except FileExistsError as exc: + raise FileExistsError("An existing database must not be overwritten") from exc + return { + "verified": True, "isolated_database_created": True, + "object_count": len(validated), + "object_ids": [plan["object_id"] for plan, _ in validated], + "verified_bytes": sum(plan["size"] for plan, _ in validated), + "verified_chunks": sum(len(refs) for _, refs in validated), + "telegram_uploads": 0, "telegram_deletions": 0, + "production_database_modified": False, + "historic_versions_reconstructed": False, + "complete_unmapped_history_discovered": False, + } + + +async def restore_isolated_catalog(plan: dict, transport, destination: Path) -> dict: + """Backward-compatible single-object adapter for the original DR CLI.""" + result = await restore_isolated_catalog_batch([plan], transport, destination) + return { + **result, + "object_id": plan["object_id"], + "size": plan["size"], + "sha256": plan["sha256"], + "chunks": len(plan["chunks"]), + } diff --git a/teledrive/storage/reconcile_inventory.py b/teledrive/storage/reconcile_inventory.py new file mode 100644 index 00000000..6392c2c5 --- /dev/null +++ b/teledrive/storage/reconcile_inventory.py @@ -0,0 +1,98 @@ +"""Offline preflight: compare discovered Telegram markers with independently supplied plans. + +No bytes, original bucket/key or historical versions can be inferred from a +caption. Matching coordinates are NOT proof of remote ownership or integrity. +""" +from __future__ import annotations + +import hashlib +import json + +from .rebuild import validate_batch + + +def reconcile_inventory(inventory: dict, plans: list[dict]) -> dict: + validated = validate_batch(plans) + if not isinstance(inventory, dict) or inventory.get("schema_version") != 1: + raise ValueError("Unsupported Telegram inventory schema") + scope = inventory.get("scope") + candidates = inventory.get("candidate_objects") + if (not isinstance(scope, dict) or type(scope.get("chat_id")) is not int + or not isinstance(scope.get("account"), str) or not scope["account"] + or not isinstance(candidates, list) or len(candidates) > 10000 + or type(inventory.get("scanned_messages")) is not int + or not 0 <= inventory["scanned_messages"] <= 10000 + or type(inventory.get("truncated")) is not bool + or type(inventory.get("candidate_object_count")) is not int + or inventory["candidate_object_count"] != len(candidates) + or type(inventory.get("candidate_chunk_count")) is not int): + raise ValueError("Malformed bounded Telegram inventory") + + observed = {} + seen_messages = set() + total = 0 + for candidate in candidates: + if (not isinstance(candidate, dict) + or not isinstance(candidate.get("object_id"), str) + or candidate["object_id"] in observed + or not isinstance(candidate.get("chunks"), list) + or len(candidate["chunks"]) > 4096): + raise ValueError("Ambiguous object marker inventory") + parts = [] + for part in candidate["chunks"]: + if (not isinstance(part, dict) + or set(part) != {"ordinal", "account", "chat_id", + "message_id", "checksum_prefix"} + or type(part["ordinal"]) is not int or part["ordinal"] < 0 + or type(part["chat_id"]) is not int + or type(part["message_id"]) is not int or part["message_id"] <= 0 + or part["account"] != scope["account"] + or part["chat_id"] != scope["chat_id"] + or not isinstance(part["checksum_prefix"], str) + or len(part["checksum_prefix"]) != 16 + or any(c not in "0123456789abcdef" for c in part["checksum_prefix"])): + raise ValueError("Untrusted Telegram candidate coordinates") + coords = (part["account"], part["chat_id"], part["message_id"]) + if coords in seen_messages: + raise ValueError("Duplicate Telegram message in inventory") + seen_messages.add(coords) + parts.append((part["ordinal"], *coords, part["checksum_prefix"])) + total += 1 + observed[candidate["object_id"]] = parts + if total != inventory["candidate_chunk_count"] or total > inventory["scanned_messages"]: + raise ValueError("Inconsistent candidate chunk counts") + outcomes = [] + for plan, refs in validated: + original = [(x.ordinal, x.account, x.chat_id, x.message_id, x.sha256[:16]) + for x in refs] + found = observed.get(plan["object_id"]) + if not refs: + result = "unproven_empty_object" + elif found is None: + result = "not_observed_in_scanned_window" + elif len(found) != len(original) or sorted(found) != sorted(original): + result = "message_identity_mismatch" + else: + result = "candidate_coordinates_match" + outcomes.append({ + "object_id": plan["object_id"], "result": result, + "matched_chunk_count": len(refs) if result == "candidate_coordinates_match" else 0, + "payload_verified": False, + }) + matched = all(x["result"] == "candidate_coordinates_match" for x in outcomes) + fingerprint = hashlib.sha256(json.dumps( + {"plans": plans, "inventory": inventory}, sort_keys=True, + separators=(",", ":"), ensure_ascii=False, + ).encode("utf-8")).hexdigest() + return { + "schema_version": 1, "report_sha256": fingerprint, + "objects_checked": len(outcomes), "all_plans_match": matched, + "ready_for_independent_byte_verification": matched, + "inventory_window_truncated": inventory["truncated"], + "extra_unmapped_candidates": len(set(observed) - {p["object_id"] for p, _ in validated}), + "matches": outcomes, + "historic_versions_reconstructed": False, + "remote_ownership_verified": False, "payload_verified": False, + "automatic_import_performed": False, + "telegram_deletions": 0, "production_database_modified": False, + } diff --git a/teledrive/storage/recovery.py b/teledrive/storage/recovery.py new file mode 100644 index 00000000..dba70dba --- /dev/null +++ b/teledrive/storage/recovery.py @@ -0,0 +1,238 @@ +"""Conservative recovery and garbage collection for failed Telegram uploads. + +No Telegram messages are deleted automatically. The caller must explicitly +request cleanup, pass a minimum age gate, and provide a transport able to +prove the messages belong to that exact failed object. + +Ready and in-progress objects are never garbage-collected by this module. +""" +from __future__ import annotations + +from dataclasses import dataclass +import time +from typing import Literal, Protocol + +from sqlalchemy import and_, inspect, or_, select, update + +from .manifest import ChunkReference, ManifestError, ManifestStore, chunks, objects + + +class UnsafeCleanup(ManifestError): + """An incomplete or ambiguous scan prevents any destructive action.""" + + +@dataclass(frozen=True) +class RemoteChunk: + account: str + chat_id: int + message_id: int + ordinal: int + checksum_prefix: str + + @property + def identity(self) -> tuple[str, int, int]: + return self.account, self.chat_id, self.message_id + + +@dataclass(frozen=True) +class Discovery: + matches: tuple[RemoteChunk, ...] + complete: bool + scanned: int + + +@dataclass(frozen=True) +class CleanupReport: + object_id: str + journaled: int + discovered: int + orphaned: int + missing: int + deleted: int + dry_run: bool + cleaned: bool + + +class RecoveryTransport(Protocol): + async def discover(self, object_id: str, *, scan_limit: int) -> Discovery: + """Scan known targets for outgoing uploads with an exact object marker.""" + ... + + async def probe(self, chunk: RemoteChunk, *, object_id: str) -> Literal["verified", "missing", "mismatch"]: + """Prove outgoing message caption matches this object, ordinal, digest prefix.""" + ... + + async def delete_verified(self, chunk: RemoteChunk, *, object_id: str) -> bool: + """Delete only after verifying ownership; return True only if absent afterward.""" + ... + + +def _from_journal(part: ChunkReference) -> RemoteChunk: + return RemoteChunk( + account=part.account, chat_id=part.chat_id, message_id=part.message_id, + ordinal=part.ordinal, checksum_prefix=part.sha256[:16], + ) + + +class RecoveryManager: + def __init__(self, manifests: ManifestStore, transport: RecoveryTransport, *, + min_age_seconds: int = 24 * 60 * 60, scan_limit: int = 10000): + if min_age_seconds < 60 or scan_limit < 1: + raise ValueError("Unsafe recovery age/scan limits") + self.manifests = manifests + self.transport = transport + self.min_age_seconds = min_age_seconds + self.scan_limit = scan_limit + + def _eligible(self, object_id: str, now: int) -> None: + row = self.manifests.db.connection.execute( + select(objects.c.status, objects.c.updated_at).where(objects.c.id == object_id) + ).first() + if row is None: + raise FileNotFoundError("Object not found") + if row[0] != "failed": + raise UnsafeCleanup("Only failed objects may be cleaned up") + if int(row[1]) > now - self.min_age_seconds: + raise UnsafeCleanup("Failure has not aged beyond the cleanup safety interval") + + def _require_no_sql_references(self, object_id: str) -> None: + """A remote ownership marker alone never authorizes physical deletion. + + Checks every installed reference table. Historic published versions, + logical deletions, aliases, active multipart parts, unresolved MTProto + send intents and active durable jobs remain retained regardless of the + current S3 HEAD. Rechecked during the delete loop because remote + operations release SQL locks. + """ + from teledrive.storage.catalog import heads, published_versions, removed_versions + from teledrive.storage.manifest import object_aliases, writer_fences + from teledrive.s3.multipart import parts + from teledrive.storage.user_mode import mtproto_send_intents + from teledrive.storage.gc_safety import ACTIVE_JOB_STATES + from teledrive.storage.jobs import jobs as upload_jobs + + connection = self.manifests.db.connection + inspector = inspect(self.manifests.db.engine) + checks = ( + ("storage_object_heads", heads.c.object_id == object_id, heads), + ("storage_published_versions", published_versions.c.object_id == object_id, published_versions), + ("storage_removed_versions", removed_versions.c.object_id == object_id, removed_versions), + ("storage_writer_fences", writer_fences.c.object_id == object_id, writer_fences), + ("storage_object_aliases", or_( + object_aliases.c.object_id == object_id, + object_aliases.c.source_object_id == object_id, + ), object_aliases), + ("storage_multipart_parts", parts.c.object_id == object_id, parts), + ("storage_mtproto_send_intents", + mtproto_send_intents.c.object_id == object_id, mtproto_send_intents), + ) + for name, condition, table in checks: + if inspector.has_table(name): + if connection.execute(select(table).where(condition).limit(1)).first(): + raise UnsafeCleanup("Telegram media retained by another SQL reference") + # Active durable jobs can still publish this object (by identity or + # by the same bucket/key), so they fence physical cleanup as well. + if inspector.has_table("storage_upload_jobs"): + row = connection.execute(select( + objects.c.bucket, objects.c.object_key + ).where(objects.c.id == object_id)).first() + conditions = [ + upload_jobs.c.object_id == object_id, + upload_jobs.c.expected_head_id == object_id, + ] + if row is not None: + bucket, key = row + conditions.append(and_( + upload_jobs.c.bucket == bucket, + upload_jobs.c.object_key == key, + )) + if connection.execute(select(upload_jobs).where( + upload_jobs.c.state.in_(ACTIVE_JOB_STATES), or_(*conditions), + ).limit(1)).first(): + raise UnsafeCleanup("Telegram media retained by another SQL reference") + + async def cleanup(self, object_id: str, *, dry_run: bool = True) -> CleanupReport: + """Reconcile journal+Telegram, optionally delete; must run with uploader stopped. + + Acquires a per-object DB lease to serialize cleanup workers, and + checks readiness/age both before and after scanning. If a Telegram + target is not exhaustively scanned, no messages are deleted. + """ + now = int(time.time()) + self._eligible(object_id, now) + db = self.manifests.db + lease_name = f"storage-gc:{object_id}" + holder = f"cleanup-{object_id}-{time.time_ns()}" + if not db.acquire_lease(lease_name, holder, now, 300): + raise UnsafeCleanup("Another cleanup worker owns this object") + try: + self._eligible(object_id, now) + journal = [_from_journal(p) for p in self.manifests.recovery_parts(object_id)] + # Every journaled reference must either be missing or be verified. + # If it points to unrelated Telegram media, refuse ALL deletion. + missing = 0 + for part in journal: + verdict = await self.transport.probe(part, object_id=object_id) + if verdict == "mismatch": + raise UnsafeCleanup("Journaled message does not belong to this object") + if verdict == "missing": + missing += 1 + if verdict not in ("verified", "missing", "mismatch"): + raise UnsafeCleanup("Transport returned an invalid proof") + + found = await self.transport.discover(object_id, scan_limit=self.scan_limit) + if not found.complete: + raise UnsafeCleanup("Telegram scan was truncated or incomplete") + all_refs = {part.identity: part for part in journal} + journal_keys = set(all_refs) + for part in found.matches: + if part.identity in all_refs and all_refs[part.identity] != part: + raise UnsafeCleanup("Conflicting Telegram chunk metadata") + all_refs[part.identity] = part + # A forged/malformed discovery must not cause arbitrary deletion. + for part in found.matches: + if await self.transport.probe(part, object_id=object_id) != "verified": + raise UnsafeCleanup("Discovered message ownership cannot be proved") + + orphaned = len(set(all_refs) - journal_keys) + if dry_run: + return CleanupReport(object_id, len(journal), len(found.matches), + orphaned, missing, 0, True, False) + + self._eligible(object_id, int(time.time())) + self._require_no_sql_references(object_id) + deleted = 0 + for part in all_refs.values(): + # Lease expiration must fail closed, never continue deleting. + if not db.renew_lease(lease_name, holder, int(time.time()), 300): + raise UnsafeCleanup("Cleanup lease lost") + self._require_no_sql_references(object_id) + verdict = await self.transport.probe(part, object_id=object_id) + if verdict == "missing": + continue + if verdict != "verified": + raise UnsafeCleanup("Telegram ownership changed during cleanup") + if not await self.transport.delete_verified(part, object_id=object_id): + raise UnsafeCleanup("Telegram deletion not verified; cleanup can be retried") + deleted += 1 + + after = await self.transport.discover(object_id, scan_limit=self.scan_limit) + if not after.complete or after.matches: + raise UnsafeCleanup("Residual Telegram messages remain after cleanup") + if not db.renew_lease(lease_name, holder, int(time.time()), 300): + raise UnsafeCleanup("Cleanup lease lost during final scan") + self._eligible(object_id, int(time.time())) + self._require_no_sql_references(object_id) + + # Do not publish a tombstone if state was changed by another actor. + result = db.connection.execute( + update(objects).where( + objects.c.id == object_id, objects.c.status == "failed" + ).values(status="cleaned", updated_at=int(time.time()))) + db.commit() + if result.rowcount != 1: + raise UnsafeCleanup("Object state changed during cleanup") + return CleanupReport(object_id, len(journal), len(found.matches), + orphaned, missing, deleted, False, True) + finally: + db.release_lease(lease_name, holder) diff --git a/teledrive/storage/target_resolver.py b/teledrive/storage/target_resolver.py new file mode 100644 index 00000000..9b64dd4c --- /dev/null +++ b/teledrive/storage/target_resolver.py @@ -0,0 +1,55 @@ +"""Resolve Telegram E2E destinations from private channel titles or peer IDs. + +StringSession does not retain Telegram entity caches across processes. Resolve +private channels using the authorized user's dialogs to preserve access hashes. +""" + +from __future__ import annotations + +from typing import Any + + +async def resolve_telegram_target( + client: Any, + target_spec: str | None, + channel_name: str | None = None, +) -> Any: + """Resolve a Telegram destination without guessing private-channel entities. + + If channel_name is configured, find exactly one broadcast channel with the + specified title in the authenticated account's dialogs. Channel-name + matching takes precedence over the legacy TELEDRIVE_E2E_TARGET variable. + """ + name = (channel_name or "").strip() + if name: + matches = [] + async for dialog in client.iter_dialogs(): + if (dialog.is_channel and not dialog.is_group + and not getattr(dialog.entity, "username", None) + and dialog.name.casefold() == name.casefold()): + matches.append(dialog.entity) + if len(matches) > 1: + break + if not matches: + raise ValueError( + f"No broadcast channel named {name!r} found in the test account's dialogs. " + "Create or join that private channel using the test account." + ) + if len(matches) > 1: + raise ValueError( + f"Multiple Telegram broadcast channels named {name!r} found. " + "Give the test channel a unique name." + ) + return matches[0] + + target = (target_spec or "me").strip() or "me" + if target.lstrip("-").isdigit(): + target_id = int(target) + async for dialog in client.iter_dialogs(): + if dialog.id == target_id: + return dialog.entity + raise ValueError( + f"Telegram E2E target {target_id} is not in the test account's dialogs. " + "Join/create the test channel with the same authorized account." + ) + return await client.get_entity(target) diff --git a/teledrive/storage/telethon_adapter.py b/teledrive/storage/telethon_adapter.py new file mode 100644 index 00000000..d7fba04d --- /dev/null +++ b/teledrive/storage/telethon_adapter.py @@ -0,0 +1,163 @@ +"""Telethon-backed chunk I/O. + +Sessions must already be authorized. Telegram is the only payload backend; +this adapter does not silently fall back to a local/S3 storage provider. +""" +from __future__ import annotations + +from pathlib import Path +import re +from uuid import UUID +from typing import Any, Mapping + +from .manifest import ChunkReference +from .recovery import Discovery, RemoteChunk + +# The unified USER-MTProto runtime splits large objects into bounded Telegram +# chunks; this is the app-level chunk size shared by REST, S3 and local jobs. +RECOMMENDED_CHUNK_SIZE = 16 * 1024 * 1024 + + +class TelethonChunkTransport: + """Use configured account clients for writes and original account for reads. + + Each mapping value is (connected Telethon client, Telegram target entity). + The application must keep the sessions connected while requests run. + """ + + def __init__(self, accounts: Mapping[str, tuple[Any, Any]]): + if not accounts: + raise ValueError("At least one authorized Telegram account is required") + self.accounts = dict(accounts) + self._cursor = 0 + + async def upload(self, path: Path, *, object_id: str, ordinal: int, + checksum: str) -> tuple[str, int, int]: + names = list(self.accounts) + account = names[self._cursor % len(names)] + self._cursor += 1 + client, target = self.accounts[account] + # A stable object/part marker allows later orphan reconciliation after + # send_file succeeded but DB commit failed or the process was killed. + caption = f"teledrive:{object_id}:{ordinal}:{checksum[:16]}" + message = await client.send_file( + target, str(path), caption=caption, force_document=True + ) + if message is None or not getattr(message, "id", None): + raise RuntimeError("Telegram did not return a valid message") + chat_id = getattr(message, "chat_id", None) + if chat_id is None: + chat_id = await client.get_peer_id(target) + return account, int(chat_id), int(message.id) + + async def download(self, reference: ChunkReference, destination: Path) -> None: + account = self.accounts.get(reference.account) + if account is None: + raise FileNotFoundError("Telegram account holding this chunk is unavailable") + client, target = account + # Re-fetch the original message (rather than caching file_reference). + # Reuse the already-resolved target where it matches the saved chat ID; + # StringSession does not necessarily persist Telegram entity caches. + peer = reference.chat_id + if int(await client.get_peer_id(target)) == reference.chat_id: + peer = target + message = await client.get_messages(peer, ids=reference.message_id) + if message is None or not getattr(message, "media", None): + raise FileNotFoundError("Telegram message/media was deleted or inaccessible") + result = await client.download_media(message, file=str(destination)) + if result is None: + raise FileNotFoundError("Telegram media download failed") + actual = Path(result) + if actual != destination: + # Telethon may append a MIME extension to an extensionless target. + if actual.resolve().parent != destination.parent.resolve(): + raise RuntimeError("Unexpected Telegram download path") + actual.replace(destination) + if not destination.is_file() or destination.is_symlink(): + raise FileNotFoundError("Telegram media download failed") + + async def delete(self, reference: ChunkReference) -> None: + """Explicit cleanup hook; never run automatically before reconciliation.""" + account = self.accounts.get(reference.account) + if account is None: + raise FileNotFoundError("Telegram account unavailable for cleanup") + client, _ = account + await client.delete_messages(reference.chat_id, [reference.message_id]) + + + @staticmethod + def _parse_marker(caption: str | None, object_id: str) -> tuple[int, str] | None: + # Never accept substrings or prefixes, only the exact upload marker. + try: + UUID(object_id) + except ValueError: + return None + matched = re.fullmatch( + rf"teledrive:{re.escape(object_id)}:([0-9]{{1,10}}):([0-9a-f]{{16}})", + caption or "", + ) + if matched is None: + return None + return int(matched.group(1)), matched.group(2) + + async def _message_for(self, part: RemoteChunk): + account = self.accounts.get(part.account) + if account is None: + raise RuntimeError("Stored Telegram account is not configured") + client, target = account + peer_id = int(await client.get_peer_id(target)) + if peer_id != part.chat_id: + # Do not inspect or delete a message from a different target. + raise RuntimeError("Configured Telegram target differs from the original chunk") + return client, target, await client.get_messages(target, ids=part.message_id) + + async def probe(self, chunk: RemoteChunk, *, object_id: str) -> str: + _, _, message = await self._message_for(chunk) + if message is None: + return "missing" + marker = self._parse_marker(getattr(message, "message", None), object_id) + actual_chat = getattr(message, "chat_id", None) + if (not getattr(message, "out", False) or marker is None + or marker != (chunk.ordinal, chunk.checksum_prefix) + or (actual_chat is not None and int(actual_chat) != chunk.chat_id)): + return "mismatch" + return "verified" + + async def discover(self, object_id: str, *, scan_limit: int) -> Discovery: + if scan_limit < 1: + raise ValueError("scan_limit must be positive") + found: list[RemoteChunk] = [] + scanned = 0 + complete = True + for name, (client, target) in self.accounts.items(): + chat_id = int(await client.get_peer_id(target)) + local_count = 0 + async for message in client.iter_messages(target, limit=scan_limit + 1): + local_count += 1 + scanned += 1 + if local_count > scan_limit: + complete = False + break + marker = self._parse_marker(getattr(message, "message", None), object_id) + if marker is None: + continue + # Channel messages by other admins may look like uploads but + # cannot be proven owned by this TeleDrive account. + if not getattr(message, "out", False): + complete = False + continue + remote_chat = getattr(message, "chat_id", None) + if remote_chat is not None and int(remote_chat) != chat_id: + complete = False + continue + found.append(RemoteChunk(name, chat_id, int(message.id), *marker)) + return Discovery(tuple(found), complete, scanned) + + async def delete_verified(self, chunk: RemoteChunk, *, object_id: str) -> bool: + if await self.probe(chunk, object_id=object_id) != "verified": + return False + client, target, _ = await self._message_for(chunk) + await client.delete_messages(target, [chunk.message_id], revoke=True) + # Do not assume Telegram accepted deletion. A later retry can safely + # continue if eventual consistency makes the message still visible. + return await self.probe(chunk, object_id=object_id) == "missing" diff --git a/teledrive/storage/user_mode.py b/teledrive/storage/user_mode.py new file mode 100644 index 00000000..1e4c0e0f --- /dev/null +++ b/teledrive/storage/user_mode.py @@ -0,0 +1,520 @@ +"""Owner-authorized user MTProto transport for the single REST/S3 catalog. + +Only an existing user StringSession is accepted. Channel targets are required: +Saved Messages, bot sessions and private chat destinations are intentionally +not storage targets. The session runs on the web application's ASGI event loop. +An acknowledged send is persisted before the object engine writes its chunk. +Pending sends are never retried automatically after an ambiguous failure. +""" +from __future__ import annotations + +import asyncio +from dataclasses import replace +import os +import platform +import re +import time +from pathlib import Path +from typing import Any + +from sqlalchemy import BigInteger, Column, Integer, MetaData, String, Table, func, insert, select, update +from sqlalchemy.exc import IntegrityError + +from .mtproto_rate_guard import MtprotoRateGuard +from .telethon_adapter import TelethonChunkTransport + +_meta = MetaData() +mtproto_send_intents = Table( + "storage_mtproto_send_intents", _meta, + Column("account", String(255), primary_key=True), + Column("chat_id", BigInteger, primary_key=True), + Column("object_id", String(36), primary_key=True), + Column("ordinal", Integer, primary_key=True), + Column("sha256", String(64), nullable=False), + Column("size", BigInteger, nullable=False), + Column("message_id", BigInteger), +) + + +def transport_mode() -> str: + """Only one live transport exists: Telethon USER MTProto.""" + return "telethon" + + +def _session_from_file() -> str: + filename = os.environ.get("TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE", "").strip() + return _session_from_file_for(filename) + + +def _session_from_file_for(filename: str) -> str: + if not filename: + raise RuntimeError("Set TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE for user storage") + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(filename, flags) + with os.fdopen(fd, "r", encoding="ascii") as handle: + value = handle.read(8193).strip() + if not value or len(value) > 8192: + raise RuntimeError("Invalid authorized user session file") + return value + + +def _split_list(value: str, name: str) -> list[str]: + items = [part.strip() for part in value.split(",")] + if not items or any(not part for part in items): + raise RuntimeError(f"Invalid {name} credential list") + return items + + +def _validate_set(label: str, api_id: int, api_hash: str, + session_file: str) -> tuple[str, int, str, str]: + if (api_id <= 0 + or not re.fullmatch(r"[0-9a-fA-F]{32}", api_hash) + or not re.fullmatch(r"[A-Za-z0-9_-]{1,64}", label) + or label == "api-bot"): + raise RuntimeError("Invalid MTProto credential set") + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(session_file, flags) + with os.fdopen(descriptor, "r", encoding="ascii") as handle: + value = handle.read(8193).strip() + except OSError: + raise RuntimeError("Invalid authorized user session file") from None + if not value or len(value) > 8192: + raise RuntimeError("Invalid authorized user session file") + return label, api_id, api_hash, session_file + + +def _config_storage_section() -> dict | None: + """Read the `storage:` section from the deployment config (config.yaml).""" + import yaml + + config_path = os.environ.get("TELEDRIVE_CONFIG", "config/config.yaml").strip() + if not config_path or not os.path.isfile(config_path): + return None + try: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + descriptor = os.open(config_path, flags) + with os.fdopen(descriptor, "r", encoding="utf-8") as handle: + document = yaml.safe_load(handle) or {} + except OSError: + raise RuntimeError("Invalid deployment configuration file") from None + section = document.get("storage") + if section is None: + return None + if not isinstance(section, dict): + raise RuntimeError("Invalid storage section in configuration") + return section + + +def _config_api_sets(section: dict | None) -> list[tuple[str, int, str, str]] | None: + """Read the `apis:` list from the storage section. + + Each entry needs api_id, api_hash and session_file; account labels are + optional and default to mtproto-user, mtproto-user-2, ... Returns None + when there is no `apis:` section. + """ + entries = section.get("apis") if section else None + if not entries: + return None + if not isinstance(entries, list) or not entries: + raise RuntimeError("Invalid apis credential list in configuration") + sets = [] + labels: set[str] = set() + for index, entry in enumerate(entries): + if not isinstance(entry, dict): + raise RuntimeError("Invalid MTProto credential set") + label = str(entry.get("account") or ( + "mtproto-user" if index == 0 else f"mtproto-user-{index + 1}" + )).strip() + try: + api_id = int(entry.get("api_id")) + except (TypeError, ValueError): + raise RuntimeError("Invalid MTProto credential set") from None + api_hash = str(entry.get("api_hash") or "").strip() + session_file = str(entry.get("session_file") or "").strip() + if not session_file: + raise RuntimeError("Invalid authorized user session file") + if label in labels: + raise RuntimeError("Invalid MTProto credential set") + labels.add(label) + sets.append(_validate_set(label, api_id, api_hash, session_file)) + return sets + + +def _credential_sets() -> list[tuple[str, int, str, str]]: + """Parse one or more authorized Telegram API credential sets. + + The deployment config (config.yaml) `storage.apis:` list is + authoritative. Without it, TELEDRIVE_STORAGE_MTPROTO_API_ID / _API_HASH + / _SESSION_FILE accept comma-separated lists of equal length (legacy + fallback). Each set owns its own session; account labels default to + mtproto-user, mtproto-user-2, ... + """ + section = _config_storage_section() + configured = _config_api_sets(section) + if configured is None: + raise RuntimeError( + "storage.apis in config.yaml must list at least one " + "{api_id, api_hash, session_file} credential set" + ) + return configured + + +class UserSessionRuntime: + """One or more user-authorized clients owned by the ASGI lifespan. + + Every credential set must resolve the SAME dedicated broadcast channel. + round_robin distributes sends across sets; failover keeps the current + set until a definite flood-wait rejection, then advances. + """ + + def __init__(self): + self.client = None + self.transport = None + self.account = None + self.chat_id = None + self.strategy = "round_robin" + self._sets: dict[str, tuple[Any, Any]] = {} + self._labels: list[str] = [] + self._index = 0 + + def transports(self) -> dict[str, tuple[Any, Any]]: + return dict(self._sets) + + async def start(self) -> None: + if transport_mode() != "telethon": + return + from telethon import TelegramClient + from telethon.sessions import StringSession + from telethon.tl.types import Channel + + if self._sets: + raise RuntimeError("User storage clients already started") + storage_section = _config_storage_section() + strategy = str((storage_section or {}).get("strategy", "round_robin")).strip().lower() + if strategy not in ("single", "round_robin", "failover", "parallel"): + raise RuntimeError( + "MTProto strategy must be single, round_robin, failover or parallel" + ) + self.strategy = strategy + target_id = str((storage_section or {}).get("channel", "")).strip() + if (not target_id or target_id.lower() in ("me", "self", "saved messages")): + raise RuntimeError("Invalid dedicated channel configuration") + sets = _credential_sets() + clients: list[tuple[str, Any]] = [] + resolved_chat: int | None = None + try: + for label, api_id, api_hash, session_file in sets: + session = _session_from_file_for(session_file) + # These are honest TeleDrive client details, not a forged + # official Telegram fingerprint. MTProto has no HTTP User-Agent. + device_model = ( + "TeleDrive Storage Client" if len(sets) == 1 + else "TeleDrive Storage Client:" + label + ) + client = TelegramClient( + StringSession(session), api_id, api_hash, + device_model=device_model, + system_version=platform.system(), + app_version="TeleDrive 2.1", + lang_code="en", system_lang_code="en", + receive_updates=False, flood_sleep_threshold=0, + ) + await client.connect() + clients.append((label, client)) + identity = await client.get_me() + if identity is None or getattr(identity, "bot", False): + raise RuntimeError( + "MTProto storage requires an authorized Telegram USER session" + ) + if target_id.lstrip("-").isdigit(): + # StringSession carries no entity cache. A private channel + # ID must be resolved from the user's own dialogs. + wanted = int(target_id) + channel = None + async for dialog in client.iter_dialogs(limit=5000): + if int(dialog.id) == wanted: + channel = dialog.entity + break + if channel is None: + raise RuntimeError( + "Configured channel not found in authorized user's dialogs" + ) + else: + channel = await client.get_entity(target_id) + if not isinstance(channel, Channel) or not getattr( + channel, "broadcast", False): + raise RuntimeError( + "MTProto storage target must be a channel, not Saved Messages" + ) + rights = getattr(channel, "admin_rights", None) + if not (getattr(channel, "creator", False) + or (rights is not None + and getattr(rights, "post_messages", False))): + raise RuntimeError("User must have channel posting rights") + chat_id = int(await client.get_peer_id(channel)) + if resolved_chat is None: + resolved_chat = chat_id + elif chat_id != resolved_chat: + raise RuntimeError( + "Every MTProto credential set must target the same channel" + ) + self._sets[label] = (client, channel) + self._labels.append(label) + except BaseException: + for _, client in clients: + await client.disconnect() + self._sets.clear() + self._labels.clear() + raise + self.account = self._labels[0] + self.chat_id = resolved_chat + self.client = self._sets[self._labels[0]][0] + self.transport = TelethonChunkTransport({ + label: pair for label, pair in self._sets.items() + }) + + async def stop(self) -> None: + clients = [pair[0] for pair in self._sets.values()] + self.client = self.transport = None + self.account = self.chat_id = None + self._sets.clear() + self._labels.clear() + self._index = 0 + for client in clients: + await client.disconnect() + + def advance(self) -> str: + """Move to the next credential set and return its account label.""" + if not self._labels: + # Legacy single-client wiring without start(): no rotation. + return self.account + self._index = (self._index + 1) % len(self._labels) + self.account = self._labels[self._index] + self.client = self._sets[self.account][0] + return self.account + + def require(self, label: str | None = None): + if self._sets and self._labels: + selected = label or self._labels[self._index] + client, channel = self._sets[selected] + if not client.is_connected(): + raise RuntimeError("User Telegram session is not connected") + if label is None: + self.account = selected + return TelethonChunkTransport({selected: (client, channel)}) + # Legacy single-client wiring without start(). + if (self.client is None or self.transport is None + or not self.client.is_connected()): + raise RuntimeError("User Telegram session is not connected") + return self.transport + + +class UserSendJournal: + def __init__(self, db): + self.db = db + _meta.create_all(db.engine) + + def start(self, account: str, chat_id: int, object_id: str, ordinal: int, + sha256: str, size: int) -> None: + if (not account or not chat_id or ordinal < 0 or size < 1 + or size > 1024 * 1024 * 1024 + or not re.fullmatch(r"[0-9a-f]{64}", sha256)): + raise ValueError("Invalid MTProto upload intent") + try: + self.db.connection.execute(insert(mtproto_send_intents).values( + account=account, chat_id=chat_id, object_id=object_id, + ordinal=ordinal, sha256=sha256, size=size, message_id=None, + )) + self.db.commit() + except IntegrityError: + self.db.connection.rollback() + raise RuntimeError("Unresolved MTProto upload intent: reconcile before retry") from None + + def acknowledged(self, account: str, chat_id: int, object_id: str, + ordinal: int, message_id: int) -> None: + changed = self.db.connection.execute(update(mtproto_send_intents).where( + mtproto_send_intents.c.account == account, + mtproto_send_intents.c.chat_id == chat_id, + mtproto_send_intents.c.object_id == object_id, + mtproto_send_intents.c.ordinal == ordinal, + mtproto_send_intents.c.message_id.is_(None), + ).values(message_id=message_id)) + if changed.rowcount != 1: + self.db.connection.rollback() + raise RuntimeError("MTProto upload receipt could not be journaled") + self.db.commit() + + def unresolved_count(self) -> int: + """Durable intents whose remote acceptance is still unknown.""" + return int(self.db.connection.execute( + select(func.count()).select_from(mtproto_send_intents).where( + mtproto_send_intents.c.message_id.is_(None) + ) + ).scalar_one()) + + def clear_rejected_send(self, account: str, chat_id: int, object_id: str, + ordinal: int, checksum: str, size: int) -> None: + """Delete one intent ONLY after a definite, verified rejection. + + A flood-wait or a hard Telegram validation error proves the message + was not accepted. Unknown outcomes must never reach this method. + """ + if (not account or not chat_id or not object_id or len(object_id) > 36 + or ordinal < 0 or size < 1 + or not re.fullmatch(r"[0-9a-f]{64}", checksum)): + raise ValueError("Invalid MTProto rejection identity") + from sqlalchemy import delete + result = self.db.connection.execute(delete(mtproto_send_intents).where( + mtproto_send_intents.c.account == account, + mtproto_send_intents.c.chat_id == chat_id, + mtproto_send_intents.c.object_id == object_id, + mtproto_send_intents.c.ordinal == ordinal, + mtproto_send_intents.c.sha256 == checksum, + mtproto_send_intents.c.size == size, + mtproto_send_intents.c.message_id.is_(None), + )) + if result.rowcount != 1: + self.db.connection.rollback() + raise RuntimeError("MTProto rejection could not clear the exact intent") + self.db.commit() + + +class UnifiedUserTransport: + """All live I/O uses USER MTProto; old same-channel Bot messages remain readable.""" + + def __init__(self, runtime: UserSessionRuntime, journal: UserSendJournal, *, + rate=None, min_upload_interval: float = 1.1): + if not 0 <= min_upload_interval <= 60: + raise ValueError("Invalid MTProto minimum upload interval") + self.runtime = runtime + self.journal = journal + self.rate = rate if rate is not None else MtprotoRateGuard(journal.db) + self.min_upload_interval = min_upload_interval + self._next_upload = 0.0 + self._select_lock = asyncio.Lock() + + async def verify_target(self): + self.runtime.require() + + def parallel_degree(self) -> int: + """Concurrent send capacity: parallel strategy uses every set.""" + transports = getattr(self.runtime, "transports", None) + labels = list(transports()) if callable(transports) else None + strategy = getattr(self.runtime, "strategy", "round_robin") + if strategy == "parallel" and labels: + return len(labels) + return 1 + + async def _select(self, *, strategy: str, labels: list[str], + advance) -> str: + """Atomically pick the account label for this send. + + round_robin and parallel pre-rotate so concurrent callers always + receive distinct sets; single and failover stay on the current set + (failover advances only after a definite rejection). + """ + async with self._select_lock: + account = self.runtime.account + if (strategy in ("round_robin", "parallel") + and len(labels) > 1 and callable(advance)): + advance() + return account + + async def upload(self, path: Path, *, object_id: str, ordinal: int, + checksum: str): + from telethon.errors import FloodWaitError + + size = path.stat().st_size + chat_id = self.runtime.chat_id + # One or more authorized credential sets share the SAME channel. + # A definite flood-wait proves the send was REJECTED, so the failed + # set's intent is cleared and the next set retries with its own + # fresh intent. Ambiguous outcomes never trigger cross-set retries. + # Single-set runtimes keep the original intent across the bounded + # same-set retry (legacy behavior). + transports = getattr(self.runtime, "transports", None) + labels = list(transports()) if callable(transports) else None + strategy = getattr(self.runtime, "strategy", "round_robin") + advance = getattr(self.runtime, "advance", None) + if labels: + account = await self._select( + strategy=strategy, labels=labels, advance=advance) + else: + account = self.runtime.account + self.journal.start(account, chat_id, object_id, ordinal, + checksum, size) + total_attempts = max(len(labels) * 2, 2) if labels else 2 + for attempt in range(total_attempts): + if attempt and labels and len(labels) > 1 and callable(advance): + self.journal.clear_rejected_send( + account, chat_id, object_id, ordinal, checksum, size, + ) + advance() + account = self.runtime.account + self.journal.start(account, chat_id, object_id, ordinal, + checksum, size) + try: + try: + delegate = self.runtime.require(account) + except TypeError: + delegate = self.runtime.require() + # The SQL slot is shared by every worker using this + # account/channel, not just this transport instance. + shared_wait = self.rate.reserve_send_slot( + account, chat_id, self.min_upload_interval, + ) + pause = max(shared_wait, self._next_upload - time.monotonic(), 0.0) + if pause: + await asyncio.sleep(pause) + # A different worker may receive a flood-wait after we + # reserved this slot. Enforce the server cooldown even for + # transports with min_upload_interval=0 (no local pacing). + while True: + backoff = self.rate.shared_cooldown_wait(account, chat_id) + if not backoff: + break + await asyncio.sleep(min(backoff, 30.0)) + received_account, received_chat, message_id = await delegate.upload( + path, object_id=object_id, ordinal=ordinal, checksum=checksum, + ) + if (received_account, received_chat) != (account, chat_id): + raise RuntimeError("MTProto send returned an unexpected account or channel") + self.journal.acknowledged(account, chat_id, object_id, ordinal, message_id) + self._next_upload = time.monotonic() + return received_account, received_chat, message_id + except FloodWaitError as exc: + # Publish the cooldown for every worker before sleeping. + wait = int(getattr(exc, "seconds", 1) or 1) + self.rate.impose_shared_cooldown(account, chat_id, min(wait, 3600)) + if attempt + 1 < total_attempts: + continue + raise RuntimeError( + "MTProto flood-wait persists; reconcile protected intent before retry" + ) from None + except Exception: + raise RuntimeError( + "MTProto send outcome uncertain; reconcile protected intent before retry" + ) from None + raise RuntimeError( + "MTProto send outcome uncertain; reconcile protected intent before retry" + ) + + async def download(self, reference, destination): + transports = getattr(self.runtime, "transports", None) + if callable(transports) and reference.account in transports(): + client, channel = transports()[reference.account] + return await TelethonChunkTransport({ + reference.account: (client, channel), + }).download(reference, destination) + if reference.account == self.runtime.account: + return await self.runtime.require().download(reference, destination) + if reference.account == "api-bot" and reference.chat_id == self.runtime.chat_id: + # Only the ORIGINAL verified channel/message coordinates may be + # rebound to a user session; immutable checksums still apply. + rebound = replace(reference, account=self.runtime.account) + return await self.runtime.require().download(rebound, destination) + raise FileNotFoundError( + "Historical source inaccessible to authorized Telethon user; " + "do not remap remote media" + ) diff --git a/teledrive/uploader.py b/teledrive/uploader.py new file mode 100644 index 00000000..91484010 --- /dev/null +++ b/teledrive/uploader.py @@ -0,0 +1,290 @@ +#!/usr/bin/env python3 +import argparse +import asyncio +from contextlib import contextmanager +import logging +import getpass +import os +import random +import re +import secrets +import shutil +import stat +import string +import tempfile +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Callable, Iterable, Optional, Union + +from .db import Database, DBRow, connect_database, normalize_database_url, redact_database_url, sqlite_database_path + +import yaml +from telethon import TelegramClient +from telethon.errors import FloodWaitError, RPCError, SessionPasswordNeededError + + +# Telegram's free-account limit is 4,000 parts. Telethon uses 512 KiB parts, +# so keep split files below the resulting ~2 GiB boundary with some headroom. +TELEGRAM_FREE_FILE_LIMIT = 1_900_000_000 +SPLIT_BUFFER_SIZE = 1024 * 1024 +ACCOUNT_STRATEGIES = {"single", "round_robin", "failover", "parallel"} +RUN_HOLDER = "" +LEASE_NAME = "" +LEASE_TTL_SECONDS = 0 +CLAIM_STALE_SECONDS = int(os.environ.get("TELEDRIVE_CLAIM_STALE_SECONDS", "86400")) +SPLIT_MIN_FREE_BYTES = int(os.environ.get("TELEDRIVE_SPLIT_MIN_FREE_BYTES", str(512 * 1024 * 1024))) + + +@dataclass(frozen=True) +@dataclass +class AppConfig: + api_id: int + api_hash: str + phone: Optional[str] + target: Union[str, int] + session_path: str + strategy: str + web: dict + jobs: dict + storage: dict + state: dict + source_dir: str + allowed_extensions: list[str] + max_file_size_mb: int + sleep_min_seconds: int + sleep_max_seconds: int + max_files_per_run: int + max_files_per_day: int + retry_attempts: int + backoff_base_seconds: int + floodwait_buffer_seconds: int + caption_template: str + send_mode: str + db_path: str + log_path: str + log_level: str + + + + +def _safe_account_token(name: str) -> str: + token = re.sub(r"[^A-Za-z0-9_.-]+", "_", name.strip()).strip("._-") + return token or "account" + + +def load_config(path: str) -> AppConfig: + config_dir = Path(path).expanduser().resolve().parent + + def config_path(value: object) -> str: + candidate = Path(str(value)).expanduser() + return str(candidate if candidate.is_absolute() else config_dir / candidate) + + with open(path, "r", encoding="utf-8") as f: + raw = yaml.safe_load(f) or {} + + tg = raw.get("telegram") or {} + up = raw.get("upload") or {} + st = raw.get("state") or {} + lg = raw.get("logging") or {} + web_section = raw.get("web") or {} + jobs_section = raw.get("jobs") or {} + storage_section = raw.get("storage") or {} + + strategy = str(tg.get("strategy", "single")).strip().lower() + default_api_id = int(tg.get("api_id", 0) or 0) + default_api_hash = str(tg.get("api_hash", "")) + default_target: Union[str, int] = tg.get("target", "me") + + # Legacy direct-send scanner is gone; the unified service uses + # storage.apis for credentials. Tolerate a legacy telegram: section but + # do not parse accounts. + return AppConfig( + api_id=default_api_id, + api_hash=default_api_hash, + phone=None, + target=default_target, + session_path=config_path("data/telethon_user"), + strategy=strategy, + source_dir=config_path(up.get("source_dir", "/uploads")), + allowed_extensions=[str(x).lower() for x in up.get("allowed_extensions", [])], + max_file_size_mb=int(up.get("max_file_size_mb", 0)), + sleep_min_seconds=int(up.get("sleep_min_seconds", 60)), + sleep_max_seconds=int(up.get("sleep_max_seconds", 90)), + max_files_per_run=int(up.get("max_files_per_run", 12)), + max_files_per_day=int(up.get("max_files_per_day", 100)), + retry_attempts=int(up.get("retry_attempts", 5)), + backoff_base_seconds=int(up.get("backoff_base_seconds", 10)), + floodwait_buffer_seconds=int(up.get("floodwait_buffer_seconds", 5)), + caption_template=str(up.get("caption_template", "{name}")), + send_mode=str(up.get("send_mode", "document")).lower(), + db_path=normalize_database_url( + st.get("database_url") or st.get("db_path", "data/state.db"), + config_dir=config_dir, + ), + log_path=config_path(lg.get("log_path", "/data/uploader.log")), + log_level=str(lg.get("level", "INFO")), + web=web_section, + jobs=jobs_section, + storage=storage_section, + state=st, + ) + + +def ensure_parent(path: str) -> None: + Path(path).parent.mkdir(parents=True, exist_ok=True) + + +_LOG_SECRET_URL_RE = re.compile(r"([A-Za-z][A-Za-z0-9+.-]*://[^:/@\s]+:)[^@\s]+(@)") +_LOG_QUERY_SECRET_RE = re.compile( + r"(?i)([?&](?:password|passwd|pwd|token|access_token|secret|api_key|apikey)=)[^&#\s]+" +) + + +def _scrub_log_text(value: str) -> str: + value = _LOG_SECRET_URL_RE.sub(r"\1***\2", value) + return _LOG_QUERY_SECRET_RE.sub(r"\1***", value) + + +def _safe_error_text(value: object, limit: int = 1000) -> str: + cleaned = _scrub_log_text(str(value)).replace("\r", "\\r").replace("\n", "\\n") + return cleaned[:limit] + + +class _RedactingFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + return _scrub_log_text(super().format(record)) + + +def setup_logging(cfg: AppConfig) -> None: + ensure_parent(cfg.log_path) + level = getattr(logging, cfg.log_level.upper(), logging.INFO) + handlers = [ + logging.FileHandler(cfg.log_path, encoding="utf-8"), + logging.StreamHandler(), + ] + logging.basicConfig(level=level, handlers=handlers) + formatter = _RedactingFormatter("%(asctime)s [%(levelname)s] %(message)s") + for handler in handlers: + handler.setFormatter(formatter) + + +def connect_db(db_path: str) -> Database: + return connect_database(db_path) + + + +def _path_within(path: str, root: str) -> bool: + try: + return os.path.commonpath([os.path.realpath(path), os.path.realpath(root)]) == os.path.realpath(root) + except ValueError: + return False + + +def _validate_allowed_roots(path: str, env_name: str) -> None: + raw = os.environ.get(env_name, "").strip() + if not raw: + return + roots = [item.strip() for item in raw.split(os.pathsep) if item.strip()] + if not roots or not any(_path_within(path, root) for root in roots): + raise ValueError(f"{path} is outside allowed roots configured by {env_name}") + + +def iter_source_files(source_dir: str) -> Iterable[str]: + source_real = os.path.realpath(source_dir) + for root, dirs, files in os.walk(source_real, followlinks=False): + dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d))] + for name in files: + path = os.path.abspath(os.path.join(root, name)) + if os.path.islink(path): + logging.warning("Skipping symlink: %r", path) + continue + if not _path_within(path, source_real): + logging.warning("Skipping path outside source root: %r", path) + continue + try: + file_stat = os.lstat(path) + except OSError: + continue + if not stat.S_ISREG(file_stat.st_mode): + logging.warning("Skipping non-regular file: %r", path) + continue + yield path + + +def file_allowed(path: str, cfg: AppConfig) -> bool: + if cfg.allowed_extensions: + ext = os.path.splitext(path)[1].lower() + if ext not in cfg.allowed_extensions: + return False + if cfg.max_file_size_mb > 0: + max_bytes = cfg.max_file_size_mb * 1024 * 1024 + try: + if os.path.getsize(path) > max_bytes: + return False + except OSError: + return False + return True + + + + +def build_caption(path: str, template: str) -> str: + name = os.path.basename(path) + stem, ext = os.path.splitext(name) + allowed = {"name", "stem", "ext"} + formatter = string.Formatter() + for _, field_name, format_spec, conversion in formatter.parse(template): + if field_name is None: + continue + if field_name not in allowed or format_spec or conversion: + raise ValueError("caption_template may only use plain {name}, {stem}, and {ext} placeholders") + # Guard against attribute traversal via field_name like "name.__class__" + return template.format(name=name, stem=stem, ext=ext)[:1024] + + + +def validate_config(cfg: AppConfig) -> None: + if cfg.strategy not in ACCOUNT_STRATEGIES: + raise ValueError("telegram.strategy must be single, round_robin, failover, or parallel") + if not cfg.accounts or not any(a.enabled for a in cfg.accounts): + raise ValueError("At least one Telegram account must be enabled") + names, sessions = set(), set() + for account in cfg.accounts: + if not re.fullmatch(r"[A-Za-z0-9_.-]{1,64}", account.name): + raise ValueError("Telegram account name must be 1-64 characters using only letters, digits, '.', '_' or '-'") + if account.name in names: raise ValueError(f"Duplicate Telegram account name: {account.name}") + names.add(account.name) + session = os.path.abspath(account.session_path) + if session in sessions: raise ValueError(f"Telegram accounts must use different session_path values: {account.session_path}") + sessions.add(session) + if account.enabled: + if account.api_id <= 0: raise ValueError(f"Telegram account '{account.name}' has an invalid api_id") + if not account.api_hash.strip(): raise ValueError(f"Telegram account '{account.name}' api_hash must not be empty") + if account.phone and not re.fullmatch(r"\+?[0-9]{5,20}", str(account.phone)): + raise ValueError(f"Telegram account '{account.name}' has an invalid phone number") + target_text = str(account.target) + if not target_text.strip(): raise ValueError(f"Telegram account '{account.name}' target must not be empty") + if len(target_text) > 256 or any(ord(ch) < 32 for ch in target_text): + raise ValueError(f"Telegram account '{account.name}' target contains invalid characters") + if cfg.sleep_min_seconds <= 0 or cfg.sleep_max_seconds <= 0: raise ValueError("Sleep values must be > 0") + if cfg.sleep_min_seconds > cfg.sleep_max_seconds: raise ValueError("sleep_min_seconds cannot be greater than sleep_max_seconds") + if cfg.max_files_per_run <= 0 or cfg.max_files_per_day <= 0: raise ValueError("Upload limits must be > 0") + if cfg.retry_attempts <= 0: raise ValueError("retry_attempts must be > 0") + if cfg.backoff_base_seconds < 0 or cfg.floodwait_buffer_seconds < 0: raise ValueError("Backoff and floodwait buffer values cannot be negative") + if cfg.max_file_size_mb < 0: raise ValueError("max_file_size_mb cannot be negative") + if cfg.send_mode not in {"document", "media", "auto"}: raise ValueError("send_mode must be document, media, or auto") + try: build_caption("example.txt", cfg.caption_template) + except (KeyError, ValueError, IndexError) as exc: raise ValueError("caption_template may only use {name}, {stem}, and {ext}") from exc + validate_storage_paths(cfg) + + +def validate_storage_paths(cfg: AppConfig) -> None: + if not os.path.isdir(cfg.source_dir): + raise ValueError(f"source_dir not found: {cfg.source_dir}") + _validate_allowed_roots(cfg.source_dir, "TELEDRIVE_ALLOWED_SOURCE_ROOTS") + _validate_allowed_roots(cfg.log_path, "TELEDRIVE_ALLOWED_LOG_ROOTS") + sqlite_path = sqlite_database_path(cfg.db_path) + if sqlite_path is not None: + _validate_allowed_roots(sqlite_path, "TELEDRIVE_ALLOWED_SQLITE_ROOTS") + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/e2e/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/e2e/test_database_live.py b/tests/e2e/test_database_live.py new file mode 100644 index 00000000..245e39b4 --- /dev/null +++ b/tests/e2e/test_database_live.py @@ -0,0 +1,1036 @@ +"""Live database integration tests; skipped unless CI supplies an external DB URL.""" +from __future__ import annotations + +import os +import time + +import pytest + +from teledrive.db import connect_database + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_external_db_schema_queue_lease_and_account_history(env_name): + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + first = connect_database(url) + second = connect_database(url) + now = int(time.time()) + token = f"ci-{env_name}-{now}" + path = f"/uploads/{token}.bin" + try: + first.insert_pending(path, 123, 1.0, now) + first.commit() + + assert first.claim_file(path, token + "-a", now, 3600) + assert not second.claim_file(path, token + "-b", now, 3600) + + first.mark_uploaded(path, 99, "primary", now) + first.set_account_state( + "primary", "ready", last_error=None, + cooldown_until=None, last_success_ts=now, now=now, + ) + first.commit() + + history = first.list_files("history", 200) + uploaded = next(row for row in history if row["path"] == path) + assert uploaded["tg_account"] == "primary" + assert uploaded["tg_message_id"] == 99 + assert first.uploaded_counts_by_account()["primary"] >= 1 + assert first.all_account_states()["primary"]["state"] == "ready" + + assert first.acquire_lease(token, "holder-a", now, 30) + assert not second.acquire_lease(token, "holder-b", now, 30) + assert first.renew_lease(token, "holder-a", now, 30) + first.release_lease(token, "holder-a") + assert second.acquire_lease(token, "holder-b", now, 30) + second.release_lease(token, "holder-b") + finally: + first.close() + second.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_storage_manifest_round_trip_on_real_database(env_name): + """Verify the new chunk journal and ready visibility against a real SQL server.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from hashlib import sha256 + from teledrive.storage.manifest import ManifestStore, ChunkReference + + connection = connect_database(url) + manifests = ManifestStore(connection) + now = int(time.time()) + try: + ident = manifests.begin("ci-bucket", f"test/{env_name}/{now}", 8, now) + with pytest.raises(FileNotFoundError): + manifests.get(ident) + payload = b"abcdefgh" + manifests.append(ident, ChunkReference( + ordinal=0, offset=0, size=8, sha256=sha256(payload).hexdigest(), + account="ci", chat_id=123456, message_id=int(now * 100) % 2000000000, + ), now) + ready = manifests.finish(ident, sha256(payload).hexdigest(), now) + assert ready.size == 8 + assert ready.sha256 == sha256(payload).hexdigest() + assert len(manifests.parts(ident)) == 1 + finally: + connection.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_failed_chunk_cleanup_transaction_is_durable(env_name): + """Real DB: recovery transitions failed -> cleaned and cannot alter READY data.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + import asyncio + from hashlib import sha256 + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore, ChunkReference + from teledrive.storage.recovery import Discovery, RecoveryManager, UnsafeCleanup + + class MissingRemote: + async def discover(self, identifier, *, scan_limit): + return Discovery((), True, 0) + + async def probe(self, part, *, object_id): + return "missing" + + async def delete_verified(self, part, *, object_id): + raise AssertionError("Deleted missing remote media") + + older = int(time.time()) - 7200 + connection = connect_database(url) + try: + manifests = ManifestStore(connection) + unique = uuid4().hex + object_id = manifests.begin("ci", f"cleanup/{unique}", 4, older) + part = ChunkReference( + ordinal=0, offset=0, size=4, sha256=sha256(b"data").hexdigest(), + account="ci", chat_id=123456, message_id=(int(unique[:8], 16) % 2000000000) + 1, + ) + manifests.append(object_id, part, older) + manifests.fail(object_id, "simulated timeout", older) + service = RecoveryManager(manifests, MissingRemote(), min_age_seconds=3600) + preview = asyncio.run(service.cleanup(object_id)) + assert preview.missing == 1 and preview.deleted == 0 and preview.dry_run + result = asyncio.run(service.cleanup(object_id, dry_run=False)) + assert result.cleaned and result.deleted == 0 + assert object_id not in [row[0] for row in manifests.interrupted()] + with pytest.raises(UnsafeCleanup): + asyncio.run(service.cleanup(object_id, dry_run=False)) + finally: + connection.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_writer_fence_blocks_stale_commit_on_separate_connection(env_name): + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from hashlib import sha256 + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore, ManifestError + + a = connect_database(url) + b = connect_database(url) + try: + now = int(time.time()) - 3600 + first = ManifestStore(a) + second = ManifestStore(b) + ident = first.begin("ci", f"live-writer/{uuid4().hex}", 0, now) + first.start_writer(ident, "worker-before-crash", now, ttl=120) + assert not second.heartbeat_writer(ident, "wrong-owner", now + 1) + abandoned = second.abandon_stale_pending(now=now + 3601, min_age_seconds=60) + assert ident in abandoned + assert not first.heartbeat_writer(ident, "worker-before-crash", now + 3602) + with pytest.raises(ManifestError): + first.finish(ident, sha256(b"").hexdigest(), now + 3602, + writer_token="worker-before-crash") + assert (ident, "failed") in first.interrupted() + finally: + a.close() + b.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_bucket_heads_visible_across_live_database_connections(env_name): + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from hashlib import sha256 + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore + from teledrive.storage.catalog import BucketCatalog + + a = connect_database(url) + b = connect_database(url) + try: + first = ManifestStore(a) + second = ManifestStore(b) + one = BucketCatalog(first) + two = BucketCatalog(second) + bucket = "ci-" + uuid4().hex[:20] + one.create_bucket(bucket, int(time.time())) + key = "backup/ü/snapshot.bin" + ident = first.begin(bucket, key, 0, int(time.time())) + first.finish(ident, sha256(b"").hexdigest(), int(time.time())) + first_head = one.publish(ident, now=int(time.time()), expected_generation=0) + assert first_head.generation == 1 + assert two.get(bucket, key).item.id == ident + + next_id = second.begin(bucket, key, 0, int(time.time())) + second.finish(next_id, sha256(b"").hexdigest(), int(time.time())) + assert two.publish(next_id, now=int(time.time()), expected_generation=1).generation == 2 + with pytest.raises(ValueError): + one.publish(ident, now=int(time.time()), expected_generation=1) + assert one.get(bucket, key).item.id == next_id + finally: + a.close() + b.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_s3_multipart_atomic_chunk_reparent_on_live_sql(env_name, tmp_path): + """Move existing Telegram message references without a duplicate upload. + + This exercises the actual PostgreSQL/MySQL unique-message constraint, + multi-row object/chunk updates and committed head visibility. + """ + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + import asyncio + import hashlib + from uuid import uuid4 + + from teledrive.s3.multipart import MultipartStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ChunkReference, ManifestStore + from teledrive.storage.engine import TelegramObjectStorage + + data1 = b"A" * (5 * 1024 * 1024) + data2 = b"tail" + payloads = {} + class FakeTelegram: + async def upload(self, *args, **kwargs): + raise AssertionError("Multipart Complete must not upload Telegram data") + + async def download(self, reference, destination): + destination.write_bytes(payloads[reference.message_id]) + + db = connect_database(url) + second = connect_database(url) + try: + manifests = ManifestStore(db) + catalog = BucketCatalog(manifests) + mpu = MultipartStore(catalog) + name = "ci-" + uuid4().hex[:20] + key = "multipart/atomic.bin" + now = int(time.time()) + catalog.create_bucket(name, now) + upload_id = mpu.initiate(name, key, "ci-access-key") + selected = [] + for idx, data in enumerate((data1, data2), start=1): + manifest_id = manifests.begin(name, key, len(data), now) + message_id = int(uuid4().hex[:8], 16) % 2_000_000_000 + 1 + payloads[message_id] = data + manifests.append(manifest_id, ChunkReference( + ordinal=0, offset=0, size=len(data), + sha256=hashlib.sha256(data).hexdigest(), + account="ci", chat_id=-10012345, message_id=message_id, + ), now) + manifests.finish(manifest_id, hashlib.sha256(data).hexdigest(), now) + etag = hashlib.md5(data, usedforsecurity=False).hexdigest() + mpu.record_part(upload_id, name, key, "ci-access-key", number=idx, + object_id=manifest_id, etag=etag) + selected.append((idx, etag)) + engine = TelegramObjectStorage(manifests, FakeTelegram(), + chunk_size=1024 * 1024, scratch_directory=tmp_path) + done = asyncio.run(mpu.complete(upload_id, name, key, "ci-access-key", + selected, engine)) + assert done.size == len(data1) + len(data2) + # Verify durable committed read through a different DB connection. + other_catalog = BucketCatalog(ManifestStore(second)) + ready = other_catalog.get(name, key).item + assert ready.id == done.object_id + assert ready.sha256 == hashlib.sha256(data1 + data2).hexdigest() + assert len(ManifestStore(second).parts(ready.id)) == 2 + async def read(): + other_engine = TelegramObjectStorage( + ManifestStore(second), FakeTelegram(), scratch_directory=tmp_path + ) + return b"".join([piece async for piece in other_engine.iter_bytes(ready.id)]) + assert asyncio.run(read()) == data1 + data2 + finally: + db.close() + second.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_multipart_recovery_fences_prior_worker_across_live_sql_connections(env_name): + """A worker killed during Telegram verification may not commit after reclaim. + + This uses separate DB connections on real Postgres/MySQL to prove + the lease token is invalidated atomically and a new writer can claim. + """ + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.s3.multipart import MultipartStore, completion_leases + from sqlalchemy import select + + a, b = connect_database(url), connect_database(url) + try: + left = MultipartStore(BucketCatalog(ManifestStore(a))) + right = MultipartStore(BucketCatalog(ManifestStore(b))) + now = int(time.time()) + bucket = "ci-" + uuid4().hex[:20] + left.catalog.create_bucket(bucket, now) + uid = left.initiate(bucket, "crashed/multipart.bin", "ci-owner") + old_token = left._claim_completion(uid, now - 7200, 120) + assert right.recover_stale_completion(uid, dry_run=True, now=now) + assert right.recover_stale_completion(uid, dry_run=False, now=now) + assert not left._renew_completion(uid, old_token, now, 120) + assert not left._reset_owned_completion(uid, old_token) + new_token = right._claim_completion(uid, now, 120) + assert new_token != old_token + assert not left._reset_owned_completion(uid, old_token) + assert right._renew_completion(uid, new_token, now, 120) + assert right._reset_owned_completion(uid, new_token) + assert right.require(uid, bucket, "crashed/multipart.bin", "ci-owner")["state"] == "active" + assert not b.connection.execute(select(completion_leases.c.upload_id) + .where(completion_leases.c.upload_id == uid)).first() + finally: + a.close() + b.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_copy_object_alias_preserves_telegram_uniqueness_live_sql(env_name, tmp_path): + """Real RDBMS + two connections: immutable source, no duplicate Telegram refs.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + import asyncio + import hashlib + from uuid import uuid4 + from sqlalchemy import func, select + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.copy import copy_object + from teledrive.storage.engine import TelegramObjectStorage + from teledrive.storage.manifest import ManifestStore, ChunkReference, chunks, object_aliases + + now = int(time.time()) + payload = b"PostgreSQL-MySQL-immutable-CopyObject" * 37 + bucket = "ci-" + uuid4().hex[:20] + key = "source.dat" + chat_id = -1001234567890 + message_id = int(uuid4().hex[:8], 16) % 1_900_000_000 + 1 + + first = connect_database(url) + second = connect_database(url) + try: + manifests = ManifestStore(first) + catalog = BucketCatalog(manifests) + catalog.create_bucket(bucket, now) + obj = manifests.begin(bucket, key, len(payload), now) + manifests.append(obj, ChunkReference( + ordinal=0, offset=0, size=len(payload), + sha256=hashlib.sha256(payload).hexdigest(), account="ci-copy", + chat_id=chat_id, message_id=message_id, + ), now) + manifests.finish(obj, hashlib.sha256(payload).hexdigest(), now) + catalog.publish(obj, now=now) + + copied = copy_object(catalog, bucket, key, bucket, "copy.dat", now=now) + assert copied.object_id != obj + assert copied.source_object_id == obj + assert copied.etag == hashlib.sha256(payload).hexdigest() + catalog.remove_current(bucket, key) + + from_second = BucketCatalog(ManifestStore(second)) + clone = from_second.get(bucket, "copy.dat").item + assert clone.id == copied.object_id + assert clone.size == len(payload) + assert first.connection.execute( + select(func.count()).select_from(chunks).where(chunks.c.object_id == obj) + ).scalar_one() == 1 + assert first.connection.execute( + select(func.count()).select_from(chunks).where( + chunks.c.object_id == copied.object_id + ) + ).scalar_one() == 0 + assert second.connection.execute(select(object_aliases.c.source_object_id) + .where(object_aliases.c.object_id == copied.object_id)).scalar_one() == obj + + class FakeTelegram: + async def download(self, ref, dest): + assert ref.account == "ci-copy" and ref.message_id == message_id + dest.write_bytes(payload) + engine = TelegramObjectStorage(ManifestStore(second), FakeTelegram(), + scratch_directory=tmp_path) + async def read(): + return b"".join([part async for part in engine.iter_bytes(clone.id)]) + assert asyncio.run(read()) == payload + finally: + first.close() + second.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_object_tags_survive_database_restart_and_copy_on_live_sql(env_name): + """Tags persist without Telegram bytes; copies snapshot rather than share tags.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + import hashlib + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + from teledrive.storage.copy import copy_object + from teledrive.s3.tagging import ObjectTagStore + + bucket = "ci-" + uuid4().hex[:20] + now = int(time.time()) + conn = connect_database(url) + try: + manifests = ManifestStore(conn) + catalog = BucketCatalog(manifests) + catalog.create_bucket(bucket, now) + original_id = manifests.begin(bucket, "source", 0, now) + manifests.finish(original_id, hashlib.sha256(b"").hexdigest(), now) + catalog.publish(original_id, now=now) + tagging = ObjectTagStore(catalog) + assert tagging.replace_current(bucket, "source", [ + ("project", "TeleDrive"), ("language", "Türkçe") + ]) == original_id + clone = copy_object(catalog, bucket, "source", bucket, "copy") + assert clone.source_object_id == original_id + finally: + conn.close() + + # Cross-process durability is modeled by a new SQL connection. + reopened = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + tagging = ObjectTagStore(catalog) + assert tagging.get(catalog.get(bucket, "copy").item.id) == [ + ("language", "Türkçe"), ("project", "TeleDrive") + ] + tagging.replace_current(bucket, "source", [("project", "changed")]) + assert tagging.get(catalog.get(bucket, "copy").item.id) == [ + ("language", "Türkçe"), ("project", "TeleDrive") + ] + assert tagging.get(catalog.get(bucket, "source").item.id) == [ + ("project", "changed") + ] + finally: + reopened.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_s3_bucket_tags_persist_and_delete_cleanly_on_live_sql(env_name): + """Real SQL FK/table lifecycle, independent versioned metadata and deletion.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + from teledrive.s3.tagging import BucketTagStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + + bucket = "ci-" + uuid4().hex[:20] + db = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket(bucket, int(time.time())) + tags = BucketTagStore(catalog) + tags.replace(bucket, [("team", "TeleDrive"), ("language", "Türkçe")]) + assert tags.get(bucket) == [("language", "Türkçe"), ("team", "TeleDrive")] + finally: + db.close() + + reopened = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + tags = BucketTagStore(catalog) + assert tags.get(bucket) == [("language", "Türkçe"), ("team", "TeleDrive")] + tags.replace(bucket, [("team", "changed")]) + assert tags.get(bucket) == [("team", "changed")] + catalog.delete_bucket(bucket) + assert bucket not in catalog.list_buckets() + finally: + reopened.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_s3_http_metadata_is_immutable_and_transactional_on_live_database(env_name): + """Cross-session: PUT/COPY metadata persists with version snapshot, not payload bytes.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + import hashlib + from uuid import uuid4 + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + from teledrive.storage.copy import copy_object + from teledrive.s3.object_metadata import ObjectMetadataStore + + bucket = "ci-" + uuid4().hex[:20] + original = { + "content-type": "application/vnd.teledrive.data", + "x-amz-meta-publisher": "bot-storage", + "cache-control": "private, max-age=70", + } + now = int(time.time()) + db = connect_database(url) + try: + manifests = ManifestStore(db) + catalog = BucketCatalog(manifests) + catalog.create_bucket(bucket, now) + oid = manifests.begin(bucket, "original", 0, now) + manifests.finish(oid, hashlib.sha256(b"").hexdigest(), now) + ObjectMetadataStore(catalog).attach_uncommitted(oid, original) + catalog.publish(oid, now=now) + cloned = copy_object(catalog, bucket, "original", bucket, "copy") + assert cloned.source_object_id == oid + finally: + db.close() + + reopened = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + props = ObjectMetadataStore(catalog) + assert props.get(catalog.get(bucket, "copy").item.id) == original + updated = copy_object( + catalog, bucket, "original", bucket, "replaced", + replacement_metadata={"content-type": "text/plain", "x-amz-meta-publisher": "sdk"}, + ) + assert props.get(updated.object_id)["content-type"] == "text/plain" + assert props.get(oid) == original + assert props.get(cloned.object_id) == original + finally: + reopened.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_bucket_cors_policy_durable_and_delete_safe_on_real_database(env_name): + """Cross-session CORS policy changes must not leave dangling bucket rows.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + from teledrive.s3.cors import BucketCORSStore, parse_cors_xml + + bucket = "cors-ci-" + uuid4().hex[:18] + xml = ( + b"" + b"https://*.example.comPUT" + b"x-amz-*" + b"300" + ) + rules = parse_cors_xml(xml) + first = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(first)) + catalog.create_bucket(bucket, int(time.time())) + BucketCORSStore(catalog).replace(bucket, rules) + finally: + first.close() + reopened = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + cors = BucketCORSStore(catalog) + assert cors.get(bucket) == rules + catalog.delete_bucket(bucket) + assert bucket not in catalog.list_buckets() + with pytest.raises(FileNotFoundError): + cors.get(bucket) + finally: + reopened.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_conditional_s3_head_cas_blocks_cross_connection_aba(env_name): + """Real DB conflict: old generation must not authorize a replaced object. + + Includes DELETE/RECREATE ABA (both generations equal 1), where only + comparing the generation would incorrectly allow stale writes/deletes. + """ + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from hashlib import sha256 + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore + from teledrive.storage.catalog import BucketCatalog + + a, b = connect_database(url), connect_database(url) + try: + one, two = BucketCatalog(ManifestStore(a)), BucketCatalog(ManifestStore(b)) + name = "ci-" + uuid4().hex[:20] + object_key = "conditional/aba.bin" + now = int(time.time()) + one.create_bucket(name, now) + + def new_empty(catalog): + ident = catalog.manifests.begin(name, object_key, 0, now) + catalog.manifests.finish(ident, sha256(b"").hexdigest(), now) + return ident + + first = new_empty(one) + one.publish(first, now=now) + snapshot = one.get(name, object_key) + a.commit() + assert snapshot.generation == 1 + assert two.remove_current(name, object_key) + second = new_empty(two) + two.publish(second, now=now) + assert two.get(name, object_key).generation == 1 + candidate = new_empty(one) + + with pytest.raises(ValueError, match="object ID precondition"): + one.publish(candidate, now=now, + expected_generation=snapshot.generation, + expected_object_id=snapshot.item.id) + a.connection.rollback() + with pytest.raises(ValueError, match="object ID precondition"): + one.remove_current(name, object_key, + expected_generation=snapshot.generation, + expected_object_id=snapshot.item.id) + a.connection.rollback() + assert one.get(name, object_key).item.id == second + finally: + a.close() + b.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_operator_stale_pending_preview_and_per_object_fence_live_sql(env_name): + """P1 dry-run and per-object execute remain fail-closed across DB workers.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + from teledrive.storage.manifest import ManifestStore + + a, b = connect_database(url), connect_database(url) + try: + first, second = ManifestStore(a), ManifestStore(b) + now = int(time.time()) + old = now - 7200 + stale = first.begin("ci", f"stale/{uuid4().hex}", 0, old) + first.start_writer(stale, "old-worker", old, ttl=120) + live = first.begin("ci", f"live/{uuid4().hex}", 0, now) + first.start_writer(live, "live-worker", now, ttl=120) + no_token = first.begin("ci", f"unfenced/{uuid4().hex}", 0, old) + + preview = second.stale_pending(now=now, min_age_seconds=3600) + assert [record for record in preview if record["id"] == stale] + assert not [record for record in preview if record["id"] in (live, no_token)] + assert not any("token" in row for row in preview) + # Flush the preview's read transaction before the separate worker + # writes, required on PostgreSQL and MySQL to avoid stale snapshots. + b.commit() + assert second.abandon_stale_pending( + now=now, min_age_seconds=3600, object_id=live, + ) == [] + assert second.abandon_stale_pending( + now=now, min_age_seconds=3600, object_id=no_token, + ) == [] + assert second.abandon_stale_pending( + now=now, min_age_seconds=3600, object_id=stale, + ) == [stale] + assert not first.heartbeat_writer(stale, "old-worker", now + 1) + assert first.heartbeat_writer(live, "live-worker", now + 1) + statuses = dict(first.interrupted()) + assert statuses[stale] == "failed" + assert statuses[live] == "pending" + assert statuses[no_token] == "pending" + finally: + a.close() + b.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_concurrent_durable_job_claim_single_winner_on_live_sql(env_name, tmp_path): + """Four independent SQL sessions race for one job; only one may own it.""" + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + from uuid import uuid4 + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.jobs import StorageUploadJobs + from teledrive.storage.manifest import ManifestStore + + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + filename = tmp_path / "only-once.bin" + filename.write_bytes(b"never-twice") + connection = connect_database(url) + now = int(time.time()) + try: + bucket = "ci-" + uuid4().hex[:20] + queue = StorageUploadJobs(BucketCatalog(ManifestStore(connection))) + queue.catalog.create_bucket(bucket, now) + job_id = queue.enqueue(bucket, "only-once.bin", filename, + allowed_root=tmp_path, now=now) + gate = Barrier(4) + + def race(): + db = connect_database(url) + try: + local = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + db.commit() + gate.wait(timeout=20) + return local.claim_next(now=now, ttl=120) + finally: + db.close() + + with ThreadPoolExecutor(max_workers=4) as pool: + claimed = list(pool.map(lambda _: race(), range(4))) + winners = [job for job in claimed if job is not None] + assert len(winners) == 1, claimed + winner = winners[0] + assert winner["id"] == job_id + assert winner["state"] == "running" and winner["attempts"] == 1 + # Only the actual claimant may renew or stage the durable job. + wrong = connect_database(url) + try: + competitor = StorageUploadJobs(BucketCatalog(ManifestStore(wrong))) + assert not competitor.renew(job_id, "forged-token", now=now) + assert competitor.claim_next(now=now + 1, ttl=120) is None + finally: + wrong.close() + assert queue.renew(job_id, winner["lease_token"], now=now) + finally: + connection.close() + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_legacy_scan_id_is_idempotent_under_live_sql_races(env_name, tmp_path): + """Two importers concurrently enqueue the identical file revision once.""" + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + from uuid import uuid4 + from sqlalchemy import select, func + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.jobs import StorageUploadJobs, jobs + from teledrive.storage.manifest import ManifestStore + + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + path = tmp_path / "same-revision.txt" + path.write_bytes(b"import-this-once") + bucket = "ci-" + uuid4().hex[:20] + initial = connect_database(url) + try: + StorageUploadJobs(BucketCatalog(ManifestStore(initial))).catalog.create_bucket( + bucket, int(time.time()) + ) + finally: + initial.close() + gate = Barrier(2) + + def insert_once(): + db = connect_database(url) + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + db.commit() + gate.wait(timeout=20) + return manager.enqueue(bucket, "same-revision.txt", path, + allowed_root=tmp_path, deduplicate=True) + finally: + db.close() + + with ThreadPoolExecutor(max_workers=2) as pool: + identifiers = list(pool.map(lambda _: insert_once(), range(2))) + assert identifiers[0] == identifiers[1] + check = connect_database(url) + try: + matches = check.connection.execute( + select(func.count()).select_from(jobs).where(jobs.c.id == identifiers[0]) + ).scalar_one() + assert matches == 1 + record = StorageUploadJobs(BucketCatalog(ManifestStore(check))).list_jobs( + limit=1000 + ) + selected = [row for row in record if row["id"] == identifiers[0]] + assert len(selected) == 1 + assert selected[0]["attempts"] == 0 + assert selected[0]["state"] == "queued" + finally: + check.close() + + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_real_sql_revoked_version_tagging_is_fenced_on_new_connection(env_name): + """Two real DB sessions: a logical S3 delete revokes direct SQL tag writes.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + + import hashlib + from uuid import uuid4 + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + from teledrive.s3.tagging import ObjectTagStore, TaggingError + + db1 = connect_database(url) + db2 = connect_database(url) + try: + one = BucketCatalog(ManifestStore(db1)) + two = BucketCatalog(ManifestStore(db2)) + bucket = "ci-" + uuid4().hex[:19] + key = "history/İstanbul.bin" + one.create_bucket(bucket, int(time.time())) + version = one.manifests.begin(bucket, key, 0, int(time.time())) + one.manifests.finish(version, hashlib.sha256(b"").hexdigest(), int(time.time())) + one.publish(version, now=int(time.time())) + ObjectTagStore(two).replace_published(bucket, key, version, [("a", "before")]) + # Close the second connection's read transaction before delete so + # MySQL repeatable-read never sees an intentionally stale snapshot. + db2.commit() + assert one.delete_version(bucket, key, version) + with pytest.raises(TaggingError) as denied: + ObjectTagStore(two).replace_published( + bucket, key, version, [("a", "after")], + ) + assert denied.value.code == "NoSuchVersion" + assert ObjectTagStore(two).get(version) == [("a", "before")] + assert not one.is_visible_version(bucket, key, version) + assert not two.is_visible_version(bucket, key, version) + finally: + db1.close() + db2.close() + + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_four_workers_compete_to_revoke_one_published_version(env_name): + """Real PG/MySQL row lock: one logical DELETE, no double-revoke/500 error.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from concurrent.futures import ThreadPoolExecutor + from threading import Barrier + from uuid import uuid4 + from hashlib import sha256 + from sqlalchemy import select, func + from teledrive.storage.catalog import BucketCatalog, removed_versions + from teledrive.storage.manifest import ManifestStore, objects + from teledrive.s3.tagging import ObjectTagStore, TaggingError + + bucket = "ci-" + uuid4().hex[:20] + key = "concurrent/revoked-version.bin" + db = connect_database(url) + try: + cat = BucketCatalog(ManifestStore(db)) + cat.create_bucket(bucket, int(time.time())) + ver = cat.manifests.begin(bucket, key, 0, int(time.time())) + cat.manifests.finish(ver, sha256(b"").hexdigest(), int(time.time())) + cat.publish(ver, now=int(time.time())) + finally: + db.close() + + gate = Barrier(4) + + def compete(_): + conn = connect_database(url) + try: + store = BucketCatalog(ManifestStore(conn)) + conn.commit() + gate.wait(timeout=20) + return store.delete_version(bucket, key, ver) + finally: + conn.close() + + with ThreadPoolExecutor(max_workers=4) as pool: + results = list(pool.map(compete, range(4))) + assert sorted(results) == [False, False, False, True] + + verification = connect_database(url) + try: + cat = BucketCatalog(ManifestStore(verification)) + assert not cat.is_visible_version(bucket, key, ver) + assert cat.list_versions(bucket, key) == [] + with pytest.raises(FileNotFoundError): + cat.get(bucket, key) + rows = verification.connection.execute(select(func.count()).select_from( + removed_versions + ).where(removed_versions.c.object_id == ver)).scalar_one() + assert rows == 1 + # Physical Telegram reference and immutable manifest are still + # retained for audited recovery; logical delete never performs GC. + status = verification.connection.execute(select(objects.c.status).where( + objects.c.id == ver + )).scalar_one() + assert status == "ready" + with pytest.raises(TaggingError) as denied: + ObjectTagStore(cat).replace_published(bucket, key, ver, [("bad", "yes")]) + assert denied.value.code == "NoSuchVersion" + finally: + verification.close() + + + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_version_revocation_bypasses_stale_repeatable_read_snapshots(env_name): + """Live SQL credentials/read transactions must not resurrect revoked IDs.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + import hashlib + from uuid import uuid4 + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + from teledrive.s3.tagging import ObjectTagStore, TaggingError + + bucket = "ci-" + uuid4().hex[:19] + key = "snapshots/old-version.bin" + seed = connect_database(url) + first_reader = connect_database(url) + second_reader = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(seed)) + catalog.create_bucket(bucket, int(time.time())) + identifier = catalog.manifests.begin(bucket, key, 0, int(time.time())) + catalog.manifests.finish(identifier, hashlib.sha256(b"").hexdigest(), + int(time.time())) + catalog.publish(identifier, now=int(time.time())) + reader1 = BucketCatalog(ManifestStore(first_reader)) + reader2 = BucketCatalog(ManifestStore(second_reader)) + # Both independent sessions establish a snapshot *before* revocation. + assert reader1.is_visible_version(bucket, key, identifier) + assert reader2.is_visible_version(bucket, key, identifier) + assert catalog.delete_version(bucket, key, identifier) + # Locking reads override old repeatable-read visibility; duplicate + # DELETE resolves to False rather than raising an IntegrityError. + assert reader1.delete_version(bucket, key, identifier) is False + tagger = ObjectTagStore(reader2) + with pytest.raises(TaggingError) as refused: + tagger.replace_published(bucket, key, identifier, [("backdoor", "no")]) + assert refused.value.code == "NoSuchVersion" + first_reader.commit() + second_reader.commit() + assert not reader1.is_visible_version(bucket, key, identifier) + assert not reader2.is_visible_version(bucket, key, identifier) + finally: + seed.close() + first_reader.close() + second_reader.close() + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_live_sql_serializes_six_independent_user_send_workers_and_flood(env_name): + """Actual PostgreSQL/MySQL transactions serialize competing channel sends. + + Six independent SQL connections start simultaneously with an identical + clock tick. A definite flood-wait then adds a common cooldown; a + different USER account/channel is unaffected. No physical Telegram load. + """ + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from concurrent.futures import ThreadPoolExecutor + from uuid import uuid4 + from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + + chat = -100_000_000_000 - int(uuid4().hex[:8], 16) + account = "rate-ci-" + uuid4().hex[:16] + t = 9_000_000_000 + prepared = connect_database(url) + try: + idx = MtprotoRateGuard(prepared) + assert idx.reserve_send_slot(account, chat, 0.9, now_ms=t) == 0 + finally: + prepared.close() + + def one_worker(_): + conn = connect_database(url) + try: + return MtprotoRateGuard(conn).reserve_send_slot( + account, chat, 0.9, now_ms=t, + ) + finally: + conn.close() + + with ThreadPoolExecutor(max_workers=6) as group: + waits = sorted(group.map(one_worker, range(6))) + assert waits == [0.9, 1.8, 2.7, 3.6, 4.5, 5.4] + + first, second = connect_database(url), connect_database(url) + try: + original_count = MtprotoRateGuard(second).total_explicit_flood_observations() + MtprotoRateGuard(first).impose_shared_cooldown( + account, chat, 30, now_ms=t, + ) + assert MtprotoRateGuard(second).total_explicit_flood_observations() == original_count + 1 + assert MtprotoRateGuard(second).reserve_send_slot( + account, chat, 0.9, now_ms=t, + ) == 30 + assert MtprotoRateGuard(first).reserve_send_slot( + account + "-other", chat, 0.9, now_ms=t, + ) == 0 + finally: + first.close() + second.close() + +@pytest.mark.parametrize("env_name", ["DB_TEST_POSTGRES_URL", "DB_TEST_MYSQL_URL"]) +def test_live_sql_flood_observed_after_another_process_reserves_slot(env_name): + """Real database isolation must not hide newly committed flood backoff.""" + url = os.getenv(env_name) + if not url: + pytest.skip(f"{env_name} not configured") + from uuid import uuid4 + from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + + suffix = uuid4().hex[:16] + account = "late-flood-" + suffix + chat_id = -200_000_000 - int(uuid4().hex[:6], 16) + before = connect_database(url) + after = connect_database(url) + t = 5_000_000 + try: + first = MtprotoRateGuard(before) + second = MtprotoRateGuard(after) + assert first.reserve_send_slot( + account, chat_id, 1.5, now_ms=t, + ) == 0 + assert second.reserve_send_slot( + account, chat_id, 1.5, now_ms=t, + ) == 1.5 + assert second.shared_cooldown_wait(account, chat_id, now_ms=t) == 0 + initial_flood = second.total_explicit_flood_observations() + first.impose_shared_cooldown( + account, chat_id, 17, now_ms=t, + ) + assert second.total_explicit_flood_observations() == initial_flood + 1 + assert second.shared_cooldown_wait(account, chat_id, now_ms=t) == 17 + assert first.shared_cooldown_wait( + account, chat_id, now_ms=t + 17000, + ) == 0 + assert second.reserve_send_slot( + account, chat_id, 1.5, now_ms=t, + ) == 17 + first.impose_shared_cooldown( + account, chat_id, 3, now_ms=t, + ) + assert second.total_explicit_flood_observations() == initial_flood + 2 + assert second.shared_cooldown_wait(account, chat_id, now_ms=t) == 17 + assert second.shared_cooldown_wait( + account + "-other", chat_id, now_ms=t, + ) == 0 + finally: + after.close() + before.close() diff --git a/tests/e2e/test_telegram_history_live_e2e.py b/tests/e2e/test_telegram_history_live_e2e.py new file mode 100644 index 00000000..0c5d811e --- /dev/null +++ b/tests/e2e/test_telegram_history_live_e2e.py @@ -0,0 +1,103 @@ +"""Dedicated real Telegram test-channel candidate inventory, never production history.""" +from __future__ import annotations +import asyncio +import hashlib +import os +import tempfile +from pathlib import Path +from uuid import uuid4 + +import pytest +from telethon import TelegramClient +from telethon.sessions import StringSession +from teledrive.storage.history_inventory import inventory_channel +from teledrive.storage.reconcile_inventory import reconcile_inventory +from teledrive.storage.rebuild import restore_isolated_catalog +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.telethon_adapter import TelethonChunkTransport +from teledrive.storage.target_resolver import resolve_telegram_target + + +def test_real_test_channel_discovery_without_inventory_mutations(): + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Dedicated E2E session is not configured") + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-discovery-ci-") as folder: + source = Path(folder) / "fixture.bin" + body = b"CI owned marker fixture " + os.urandom(32) + source.write_bytes(body) + client = TelegramClient(StringSession(session), int(api_id), api_hash) + target, sent = None, None + try: + await client.connect() + if not await client.is_user_authorized(): + raise RuntimeError("Telegram test session is not authorized") + target = await resolve_telegram_target( + client, os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + ident = str(uuid4()) + digest = hashlib.sha256(body).hexdigest() + transport = TelethonChunkTransport({"ci": (client, target)}) + sent = await transport.upload( + source, object_id=ident, ordinal=0, checksum=digest, + ) + report = await inventory_channel( + client, target, account="ci", scan_limit=40, + ) + candidate = next(o for o in report["candidate_objects"] + if o["object_id"] == ident) + assert candidate["candidate_ordinals_contiguous"] is True + assert candidate["chunks"] == [{ + "ordinal": 0, "checksum_prefix": digest[:16], + "account": sent[0], "chat_id": sent[1], "message_id": sent[2], + }] + assert report["payload_downloads"] == 0 + assert report["telegram_mutations"] == 0 + assert report["restorable_without_independent_manifest"] is False + assert await client.get_messages(target, ids=sent[2]) is not None + # The fixture independently supplies bucket/key, exact length + # and full SHA-256. A marker alone never supplies this data. + plan = { + "object_id": ident, "bucket": "recovery-ci", + "key": "isolated/channel-fixture.bin", + "size": len(body), "sha256": digest, + "chunks": [{ + "ordinal": 0, "account": sent[0], "chat_id": sent[1], + "message_id": sent[2], "size": len(body), "sha256": digest, + }], + } + preflight = reconcile_inventory(report, [plan]) + assert preflight["all_plans_match"] + assert preflight["payload_verified"] is False + isolated_path = Path(folder) / "isolated-rebuilt.sqlite" + restored = await restore_isolated_catalog( + plan, transport, isolated_path, + ) + assert restored["verified"] and restored["telegram_deletions"] == 0 + rebuilt_db = connect_database(str(isolated_path)) + try: + catalog = BucketCatalog(ManifestStore(rebuilt_db)) + head = catalog.get("recovery-ci", "isolated/channel-fixture.bin").item + assert head.id == ident and head.size == len(body) + assert head.sha256 == digest + refs = catalog.manifests.parts(head.id) + assert len(refs) == 1 and refs[0].message_id == sent[2] + finally: + rebuilt_db.close() + assert await client.get_messages(target, ids=sent[2]) is not None + finally: + try: + if target is not None and sent is not None: + assert int(await client.get_peer_id(target)) == sent[1] + await client.delete_messages(target, [sent[2]], revoke=True) + finally: + await client.disconnect() + + asyncio.run(scenario()) diff --git a/tests/e2e/test_telegram_multiworker_e2e.py b/tests/e2e/test_telegram_multiworker_e2e.py new file mode 100644 index 00000000..a5c0e074 --- /dev/null +++ b/tests/e2e/test_telegram_multiworker_e2e.py @@ -0,0 +1,400 @@ +"""Opt-in real Telegram + PostgreSQL + THREE separate worker processes. + +The process workers share only a real PostgreSQL server and a dedicated +Telethon USER session/channel: each opens a distinct SQL connection and its +own MTProto client. The production SQL rate guard serializes Telegram send +starts across workers. Only this test's owned Telegram message IDs are +cleaned up after byte-verified readback. +""" +from __future__ import annotations + +import asyncio +import hashlib +import os +import subprocess +import sys +import tempfile +import time +from pathlib import Path +from uuid import uuid4 + +import pytest +from sqlalchemy import select + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore, chunks +from teledrive.storage.target_resolver import resolve_telegram_target +from teledrive.storage.telethon_adapter import TelethonChunkTransport +from teledrive.storage.user_mode import UserSendJournal + + +WORKER = r""" +import asyncio, os, sys, traceback +from pathlib import Path +from telethon import TelegramClient +from telethon.sessions import StringSession +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.target_resolver import resolve_telegram_target +from teledrive.storage.telethon_adapter import TelethonChunkTransport +from teledrive.storage.user_mode import UnifiedUserTransport, UserSendJournal + +async def run(): + db_url, source, ready, go, target, account = sys.argv[1:] + Path(ready).write_text("ready", encoding="ascii") + for _ in range(500): + if Path(go).exists(): + break + await asyncio.sleep(0.03) + else: + raise RuntimeError("Worker synchronization expired") + db = connect_database(db_url) + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + channel_name = os.getenv("TELEDRIVE_E2E_CHANNEL_NAME") or None + # One USER StringSession must never hold concurrent connections: + # serialize the whole per-job connect/upload/disconnect window + # across workers with a file lock in the shared source tree. + import fcntl + lock = open(Path(source) / "mtproto-connect.lock", "w") + for _ in range(12): + fcntl.flock(lock, fcntl.LOCK_EX) + try: + # Each job uses a fresh, short-lived client. + client = TelegramClient( + StringSession(os.environ["TELEDRIVE_E2E_SESSION"]), + int(os.environ["TELEDRIVE_E2E_API_ID"]), + os.environ["TELEDRIVE_E2E_API_HASH"], + device_model="TeleDrive Storage Client", + app_version="TeleDrive 2.1", + receive_updates=False, flood_sleep_threshold=60, + ) + await client.connect() + try: + resolved = await resolve_telegram_target( + client, target or None, channel_name, + ) + peer_chat_id = int(await client.get_peer_id(resolved)) + from types import SimpleNamespace + runtime = SimpleNamespace( + account=account, chat_id=peer_chat_id, + ) + runtime.require = lambda: TelethonChunkTransport({ + account: (client, resolved), + }) + + transport = UnifiedUserTransport( + runtime, UserSendJournal(db), min_upload_interval=1.25, + ) + work = await manager.run_next( + transport, allowed_root=Path(source), + scratch_directory=Path(source), chunk_size=256 * 1024, + ) + if work is None: + return + if work["state"] != "done": + raise RuntimeError("Live multiworker upload was not committed") + finally: + await client.disconnect() + finally: + fcntl.flock(lock, fcntl.LOCK_UN) + raise RuntimeError("Worker claimed too many jobs") + finally: + db.close() + +try: + asyncio.run(run()) +except BaseException as exc: + print(f"WORKER-FAILED {type(exc).__name__}: {exc}", file=sys.stderr) + traceback.print_exc(file=sys.stderr) + raise SystemExit(1) +""" + + +@pytest.mark.skipif( + os.getenv("TELEDRIVE_E2E_MULTIWORKER") != "1", + reason="Real PostgreSQL and USER MTProto multiworker soak requires explicit opt-in", +) +def test_live_three_worker_user_mtproto_soak_with_remote_checksum_cleanup(): + url = os.getenv("DB_TEST_POSTGRES_URL", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + target = os.getenv("TELEDRIVE_E2E_TARGET", "") + channel_name = os.getenv("TELEDRIVE_E2E_CHANNEL_NAME", "") + if not (url and session and api_id and api_hash and (target or channel_name)): + pytest.fail("Opt-in multiworker Telegram E2E needs PG and USER credentials") + from telethon import TelegramClient + from telethon.sessions import StringSession + + with tempfile.TemporaryDirectory(prefix="teledrive-pg-user-soak-") as folder: + root = Path(folder) + database = connect_database(url) + account = "user-ci-multi-" + uuid4().hex[:12] + bucket = "ci-pg-" + uuid4().hex[:16] + sizes = (0, 1, 65537, 262159, 1024 * 1024 + 11, + 2 * 1024 * 1024 + 9) + expected = {} + processes = [] + try: + catalog = BucketCatalog(ManifestStore(database)) + catalog.create_bucket(bucket, int(time.time())) + UserSendJournal(database) # create SQL intents/slots before workers race + queue = StorageUploadJobs(catalog) + for n, size in enumerate(sizes): + data = (bytes((0, 255)) * ((size + 1)//2))[:size] if n % 2 == 0 else os.urandom(size) + filename = root / f"source-{n}.bin" + filename.write_bytes(data) + key = f"multiworker/{n}-{size}.bin" + queue.enqueue(bucket, key, filename, allowed_root=root) + expected[key] = data + assert len(queue.list_jobs(limit=50)) >= len(sizes) + go = root / "all-workers-start" + for n in range(3): + ready = root / f"worker-{n}.ready" + worker = subprocess.Popen( + [sys.executable, "-c", WORKER, url, str(root), + str(ready), str(go), target, account], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=Path(__file__).resolve().parents[2], + ) + processes.append((worker, ready)) + deadline = time.monotonic() + 60 + while not all(ready.exists() for _, ready in processes): + if time.monotonic() >= deadline: + raise RuntimeError("Three SQL worker processes did not start") + if any(proc.poll() is not None for proc, _ in processes): + for proc, _ in processes: + if proc.poll() is not None: + _, stderr = proc.communicate(timeout=10) + raise RuntimeError( + "Real USER worker failed before the start gate: " + + stderr.decode(errors="replace")[-1500:] + ) + raise RuntimeError("Real USER worker failed before the start gate") + time.sleep(0.05) + go.write_text("go", encoding="ascii") + for proc, _ in processes: + stdout, stderr = proc.communicate(timeout=480) + assert proc.returncode == 0, stderr.decode(errors="replace")[-2000:] + database.close() + database = connect_database(url) + catalog = BucketCatalog(ManifestStore(database)) + jobs = [ + entry for entry in StorageUploadJobs(catalog).list_jobs(limit=1000) + if entry["bucket"] == bucket + ] + assert len(jobs) == len(sizes) + assert all(row["state"] == "done" and row["attempts"] == 1 for row in jobs) + # Readback phase runs after all workers disconnected their + # clients, so only this process holds the USER session now. + # One asyncio.run scope: a connected Telethon client is bound + # to its event loop and cannot move between them. + def readback_scenario(): + async def scenario(): + client = TelegramClient( + StringSession(session), int(api_id), api_hash, + device_model="TeleDrive Storage Client", + app_version="TeleDrive 2.1", + receive_updates=False, flood_sleep_threshold=60, + ) + await client.connect() + try: + resolved = await resolve_telegram_target( + client, target or None, channel_name or None, + ) + chat = int(await client.get_peer_id(resolved)) + transport = TelethonChunkTransport({ + account: (client, resolved), + }) + engine = TelegramObjectStorage( + catalog.manifests, transport, + scratch_directory=root, + ) + for key, body in expected.items(): + item = catalog.get(bucket, key).item + assert item.sha256 == hashlib.sha256(body).hexdigest() + destination = root / f"restored-{uuid4().hex}.bin" + await engine.download_to_path(item.id, destination) + assert destination.read_bytes() == body + received = await _collect_range( + engine, item.id, len(body), len(body)//2, + ) + assert received == body[len(body)//2:min(len(body), len(body)//2+37)] + assert UserSendJournal(database).unresolved_count() == 0 + # Only the globally unique account alias created by + # this run. + owned = database.connection.execute( + select(chunks.c.message_id).where( + chunks.c.account == account, + chunks.c.chat_id == chat, + ) + ).scalars().all() + if owned: + await client.delete_messages( + resolved, + sorted(set(int(mid) for mid in owned)), + ) + finally: + await client.disconnect() + + return asyncio.run(scenario()) + + readback_scenario() + finally: + for proc, _ in processes: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=10) + database.close() + + +async def _collect_range(engine, ident, size, start): + return b"".join([ + part async for part in engine.iter_bytes( + ident, start=start, end=min(size, start+37), + block_size=7, + ) + ]) + + +def test_embedded_worker_script_is_valid_python(): + """The subprocess worker must compile before any live CI run.""" + compile(WORKER, "", "exec") + + +def test_worker_script_processes_jobs_with_fake_telethon(tmp_path): + """End-to-end worker flow (SQL jobs + fake MTProto) without live Telegram. + + Spawns the embedded worker subprocess with a stubbed telethon package so + the claim/connect/upload/journal loop is verified before any live CI run. + """ + import textwrap + from types import SimpleNamespace + + fake_telethon = tmp_path / "fake_telethon" + package = fake_telethon / "telethon" + package.mkdir(parents=True) + (package / "__init__.py").write_text( + "from .client import TelegramClient\n", encoding="utf-8" + ) + (package / "sessions.py").write_text( + "StringSession = lambda value: value\n", encoding="utf-8" + ) + (package / "errors.py").write_text( + "class FloodWaitError(Exception):\n" + " def __init__(self, request=None, capture=0):\n" + " self.seconds = int(capture)\n" + " super().__init__('flood wait %d' % self.seconds)\n", + encoding="utf-8", + ) + (package / "client.py").write_text(textwrap.dedent(""" + from types import SimpleNamespace + + class TelegramClient: + def __init__(self, session, api_id, api_hash, **kwargs): + self.session = session + self.posts = {} + self.next_id = 1000 + + async def connect(self): + pass + + async def disconnect(self): + pass + + def is_connected(self): + return True + + async def get_peer_id(self, target): + return -100123456 + + async def iter_dialogs(self, limit=None): + entity = SimpleNamespace(username=None) + dialog = SimpleNamespace( + id=-100123, is_channel=True, is_group=False, + entity=entity, name="Test Channel", + ) + if limit: + assert limit > 0 + yield dialog + else: + yield dialog + + async def get_entity(self, target): + return SimpleNamespace(username=None) + + async def send_file(self, target, path, caption=None, + force_document=False): + from pathlib import Path + assert caption.startswith("teledrive:") + self.next_id += 1 + self.posts[self.next_id] = Path(path).read_bytes() + return SimpleNamespace(id=self.next_id, chat_id=-100123456) + """), encoding="utf-8") + + db_url = "sqlite:///" + str(tmp_path / "state.db") + from teledrive.db import connect_database + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.jobs import StorageUploadJobs + from teledrive.storage.manifest import ManifestStore + + source = tmp_path / "sources" + source.mkdir() + body = b"worker-job-bytes" * 100 + payload = source / "job.bin" + payload.write_bytes(body) + db = connect_database(db_url) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("worker-ci", 1) + StorageUploadJobs(catalog).enqueue( + "worker-ci", "a.bin", payload, allowed_root=source, + ) + finally: + db.close() + + ready = tmp_path / "ready" + go = tmp_path / "go" + env = { + "PYTHONPATH": str(fake_telethon), + "TELEDRIVE_E2E_SESSION": "fake-session", + "TELEDRIVE_E2E_API_ID": "12345", + "TELEDRIVE_E2E_API_HASH": "a" * 32, + } + import os + proc_env = dict(os.environ) + proc_env.update(env) + # The live workflow exports these; the fake-telethon run must resolve + # by the numeric target argv instead of a real channel title. + proc_env.pop("TELEDRIVE_E2E_CHANNEL_NAME", None) + proc_env.pop("TELEDRIVE_E2E_TARGET", None) + process = subprocess.Popen( + [sys.executable, "-c", WORKER, db_url, str(source), + str(ready), str(go), "-100123", "worker-account"], + env=proc_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, cwd=Path(__file__).resolve().parents[2], + ) + deadline = time.monotonic() + 30 + while not ready.exists(): + if time.monotonic() >= deadline or process.poll() is not None: + process.kill() + raise AssertionError("worker did not reach the ready gate") + time.sleep(0.05) + go.write_text("go", encoding="ascii") + stdout, stderr = process.communicate(timeout=120) + assert process.returncode == 0, stderr[-2000:] + db = connect_database(db_url) + try: + catalog = BucketCatalog(ManifestStore(db)) + rows = StorageUploadJobs(catalog).list_jobs(limit=10) + assert len(rows) == 1 and rows[0]["state"] == "done" + item = catalog.get("worker-ci", "a.bin").item + assert item.sha256 == hashlib.sha256(body).hexdigest() + finally: + db.close() diff --git a/tests/e2e/test_telegram_storage_e2e.py b/tests/e2e/test_telegram_storage_e2e.py new file mode 100644 index 00000000..d4ccc751 --- /dev/null +++ b/tests/e2e/test_telegram_storage_e2e.py @@ -0,0 +1,442 @@ +"""Manual-only Telegram round trip. Requires dedicated test account credentials. + +No other CI job requires these secrets. Do not point at a personal account. +""" +from __future__ import annotations + +import asyncio +import hashlib +import os +import tempfile +from pathlib import Path + +from telethon import TelegramClient +from telethon.sessions import StringSession + +import pytest + +from teledrive.db import connect_database +from teledrive.storage import ManifestStore, TelegramObjectStorage +from teledrive.storage.telethon_adapter import TelethonChunkTransport +from teledrive.storage.target_resolver import resolve_telegram_target + + +def test_real_telegram_upload_download_and_range(): + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Manual E2E requires dedicated Telegram test credentials") + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-e2e-") as folder: + root = Path(folder) + original = root / "input.bin" + content = os.urandom(4097) + original.write_bytes(content) + conn = connect_database(str(root / "manifest.db")) + store = ManifestStore(conn) + client = TelegramClient(StringSession(session), int(api_id), api_hash) + await client.connect() + uploaded_refs = [] + target = None + try: + if not await client.is_user_authorized(): + raise RuntimeError("E2E StringSession is expired or not authorized") + target = await resolve_telegram_target( + client, + os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + backend = TelegramObjectStorage( + store, TelethonChunkTransport({"ci": (client, target)}), + chunk_size=2048, + ) + item = await backend.put_file("ci", "ci/roundtrip.bin", original) + uploaded_refs = store.parts(item.id) + original.unlink() + + output = root / "restored.bin" + await backend.download_to_path(item.id, output) + assert output.read_bytes() == content + assert hashlib.sha256(output.read_bytes()).hexdigest() == item.sha256 + + range_bytes = bytearray() + async for part in backend.iter_bytes(item.id, start=2047, end=2051): + range_bytes.extend(part) + assert bytes(range_bytes) == content[2047:2051] + assert len(uploaded_refs) == 3 + finally: + # Cleanup is scoped to the test target and only messages + # uploaded by this test invocation. + for reference in uploaded_refs: + if target is not None: + await client.delete_messages(target, [reference.message_id]) + await client.disconnect() + conn.close() + + asyncio.run(scenario()) + + +def test_real_telegram_unindexed_chunk_cleanup(): + """Manual E2E: Telegram accepted upload, DB journal commit was lost. + + Only the message created in this test is eligible for deletion. + """ + import time + from teledrive.storage.recovery import RecoveryManager + + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Manual E2E requires dedicated Telegram test credentials") + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-recovery-e2e-") as folder: + root = Path(folder) + upload = root / "orphan.bin" + body = os.urandom(87) + upload.write_bytes(body) + conn = connect_database(str(root / "manifest.db")) + store = ManifestStore(conn) + client = TelegramClient(StringSession(session), int(api_id), api_hash) + await client.connect() + uploaded_id = None + target = None + try: + if not await client.is_user_authorized(): + raise RuntimeError("E2E Telegram test session not authorized") + target = await resolve_telegram_target( + client, + os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + transport = TelethonChunkTransport({"ci": (client, target)}) + old = int(time.time()) - 7200 + identifier = store.begin("ci", "unindexed-test", len(body), old) + digest = hashlib.sha256(body).hexdigest() + account, chat_id, uploaded_id = await transport.upload( + upload, object_id=identifier, ordinal=0, checksum=digest + ) + assert account == "ci" and chat_id != 0 and uploaded_id > 0 + # Simulate a crash between Telegram's successful response + # and SQL append, without relying on mocks. + store.fail(identifier, "simulated metadata commit crash", old) + manager = RecoveryManager(store, transport, min_age_seconds=3600) + preview = await manager.cleanup(identifier) + assert preview.orphaned == 1 and preview.deleted == 0 + assert await client.get_messages(target, ids=uploaded_id) is not None + done = await manager.cleanup(identifier, dry_run=False) + assert done.cleaned and done.deleted == 1 + assert await client.get_messages(target, ids=uploaded_id) is None + finally: + if uploaded_id is not None and target is not None: + await client.delete_messages(target, [uploaded_id], revoke=True) + await client.disconnect() + conn.close() + + asyncio.run(scenario()) + + +@pytest.mark.parametrize("size,chunk_size", [ + pytest.param(0, 2048, id="empty-0B"), + pytest.param(1, 2048, id="single-byte"), + pytest.param(2047, 2048, id="chunk-minus-one"), + pytest.param(2048, 2048, id="exact-chunk"), + pytest.param(2049, 2048, id="chunk-plus-one"), + pytest.param(65537, 32768, id="64KiB-plus-one"), + pytest.param(1048593, 524288, id="1MiB-plus-17"), + pytest.param(2 * 1024 * 1024 + 17, 1024 * 1024, id="2MiB-plus-17"), + pytest.param(3 * 1024 * 1024 + 31, 1024 * 1024, id="3MiB-plus-31"), + pytest.param(4 * 1024 * 1024 + 1, 1024 * 1024, id="4MiB-plus-one-five-chunks"), +]) +def test_real_telethon_varied_payload_lengths_with_remote_readback(size, chunk_size): + """Live user-account Telegram: SHA of every remote part and crossed ranges. + + Every send ACK is tracked *before* the manifest journal: even an assertion + failure during publish attempts to delete only this test's new messages. + """ + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Dedicated real Telegram user account E2E not configured") + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-user-size-matrix-") as folder: + root = Path(folder) + source = root / "source.bin" + # Binary 0x00/0xFF alternation supplements random payloads. + expected = ((bytes((0, 255)) * ((size + 1) // 2))[:size] + if size == 65537 else os.urandom(size)) + source.write_bytes(expected) + db_path = root / "manifest.db" + conn = connect_database(str(db_path)) + client = TelegramClient(StringSession(session), int(api_id), api_hash) + target = None + uploaded = [] + try: + await client.connect() + if not await client.is_user_authorized(): + raise RuntimeError("E2E Telegram test session expired") + target = await resolve_telegram_target( + client, os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + class TrackedTelegram(TelethonChunkTransport): + async def upload(self, path, *, object_id, ordinal, checksum): + result = await super().upload( + path, object_id=object_id, + ordinal=ordinal, checksum=checksum, + ) + uploaded.append(result) + # Avoid uncontrolled bursts to a shared CI channel. + await asyncio.sleep(0.75) + return result + + transport = TrackedTelegram({"ci": (client, target)}) + engine = TelegramObjectStorage( + ManifestStore(conn), transport, chunk_size=chunk_size, + scratch_directory=root, + ) + obj = await engine.put_file("ci", f"live/sizes/{size}.bin", source) + assert obj.size == size + assert obj.sha256 == hashlib.sha256(expected).hexdigest() + parts = engine.manifests.parts(obj.id) + assert len(parts) == (size + chunk_size - 1) // chunk_size + assert len(uploaded) == len(parts) + assert sum(p.size for p in parts) == size + for index, part in enumerate(parts): + assert part.ordinal == index and part.offset == index * chunk_size + assert part.sha256 == hashlib.sha256( + expected[part.offset:part.offset + part.size] + ).hexdigest() + assert (part.account, part.chat_id, part.message_id) == uploaded[index] + + # Reconstruct from SQL + Telegram on a clean database session. + conn.close() + conn = connect_database(str(db_path)) + source.unlink() + recreated = TelegramObjectStorage( + ManifestStore(conn), TrackedTelegram({"ci": (client, target)}), + chunk_size=chunk_size, scratch_directory=root, + ) + destination = root / "downloaded.bin" + await recreated.download_to_path(obj.id, destination) + assert destination.read_bytes() == expected + for start, end in ( + (0, min(size, 15)), + (min(size, max(0, chunk_size - 2)), min(size, chunk_size + 9)), + (max(0, size - 1), size), + (size, size), + ): + blocks = [b async for b in recreated.iter_bytes( + obj.id, start=start, end=end, block_size=7, + )] + assert b"".join(blocks) == expected[start:end] + finally: + try: + if target is not None: + for _, chat, message in uploaded: + assert int(await client.get_peer_id(target)) == chat + await client.delete_messages( + target, [message], revoke=True, + ) + finally: + await client.disconnect() + conn.close() + + asyncio.run(scenario()) + + +@pytest.mark.skipif( + os.getenv("TELEDRIVE_E2E_STRESS") != "1", + reason="5/16 MiB live Telethon size stress explicitly opt-in", +) +@pytest.mark.parametrize("size", [ + pytest.param(5 * 1024 * 1024 + 1, id="5MiB-plus-one"), + pytest.param(16 * 1024 * 1024 + 17, id="16MiB-plus-17"), +]) +def test_live_telethon_large_payload_chunk_boundaries_and_remote_restore(size): + """Real user Telegram uploads >5MiB, downloads every chunk after SQL restart.""" + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Dedicated test account and channel required") + chunk_size = 4 * 1024 * 1024 + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-user-large-") as folder: + root = Path(folder) + db_path = root / "large.db" + original = root / "payload.bin" + payload = os.urandom(size) + original.write_bytes(payload) + db = connect_database(str(db_path)) + client = TelegramClient(StringSession(session), int(api_id), api_hash) + target = None + acked = [] + try: + await client.connect() + if not await client.is_user_authorized(): + raise RuntimeError("Telethon test session is no longer authorized") + target = await resolve_telegram_target( + client, os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + + class OwnedRemote(TelethonChunkTransport): + async def upload(self, path, *, object_id, ordinal, checksum): + result = await super().upload( + path, object_id=object_id, + ordinal=ordinal, checksum=checksum, + ) + acked.append(result) + await asyncio.sleep(0.75) + return result + + transport = OwnedRemote({"ci": (client, target)}) + engine = TelegramObjectStorage( + ManifestStore(db), transport, chunk_size=chunk_size, + scratch_directory=root, + ) + item = await engine.put_file("ci", "p1/large.bin", original) + parts = engine.manifests.parts(item.id) + assert len(parts) == (size + chunk_size - 1) // chunk_size + assert len(parts) == len(acked) + assert item.sha256 == hashlib.sha256(payload).hexdigest() + for index, part in enumerate(parts): + fragment = payload[part.offset:part.offset + part.size] + assert part.ordinal == index and part.size == len(fragment) + assert part.offset == index * chunk_size + assert part.sha256 == hashlib.sha256(fragment).hexdigest() + assert (part.account, part.chat_id, part.message_id) == acked[index] + + # No original file or in-memory source from the first SQL + # session is used by this recovery path. + db.close() + db = connect_database(str(db_path)) + original.unlink() + new_engine = TelegramObjectStorage( + ManifestStore(db), OwnedRemote({"ci": (client, target)}), + chunk_size=chunk_size, scratch_directory=root, + ) + target_path = root / "restored.bin" + await new_engine.download_to_path(item.id, target_path) + assert target_path.stat().st_size == size + assert hashlib.sha256(target_path.read_bytes()).hexdigest() == item.sha256 + for start, end in ( + (chunk_size - 3, chunk_size + 5), + (max(0, size - 64), size), + (size, size), + ): + blocks = [block async for block in new_engine.iter_bytes( + item.id, start=start, end=end, + block_size=7, + )] + assert b"".join(blocks) == payload[start:end] + finally: + try: + if target is not None: + for _, peer, mid in acked: + assert int(await client.get_peer_id(target)) == peer + await client.delete_messages(target, [mid], revoke=True) + finally: + await client.disconnect() + db.close() + asyncio.run(scenario()) + + +def test_live_user_session_unified_transport_and_history_marker(monkeypatch): + """Real owner-authorized USER session: unified engine and read-only history. + + This test uses the dedicated CI channel only and removes only the message + uploaded by its own fixture. It never touches production Telegram media. + """ + api_id = os.getenv("TELEDRIVE_E2E_API_ID", "") + api_hash = os.getenv("TELEDRIVE_E2E_API_HASH", "") + session = os.getenv("TELEDRIVE_E2E_SESSION", "") + if not all((api_id, api_hash, session)): + pytest.skip("Dedicated user Telegram E2E session is not configured") + + from teledrive.storage.user_mode import ( + UserSessionRuntime, UserSendJournal, UnifiedUserTransport, + ) + from teledrive.storage.history_inventory import inventory_channel + from teledrive.storage.gc_safety import GCReferenceGuard + from teledrive.storage.catalog import BucketCatalog + + async def scenario(): + with tempfile.TemporaryDirectory(prefix="teledrive-user-transport-") as folder: + root = Path(folder) + session_file = root / "mtproto-user.session" + session_file.write_text(session + "\n", encoding="ascii") + session_file.chmod(0o600) + # Resolve the channel with the same trusted, preauthorized user. + resolver = TelegramClient(StringSession(session), int(api_id), api_hash) + await resolver.connect() + try: + target = await resolve_telegram_target( + resolver, os.getenv("TELEDRIVE_E2E_TARGET"), + os.getenv("TELEDRIVE_E2E_CHANNEL_NAME"), + ) + channel_id = int(await resolver.get_peer_id(target)) + finally: + await resolver.disconnect() + config_path = root / "config.yaml" + config_path.write_text( + "storage:\n" + f' channel: "{channel_id}"\n' + " strategy: round_robin\n" + " apis:\n" + f" - api_id: {api_id}\n" + f' api_hash: "{api_hash}"\n' + f" session_file: {session_file}\n", + encoding="ascii", + ) + monkeypatch.setenv("TELEDRIVE_CONFIG", str(config_path)) + runtime = UserSessionRuntime() + await runtime.start() + db = connect_database(str(root / "isolated.db")) + # Run the exact catalog schema that a real REST/S3 installation + # creates, rather than weakening the fail-closed GC graph checker. + catalog = BucketCatalog(ManifestStore(db)) + published = [] + try: + content = os.urandom(4221) + source = root / "source.bin" + source.write_bytes(content) + backend = TelegramObjectStorage( + catalog.manifests, + UnifiedUserTransport(runtime, UserSendJournal(db)), + chunk_size=8192, + ) + item = await backend.put_file("ci", "unified-user-test", source) + published = backend.manifests.parts(item.id) + source.unlink() + output = root / "verified.bin" + await backend.download_to_path(item.id, output) + assert output.read_bytes() == content + report = await inventory_channel( + runtime.client, + runtime.transport.accounts[runtime.account][1], + account=runtime.account, scan_limit=100, + ) + assert any( + obj["object_id"] == item.id + for obj in report["candidate_objects"] + ) + safety = GCReferenceGuard(backend.manifests).evidence(item.id) + assert not safety["deletion_permitted"] + assert "mtproto_send_intent_or_receipt" in safety["reference_blockers"] + finally: + for part in published: + await runtime.client.delete_messages( + runtime.transport.accounts[runtime.account][1], + [part.message_id], + ) + db.close() + await runtime.stop() + asyncio.run(scenario()) diff --git a/tests/meta/__init__.py b/tests/meta/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/meta/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/meta/test_acceptance_cli.py b/tests/meta/test_acceptance_cli.py new file mode 100644 index 00000000..ab430731 --- /dev/null +++ b/tests/meta/test_acceptance_cli.py @@ -0,0 +1,149 @@ +"""Offline, no-secret, no-network regressions for P1 operator wizard.""" +from __future__ import annotations + +import asyncio +import json +import os +import stat +import subprocess +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.ops import acceptance as w + + +def test_report_is_unique_private_and_no_automatic_signoff(tmp_path): + record = {"stages": {"multi_host_soak": {"status": "PASS", "reason": "ok"}}} + first = w.save_report(tmp_path, record) + second = w.save_report(tmp_path, record) + assert first != second + assert stat.S_IMODE(first.stat().st_mode) == 0o600 + saved = json.loads(first.read_text()) + assert saved["production_acceptance_signed_off"] is False + assert saved["operator_review_required"] is True + assert saved["physical_telegram_deletions"] == 0 + + +def test_github_e2e_only_accepts_real_success_not_skipped(monkeypatch): + mocked = [ + {"workflowName": name, "conclusion": conclusion, + "headSha": "ab" * 20, "databaseId": i + 1} + for i, (name, conclusion) in enumerate([ + (w.WORKFLOWS["user"], "success"), + (w.WORKFLOWS["multiworker"], "success"), + ]) + ] + monkeypatch.setattr(w.shutil, "which", lambda name: "/usr/bin/gh") + monkeypatch.setattr(w, "safe_run", lambda *args, **kwargs: + SimpleNamespace(returncode=0, stdout=json.dumps(mocked))) + result = {"stages": {}} + w.github_stage(result) + assert result["stages"]["github_e2e"]["status"] == "PASS" + assert "bot" not in result["github"] + + +@pytest.mark.parametrize("bad", ["not-a-number", "0", "-5"]) +def test_user_channel_fails_without_a_valid_integer_before_telegram(monkeypatch, bad): + prompts = iter(["account", "target", "bucket", "key"]) + monkeypatch.setattr(w, "secret", lambda label: bad if "kanal" in label else "fake-value") + monkeypatch.setattr(w, "ask", lambda label, default="": next(prompts)) + with pytest.raises((ValueError, StopIteration)): + w.ask_user_identity() + + +def test_soak_unavailable_is_blocked_not_green(monkeypatch, tmp_path): + monkeypatch.setattr(w, "yes", lambda *args, **kwargs: False) + result = {"stages": {}} + w.soak_stage(result, tmp_path) + assert result["stages"]["multi_host_soak"]["status"] == "BLOCKED" + assert not list(tmp_path.iterdir()) + + +def test_soak_redacts_origins_and_basic_auth(monkeypatch, tmp_path): + approvals = iter([True, True]) + addresses = iter(["https://node-a.example.org", "https://node-b.example.org", "admin"]) + monkeypatch.setattr(w, "yes", lambda *args, **kwargs: next(approvals)) + monkeypatch.setattr(w, "ask", lambda *args, **kwargs: next(addresses)) + monkeypatch.setattr(w, "secret", lambda *args, **kwargs: "very-secret-password") + + def run(args, *, env=None, **kwargs): + assert env["TELEDRIVE_P1_BASIC_PASSWORD"] == "very-secret-password" + target = Path(args[args.index("--output") + 1]) + target.write_text(json.dumps({ + "sample_count": 61, + "duration_seconds": 3600, + "telemetry_gate_passed": True, + "final_shared_counter_agreement": True, + "explicit_429_deltas_by_host": {"host-a": 3, "host-b": 3}, + })) + return SimpleNamespace(returncode=0) + monkeypatch.setattr(w, "safe_run", run) + record = {"stages": {}} + w.soak_stage(record, tmp_path) + assert record["stages"]["multi_host_soak"]["status"] == "PASS" + assert "very-secret-password" not in json.dumps(record) + assert "example.org" not in json.dumps(record) + assert "production_acceptance_signed_off" not in json.dumps(record) + + +def test_no_native_restore_to_same_host_even_different_database(tmp_path): + credentials = {"host": "db.internal", "port": 5432, "dbname": "prod", + "user": "owner", "password": "secret"} + staging = dict(credentials, dbname="staging") + with pytest.raises(ValueError, match="Aynı DB sunucusuna"): + w.native_restore("postgresql", credentials, staging, + tmp_path / "backup.dump", tmp_path) + assert not list(tmp_path.iterdir()) + + +def test_mtproto_ack_wizard_never_uses_apply(monkeypatch, tmp_path): + session = tmp_path / "session" + session.write_text("string-session") + session.chmod(0o600) + monkeypatch.setattr(w, "yes", lambda *args, **kwargs: True) + answers = iter(["account", "object-id", "0", "api-id"]) + monkeypatch.setattr(w, "ask", lambda *args, **kwargs: next(answers)) + def secret(label): + if "kanal" in label: + return "-1001234567890" + if "SQL" in label or "veritabanı" in label: + return "/tmp/disposable-staging.db" + if "Telegram message" in label: + return "12345" + if "SHA-256" in label: + return "a" * 64 + if "boyutu" in label: + return "7" + return "not-real-secret" + monkeypatch.setattr(w, "secret", secret) + from scripts.ops import reconcile_mtproto_receipt + async def fake_verify(args): + assert args.apply is False + assert args.database.endswith("disposable-staging.db") + return {"verified": True, "applied": False} + monkeypatch.setattr(reconcile_mtproto_receipt, "run", fake_verify) + result = {"stages": {}} + w.ack_stage(result, session) + assert result["stages"]["mtproto_ack"]["status"] == "PASS" + assert result["ack"]["applied"] is False + assert os.environ.get("TELEDRIVE_RECONCILE_SESSION") != "string-session" + + +def test_session_step_uses_async_login_and_never_displays_session(monkeypatch, tmp_path): + path = tmp_path / "session" + async def fake_login(output): + assert output == path + return True + monkeypatch.setattr(w, "yes", lambda *args, **kwargs: True) + monkeypatch.setattr(w, "session_login", fake_login) + record = {"stages": {}} + w.session_stage(record, path) + assert record["stages"]["telegram_session"]["status"] == "PASS" + + +def test_script_has_tty_gate_instead_of_stdin_heredoc(): + assert w.main.__code__.co_name == "main" + assert "--evidence-dir" in w.main.__code__.co_consts + assert "python - <= 1 + assert result["verified_bytes"] == result["verified_roundtrips"] * 4097 + assert result["p95_roundtrip_ms"] >= 0 + assert result["real_telegram_tested"] is False + assert result["physical_hosts_tested"] is False + assert result["production_acceptance_signed_off"] is False + + +@pytest.mark.parametrize("params", [ + {"duration_seconds": 0, "object_bytes": 1024}, + {"duration_seconds": 21601, "object_bytes": 1024}, + {"duration_seconds": 1, "object_bytes": 0}, + {"duration_seconds": 1, "object_bytes": 4 * 1024 * 1024 + 1}, + {"duration_seconds": 1, "object_bytes": 1, "interval_seconds": -1}, +]) +def test_invalid_soak_bounds_fail_closed(params): + with pytest.raises(ValueError): + run(**params) + + +def test_disk_media_retention_survives_database_restart(tmp_path): + result = run(duration_seconds=1, object_bytes=65537, + interval_seconds=0, workspace=tmp_path / "disposable") + assert result["sql_restarts"] >= 1 + assert result["historical_rereads"] >= 3 + assert result["retained_telegram_bytes"] == result["verified_bytes"] + assert result["retained_telegram_chunks"] >= result["verified_roundtrips"] + assert result["real_telegram_tested"] is False + + +def test_synthetic_message_corruption_is_visible_to_original_storage_engine(tmp_path): + from scripts.ops.capacity_probe import IsolatedMedia + from teledrive.db import connect_database + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.engine import TelegramObjectStorage, IntegrityError + from teledrive.storage.manifest import ManifestStore + import hashlib + import asyncio + + scratch = tmp_path / "workspace" + scratch.mkdir() + media = IsolatedMedia(scratch / "telegram") + db = connect_database(str(scratch / "db.sqlite")) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("capacity-test", 100) + src = scratch / "source.bin" + src.write_bytes(b"corruption-target") + item = asyncio.run(TelegramObjectStorage( + catalog.manifests, media, chunk_size=3, scratch_directory=scratch, + ).put_file("capacity-test", "broken", src)) + catalog.publish(item.id, now=100) + src.unlink() + media.path(1).write_bytes(b"BAD") + with pytest.raises(IntegrityError): + asyncio.run(TelegramObjectStorage( + catalog.manifests, media, chunk_size=3, scratch_directory=scratch, + ).download_to_path(item.id, scratch / "output")) + assert not (scratch / "output").exists() + finally: + db.close() diff --git a/tests/meta/test_ci_workflows.py b/tests/meta/test_ci_workflows.py new file mode 100644 index 00000000..8e222d3c --- /dev/null +++ b/tests/meta/test_ci_workflows.py @@ -0,0 +1,1107 @@ +"""Contract checks for the GitHub Actions pipeline itself. + +These run without GitHub secrets or live Telegram access and catch accidental +false-green E2E jobs, unsafe release permissions, and missing test coverage. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOWS = ROOT / ".github" / "workflows" +EXPECTED = ( + "quality-security.yml", + "integration-storage.yml", + "integration-databases.yml", + "live-telegram-user.yml", + "live-telegram-multiworker.yml", + "delivery-release.yml", + "resilience-docker.yml", +) + + +DISPLAY_NAMES = { + "quality-security.yml": "01 · Quality | Security & Regression", + "integration-databases.yml": "02 · Integration | Databases", + "integration-storage.yml": "02 · Integration | REST & S3", + "resilience-docker.yml": "03 · Resilience | Docker Fault Injection", + "live-telegram-user.yml": "04 · Live Telegram | User MTProto E2E", + "live-telegram-multiworker.yml": "04 · Live Telegram | Multiworker PostgreSQL E2E", + "delivery-release.yml": "05 · Delivery | Release Gate & Packages", +} + + +@pytest.mark.parametrize("name", EXPECTED) +def test_workflow_display_groups_and_summary_preserved(name): + """Presentation-only labels must remain complete and unambiguous.""" + document = workflow(name) + assert document["name"] == DISPLAY_NAMES[name] + for job in document["jobs"].values(): + assert job.get("name"), name + result_steps = [step for step in job["steps"] if step.get("name") == "Publish CI run summary"] + assert len(result_steps) == 1, name + assert result_steps[0]["if"] == "always()" + assert "job.status" in result_steps[0]["run"] + + +def workflow(name: str): + return yaml.load((WORKFLOWS / name).read_text(encoding="utf-8"), + Loader=yaml.BaseLoader) + + +def step(job, name: str): + return next(s for s in job["steps"] if s["name"] == name) + + +@pytest.mark.parametrize("name", EXPECTED) +def test_workflow_parses_and_has_jobs_with_no_continue_on_error(name): + path = WORKFLOWS / name + assert path.is_file(), f"Missing {name}" + doc = workflow(name) + assert isinstance(doc, dict) and isinstance(doc.get("jobs"), dict) + assert doc["jobs"], name + # Explicitly reject duplicate YAML keys that normal loaders may silently overwrite. + raw = path.read_text(encoding="utf-8") + assert not re.search(r"(?m)^( name: [^\n]+\n(?: [^\n]*\n){0,5} name: )", raw) + assert not re.search(r"(?m)^\s*continue-on-error:\s*true", path.read_text()) + + +@pytest.mark.parametrize("name", EXPECTED) +def test_all_external_actions_are_pinned_to_full_commit_sha(name): + path = WORKFLOWS / name + for action in re.findall(r"(?m)^\s*uses:\s*(\S+)", path.read_text()): + if action.startswith("./"): + continue + assert re.fullmatch(r".+@[0-9a-f]{40}", action), (name, action) + + +@pytest.mark.parametrize("name,job_name,test_file", ( + ("live-telegram-user.yml", "telegram-roundtrip", + "tests/e2e/test_telegram_storage_e2e.py"), +)) +def test_live_e2e_is_push_manual_only_fails_closed_and_never_aborts_cleanup( + name, job_name, test_file +): + doc = workflow(name) + events = doc["on"] + assert "workflow_dispatch" in events + assert "push" in events and "pull_request" not in events + assert doc["permissions"]["contents"] == "read" + assert doc["concurrency"]["cancel-in-progress"] == "false" + assert (ROOT / test_file).is_file() + + job = doc["jobs"][job_name] + checks = next(s for s in job["steps"] if s.get("id") == "configured") + script = checks["run"] + assert "::error::" in script and "exit 1" in script + assert 'enabled=false' not in script # no misleading green workflow + assert 'enabled=true' in script + assert any(test_file in s.get("run", "") for s in job["steps"]) + assert not any(s.get("continue-on-error") == "true" for s in job["steps"]) + + +def test_release_requires_regressions_before_build_and_tag_before_publication(): + doc = workflow("delivery-release.yml") + assert doc["permissions"]["contents"] == "read" + assert "tags" in doc["on"]["push"] + jobs = doc["jobs"] + verify = jobs["verify"] + assert "git merge-base --is-ancestor" in "\n".join( + s.get("run", "") for s in verify["steps"] + ) + assert any("pytest" in s.get("run", "") for s in verify["steps"]) + assert any("pip-audit" in s.get("run", "") for s in verify["steps"]) + assert jobs["build"]["needs"] == "verify" + assert jobs["publish"]["needs"] == "build" + assert "github.ref_type == 'tag'" in jobs["publish"]["if"] + assert jobs["publish"]["permissions"]["contents"] == "write" + + +def test_security_includes_storage_and_api_tests(): + doc = workflow("quality-security.yml") + assert doc["permissions"]["contents"] == "read" + scripts = "\n".join(s.get("run", "") for s in doc["jobs"]["security"]["steps"]) + for path in ( + "tests/meta/test_ci_workflows.py", + "tests/storage/test_storage_user_mode.py", + "tests/storage/test_mtproto_rate_guard.py", + "tests/storage/test_telegram_target_resolution.py", + "tests/storage/test_storage_recovery.py", + "tests/web/test_api_keys.py", + "tests/web/test_rest_api.py", + "tests/s3/test_s3_sigv4_defense.py", + "tests/s3/test_s3_fail_closed.py", + "tests/s3/test_s3_contract.py", + ): + assert path in scripts + for tool in ("pip-audit", "bandit", "docker build", "trivy"): + if tool == "trivy": + assert any("trivy-action@" in s.get("uses", "") + for s in doc["jobs"]["security"]["steps"]) + else: + assert tool in scripts + + +def test_db_and_desktop_matrix_and_storage_contracts_are_present(): + db = workflow("integration-databases.yml")["jobs"]["database"] + assert {"mysql", "postgres"} <= set(db["services"]) + assert any("tests/e2e/test_database_live.py" in s.get("run", "") + and "tests/storage/test_storage_engine.py" in s.get("run", "") + and "tests/storage/test_storage_recovery.py" in s.get("run", "") + for s in db["steps"]) + + api = workflow("integration-storage.yml")["jobs"]["storage-api"] + commands = "\n".join(s.get("run", "") for s in api["steps"]) + for path in ("tests/web/test_rest_api.py", "tests/web/test_rest_transfer.py", + "tests/storage/test_storage_catalog.py", "tests/storage/test_storage_user_mode.py", + "tests/s3/test_s3_permissions.py"): + assert path in commands + + +def test_final_p0_contract_runs_on_every_database_and_desktop_push(): + for name in ("integration-databases.yml", "integration-storage.yml"): + rules = workflow(name)["on"] + for event in ("push", "pull_request"): + assert event in rules + assert "paths" not in rules[event] + assert "paths-ignore" not in rules[event] + commands = "\n".join( + s.get("run", "") + for s in workflow("integration-storage.yml")["jobs"]["storage-api"]["steps"] + ) + assert "tests/s3/test_s3_contract.py" in commands + + + +def test_acceptance_report_updates_cannot_skip_required_ci(): + for name in ("integration-databases.yml", "integration-storage.yml"): + events = workflow(name)["on"] + assert "paths" not in events["push"] + assert "paths" not in events["pull_request"] + + + +def test_status_audit_rechecks_every_release_workflow(): + for name in ("integration-databases.yml", "integration-storage.yml"): + triggers = workflow(name)["on"] + for event in ("push", "pull_request"): + assert "paths" not in triggers[event] + release = workflow("delivery-release.yml")["on"]["push"] + assert "paths" not in release + assert "tags" in release and "branches" in release + + + +def test_storage_ci_runs_legacy_migration_on_every_change(): + events = workflow("integration-storage.yml")["on"] + for event in ("push", "pull_request"): + assert "paths" not in events[event] + command = "\n".join( + x.get("run", "") for x in workflow("integration-storage.yml")["jobs"]["storage-api"]["steps"] + ) + for path in ( + "tests/storage/test_legacy_bridge.py", + "tests/storage/test_legacy_import_admin.py", + "tests/storage/test_storage_jobs.py", + ): + assert path in command + + +def test_mtproto_send_intent_contract_and_operator_health_run_on_every_push(): + storage = workflow("integration-storage.yml") + for event in ("push", "pull_request"): + assert "paths" not in storage["on"][event] + executed = "\n".join( + s.get("run", "") for s in storage["jobs"]["storage-api"]["steps"] + ) + for test_file in ( + "tests/storage/test_storage_user_mode.py", + "tests/storage/test_storage_ops_health.py", + "tests/storage/test_mtproto_rate_guard.py", + ): + assert test_file in executed + security_steps = "\n".join( + s.get("run", "") for s in workflow("quality-security.yml")["jobs"]["security"]["steps"] + ) + assert "tests/storage/test_storage_user_mode.py" in security_steps + assert "tests/storage/test_mtproto_rate_guard.py" in security_steps + + +def test_production_acceptance_dr_ack_cutover_gates_are_run_in_ci(): + """Safety-critical operator artifacts must stay in non-skipped CI gates.""" + for filename in ("integration-storage.yml", "quality-security.yml", "delivery-release.yml"): + content = (WORKFLOWS / filename).read_text(encoding="utf-8") + for path in ( + "tests/storage/test_mtproto_rate_guard.py", + "tests/storage/test_telegram_dr_rehearsal.py", + "tests/storage/test_legacy_cutover_acceptance.py", + ): + assert path in content, (filename, path) + + +def test_mainline_multi_process_live_postgres_user_acceptance(): + name = "live-telegram-multiworker.yml" + path = WORKFLOWS / name + assert path.is_file() + content = path.read_text(encoding="utf-8") + wf = workflow(name) + assert "workflow_dispatch" in wf["on"] + assert "push" in wf["on"] and "pull_request" not in wf["on"] + assert wf["permissions"]["contents"] == "read" + job = wf["jobs"]["three-process-telegram"] + assert "concurrency" not in wf # job-level serialization with the user E2E + assert job["concurrency"]["cancel-in-progress"] == "false" + assert job["concurrency"]["group"] != workflow( + "live-telegram-user.yml" + )["concurrency"]["group"] + assert "postgres" in job["services"] + assert "postgres:17-alpine" in job["services"]["postgres"]["image"] + assert "if" not in job # every main push runs the P1 multiworker acceptance + assert "TELEDRIVE_E2E_SESSION" in job["env"] + assert "TELEDRIVE_E2E_API_ID" in job["env"] + assert "TELEDRIVE_E2E_API_HASH" in job["env"] + assert "TELEDRIVE_E2E_TARGET" in job["env"] + assert "DB_TEST_POSTGRES_URL" in job["env"] + assert job["env"]["TELEDRIVE_E2E_MULTIWORKER"] == "1" + assert any("tests/e2e/test_telegram_multiworker_e2e.py" in step.get( + "run", "") for step in job["steps"]) + assert "continue-on-error" not in content + for action in re.findall(r"(?m)^\s*uses:\s*(\S+)", content): + assert re.fullmatch(r".+@[0-9a-f]{40}", action) + + +@pytest.mark.parametrize("name", [ + "live-telegram-multiworker.yml", +]) +def test_user_target_is_masked_secret_before_any_live_step(name): + """Reject accidental reintroduction of clear-text Actions channel IDs.""" + raw = (WORKFLOWS / name).read_text(encoding="utf-8") + doc = workflow(name) + job = doc["jobs"]["three-process-telegram"] + target = job["env"]["TELEDRIVE_E2E_TARGET"] + assert target == "${{ secrets.TELEDRIVE_E2E_TARGET }}" + assert job["env"]["TELEDRIVE_E2E_CHANNEL_NAME"] == ( + "${{ secrets.TELEDRIVE_E2E_CHANNEL_NAME }}" + ) + assert "vars.TELEDRIVE_E2E_TARGET" not in raw + assert not re.search(r"(?m)^\s*TELEDRIVE_E2E_TARGET:\s*['\"]?-100[0-9]+", raw) + mask = job["steps"][0] + assert mask["name"] == "Mask dedicated Telegram test target" + assert mask["shell"] == "bash" + assert "::add-mask::$TELEDRIVE_E2E_TARGET" in mask["run"] + # The credentials step must fail closed when neither target nor channel + # name is configured; the numeric mask applies only when TARGET is set. + configured = next(s for s in job["steps"] if s.get("name") == + "Configure dedicated live USER credentials") + assert "exit 1" in configured["run"] + assert "TELEDRIVE_E2E_CHANNEL_NAME" in configured["run"] + assert job["steps"][1]["name"] == "Checkout" + + +def test_telegram_setup_writes_user_target_as_secret_and_avoids_value_output(): + content = (ROOT / "scripts" / "dev" / "setup_telegram_e2e.py").read_text("utf-8") + assert 'store_secret(repo, "TELEDRIVE_E2E_TARGET"' in content + assert 'store_variable(repo, "TELEDRIVE_E2E_TARGET"' not in content + assert "def secure_existing_user_target(" in content + assert '["gh", "variable", "delete"' in content + assert "--secure-user-target" in content + assert "configure-bot" not in content + + +def test_telethon_optional_numeric_target_uses_secret_not_public_variable(): + """A numeric user-session peer ID needs masking as much as a Bot peer.""" + raw = (WORKFLOWS / "live-telegram-user.yml").read_text("utf-8") + job = workflow("live-telegram-user.yml")["jobs"]["telegram-roundtrip"] + assert job["env"]["TELEDRIVE_E2E_TARGET"] == "${{ secrets.TELEDRIVE_E2E_TARGET }}" + assert "vars.TELEDRIVE_E2E_TARGET" not in raw + assert job["steps"][0]["name"] == "Mask optional Telegram user target" + assert "::add-mask::$TELEDRIVE_E2E_TARGET" in job["steps"][0]["run"] + assert job["steps"][1]["name"] == "Checkout" + + +def test_e2e_chat_bootstrap_migration_hides_value_and_removes_old_variable( + monkeypatch, capsys, +): + from scripts.dev import setup_telegram_e2e as setup + + example_private_peer = "-100" + "987654321012" + observed = {} + monkeypatch.setattr(setup, "require_github_cli", lambda repo: None) + from subprocess import CompletedProcess + subprocess_calls = [] + + def masked_variable_read(cmd, **kwargs): + subprocess_calls.append((cmd, kwargs)) + return CompletedProcess(cmd, 0, stdout=example_private_peer.encode(), stderr=b"") + + monkeypatch.setattr(setup.subprocess, "run", masked_variable_read) + monkeypatch.setattr( + setup.getpass, "getpass", + lambda prompt: pytest.fail("Existing Actions variable must not be prompted"), + ) + monkeypatch.setattr( + setup, "store_secret", + lambda repo, key, value: observed.update( + {"repository": repo, "name": key, "private_value": value} + ), + ) + monkeypatch.setattr( + setup, "remove_public_user_target_variable", + lambda repo: observed.update({"removed_variable_from": repo}), + ) + setup.secure_existing_user_target("some-owner/some-repo") + assert observed == { + "repository": "some-owner/some-repo", + "name": "TELEDRIVE_E2E_TARGET", + "private_value": example_private_peer, + "removed_variable_from": "some-owner/some-repo", + } + captured = capsys.readouterr() + assert example_private_peer not in captured.out + captured.err + assert "Saved GitHub Actions secret" in captured.out + assert subprocess_calls == [( + ["gh", "variable", "get", "TELEDRIVE_E2E_TARGET", + "--repo", "some-owner/some-repo"], + {"capture_output": True, "check": False}, + )] + + +@pytest.mark.parametrize("bad", [ + "", "not-a-channel", "123456789", "-100", "-100NOTNUMERIC", + "-1001111", "-10012345678901234567890", +]) +def test_e2e_chat_bootstrap_rejects_invalid_private_channel_before_write( + bad, monkeypatch, +): + from scripts.dev import setup_telegram_e2e as setup + + writes = [] + monkeypatch.setattr(setup, "require_github_cli", lambda repo: None) + from subprocess import CompletedProcess + def no_existing_variable(cmd, **kwargs): + if cmd[:3] == ["gh", "variable", "get"]: + return CompletedProcess(cmd, 1, stdout=b"", stderr=b"") + assert cmd[:3] == ["gh", "variable", "list"] + return CompletedProcess(cmd, 0, stdout=b"[]", stderr=b"") + + monkeypatch.setattr(setup.subprocess, "run", no_existing_variable) + monkeypatch.setattr(setup.getpass, "getpass", lambda prompt: bad) + monkeypatch.setattr( + setup, "store_secret", + lambda *args: writes.append(args), + ) + with pytest.raises(ValueError, match="numeric private channel"): + setup.secure_existing_user_target("some-owner/some-repo") + assert not writes + + +@pytest.mark.parametrize("present,delete_success", [ + (True, True), (True, False), (False, True), +]) +def test_e2e_chat_variable_cleanup_never_logs_unmasked_value( + present, delete_success, monkeypatch, capsys, +): + from scripts.dev import setup_telegram_e2e as setup + from subprocess import CompletedProcess + import json + + calls = [] + sample_id = "-100" + "987654321012" + names = ([{"name": "TELEDRIVE_E2E_TARGET"}] if present + else [{"name": "AN_UNRELATED_VARIABLE"}]) + + def mock_run(cmd, **kwargs): + calls.append((cmd, kwargs)) + if cmd[:3] == ["gh", "variable", "list"]: + refreshed = not present or (len(calls) > 1 and delete_success) + rows = ([{"name": "AN_UNRELATED_VARIABLE"}] if refreshed else names) + return CompletedProcess(cmd, 0, stdout=json.dumps(rows).encode(), + stderr=sample_id.encode()) + assert cmd[:3] == ["gh", "variable", "delete"] + return CompletedProcess(cmd, 0 if delete_success else 1, + stdout=b"", stderr=sample_id.encode()) + + monkeypatch.setattr(setup.subprocess, "run", mock_run) + if present and not delete_success: + with pytest.raises(RuntimeError, match="could not be deleted"): + setup.remove_public_user_target_variable("some-owner/some-repo") + else: + setup.remove_public_user_target_variable("some-owner/some-repo") + assert calls[0][0] == [ + "gh", "variable", "list", "--repo", "some-owner/some-repo", + "--json", "name", + ] + assert all(kwargs["capture_output"] for _, kwargs in calls) + assert [cmd[2] for cmd, _ in calls] == ( + ["list", "delete"] if present and not delete_success else + ["list", "delete", "list"] if present else ["list", "list"] + ) + captured = capsys.readouterr() + assert sample_id not in captured.out + captured.err + + +@pytest.mark.parametrize("kind", ["permission", "malformed-json", "wrong-shape"]) +def test_e2e_chat_variable_cleanup_fails_closed_on_unverifiable_state( + kind, monkeypatch, +): + from scripts.dev import setup_telegram_e2e as setup + from subprocess import CompletedProcess + if kind == "permission": + result = CompletedProcess([], 1, stdout=b"", stderr=b"denied") + elif kind == "malformed-json": + result = CompletedProcess([], 0, stdout=b"not-json", stderr=b"") + else: + result = CompletedProcess([], 0, stdout=b'{"name":"x"}', stderr=b"") + monkeypatch.setattr(setup.subprocess, "run", + lambda cmd, **kw: result) + with pytest.raises(RuntimeError, match="Actions variable"): + setup.remove_public_user_target_variable("some-owner/some-repo") + + +def test_e2e_chat_migration_rejects_untrusted_existing_variable_without_writing( + monkeypatch, +): + from scripts.dev import setup_telegram_e2e as setup + from subprocess import CompletedProcess + + monkeypatch.setattr(setup, "require_github_cli", lambda repo: None) + monkeypatch.setattr( + setup.subprocess, "run", + lambda cmd, **kwargs: CompletedProcess( + cmd, 0, stdout=b"unsafe\r\n::warning::hijack", stderr=b"", + ), + ) + monkeypatch.setattr( + setup.getpass, "getpass", + lambda prompt: pytest.fail("Malformed available variable must not be prompted"), + ) + monkeypatch.setattr( + setup, "store_secret", + lambda *args: pytest.fail("Unvalidated channel ID must never be stored"), + ) + with pytest.raises(ValueError, match="numeric private channel"): + setup.secure_existing_user_target("some-owner/some-repo") + + +def test_e2e_chat_migration_unreadable_existing_variable_fails_closed( + monkeypatch, +): + """A permission/network read error must not silently become a prompt.""" + from subprocess import CompletedProcess + from scripts.dev import setup_telegram_e2e as setup + + monkeypatch.setattr(setup, "require_github_cli", lambda repo: None) + commands = [] + + def read_failure(cmd, **kwargs): + assert kwargs["capture_output"] is True + commands.append(cmd[:3]) + if cmd[:3] == ["gh", "variable", "get"]: + return CompletedProcess(cmd, 1, stdout=b"", stderr=b"denied") + assert cmd[:3] == ["gh", "variable", "list"] + return CompletedProcess( + cmd, 0, + stdout=b'[{"name":"TELEDRIVE_E2E_TARGET"}]', + stderr=b"", + ) + + monkeypatch.setattr(setup.subprocess, "run", read_failure) + monkeypatch.setattr( + setup.getpass, "getpass", + lambda _: pytest.fail("Do not prompt when the original variable exists"), + ) + monkeypatch.setattr( + setup, "store_secret", + lambda *args: pytest.fail("Do not overwrite secrets after a read error"), + ) + with pytest.raises(RuntimeError, match="cannot be securely read"): + setup.secure_existing_user_target("some-owner/some-repo") + assert commands == [ + ["gh", "variable", "get"], ["gh", "variable", "list"], + ] + + +def test_e2e_chat_migration_missing_variable_uses_hidden_input( + monkeypatch, capsys, +): + from subprocess import CompletedProcess + from scripts.dev import setup_telegram_e2e as setup + + private_peer = "-100" + "123450987654" + commands = [] + saved = [] + monkeypatch.setattr(setup, "require_github_cli", lambda repo: None) + + def missing_variable(cmd, **kwargs): + assert kwargs["capture_output"] is True + commands.append(cmd[:3]) + if cmd[:3] == ["gh", "variable", "get"]: + return CompletedProcess(cmd, 1, stdout=b"", stderr=b"not found") + assert cmd[:3] == ["gh", "variable", "list"] + return CompletedProcess(cmd, 0, stdout=b"[]", stderr=b"") + + monkeypatch.setattr(setup.subprocess, "run", missing_variable) + monkeypatch.setattr(setup.getpass, "getpass", lambda prompt: private_peer) + monkeypatch.setattr( + setup, "store_secret", + lambda repo, key, value: saved.append((repo, key, value)), + ) + monkeypatch.setattr(setup, "remove_public_user_target_variable", lambda _: None) + setup.secure_existing_user_target("some-owner/some-repo") + assert saved == [( + "some-owner/some-repo", "TELEDRIVE_E2E_TARGET", private_peer, + )] + assert commands == [ + ["gh", "variable", "get"], ["gh", "variable", "list"], + ] + printed = capsys.readouterr() + assert private_peer not in printed.out + printed.err + + +def test_e2e_chat_variable_cleanup_detects_racy_recreation( + monkeypatch, capsys, +): + """A GitHub CLI success is insufficient if the variable still exists.""" + from subprocess import CompletedProcess + from scripts.dev import setup_telegram_e2e as setup + + private_peer = "-100" + "888777666555" + commands = [] + + def variable_reappears(cmd, **kwargs): + assert kwargs["capture_output"] is True + commands.append(cmd[:3]) + if cmd[:3] == ["gh", "variable", "list"]: + return CompletedProcess( + cmd, 0, + stdout=b'[{"name":"TELEDRIVE_E2E_TARGET"}]', + stderr=private_peer.encode(), + ) + assert cmd[:3] == ["gh", "variable", "delete"] + return CompletedProcess(cmd, 0, stdout=b"", stderr=private_peer.encode()) + + monkeypatch.setattr(setup.subprocess, "run", variable_reappears) + with pytest.raises(RuntimeError, match="still present"): + setup.remove_public_user_target_variable("some-owner/some-repo") + assert commands == [ + ["gh", "variable", "list"], ["gh", "variable", "delete"], + ["gh", "variable", "list"], + ] + printed = capsys.readouterr() + assert private_peer not in printed.out + printed.err + + +def test_secure_user_target_cli_help_requires_no_telethon_install(): + """The GitHub-only secret migration must be usable without Telegram SDK.""" + import subprocess + import sys + + command = [ + sys.executable, "-S", + str(ROOT / "scripts" / "dev" / "setup_telegram_e2e.py"), + "--help", + ] + finished = subprocess.run( + command, capture_output=True, text=True, check=False, timeout=15, + ) + assert finished.returncode == 0, finished.stderr + assert "--secure-user-target" in finished.stdout + assert "ModuleNotFoundError" not in finished.stderr + + +def test_user_target_variable_discovery_parses_full_name_listing(monkeypatch): + """The names-only JSON parser must not truncate a long variable list.""" + from subprocess import CompletedProcess + import json + from scripts.dev import setup_telegram_e2e as setup + + legacy_name = "TELEDRIVE_E2E_TARGET" + entries = [{"name": f"OTHER_{n}"} for n in range(40)] + entries.append({"name": legacy_name}) + calls = [] + + def all_variable_names(cmd, **kwargs): + calls.append(cmd) + assert kwargs["capture_output"] is True + assert cmd[-2:] == ["--json", "name"] + return CompletedProcess(cmd, 0, stdout=json.dumps(entries).encode()) + + monkeypatch.setattr(setup.subprocess, "run", all_variable_names) + names = setup.legacy_user_target_variable_names("some-owner/some-repo") + assert legacy_name in names + assert len(names) == 41 + assert len(calls) == 1 + + +def test_live_workers_use_independent_queues_without_cancelling_user_e2e(): + """Keep the multiworker and user pending queues distinct.""" + user_group = "teledrive-live-user-${{ github.repository }}" + user = workflow("live-telegram-user.yml") + workers = workflow("live-telegram-multiworker.yml") + assert user["concurrency"]["group"] == user_group + worker_group = workers["jobs"]["three-process-telegram"]["concurrency"]["group"] + assert worker_group == "teledrive-live-multiworker-${{ github.repository }}" + assert len({user_group, worker_group}) == 2 + for value in ( + user["concurrency"], + workers["jobs"]["three-process-telegram"]["concurrency"], + ): + assert value["cancel-in-progress"] == "false" + + +def test_every_test_file_is_referenced_by_a_real_ci_workflow(): + """New regression modules cannot silently sit outside GitHub Actions.""" + workflow_sources = "\n".join( + (WORKFLOWS / name).read_text(encoding="utf-8") + for name in EXPECTED + ) + missing = [ + str(path.relative_to(ROOT)) + for path in sorted((ROOT / "tests").rglob("test_*.py")) + if str(path.relative_to(ROOT)) not in workflow_sources + ] + assert not missing, f"Unscheduled regression test modules: {missing}" + + +def test_critical_ci_is_unfiltered_at_every_commit_sha(): + for name in ("integration-databases.yml", "integration-storage.yml"): + events = workflow(name)["on"] + for trigger in ("push", "pull_request"): + assert "paths" not in events[trigger] + assert "paths-ignore" not in events[trigger] + assert "paths" not in workflow("delivery-release.yml")["on"]["push"] + + +@pytest.mark.parametrize("name", EXPECTED) +def test_mainline_only_push_triggers_after_p1_merge(name): + """Keep retired development branches out of release-gating workflows.""" + config = workflow(name) + assert "push" in config["on"] + branches = config["on"]["push"].get("branches", []) + assert "main" in branches, name + assert "feature/docker-webui-multiaccount" not in branches, name + + +def test_live_multiworker_runs_on_main_every_push(): + """Multiworker runs on every main push without any opt-in marker.""" + worker = workflow("live-telegram-multiworker.yml") + triple = worker["jobs"]["three-process-telegram"] + assert "if" not in triple # no opt-in marker required for P1 acceptance + assert triple["concurrency"]["cancel-in-progress"] == "false" + + +def test_reusable_docker_fault_injection_workflow_contract(): + """Independent Docker fault injection persists across roadmap phases.""" + assert sorted(p.name for p in WORKFLOWS.glob("*.yml")) == sorted([*EXPECTED, "endurance-capacity.yml"]) + doc = workflow("resilience-docker.yml") + assert doc["name"] == "03 · Resilience | Docker Fault Injection" + assert "workflow_dispatch" in doc["on"] + assert "push" in doc["on"] and "pull_request" in doc["on"] + assert "paths" not in doc["on"]["push"] + assert doc["permissions"]["contents"] == "read" + commands = "\n".join(s.get("run", "") for s in + doc["jobs"]["disposable-docker-fault-injection"]["steps"]) + assert "scripts/dev/run_docker_fault_injection.sh" in commands + assert (ROOT / ".github/ci/fault-injection/compose.yml").is_file() + assert (ROOT / ".github/ci/fault-injection/Dockerfile").is_file() + assert (ROOT / ".github/ci/fault-injection/Dockerfile.dockerignore").is_file() + assert (ROOT / "scripts/dev/docker_fault_scenarios.py").is_file() + assert (ROOT / "docs/CI_WORKFLOWS.md").is_file() + assert "DEV-13" in (ROOT / "docs/CI_WORKFLOWS.md").read_text(encoding="utf-8") + + +def test_docker_fault_config_has_every_required_runtime_upload_field(): + """A running HTTP healthcheck does not prove the WebUI config can load. + + Catches the actual Docker DEV regression from 1f32159, where the + generated config omitted sleep_min_seconds and storage health returned + HTTP 500 (while both /api/health probes remained green). + """ + source = (ROOT / "teledrive" / "uploader.py").read_text(encoding="utf-8") + harness = (ROOT / "scripts" / "dev" / "run_docker_fault_injection.sh").read_text( + encoding="utf-8", + ) + required = set(re.findall(r'up\.get\("([^"]+)"', source)) + required |= set(re.findall(r'up\["([^"]+)"\]', source)) + assert required + missing = sorted(field for field in required if field + ":" not in harness) + assert not missing, f"Docker fault config missing required upload fields: {missing}" + # Verify the ACTUAL production config parser in each clean Web container, + # not only the permissive Uvicorn /api/health startup probe. + assert "for node in web-a web-b" in harness + assert 'load_config("/config/config.yaml")' in harness + assert 'exec -T "$node"' in harness + + +def test_disposable_docker_dev_acceptance_has_real_restart_and_restore_gates(): + """Verify the acceptance runner does not regress into metadata-only mocks.""" + runner = (ROOT / "scripts" / "dev" / "run_docker_fault_injection.sh").read_text( + encoding="utf-8", + ) + cases = (ROOT / "scripts" / "dev" / "docker_fault_scenarios.py").read_text( + encoding="utf-8", + ) + assert 'kill -s SIGKILL web-a' in runner + assert '"${COMPOSE[@]}" restart postgres' in runner + assert '"${COMPOSE[@]}" run -T --rm scenarios restarted' in runner + assert '"${COMPOSE[@]}" run -T --rm scenarios restored' in runner + assert '"${COMPOSE[@]}" run -T --rm scenarios acceptance' in runner + assert '{"seed", "restarted", "restored", "acceptance"}' in runner + assert 'docker_removed' in runner and 'and docker_removed' in runner + assert 'checks_count") == 13' in runner + assert '"DEV-13"' in cases and '"DEV-11"' in cases + assert 'InterruptedDownload' in cases + assert 'raise TimeoutError("injected transport disconnect")' in cases + assert 'missing.replace(media)' in cases + assert 'tempfile' not in runner or 'docker compose' in runner + + +def test_disposable_dev_acceptance_does_not_require_live_telegram_credentials(): + """No production secret may be mounted inside disposable Compose.""" + compose = (ROOT / ".github/ci/fault-injection/compose.yml").read_text( + encoding="utf-8", + ) + runner = (ROOT / "scripts" / "dev" / "run_docker_fault_injection.sh").read_text( + encoding="utf-8", + ) + for private in ("TELEDRIVE_E2E_SESSION", "TELEDRIVE_E2E_API_HASH", + "TELEDRIVE_STORAGE_MTPROTO_API_HASH"): + assert private not in compose + assert 'if [[ -n "${TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE:-}"' in runner + assert 'down -v --remove-orphans' in runner + assert 'production_acceptance_signed_off": False' in runner + + + +def test_release_publish_marker_requires_main_and_same_sha_guard(): + release = workflow("delivery-release.yml") + build = release["jobs"]["build"] + publish = release["jobs"]["publish"] + for job in (build, publish): + assert "[publish-release]" in job["if"] + assert "refs/heads/main" in job["if"] + assert "github.ref_type == 'tag'" in job["if"] + assert publish["needs"] == "build" + script = "\\n".join(step.get("run", "") for step in publish["steps"]) + assert "scripts/ops/check_release_matrix.py" in script + assert "gh release create" in script + + +def test_canonical_docs_and_generated_release_notes(): + """Split legacy status files must not be recreated or break the release.""" + for name in ( + "DOCKER_FAULT_INJECTION.md", + "P0_ACCEPTANCE.md", + "P1_DEVELOPMENT_ACCEPTANCE.md", + "P1_PRODUCTION_ACCEPTANCE.md", + "P2_RECOVERY_AND_GC.md", + "RELEASE_NOTES_v1.0.7.md", + ): + assert not (ROOT / "docs" / name).exists(), name + for name, required_sections in ( + ("CI_WORKFLOWS.md", ("DEV-01", "DEV-13", "resilience-docker.yml")), + ("IMPLEMENTATION_STATUS.md", ("P0", "P1", "P2", "verify_production_bundle.py")), + ): + content = (ROOT / "docs" / name).read_text(encoding="utf-8") + assert all(section in content for section in required_sections), name + text = (WORKFLOWS / "delivery-release.yml").read_text(encoding="utf-8") + assert "--generate-notes" in text + assert "--notes-file" not in text + + +def test_explicit_dependencies_and_isolated_docker_build_contexts(): + """Keep desktop GUI out of production API image and keep CI Docker isolated.""" + server = [line.strip().lower() for line in + (ROOT / "requirements.txt").read_text("utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#")] + assert any(line.startswith("fastapi") for line in server) + assert not any(line.startswith("pyside6") for line in server) + web = (ROOT / "Dockerfile").read_text("utf-8") + isolated = (ROOT / ".github/ci/fault-injection/Dockerfile").read_text("utf-8") + compose = (ROOT / ".github/ci/fault-injection/compose.yml").read_text("utf-8") + assert "requirements.txt" in web and "requirements.txt" in isolated + assert "context: ../../.." in compose + assert "dockerfile: .github/ci/fault-injection/Dockerfile" in compose + prod_compose = yaml.safe_load((ROOT / "docker-compose.yml").read_text("utf-8")) + assert list(prod_compose["services"]) == ["teledrive"] + service = prod_compose["services"]["teledrive"] + assert "profiles" not in service + assert len(service["ports"]) == 2 + assert any(":8080" in entry for entry in service["ports"]) + assert any(":9000" in entry for entry in service["ports"]) + assert service["environment"] == { + "TELEDRIVE_CONFIG": "/config/config.yaml", + "TZ": "${TZ:-Europe/Istanbul}", + } + assert service.get("healthcheck", {}).get("test") + assert service.get("logging", {}).get("driver") == "json-file" + assert "web.unified" in (ROOT / "Dockerfile").read_text("utf-8") + assert (ROOT / "docs/SECURITY.md").is_file() + assert not (ROOT / "SECURITY.md").exists() + + +def test_pip_cache_keys_follow_explicit_target_requirements(): + """Renamed manifest paths must be used by setup-python caching too.""" + for workflow_name in ( + "quality-security.yml", "integration-storage.yml", + "integration-databases.yml", + "delivery-release.yml", + ): + job = next(iter(workflow(workflow_name)["jobs"].values())) + setup = next(s for s in job["steps"] if s.get("name") == "Set up Python") + expected = "requirements.txt" + assert setup["with"]["cache-dependency-path"] == expected + endurance = workflow("endurance-capacity.yml") + assert endurance["name"] == "06 · Endurance | Isolated Capacity Soak" + assert "workflow_dispatch" in endurance["on"] + + +def test_general_purpose_test_file_names_not_roadmap_phases(): + """Test names describe behavior rather than internal rollout priorities.""" + for path in (ROOT / "tests").glob("test_*.py"): + assert not re.match(r"test_(?:p0|p1|p2|s3_p0)(?:_|\\.)", path.name), path.name + for script in (ROOT / "scripts").rglob("*.py"): + assert not re.match(r"(?:p0|p1|p2)_", script.name), script.name + + +def test_public_markdown_is_english_structured_and_locally_linked(): + """Keep public guides consistent after the documentation consolidation.""" + from urllib.parse import unquote + + guides = ( + "README.md", + "CONTRIBUTING.md", + "TODO.md", + "docs/CI_WORKFLOWS.md", + "docs/S3_SPECS.md", + "docs/IMPLEMENTATION_STATUS.md", + "docs/BACKUP_RESTORE.md", + "docs/SECURITY.md", + ) + turkish_glyphs = re.compile(r"[ğĞıİşŞçÇöÖüÜ]") + for name in guides: + path = ROOT / name + content = path.read_text(encoding="utf-8") + # Markdown fence contents may contain shell comments beginning '#'. + # Those are source code, not document headings. + prose_lines = [] + fenced = False + for line in content.splitlines(): + if line.startswith(("~~~", "```")): + fenced = not fenced + continue + if not fenced: + prose_lines.append(line) + prose = "\n".join(prose_lines) + headings = re.findall(r"(?m)^#{1,6} (.+)$", prose) + assert content.startswith("# ") and len( + re.findall(r"(?m)^# ", prose) + ) == 1, name + assert sum(heading[0].isdigit() for heading in headings[1:]) >= 3, name + assert not turkish_glyphs.search(content), name + + for target in re.findall(r"!?\[[^\]]*\]\(([^)]+)\)", content): + target = target.split("#", 1)[0].strip() + if not target or target.startswith( + ("http://", "https://", "mailto:", "data:") + ): + continue + resolved = (path.parent / unquote(target)).resolve() + assert resolved.is_relative_to(ROOT), (name, target) + assert resolved.exists(), (name, target) + + readme = (ROOT / "README.md").read_text(encoding="utf-8") + assert "docker compose up -d --build" in readme + assert "one container" in readme.lower() + assert "a separate Docker Compose profile" not in readme + assert not (ROOT / "website").exists() + +def test_gc_references_and_live_inventory_are_ci_gates(): + for name in ("quality-security.yml", "integration-storage.yml"): + all_commands = "\\n".join( + step.get("run", "") + for job in workflow(name)["jobs"].values() for step in job["steps"] + ) + assert "tests/storage/test_gc_reference_planner.py" in all_commands + user = workflow("live-telegram-user.yml")["jobs"]["telegram-roundtrip"] + assert any("tests/e2e/test_telegram_history_live_e2e.py" in s.get("run", "") + for s in user["steps"]) + +def test_offline_inventory_reconciliation_is_required_in_ci(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(step.get("run", "") + for job in workflow(name)["jobs"].values() for step in job["steps"]) + assert "tests/storage/test_telegram_inventory_reconcile.py" in commands + +def test_bucket_versioning_controls_are_exercised_in_all_storage_ci(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(s.get("run", "") for job in workflow(name)["jobs"].values() + for s in job["steps"]) + assert "tests/s3/test_s3_bucket_versioning.py" in commands + +def test_immutable_public_access_controls_have_actual_s3_regression_coverage(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(s.get("run", "") for j in workflow(name)["jobs"].values() + for s in j["steps"]) + assert "tests/s3/test_s3_public_access_block.py" in commands + +def test_single_container_passes_opt_in_legacy_ingest_configuration(): + import yaml + compose = yaml.safe_load((ROOT / "docker-compose.yml").read_text()) + services = compose["services"] + assert list(services) == ["teledrive"] + # Service settings live in config.yaml only; the Compose environment + # carries just the config path and the timezone. + env = services["teledrive"]["environment"] + assert "TELEDRIVE_CONFIG" in env and "TZ" in env + assert "TELEDRIVE_JOB_AUTO_RUN" not in env + example = (ROOT / "config.example.yaml").read_text() + for key in ( + "auto_run", + ): + assert key in example + assert "jobs:" in example + assert "tests/web/test_unified_service.py" in ( + WORKFLOWS / "integration-storage.yml" + ).read_text() + +def test_real_s3_lifecycle_has_single_process_schedule_and_ci(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(s.get("run","") for job in workflow(name)["jobs"].values() + for s in job["steps"]) + assert "tests/s3/test_s3_lifecycle.py" in commands + assert "lifecycle_scheduler_loop()" in (ROOT / "web/app.py").read_text() + +def test_s3_compatibility_matrix_cannot_relabel_safe_lifecycle_as_full_aws(): + specs = (ROOT / "docs/S3_SPECS.md").read_text() + assert "**Inventory totals:** 11 supported, 27 partial, 78 unsupported;" in specs + for action in ( + "GetBucketLifecycleConfiguration", "PutBucketLifecycleConfiguration", + "DeleteBucketLifecycle", + ): + prefix = "| [" + action + "](https://docs.aws.amazon.com" + matches = [line for line in specs.splitlines() if line.startswith(prefix)] + assert len(matches) == 1 + assert "🟡 Partial" in matches[0] + assert "AbortIncompleteMultipartUpload" in specs + assert "never deletes an object or Telegram media" in specs + +def test_s3_fail_closed_116_operation_manifest_suite_is_required(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(s.get("run", "") + for job in workflow(name)["jobs"].values() for s in job["steps"]) + assert "tests/s3/test_s3_route_guard.py" in commands + +def test_private_bucket_policy_status_regression_is_mandatory(): + for name in ("quality-security.yml", "integration-storage.yml"): + source = "\n".join(s.get("run", "") + for job in workflow(name)["jobs"].values() for s in job["steps"]) + assert "tests/s3/test_s3_policy_status.py" in source + + +def test_todo_has_real_markdown_checklist_line_breaks_and_s3_tagging_contract(): + text = (ROOT / "TODO.md").read_text() + assert r"\n- [x]" not in text + assert r"\n- [ ]" not in text + assert "- [x] Complete AWS-compatible PutBucketTagging" in text + specs = (ROOT / "docs/S3_SPECS.md").read_text() + assert r"\n\n**Implementation evidence:**" not in specs + target = "| [PutBucketTagging](https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketTagging.html)" + assert len([line for line in specs.splitlines() if line.startswith(target)]) == 1 + assert any(line.startswith(target) and "✅ Supported" in line + for line in specs.splitlines()) + import json + rows = json.loads((ROOT / "teledrive/s3/operation_manifest.json").read_text())["operations"] + assert next(x["status"] for x in rows if x["name"] == "PutBucketTagging") == "supported" + +def test_bucket_tagging_promoted_contract_is_required_in_both_regression_ci(): + for name in ("quality-security.yml", "integration-storage.yml"): + text = "\n".join(step.get("run", "") + for job in workflow(name)["jobs"].values() for step in job["steps"]) + assert "tests/s3/test_s3_tagging.py" in text + + +def test_paginated_bucket_listing_contract_is_required_in_both_ci_suites(): + for name in ("quality-security.yml", "integration-storage.yml"): + commands = "\n".join(s.get("run", "") + for job in workflow(name)["jobs"].values() for s in job["steps"]) + assert "tests/s3/test_s3_bucket_listing.py" in commands + specs = (ROOT / "docs/S3_SPECS.md").read_text() + action = "| [ListBuckets](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListBuckets.html)" + assert len([line for line in specs.splitlines() if line.startswith(action)]) == 1 + assert any(line.startswith(action) and "✅ Supported" in line + for line in specs.splitlines()) + import json + manifest = json.loads((ROOT / "teledrive/s3/operation_manifest.json").read_text()) + assert next(x["status"] for x in manifest["operations"] if x["name"] == "ListBuckets") == "supported" + + +def test_s3_specs_markdown_tables_render_as_complete_gfm_tables(): + """All GitHub-flavored Markdown tables need contiguous rows and delimiter.""" + content = (ROOT / "docs/S3_SPECS.md").read_text(encoding="utf-8") + lines = content.splitlines() + for artifact in (r"\n\n", r"\n|", r"\n- "): + assert artifact not in content, ("literal escaped newline", artifact) + groups = [] + current = [] + for lineno, line in enumerate(lines, 1): + if line.startswith("|"): + current.append((lineno, line)) + elif current: + groups.append(current) + current = [] + if current: + groups.append(current) + assert groups, "S3 compatibility tables are missing" + for group in groups: + start = group[0][0] + assert len(group) >= 2, ("orphan Markdown table row", start) + columns = group[0][1].count("|") + separator_cells = group[1][1].strip().strip("|").split("|") + assert len(separator_cells) == columns - 1 and all( + re.fullmatch(r":?-{3,}:?", cell.strip()) + for cell in separator_cells + ), ("missing GitHub-flavored Markdown table delimiter", start) + assert all(row.count("|") == columns for _, row in group), ( + "mismatched table columns", start + ) + # Supported/partial operation table must remain ONE contiguous table. + table = next(group for group in groups if group[0][1].startswith( + "| AWS S3 API operation | Status | Implemented scope" + )) + entries = [row for _, row in table[2:]] + for operation in ("GetBucketLifecycleConfiguration", + "PutBucketLifecycleConfiguration", + "DeleteBucketLifecycle"): + assert any(f"[{operation}]" in row for row in entries), operation + assert sum( + line.startswith("| [") and + "](https://docs.aws.amazon.com/AmazonS3/latest/API/" in line + for line in lines + ) == 116 + + +def test_merged_branch_cleanup_requires_exact_pr_merge_and_nonprotected_ref(): + delivery = workflow("delivery-release.yml") + cleanup = delivery["jobs"]["cleanup-merged-branches"] + assert cleanup["needs"] == "verify" + assert cleanup["permissions"]["contents"] == "write" + assert cleanup["permissions"]["pull-requests"] == "read" + assert "refs/heads/main" in cleanup["if"] + assert "pull_request" not in cleanup["if"] + commands = "\n".join(step.get("run", "") for step in cleanup["steps"]) + assert "scripts/ops/cleanup_merged_branches.py --execute" in commands + script = (ROOT / "scripts" / "ops" / "cleanup_merged_branches.py").read_text(encoding="utf-8") + for required in ( + "merged_at", "base", "main", "protected", "state=open", + "head", "sha", "--execute", "GITHUB_REPOSITORY", + "git/ref/heads/", "git/refs/heads/", + ): + assert required in script + assert "branch -D" not in script and "git push --delete" not in script diff --git a/tests/meta/test_host_soak.py b/tests/meta/test_host_soak.py new file mode 100644 index 00000000..2c4d586a --- /dev/null +++ b/tests/meta/test_host_soak.py @@ -0,0 +1,135 @@ +"""P1 production telemetry gate: safe network configuration and no fake signoff.""" +from __future__ import annotations + +import pytest + +from scripts.ops.monitor_host_soak import ( + collect, parse_hosts, selected_metrics, summarize, +) + + +def fixture_metrics(ack_count=0, **overrides): + data = { + "bot_explicit_429_total": ack_count, + "unresolved_bot_send_count": 0, + "unjournaled_bot_ack_count": 0, + "stale_pending_sample_count": 0, + "stale_multipart_sample_count": 0, + "job_states": {"queued": 0, "running": 0, "failed": 0}, + "samples_truncated": False, + "physical_deletion_enabled": False, + } + data.update(overrides) + return data + + +def test_two_distinct_https_health_endpoints_required(): + values = parse_hosts( + '[{"label":"west","url":"https://west.example.org"},' + '{"label":"east","url":"https://east.example.org/"}]' + ) + assert [host["label"] for host in values] == ["west", "east"] + assert [host["origin"] for host in values] == [ + "https://west.example.org", "https://east.example.org", + ] + + +@pytest.mark.parametrize("input_value", [ + "[]", + '[{"label":"alone","url":"https://a.example.org"}]', + '[{"label":"one","url":"http://a.example.org"},' + '{"label":"two","url":"https://b.example.org"}]', + '[{"label":"one","url":"https://user:pass@a.example.org"},' + '{"label":"two","url":"https://b.example.org"}]', + '[{"label":"one","url":"https://a.example.org/redirect"},' + '{"label":"two","url":"https://b.example.org"}]', + '[{"label":"one","url":"https://a.example.org"},' + '{"label":"one","url":"https://b.example.org"}]', + '[{"label":"one","url":"https://a.example.org"},' + '{"label":"two","url":"https://A.EXAMPLE.ORG"}]', + '[{"label":"one","url":"https://a.example.org?secret=1"},' + '{"label":"two","url":"https://b.example.org"}]', +]) +def test_unsafe_host_configuration_fails_closed(input_value): + with pytest.raises(ValueError): + parse_hosts(input_value) + + +@pytest.mark.parametrize("bad", [ + {"bot_explicit_429_total": True}, + {"bot_explicit_429_total": -1}, + {"job_states": {"queued": "one"}}, + {"samples_truncated": True}, + {"physical_deletion_enabled": True}, +]) +def test_unsafe_metrics_never_count_as_accepted(bad): + with pytest.raises(ValueError): + selected_metrics(fixture_metrics(**bad)) + + +def test_hour_soak_simulation_observes_shared_counter_without_claiming_signoff(): + # CI only compresses the telemetry loop; CLI enforces a real hour. + hosts = [ + {"label": "west", "origin": "https://west.example.org"}, + {"label": "east", "origin": "https://east.example.org"}, + ] + clock_value = [0.0] + + def clock(): + return clock_value[0] + + def sleep(seconds): + clock_value[0] += seconds + + def probe(_origin): + return fixture_metrics(3 if clock_value[0] >= 2 else 1) + + report = collect(hosts, probe, duration=4, interval=2, clock=clock, sleep=sleep) + assert report["telemetry_gate_passed"] is True + assert report["sample_count"] == 3 + assert report["explicit_429_deltas_by_host"] == {"west": 2, "east": 2} + assert report["production_acceptance_signed_off"] is False + assert report["physical_host_independence_verified"] is False + assert report["original_media_restore_verified"] is False + assert report["legacy_cutover_approved"] is False + assert report["telegram_mutations_performed"] == 0 + assert "url" not in str(report) + assert "password" not in str(report) + + +@pytest.mark.parametrize("scenario", [ + "no-actual-429", + "second-host-disagrees", + "unresolved-bot-ack", + "backlog-not-drained", + "counter-decreased", + "missed-probe", +]) +def test_unsafe_soak_evidence_must_not_pass(scenario): + def observation(time, left=1, right=1, **changes): + a = selected_metrics(fixture_metrics(left, **changes)) + b = selected_metrics(fixture_metrics(right, **changes)) + return {"elapsed": time, "hosts": {"west": a, "east": b}} + + beginning = observation(0) + ending = observation(3600, left=3, right=3) + if scenario == "no-actual-429": + ending = observation(3600) + elif scenario == "second-host-disagrees": + ending = observation(3600, left=3, right=2) + elif scenario == "unresolved-bot-ack": + ending = observation(3600, left=3, right=3, unresolved_bot_send_count=1) + elif scenario == "backlog-not-drained": + ending = observation(3600, left=3, right=3, + job_states={"queued": 1, "running": 0, "failed": 0}) + elif scenario == "counter-decreased": + beginning = observation(0, left=3, right=3) + ending = observation(3600, left=2, right=2) + samples = [beginning, ending] + if scenario == "missed-probe": + with pytest.raises(ValueError, match="Unobserved gap"): + summarize(samples, duration=3600, interval=60) + else: + summary = summarize(samples, duration=3600, interval=3600) + assert summary["telemetry_gate_passed"] is False + assert summary["production_acceptance_signed_off"] is False diff --git a/tests/meta/test_production_bundle.py b/tests/meta/test_production_bundle.py new file mode 100644 index 00000000..dacdad9f --- /dev/null +++ b/tests/meta/test_production_bundle.py @@ -0,0 +1,95 @@ +"""Signed production evidence alone never asserts deployment acceptance.""" +from __future__ import annotations +import base64 +import copy +from hashlib import sha256 +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, PublicFormat +from scripts.ops.verify_production_bundle import canonical_payload, verify_bundle + +COMMIT = "a" * 40 + + +def fixture(): + one, two = Ed25519PrivateKey.generate(), Ed25519PrivateKey.generate() + pubs = tuple(k.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + for k in (one, two)) + hashed = lambda text: sha256(text.encode()).hexdigest() + report = { + "release_sha": COMMIT, + "original_telegram_data_attested": True, + "no_legacy_source_attested": True, + "topology": { + "independent_hosts_owner_attested": True, + "hosts": [ + {"label": label, "machine_id_sha256": hashed(label), + "shared_database_sha256": hashed("same-sql-db")} + for label in ("host-a", "host-b") + ], + }, + "soak": { + "telemetry_gate_passed": True, "duration_seconds": 3600, + "host_labels": ["host-a", "host-b"], + "final_shared_counter_agreement": True, + "explicit_429_deltas_by_host": {"host-a": 1, "host-b": 1}, + "telegram_mutations_performed": 0, + }, + "dr": { + "verified": True, "source_database_modified": False, + "telegram_uploads": 0, "telegram_deletions": 0, + "archive_sha256": hashed("backup"), "sha256": hashed("original"), + "size": 100, "readback_bytes": 100, + }, + "cutover": {"not_applicable": True}, + } + def sign(entry): + entry = copy.deepcopy(entry) + raw = canonical_payload(entry) + entry["approvals"] = { + role: base64.b64encode(key.sign(raw)).decode() + for role, key in (("owner", one), ("operator", two)) + } + return entry + return report, pubs, sign + + +def test_complete_attested_bundle_is_not_automatic_production_signoff(): + data, keys, sign = fixture() + result = verify_bundle(sign(data), expected_sha=COMMIT, + owner_public_key=keys[0], operator_public_key=keys[1]) + assert result["technical_evidence_verified"] + assert result["production_acceptance_signed_off"] is False + assert result["physical_telegram_deletion_enabled"] is False + + +@pytest.mark.parametrize("change", [ + lambda d: d["topology"]["hosts"][1].update(machine_id_sha256=d["topology"]["hosts"][0]["machine_id_sha256"]), + lambda d: d["soak"].update(duration_seconds=3599), + lambda d: d["soak"].update(telemetry_gate_passed=False), + lambda d: d["dr"].update(sha256="0"), + lambda d: d["cutover"].update(not_applicable=False), + lambda d: d.update(original_telegram_data_attested=False), + lambda d: d.update(no_legacy_source_attested=False), +]) +def test_invalid_production_receipts_rejected_even_if_approved(change): + data, keys, sign = fixture() + change(data) + with pytest.raises(ValueError): + verify_bundle(sign(data), expected_sha=COMMIT, + owner_public_key=keys[0], operator_public_key=keys[1]) + + +def test_modified_bundle_or_reused_owner_key_fails_closed(): + data, keys, sign = fixture() + signed = sign(data) + signed["soak"]["duration_seconds"] += 1 + with pytest.raises(ValueError): + verify_bundle(signed, expected_sha=COMMIT, + owner_public_key=keys[0], operator_public_key=keys[1]) + with pytest.raises(ValueError): + verify_bundle(sign(data), expected_sha=COMMIT, + owner_public_key=keys[0], operator_public_key=keys[0]) + with pytest.raises(ValueError): + verify_bundle(sign(data), expected_sha="b"*40, + owner_public_key=keys[0], operator_public_key=keys[1]) diff --git a/tests/meta/test_release_matrix.py b/tests/meta/test_release_matrix.py new file mode 100644 index 00000000..3e100f2b --- /dev/null +++ b/tests/meta/test_release_matrix.py @@ -0,0 +1,26 @@ +from scripts.ops.check_release_matrix import REQUIRED, check + +def sample(status="completed", conclusion="success", sha="abc"): + return {"workflow_runs": [ + {"name": name, "head_sha": sha, "id": i, "created_at": "2026-10-14T00:00:00Z", + "status": status, "conclusion": conclusion} + for i, name in enumerate(sorted(REQUIRED)) + ]} + +def test_all_completed_success(): + assert check(sample(), "abc") == (0, []) + +def test_skipped_failed_cancelled_are_blocked(): + for outcome in ("skipped", "failure", "cancelled"): + assert check(sample(conclusion=outcome), "abc")[0] == 1 + +def test_wrong_sha_and_unfinished_cannot_release(): + assert check(sample(sha="wrong"), "abc")[0] == 2 + assert check(sample(status="in_progress", conclusion=None), "abc")[0] == 2 + +def test_newest_same_sha_run_wins(): + obj = sample() + obj["workflow_runs"].append({"name": sorted(REQUIRED)[0], "head_sha": "abc", + "id": 999, "created_at": "2026-10-14T01:00:00Z", + "status": "completed", "conclusion": "failure"}) + assert check(obj, "abc")[0] == 1 diff --git a/tests/s3/__init__.py b/tests/s3/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/s3/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/s3/test_s3_additional_checksums.py b/tests/s3/test_s3_additional_checksums.py new file mode 100644 index 00000000..76e5e151 --- /dev/null +++ b/tests/s3/test_s3_additional_checksums.py @@ -0,0 +1,196 @@ +"""Signed checksums are validated independently before Telegram uploads.""" +from __future__ import annotations + +import base64 +import hashlib +import zlib + +import pytest + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.checksums import crc32c, crc64nvme +from tests.s3.test_s3_multipart import begin + + +def _b64(raw): + return base64.b64encode(raw).decode("ascii") + + +def test_valid_sha1_crc32_and_sha256_headers_bind_object_bytes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"sha1-crc32-integrity" * 19 + headers = { + "x-amz-checksum-sha1": _b64(hashlib.sha1(data, usedforsecurity=False).digest()), + "x-amz-checksum-sha256": _b64(hashlib.sha256(data).digest()), + "x-amz-checksum-crc32": _b64((zlib.crc32(data) & 0xffffffff).to_bytes(4, "big")), + } + result = signed(service, "PUT", "/backup-bucket/checked.bin", data, + extra_headers=headers) + assert result.status_code == 200, result.text + assert signed(service, "GET", "/backup-bucket/checked.bin").content == data + + +@pytest.mark.parametrize("bad_header", [ + {"x-amz-checksum-sha1": _b64(b"0" * 20)}, + {"x-amz-checksum-sha256": _b64(b"0" * 32)}, + {"content-md5": _b64(b"0" * 16)}, + {"x-amz-sdk-checksum-algorithm": "SHA1"}, +]) +def test_bad_extra_checksums_fail_even_when_signed_body_sha256_is_correct( + service, bad_header +): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + before = service[2].uploads + denied = signed(service, "PUT", "/backup-bucket/never.bin", b"valid-sha256", + extra_headers=bad_header) + assert denied.status_code == 400, denied.text + assert b"BadDigest" in denied.content + assert service[2].uploads == before + + +def test_unknown_sdk_checksum_algorithm_does_not_silently_succeed(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + before = service[2].uploads + result = signed(service, "PUT", "/backup-bucket/no-false-ack", b"content", + extra_headers={"x-amz-sdk-checksum-algorithm": "CRC64NOTREAL"}) + assert result.status_code == 501, result.text + assert b"NotImplemented" in result.content + assert service[2].uploads == before + + +def test_bad_multipart_checksum_does_not_contact_telegram(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + before = service[2].uploads + response = signed(service, "PUT", + f"/backup-bucket/large.bin?uploadId={uid}&partNumber=1", + b"valid", extra_headers={ + "x-amz-checksum-sha1": _b64(b"0" * 20), + }) + # This uploadId targets a different key from the default begin() key; + # validate on the matching path in the next request instead. + assert response.status_code == 404 + response = signed(service, "PUT", + f"/backup-bucket/asset.bin?uploadId={uid}&partNumber=1", + b"valid", extra_headers={ + "x-amz-checksum-sha1": _b64(b"0" * 20), + }) + assert response.status_code == 400 + assert service[2].uploads == before + + +def test_crc32c_known_vector_and_incremental_streaming(): + assert crc32c(b"123456789") == 0xE3069283 + assert crc32c(b"") == 0 + assert crc32c(b"456789", crc32c(b"123")) == 0xE3069283 + assert crc32c(b"second", crc32c(b"first")) == crc32c(b"firstsecond") + with pytest.raises(ValueError): + crc32c(b"x", prior=-1) + + +def test_crc32c_signed_put_with_sdk_algorithm(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + content = (b"Castagnoli stream integrity" * 189) + b"tail" + checksum = _b64(crc32c(content).to_bytes(4, "big")) + answer = signed(service, "PUT", "/backup-bucket/crc32c.bin", content, + extra_headers={"x-amz-checksum-crc32c": checksum, + "x-amz-sdk-checksum-algorithm": "CRC32C"}) + assert answer.status_code == 200, answer.text + assert signed(service, "GET", "/backup-bucket/crc32c.bin").content == content + + +def test_crc32c_mismatch_prevents_telegram_put(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + before = service[2].uploads + denied = signed(service, "PUT", "/backup-bucket/no-crc.bin", b"bound-sha256", + extra_headers={"x-amz-checksum-crc32c": _b64(b"\0" * 4)}) + assert denied.status_code == 400 and b"BadDigest" in denied.content + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/no-crc.bin").status_code == 404 + + +def test_crc32c_multipart_part_roundtrip_and_negative(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + content = b"multipart-crc32c-part" * 33 + path = f"/backup-bucket/asset.bin?uploadId={uid}&partNumber=1" + checksum = {"x-amz-checksum-crc32c": _b64(crc32c(content).to_bytes(4, "big"))} + before = service[2].uploads + rejected = signed(service, "PUT", path, content + b"-mutated", + extra_headers=checksum) + assert rejected.status_code == 400 and b"BadDigest" in rejected.content + assert service[2].uploads == before + valid = signed(service, "PUT", path, content, extra_headers=checksum) + assert valid.status_code == 200, valid.text + assert service[2].uploads == before + 1 + + +def test_crc64nvme_known_vector_and_incremental_streaming(): + assert crc64nvme(b"123456789") == 0xAE8B14860A799888 + assert crc64nvme(b"") == 0 + assert crc64nvme(b"456789", crc64nvme(b"123")) == 0xAE8B14860A799888 + assert crc64nvme(b"second", crc64nvme(b"first")) == crc64nvme(b"firstsecond") + for invalid in (-1, 1 << 64): + with pytest.raises(ValueError): + crc64nvme(b"x", prior=invalid) + + +def test_crc64nvme_signed_put_verifies_and_preserves_bytes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = bytes(range(256)) * 13 + b"nvme-check" + declared = _b64(crc64nvme(data).to_bytes(8, "big")) + uploaded = signed(service, "PUT", "/backup-bucket/crc64.bin", data, + extra_headers={"x-amz-checksum-crc64nvme": declared, + "x-amz-sdk-checksum-algorithm": "CRC64NVME"}) + assert uploaded.status_code == 200, uploaded.text + assert signed(service, "GET", "/backup-bucket/crc64.bin").content == data + + +@pytest.mark.parametrize("headers", [ + {"x-amz-checksum-crc64nvme": _b64(b"0" * 8)}, + {"x-amz-checksum-crc64nvme": "not-base64"}, + {"x-amz-sdk-checksum-algorithm": "CRC64NVME"}, + {"x-amz-checksum-crc64nvme": _b64(b"0" * 8), + "x-amz-checksum-crc32c": _b64(b"0" * 4)}, +]) +def test_crc64nvme_bad_or_missing_digest_never_uploads(service, headers): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + before = service[2].uploads + denied = signed(service, "PUT", "/backup-bucket/bad-crc64.bin", b"payload", + extra_headers=headers) + assert denied.status_code == 400 and b"BadDigest" in denied.content + assert service[2].uploads == before + assert signed(service, "HEAD", "/backup-bucket/bad-crc64.bin").status_code == 404 + + +def test_crc64nvme_multipart_part_accepts_correct_digest_only(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + upload_id = begin(service) + part = b"crc64-multipart" * 177 + path = f"/backup-bucket/asset.bin?uploadId={upload_id}&partNumber=1" + before = service[2].uploads + bad = signed(service, "PUT", path, part, + extra_headers={"x-amz-checksum-crc64nvme": _b64(b"X" * 8)}) + assert bad.status_code == 400 and b"BadDigest" in bad.content + assert service[2].uploads == before + good = signed(service, "PUT", path, part, + extra_headers={"x-amz-checksum-crc64nvme": + _b64(crc64nvme(part).to_bytes(8, "big")), + "x-amz-sdk-checksum-algorithm": "CRC64NVME"}) + assert good.status_code == 200, good.text + assert service[2].uploads == before + 1 + + +def test_crc64nvme_declared_on_metadata_xml_cannot_bypass_validation(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/tags.bin", b"data").status_code == 200 + before = service[2].uploads + wrong = {"x-amz-checksum-crc64nvme": _b64(b"0" * 8)} + # XML control-plane writes share the checksum declaration parser. + bad_cors = signed(service, "PUT", "/backup-bucket?cors", + b"", extra_headers=wrong) + assert bad_cors.status_code == 400 and b"BadDigest" in bad_cors.content + bad_tags = signed(service, "PUT", "/backup-bucket/tags.bin?tagging", + b"", extra_headers=wrong) + assert bad_tags.status_code == 400 and b"BadDigest" in bad_tags.content + assert service[2].uploads == before diff --git a/tests/s3/test_s3_admin.py b/tests/s3/test_s3_admin.py new file mode 100644 index 00000000..126f6d87 --- /dev/null +++ b/tests/s3/test_s3_admin.py @@ -0,0 +1,85 @@ +"""Admin S3 credential issuance contract and immediate revoke with signed S3 calls.""" +from __future__ import annotations + +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient +import pytest + +import web.app as webapp +import web.s3 as gateway +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore + + +@pytest.fixture +def admin(tmp_path, monkeypatch): + url = str(tmp_path / "metadata.db") + key = Fernet.generate_key() + monkeypatch.setattr(webapp, "db_connect", lambda: connect_database(url)) + monkeypatch.setattr(gateway, "db_connect", lambda: connect_database(url)) + monkeypatch.setattr(webapp, "load_master_key", lambda: key) + monkeypatch.setattr(gateway, "load_master_key", lambda: key) + monkeypatch.setattr(webapp, "WEB_USER", "testadmin") + monkeypatch.setattr(webapp, "WEB_PASSWORD", "secure-auth-password") + webapp.AUTH_FAILURES.clear() + # No ASGI lifespan: endpoint authorization contract is isolated from + # Docker startup paths (/config, /data) and Telegram scheduler. + client = TestClient(webapp.app, base_url="http://localhost") + try: + yield client, url, key + finally: + client.close() + + +def test_issuance_requires_basic_auth_csrf_and_one_time_secret(admin): + client, url, master = admin + path = "/api/v1/admin/s3-credentials" + payload = {"name": "SDK-backup", "permissions": ["s3:GetObject", "s3:ListBucket"]} + assert client.post(path, json=payload).status_code == 403 or client.post(path, json=payload).status_code == 401 + assert client.post(path, auth=("testadmin", "secure-auth-password"), + json=payload).status_code == 403 + issued = client.post(path, auth=("testadmin", "secure-auth-password"), + headers={"x-teledrive-csrf": "1"}, json=payload) + assert issued.status_code == 201, issued.text + result = issued.json() + assert result["secret_key"] + assert result["access_key"].startswith("TD") + listing = client.get(path, auth=("testadmin", "secure-auth-password")) + assert listing.status_code == 200 + rows = listing.json()["credentials"] + assert rows[0]["access_key"] == result["access_key"] + assert result["secret_key"] not in listing.text + connection = connect_database(url) + try: + assert S3CredentialStore(connection, master).resolve( + result["access_key"], "s3:GetObject" + ) == result["secret_key"] + finally: + connection.close() + revoke_path = f"{path}/{result['id']}" + assert client.delete(revoke_path, auth=("testadmin", "secure-auth-password")).status_code == 403 + assert client.delete(revoke_path, auth=("testadmin", "secure-auth-password"), + headers={"x-teledrive-csrf": "1"}).status_code == 200 + connection = connect_database(url) + try: + assert S3CredentialStore(connection, master).resolve( + result["access_key"], "s3:GetObject" + ) is None + finally: + connection.close() + + +def test_scope_validation_and_missing_master_key(admin, monkeypatch): + client, _, _ = admin + url = "/api/v1/admin/s3-credentials" + auth = ("testadmin", "secure-auth-password") + headers = {"x-teledrive-csrf": "1"} + invalid = client.post(url, auth=auth, headers=headers, + json={"name": "bad", "permissions": ["s3:AdminAll"]}) + assert invalid.status_code == 422 + monkeypatch.setattr(webapp, "load_master_key", lambda: (_ for _ in ()).throw( + RuntimeError("Missing S3 master key") + )) + missing = client.get(url, auth=auth) + assert missing.status_code == 503 + assert "Missing S3 master key" in missing.text diff --git a/tests/s3/test_s3_batch_delete.py b/tests/s3/test_s3_batch_delete.py new file mode 100644 index 00000000..0550f73e --- /dev/null +++ b/tests/s3/test_s3_batch_delete.py @@ -0,0 +1,212 @@ +"""S3 Multi-Object Delete: real signed HTTP, XML, scopes, and Telegram retention. + +DeleteObjects updates ONLY the current object heads. Its underlying Telegram +messages remain intact for reference-audited GC/version retention. +""" +from __future__ import annotations + +import base64 +import hashlib +import os +import socket +import threading +import time +import xml.etree.ElementTree as ET + +import boto3 +import pytest +import uvicorn +from botocore.config import Config + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +import web.s3 as gateway + +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def delete_xml(keys, *, quiet=False, version=None): + root = ET.Element("Delete") + if quiet: + ET.SubElement(root, "Quiet").text = "true" + for key in keys: + item = ET.SubElement(root, "Object") + ET.SubElement(item, "Key").text = key + if version is not None: + ET.SubElement(item, "VersionId").text = version + return ET.tostring(root, encoding="utf-8") + + +def batch(svc, body, *, md5=True, crc32=None, access=None, secret=None): + headers = {} + if md5: + headers["Content-MD5"] = base64.b64encode( + hashlib.md5(body, usedforsecurity=False).digest() + ).decode("ascii") + if crc32 is not None: + headers["x-amz-checksum-crc32"] = crc32 + return signed(svc, "POST", "/backup-bucket?delete", body, + extra_headers=headers, access=access, secret=secret) + + +def make_bucket(svc): + assert signed(svc, "PUT", "/backup-bucket").status_code == 200 + + +def make_objects(svc, names): + for key in names: + response = signed(svc, "PUT", "/backup-bucket/" + key, key.encode("utf-8")) + assert response.status_code == 200, response.text + + +def deleted_keys(result): + root = ET.fromstring(result.content) + return [child.findtext(NS+"Key") for child in root.findall(NS+"Deleted")] + + +def error_code(result): + return ET.fromstring(result.content).findtext("Code") + + +def test_batch_delete_only_unlinks_heads_and_retains_remote_parts(service): + make_bucket(service) + make_objects(service, ["a.bin", "folder/b.bin", "keep.bin"]) + before_remote = dict(service[2].parts) + uploads = service[2].uploads + payload = delete_xml(["folder/b.bin", "missing.bin", "a.bin"]) + result = batch(service, payload) + assert result.status_code == 200, result.text + assert deleted_keys(result) == ["folder/b.bin", "missing.bin", "a.bin"] + assert signed(service, "GET", "/backup-bucket/a.bin").status_code == 404 + assert signed(service, "GET", "/backup-bucket/folder/b.bin").status_code == 404 + kept = signed(service, "GET", "/backup-bucket/keep.bin") + assert kept.status_code == 200 and kept.content == b"keep.bin" + assert service[2].parts == before_remote + assert service[2].uploads == uploads + # Historical manifests remain reachable by immutable internal object ID. + db = service[3]() + try: + manifests = ManifestStore(db) + assert len(manifests.interrupted()) == 0 + assert any(body == b"a.bin" for body in before_remote.values()) + finally: + db.close() + again = batch(service, payload) + assert again.status_code == 200 + assert deleted_keys(again) == ["folder/b.bin", "missing.bin", "a.bin"] + + +def test_batch_delete_quiet_omits_successes(service): + make_bucket(service) + make_objects(service, ["first.bin", "other.bin"]) + result = batch(service, delete_xml(["first.bin", "absent.bin"], quiet=True)) + assert result.status_code == 200 + assert ET.fromstring(result.content).findall(NS+"Deleted") == [] + assert signed(service, "GET", "/backup-bucket/first.bin").status_code == 404 + assert signed(service, "GET", "/backup-bucket/other.bin").status_code == 200 + + +def test_corrupted_signed_body_and_bad_md5_rejected_before_any_delete(service): + make_bucket(service) + make_objects(service, ["target.bin"]) + legitimate = delete_xml(["target.bin"]) + # Valid signature/sha256 but malformed MD5. + bad_md5 = signed(service, "POST", "/backup-bucket?delete", legitimate, + extra_headers={"Content-MD5": "AAAA"}) + assert bad_md5.status_code == 400 + assert error_code(bad_md5) == "BadDigest" + missing_checksum = batch(service, legitimate, md5=False) + assert missing_checksum.status_code == 400 + assert error_code(missing_checksum) == "InvalidRequest" + assert signed(service, "GET", "/backup-bucket/target.bin").status_code == 200 + + +@pytest.mark.parametrize("payload", [ + b"]>&payload;", + b"target.bin", + delete_xml(["target.bin"], version="unknown-version"), + delete_xml(["target.bin"] * 1001), + b"", +]) +def test_invalid_batch_is_fully_rejected_before_mutation(service, payload): + make_bucket(service) + make_objects(service, ["target.bin"]) + res = batch(service, payload) + assert res.status_code in (400, 501) + assert error_code(res) in ("MalformedXML", "NotImplemented", "InvalidObjectName") + assert signed(service, "GET", "/backup-bucket/target.bin").status_code == 200 + + +def test_batch_denies_key_without_delete_scope_and_missing_bucket(service): + make_bucket(service) + make_objects(service, ["target.bin"]) + db = service[3]() + try: + reader = S3CredentialStore(db, service[4]).create( + "list-and-read", ["s3:ListBucket", "s3:GetObject"] + ) + finally: + db.close() + res = batch(service, delete_xml(["target.bin"]), + access=reader.access_key, secret=reader.secret_key) + assert res.status_code == 403 and error_code(res) == "AccessDenied" + assert signed(service, "GET", "/backup-bucket/target.bin").status_code == 200 + missing = signed(service, "POST", "/missing-bucket?delete", delete_xml(["x.bin"]), + extra_headers={"Content-MD5": base64.b64encode( + hashlib.md5(delete_xml(["x.bin"]), usedforsecurity=False).digest() + ).decode("ascii")}) + assert missing.status_code == 404 and error_code(missing) == "NoSuchBucket" + + +def test_boto3_delete_objects_with_real_http_and_crc32(service): + _, cred, remote, _, _ = service + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, + lifespan="off", log_level="error", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + client = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=cred.access_key, + aws_secret_access_key=cred.secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}, + ), + ) + client.create_bucket(Bucket="sdk-delete") + for key in ["one", "two", "retain"]: + client.put_object(Bucket="sdk-delete", Key=key, Body=key.encode()) + before_parts = dict(remote.parts) + res = client.delete_objects( + Bucket="sdk-delete", + Delete={"Objects": [{"Key": "one"}, {"Key": "two"}, {"Key": "absent"}], + "Quiet": False}, + ) + assert sorted(x["Key"] for x in res["Deleted"]) == ["absent", "one", "two"] + remaining = client.list_objects_v2(Bucket="sdk-delete") + assert [x["Key"] for x in remaining["Contents"]] == ["retain"] + assert client.get_object(Bucket="sdk-delete", Key="retain")["Body"].read() == b"retain" + assert remote.parts == before_parts + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() diff --git a/tests/s3/test_s3_bucket_listing.py b/tests/s3/test_s3_bucket_listing.py new file mode 100644 index 00000000..4ac69a4e --- /dev/null +++ b/tests/s3/test_s3_bucket_listing.py @@ -0,0 +1,237 @@ +"""Real SigV4 + SQL ListBuckets pagination (AWS S3 private-region contract). + +Object bytes never reach Telegram: ListBuckets accesses only SQL metadata. +Cross-key tampering and reuse of prefix/region cursors must fail closed. +""" +from __future__ import annotations + +import hashlib +import socket +import threading +import time +from xml.etree import ElementTree as ET +from urllib.parse import quote + +import boto3 +import pytest +import uvicorn +from botocore.config import Config +from botocore.exceptions import ClientError +from sqlalchemy import insert + +import web.s3 as gateway +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog, buckets +from teledrive.storage.manifest import ManifestStore +from tests.s3.test_s3_gateway import service, signed + +_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def _xml_bucket_names(reply) -> tuple[list[str], str | None]: + assert reply.status_code == 200, reply.text + root = ET.fromstring(reply.content) + assert root.tag == _NS + "ListAllMyBucketsResult" + return [el.text for el in root.findall(_NS + "Buckets/" + _NS + "Bucket/" + _NS + "Name")], ( + root.findtext(_NS + "ContinuationToken") + ) + + +def test_unpaginated_buckets_contain_sql_creation_date_and_stable_owner(service): + assert signed(service, "PUT", "/list-ci-apple").status_code == 200 + assert signed(service, "PUT", "/list-ci-banana").status_code == 200 + answer = signed(service, "GET", "/") + names, token = _xml_bucket_names(answer) + assert names == ["list-ci-apple", "list-ci-banana"] and token is None + root = ET.fromstring(answer.content) + owner_id = root.findtext(_NS + "Owner/" + _NS + "ID") + assert owner_id and len(owner_id) == 64 + created = root.findall(_NS + "Buckets/" + _NS + "Bucket/" + _NS + "CreationDate") + assert len(created) == 2 + assert all(date.text and date.text.endswith("Z") for date in created) + assert all(date.text[:4] == time.strftime("%Y", time.gmtime()) for date in created) + assert not root.findall(_NS + "Buckets/" + _NS + "Bucket/" + _NS + "BucketRegion") + paged = signed(service, "GET", "/?max-buckets=2") + assert ET.fromstring(paged.content).findtext(_NS + "Owner/" + _NS + "ID") == owner_id + assert [r.text for r in ET.fromstring(paged.content).findall( + _NS + "Buckets/" + _NS + "Bucket/" + _NS + "BucketRegion" + )] == ["us-east-1", "us-east-1"] + assert service[2].uploads == service[2].downloads == 0 + + +def test_s3_list_buckets_paginated_prefix_and_page_boundaries(service): + names = ["list-ci-alpha", "list-ci-beta", "list-ci-gamma", "other-bucket"] + for name in reversed(names): + assert signed(service, "PUT", "/" + name).status_code == 200 + requested = [] + token = None + for index in range(4): + query = "/?max-buckets=1&prefix=list-ci-&bucket-region=us-east-1" + if token: + query += "&continuation-token=" + quote(token, safe="") + result = signed(service, "GET", query) + found, next_token = _xml_bucket_names(result) + if index < 3: + assert len(found) == 1 + requested.extend(found) + token = next_token + if token is None: + break + assert requested == sorted([n for n in names if n.startswith("list-ci-")]) + assert token is None + assert _xml_bucket_names(signed(service, "GET", + "/?prefix=OTHER&max-buckets=1"))[0] == [] + assert _xml_bucket_names(signed(service, "GET", + "/?prefix=list-ci-&max-buckets=10000"))[0] == requested + + +def test_cursor_valid_with_concurrent_bucket_create_but_not_duplicate(service): + for name in ("list-ci-a", "list-ci-c"): + assert signed(service, "PUT", "/" + name).status_code == 200 + first, token = _xml_bucket_names( + signed(service, "GET", "/?prefix=list-ci-&max-buckets=1") + ) + assert first == ["list-ci-a"] and token + # Insert a record lexicographically after the consumed high-water mark. + assert signed(service, "PUT", "/list-ci-b").status_code == 200 + second, token2 = _xml_bucket_names(signed( + service, "GET", "/?max-buckets=1&prefix=list-ci-&continuation-token=" + + quote(token, safe=""), + )) + assert second == ["list-ci-b"] and token2 + last, token3 = _xml_bucket_names(signed( + service, "GET", "/?max-buckets=1&prefix=list-ci-&continuation-token=" + + quote(token2, safe=""), + )) + assert last == ["list-ci-c"] and token3 is None + + +def test_list_buckets_cursor_hmac_tamper_context_and_permission_scope(service): + for name in ("list-ci-a", "list-ci-b"): + assert signed(service, "PUT", "/" + name).status_code == 200 + _, token = _xml_bucket_names(signed(service, "GET", "/?max-buckets=1")) + assert token + db = service[3]() + try: + keys = S3CredentialStore(db, service[4]) + owner2 = keys.create("second-list-key", ["s3:ListAllMyBuckets"]) + read_only = keys.create("object-read-only", ["s3:GetObject"]) + finally: + db.close() + trials = [ + "/?max-buckets=1&continuation-token=" + quote(token + "tampered"), + "/?max-buckets=1&prefix=different&continuation-token=" + quote(token), + "/?max-buckets=1&bucket-region=us-east-2&continuation-token=" + quote(token), + ] + for path in trials: + denied = signed(service, "GET", path) + assert denied.status_code == 400, denied.text + assert b"Invalid" in denied.content + other = signed(service, "GET", "/?continuation-token=" + quote(token), + access=owner2.access_key, secret=owner2.secret_key) + assert other.status_code == 400 + assert signed(service, "GET", "/", access=owner2.access_key, + secret=owner2.secret_key).status_code == 200 + assert signed(service, "GET", "/", access=read_only.access_key, + secret=read_only.secret_key).status_code == 403 + assert service[0].get("/").status_code == 403 + assert service[2].uploads == service[2].downloads == 0 + + +@pytest.mark.parametrize("path,expected", [ + ("/?max-buckets=0", 400), + ("/?max-buckets=-1", 400), + ("/?max-buckets=10001", 400), + ("/?max-buckets=2.0", 400), + ("/?max-buckets=2&max-buckets=3", 400), + ("/?prefix=a&prefix=b", 400), + ("/?continuation-token=", 400), + ("/?continuation-token=" + "x"*1025, 400), + ("/?bucket-region=eu-west-1", 400), + ("/?bucket-region=us-east-1&bucket-region=us-east-1", 400), + ("/?prefix=" + "z"*256, 400), + ("/?acl", 501), + ("/?policy", 501), + ("/?list-type=2", 501), + ("/?x-id=ListBuckets&unexpected=x", 501), +]) +def test_invalid_root_list_parameters_fail_closed(service, path, expected): + response = signed(service, "GET", path) + assert response.status_code == expected, (path, response.text) + assert service[2].uploads == service[2].downloads == 0 + + +def test_sql_page_limit_10000_and_10001st_record(service): + """Do not load all buckets before limiting even for a default root GET.""" + db = service[3]() + try: + BucketCatalog(ManifestStore(db)) # create shared SQL bucket schema + db.connection.execute(insert(buckets), [ + {"name": f"test-list-{i:05d}", "created_at": 1714000000} + for i in range(10001) + ]) + db.commit() + finally: + db.close() + first, cursor = _xml_bucket_names(signed(service, "GET", "/")) + assert len(first) == 10000 and cursor + assert first[0] == "test-list-00000" and first[-1] == "test-list-09999" + tail, final = _xml_bucket_names( + signed(service, "GET", "/?max-buckets=10000&continuation-token=" + quote(cursor)), + ) + assert tail == ["test-list-10000"] and final is None + + +def test_boto3_list_buckets_parameters_actual_http(service): + assert signed(service, "PUT", "/list-ci-a").status_code == 200 + assert signed(service, "PUT", "/list-ci-b").status_code == 200 + assert signed(service, "PUT", "/other-bucket").status_code == 200 + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(128) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", + log_level="error", + )) + worker = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, + daemon=True) + worker.start() + try: + end = time.monotonic() + 10 + while not server.started and time.monotonic() < end: + time.sleep(0.03) + assert server.started + key = service[1] + sdk = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + region_name="us-east-1", aws_access_key_id=key.access_key, + aws_secret_access_key=key.secret_key, + config=Config(signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"max_attempts": 0}), + ) + page1 = sdk.list_buckets(MaxBuckets=1, Prefix="list-ci-", BucketRegion="us-east-1") + assert len(page1["Buckets"]) == 1 + assert page1["Buckets"][0]["Name"] == "list-ci-a" + assert page1["Buckets"][0]["BucketRegion"] == "us-east-1" + assert page1["Buckets"][0]["CreationDate"].year >= 2025 + assert page1["Owner"]["ID"] + page2 = sdk.list_buckets( + MaxBuckets=1, Prefix="list-ci-", BucketRegion="us-east-1", + ContinuationToken=page1["ContinuationToken"], + ) + assert [b["Name"] for b in page2["Buckets"]] == ["list-ci-b"] + assert "ContinuationToken" not in page2 + all_buckets = sdk.list_buckets() + assert {b["Name"] for b in all_buckets["Buckets"]} == { + "list-ci-a", "list-ci-b", "other-bucket", + } + with pytest.raises(ClientError): + sdk.list_buckets(MaxBuckets=10001) + assert service[2].uploads == service[2].downloads == 0 + finally: + server.should_exit = True + worker.join(timeout=10) + sock.close() + assert not worker.is_alive() diff --git a/tests/s3/test_s3_bucket_location.py b/tests/s3/test_s3_bucket_location.py new file mode 100644 index 00000000..d6c1566d --- /dev/null +++ b/tests/s3/test_s3_bucket_location.py @@ -0,0 +1,77 @@ +"""S3 bucket location contract, policy isolation and real boto3 discovery.""" +from __future__ import annotations + +import socket +import threading +import time +from xml.etree import ElementTree as ET + +import boto3 +import uvicorn +from botocore.config import Config + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore + + +def test_get_bucket_location_exact_s3_xml_and_missing_bucket(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + location = signed(service, "GET", "/backup-bucket?location") + assert location.status_code == 200, location.text + root = ET.fromstring(location.content) + assert root.tag == "{http://s3.amazonaws.com/doc/2006-03-01/}LocationConstraint" + assert root.text is None + missing = signed(service, "GET", "/missing-bucket?location") + assert missing.status_code == 404 + assert b"NoSuchBucket" in missing.content + + +def test_bucket_location_requires_independent_s3_permission(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + db = service[3]() + try: + store = S3CredentialStore(db, service[4]) + reader = store.create("bucket-locator", ["s3:GetBucketLocation"]) + listing_only = store.create("bucket-list", ["s3:ListBucket"]) + finally: + db.close() + assert signed(service, "GET", "/backup-bucket?location", + access=reader.access_key, secret=reader.secret_key).status_code == 200 + assert signed(service, "GET", "/backup-bucket?list-type=2", + access=reader.access_key, secret=reader.secret_key).status_code == 403 + assert signed(service, "GET", "/backup-bucket?location", + access=listing_only.access_key, secret=listing_only.secret_key).status_code == 403 + + +def test_real_boto3_get_bucket_location_http(service): + import web.s3 as gateway + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error")) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + client = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=service[1].access_key, + aws_secret_access_key=service[1].secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + retries={"max_attempts": 0}), + ) + client.create_bucket(Bucket="sdk-location") + assert client.get_bucket_location(Bucket="sdk-location")["LocationConstraint"] is None + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() diff --git a/tests/s3/test_s3_bucket_versioning.py b/tests/s3/test_s3_bucket_versioning.py new file mode 100644 index 00000000..1835ac9f --- /dev/null +++ b/tests/s3/test_s3_bucket_versioning.py @@ -0,0 +1,168 @@ +"""S3 bucket versioning: honest always-on immutable version semantics.""" +from __future__ import annotations + +import hashlib +import socket +import threading + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError +import pytest + +from tests.s3.test_s3_gateway import service, signed +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.versioning import parse_versioning_xml, VersioningError + +VALID = ( + b'' + b'Enabled' +) + + +def test_versioning_get_put_idempotent_and_real_object_history(service): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + path = "/versioning-ci?versioning" + response = signed(service, "GET", path) + assert response.status_code == 200, response.text + assert b"Enabled" in response.content + assert signed(service, "PUT", path, VALID).status_code == 200 + assert signed(service, "PUT", path, VALID).status_code == 200 + first = signed(service, "PUT", "/versioning-ci/data.bin", b"one") + second = signed(service, "PUT", "/versioning-ci/data.bin", b"two") + assert first.status_code == second.status_code == 200 + older = first.headers["x-amz-version-id"] + newest = second.headers["x-amz-version-id"] + assert older != newest + assert signed(service, "GET", + "/versioning-ci/data.bin?versionId=" + older).content == b"one" + assert signed(service, "GET", "/versioning-ci/data.bin").content == b"two" + assert remote.uploads == 2 + + +@pytest.mark.parametrize("bad,code", [ + (b"Suspended", 501), + (b"Disabled", 400), + (b"", 400), + (b"DisabledEnabled", 501), + (b"EnabledEnabled", 400), + (b']>&a;', 400), + (b"x" * 16385, 400), +]) +def test_unsafe_and_malformed_versioning_requests_do_not_change_history(service, bad, code): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + before = remote.uploads + response = signed(service, "PUT", "/versioning-ci?versioning", bad) + assert response.status_code == code, response.text + assert remote.uploads == before + assert b"Enabled" in signed(service, "GET", "/versioning-ci?versioning").content + + +def test_versioning_control_permissions_are_separate_from_putobject(service): + client, _, remote, connect, master = service + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + db = connect() + try: + credentials = S3CredentialStore(db, master) + reader = credentials.create("versioning-read", ["s3:GetBucketVersioning"]) + writer = credentials.create("versioning-write", ["s3:PutBucketVersioning"]) + object_key = credentials.create("versioning-objects", ["s3:PutObject"]) + finally: + db.close() + path = "/versioning-ci?versioning" + assert signed(service, "GET", path, access=reader.access_key, secret=reader.secret_key).status_code == 200 + assert signed(service, "PUT", path, VALID, access=reader.access_key, secret=reader.secret_key).status_code == 403 + assert signed(service, "GET", path, access=writer.access_key, secret=writer.secret_key).status_code == 403 + assert signed(service, "PUT", path, VALID, access=writer.access_key, secret=writer.secret_key).status_code == 200 + assert signed(service, "GET", path, access=object_key.access_key, secret=object_key.secret_key).status_code == 403 + assert signed(service, "PUT", path, VALID, access=object_key.access_key, secret=object_key.secret_key).status_code == 403 + assert remote.uploads == 0 + + +def test_versioning_checksums_are_validated_even_with_signed_correct_sha256(service): + import base64 + import zlib + + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + correct_crc32 = base64.b64encode( + (zlib.crc32(VALID) & 0xffffffff).to_bytes(4, "big"), + ).decode("ascii") + verified = signed(service, "PUT", "/versioning-ci?versioning", VALID, + extra_headers={ + "x-amz-checksum-crc32": correct_crc32, + "x-amz-sdk-checksum-algorithm": "CRC32", + }) + assert verified.status_code == 200, verified.text + bad_crc = signed(service, "PUT", "/versioning-ci?versioning", VALID, + extra_headers={ + "x-amz-checksum-crc32": base64.b64encode( + b"nope", + ).decode("ascii"), + "x-amz-sdk-checksum-algorithm": "CRC32", + }) + assert bad_crc.status_code == 400, bad_crc.text + bad_md5 = signed(service, "PUT", "/versioning-ci?versioning", VALID, + extra_headers={ + "Content-MD5": base64.b64encode(b"\x00" * 16).decode(), + }) + assert bad_md5.status_code == 400, bad_md5.text + + +def test_missing_bucket_and_ambiguous_selectors_fail_closed(service): + assert signed(service, "GET", "/missing-versioning?versioning").status_code == 404 + assert signed(service, "PUT", "/missing-versioning?versioning", VALID).status_code == 404 + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + for path in ("/versioning-ci?versioning&versioning", + "/versioning-ci?versioning&policy"): + assert signed(service, "GET", path).status_code in (400, 501) + assert signed(service, "DELETE", "/versioning-ci?versioning").status_code == 501 + assert signed(service, "GET", "/versioning-ci/data.bin?versioning").status_code == 501 + assert signed(service, "DELETE", "/versioning-ci/data.bin?versioning").status_code == 501 + + +def test_real_boto3_get_put_bucket_versioning(service): + import uvicorn + import web.s3 as gateway + + _, creds, _, _, _ = service + assert signed(service, "PUT", "/versioning-ci").status_code == 200 + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(128) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, log_level="error", + lifespan="off", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, + daemon=True) + thread.start() + try: + import time + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(0.02) + assert server.started + client = boto3.client("s3", endpoint_url=f"http://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + config=Config(signature_version="s3v4", + s3={"addressing_style": "path"})) + assert client.get_bucket_versioning(Bucket="versioning-ci")["Status"] == "Enabled" + reply = client.put_bucket_versioning( + Bucket="versioning-ci", + VersioningConfiguration={"Status": "Enabled"}, + ) + assert reply["ResponseMetadata"]["HTTPStatusCode"] == 200 + with pytest.raises(ClientError): + client.put_bucket_versioning( + Bucket="versioning-ci", + VersioningConfiguration={"Status": "Suspended"}, + ) + finally: + server.should_exit = True + thread.join(timeout=10) diff --git a/tests/s3/test_s3_cli_compat.py b/tests/s3/test_s3_cli_compat.py new file mode 100644 index 00000000..88f1c6a5 --- /dev/null +++ b/tests/s3/test_s3_cli_compat.py @@ -0,0 +1,234 @@ +"""Real AWS CLI + rclone S3 compatibility over HTTP with Telegram-backed bytes. + +Runs the actual client executables against Uvicorn in a background thread. +No network AWS/S3 backend is used: the gateway uses real SQLite manifests and +an in-memory Telegram protocol fake. Every downloaded byte is compared. + +CI installs both binaries; local runs skip missing tools unless +TELEDRIVE_REQUIRE_CLI_CLIENTS=1 is set. +""" +from __future__ import annotations + +import hashlib +import os +import shutil +import socket +import subprocess +import threading +import time +from pathlib import Path + +import pytest +import uvicorn + +import web.s3 as gateway +from tests.s3.test_s3_gateway import service, signed + + +@pytest.fixture +def live_s3(service): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started, "S3 ASGI server did not start" + yield f"http://127.0.0.1:{port}", service[1], service[2] + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive(), "S3 ASGI server failed to stop" + + +def _client(name): + command = shutil.which(name) + if command: + return command + if os.getenv("TELEDRIVE_REQUIRE_CLI_CLIENTS") == "1": + pytest.fail(f"Required real S3 client is absent: {name}") + pytest.skip(f"Install {name} to run its real integration tests") + + +def _run(args, *, env, timeout=90): + done = subprocess.run(args, capture_output=True, text=True, env=env, + timeout=timeout, check=False) + assert done.returncode == 0, ( + f"Real S3 client failed: {args[0]} {args[1:]}\n" + f"stdout: {done.stdout[-2500:]}\nstderr: {done.stderr[-3500:]}" + ) + return done.stdout + + +def _data(tmp_path: Path) -> bytes: + content = bytes(range(256)) * 121 + b"\x00\xff\x80binary" + (tmp_path / "source.bin").write_bytes(content) + return content + + +def test_real_aws_cli_upload_download_list_delete(live_s3, tmp_path): + """Independent AWS CLI binary, including high-level cp and multipart threshold.""" + aws = _client("aws") + endpoint, issued, transport = live_s3 + env = dict(os.environ) + env.update({ + "AWS_ACCESS_KEY_ID": issued.access_key, + "AWS_SECRET_ACCESS_KEY": issued.secret_key, + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_MAX_ATTEMPTS": "1", + "AWS_PAGER": "", + "AWS_CONFIG_FILE": str(tmp_path / "aws-config"), + }) + (tmp_path / "aws-config").write_text( + "[default]\ns3 =\n addressing_style = path\n" + " payload_signing_enabled = true\n", encoding="utf-8" + ) + def cli(*args): + return _run([aws, "--endpoint-url", endpoint, *args], env=env) + + cli("s3api", "create-bucket", "--bucket", "aws-cli-test") + content = _data(tmp_path) + cli("s3api", "put-object", "--bucket", "aws-cli-test", + "--key", "bin/source.bin", "--body", str(tmp_path / "source.bin")) + cli("s3api", "head-object", "--bucket", "aws-cli-test", + "--key", "bin/source.bin") + cli("s3api", "get-object", "--bucket", "aws-cli-test", + "--key", "bin/source.bin", str(tmp_path / "retrieved.bin")) + assert (tmp_path / "retrieved.bin").read_bytes() == content + assert hashlib.sha256(content).hexdigest() == hashlib.sha256( + (tmp_path / "retrieved.bin").read_bytes() + ).hexdigest() + listing = cli("s3api", "list-objects-v2", "--bucket", "aws-cli-test", + "--prefix", "bin/") + assert "bin/source.bin" in listing + cli("s3api", "delete-object", "--bucket", "aws-cli-test", + "--key", "bin/source.bin") + assert transport.uploads >= 1 and transport.downloads >= 1 + + +def test_real_rclone_sync_and_integrity(live_s3, tmp_path, service): + """rclone real S3 remote: put, list, copy-to-local, replace and delete.""" + rclone = _client("rclone") + endpoint, issued, transport = live_s3 + env = dict(os.environ) + env.update({ + "RCLONE_CONFIG_TD_TYPE": "s3", + "RCLONE_CONFIG_TD_PROVIDER": "Other", + "RCLONE_CONFIG_TD_ACCESS_KEY_ID": issued.access_key, + "RCLONE_CONFIG_TD_SECRET_ACCESS_KEY": issued.secret_key, + "RCLONE_CONFIG_TD_ENDPOINT": endpoint, + "RCLONE_CONFIG_TD_REGION": "us-east-1", + "RCLONE_CONFIG_TD_FORCE_PATH_STYLE": "true", + "RCLONE_CONFIG_TD_NO_CHECK_BUCKET": "true", + "RCLONE_CONFIG": str(tmp_path / "rclone.conf"), + }) + (tmp_path / "rclone.conf").write_text("", encoding="utf-8") + def cli(*args): + return _run([rclone, "--retries", "1", "--low-level-retries", "1", + "--contimeout", "5s", "--timeout", "60s", *args], env=env) + + # no_check_bucket deliberately disables rclone bucket probing/creation. + # Provision via the signed S3 endpoint first, then verify rclone's + # independent PUT/GET/list/overwrite/delete client behavior. + created = signed(service, "PUT", "/rclone-test") + assert created.status_code == 200, created.text + assert "rclone-test" in cli("lsd", "td:") + content = _data(tmp_path) + cli("copyto", str(tmp_path / "source.bin"), + "td:rclone-test/deep/path/one.bin") + listing = cli("lsf", "td:rclone-test/deep/path", "--files-only") + assert "one.bin" in listing + cli("copyto", "td:rclone-test/deep/path/one.bin", + str(tmp_path / "rclone-downloaded.bin")) + assert (tmp_path / "rclone-downloaded.bin").read_bytes() == content + # A second write must replace current metadata without corrupting + # old Telegram immutable manifests. + updated = b"updated-content" * 517 + (tmp_path / "source.bin").write_bytes(updated) + cli("copyto", str(tmp_path / "source.bin"), "td:rclone-test/deep/path/one.bin") + cli("copyto", "td:rclone-test/deep/path/one.bin", + str(tmp_path / "rclone-replaced.bin")) + assert (tmp_path / "rclone-replaced.bin").read_bytes() == updated + cli("deletefile", "td:rclone-test/deep/path/one.bin") + assert cli("lsf", "td:rclone-test/deep/path", "--files-only").strip() == "" + assert transport.uploads >= 2 and transport.downloads >= 2 + + # Force the actual Go rclone executable down the multipart S3 path, + # including CreateMultipartUpload, UploadPart, Complete and range-safe GET. + # The fake Telegram transport stores the full binary body and the + # independently read-back SHA-256 must match. + multipart_body = os.urandom(5 * 1024 * 1024 + 513) + large_path = tmp_path / "large.bin" + large_path.write_bytes(multipart_body) + uploads_before = transport.uploads + cli("copyto", str(large_path), "td:rclone-test/deep/path/multipart.bin", + "--s3-upload-cutoff", "5M", "--s3-chunk-size", "5M") + restored = tmp_path / "multipart-restored.bin" + cli("copyto", "td:rclone-test/deep/path/multipart.bin", str(restored)) + assert hashlib.sha256(restored.read_bytes()).digest() == hashlib.sha256( + multipart_body + ).digest() + assert transport.uploads >= uploads_before + 2, ( + "Expected at least two Telegram chunks, proving multipart rather than one PUT" + ) + + +def test_real_aws_cli_high_level_sync_round_trip_and_delete(live_s3, tmp_path): + """Exercise aws s3 sync directory semantics, not just s3api single-object calls.""" + aws = _client("aws") + endpoint, issued, remote = live_s3 + env = dict(os.environ) + env.update({ + "AWS_ACCESS_KEY_ID": issued.access_key, + "AWS_SECRET_ACCESS_KEY": issued.secret_key, + "AWS_DEFAULT_REGION": "us-east-1", + "AWS_EC2_METADATA_DISABLED": "true", + "AWS_MAX_ATTEMPTS": "1", + "AWS_PAGER": "", + "AWS_CONFIG_FILE": str(tmp_path / "aws-config"), + }) + (tmp_path / "aws-config").write_text( + "[default]\ns3 =\n addressing_style = path\n" + " payload_signing_enabled = true\n", encoding="utf-8", + ) + + def cli(*args): + return _run([aws, "--endpoint-url", endpoint, *args], env=env) + + cli("s3api", "create-bucket", "--bucket", "aws-sync-test") + original = tmp_path / "original" + original.mkdir() + (original / "sub").mkdir() + first = b"hello" + bytes(range(256)) * 37 + second = b"world" + b"\x00\xff" * 500 + (original / "sub" / "one.bin").write_bytes(first) + (original / "two.bin").write_bytes(second) + cli("s3", "sync", str(original), "s3://aws-sync-test/archive/", + "--size-only") + downloaded = tmp_path / "downloaded" + cli("s3", "sync", "s3://aws-sync-test/archive/", str(downloaded), + "--size-only") + assert (downloaded / "sub" / "one.bin").read_bytes() == first + assert (downloaded / "two.bin").read_bytes() == second + + changed = b"changed" * 1200 + (original / "sub" / "one.bin").write_bytes(changed) + (original / "two.bin").unlink() + cli("s3", "sync", str(original), "s3://aws-sync-test/archive/", + "--size-only", "--delete") + dest2 = tmp_path / "downloaded2" + cli("s3", "sync", "s3://aws-sync-test/archive/", str(dest2), + "--size-only") + assert (dest2 / "sub" / "one.bin").read_bytes() == changed + assert not (dest2 / "two.bin").exists() + assert remote.uploads >= 3 and remote.downloads >= 3 diff --git a/tests/s3/test_s3_conditional_writes.py b/tests/s3/test_s3_conditional_writes.py new file mode 100644 index 00000000..dead6710 --- /dev/null +++ b/tests/s3/test_s3_conditional_writes.py @@ -0,0 +1,265 @@ +"""Conditional S3 writes: actual signed HTTP and real SQL concurrency boundaries. + +An ETag check without an atomic publish would be a time-of-check/time-of-use +vulnerability. These tests trigger concurrent head mutations during Telegram +sendDocument and verify that the conditional writer CANNOT replace the winner. +""" +from __future__ import annotations + +import asyncio +import hashlib +import time + +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials + +from tests.s3.test_s3_gateway import service, signed +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +def put(svc, body: bytes, key: str = "race.bin", **headers): + return signed(svc, "PUT", f"/conditional/{key}", body, + extra_headers={name.replace("_", "-"): value + for name, value in headers.items()}) + + +def read(svc, key: str = "race.bin"): + return signed(svc, "GET", f"/conditional/{key}") + + +def test_create_if_absent_and_etag_cas_prevent_stale_writes(service): + assert signed(service, "PUT", "/conditional").status_code == 200 + first = put(service, b"first", If_None_Match="*") + assert first.status_code == 200, first.text + uploads = service[2].uploads + conflict = put(service, b"must-not-write", If_None_Match="*") + assert conflict.status_code == 412 + assert service[2].uploads == uploads + current_tag = first.headers["etag"] + denied = put(service, b"wrong", If_Match='"not-the-etag"') + assert denied.status_code == 412 + assert service[2].uploads == uploads + matching = put(service, b"second", If_Match=current_tag) + assert matching.status_code == 200, matching.text + stale = put(service, b"third", If_Match=current_tag) + assert stale.status_code == 412 + assert read(service).content == b"second" + + +def test_signed_conditional_delete_is_atomic_and_missing_match_is_412(service): + assert signed(service, "PUT", "/conditional").status_code == 200 + missing = signed(service, "DELETE", "/conditional/race.bin", + extra_headers={"If-Match": "*"}) + assert missing.status_code == 412 + first = put(service, b"important") + etag = first.headers["etag"] + denied = signed(service, "DELETE", "/conditional/race.bin", + extra_headers={"If-Match": '"bogus"'}) + assert denied.status_code == 412 + assert read(service).content == b"important" + done = signed(service, "DELETE", "/conditional/race.bin", + extra_headers={"If-Match": etag}) + assert done.status_code == 204 + assert read(service).status_code == 404 + again = signed(service, "DELETE", "/conditional/race.bin", + extra_headers={"If-Match": etag}) + assert again.status_code == 412 + + +def test_unsigned_write_precondition_cannot_be_added_to_signed_request(service): + assert signed(service, "PUT", "/conditional").status_code == 200 + existing = put(service, b"keep") + assert existing.status_code == 200 + raw = b"overwrite" + req = AWSRequest(method="PUT", url="http://localhost:9000/conditional/race.bin", + data=raw, headers={ + "x-amz-content-sha256": hashlib.sha256(raw).hexdigest(), + }) + req.context["payload_signing_enabled"] = True + S3SigV4Auth( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(req) + forged = dict(req.headers) + forged["If-Match"] = existing.headers["etag"] + before = service[2].uploads + denied = service[0].put("/conditional/race.bin", content=raw, headers=forged) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + assert service[2].uploads == before + assert read(service).content == b"keep" + + +def test_competing_writer_during_telegram_upload_cannot_steal_conditional_head( + service, monkeypatch): + assert signed(service, "PUT", "/conditional").status_code == 200 + first = put(service, b"old") + assert first.status_code == 200 + remote = service[2] + original = remote.upload + intrusions = [] + + async def concurrent_winner(path, *, object_id, ordinal, checksum): + if not intrusions: + # Separate transaction commits while first writer is awaiting Telegram. + connection = service[3]() + try: + catalog = BucketCatalog(ManifestStore(connection)) + moment = int(time.time()) + winner_id = catalog.manifests.begin("conditional", "race.bin", 0, moment) + catalog.manifests.finish(winner_id, hashlib.sha256(b"").hexdigest(), moment) + catalog.publish(winner_id, now=moment) + intrusions.append(winner_id) + finally: + connection.close() + return await original(path, object_id=object_id, + ordinal=ordinal, checksum=checksum) + + monkeypatch.setattr(remote, "upload", concurrent_winner) + stale = put(service, b"stale-write-must-not-win", If_Match=first.headers["etag"]) + assert intrusions + assert stale.status_code == 412, stale.text + assert read(service).status_code == 200 + assert read(service).content == b"" + + +def test_competing_create_during_upload_rejects_if_none_match(service, monkeypatch): + assert signed(service, "PUT", "/conditional").status_code == 200 + remote = service[2] + original = remote.upload + injected = [] + + async def concurrent_create(path, *, object_id, ordinal, checksum): + if not injected: + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + moment = int(time.time()) + winner = catalog.manifests.begin("conditional", "race.bin", 0, moment) + catalog.manifests.finish(winner, hashlib.sha256(b"").hexdigest(), moment) + catalog.publish(winner, now=moment) + injected.append(winner) + finally: + db.close() + return await original(path, object_id=object_id, + ordinal=ordinal, checksum=checksum) + + monkeypatch.setattr(remote, "upload", concurrent_create) + result = put(service, b"should-not-win", If_None_Match="*") + assert injected + assert result.status_code == 412, result.text + assert read(service).content == b"" + + +def test_delete_recreate_same_generation_is_not_equal_object_identity(service): + """Object IDs fence ABA even when generation resets after unlink/recreate.""" + assert signed(service, "PUT", "/conditional").status_code == 200 + original = put(service, b"A") + assert original.status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + observed = catalog.get("conditional", "race.bin") + assert observed.generation == 1 + assert catalog.remove_current("conditional", "race.bin", + expected_generation=1, + expected_object_id=observed.item.id) + now = int(time.time()) + replacement_id = catalog.manifests.begin("conditional", "race.bin", 0, now) + catalog.manifests.finish(replacement_id, hashlib.sha256(b"").hexdigest(), now) + catalog.publish(replacement_id, now=now) + assert catalog.get("conditional", "race.bin").generation == 1 + try: + catalog.remove_current("conditional", "race.bin", + expected_generation=1, + expected_object_id=observed.item.id) + except ValueError: + pass + else: + raise AssertionError("ABA guard allowed old object ID to unlink a new version") + staged = catalog.manifests.begin("conditional", "race.bin", 0, now) + catalog.manifests.finish(staged, hashlib.sha256(b"").hexdigest(), now) + try: + catalog.publish(staged, now=now, expected_generation=1, + expected_object_id=observed.item.id) + except ValueError: + pass + else: + raise AssertionError("ABA guard allowed stale writer to overwrite a new version") + assert catalog.get("conditional", "race.bin").item.id == replacement_id + finally: + db.close() + + +def test_real_boto3_http_conditional_put_contract(service): + """boto3 (not synthetic requests) must observe 412 and preserve object bytes.""" + import socket + import threading + import time + import boto3 + import uvicorn + from botocore.config import Config + from botocore.exceptions import ClientError + import web.s3 as gateway + + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + cred = service[1] + sdk = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=cred.access_key, + aws_secret_access_key=cred.secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}, + ), + ) + sdk.create_bucket(Bucket="sdk-condition") + first = sdk.put_object(Bucket="sdk-condition", Key="key", Body=b"one", + IfNoneMatch="*") + try: + sdk.put_object(Bucket="sdk-condition", Key="key", + Body=b"should-not-overwrite", IfNoneMatch="*") + except ClientError as exc: + assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 412 + assert exc.response["Error"]["Code"] == "PreconditionFailed" + else: + raise AssertionError("SDK conditional create ignored existing object") + try: + second = sdk.put_object(Bucket="sdk-condition", Key="key", + Body=b"two", IfMatch=first["ETag"]) + except ClientError as exc: + raise AssertionError(f"Conditional PUT SDK wire failure: {exc.response!r}") from exc + assert second["ETag"] != first["ETag"] + try: + sdk.put_object(Bucket="sdk-condition", Key="key", + Body=b"stale", IfMatch=first["ETag"]) + except ClientError as exc: + assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 412 + else: + raise AssertionError("SDK stale IfMatch overwrote current object") + assert sdk.get_object(Bucket="sdk-condition", Key="key")["Body"].read() == b"two" + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() diff --git a/tests/s3/test_s3_conditions.py b/tests/s3/test_s3_conditions.py new file mode 100644 index 00000000..50beb5f5 --- /dev/null +++ b/tests/s3/test_s3_conditions.py @@ -0,0 +1,98 @@ +"""Conditional S3 reads must save Telegram bandwidth and preserve ETag semantics.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from email.utils import format_datetime + +from tests.s3.test_s3_gateway import service, signed + + +def setup_object(service, body=b"cached-object-bytes"): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + put = signed(service, "PUT", "/backup-bucket/cache.bin", body) + assert put.status_code == 200, put.text + return put.headers["etag"] + + +def get(service, **headers): + return signed(service, "GET", "/backup-bucket/cache.bin", + extra_headers={key.replace("_", "-"): value + for key, value in headers.items()}) + + +def head(service, **headers): + return signed(service, "HEAD", "/backup-bucket/cache.bin", + extra_headers={key.replace("_", "-"): value + for key, value in headers.items()}) + + +def test_if_none_match_returns_304_without_telegram_io(service): + tag = setup_object(service) + remote = service[2] + before = remote.downloads + response = get(service, If_None_Match=tag) + assert response.status_code == 304 + assert response.content == b"" + assert response.headers["etag"] == tag + assert remote.downloads == before + + wildcard = head(service, If_None_Match="*") + assert wildcard.status_code == 304 + assert remote.downloads == before + + +def test_if_match_blocks_bad_etag_without_downloading_chunks(service): + tag = setup_object(service) + remote = service[2] + before = remote.downloads + blocked = get(service, If_Match='"unexpected"') + assert blocked.status_code == 412 + assert b"PreconditionFailed" in blocked.content + assert remote.downloads == before + success = get(service, If_Match=tag) + assert success.status_code == 200 + assert success.content == b"cached-object-bytes" + assert remote.downloads > before + + +def test_if_modified_and_unmodified_since_use_stable_manifest_timestamp(service): + setup_object(service) + head_response = head(service) + assert head_response.status_code == 200 + modified = head_response.headers["last-modified"] + before = service[2].downloads + unchanged = get(service, If_Modified_Since=modified) + assert unchanged.status_code == 304 + assert service[2].downloads == before + + yesterday = format_datetime(datetime.now(timezone.utc) - timedelta(days=1), usegmt=True) + prohibited = get(service, If_Unmodified_Since=yesterday) + assert prohibited.status_code == 412 + assert service[2].downloads == before + latest = get(service, If_Modified_Since=yesterday) + assert latest.status_code == 200 + assert latest.content == b"cached-object-bytes" + + # Correct If-Match suppresses an old If-Unmodified-Since per RFC semantics. + tag = head_response.headers["etag"] + matching = get(service, If_Match=tag, If_Unmodified_Since=yesterday) + assert matching.status_code == 200 + + +def test_failed_write_conditions_reject_before_telegram_upload(service): + tag = setup_object(service) + before = service[2].uploads + write = signed(service, "PUT", "/backup-bucket/cache.bin", b"unsafe-overwrite", + extra_headers={"If-None-Match": "*"}) + assert write.status_code == 412 + assert b"PreconditionFailed" in write.content + deletion = signed(service, "DELETE", "/backup-bucket/cache.bin", + extra_headers={"If-Match": '"different"'}) + assert deletion.status_code == 412 + assert service[2].uploads == before + assert get(service).content == b"cached-object-bytes" + # A correct conditional DELETE must atomically unlink the current head. + permitted = signed(service, "DELETE", "/backup-bucket/cache.bin", + extra_headers={"If-Match": tag}) + assert permitted.status_code == 204 + assert get(service).status_code == 404 diff --git a/tests/s3/test_s3_contract.py b/tests/s3/test_s3_contract.py new file mode 100644 index 00000000..0690aac1 --- /dev/null +++ b/tests/s3/test_s3_contract.py @@ -0,0 +1,226 @@ +"""Release-blocking P0 failure matrix over production S3 HTTP and SQLite. + +Run with deterministic Telegram-compatible fake to prove crash atomicity, +authentication revocation, and multi-chunk range behavior without touching +any user or production Telegram messages. +""" +from __future__ import annotations + +import base64 +import hashlib +import os +import time + +import boto3 +import pytest +from botocore.config import Config +from sqlalchemy import select + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import begin, complete, part +from tests.s3.test_s3_presigned import request_url +from teledrive.db import Database +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.multipart import MultipartStore, parts, uploads +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +import web.s3 as gateway + + +def _mpu(service): + db = service[3]() + return db, MultipartStore(BucketCatalog(ManifestStore(db))) + + +def test_failed_sql_commit_during_complete_rolls_back_and_retry_preserves_bytes( + service, monkeypatch +): + """Crash after chunk reparenting but before final SQL commit is all-or-nothing.""" + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/asset.bin", b"old-head").status_code == 200 + uid = begin(service) + payload = b"remote-message-owned-by-staged-part" * 120 + etag = part(service, uid, 1, payload) + original_uploads = service[2].uploads + + db, manager = _mpu(service) + try: + staging_id = db.connection.execute( + select(parts.c.object_id).where( + parts.c.upload_id == uid, parts.c.number == 1 + ) + ).scalar_one() + before_refs = manager.manifests.parts(staging_id) + assert len(before_refs) == 1 + finally: + db.close() + + original_commit = Database.commit + faulted = [] + + def fail_at_final_commit(self): + # Stage complete has an in-flight "completed" row only during the + # publication transaction; the earlier lease claim is "completing". + state = self.connection.execute( + select(uploads.c.state).where(uploads.c.id == uid) + ).scalar_one_or_none() + if state == "completed" and not faulted: + faulted.append(True) + raise RuntimeError("injected SQL commit outage") + return original_commit(self) + + monkeypatch.setattr(Database, "commit", fail_at_final_commit) + interrupted = complete(service, uid, [(1, etag)]) + assert interrupted.status_code == 503, interrupted.text + assert faulted, "Fault injection must reach the final publish transaction" + assert service[2].uploads == original_uploads + monkeypatch.setattr(Database, "commit", original_commit) + + # The previously published HEAD and staged message IDs survive across + # fresh SQL sessions. There must be no half-visible final object. + assert signed(service, "GET", "/backup-bucket/asset.bin").content == b"old-head" + db, manager = _mpu(service) + try: + assert manager.require(uid, "backup-bucket", "asset.bin", + service[1].access_key)["state"] == "active" + assert manager.manifests.parts(staging_id) == before_refs + assert manager.catalog.get("backup-bucket", "asset.bin").item.size == len(b"old-head") + finally: + db.close() + + done = complete(service, uid, [(1, etag)]) + assert done.status_code == 200, done.text + assert signed(service, "GET", "/backup-bucket/asset.bin").content == payload + assert service[2].uploads == original_uploads + retry = complete(service, uid, [(1, etag)]) + assert retry.status_code == 200, retry.text + assert service[2].uploads == original_uploads + + +def test_telegram_read_failure_during_complete_leaves_staging_retryable( + service, monkeypatch +): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + payload = b"recoverable-multipart-part" * 100 + etag = part(service, uid, 1, payload) + original_download = service[2].download + count = [] + + async def broken_download(reference, destination): + count.append(reference.message_id) + raise RuntimeError("injected Telegram retrieval outage") + + monkeypatch.setattr(service[2], "download", broken_download) + interrupted = complete(service, uid, [(1, etag)]) + assert count + assert interrupted.status_code == 503, interrupted.text + monkeypatch.setattr(service[2], "download", original_download) + assert signed(service, "GET", "/backup-bucket/asset.bin").status_code == 404 + db, manager = _mpu(service) + try: + assert manager.require(uid, "backup-bucket", "asset.bin", + service[1].access_key)["state"] == "active" + assert len(manager.list_parts(uid, "backup-bucket", "asset.bin", + service[1].access_key)) == 1 + finally: + db.close() + before = service[2].uploads + assert complete(service, uid, [(1, etag)]).status_code == 200 + assert signed(service, "GET", "/backup-bucket/asset.bin").content == payload + assert service[2].uploads == before + + +def test_expired_and_revoked_presigned_get_put_never_touch_telegram(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"keep-immutable" + assert signed(service, "PUT", "/backup-bucket/asset.bin", body).status_code == 200 + connection = service[3]() + try: + key = S3CredentialStore(connection, service[4]).create( + "presigned-revocable", ["s3:GetObject", "s3:PutObject"] + ) + finally: + connection.close() + + s3 = boto3.client("s3", endpoint_url="http://localhost:9000", + aws_access_key_id=key.access_key, + aws_secret_access_key=key.secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required")) + read = s3.generate_presigned_url( + "get_object", Params={"Bucket": "backup-bucket", "Key": "asset.bin"}, + ExpiresIn=600, + ) + assert request_url(service[0], "GET", read).content == body + unauthorized = b"must-not-upload" + checksum = base64.b64encode(hashlib.sha256(unauthorized).digest()).decode("ascii") + write = s3.generate_presigned_url( + "put_object", + Params={"Bucket": "backup-bucket", "Key": "new.bin", + "ChecksumSHA256": checksum, "IfNoneMatch": "*"}, + ExpiresIn=600, HttpMethod="PUT", + ) + + db = service[3]() + try: + assert S3CredentialStore(db, service[4]).revoke(key.id) + finally: + db.close() + before = service[2].uploads + for outcome in ( + request_url(service[0], "GET", read), + request_url(service[0], "PUT", write, content=unauthorized, + headers={"x-amz-checksum-sha256": checksum, + "If-None-Match": "*"}), + ): + assert outcome.status_code == 403 + assert b"AccessDenied" in outcome.content + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/asset.bin").content == body + assert signed(service, "GET", "/backup-bucket/new.bin").status_code == 404 + + +def test_large_multi_chunk_range_and_head_over_fresh_sql_connections( + service, monkeypatch +): + """Exercise genuine multi-chunk reads, not just slicing a single blob.""" + monkeypatch.setattr(gateway, "RECOMMENDED_CHUNK_SIZE", 2 * 1024 * 1024) + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = os.urandom(5 * 1024 * 1024 + 137) + wrote = signed(service, "PUT", "/backup-bucket/chunks.bin", payload) + assert wrote.status_code == 200, wrote.text + assert service[2].uploads == 3 + downloads_before_head = service[2].downloads + head = signed(service, "HEAD", "/backup-bucket/chunks.bin") + assert head.status_code == 200 + assert int(head.headers["content-length"]) == len(payload) + assert service[2].downloads == downloads_before_head + + for offset, length in [ + (2 * 1024 * 1024 - 19, 61), + (4 * 1024 * 1024 - 3, 19), + (len(payload) - 91, 91), + (0, len(payload)), + ]: + # Every request gets a new independent DB connection from the + # gateway fixture, including after the writer connection closes. + result = signed( + service, "GET", "/backup-bucket/chunks.bin", + extra_headers={"Range": f"bytes={offset}-{offset+length-1}"}, + ) + assert result.status_code == 206, result.text[:200] + assert result.content == payload[offset:offset + length] + assert result.headers["content-range"] == ( + f"bytes {offset}-{offset+length-1}/{len(payload)}" + ) + assert result.headers["content-length"] == str(length) + suffix = signed(service, "GET", "/backup-bucket/chunks.bin", + extra_headers={"Range": "bytes=-47"}) + assert suffix.status_code == 206 and suffix.content == payload[-47:] + invalid = signed(service, "GET", "/backup-bucket/chunks.bin", + extra_headers={"Range": f"bytes={len(payload)}-{len(payload)+100}"}) + assert invalid.status_code == 416 + assert signed(service, "GET", "/backup-bucket/chunks.bin").content == payload diff --git a/tests/s3/test_s3_copy.py b/tests/s3/test_s3_copy.py new file mode 100644 index 00000000..e4d2e351 --- /dev/null +++ b/tests/s3/test_s3_copy.py @@ -0,0 +1,331 @@ +"""Real signed S3 CopyObject contracts with Telegram-only immutable data. + +These tests verify source-read AND destination-write scope, ETag preconditions, +zero Telegram transfers, cross-bucket durability and alias correctness. +""" +from __future__ import annotations + +import hashlib +from urllib.parse import quote + +import boto3 +from botocore.config import Config +import pytest +from sqlalchemy import select + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import begin, complete, part +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore, chunks, object_aliases + + +def copy(svc, source, target, **headers): + return signed( + svc, "PUT", target, + extra_headers={"x-amz-copy-source": source, **headers}, + ) + + +def test_metadata_only_copy_durable_after_source_delete_and_overwrite(service): + assert signed(service, "PUT", "/origin-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"telegram-referenced-payload" * 200 + assert signed(service, "PUT", "/origin-bucket/original.bin", body).status_code == 200 + before_uploads = service[2].uploads + before_downloads = service[2].downloads + response = copy(service, "/origin-bucket/original.bin", "/backup-bucket/copy.bin") + assert response.status_code == 200, response.text + assert service[2].uploads == before_uploads + assert service[2].downloads == before_downloads + etag = '"' + hashlib.sha256(body).hexdigest() + '"' + assert etag.encode() in response.content + assert signed(service, "HEAD", "/backup-bucket/copy.bin").headers["etag"] == etag + + connection = service[3]() + try: + manifests = ManifestStore(connection) + original = BucketCatalog(manifests).get("origin-bucket", "original.bin").item + clone = BucketCatalog(manifests).get("backup-bucket", "copy.bin").item + assert original.id != clone.id + alias = connection.connection.execute( + select(object_aliases.c.source_object_id) + .where(object_aliases.c.object_id == clone.id) + ).scalar_one() + assert alias == original.id + assert len(manifests.parts(clone.id)) == len(manifests.parts(original.id)) == 1 + # A copied object must never duplicate the same Telegram reference + # and violate the global unique-message constraint. + assert connection.connection.execute(select(chunks.c.object_id)).all() == [(original.id,)] + finally: + connection.close() + + assert signed(service, "DELETE", "/origin-bucket/original.bin").status_code == 204 + assert signed(service, "PUT", "/origin-bucket/original.bin", b"replaced").status_code == 200 + assert signed(service, "GET", "/backup-bucket/copy.bin").content == body + assert signed(service, "GET", "/origin-bucket/original.bin").content == b"replaced" + + +def test_copy_source_without_leading_slash_matches_boto3(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/src", b"without-slash").status_code == 200 + result = copy(service, "backup-bucket/src", "/backup-bucket/target") + assert result.status_code == 200, result.text + assert signed(service, "GET", "/backup-bucket/target").content == b"without-slash" + + +def test_copy_chain_flattens_and_remains_readable_when_heads_disappear(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"chain-immutable" * 333 + assert signed(service, "PUT", "/backup-bucket/first", payload).status_code == 200 + assert copy(service, "/backup-bucket/first", "/backup-bucket/second").status_code == 200 + assert copy(service, "/backup-bucket/second", "/backup-bucket/third").status_code == 200 + assert signed(service, "DELETE", "/backup-bucket/first").status_code == 204 + assert signed(service, "DELETE", "/backup-bucket/second").status_code == 204 + assert signed(service, "GET", "/backup-bucket/third").content == payload + conn = service[3]() + try: + manifests = ManifestStore(conn) + head = BucketCatalog(manifests).get("backup-bucket", "third") + first_id = manifests.resolve_content_id(head.item.id) + assert first_id != head.item.id + assert conn.connection.execute(select(object_aliases.c.source_object_id) + .where(object_aliases.c.object_id == head.item.id)).scalar_one() == first_id + finally: + conn.close() + + +def test_copy_preconditions_are_signed_and_failed_copy_never_publishes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"safe-current" + assert signed(service, "PUT", "/backup-bucket/source", data).status_code == 200 + assert signed(service, "PUT", "/backup-bucket/dest", b"previous").status_code == 200 + actual = '"' + hashlib.sha256(data).hexdigest() + '"' + before = service[2].uploads + unmatched = copy(service, "/backup-bucket/source", "/backup-bucket/dest", + **{"x-amz-copy-source-if-match": '"not-the-etag"'}) + assert unmatched.status_code == 412 + assert b"PreconditionFailed" in unmatched.content + matched_none = copy(service, "/backup-bucket/source", "/backup-bucket/dest", + **{"x-amz-copy-source-if-none-match": actual}) + assert matched_none.status_code == 412 + assert signed(service, "GET", "/backup-bucket/dest").content == b"previous" + works = copy(service, "/backup-bucket/source", "/backup-bucket/dest", + **{"x-amz-copy-source-if-match": actual}) + assert works.status_code == 200, works.text + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/dest").content == data + + +def test_copy_requires_both_get_and_put_permissions(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source", b"read-permission").status_code == 200 + db = service[3]() + try: + store = S3CredentialStore(db, service[4]) + only_write = store.create("copy-denied", ["s3:PutObject"]) + only_read = store.create("copy-readonly", ["s3:GetObject"]) + finally: + db.close() + denied = signed(service, "PUT", "/backup-bucket/out", + extra_headers={"x-amz-copy-source": "/backup-bucket/source"}, + access=only_write.access_key, secret=only_write.secret_key) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + rejected = signed(service, "PUT", "/backup-bucket/out", + extra_headers={"x-amz-copy-source": "/backup-bucket/source"}, + access=only_read.access_key, secret=only_read.secret_key) + assert rejected.status_code == 403 + assert signed(service, "GET", "/backup-bucket/out").status_code == 404 + + +def test_copy_does_not_ignore_unsupported_metadata_changes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source", b"payload").status_code == 200 + before = service[2].uploads + unsupported_directive = copy(service, "/backup-bucket/source", "/backup-bucket/out", + **{"x-amz-metadata-directive": "INVALID"}) + assert unsupported_directive.status_code == 400 + replacement = copy(service, "/backup-bucket/source", "/backup-bucket/out", + **{"x-amz-metadata-directive": "REPLACE", + "Content-Type": "text/plain", "x-amz-meta-purpose": "copy"}) + assert replacement.status_code == 200 + assert signed(service, "HEAD", "/backup-bucket/out").headers["x-amz-meta-purpose"] == "copy" + ignored_range = copy(service, "/backup-bucket/source", "/backup-bucket/out", + **{"x-amz-copy-source-range": "bytes=0-2"}) + assert ignored_range.status_code == 501 + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/out").content == b"payload" + + +@pytest.mark.parametrize("key", ["folder/a%20space.txt", "caf%C3%A9.bin", "percent%2525.txt"]) +def test_copy_preserves_percent_encoded_source_names(service, key): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = key.encode() + assert signed(service, "PUT", "/backup-bucket/"+key, body).status_code == 200 + response = copy(service, "/backup-bucket/"+key, "/backup-bucket/destination.bin") + assert response.status_code == 200, response.text + assert signed(service, "GET", "/backup-bucket/destination.bin").content == body + + +def test_copy_of_multipart_keeps_composite_etag_without_new_media(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service, key="mpu.bin") + etag = part(service, uid, 1, b"multipart-one-part", key="mpu.bin") + finished = complete(service, uid, [(1, etag)], key="mpu.bin") + assert finished.status_code == 200, finished.text + composite = '"' + hashlib.md5( + bytes.fromhex(etag.strip('"')), usedforsecurity=False + ).hexdigest() + '-1"' + before = service[2].uploads + response = copy(service, "/backup-bucket/mpu.bin", "/backup-bucket/clone.bin") + assert response.status_code == 200, response.text + assert composite.encode() in response.content + assert service[2].uploads == before + assert signed(service, "HEAD", "/backup-bucket/clone.bin").headers["etag"] == composite + assert signed(service, "GET", "/backup-bucket/clone.bin").content == b"multipart-one-part" + + +def test_real_boto3_copy_object_over_http(service): + import socket + import threading + import time + import uvicorn + import web.s3 as gateway + + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error" + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(.05) + assert server.started + client = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=service[1].access_key, + aws_secret_access_key=service[1].secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}), + ) + client.create_bucket(Bucket="sdk-copy") + body = b"real-http-s3-copy" * 19 + client.put_object(Bucket="sdk-copy", Key="old", Body=body) + before = service[2].uploads + copied = client.copy_object(Bucket="sdk-copy", Key="new", + CopySource={"Bucket":"sdk-copy", "Key":"old"}) + assert copied["CopyObjectResult"]["ETag"] == '"' + hashlib.sha256(body).hexdigest() + '"' + assert service[2].uploads == before + client.delete_object(Bucket="sdk-copy", Key="old") + assert client.get_object(Bucket="sdk-copy", Key="new")["Body"].read() == body + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() + + +def test_source_version_change_fails_without_publishing_destination(service): + from teledrive.storage.copy import CopyConflict, copy_object + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source", b"original").status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + original_id = catalog.get("backup-bucket", "source").item.id + finally: + db.close() + assert signed(service, "PUT", "/backup-bucket/source", b"replacement").status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + with pytest.raises(CopyConflict, match="changed"): + copy_object(catalog, "backup-bucket", "source", + "backup-bucket", "destination", + expected_source_id=original_id) + with pytest.raises(FileNotFoundError): + catalog.get("backup-bucket", "destination") + finally: + db.close() + assert signed(service, "GET", "/backup-bucket/source").content == b"replacement" + + +def test_changed_copy_source_header_invalidates_sigv4_signature(service): + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source-one", b"one").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source-two", b"two").status_code == 200 + req = AWSRequest( + method="PUT", url="http://localhost:9000/backup-bucket/target", + data=b"", headers={ + "x-amz-copy-source": "/backup-bucket/source-one", + "x-amz-content-sha256": hashlib.sha256(b"").hexdigest(), + }, + ) + req.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1").add_auth(req) + modified = dict(req.headers) + modified["X-Amz-Copy-Source"] = "/backup-bucket/source-two" + result = service[0].put("/backup-bucket/target", content=b"", headers=modified) + assert result.status_code == 403 + assert b"SignatureDoesNotMatch" in result.content + assert signed(service, "GET", "/backup-bucket/target").status_code == 404 + + +def test_copy_metadata_directive_cannot_be_injected_into_signed_request(service): + """An intermediary must not strip or replace immutable copy metadata.""" + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + source = signed( + service, "PUT", "/backup-bucket/source-copy-metadata", b"persistent", + extra_headers={"x-amz-meta-retain": "yes"}, + ) + assert source.status_code == 200, source.text + path = "/backup-bucket/unsigned-directive" + request = AWSRequest( + method="PUT", url="http://localhost:9000" + path, data=b"", + headers={ + "x-amz-content-sha256": hashlib.sha256(b"").hexdigest(), + "x-amz-copy-source": "/backup-bucket/source-copy-metadata", + }, + ) + request.context["payload_signing_enabled"] = True + issued = service[1] + S3SigV4Auth( + Credentials(issued.access_key, issued.secret_key), + "s3", "us-east-1", + ).add_auth(request) + injected = dict(request.headers) + assert "x-amz-metadata-directive" not in injected + injected["x-amz-metadata-directive"] = "REPLACE" + denied = service[0].put(path, content=b"", headers=injected) + assert denied.status_code == 403, denied.text + assert b"AccessDenied" in denied.content + assert signed(service, "GET", path).status_code == 404 + + original = copy( + service, "/backup-bucket/source-copy-metadata", + "/backup-bucket/signed-metadata-copy", + ) + assert original.status_code == 200, original.text + head = signed(service, "HEAD", "/backup-bucket/signed-metadata-copy") + assert head.headers["x-amz-meta-retain"] == "yes" + assert service[2].uploads == 1 # copy never reuploads Telegram bytes diff --git a/tests/s3/test_s3_cors.py b/tests/s3/test_s3_cors.py new file mode 100644 index 00000000..c25afc2d --- /dev/null +++ b/tests/s3/test_s3_cors.py @@ -0,0 +1,284 @@ +"""Bucket CORS S3 compatibility: boto3 XML contract and browser preflights. + +Real SQLite and AWS SigV4; fake Telegram transport is used only to measure +the absence of payload calls. CORS never bypasses signed S3 permissions. +""" +from __future__ import annotations + +import hashlib +import os +import socket +import threading +import time +from xml.etree import ElementTree as ET + +import boto3 +from botocore.config import Config + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.cors import parse_cors_xml, CORSError + +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def config_xml(origins=None, methods=None, headers=None, expose=None, max_age=180): + node = ET.Element("CORSConfiguration") + rule = ET.SubElement(node, "CORSRule") + for origin in origins or ["https://app.example.com"]: + ET.SubElement(rule, "AllowedOrigin").text = origin + for method in methods or ["GET", "PUT"]: + ET.SubElement(rule, "AllowedMethod").text = method + for value in headers or ["authorization", "content-type", "x-amz-*"]: + ET.SubElement(rule, "AllowedHeader").text = value + for value in expose or ["ETag"]: + ET.SubElement(rule, "ExposeHeader").text = value + ET.SubElement(rule, "MaxAgeSeconds").text = str(max_age) + return ET.tostring(node) + + +def create_bucket(svc, name="cors-bucket"): + assert signed(svc, "PUT", "/" + name).status_code == 200 + + +def test_signed_bucket_cors_round_trip_preflight_and_regular_get(service): + client, _, fake, _, _ = service + create_bucket(service) + url = "/cors-bucket?cors" + baseline = signed(service, "GET", url) + assert baseline.status_code == 404 and b"NoSuchCORSConfiguration" in baseline.content + prior = (fake.uploads, fake.downloads) + assert client.options("/cors-bucket/file.bin", headers={ + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET" + }).status_code == 403 + created = signed(service, "PUT", url, config_xml()) + assert created.status_code == 200, created.text + config = signed(service, "GET", url) + assert config.status_code == 200 + parsed = ET.fromstring(config.content) + assert parsed.findtext(NS+"CORSRule/"+NS+"MaxAgeSeconds") == "180" + assert parsed.findtext(NS+"CORSRule/"+NS+"AllowedOrigin") == "https://app.example.com" + preflight = client.options("/cors-bucket/file.bin", headers={ + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "PUT", + "Access-Control-Request-Headers": "Content-Type,X-Amz-Meta-Custom", + }) + assert preflight.status_code == 200, preflight.text + assert preflight.headers["access-control-allow-origin"] == "https://app.example.com" + assert "PUT" in preflight.headers["access-control-allow-methods"] + assert preflight.headers["access-control-max-age"] == "180" + assert "content-type" in preflight.headers["access-control-allow-headers"].lower() + assert "authorization" not in preflight.headers + assert (fake.uploads, fake.downloads) == prior + # CORS is not authorization: unsiged GET remains 403 even for allowed origins. + unauth = client.get("/cors-bucket/private.bin", headers={"Origin": "https://app.example.com"}) + assert unauth.status_code == 403 + assert unauth.headers["access-control-allow-origin"] == "https://app.example.com" + wrong_origin = client.options("/cors-bucket/private.bin", headers={ + "Origin": "https://evil.example.net", + "Access-Control-Request-Method": "GET", + }) + assert wrong_origin.status_code == 403 + wrong_header = client.options("/cors-bucket/private.bin", headers={ + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": "X-Untrusted", + }) + assert wrong_header.status_code == 403 + removed = signed(service, "DELETE", url) + assert removed.status_code == 204 + assert signed(service, "GET", url).status_code == 404 + assert client.options("/cors-bucket/file.bin", headers={ + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + }).status_code == 403 + + +def test_cors_does_not_leak_across_buckets_and_is_durable(service): + create_bucket(service) + assert signed(service, "PUT", "/empty-bucket").status_code == 200 + assert signed(service, "PUT", "/cors-bucket?cors", config_xml()).status_code == 200 + client, _, _, connect, _ = service + assert client.options("/empty-bucket/x", headers={ + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + }).status_code == 403 + db = connect() + try: + from teledrive.s3.cors import BucketCORSStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + assert BucketCORSStore(BucketCatalog(ManifestStore(db))).get("cors-bucket")[0]["max_age"] == 180 + finally: + db.close() + assert signed(service, "DELETE", "/cors-bucket").status_code == 204 + assert signed(service, "PUT", "/cors-bucket").status_code == 200 + assert signed(service, "GET", "/cors-bucket?cors").status_code == 404 + + +def test_scoped_get_put_delete_cors_independent_of_object_permissions(service): + create_bucket(service) + db = service[3]() + try: + keys = S3CredentialStore(db, service[4]) + obj = keys.create("object-reader", ["s3:GetObject"]) + reader = keys.create("cors-reader", ["s3:GetBucketCORS"]) + writer = keys.create("cors-writer", ["s3:PutBucketCORS"]) + finally: + db.close() + path = "/cors-bucket?cors" + unauthorized = signed(service, "PUT", path, config_xml(), + access=obj.access_key, secret=obj.secret_key) + assert unauthorized.status_code == 403 + allowed = signed(service, "PUT", path, config_xml(), + access=writer.access_key, secret=writer.secret_key) + assert allowed.status_code == 200, allowed.text + denied = signed(service, "GET", path, + access=writer.access_key, secret=writer.secret_key) + assert denied.status_code == 403 + visible = signed(service, "GET", path, + access=reader.access_key, secret=reader.secret_key) + assert visible.status_code == 200 + cannot_delete = signed(service, "DELETE", path, + access=reader.access_key, secret=reader.secret_key) + assert cannot_delete.status_code == 403 + can_delete = signed(service, "DELETE", path, + access=writer.access_key, secret=writer.secret_key) + assert can_delete.status_code == 204 + + +def test_malformed_xml_entity_attack_bad_digest_preserve_previous_config(service): + create_bucket(service) + path = "/cors-bucket?cors" + original = config_xml() + assert signed(service, "PUT", path, original).status_code == 200 + for malformed in [ + b']>&bomb;', + b"GET", + config_xml(origins=["https://a.example.com/path"]), + config_xml(methods=["TRACE"]), + config_xml(origins=["https://*.example.com.evil.test:bad"]), + config_xml(max_age=100000), + ]: + rejected = signed(service, "PUT", path, malformed) + assert rejected.status_code == 400, (malformed[:100], rejected.text) + assert signed(service, "GET", path).status_code == 200 + # Original rule remains even after malformed and hostile policy writes. + assert b"https://app.example.com" in signed(service, "GET", path).content + assert signed(service, "PUT", path, b"x" * (64 * 1024 + 1)).status_code == 400 + + +def test_real_boto3_bucket_cors_over_http_socket(service): + import uvicorn + import web.s3 as gateway + + client, creds, remote, _, _ = service + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, log_level="error", lifespan="off" + )) + worker = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + worker.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + sdk = boto3.client("s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", s3={"addressing_style": "path", + "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts":0}), + ) + sdk.create_bucket(Bucket="cors-sdk") + before=(remote.uploads, remote.downloads) + result = sdk.put_bucket_cors(Bucket="cors-sdk", CORSConfiguration={"CORSRules":[{ + "AllowedOrigins":["https://console.example.com"], "AllowedMethods":["GET", "HEAD"], + "AllowedHeaders":["x-amz-*"], "ExposeHeaders":["ETag"], "MaxAgeSeconds":600, + }]}) + assert result["ResponseMetadata"]["HTTPStatusCode"] == 200 + info = sdk.get_bucket_cors(Bucket="cors-sdk") + assert info["CORSRules"][0]["AllowedOrigins"] == ["https://console.example.com"] + assert info["CORSRules"][0]["MaxAgeSeconds"] == 600 + sdk.delete_bucket_cors(Bucket="cors-sdk") + import botocore.exceptions + try: + sdk.get_bucket_cors(Bucket="cors-sdk") + assert False, "Deleted CORS must return NoSuchCORSConfiguration" + except botocore.exceptions.ClientError as exc: + assert exc.response["ResponseMetadata"]["HTTPStatusCode"] == 404 + assert exc.response["Error"]["Code"] == "NoSuchCORSConfiguration" + # Separately verify TeleDrive's exact S3 XML error contract, rather + # than relying on botocore's lossy HTTP error parser. + raw = signed(service, "GET", "/cors-sdk?cors") + assert raw.status_code == 404 + assert ET.fromstring(raw.content).findtext("Code") == "NoSuchCORSConfiguration" + assert (remote.uploads, remote.downloads) == before + finally: + server.should_exit = True + worker.join(timeout=10) + sock.close() + assert not worker.is_alive() + + +def test_wildcard_subdomain_cors_does_not_match_lookalikes(service): + create_bucket(service) + body = config_xml( + origins=["https://*.example.com"], + methods=["GET", "POST"], + headers=["x-amz-*"], + expose=["ETag"], + ) + assert signed(service, "PUT", "/cors-bucket?cors", body).status_code == 200 + client = service[0] + allowed = client.options("/cors-bucket/one.txt", headers={ + "Origin": "https://console.example.com", + "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": "X-Amz-Date,X-Amz-Meta-Project", + }) + assert allowed.status_code == 200 + assert allowed.headers["access-control-allow-origin"] == "https://console.example.com" + for origin in ( + "https://example.com", + "https://console.example.com.evil.invalid", + "http://console.example.com", + "https://sub.deeper.example.com", + "null", + ): + denied = client.options("/cors-bucket/one.txt", headers={ + "Origin": origin, "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": "X-Amz-Date", + }) + assert denied.status_code == 403, origin + + +def test_preflight_rejects_header_abuse_and_does_not_authenticate_uploads(service): + create_bucket(service) + assert signed(service, "PUT", "/cors-bucket?cors", config_xml( + origins=["*"], methods=["GET", "PUT"], headers=["*"] + )).status_code == 200 + client, _, transport, _, _ = service + huge = client.options("/cors-bucket/item", headers={ + "Origin": "https://client.example.net", + "Access-Control-Request-Method": "PUT", + "Access-Control-Request-Headers": ",".join(f"X-Foo-{x}" for x in range(33)), + }) + assert huge.status_code == 403 + before = transport.uploads + # Origin and successful OPTIONS do not authorize a subsequent anonymous PUT. + assert client.options("/cors-bucket/item", headers={ + "Origin": "https://client.example.net", + "Access-Control-Request-Method": "PUT", + }).status_code == 200 + direct = client.put("/cors-bucket/item", content=b"must-not-upload", + headers={"Origin": "https://client.example.net"}) + assert direct.status_code == 403 + assert before == transport.uploads diff --git a/tests/s3/test_s3_credentials.py b/tests/s3/test_s3_credentials.py new file mode 100644 index 00000000..14139a6b --- /dev/null +++ b/tests/s3/test_s3_credentials.py @@ -0,0 +1,68 @@ +"""S3 key lifecycle and signature abuse regression tests.""" +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +import pytest +from cryptography.fernet import Fernet +from sqlalchemy import select +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore, credentials, load_master_key + + +def test_secret_is_encrypted_and_survives_database_restart(tmp_path): + path = str(tmp_path / "state.db") + master = Fernet.generate_key() + db = connect_database(path) + try: + keys = S3CredentialStore(db, master) + value = keys.create("sync", ["s3:GetObject"], now=1000) + row = db.connection.execute(select(credentials.c.encrypted_secret)).scalar_one() + assert value.secret_key not in row + assert value.access_key not in row + assert keys.resolve(value.access_key, "s3:GetObject", now=1001) == value.secret_key + assert keys.resolve(value.access_key, "s3:DeleteObject", now=1001) is None + assert "secret_key" not in str(keys.list()) + assert keys.revoke(value.id) + assert not keys.revoke(value.id) + assert keys.resolve(value.access_key, "s3:GetObject", now=1001) is None + finally: + db.close() + reopened = connect_database(path) + try: + keys = S3CredentialStore(reopened, master) + assert keys.resolve(value.access_key, "s3:GetObject", now=1001) is None + finally: + reopened.close() + + +def test_expiration_and_invalid_scopes(tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + keys = S3CredentialStore(db, Fernet.generate_key()) + with pytest.raises(ValueError): + keys.create("bad", ["admin:*"]) + with pytest.raises(ValueError): + keys.create("expired", ["s3:GetObject"], expires_at=2, now=2) + issued = keys.create("temp", ["s3:GetObject"], expires_at=110, now=100) + assert keys.resolve(issued.access_key, "s3:GetObject", now=109) + assert keys.resolve(issued.access_key, "s3:GetObject", now=110) is None + finally: + db.close() + + +def test_master_key_file_and_fail_closed(monkeypatch, tmp_path): + key = Fernet.generate_key() + keyfile = tmp_path / "master" + keyfile.write_bytes(key + b"\n") + import web.app as webapp + import os + os.environ["TELEDRIVE_S3_MASTER_KEY_FILE"] = str(keyfile) + assert load_master_key() == key + keyfile.write_text("bad-key") + with pytest.raises(RuntimeError): + load_master_key() + monkeypatch.delenv("TELEDRIVE_S3_MASTER_KEY_FILE") + monkeypatch.delenv("TELEDRIVE_S3_MASTER_KEY", raising=False) + with pytest.raises(RuntimeError): + load_master_key() diff --git a/tests/s3/test_s3_delete_markers.py b/tests/s3/test_s3_delete_markers.py new file mode 100644 index 00000000..9bb40197 --- /dev/null +++ b/tests/s3/test_s3_delete_markers.py @@ -0,0 +1,360 @@ +"""S3 versioned tombstones: publication fencing and logical VersionId deletion. + +All writes use real signed TestClient HTTP and SQLite transactions. Telegram +fake tracks uploads; physical deletion is categorically forbidden. +""" +from __future__ import annotations + +import hashlib +from xml.etree import ElementTree as ET + +import pytest + +from tests.s3.test_s3_gateway import service, signed +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def _current_id(service, key="file.bin"): + conn = service[3]() + try: + return BucketCatalog(ManifestStore(conn)).get("backup-bucket", key).item.id + finally: + conn.close() + + +def _versions(service, *, suffix=""): + result = signed(service, "GET", "/backup-bucket?versions" + suffix) + assert result.status_code == 200, result.text + root = ET.fromstring(result.content) + rows = [] + for node in root: + if node.tag in (NS + "Version", NS + "DeleteMarker"): + rows.append(( + node.findtext(NS + "Key"), + node.findtext(NS + "VersionId"), + node.tag == NS + "DeleteMarker", + node.findtext(NS + "IsLatest") == "true", + )) + return rows, root + + +def test_delete_marker_hides_latest_but_preserves_historical_bytes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"first").status_code == 200 + previous = _current_id(service) + before = service[2].uploads + deletion = signed(service, "DELETE", "/backup-bucket/file.bin") + assert deletion.status_code == 204, deletion.text + assert deletion.headers["x-amz-delete-marker"] == "true" + marker = deletion.headers["x-amz-version-id"] + assert marker and marker != previous + latest = signed(service, "GET", "/backup-bucket/file.bin") + assert latest.status_code == 404 + assert latest.headers["x-amz-delete-marker"] == "true" + assert latest.headers["x-amz-version-id"] == marker + assert signed(service, "HEAD", "/backup-bucket/file.bin").status_code == 404 + historical = signed(service, "GET", f"/backup-bucket/file.bin?versionId={previous}") + assert historical.status_code == 200 and historical.content == b"first" + selected = signed(service, "GET", f"/backup-bucket/file.bin?versionId={marker}") + assert selected.status_code == 405 + assert selected.headers["x-amz-delete-marker"] == "true" + assert selected.headers["x-amz-version-id"] == marker + rows, _ = _versions(service) + assert rows == [("file.bin", marker, True, True), + ("file.bin", previous, False, False)] + assert service[2].uploads == before + + +def test_delete_marker_version_removal_restores_prior_object_without_reupload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"old").status_code == 200 + old = _current_id(service) + assert signed(service, "PUT", "/backup-bucket/file.bin", b"latest").status_code == 200 + current = _current_id(service) + marker_response = signed(service, "DELETE", "/backup-bucket/file.bin") + marker = marker_response.headers["x-amz-version-id"] + before = service[2].uploads + unlinked = signed(service, "DELETE", f"/backup-bucket/file.bin?versionId={marker}") + assert unlinked.status_code == 204 + assert unlinked.headers["x-amz-delete-marker"] == "true" + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"latest" + assert _current_id(service) == current + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={marker}").status_code == 404 + remove_current = signed(service, "DELETE", f"/backup-bucket/file.bin?versionId={current}") + assert remove_current.status_code == 204 + assert _current_id(service) == old + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"old" + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={current}").status_code == 404 + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={old}").content == b"old" + rows, _ = _versions(service) + assert rows == [("file.bin", old, False, True)] + assert service[2].uploads == before + + +def test_multiple_markers_are_immutable_and_current_last_marker_can_be_removed(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"payload").status_code == 200 + first = signed(service, "DELETE", "/backup-bucket/file.bin").headers["x-amz-version-id"] + second = signed(service, "DELETE", "/backup-bucket/file.bin").headers["x-amz-version-id"] + assert first != second + assert signed(service, "DELETE", f"/backup-bucket/file.bin?versionId={second}").status_code == 204 + last = signed(service, "GET", "/backup-bucket/file.bin") + assert last.status_code == 404 and last.headers["x-amz-version-id"] == first + assert signed(service, "DELETE", f"/backup-bucket/file.bin?versionId={first}").status_code == 204 + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"payload" + + +def test_missing_foreign_staged_and_deleted_version_ids_never_leak(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/one.bin", b"one").status_code == 200 + actual = _current_id(service, "one.bin") + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + staged = cat.manifests.begin("backup-bucket", "one.bin", 0, 123) + cat.manifests.finish(staged, hashlib.sha256(b"").hexdigest(), 123) + finally: + db.close() + before = service[2].uploads + for key, version in ( + ("one.bin", "nonexistent"), + ("other.bin", actual), + ("one.bin", staged), + ): + response = signed(service, "DELETE", f"/backup-bucket/{key}?versionId={version}") + assert response.status_code == 404 + for path in ( + "/backup-bucket/one.bin?versionId=", + "/backup-bucket/one.bin?versionId=abc&versionId=xyz", + ): + assert signed(service, "DELETE", path).status_code == 400 + # Tagging is an independent versioned operation. Unknown IDs return + # NoSuchVersion, not a malformed delete-version selector. + assert signed(service, "DELETE", + "/backup-bucket/one.bin?tagging&versionId=abc").status_code == 404 + assert signed(service, "GET", "/backup-bucket/one.bin").content == b"one" + assert service[2].uploads == before + + +def test_latest_marker_can_be_replaced_using_signed_if_none_match_and_copy(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source.bin", b"copy-source").status_code == 200 + marker = signed(service, "DELETE", "/backup-bucket/file.bin").headers["x-amz-version-id"] + before = service[2].uploads + copied = signed(service, "PUT", "/backup-bucket/file.bin", b"", + extra_headers={"x-amz-copy-source": "/backup-bucket/source.bin"}) + assert copied.status_code == 200, copied.text + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"copy-source" + assert service[2].uploads == before + assert signed(service, "DELETE", "/backup-bucket/file.bin").status_code == 204 + written = signed(service, "PUT", "/backup-bucket/file.bin", b"new", + extra_headers={"If-None-Match": "*"}) + assert written.status_code == 200, written.text + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"new" + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={marker}").status_code == 405 + + +def test_removed_version_is_inaccessible_through_copy_and_tagging(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source.bin", b"private-version").status_code == 200 + source_id = _current_id(service, "source.bin") + assert signed(service, "DELETE", f"/backup-bucket/source.bin?versionId={source_id}").status_code == 204 + before = service[2].uploads + failed_copy = signed(service, "PUT", "/backup-bucket/after.bin", b"", + extra_headers={"x-amz-copy-source": + f"/backup-bucket/source.bin?versionId={source_id}"}) + assert failed_copy.status_code == 404 + failed_tags = signed(service, "GET", f"/backup-bucket/source.bin?tagging&versionId={source_id}") + assert failed_tags.status_code == 404 + assert service[2].uploads == before + + +def test_delete_marker_listing_pagination_across_versions(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for key, value in (("a.bin", b"one"), ("a.bin", b"two"), ("b.bin", b"three")): + assert signed(service, "PUT", f"/backup-bucket/{key}", value).status_code == 200 + marker = signed(service, "DELETE", "/backup-bucket/a.bin").headers["x-amz-version-id"] + collected = [] + suffix = "&max-keys=1" + for _ in range(6): + page, root = _versions(service, suffix=suffix) + assert len(page) == 1 + collected.extend(page) + if root.findtext(NS + "IsTruncated") == "false": + break + suffix = ("&max-keys=1&key-marker=" + root.findtext(NS + "NextKeyMarker") + + "&version-id-marker=" + root.findtext(NS + "NextVersionIdMarker")) + assert len(collected) == 4 + assert collected[0] == ("a.bin", marker, True, True) + assert len({row[1] for row in collected}) == 4 + assert collected[-1][0] == "b.bin" + + +def test_delete_last_version_leaves_missing_head_and_keeps_remote(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"keep-on-telegram").status_code == 200 + only = _current_id(service) + before = service[2].uploads + assert signed(service, "DELETE", f"/backup-bucket/file.bin?versionId={only}").status_code == 204 + assert signed(service, "GET", "/backup-bucket/file.bin").status_code == 404 + rows, _ = _versions(service) + assert rows == [] + assert service[2].uploads == before + + +def test_version_tagging_selector_is_not_shadowed_by_version_delete(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"kept").status_code == 200 + version = _current_id(service) + deleted_tags = signed(service, "DELETE", + f"/backup-bucket/file.bin?tagging&versionId={version}") + assert deleted_tags.status_code == 204, deleted_tags.text + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={version}").content == b"kept" + # An operation selector collision is rejected instead of deleting bytes. + invalid = signed(service, "DELETE", + f"/backup-bucket/file.bin?tagging&versionId={version}&bogus=1") + assert invalid.status_code in (400, 501) + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"kept" + + +def test_delete_version_requires_delete_scope_and_signed_query(service): + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + from teledrive.s3.credentials import S3CredentialStore + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"protect").status_code == 200 + version = _current_id(service) + db = service[3]() + try: + read_only = S3CredentialStore(db, service[4]).create( + "read-only-version", ["s3:GetObject", "s3:ListBucketVersions"], + ) + finally: + db.close() + denied = signed(service, "DELETE", + f"/backup-bucket/file.bin?versionId={version}", + access=read_only.access_key, secret=read_only.secret_key) + assert denied.status_code == 403 + path = f"/backup-bucket/file.bin?versionId={version}" + req = AWSRequest(method="DELETE", url="http://localhost:9000" + path, + data=b"", headers={ + "x-amz-content-sha256": hashlib.sha256(b"").hexdigest(), + }) + S3SigV4Auth(Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1").add_auth(req) + # Keep the authenticated header signature but change the query selector. + changed = service[0].delete( + "/backup-bucket/file.bin?versionId=forged", headers=dict(req.headers) + ) + assert changed.status_code == 403 + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={version}").content == b"protect" + + +def test_marker_is_retained_by_online_metadata_snapshot(tmp_path, service): + from scripts.ops.backup_metadata import backup_sqlite + from scripts.ops.verify_metadata_backup import verify_snapshot + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"backup-me").status_code == 200 + marker = signed(service, "DELETE", "/backup-bucket/file.bin").headers["x-amz-version-id"] + database = tmp_path / "state.db" + snapshot = tmp_path / "snapshot.db" + checksum = backup_sqlite(database, snapshot) + result = verify_snapshot(snapshot, checksum) + assert result["valid"] is True + assert result["versions"] == 2 + assert result["heads"] == 1 + assert result["telegram_chunk_refs"] == 1 + assert signed(service, "GET", f"/backup-bucket/file.bin?versionId={marker}").status_code == 405 + + +def test_batch_delete_produces_visible_markers_and_never_deletes_telegram(service): + import base64 + from tests.s3.test_s3_batch_delete import delete_xml + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"batch-data").status_code == 200 + data = delete_xml(["file.bin", "unlisted.bin"]) + before = dict(service[2].parts) + digest = base64.b64encode(hashlib.md5(data, usedforsecurity=False).digest()).decode() + response = signed(service, "POST", "/backup-bucket?delete", data, + extra_headers={"Content-MD5": digest}) + assert response.status_code == 200, response.text + entries, _ = _versions(service) + assert sum(is_marker for _, _, is_marker, _ in entries) == 2 + assert signed(service, "GET", "/backup-bucket/file.bin").status_code == 404 + assert service[2].parts == before + + + +def test_removed_version_is_invisible_to_all_tagging_methods_and_direct_store(service): + """S3 gateway and SQL service share the same irreversible removal fence.""" + from teledrive.s3.tagging import ObjectTagStore, TaggingError + from sqlalchemy import select + from teledrive.s3.tagging import object_tags + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"revoked-version-must-stay-hidden" * 4097 + assert signed(service, "PUT", "/backup-bucket/secure.bin", data).status_code == 200 + version = _current_id(service, "secure.bin") + assert signed(service, "PUT", + f"/backup-bucket/secure.bin?tagging&versionId={version}", + b"safeoriginal").status_code == 200 + before = service[2].uploads + removed = signed(service, "DELETE", + f"/backup-bucket/secure.bin?versionId={version}") + assert removed.status_code == 204 + for method in ("GET", "PUT", "DELETE"): + path = f"/backup-bucket/secure.bin?tagging&versionId={version}" + body = (b"safe" + b"attack") if method == "PUT" else b"" + res = signed(service, method, path, body) + assert res.status_code == 404, (method, res.text) + assert b"NoSuchVersion" in res.content + + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + tags = ObjectTagStore(cat) + assert not cat.is_visible_version("backup-bucket", "secure.bin", version) + with pytest.raises(TaggingError) as denied: + tags.replace_published("backup-bucket", "secure.bin", + version, [("safe", "attack")]) + assert denied.value.code == "NoSuchVersion" + original = db.connection.execute(select( + object_tags.c.tag_key, object_tags.c.tag_value + ).where(object_tags.c.object_id == version)).all() + assert [tuple(x) for x in original] == [("safe", "original")] + finally: + db.close() + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/secure.bin").status_code == 404 + + +def test_published_version_visibility_requires_untampered_sql_manifest_identity(service): + """A corrupted published index may not make another object's bytes visible.""" + from sqlalchemy import update + from teledrive.storage.manifest import objects + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/other.bin", b"private" * 791).status_code == 200 + version = _current_id(service, "other.bin") + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + assert catalog.is_visible_version("backup-bucket", "other.bin", version) + db.connection.execute(update(objects).where( + objects.c.id == version + ).values(object_key="wrong-key.bin")) + db.commit() + assert not catalog.is_visible_version("backup-bucket", "other.bin", version) + finally: + db.close() + res = signed(service, "GET", + f"/backup-bucket/other.bin?versionId={version}") + assert res.status_code in (404, 409), res.text diff --git a/tests/s3/test_s3_fail_closed.py b/tests/s3/test_s3_fail_closed.py new file mode 100644 index 00000000..73e0920e --- /dev/null +++ b/tests/s3/test_s3_fail_closed.py @@ -0,0 +1,137 @@ +"""P0 acceptance: crash, revoked credentials and fail-closed S3 operations. + +Uses the production HTTP handler, SigV4, real SQLite and deterministic +Telegram-compatible I/O. No production Telegram messages can be deleted. +""" +from __future__ import annotations + +import hashlib +import time + +import pytest + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import begin, part, complete +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +@pytest.mark.parametrize("after_remote_ack", [False, True]) +def test_remote_crash_before_or_after_ack_never_overwrites_ready_head( + service, monkeypatch, after_remote_ack +): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/asset.bin", + b"already-safe").status_code == 200 + remote = service[2] + original_call = remote.upload + old_uploads = remote.uploads + + async def crashed_send(path, *, object_id, ordinal, checksum): + if after_remote_ack: + await original_call(path, object_id=object_id, + ordinal=ordinal, checksum=checksum) + raise RuntimeError("simulated crash before SQL journal commit") + + monkeypatch.setattr(remote, "upload", crashed_send) + failed = signed(service, "PUT", "/backup-bucket/asset.bin", b"new-unpublished") + assert failed.status_code == 503, failed.text + assert remote.uploads == old_uploads + int(after_remote_ack) + assert signed(service, "GET", "/backup-bucket/asset.bin").content == b"already-safe" + + db = service[3]() + try: + store = ManifestStore(db) + states = store.interrupted() + assert len(states) == 1 + assert states[0][1] in ("pending", "failed") + # A failed/unindexed Telegram send must never become visible as a + # second S3 head, and physical cleanup must remain explicit. + assert BucketCatalog(store).get("backup-bucket", "asset.bin").item.size == len(b"already-safe") + finally: + db.close() + + +@pytest.mark.parametrize("method,path,content,headers", [ + ("PUT", "/backup-bucket/forbidden.bin", b"unauthorized", {}), + ("DELETE", "/backup-bucket/asset.bin", b"", {}), + ("POST", "/backup-bucket/asset.bin?uploads", b"", {}), + ("PUT", "/backup-bucket/copied.bin", b"", {"x-amz-copy-source": "/backup-bucket/asset.bin"}), + ("GET", "/backup-bucket/asset.bin?tagging", b"", {}), + ("GET", "/backup-bucket?uploads", b"", {}), +]) +def test_read_only_s3_credential_cannot_mutate_or_enumerate( + service, method, path, content, headers +): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/asset.bin", + b"original").status_code == 200 + db = service[3]() + try: + reader = S3CredentialStore(db, service[4]).create( + "read-only acceptance", ["s3:GetObject"] + ) + finally: + db.close() + assert signed(service, "HEAD", "/backup-bucket/asset.bin", + access=reader.access_key, secret=reader.secret_key).status_code == 200 + assert signed(service, "GET", "/backup-bucket/asset.bin", + access=reader.access_key, secret=reader.secret_key).content == b"original" + uploads = service[2].uploads + rejected = signed(service, method, path, content, extra_headers=headers, + access=reader.access_key, secret=reader.secret_key) + assert rejected.status_code == 403, rejected.text + assert b"AccessDenied" in rejected.content + assert service[2].uploads == uploads + assert signed(service, "GET", "/backup-bucket/asset.bin").content == b"original" + + +def test_revocation_between_multipart_upload_and_complete_fails_closed(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/asset.bin", b"preserve").status_code == 200 + uid = begin(service) + etag = part(service, uid, 1, b"staged-before-revocation") + original_uploads = service[2].uploads + db = service[3]() + try: + credentials = S3CredentialStore(db, service[4]) + reader = credentials.create("fresh-reader", ["s3:GetObject"]) + assert credentials.revoke(service[1].id) + finally: + db.close() + assert signed(service, "PUT", + f"/backup-bucket/asset.bin?partNumber=2&uploadId={uid}", + b"must-not-upload").status_code == 403 + assert complete(service, uid, [(1, etag)]).status_code == 403 + assert signed(service, "DELETE", + f"/backup-bucket/asset.bin?uploadId={uid}").status_code == 403 + assert signed(service, "GET", "/backup-bucket/asset.bin", + access=reader.access_key, secret=reader.secret_key).content == b"preserve" + assert service[2].uploads == original_uploads + + +def test_expired_credential_denied_by_real_sql_resolver_without_sleep(service, monkeypatch): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + now = int(time.time()) + db = service[3]() + try: + expiring = S3CredentialStore(db, service[4]).create( + "expires-in-one-second", ["s3:PutObject", "s3:GetObject"], + now=now, expires_at=now + 1, + ) + finally: + db.close() + original_resolve = S3CredentialStore.resolve + + def expired(self, access_key, permission): + return original_resolve(self, access_key, permission, now=now + 2) + + monkeypatch.setattr(S3CredentialStore, "resolve", expired) + before = service[2].uploads + denied = signed(service, "PUT", "/backup-bucket/expired.bin", + b"must-never-reach-telegram", access=expiring.access_key, + secret=expiring.secret_key) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + assert service[2].uploads == before diff --git a/tests/s3/test_s3_gateway.py b/tests/s3/test_s3_gateway.py new file mode 100644 index 00000000..d9824901 --- /dev/null +++ b/tests/s3/test_s3_gateway.py @@ -0,0 +1,428 @@ +"""S3 SigV4 and Telegram-backed HTTP behavior tests. + +Botocore signs real HTTP wire requests; a deterministic fake USER MTProto +transport holds real bytes while SQLite stores only metadata and encrypted keys. +""" +from __future__ import annotations + +import hashlib +import os +from urllib.parse import quote + +import pytest +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +import web.app as admin +import web.s3 as gateway +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore, DEFAULT_PERMISSIONS + + +class FakeTelegram: + """Disk-backed USER MTProto stand-in with the engine transport contract.""" + + def __init__(self): + self.parts = {} + self.uploads = 0 + self.downloads = 0 + + async def upload(self, path, *, object_id, ordinal, checksum): + self.uploads += 1 + fid = f"file-{self.uploads}" + self.parts[fid] = path.read_bytes() + return "mtproto-user", -1001234567890, self.uploads + + async def verify_target(self): + pass + + async def download(self, reference, destination): + self.downloads += 1 + destination.write_bytes(self.parts[f"file-{reference.message_id}"]) + + +@pytest.fixture +def service(monkeypatch, tmp_path): + key = Fernet.generate_key() + database_path = tmp_path / "state.db" + db = connect_database(str(database_path)) + cred_store = S3CredentialStore(db, key) + issued = cred_store.create("S3 integration", list(DEFAULT_PERMISSIONS)) + db.close() + telegram = FakeTelegram() + connection = lambda: connect_database(str(database_path)) + factory = lambda db: telegram + monkeypatch.setattr(gateway, "db_connect", connection) + monkeypatch.setattr(gateway, "storage_transport", factory) + monkeypatch.setattr(gateway, "load_master_key", lambda: key) + monkeypatch.setattr(admin, "STORAGE_SCRATCH_DIR", str(tmp_path / "staging")) + monkeypatch.setattr(admin, "REST_MAX_UPLOAD_BYTES", 8 * 1024 * 1024) + client = TestClient(gateway.app, base_url="http://localhost:9000") + try: + yield client, issued, telegram, connection, key + finally: + client.close() + + +def signed(service, method, path, body=b"", *, extra_headers=None, access=None, secret=None, date=None): + client, issued, _, _, _ = service + headers = {"x-amz-content-sha256": hashlib.sha256(body).hexdigest()} + headers.update(extra_headers or {}) + url = "http://localhost:9000" + path + request = AWSRequest(method=method, url=url, data=body, headers=headers) + request.context["payload_signing_enabled"] = headers["x-amz-content-sha256"] != "UNSIGNED-PAYLOAD" + S3SigV4Auth(Credentials(access or issued.access_key, + secret or issued.secret_key), "s3", "us-east-1").add_auth(request) + if date: + request.headers["X-Amz-Date"] = date + return client.request(method, path, content=body, headers=dict(request.headers)) + + +def test_s3_basic_bucket_object_crud_and_range(service): + client, creds, telegram, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + buckets = signed(service, "GET", "/") + assert buckets.status_code == 200 and b"backup-bucket" in buckets.content + payload = os.urandom(8193) + uploaded = signed(service, "PUT", "/backup-bucket/file.bin", payload) + assert uploaded.status_code == 200, uploaded.text + assert telegram.uploads == 1 + meta = signed(service, "HEAD", "/backup-bucket/file.bin") + assert meta.status_code == 200 + assert meta.headers["content-length"] == str(len(payload)) + assert telegram.downloads == 0 + response = signed(service, "GET", "/backup-bucket/file.bin") + assert response.status_code == 200 + assert response.content == payload + rng = signed(service, "GET", "/backup-bucket/file.bin", + extra_headers={"Range": "bytes=8190-8192"}) + assert rng.status_code == 206 and rng.content == payload[-3:] + listed = signed(service, "GET", "/backup-bucket?list-type=2&prefix=file") + assert listed.status_code == 200 and b"file.bin" in listed.content + assert signed(service, "DELETE", "/backup-bucket/file.bin").status_code == 204 + missing = signed(service, "GET", "/backup-bucket/file.bin") + assert missing.status_code == 404 and b"NoSuchKey" in missing.content + # Physical Telegram parts are deliberately retained for future GC. + assert telegram.uploads == 1 + + +def test_signature_required_tampering_rejected_before_upload(service): + client, issued, bot, _, _ = service + assert client.get("/").status_code == 403 + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + wrong_key = signed(service, "GET", "/", secret="incorrect") + assert wrong_key.status_code == 403 + assert b"SignatureDoesNotMatch" in wrong_key.content + good = AWSRequest(method="PUT", url="http://localhost:9000/backup-bucket/tamper", + data=b"original", + headers={"x-amz-content-sha256": hashlib.sha256(b"original").hexdigest()}) + good.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(issued.access_key, issued.secret_key), + "s3", "us-east-1").add_auth(good) + before = bot.uploads + modified = client.put("/backup-bucket/tamper", content=b"changed", + headers=dict(good.headers)) + assert modified.status_code == 400 + assert bot.uploads == before + assert b"XAmzContentSHA256Mismatch" in modified.content + + +def test_revoke_and_scope_are_enforced_independently(service): + _, issued, _, connection, key = service + assert signed(service, "GET", "/").status_code == 200 + db = connection() + try: + restricted = S3CredentialStore(db, key).create("reader", ["s3:GetObject"]) + assert S3CredentialStore(db, key).revoke(issued.id) + finally: + db.close() + assert signed(service, "GET", "/").status_code == 403 + assert signed(service, "PUT", "/new-bucket", access=restricted.access_key, + secret=restricted.secret_key).status_code == 403 + + +def test_unsigned_payload_and_unsupported_operations_fail_closed(service): + client, _, bot, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + # Botocore's S3SigV4Auth always replaces UNSIGNED-PAYLOAD with a + # SHA-256 for PUT. Mutate the actual signed wire request to verify the + # gateway rejects the unsigned upload before touching Telegram. + req = AWSRequest(method="PUT", url="http://localhost:9000/backup-bucket/unsafe", + data=b"stuff", headers={ + "x-amz-content-sha256": hashlib.sha256(b"stuff").hexdigest(), + }) + req.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1").add_auth(req) + bad_headers = dict(req.headers) + bad_headers["X-Amz-Content-SHA256"] = "UNSIGNED-PAYLOAD" + response = client.put("/backup-bucket/unsafe", content=b"stuff", headers=bad_headers) + assert response.status_code == 400 + assert bot.uploads == 0 + initiated = signed(service, "POST", "/backup-bucket/object?uploads") + assert initiated.status_code == 200 + assert b"UploadId" in initiated.content + listed = signed(service, "GET", "/backup-bucket?uploads") + assert listed.status_code == 200 + assert b"ListMultipartUploadsResult" in listed.content + unsupported = signed(service, "GET", "/backup-bucket?policy") + assert unsupported.status_code == 501 + assert b"NotImplemented" in unsupported.content + + +def test_catalog_pagination_with_signed_s3_queries(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for name in ("a.txt", "b.txt", "c.txt"): + assert signed(service, "PUT", f"/backup-bucket/{name}", name.encode()).status_code == 200 + first = signed(service, "GET", "/backup-bucket?list-type=2&max-keys=2") + assert first.status_code == 200 + import xml.etree.ElementTree as ET + root = ET.fromstring(first.content) + tag = "{http://s3.amazonaws.com/doc/2006-03-01/}" + assert root.findtext(tag+"IsTruncated") == "true" + token = root.findtext(tag+"NextContinuationToken") + second = signed(service, "GET", + "/backup-bucket?list-type=2&max-keys=2&continuation-token="+quote(token)) + assert second.status_code == 200 + keys = [item.findtext(tag+"Key") for item in ET.fromstring(second.content).findall(tag+"Contents")] + assert keys == ["c.txt"] + + +def test_real_boto3_sdk_over_http_loopback(service): + """Exercise an actual boto3 S3 client rather than only hand-built requests.""" + import socket + import threading + import time + import boto3 + import uvicorn + from botocore.config import Config + + _, issued, bot, _, _ = service + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + settings = uvicorn.Config(gateway.app, host="127.0.0.1", port=port, + log_level="error", lifespan="off") + server = uvicorn.Server(settings) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started, "ASGI S3 server did not start" + sdk = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=issued.access_key, aws_secret_access_key=issued.secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}, + ), + ) + assert sdk.create_bucket(Bucket="sdk-bucket")["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert any(b["Name"] == "sdk-bucket" for b in sdk.list_buckets()["Buckets"]) + body = os.urandom(6003) + put_result = sdk.put_object(Bucket="sdk-bucket", Key="nested/data.bin", Body=body) + assert put_result["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sdk.head_object(Bucket="sdk-bucket", Key="nested/data.bin")["ContentLength"] == len(body) + read = sdk.get_object(Bucket="sdk-bucket", Key="nested/data.bin")["Body"].read() + assert read == body + keys = [item["Key"] for item in sdk.list_objects_v2( + Bucket="sdk-bucket", Prefix="nested/" + )["Contents"]] + assert keys == ["nested/data.bin"] + sdk.delete_object(Bucket="sdk-bucket", Key="nested/data.bin") + assert bot.uploads >= 1 and bot.downloads >= 1 + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive(), "ASGI S3 server did not stop" + + +@pytest.mark.parametrize("raw_key", [ + "a%20file.txt", "caf%C3%A9.txt", "folder%2Fencoded-slash.bin", + "x%2525literal-percent.txt", +]) +def test_url_encoded_keys_preserve_s3_sigv4_semantics(service, raw_key): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + content = b"encoded-key-content:" + raw_key.encode("ascii") + uri = "/backup-bucket/" + raw_key + uploaded = signed(service, "PUT", uri, content) + assert uploaded.status_code == 200, uploaded.text + retrieved = signed(service, "GET", uri) + assert retrieved.status_code == 200, retrieved.text + assert retrieved.content == content + + +def test_encoded_path_tampering_invalidates_signature(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"same-body" + req = AWSRequest(method="PUT", url="http://localhost:9000/backup-bucket/a%20b.txt", + data=payload, headers={ + "x-amz-content-sha256": hashlib.sha256(payload).hexdigest(), + }) + req.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1").add_auth(req) + tampered = service[0].put("/backup-bucket/a%20c.txt", headers=dict(req.headers), + content=payload) + assert tampered.status_code == 403 + assert b"SignatureDoesNotMatch" in tampered.content + + +def test_s3_delimiter_common_prefixes_and_safe_pagination(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for key in ("dir-a/1", "dir-a/2", "dir-a/3", "dir-b/one", "root.txt"): + result = signed(service, "PUT", "/backup-bucket/" + key, key.encode()) + assert result.status_code == 200, result.text + import xml.etree.ElementTree as ET + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" + token = None + all_objects = [] + all_prefixes = [] + for _ in range(5): + query = "/backup-bucket?list-type=2&delimiter=%2F&max-keys=1" + if token: + query += "&continuation-token=" + quote(token) + response = signed(service, "GET", query) + assert response.status_code == 200, response.text + root = ET.fromstring(response.content) + all_objects.extend(c.findtext(ns+"Key") for c in root.findall(ns+"Contents")) + all_prefixes.extend(c.findtext(ns+"Prefix") for c in root.findall(ns+"CommonPrefixes")) + token = root.findtext(ns+"NextContinuationToken") + if root.findtext(ns+"IsTruncated") == "false": + break + else: + pytest.fail("S3 continuation token did not terminate") + assert all_prefixes == ["dir-a/", "dir-b/"] + assert all_objects == ["root.txt"] + assert token is None + + +def test_s3_cursor_cannot_be_modified_or_reused_with_a_different_prefix(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for key in ("a/one", "b/two", "c/three"): + assert signed(service, "PUT", "/backup-bucket/" + key, b"data").status_code == 200 + import xml.etree.ElementTree as ET + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" + first = signed(service, "GET", "/backup-bucket?list-type=2&max-keys=1") + token = ET.fromstring(first.content).findtext(ns+"NextContinuationToken") + assert token + corrupted = token[:-2] + ("aa" if token[-2:] != "aa" else "bb") + result = signed(service, "GET", + "/backup-bucket?list-type=2&max-keys=1&continuation-token="+quote(corrupted)) + assert result.status_code == 400 + assert b"InvalidArgument" in result.content + second = signed(service, "GET", + "/backup-bucket?list-type=2&prefix=a%2F&max-keys=1&continuation-token="+quote(token)) + assert second.status_code == 400 + + +def test_s3_folder_larger_than_backend_page_is_not_repeated(): + """A single CommonPrefixes entry spanning >1000 raw keys still counts once.""" + from types import SimpleNamespace + from web.s3 import _page + names = [f"huge/{n:04d}" for n in range(1201)] + ["next/item", "root"] + class FakeCatalog: + def list_objects(self, bucket, *, prefix, after, limit): + selected = [name for name in names if name.startswith(prefix) and name > after][:limit] + return [SimpleNamespace(item=SimpleNamespace(key=key)) for key in selected] + catalog = FakeCatalog() + keys, common, after, skip, more = _page(catalog, "bucket", "", "/", "", 1, "") + assert keys == [] and common == ["huge/"] and more + keys2, common2, after2, skip2, more2 = _page(catalog, "bucket", "", "/", after, 1, skip) + assert common2 == ["next/"] and keys2 == [] + keys3, common3, _, _, more3 = _page(catalog, "bucket", "", "/", after2, 1, skip2) + assert [x.item.key for x in keys3] == ["root"] + assert common3 == [] and not more3 + + +def test_signed_md5_allows_standard_unsigned_payload_without_sacrificing_integrity(service): + """The rclone/aws-sdk variant must sign Content-MD5 and verify actual bytes. + + A mutated body or an omitted signed checksum is rejected *before* upload. + Presigned PUT still requires its SHA-256 checksum-binding policy. + """ + import base64 + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"standard-unsigned-payload-with-content-md5" * 37 + digest = base64.b64encode(hashlib.md5(body, usedforsecurity=False).digest()).decode() + url = "http://localhost:9000/backup-bucket/rclone-compatible.bin" + request = AWSRequest( + method="PUT", url=url, data=body, headers={"Content-MD5": digest}, + ) + class RcloneStyleUnsignedSigner(S3SigV4Auth): + # S3SigV4Auth calculates SHA-256 even if payload_signing_enabled=False; + # override the body canonicalization for rclone's actual wire shape. + def payload(self, request): + return "UNSIGNED-PAYLOAD" + + RcloneStyleUnsignedSigner( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(request) + headers = dict(request.headers) + assert headers.get("X-Amz-Content-SHA256") == "UNSIGNED-PAYLOAD" + assert "content-md5" in headers["Authorization"].lower() + before = service[2].uploads + denied = service[0].put("/backup-bucket/rclone-compatible.bin", + content=b"replaced-body", headers=headers) + assert denied.status_code == 400 + assert service[2].uploads == before + allowed = service[0].put("/backup-bucket/rclone-compatible.bin", + content=body, headers=headers) + assert allowed.status_code == 200, allowed.text + assert service[2].uploads == before + 1 + assert signed(service, "GET", "/backup-bucket/rclone-compatible.bin").content == body + + unsigned = AWSRequest(method="PUT", url="http://localhost:9000/backup-bucket/no-checksum", + data=body) + RcloneStyleUnsignedSigner( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(unsigned) + assert dict(unsigned.headers)["X-Amz-Content-SHA256"] == "UNSIGNED-PAYLOAD" + denied = service[0].put("/backup-bucket/no-checksum", + content=body, headers=dict(unsigned.headers)) + assert denied.status_code == 400 + assert service[2].uploads == before + 1 + + +def test_private_acl_and_standard_class_hints_do_not_weaken_private_storage(service): + """Only the default private/standard hints map to actual gateway behavior.""" + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + value = b"retained-in-telegram" + before = service[2].uploads + ok = signed(service, "PUT", "/backup-bucket/default-hints.txt", value, + extra_headers={"x-amz-acl": "private", + "x-amz-storage-class": "STANDARD"}) + assert ok.status_code == 200, ok.text + assert signed(service, "GET", "/backup-bucket/default-hints.txt").content == value + assert service[2].uploads == before + 1 + + for name, bad in [ + ("x-amz-acl", "public-read"), + ("x-amz-acl", "authenticated-read"), + ("x-amz-storage-class", "GLACIER"), + ("x-amz-storage-class", "STANDARD_IA"), + ]: + refused = signed(service, "PUT", "/backup-bucket/refused", + value, extra_headers={name: bad}) + assert refused.status_code == 501 + assert b"NotImplemented" in refused.content + assert service[2].uploads == before + 1 + assert signed(service, "GET", "/backup-bucket/refused").status_code == 404 diff --git a/tests/s3/test_s3_lifecycle.py b/tests/s3/test_s3_lifecycle.py new file mode 100644 index 00000000..9ec0365f --- /dev/null +++ b/tests/s3/test_s3_lifecycle.py @@ -0,0 +1,186 @@ +"""S3 lifecycle aborts only old multipart sessions, retaining remote media.""" +from __future__ import annotations +import time +from xml.etree import ElementTree as ET +import pytest +from tests.s3.test_s3_gateway import service, signed +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +from teledrive.s3.multipart import MultipartStore +from teledrive.s3.lifecycle import BucketLifecycleStore +from teledrive.s3.credentials import S3CredentialStore + + +def xml(days=7, status="Enabled", prefix="upload/"): + root = ET.Element("LifecycleConfiguration") + rule = ET.SubElement(root, "Rule") + ET.SubElement(rule, "ID").text = "abort-only" + ET.SubElement(rule, "Status").text = status + filt = ET.SubElement(rule, "Filter") + ET.SubElement(filt, "Prefix").text = prefix + act = ET.SubElement(rule, "AbortIncompleteMultipartUpload") + ET.SubElement(act, "DaysAfterInitiation").text = str(days) + return ET.tostring(root) + + +def test_s3_lifecycle_crud_and_old_mpu_only(service): + assert signed(service, "PUT", "/lifecycle-ci").status_code == 200 + target = "/lifecycle-ci?lifecycle" + assert signed(service, "GET", target).status_code == 404 + assert signed(service, "PUT", target, xml()).status_code == 200 + response = signed(service, "GET", target) + assert response.status_code == 200 and b"DaysAfterInitiation" in response.content + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + mpu = MultipartStore(cat) + now = int(time.time()) + aged = mpu.initiate("lifecycle-ci", "upload/old", "owner", now=now-9*86400) + fresh = mpu.initiate("lifecycle-ci", "upload/new", "owner", now=now) + other = mpu.initiate("lifecycle-ci", "other/old", "owner", now=now-9*86400) + lifecycle = BucketLifecycleStore(cat) + assert lifecycle.apply_due(now=now) == 1 + assert lifecycle.apply_due(now=now) == 0 + active = {r.id for r in mpu.list_uploads("lifecycle-ci", "owner")[0]} + assert aged not in active and {fresh, other} <= active + finally: + db.close() + assert service[2].uploads == service[2].downloads == 0 + assert signed(service, "DELETE", target).status_code == 204 + assert signed(service, "GET", target).status_code == 404 + + +@pytest.mark.parametrize("bad,expected", [ + (b"", 400), (xml(days=0), 400), + (xml(days=3651), 400), + (xml().replace(b"AbortIncompleteMultipartUpload", b"Expiration"), 501), + (xml().replace(b"Filter", b"And"), 501), + (b']>&a;', 400), +]) +def test_unimplemented_actions_never_mutate_rule(service, bad, expected): + assert signed(service, "PUT", "/lifecycle-ci").status_code == 200 + path = "/lifecycle-ci?lifecycle" + assert signed(service, "PUT", path, xml()).status_code == 200 + res = signed(service, "PUT", path, bad) + assert res.status_code == expected + assert signed(service, "GET", path).status_code == 200 + + +def test_disabled_rule_and_permission_isolation(service): + assert signed(service, "PUT", "/lifecycle-ci").status_code == 200 + path = "/lifecycle-ci?lifecycle" + assert signed(service, "PUT", path, xml(status="Disabled", days=1)).status_code == 200 + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + mpu = MultipartStore(cat) + ident = mpu.initiate("lifecycle-ci", "upload/old", "owner", + now=int(time.time()) - 20*86400) + assert BucketLifecycleStore(cat).apply_due(now=int(time.time())) == 0 + assert ident in {r.id for r in mpu.list_uploads("lifecycle-ci", "owner")[0]} + auth = S3CredentialStore(db, service[4]) + read = auth.create("lifecycle-read", ["s3:GetLifecycleConfiguration"]) + write = auth.create("lifecycle-write", ["s3:PutLifecycleConfiguration"]) + finally: + db.close() + assert signed(service, "PUT", path, xml(), access=read.access_key, + secret=read.secret_key).status_code == 403 + assert signed(service, "PUT", path, xml(), access=write.access_key, + secret=write.secret_key).status_code == 200 + assert signed(service, "GET", path, access=write.access_key, + secret=write.secret_key).status_code == 403 + assert signed(service, "GET", path, access=read.access_key, + secret=read.secret_key).status_code == 200 + + +def test_real_boto3_lifecycle_control_api(service): + import boto3, socket, threading, uvicorn + from botocore.config import Config + import web.s3 as gateway + assert signed(service, "PUT", "/lifecycle-ci").status_code == 200 + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(128) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config(gateway.app, host="127.0.0.1", + port=port, log_level="error", lifespan="off")) + thread = threading.Thread(target=server.run, kwargs={"sockets":[sock]}, daemon=True) + thread.start() + try: + deadline=time.monotonic()+10 + while not server.started and time.monotonic()]>' + b'1' + b'&x;' + ) + attempts_before = service[2].downloads + denied = signed(service, "POST", f"/backup-bucket/asset.bin?uploadId={uid}", xml) + assert denied.status_code == 400 + assert xml_text(denied, "Code") == "MalformedXML" + assert service[2].downloads == attempts_before + # No state change: still completable with the valid part. + assert complete(service, uid, [(1, etag)]).status_code == 200 + + +def test_boto3_multipart_real_http_transport(service): + """The real boto3 API must parse S3 multipart XML and recover exact bytes.""" + import socket + import threading + import time + import boto3 + import uvicorn + from botocore.config import Config + import web.s3 as gateway + + _, credential, remote, _, _ = service + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error" + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + client = boto3.client("s3", + endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=credential.access_key, + aws_secret_access_key=credential.secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}), + ) + client.create_bucket(Bucket="sdk-multipart") + created = client.create_multipart_upload(Bucket="sdk-multipart", Key="large.bin") + uid = created["UploadId"] + payload1 = os.urandom(5*1024*1024) + payload2 = b"last-part" * 61 + uploaded1 = client.upload_part( + Bucket="sdk-multipart", Key="large.bin", UploadId=uid, + PartNumber=1, Body=payload1, + ) + uploaded2 = client.upload_part( + Bucket="sdk-multipart", Key="large.bin", UploadId=uid, + PartNumber=2, Body=payload2, + ) + first_page = client.list_parts( + Bucket="sdk-multipart", Key="large.bin", UploadId=uid, MaxParts=1, + ) + assert first_page["IsTruncated"] is True + assert first_page["NextPartNumberMarker"] == 1 + assert first_page["StorageClass"] == "STANDARD" + assert first_page["Parts"][0]["LastModified"] + next_page = client.list_parts( + Bucket="sdk-multipart", Key="large.bin", UploadId=uid, + MaxParts=1, PartNumberMarker=1, + ) + assert [x["PartNumber"] for x in next_page["Parts"]] == [2] + assert next_page["IsTruncated"] is False + pending = client.list_multipart_uploads( + Bucket="sdk-multipart", Prefix="large", Delimiter="/", + EncodingType="url", MaxUploads=1, + ) + assert pending["Prefix"] == "large" + assert pending["Delimiter"] == "/" + assert pending["EncodingType"] == "url" + assert pending["Uploads"][0]["Key"] == "large.bin" + assert pending["Uploads"][0]["StorageClass"] == "STANDARD" + before = remote.uploads + done = client.complete_multipart_upload( + Bucket="sdk-multipart", Key="large.bin", UploadId=uid, + MultipartUpload={"Parts": [ + {"PartNumber": 1, "ETag": uploaded1["ETag"]}, + {"PartNumber": 2, "ETag": uploaded2["ETag"]}, + ]}, + ) + assert done["ETag"].endswith('-2"') + assert remote.uploads == before + response = client.get_object(Bucket="sdk-multipart", Key="large.bin") + assert response["Body"].read() == payload1 + payload2 + assert client.head_object(Bucket="sdk-multipart", Key="large.bin")["ETag"] == done["ETag"] + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() + + +def test_corrupted_telegram_part_prevents_multipart_publication(service): + setup(service) + original = b"untouched-data" + assert signed(service, "PUT", "/backup-bucket/asset.bin", original).status_code == 200 + uid = begin(service) + etag = part(service, uid, 1, b"remote-corruption-test") + bot = service[2] + latest_file_id = f"file-{bot.uploads}" + assert latest_file_id in bot.parts + bot.parts[latest_file_id] = b"tampered-body" + completion = complete(service, uid, [(1, etag)]) + assert completion.status_code != 200 + # Checksum verification happens before any pointer mutation. + still_current = signed(service, "GET", "/backup-bucket/asset.bin") + assert still_current.status_code == 200 + assert still_current.content == original + # Failure can be inspected/retried; no visible truncated object. + listed = signed(service, "GET", f"/backup-bucket/asset.bin?uploadId={uid}") + assert listed.status_code == 200 + + +def test_list_multipart_uploads_is_owner_scoped_and_paginated(service): + setup(service) + ids = [begin(service, key=key) for key in ("a.bin", "b.bin", "c.bin")] + from teledrive.s3.credentials import S3CredentialStore + connection = service[3]() + try: + stranger = S3CredentialStore(connection, service[4]).create( + "stranger", ["s3:ListBucket"] + ) + finally: + connection.close() + unauthorized = signed(service, "GET", "/backup-bucket?uploads", + access=stranger.access_key, secret=stranger.secret_key) + assert unauthorized.status_code == 200 + assert ET.fromstring(unauthorized.content).findall(NS+"Upload") == [] + first = signed(service, "GET", "/backup-bucket?uploads&max-uploads=2") + assert first.status_code == 200, first.text + root = ET.fromstring(first.content) + assert [e.findtext(NS+"Key") for e in root.findall(NS+"Upload")] == ["a.bin", "b.bin"] + assert root.findtext(NS+"IsTruncated") == "true" + key_marker = root.findtext(NS+"NextKeyMarker") + upload_marker = root.findtext(NS+"NextUploadIdMarker") + from urllib.parse import quote + final = signed(service, "GET", + "/backup-bucket?uploads&max-uploads=2&key-marker="+quote(key_marker) + +"&upload-id-marker="+quote(upload_marker)) + assert final.status_code == 200, final.text + assert [e.findtext(NS+"Key") for e in ET.fromstring(final.content).findall(NS+"Upload")] == ["c.bin"] + assert signed(service, "DELETE", "/backup-bucket/b.bin?uploadId="+ids[1]).status_code == 204 + after_abort = signed(service, "GET", "/backup-bucket?uploads") + assert [e.findtext(NS+"Key") for e in ET.fromstring(after_abort.content).findall(NS+"Upload")] == [ + "a.bin", "c.bin" + ] + + + +def test_list_parts_sql_paging_and_replaced_part_metadata(service): + setup(service) + uid = begin(service) + part(service, uid, 9, b"original") + replacement = part(service, uid, 9, b"replacement") + part(service, uid, 1, b"first") + part(service, uid, 5, b"middle") + before_download = service[2].downloads + first = signed(service, "GET", + f"/backup-bucket/asset.bin?uploadId={uid}&max-parts=2") + assert first.status_code == 200, first.text + root = ET.fromstring(first.content) + assert [n.findtext(NS + "PartNumber") for n in root.findall(NS + "Part")] == ["1", "5"] + assert root.findtext(NS + "StorageClass") == "STANDARD" + assert root.findtext(NS + "IsTruncated") == "true" + assert root.findtext(NS + "NextPartNumberMarker") == "5" + for entry in root.findall(NS + "Part"): + assert entry.findtext(NS + "LastModified").endswith("Z") + assert entry.findtext(NS + "Size") + later = signed(service, "GET", + f"/backup-bucket/asset.bin?uploadId={uid}&max-parts=2&part-number-marker=5") + assert later.status_code == 200 + rest = ET.fromstring(later.content) + assert rest.findtext(NS + "IsTruncated") == "false" + parts = rest.findall(NS + "Part") + assert [p.findtext(NS + "PartNumber") for p in parts] == ["9"] + assert parts[0].findtext(NS + "ETag") == replacement + assert service[2].downloads == before_download + for marker, maxparts in ( + ("-1", "1"), ("10001", "1"), ("no", "1"), ("1.0", "1"), + ("0", "0"), ("0", "1001"), ("0", "foo"), + ): + invalid = signed(service, "GET", + f"/backup-bucket/asset.bin?uploadId={uid}" + f"&part-number-marker={marker}&max-parts={maxparts}") + assert invalid.status_code == 400, (marker, maxparts, invalid.text) + assert b"InvalidArgument" in invalid.content + assert service[2].downloads == before_download + + # A completed upload is no longer an active ListParts target. + assert complete(service, uid, [(9, replacement)]).status_code == 200 + inactive = signed(service, "GET", f"/backup-bucket/asset.bin?uploadId={uid}") + assert inactive.status_code == 404 and b"NoSuchUpload" in inactive.content + + +def test_list_multipart_uploads_grouping_unicode_encoding_and_cursors(service): + setup(service) + # Interleaved prefixes, same-key duplicate uploads and Unicode keys. + first_a = begin(service, key="assets/a/one") + begin(service, key="assets/a/two") + first_b = begin(service, key="assets/b/one") + begin(service, key="assets/b/two") + begin(service, key="assets/single") + begin(service, key="misc/ü(1).bin") + before_download = service[2].downloads + + grouped = signed(service, "GET", + "/backup-bucket?uploads&prefix=assets%2F&delimiter=%2F&max-uploads=2") + assert grouped.status_code == 200, grouped.text + root = ET.fromstring(grouped.content) + assert root.findtext(NS + "Prefix") == "assets/" + assert root.findtext(NS + "Delimiter") == "/" + assert [node.findtext(NS + "Prefix") for node in root.findall(NS+"CommonPrefixes")] == [ + "assets/a/", "assets/b/", + ] + assert root.findtext(NS + "IsTruncated") == "true" + assert root.findtext(NS + "NextKeyMarker") == "assets/b/" + from urllib.parse import quote + next_page = signed(service, "GET", + "/backup-bucket?uploads&prefix=assets%2F&delimiter=%2F&max-uploads=2" + "&key-marker=" + quote(root.findtext(NS+"NextKeyMarker"), safe="") + + "&upload-id-marker=") + assert next_page.status_code == 200, next_page.text + follow = ET.fromstring(next_page.content) + assert [node.findtext(NS+"Key") for node in follow.findall(NS+"Upload")] == ["assets/single"] + assert follow.find(NS+"CommonPrefixes") is None + assert follow.findtext(NS+"IsTruncated") == "false" + + utf8 = signed(service, "GET", + "/backup-bucket?uploads&prefix=misc%2F&encoding-type=url") + assert utf8.status_code == 200, utf8.text + encoded = ET.fromstring(utf8.content) + assert encoded.findtext(NS+"EncodingType") == "url" + assert encoded.findtext(NS+"Upload/"+NS+"Key") == "misc/%C3%BC%281%29.bin" + + same_key = begin(service, key="repeat") + same_key2 = begin(service, key="repeat") + first = signed(service, "GET", "/backup-bucket?uploads&prefix=repeat&max-uploads=1") + assert first.status_code == 200 + start = ET.fromstring(first.content) + assert start.findtext(NS+"IsTruncated") == "true" + last_id = start.findtext(NS+"NextUploadIdMarker") + last_key = start.findtext(NS+"NextKeyMarker") + next_result = signed(service, "GET", + "/backup-bucket?uploads&prefix=repeat&max-uploads=1" + "&key-marker="+quote(last_key)+"&upload-id-marker="+quote(last_id)) + assert next_result.status_code == 200 + end = ET.fromstring(next_result.content) + assert len(end.findall(NS+"Upload")) == 1 + assert end.findtext(NS+"Upload/"+NS+"UploadId") in {same_key, same_key2} - {last_id} + assert end.findtext(NS+"Upload/"+NS+"StorageClass") == "STANDARD" + assert end.findtext(NS+"IsTruncated") == "false" + + for path in ( + "/backup-bucket?uploads&encoding-type=garbage", + "/backup-bucket?uploads&max-uploads=0", + "/backup-bucket?uploads&max-uploads=1001", + "/backup-bucket?uploads&upload-id-marker=x", + ): + failure = signed(service, "GET", path) + assert failure.status_code == 400, (path, failure.text) + assert service[2].downloads == before_download + assert signed(service, "DELETE", + "/backup-bucket/assets/a/one?uploadId="+first_a).status_code == 204 + assert signed(service, "DELETE", + "/backup-bucket/assets/b/one?uploadId="+first_b).status_code == 204 + + +def test_list_multipart_pages_are_scoped_to_credential_without_telegram_reads(service): + setup(service) + uid = begin(service) + part(service, uid, 1, b"private") + db = service[3]() + try: + guest = S3CredentialStore(db, service[4]).create( + "listing-only", ["s3:ListBucket", "s3:GetObject"], + ) + finally: + db.close() + before = service[2].downloads + page = signed(service, "GET", "/backup-bucket?uploads&delimiter=%2F", + access=guest.access_key, secret=guest.secret_key) + assert page.status_code == 200 + assert ET.fromstring(page.content).findall(NS+"Upload") == [] + denied = signed(service, "GET", f"/backup-bucket/asset.bin?uploadId={uid}", + access=guest.access_key, secret=guest.secret_key) + assert denied.status_code == 404 and b"NoSuchUpload" in denied.content + assert service[2].downloads == before + +def test_bucket_delete_rejects_uncommitted_multipart_history(service): + setup(service) + uid = begin(service) + before = signed(service, "DELETE", "/backup-bucket") + assert before.status_code == 409 + assert xml_text(before, "Code") == "BucketNotEmpty" + # Upload remains usable after failed bucket deletion. + assert signed(service, "GET", "/backup-bucket?uploads").status_code == 200 + assert signed(service, "DELETE", f"/backup-bucket/asset.bin?uploadId={uid}").status_code == 204 + # Abort retains orphan-cleanup audit metadata; still not safe to delete. + after = signed(service, "DELETE", "/backup-bucket") + assert after.status_code == 409 and xml_text(after, "Code") == "BucketNotEmpty" diff --git a/tests/s3/test_s3_multipart_preconditions.py b/tests/s3/test_s3_multipart_preconditions.py new file mode 100644 index 00000000..b9e38f0a --- /dev/null +++ b/tests/s3/test_s3_multipart_preconditions.py @@ -0,0 +1,130 @@ +"""Conditional CompleteMultipartUpload: exact HEAD CAS, signed headers and retries. + +The upload writes to Telegram before Complete, but completing under a stale +If-Match/If-None-Match must not take ownership of a different object's head. +""" +from __future__ import annotations + +import hashlib +import time +from xml.etree import ElementTree as ET + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import setup, begin, part, xml_text +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + +def complete_conditional(svc, upload_id, selected, **headers): + root = ET.Element("CompleteMultipartUpload") + for number, etag in selected: + row = ET.SubElement(root, "Part") + ET.SubElement(row, "PartNumber").text = str(number) + ET.SubElement(row, "ETag").text = etag + return signed(svc, "POST", + f"/backup-bucket/asset.bin?uploadId={upload_id}", + ET.tostring(root), extra_headers={ + k.replace("_", "-"): v for k, v in headers.items() + }, + ) + + +def test_if_none_match_complete_is_one_time_and_keeps_parts_retryable(service): + setup(service) + uid = begin(service) + payload = b"conditioned-telegram-multipart" + etag = part(service, uid, 1, payload) + baseline = service[2].uploads + first = complete_conditional(service, uid, [(1, etag)], If_None_Match="*") + assert first.status_code == 200, first.text + assert service[2].uploads == baseline + assert signed(service, "GET", "/backup-bucket/asset.bin").content == payload + assert complete_conditional( + service, uid, [(1, etag)], If_None_Match="*" + ).status_code == 412 + + second_id = begin(service) + second_etag = part(service, second_id, 1, b"second-staged") + uploads = service[2].uploads + conflict = complete_conditional( + service, second_id, [(1, second_etag)], If_None_Match="*" + ) + assert conflict.status_code == 412, conflict.text + assert xml_text(conflict, "Code") == "PreconditionFailed" + assert signed(service, "GET", "/backup-bucket/asset.bin").content == payload + assert service[2].uploads == uploads + + +def test_if_match_complete_fences_inflight_head_change(service, monkeypatch): + setup(service) + old = signed(service, "PUT", "/backup-bucket/asset.bin", b"old-head") + assert old.status_code == 200 + uid = begin(service) + payload = b"staged-multipart-object" + etag = part(service, uid, 1, payload) + prior_uploads = service[2].uploads + initial_etag = old.headers["etag"] + original = TelegramObjectStorage.iter_bytes + intrusions = [] + + async def concurrently_publish(self, identifier, **kwargs): + if not intrusions: + connection = service[3]() + try: + catalog = BucketCatalog(ManifestStore(connection)) + now = int(time.time()) + winner = catalog.manifests.begin( + "backup-bucket", "asset.bin", 0, now, + ) + catalog.manifests.finish(winner, hashlib.sha256(b"").hexdigest(), now) + catalog.publish(winner, now=now) + intrusions.append(winner) + finally: + connection.close() + async for data in original(self, identifier, **kwargs): + yield data + + monkeypatch.setattr(TelegramObjectStorage, "iter_bytes", concurrently_publish) + stale = complete_conditional(service, uid, [(1, etag)], If_Match=initial_etag) + assert intrusions + assert stale.status_code == 412, stale.text + assert xml_text(stale, "Code") == "PreconditionFailed" + assert service[2].uploads == prior_uploads + monkeypatch.setattr(TelegramObjectStorage, "iter_bytes", original) + assert signed(service, "GET", "/backup-bucket/asset.bin").content == b"" + + # Staged media is not thrown away by the failed CAS; a fresh signed + # condition can safely Complete without a new Telegram upload. + new_etag = signed(service, "HEAD", "/backup-bucket/asset.bin").headers["etag"] + done = complete_conditional(service, uid, [(1, etag)], If_Match=new_etag) + assert done.status_code == 200, done.text + assert service[2].uploads == prior_uploads + assert signed(service, "GET", "/backup-bucket/asset.bin").content == payload + + +def test_unsigned_completion_precondition_is_rejected(service): + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + setup(service) + uid = begin(service) + etag = part(service, uid, 1, b"safe") + request_path = f"/backup-bucket/asset.bin?uploadId={uid}" + xml = ET.tostring(ET.Element("CompleteMultipartUpload")) + req = AWSRequest( + method="POST", url="http://localhost:9000" + request_path, + data=xml, + headers={"x-amz-content-sha256": hashlib.sha256(xml).hexdigest()}, + ) + req.context["payload_signing_enabled"] = True + S3SigV4Auth( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(req) + altered = dict(req.headers) + altered["If-None-Match"] = "*" + response = service[0].post(request_path, content=xml, headers=altered) + assert response.status_code == 403 + assert b"AccessDenied" in response.content + assert service[2].uploads == 1 diff --git a/tests/s3/test_s3_object_attributes.py b/tests/s3/test_s3_object_attributes.py new file mode 100644 index 00000000..91a2b664 --- /dev/null +++ b/tests/s3/test_s3_object_attributes.py @@ -0,0 +1,439 @@ +"""S3 GetObjectAttributes — strict signed metadata-only compatibility tests.""" +from __future__ import annotations + +import base64 +import hashlib +import os +import socket +import threading +import time +from xml.etree import ElementTree as ET + +import boto3 +from botocore.config import Config +import uvicorn + +import web.s3 as gateway +from teledrive.s3.credentials import S3CredentialStore +from tests.s3.test_s3_gateway import service, signed + + +def test_signed_object_attributes_return_exact_metadata_without_telegram_get(service): + assert signed(service, "PUT", "/attrs-bucket").status_code == 200 + payload = os.urandom(12003) + put = signed(service, "PUT", "/attrs-bucket/data.bin", payload) + assert put.status_code == 200 + fake = service[2] + before = fake.downloads + result = signed( + service, "GET", "/attrs-bucket/data.bin?attributes", + extra_headers={"x-amz-object-attributes": "ETag,ObjectSize,Checksum,StorageClass"}, + ) + assert result.status_code == 200, result.text + assert b"12003" in result.content + assert put.headers["etag"].encode() in result.content + assert b"STANDARD" in result.content + assert b"FULL_OBJECT" in result.content + assert base64.b64encode(hashlib.sha256(payload).digest()) in result.content + assert result.headers["x-amz-version-id"] == put.headers["x-amz-version-id"] + assert "last-modified" in result.headers + assert fake.downloads == before, "metadata lookup must not download Telegram chunks" + + +def test_attributes_require_separate_permission_and_header_signature(service): + assert signed(service, "PUT", "/attrs-bucket").status_code == 200 + assert signed(service, "PUT", "/attrs-bucket/data", b"secret").status_code == 200 + db = service[3]() + try: + restricted = S3CredentialStore(db, service[4]).create( + "reader-no-attributes", ["s3:GetObject"], + ) + finally: + db.close() + denied = signed( + service, "GET", "/attrs-bucket/data?attributes", + extra_headers={"x-amz-object-attributes": "ETag,ObjectSize"}, + access=restricted.access_key, secret=restricted.secret_key, + ) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + + # Sign GET normally, then inject a new unsigned attributes header. + # No amount of valid read permission can bypass SignedHeaders enforcement. + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + auth = AWSRequest(method="GET", url="http://localhost:9000/attrs-bucket/data?attributes") + S3SigV4Auth( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(auth) + headers = dict(auth.headers) + headers["x-amz-object-attributes"] = "ETag,ObjectSize" + forged = service[0].get("/attrs-bucket/data?attributes", headers=headers) + assert forged.status_code == 403 + assert b"AccessDenied" in forged.content + + +def test_unsupported_or_duplicate_attributes_refuse_false_success(service): + assert signed(service, "PUT", "/attrs-bucket").status_code == 200 + assert signed(service, "PUT", "/attrs-bucket/data", b"data").status_code == 200 + downloads = service[2].downloads + for requested, code, status in ( + ("UnknownAttribute", b"NotImplemented", 501), + ("ETag,ETag", b"InvalidArgument", 400), + ("", b"InvalidArgument", 400), + ): + response = signed( + service, "GET", "/attrs-bucket/data?attributes", + extra_headers={"x-amz-object-attributes": requested}, + ) + assert response.status_code == status, (requested, response.text) + assert code in response.content + assert service[2].downloads == downloads + + +def test_real_boto3_get_object_attributes_over_live_http(service): + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + port = sock.getsockname()[1] + sock.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + creds = service[1] + sdk = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style": "path", "payload_signing_enabled": True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts": 0}, + ), + ) + sdk.create_bucket(Bucket="sdk-attributes") + body = b"telegram-metadata-only" * 97 + put = sdk.put_object(Bucket="sdk-attributes", Key="item", Body=body) + before = service[2].downloads + attributes = sdk.get_object_attributes( + Bucket="sdk-attributes", Key="item", + ObjectAttributes=["ETag", "ObjectSize", "Checksum", "StorageClass"], + ) + assert attributes["ObjectSize"] == len(body) + assert attributes["ETag"] == put["ETag"] + assert attributes["VersionId"] == put["VersionId"] + assert attributes["StorageClass"] == "STANDARD" + assert attributes["Checksum"]["ChecksumSHA256"] == base64.b64encode( + hashlib.sha256(body).digest() + ).decode("ascii") + assert attributes["Checksum"]["ChecksumType"] == "FULL_OBJECT" + assert attributes["LastModified"] + assert service[2].downloads == before + single = sdk.get_object_attributes( + Bucket="sdk-attributes", Key="item", ObjectAttributes=["ObjectParts"], + ) + assert "ObjectParts" not in single, "A simple PUT must not expose Telegram chunks as S3 parts" + + # Real boto3 GetObjectAttributes must parse the AWS-shaped XML and + # use header pagination, not the ListParts query parameters. + created = sdk.create_multipart_upload(Bucket="sdk-attributes", Key="multi") + uid = created["UploadId"] + first_bytes = b"x" * (5 * 1024 * 1024) + last_bytes = b"genuine-second-part" * 33 + p1 = sdk.upload_part( + Bucket="sdk-attributes", Key="multi", UploadId=uid, + PartNumber=1, Body=first_bytes, + ) + p2 = sdk.upload_part( + Bucket="sdk-attributes", Key="multi", UploadId=uid, + PartNumber=2, Body=last_bytes, + ) + completed = sdk.complete_multipart_upload( + Bucket="sdk-attributes", Key="multi", UploadId=uid, + MultipartUpload={"Parts": [ + {"PartNumber": 1, "ETag": p1["ETag"]}, + {"PartNumber": 2, "ETag": p2["ETag"]}, + ]}, + ) + old_version = sdk.head_object(Bucket="sdk-attributes", Key="multi")["VersionId"] + before = service[2].downloads + page1 = sdk.get_object_attributes( + Bucket="sdk-attributes", Key="multi", + ObjectAttributes=["ETag", "ObjectParts", "Checksum"], + MaxParts=1, + ) + parts1 = page1["ObjectParts"] + assert parts1["TotalPartsCount"] == 2 + assert parts1["IsTruncated"] is True + assert parts1["NextPartNumberMarker"] == 1 + assert parts1["Parts"][0]["PartNumber"] == 1 + assert parts1["Parts"][0]["Size"] == len(first_bytes) + assert parts1["Parts"][0]["ChecksumSHA256"] == base64.b64encode( + hashlib.sha256(first_bytes).digest() + ).decode("ascii") + assert page1["ETag"] == completed["ETag"] + + page2 = sdk.get_object_attributes( + Bucket="sdk-attributes", Key="multi", + ObjectAttributes=["ObjectParts"], MaxParts=1, PartNumberMarker=1, + )["ObjectParts"] + assert page2["IsTruncated"] is False + assert page2["Parts"][0]["PartNumber"] == 2 + assert page2["Parts"][0]["Size"] == len(last_bytes) + assert page2["Parts"][0]["ChecksumSHA256"] == base64.b64encode( + hashlib.sha256(last_bytes).digest() + ).decode("ascii") + assert service[2].downloads == before, "Attribute pagination must not read Telegram" + + sdk.create_bucket(Bucket="sdk-attributes-copy") + sdk.copy_object( + Bucket="sdk-attributes-copy", Key="clone", + CopySource={"Bucket": "sdk-attributes", "Key": "multi"}, + ) + clone = sdk.get_object_attributes( + Bucket="sdk-attributes-copy", Key="clone", + ObjectAttributes=["ObjectParts", "ETag"], + ) + assert clone["ObjectParts"]["TotalPartsCount"] == 2 + assert [p["PartNumber"] for p in clone["ObjectParts"]["Parts"]] == [1, 2] + assert clone["ETag"] == completed["ETag"] + sdk.put_object(Bucket="sdk-attributes", Key="multi", Body=b"new-head") + historical = sdk.get_object_attributes( + Bucket="sdk-attributes", Key="multi", VersionId=old_version, + ObjectAttributes=["ObjectParts"], MaxParts=1, PartNumberMarker=1, + ) + assert historical["VersionId"] == old_version + assert historical["ObjectParts"]["Parts"][0]["PartNumber"] == 2 + assert service[2].downloads == before + finally: + server.should_exit = True + thread.join(timeout=10) + sock.close() + assert not thread.is_alive() + + +def test_historical_attributes_use_exact_published_version_not_current_head(service): + assert signed(service, "PUT", "/attrs-bucket").status_code == 200 + old = b"first-immutable-plaintext" * 19 + new = b"second-immutable-plaintext" * 11 + first = signed(service, "PUT", "/attrs-bucket/history", old) + second = signed(service, "PUT", "/attrs-bucket/history", new) + assert first.status_code == second.status_code == 200 + old_id = first.headers["x-amz-version-id"] + new_id = second.headers["x-amz-version-id"] + downloads = service[2].downloads + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + for version_id, body in ((old_id, old), (new_id, new)): + reply = signed( + service, "GET", f"/attrs-bucket/history?attributes&versionId={version_id}", + extra_headers={ + "x-amz-object-attributes": "Checksum,ETag,ObjectSize,StorageClass", + }, + ) + assert reply.status_code == 200, reply.text + assert reply.headers["x-amz-version-id"] == version_id + root = ET.fromstring(reply.content) + assert root.findtext(ns + "ObjectSize") == str(len(body)) + assert root.findtext(ns + "Checksum/" + ns + "ChecksumSHA256") == ( + base64.b64encode(hashlib.sha256(body).digest()).decode("ascii") + ) + assert root.findtext(ns + "Checksum/" + ns + "ChecksumType") == "FULL_OBJECT" + assert root.findtext(ns + "StorageClass") == "STANDARD" + + for path, code in ( + (f"/attrs-bucket/other-key?attributes&versionId={old_id}", 404), + ("/attrs-bucket/history?attributes&versionId=missing", 404), + (f"/attrs-bucket/history?attributes&versionId={old_id}&versionId={new_id}", 400), + (f"/attrs-bucket/history?versionId={old_id}&attributes&tagging", 501), + ): + reply = signed( + service, "GET", path, + extra_headers={"x-amz-object-attributes": "ETag"}, + ) + assert reply.status_code == code, (path, reply.text) + + assert signed(service, "DELETE", "/attrs-bucket/history").status_code == 204 + marker = signed( + service, "GET", "/attrs-bucket/history?attributes", + extra_headers={"x-amz-object-attributes": "ETag"}, + ) + assert marker.status_code == 404 and marker.headers["x-amz-delete-marker"] == "true" + historical = signed( + service, "GET", f"/attrs-bucket/history?attributes&versionId={old_id}", + extra_headers={"x-amz-object-attributes": "ETag"}, + ) + assert historical.status_code == 200 + assert signed(service, "DELETE", f"/attrs-bucket/history?versionId={old_id}").status_code == 204 + removed = signed( + service, "GET", f"/attrs-bucket/history?attributes&versionId={old_id}", + extra_headers={"x-amz-object-attributes": "ETag"}, + ) + assert removed.status_code == 404 + assert service[2].downloads == downloads + + +def test_object_parts_pagination_validation_and_unsigned_injection(service): + assert signed(service, "PUT", "/attrs-bucket").status_code == 200 + assert signed(service, "PUT", "/attrs-bucket/key", b"contents").status_code == 200 + downloads = service[2].downloads + for header, value, fields in ( + ("x-amz-max-parts", "0", "ObjectParts"), + ("x-amz-max-parts", "1001", "ObjectParts"), + ("x-amz-max-parts", "not-an-int", "ObjectParts"), + ("x-amz-max-parts", "1.0", "ObjectParts"), + ("x-amz-part-number-marker", "-1", "ObjectParts"), + ("x-amz-part-number-marker", "10001", "ObjectParts"), + ("x-amz-part-number-marker", "nope", "ObjectParts"), + ("x-amz-max-parts", "1", "ETag"), + ("x-amz-part-number-marker", "1", "ETag"), + ): + response = signed( + service, "GET", "/attrs-bucket/key?attributes", + extra_headers={"x-amz-object-attributes": fields, header: value}, + ) + assert response.status_code == 400, (header, value, response.text) + assert b"InvalidArgument" in response.content + + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + + request = AWSRequest( + method="GET", url="http://localhost:9000/attrs-bucket/key?attributes", + headers={"x-amz-object-attributes": "ObjectParts"}, + ) + S3SigV4Auth( + Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1", + ).add_auth(request) + headers = dict(request.headers) + headers["x-amz-max-parts"] = "1" + injected = service[0].get("/attrs-bucket/key?attributes", headers=headers) + assert injected.status_code == 403 + assert b"AccessDenied" in injected.content + + for attr in ("x-amz-max-parts", "x-amz-part-number-marker"): + unrelated = signed( + service, "GET", "/attrs-bucket/key", + extra_headers={attr: "1"}, + ) + assert unrelated.status_code == 501 + assert service[2].downloads == downloads + + +def test_signed_sparse_multipart_historical_copy_and_legacy_fallback(service): + from sqlalchemy import delete, select, update + from teledrive.s3.multipart import multipart_object_parts + from tests.s3.test_s3_multipart import begin, complete, part + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + first = b"a" * (5 * 1024 * 1024) + tail = b"the-last-part" + uid = begin(service) + p1 = part(service, uid, 1, first) + part(service, uid, 2, b"not-committed") + p3 = part(service, uid, 3, tail) + done = complete(service, uid, [(1, p1), (3, p3)]) + assert done.status_code == 200, done.text + version = signed(service, "HEAD", "/backup-bucket/asset.bin").headers[ + "x-amz-version-id" + ] + before = service[2].downloads + + def fetch(marker=0, limit=1, *, version_id=version, bucket="backup-bucket", + key="asset.bin"): + reply = signed( + service, "GET", + f"/{bucket}/{key}?attributes&versionId={version_id}", + extra_headers={ + "x-amz-object-attributes": "ObjectParts,Checksum", + "x-amz-max-parts": str(limit), + "x-amz-part-number-marker": str(marker), + }, + ) + assert reply.status_code == 200, reply.text + root = ET.fromstring(reply.content) + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" + parts = root.find(ns + "ObjectParts") + assert parts is not None + return parts, ns + + first_page, ns = fetch() + assert first_page.findtext(ns + "PartsCount") == "2" + assert first_page.findtext(ns + "NextPartNumberMarker") == "1" + assert first_page.findtext(ns + "IsTruncated") == "true" + assert first_page.findtext(ns + "Part/" + ns + "PartNumber") == "1" + second_page, ns = fetch(1) + assert second_page.findtext(ns + "Part/" + ns + "PartNumber") == "3" + assert second_page.findtext(ns + "Part/" + ns + "Size") == str(len(tail)) + assert second_page.findtext(ns + "IsTruncated") == "false" + assert second_page.findtext(ns + "Part/" + ns + "ChecksumSHA256") == ( + base64.b64encode(hashlib.sha256(tail).digest()).decode("ascii") + ) + empty_page, _ = fetch(3) + assert empty_page.find(ns + "Part") is None + + # Direct SQL snapshots are durable before any HEAD overwrite; these + # metadata calls never download object bytes from Telegram. + assert service[2].downloads == before + assert signed(service, "PUT", "/backup-bucket/asset.bin", b"new-head").status_code == 200 + historical, _ = fetch(1) + assert historical.findtext(ns + "Part/" + ns + "PartNumber") == "3" + copied = signed( + service, "PUT", "/backup-bucket/clone.bin", + extra_headers={"x-amz-copy-source": + f"/backup-bucket/asset.bin?versionId={version}"}, + ) + assert copied.status_code == 200, copied.text + copied_id = signed(service, "HEAD", "/backup-bucket/clone.bin").headers[ + "x-amz-version-id" + ] + cloned, _ = fetch(1, version_id=copied_id, key="clone.bin") + assert cloned.findtext(ns + "Part/" + ns + "PartNumber") == "3" + + # A pre-upgrade object has no new snapshot table. The old committed + # multipart journal is an independently verified read-only fallback. + db = service[3]() + try: + db.connection.execute(delete(multipart_object_parts).where( + multipart_object_parts.c.object_id == version + )) + db.commit() + finally: + db.close() + from_legacy, _ = fetch(1) + assert from_legacy.findtext(ns + "Part/" + ns + "PartNumber") == "3" + + # Any partial or tampered snapshot must fail closed, rather than silently + # falling back to a plausible but inconsistent historical multipart list. + db = service[3]() + try: + db.connection.execute(update(multipart_object_parts).where( + multipart_object_parts.c.object_id == copied_id, + multipart_object_parts.c.number == 3, + ).values(size=1)) + db.commit() + finally: + db.close() + corrupt = signed( + service, "GET", + f"/backup-bucket/clone.bin?attributes&versionId={copied_id}", + extra_headers={"x-amz-object-attributes": "ObjectParts"}, + ) + assert corrupt.status_code == 409 + assert b"InvalidObjectState" in corrupt.content + assert service[2].downloads == before + diff --git a/tests/s3/test_s3_permissions.py b/tests/s3/test_s3_permissions.py new file mode 100644 index 00000000..5a91d158 --- /dev/null +++ b/tests/s3/test_s3_permissions.py @@ -0,0 +1,82 @@ +"""P0 SigV4 privilege isolation across real SQL-backed S3 HTTP endpoints.""" +from __future__ import annotations + +import time + +from sqlalchemy import update + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore, credentials + + +def test_expired_and_readonly_keys_cannot_mutate_or_list_storage(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"keep-permissions-separated" + assert signed(service, "PUT", "/backup-bucket/original", payload).status_code == 200 + remote = service[2] + original_uploads = remote.uploads + now = int(time.time()) + connection = service[3]() + try: + store = S3CredentialStore(connection, service[4]) + reader = store.create("p0-read-only", ["s3:GetObject"]) + expired = store.create( + "p0-expired", ["s3:GetObject"], expires_at=now + 60, now=now + ) + connection.connection.execute( + update(credentials) + .where(credentials.c.id == expired.id) + .values(expires_at=now - 1) + ) + connection.commit() + finally: + connection.close() + + def request(method, path, body=b"", credential=reader): + return signed( + service, method, path, body, + access=credential.access_key, secret=credential.secret_key, + ) + + assert request("GET", "/backup-bucket/original").content == payload + denied = [ + request("GET", "/"), + request("GET", "/backup-bucket?list-type=2"), + request("PUT", "/backup-bucket/injected", b"not-authorized"), + request("DELETE", "/backup-bucket/original"), + request("POST", "/backup-bucket/attempt?uploads"), + request("PUT", "/backup-bucket/copied", credential=reader), + request("GET", "/backup-bucket/original", credential=expired), + ] + assert all(item.status_code == 403 for item in denied), [ + (item.status_code, item.text[:200]) for item in denied + ] + assert all(b"AccessDenied" in item.content for item in denied) + assert remote.uploads == original_uploads + assert signed(service, "GET", "/backup-bucket/original").content == payload + assert signed(service, "GET", "/backup-bucket/injected").status_code == 404 + assert signed(service, "GET", "/backup-bucket/copied").status_code == 404 + + +def test_key_revocation_applies_before_any_mutation(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + connection = service[3]() + try: + store = S3CredentialStore(connection, service[4]) + scoped = store.create("p0-revocable", ["s3:PutObject", "s3:GetObject"]) + finally: + connection.close() + before = service[2].uploads + connection = service[3]() + try: + assert S3CredentialStore(connection, service[4]).revoke(scoped.id) + finally: + connection.close() + denied = signed( + service, "PUT", "/backup-bucket/revoked", b"must-not-upload", + access=scoped.access_key, secret=scoped.secret_key, + ) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/revoked").status_code == 404 diff --git a/tests/s3/test_s3_policy_status.py b/tests/s3/test_s3_policy_status.py new file mode 100644 index 00000000..ace228d0 --- /dev/null +++ b/tests/s3/test_s3_policy_status.py @@ -0,0 +1,105 @@ +"""S3 private-only GetBucketPolicyStatus protocol and permission coverage.""" +from __future__ import annotations + +import threading +import time +from xml.etree import ElementTree as ET + +import boto3 +import pytest +import uvicorn +from botocore.config import Config + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +import web.s3 as gateway + + +def test_policy_status_always_false_and_is_read_only(service): + assert signed(service, "PUT", "/policy-ci").status_code == 200 + status = signed(service, "GET", "/policy-ci?policyStatus") + assert status.status_code == 200, status.text + root = ET.fromstring(status.content) + assert root.tag.rsplit("}", 1)[-1] == "PolicyStatus" + assert root.findtext("{http://s3.amazonaws.com/doc/2006-03-01/}IsPublic") == "false" + assert signed(service, "PUT", "/policy-ci?policyStatus").status_code == 501 + assert signed(service, "DELETE", "/policy-ci?policyStatus").status_code == 501 + assert signed(service, "HEAD", "/policy-ci").status_code == 200 + assert service[2].uploads == 0 and service[2].downloads == 0 + + +def test_policy_status_missing_or_unauthorized_bucket(service): + assert signed(service, "GET", "/policy-missing?policyStatus").status_code == 404 + assert signed(service, "PUT", "/policy-ci").status_code == 200 + assert service[0].get("/policy-ci?policyStatus").status_code == 403 + db = service[3]() + try: + credentials = S3CredentialStore(db, service[4]) + policy_reader = credentials.create( + "get-private-policy-status", ["s3:GetBucketPolicyStatus"], + ) + object_reader = credentials.create("object-reader", ["s3:GetObject"]) + finally: + db.close() + result = signed(service, "GET", "/policy-ci?policyStatus", + access=policy_reader.access_key, secret=policy_reader.secret_key) + assert result.status_code == 200, result.text + assert signed(service, "GET", "/policy-ci?policyStatus", + access=object_reader.access_key, secret=object_reader.secret_key).status_code == 403 + assert signed(service, "GET", "/policy-ci", + access=policy_reader.access_key, secret=policy_reader.secret_key).status_code == 403 + + +@pytest.mark.parametrize("method,path", [ + ("GET", "/policy-ci?policyStatus=true"), + ("GET", "/policy-ci?policyStatus&policyStatus"), + ("GET", "/policy-ci?policyStatus&tagging"), + ("GET", "/policy-ci/item?policyStatus"), + ("PUT", "/policy-ci?policyStatus"), + ("DELETE", "/policy-ci?policyStatus"), +]) +def test_policy_status_cannot_override_or_mutate_routes(service, method, path): + assert signed(service, "PUT", "/policy-ci").status_code == 200 + response = signed(service, method, path) + assert response.status_code in (400, 501), response.text + assert signed(service, "HEAD", "/policy-ci").status_code == 200 + assert service[2].uploads == 0 + + +def test_real_boto3_get_bucket_policy_status_private_only(service): + from socket import socket + assert signed(service, "PUT", "/policy-ci").status_code == 200 + sock = socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(128) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, log_level="error", + lifespan="off", + )) + worker = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, + daemon=True) + worker.start() + try: + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(.02) + assert server.started + credential = service[1] + client = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", region_name="us-east-1", + aws_access_key_id=credential.access_key, + aws_secret_access_key=credential.secret_key, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + actual = client.get_bucket_policy_status(Bucket="policy-ci") + assert actual["PolicyStatus"] == {"IsPublic": False} + # No public bucket-policy engine can be enabled via the same API. + with pytest.raises(client.exceptions.ClientError): + client.put_bucket_policy( + Bucket="policy-ci", + Policy='{"Version":"2012-10-17","Statement":[]}', + ) + finally: + server.should_exit = True + worker.join(timeout=10) diff --git a/tests/s3/test_s3_post_policy.py b/tests/s3/test_s3_post_policy.py new file mode 100644 index 00000000..a6d5cc6b --- /dev/null +++ b/tests/s3/test_s3_post_policy.py @@ -0,0 +1,1121 @@ +"""Real multipart browser POSTs with bounded, checksum-pinned SigV4 policies.""" +import base64 +import hashlib +import hmac +import json +from datetime import datetime, timedelta, timezone +from urllib.parse import quote, urlsplit, parse_qs, urlencode +from xml.etree import ElementTree as ET + +import pytest + +from tests.s3.test_s3_gateway import service, signed +import web.app as admin +from teledrive.s3.credentials import S3CredentialStore + + +def make_form(service, data, *, key="browser/upload.bin", seconds=300, + missing=None, clauses=None, issued=None, key_prefix=None, + status=None, redirect=None, content_type=None, content_type_prefix=None, + extra_fields=None, metadata_prefixes=None): + credential = issued or service[1] + current = datetime.now(timezone.utc).replace(microsecond=0) + day = current.strftime("%Y%m%d") + form = { + "key": key, + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-credential": f"{credential.access_key}/{day}/us-east-1/s3/aws4_request", + "x-amz-date": current.strftime("%Y%m%dT%H%M%SZ"), + "x-amz-checksum-sha256": base64.b64encode(hashlib.sha256(data).digest()).decode(), + } + if status is not None: + form["success_action_status"] = status + if redirect is not None: + form["success_action_redirect"] = redirect + if content_type is not None: + form["Content-Type"] = content_type + form.update(extra_fields or {}) + conditions = [{"bucket": "backup-bucket"}] + conditions.extend( + ["starts-with", "$key", key_prefix] if k == "key" and key_prefix is not None + else ["starts-with", "$Content-Type", content_type_prefix] + if k == "Content-Type" and content_type_prefix is not None + else ["starts-with", "$" + k, metadata_prefixes[k]] + if metadata_prefixes is not None and k in metadata_prefixes + else {k: v} + for k, v in form.items() + ) + conditions.append(["content-length-range", 0, 8 * 1024 * 1024]) + if missing: + conditions = [c for c in conditions if not isinstance(c, dict) or missing not in c] + if clauses is not None: + conditions = clauses + document = {"expiration": (current + timedelta(seconds=seconds)).strftime("%Y-%m-%dT%H:%M:%SZ"), + "conditions": conditions} + encoded = base64.b64encode(json.dumps(document).encode()).decode() + form["policy"] = encoded + secret = hmac.new(("AWS4" + credential.secret_key).encode(), day.encode(), hashlib.sha256).digest() + for label in ("us-east-1", "s3", "aws4_request"): + secret = hmac.new(secret, label.encode(), hashlib.sha256).digest() + form["x-amz-signature"] = hmac.new(secret, encoded.encode(), hashlib.sha256).hexdigest() + return form + + +def post(service, data, fields, *, bucket="backup-bucket"): + return service[0].post( + f"/{bucket}", data=fields, + files={"file": ("blob.bin", data, "application/octet-stream")}, + follow_redirects=False, + ) + + +@pytest.mark.parametrize("size", [0, 1, 2048, 8193, 65537, 1048579]) +def test_browser_post_signed_random_bytes_sha_and_readback(service, size): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(str(size).encode()).digest(size) + key = f"browser/{size}.bin" + upload = post(service, payload, make_form(service, payload, key=key)) + assert upload.status_code == 204, upload.text + assert upload.headers["etag"] == '"' + hashlib.sha256(payload).hexdigest() + '"' + assert service[2].uploads == (0 if size == 0 else 1) + result = signed(service, "GET", f"/backup-bucket/{key}") + assert result.status_code == 200 and result.content == payload + + +def test_browser_post_rejects_mutation_without_telegram_upload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"the-verified-upload" + form = make_form(service, body) + mutations = [ + (body+b"x", form), (body, {**form, "key": "changed.bin"}), + (body, {**form, "x-amz-signature": "0"*64}), + (body, {**form, "x-amz-checksum-sha256": base64.b64encode(b"0"*32).decode()}), + (body, {**form, "policy": base64.b64encode(b"{}").decode()}), + ] + for payload, values in mutations: + response = post(service, payload, values) + assert response.status_code in (400, 403), response.text + assert service[2].uploads == 0 + + +def test_browser_post_requires_exact_policy_conditions(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"fixed-policy" + for field in ("bucket", "key", "x-amz-algorithm", "x-amz-credential", + "x-amz-date", "x-amz-checksum-sha256"): + assert post(service, body, make_form(service, body, missing=field)).status_code == 403 + assert post(service, body, make_form(service, body), bucket="another-bucket").status_code == 403 + form = make_form(service, body) + constraints = json.loads(base64.b64decode(form["policy"]))["conditions"] + constraints[-1] = ["content-length-range", 0, len(body)-1] + assert post(service, body, make_form(service, body, clauses=constraints)).status_code == 403 + constraints[-1] = ["starts-with", "$key", "unrelated/"] + assert post(service, body, make_form(service, body, clauses=constraints)).status_code == 403 + assert service[2].uploads == 0 + + +def test_browser_post_prefix_policy_binds_bucket_checksum_and_destination(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(b"browser-prefix").digest(65537) + form = make_form( + service, data, key="browser/prefix/first.bin", key_prefix="browser/prefix/", + ) + # The signed policy authorizes a prefix, rather than the form's original key. + authorized = {**form, "key": "browser/prefix/nested/İstanbul.bin"} + assert post(service, data, authorized).status_code == 204 + # Also verify independent SigV4 GET readback through an ASCII HTTP path. + # Raw non-ASCII URL normalization is covered by the dedicated S3 path tests. + assert post( + service, data, {**form, "key": "browser/prefix/nested/check.bin"}, + ).status_code == 204 + assert signed( + service, "GET", "/backup-bucket/browser/prefix/nested/check.bin", + ).content == data + for forbidden in ("browser/outside.bin", "browser/prefix-escape.bin"): + assert post(service, data, {**form, "key": forbidden}).status_code == 403 + assert post(service, data+b"x", authorized).status_code in (400, 403) + assert post(service, data, authorized, bucket="other-bucket").status_code == 403 + assert service[2].uploads == 2 + + +@pytest.mark.parametrize("bad_condition", [ + ["starts-with", "$x-amz-checksum-sha256", ""], + ["starts-with", "$bucket", "backup"], + ["starts-with", "$key", 42], + ["starts-with", "key", "browser/"], +]) +def test_browser_post_rejects_unapproved_wildcards(service, bad_condition): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"unchanged" + valid = make_form(service, data) + constraints = json.loads(base64.b64decode(valid["policy"]))["conditions"] + constraints[1] = bad_condition + assert post(service, data, make_form(service, data, clauses=constraints)).status_code == 403 + assert service[2].uploads == 0 + + +def test_browser_post_real_boto3_filename_prefix_policy(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(b"sdk-prefix").digest(8193) + checksum = base64.b64encode(hashlib.sha256(payload).digest()).decode() + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/prefix/${filename}", + Fields={"x-amz-checksum-sha256": checksum}, + Conditions=[ + {"x-amz-checksum-sha256": checksum}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + assert "${filename}" in signed_form["fields"]["key"] + expanded = {**signed_form["fields"], "key": "sdk/prefix/nested.bin"} + assert post(service, payload, expanded).status_code == 204 + assert signed(service, "GET", "/backup-bucket/sdk/prefix/nested.bin").content == payload + assert post(service, payload, {**expanded, "key": "sdk/outside.bin"}).status_code == 403 + assert service[2].uploads == 1 + + + +@pytest.mark.parametrize("status", ["200", "201", "204"]) +@pytest.mark.parametrize("size", [0, 1, 2049, 65537]) +def test_browser_post_exact_signed_success_response_modes(service, status, size): + """Each supported status responds only after checksum and SQL publication.""" + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(f"{status}:{size}".encode()).digest(size) + key = f"browser/status/{status}/{size}.bin" + response = post(service, payload, make_form( + service, payload, key=key, status=status, + )) + assert response.status_code == int(status), response.text + etag = '"' + hashlib.sha256(payload).hexdigest() + '"' + assert response.headers["etag"] == etag + assert response.headers["x-amz-version-id"] + if status == "201": + root = ET.fromstring(response.content) + assert root.tag.endswith("PostResponse") + assert root.findtext("{*}Bucket") == "backup-bucket" + assert root.findtext("{*}Key") == key + assert root.findtext("{*}ETag") == etag + assert root.findtext("{*}Location") == response.headers["location"] + assert response.headers["location"].endswith("/backup-bucket/" + key) + else: + assert response.content == b"" + assert "location" not in response.headers + assert signed(service, "GET", "/backup-bucket/" + key).content == payload + assert service[2].uploads == (0 if size == 0 else 1) + + +def test_browser_post_201_response_escapes_key_and_url(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"correctly-escaped-xml" * 32 + key = "browser/201/İstanbul & validation?.bin" + result = post(service, payload, make_form(service, payload, key=key, status="201")) + assert result.status_code == 201, result.text + root = ET.fromstring(result.content) + assert root.findtext("{*}Key") == key + assert "&" in result.text + assert root.findtext("{*}Location") == result.headers["location"] + assert result.headers["location"].endswith( + "/backup-bucket/" + quote(key, safe="/") + ) + assert service[2].uploads == 1 + + +@pytest.mark.parametrize("invalid", [ + "", "0", "202", "302", "-1", "0201", "201 ", "True", "201\\nLocation:evil", +]) +def test_browser_post_invalid_success_status_is_denied_before_send(service, invalid): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"status-validation" + response = post(service, payload, make_form(service, payload, status=invalid)) + assert response.status_code in (400, 403) + assert service[2].uploads == 0 + + +def test_browser_post_unsigned_or_changed_status_and_redirect_rejected(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"immutable-status" + original = make_form(service, body, status="201") + valid_without_status = make_form(service, body) + mutations = [ + {**original, "success_action_status": "200"}, + {**valid_without_status, "success_action_status": "201"}, + {k: v for k, v in original.items() if k != "success_action_status"}, + {**original, "success_action_redirect": "https://example.com/"}, + {**original, "x-amz-meta-injected": "not-signed"}, + ] + for fields in mutations: + response = post(service, body, fields) + assert response.status_code in (400, 403), response.text + assert service[2].uploads == 0 + + +def test_browser_post_signed_status_policy_rejects_unsupported_wildcard(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"bound-201" + base = make_form(service, payload, status="201") + clauses = json.loads(base64.b64decode(base["policy"]))["conditions"] + clauses[-2] = ["starts-with", "$success_action_status", "20"] + assert post(service, payload, make_form( + service, payload, status="201", clauses=clauses, + )).status_code == 403 + assert service[2].uploads == 0 + + +def test_real_boto3_presigned_post_status_201_xml(service): + """SDK-generated exact signed status must survive independent verification.""" + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(b"boto3-status-201").digest(262157) + checksum = base64.b64encode(hashlib.sha256(data).digest()).decode() + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/status-201.bin", + Fields={"x-amz-checksum-sha256": checksum, "success_action_status": "201"}, + Conditions=[ + {"x-amz-checksum-sha256": checksum}, + {"success_action_status": "201"}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + response = post(service, data, signed_form["fields"]) + assert response.status_code == 201, response.text + assert ET.fromstring(response.content).findtext("{*}Key") == "sdk/status-201.bin" + assert signed(service, "GET", "/backup-bucket/sdk/status-201.bin").content == data + assert service[2].uploads == 1 + + + +@pytest.mark.parametrize("size", [0, 1, 65537]) +def test_browser_post_exact_signed_https_redirect_persists_before_303(service, size): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(str(size).encode()).digest(size) + key = f"browser/callback/{size}.bin" + callback = "https://callbacks.example.test/upload/done?tracking=abc%2F123" + response = post(service, data, make_form( + service, data, key=key, redirect=callback, + )) + assert response.status_code == 303, response.text + assert response.content == b"" + etag = '"' + hashlib.sha256(data).hexdigest() + '"' + assert response.headers["etag"] == etag + url = urlsplit(response.headers["location"]) + assert url.scheme == "https" and url.hostname == "callbacks.example.test" + assert url.path == "/upload/done" + assert parse_qs(url.query) == { + "tracking": ["abc/123"], "bucket": ["backup-bucket"], + "key": [key], "etag": [etag], + } + assert signed(service, "GET", "/backup-bucket/" + key).content == data + assert service[2].uploads == (0 if size == 0 else 1) + + +def test_browser_post_https_callback_percent_encodes_non_ascii_key(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"redirect-unicode" * 313 + key = "browser/callback/İstanbul & 東京?.bin" + response = post(service, data, make_form( + service, data, key=key, redirect="https://callbacks.example.test/complete", + )) + assert response.status_code == 303 + assert parse_qs(urlsplit(response.headers["location"]).query)["key"] == [key] + assert "%C4%B0stanbul" in response.headers["location"] + assert "%26" in response.headers["location"] + assert service[2].uploads == 1 + + +@pytest.mark.parametrize("unsafe_url", [ + "", "http://callbacks.example.test/done", + "//callbacks.example.test/done", + "javascript:alert(1)", + "https://user:secret@callbacks.example.test/done", + "https://callbacks.example.test/done#fragment", + "https://callbacks.example.test:badport/done", + "https://callbacks.example.test/done\\evil", + "https://callbacks.example.test/done\\r\\nSet-Cookie:bad=1", + "https://", +]) +def test_browser_post_unsafe_signed_redirect_cannot_upload(service, unsafe_url): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"reject-unsafe-redirect" + response = post(service, data, make_form( + service, data, redirect=unsafe_url, + )) + assert response.status_code in (400, 403), response.text + assert service[2].uploads == 0 + + +def test_browser_post_callback_requires_exact_signed_url_not_status_override(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"authenticated-callback" + base = "https://callbacks.example.test/complete" + correct = make_form(service, data, redirect=base) + other = make_form(service, data) + altered = [ + {**correct, "success_action_redirect": base + "?changed=1"}, + {**other, "success_action_redirect": base}, + {k: v for k, v in correct.items() if k != "success_action_redirect"}, + make_form(service, data, redirect=base, status="201"), + {**correct, "success_action_status": "200"}, + ] + for fields in altered: + response = post(service, data, fields) + assert response.status_code in (400, 403), response.text + assert service[2].uploads == 0 + + +def test_real_boto3_presigned_post_https_callback_and_payload_integrity(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(b"boto3-redirect").digest(128 * 1024 + 7) + checksum = base64.b64encode(hashlib.sha256(payload).digest()).decode() + callback = "https://callbacks.example.test/complete?request=43" + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/callback.bin", + Fields={"x-amz-checksum-sha256": checksum, "success_action_redirect": callback}, + Conditions=[ + {"x-amz-checksum-sha256": checksum}, + {"success_action_redirect": callback}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + result = post(service, payload, signed_form["fields"]) + assert result.status_code == 303, result.text + assert parse_qs(urlsplit(result.headers["location"]).query)["key"] == ["sdk/callback.bin"] + assert signed(service, "GET", "/backup-bucket/sdk/callback.bin").content == payload + assert service[2].uploads == 1 + + + + +@pytest.mark.parametrize("mime", [ + "application/pdf", "image/png", "text/plain; charset=utf-8", + "application/octet-stream", +]) +def test_browser_post_content_type_is_signed_persisted_and_retrievable(service, mime): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(mime.encode()).digest(65537) + key = "browser/metadata/" + hashlib.sha256(mime.encode()).hexdigest()[:12] + result = post(service, payload, make_form( + service, payload, key=key, content_type=mime, + )) + assert result.status_code == 204, result.text + read = signed(service, "GET", "/backup-bucket/" + key) + assert read.status_code == 200 and read.content == payload + assert read.headers["content-type"] == mime + head = signed(service, "HEAD", "/backup-bucket/" + key) + assert head.status_code == 200 and head.headers["content-type"] == mime + assert service[2].uploads == 1 + + +def test_browser_post_content_type_signed_prefix_accepts_only_authorized_mimes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"verified-image" * 327 + policy = make_form( + service, payload, key="browser/image-policy.bin", + content_type="image/png", content_type_prefix="image/", + ) + form = {**policy, "Content-Type": "image/webp"} + assert post(service, payload, form).status_code == 204 + got = signed(service, "HEAD", "/backup-bucket/browser/image-policy.bin") + assert got.headers["content-type"] == "image/webp" + assert post(service, payload, {**policy, "Content-Type": "text/html"}).status_code == 403 + assert service[2].uploads == 1 + + +@pytest.mark.parametrize("override", [ + "text/html", "application/javascript", "application/pdf ", + "image/png\\nSet-Cookie: evil=1", +]) +def test_browser_post_exact_mime_cannot_be_changed_or_injected(service, override): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"mime-policy-enforced" + form = make_form(service, body, content_type="application/pdf") + response = post(service, body, {**form, "Content-Type": override}) + assert response.status_code in (400, 403) + assert service[2].uploads == 0 + + +def test_browser_post_unsigned_mime_and_malformed_mime_denied_before_send(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"metadata-must-be-bound" + unsigned = make_form(service, body) + assert post(service, body, {**unsigned, "Content-Type": "text/plain"}).status_code == 403 + form = make_form(service, body, content_type="text/plain") + conditions = json.loads(base64.b64decode(form["policy"]))["conditions"] + constraints = [ + entry for entry in conditions if entry != {"Content-Type": "text/plain"} + ] + assert post(service, body, make_form( + service, body, content_type="text/plain", clauses=constraints, + )).status_code == 403 + malformed = make_form(service, body, content_type="text/" + chr(1)) + assert post(service, body, malformed).status_code == 400 + assert service[2].uploads == 0 + + +def test_real_boto3_presigned_browser_post_content_type_is_persisted(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(b"sdk-mime").digest(131089) + checksum = base64.b64encode(hashlib.sha256(payload).digest()).decode() + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/mime.pdf", + Fields={ + "Content-Type": "application/pdf", + "x-amz-checksum-sha256": checksum, + }, + Conditions=[ + {"Content-Type": "application/pdf"}, + {"x-amz-checksum-sha256": checksum}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + assert post(service, payload, signed_form["fields"]).status_code == 204 + response = signed(service, "GET", "/backup-bucket/sdk/mime.pdf") + assert response.status_code == 200 and response.content == payload + assert response.headers["content-type"] == "application/pdf" + assert service[2].uploads == 1 + + + +@pytest.mark.parametrize("size", [0, 1, 65537]) +def test_browser_post_signed_custom_metadata_and_cache_headers_roundtrip(service, size): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(f"browser-metadata:{size}".encode()).digest(size) + metadata = { + "x-amz-meta-owner": "ci-operator", + "x-amz-meta-şehir": "not-allowed", + } + # The metadata field name is strictly ASCII and must be signed. + invalid = make_form(service, data, extra_fields=metadata) + assert post(service, data, invalid).status_code == 400 + extra = { + "x-amz-meta-owner": "ci-operator", + "x-amz-meta-i18n": "Istanbul & Tokyo", + "Cache-Control": "private, max-age=120", + "Content-Disposition": "attachment; filename=remote.bin", + "Content-Language": "tr", + } + key = f"browser/custom-metadata/{size}.bin" + result = post(service, data, make_form( + service, data, key=key, content_type="application/octet-stream", + extra_fields=extra, + )) + assert result.status_code == 204, result.text + get = signed(service, "GET", f"/backup-bucket/{key}") + head = signed(service, "HEAD", f"/backup-bucket/{key}") + assert get.status_code == 200 and get.content == data + assert head.status_code == 200 + for response in (get, head): + assert response.headers["x-amz-meta-owner"] == extra["x-amz-meta-owner"] + assert response.headers["x-amz-meta-i18n"] == extra["x-amz-meta-i18n"] + assert response.headers["cache-control"] == extra["Cache-Control"] + assert response.headers["content-disposition"] == extra["Content-Disposition"] + assert response.headers["content-language"] == "tr" + assert response.headers["content-type"] == "application/octet-stream" + assert service[2].uploads == (0 if size == 0 else 1) + + +@pytest.mark.parametrize("field", [ + "x-amz-meta-owner", "Cache-Control", "Content-Disposition", + "Content-Language", +]) +def test_browser_post_metadata_mutation_or_missing_policy_denied_before_send(service, field): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"metadata-must-be-exact" + initial = {field: "signed-value"} + signed_form = make_form(service, body, extra_fields=initial) + changes = [ + {**signed_form, field: "changed"}, + {k: v for k, v in signed_form.items() if k != field}, + {**make_form(service, body), field: "unsigned"}, + ] + for mutated in changes: + response = post(service, body, mutated) + assert response.status_code in (400, 403), response.text + assert service[2].uploads == 0 + + +def test_browser_post_metadata_must_be_bounded_before_telegram_send(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"guard-aggregate-metadata" + large = { + "x-amz-meta-one": "a" * 1024, + "x-amz-meta-two": "b" * 1024, + "x-amz-meta-three": "c" * 1024, + } + form = make_form(service, body, extra_fields=large) + assert post(service, body, form).status_code == 400 + too_many = {f"x-amz-meta-field-{n}": str(n) for n in range(13)} + assert post(service, body, make_form( + service, body, extra_fields=too_many, + )).status_code == 400 + malformed = {"x-amz-meta-shell": "valid\r\nInjected: unsafe"} + assert post(service, body, make_form( + service, body, extra_fields=malformed, + )).status_code == 400 + assert service[2].uploads == 0 + + +def test_real_boto3_presigned_post_custom_metadata_cache_and_binary_readback(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = hashlib.shake_256(b"boto3-meta-and-cache").digest(262149) + checksum = base64.b64encode(hashlib.sha256(payload).digest()).decode() + fields = { + "Content-Type": "application/pdf", + "Cache-Control": "private, max-age=60", + "x-amz-meta-ci-origin": "SDK-generated", + "x-amz-meta-ci-variant": "binary", + "x-amz-checksum-sha256": checksum, + } + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + generated = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/custom-meta.pdf", + Fields=fields, Conditions=[ + {name: value} for name, value in fields.items() + ] + [["content-length-range", 0, 8 * 1024 * 1024]], + ExpiresIn=120, + ) + result = post(service, payload, generated["fields"]) + assert result.status_code == 204, result.text + response = signed(service, "GET", "/backup-bucket/sdk/custom-meta.pdf") + assert response.status_code == 200 and response.content == payload + assert response.headers["x-amz-meta-ci-origin"] == "SDK-generated" + assert response.headers["x-amz-meta-ci-variant"] == "binary" + assert response.headers["cache-control"] == "private, max-age=60" + assert response.headers["content-type"] == "application/pdf" + assert service[2].uploads == 1 + + + +def test_browser_post_metadata_non_latin1_cannot_publish_unreadable_object(service): + """HTTP server cannot emit U+6771 directly in response header values.""" + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"metadata-header-encoding-regression" * 47 + fields = make_form(service, data, extra_fields={ + "x-amz-meta-unsafe": "東京", + }) + response = post(service, data, fields) + assert response.status_code == 400, response.text + assert service[2].uploads == 0 + + +@pytest.mark.parametrize("field,content", [ + ("x-amz-meta-bad@name", "value"), + ("x-amz-meta-", "empty-name"), + ("x-amz-meta-double.dot", "unsupported-name"), + ("Content-MD5", "unbound-unsupported-checksum"), + ("x-amz-server-side-encryption", "AES256"), +]) +def test_browser_post_rejects_unsupported_signed_fields_before_upload(service, field, content): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"do-not-write-insecure-metadata" + response = post(service, body, make_form( + service, body, extra_fields={field: content}, + )) + assert response.status_code == 400, response.text + assert service[2].uploads == 0 + + +def test_browser_post_metadata_duplicate_constraints_fail_closed(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"duplicate-user-meta-must-fail" + base = make_form(service, body, extra_fields={ + "x-amz-meta-ci": "signed", + }) + constraints = json.loads(base64.b64decode(base["policy"]))["conditions"] + # Duplicate the metadata clause and drop the key constraint, rather + # than accidentally retaining the original valid policy unchanged. + constraints[1] = {"x-amz-meta-ci": "signed"} + response = post(service, body, make_form( + service, body, extra_fields={"x-amz-meta-ci": "signed"}, + clauses=constraints, + )) + assert response.status_code == 403, response.text + assert service[2].uploads == 0 + + + +@pytest.mark.parametrize("size", [1, 65537, 1048593]) +def test_browser_post_metadata_prefix_signed_value_readback(service, size): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = hashlib.shake_256(f"metadata-prefix-{size}".encode()).digest(size) + key = f"browser/policy-meta/{size}.bin" + issued = make_form( + service, body, key=key, + extra_fields={"x-amz-meta-release": "p1:initial"}, + metadata_prefixes={"x-amz-meta-release": "p1:"}, + ) + permitted = {**issued, "x-amz-meta-release": "p1:verified-on-server"} + response = post(service, body, permitted) + assert response.status_code == 204, response.text + get = signed(service, "GET", "/backup-bucket/" + key) + assert get.content == body + assert get.headers["x-amz-meta-release"] == "p1:verified-on-server" + assert post(service, body, { + **issued, "x-amz-meta-release": "p2:unauthorized", + }).status_code == 403 + assert service[2].uploads == 1 + + +def test_real_boto3_browser_post_metadata_prefix_policy_interoperability(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(b"boto3-meta-prefix").digest(131087) + checksum = base64.b64encode(hashlib.sha256(data).digest()).decode() + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/meta-prefix.bin", + Fields={ + "x-amz-meta-env": "ci:original", + "x-amz-checksum-sha256": checksum, + }, + Conditions=[ + ["starts-with", "$x-amz-meta-env", "ci:"], + {"x-amz-checksum-sha256": checksum}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], ExpiresIn=120, + ) + authorized = {**form["fields"], "x-amz-meta-env": "ci:accepted"} + result = post(service, data, authorized) + assert result.status_code == 204, result.text + get = signed(service, "GET", "/backup-bucket/sdk/meta-prefix.bin") + assert get.content == data and get.headers["x-amz-meta-env"] == "ci:accepted" + assert post(service, data, { + **authorized, "x-amz-meta-env": "prod:forbidden", + }).status_code == 403 + assert service[2].uploads == 1 + + +@pytest.mark.parametrize("bad", [ + ["starts-with", "$x-amz-meta-does-not-exist", "ci:"], + ["starts-with", "$Cache-Control", "public"], + ["starts-with", "$x-amz-credential", ""], + ["starts-with", "$x-amz-meta-env", 5], + ["starts-with", "$x-amz-meta-env", "other:"], +]) +def test_browser_post_metadata_prefix_policy_does_not_weaken_auth(service, bad): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"must-not-be-accepted" + form = make_form( + service, data, extra_fields={"x-amz-meta-env": "ci:verified"}, + ) + clauses = json.loads(base64.b64decode(form["policy"]))["conditions"] + clauses[-2] = bad + result = post(service, data, make_form( + service, data, extra_fields={"x-amz-meta-env": "ci:verified"}, + clauses=clauses, + )) + assert result.status_code == 403, result.text + assert service[2].uploads == 0 + + + +def _post_tags(service, key: str): + response = signed(service, "GET", "/backup-bucket/" + key + "?tagging") + assert response.status_code == 200, response.text + root = ET.fromstring(response.content) + ns = "{http://s3.amazonaws.com/doc/2006-03-01/}" + return sorted( + (tag.findtext(ns + "Key"), tag.findtext(ns + "Value")) + for tag in root.findall(ns + "TagSet/" + ns + "Tag") + ) + + +@pytest.mark.parametrize("size", [0, 1, 65537, 1048607]) +def test_browser_post_exact_signed_tags_and_metadata_commit_together(service, size): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = hashlib.shake_256(f"signed-form-tags:{size}".encode()).digest(size) + key = f"browser/tags/{size}.bin" + tags = urlencode([("source", "browser"), ("şehir", "İstanbul & 東京")]) + fields = make_form( + service, body, key=key, status="201", + content_type="application/octet-stream", + extra_fields={ + "x-amz-tagging": tags, + "x-amz-meta-ci": "signed", + }, + ) + result = post(service, body, fields) + assert result.status_code == 201, result.text + assert ET.fromstring(result.content).findtext("{*}Key") == key + assert _post_tags(service, key) == [ + ("source", "browser"), ("şehir", "İstanbul & 東京"), + ] + response = signed(service, "GET", "/backup-bucket/" + key) + assert response.content == body + assert response.headers["x-amz-meta-ci"] == "signed" + assert response.headers["content-type"] == "application/octet-stream" + assert service[2].uploads == (0 if size == 0 else 1) + + +def test_browser_post_presigned_tags_are_immutable_across_overwrite(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + first = b"first-signed-browse-version" * 5 + second = b"second-signed-browse-version" * 7 + assert post(service, first, make_form( + service, first, key="browser/tag-versions.bin", + extra_fields={"x-amz-tagging": "stage=first"}, + )).status_code == 204 + assert _post_tags(service, "browser/tag-versions.bin") == [("stage", "first")] + first_id = signed(service, "HEAD", "/backup-bucket/browser/tag-versions.bin").headers[ + "x-amz-version-id" + ] + assert post(service, second, make_form( + service, second, key="browser/tag-versions.bin", + extra_fields={"x-amz-tagging": "stage=second"}, + )).status_code == 204 + assert _post_tags(service, "browser/tag-versions.bin") == [("stage", "second")] + from teledrive.s3.tagging import ObjectTagStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + db = service[3]() + try: + tags = ObjectTagStore(BucketCatalog(ManifestStore(db))) + assert tags.get(first_id) == [("stage", "first")] + finally: + db.close() + assert service[2].uploads == 2 + + +@pytest.mark.parametrize("wrong", [ + "source=stolen", + "source=browser&second=inserted", + "source=browser%ZZ", + "source=browser&source=duplicate", + "aws%3Asecret=reserved", + "key0=0&key1=1&key2=2&key3=3&key4=4&key5=5&key6=6&key7=7&key8=8&key9=9&key10=10", +]) +def test_browser_post_tampered_or_malformed_tags_never_send_to_telegram(service, wrong): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"presigned-tags-must-stay-pinned" + form = make_form(service, data, extra_fields={ + "x-amz-tagging": "source=browser", + }) + altered = {**form, "x-amz-tagging": wrong} + result = post(service, data, altered) + assert result.status_code in (400, 403), result.text + assert service[2].uploads == 0 + + +def test_browser_post_signed_invalid_tagset_is_rejected_before_upload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = b"invalid-but-exactly-signed-tags" + for bad in ( + "source=ok&source=duplicated", "aws%3Ainternal=unsafe", + "source=bad%ZZ", "=empty-key", "too-many=" + "x" * 257, + ): + result = post(service, data, make_form( + service, data, extra_fields={"x-amz-tagging": bad}, + )) + assert result.status_code == 400, result.text + assert service[2].uploads == 0 + + +def test_browser_post_tagging_requires_extra_explicit_permission(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + from teledrive.s3.credentials import S3CredentialStore + db = service[3]() + try: + write_only = S3CredentialStore(db, service[4]).create( + "browser-post-put-only", ["s3:PutObject"] + ) + finally: + db.close() + data = b"requires-tagging-privilege" + without = make_form( + service, data, key="browser/put-only-allowed.bin", issued=write_only, + ) + assert post(service, data, without).status_code == 204 + with_tags = make_form( + service, data, key="browser/put-only-denied.bin", issued=write_only, + extra_fields={"x-amz-tagging": "private=blocked"}, + ) + denied = post(service, data, with_tags) + assert denied.status_code == 403, denied.text + assert service[2].uploads == 1 + + + + +def test_signed_browser_tag_and_http_metadata_survive_online_sql_dr(service, tmp_path): + """A consistent SQLite snapshot restores immutable tag+header metadata. + + Telegram bytes remain in its dedicated remote, never in the SQL backup. + """ + from pathlib import Path + from scripts.ops.backup_metadata import backup_sqlite + from teledrive.db import connect_database + from teledrive.s3.object_metadata import ObjectMetadataStore + from teledrive.s3.tagging import ObjectTagStore + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(b"browser-tag-dr").digest(2 * 1024 * 1024 + 17) + key = "browser/dr/tags-and-metadata.bin" + response = post(service, data, make_form( + service, data, key=key, extra_fields={ + "x-amz-tagging": "operator=verified&stage=ci", + "Cache-Control": "private, max-age=60", + "x-amz-meta-origin": "signed-browser", + }, + content_type="application/octet-stream", + )) + assert response.status_code == 204, response.text + version = response.headers["x-amz-version-id"] + archive = tmp_path / "durable-metadata-backup.db" + active = service[3]() + try: + path = Path(active.engine.url.database) + finally: + active.close() + digest = backup_sqlite(path, archive) + assert len(digest) == 64 + restored = connect_database(str(archive)) + try: + catalog = BucketCatalog(ManifestStore(restored)) + head = catalog.get("backup-bucket", key) + assert head.item.id == version + assert head.item.sha256 == hashlib.sha256(data).hexdigest() + assert ObjectTagStore(catalog).get(version) == [ + ("operator", "verified"), ("stage", "ci"), + ] + metadata = ObjectMetadataStore(catalog).get(version) + assert metadata["x-amz-meta-origin"] == "signed-browser" + assert metadata["cache-control"] == "private, max-age=60" + assert metadata["content-type"] == "application/octet-stream" + finally: + restored.close() + assert _post_tags(service, key) == [ + ("operator", "verified"), ("stage", "ci"), + ] + assert signed(service, "GET", "/backup-bucket/" + key).content == data + assert service[2].uploads == 1 + + +def test_browser_post_tag_sql_failure_cannot_replace_published_version(service, monkeypatch): + from teledrive.s3.tagging import ObjectTagStore + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + first = b"previous-published-original" * 37 + key = "browser/transactional-tags.bin" + good = post(service, first, make_form( + service, first, key=key, + extra_fields={"x-amz-tagging": "stage=previous"}, + )) + assert good.status_code == 204 + old_id = signed(service, "HEAD", "/backup-bucket/" + key).headers[ + "x-amz-version-id" + ] + uploads_before = service[2].uploads + original = ObjectTagStore.attach_uncommitted + + def inject_sql_failure(self, object_id, tags): + if ("stage", "failing") in tags: + raise RuntimeError("injected tag journal failure") + return original(self, object_id, tags) + + monkeypatch.setattr(ObjectTagStore, "attach_uncommitted", inject_sql_failure) + later = b"new-remote-but-not-published" * 41 + failed = post(service, later, make_form( + service, later, key=key, + extra_fields={"x-amz-tagging": "stage=failing"}, + )) + assert failed.status_code == 503, failed.text + assert service[2].uploads == uploads_before + 1 + assert signed(service, "GET", "/backup-bucket/" + key).content == first + assert _post_tags(service, key) == [("stage", "previous")] + assert signed(service, "HEAD", "/backup-bucket/" + key).headers[ + "x-amz-version-id" + ] == old_id + + +def test_real_boto3_browser_post_signed_object_tagging_and_readback(service): + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + data = hashlib.shake_256(b"boto3-browser-post-tags").digest(262153) + checksum = base64.b64encode(hashlib.sha256(data).digest()).decode() + tags = urlencode([("source", "boto3-sdk"), ("test", "binary")]) + creds = service[1] + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/with-tags.bin", + Fields={ + "x-amz-tagging": tags, + "x-amz-checksum-sha256": checksum, + }, + Conditions=[ + {"x-amz-tagging": tags}, + {"x-amz-checksum-sha256": checksum}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + result = post(service, data, signed_form["fields"]) + assert result.status_code == 204, result.text + assert _post_tags(service, "sdk/with-tags.bin") == [ + ("source", "boto3-sdk"), ("test", "binary"), + ] + assert signed(service, "GET", "/backup-bucket/sdk/with-tags.bin").content == data + assert service[2].uploads == 1 + + +def test_browser_post_rejects_expired_or_revoked_signers(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"secret" + assert post(service, body, make_form(service, body, seconds=-1)).status_code == 403 + assert post(service, body, make_form(service, body, seconds=960)).status_code == 403 + conn = service[3]() + try: + readonly = S3CredentialStore(conn, service[4]).create("readonly", ["s3:GetObject"]) + assert post(service, body, make_form(service, body, issued=readonly)).status_code == 403 + assert S3CredentialStore(conn, service[4]).revoke(service[1].id) + finally: + conn.close() + assert post(service, body, make_form(service, body)).status_code == 403 + assert service[2].uploads == 0 + + +def test_browser_post_rejects_unsigned_extra_and_duplicate_fields(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"not-writable" + form = make_form(service, body) + assert post(service, body, {**form, "x-amz-meta-hijack": "yes"}).status_code in (400, 403) + assert post(service, body, {**form, "key": "prefix/"+"$"+"{filename}"}).status_code == 400 + duplicate = [(name, (None, value)) for name, value in form.items()] + duplicate += [ + ("key", (None, form["key"])), + ("file", ("payload.bin", body, "application/octet-stream")), + ] + # HTTPX only accepts multi-valued *files* for duplicate multipart keys; + # passing a list as data creates an invalid stream, not an HTTP request. + response = service[0].post("/backup-bucket", files=duplicate) + assert response.status_code == 400 + assert service[2].uploads == 0 + + +def test_browser_post_mime_upload_limit_blocks_before_remote_send(service, monkeypatch): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + monkeypatch.setattr(admin, "REST_MAX_UPLOAD_BYTES", 4096) + body = b"X" * 8192 + assert post(service, body, make_form(service, body)).status_code == 413 + assert service[2].uploads == 0 + + +def test_real_boto3_generated_sigv4_browser_post_policy(service): + """Interoperate with the SDK's real POST presigner, not a homegrown signer.""" + import boto3 + from botocore.config import Config + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + creds = service[1] + payload = b"sdk-authenticated-browser-upload" * 41 + checksum = base64.b64encode(hashlib.sha256(payload).digest()).decode() + client = boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=creds.access_key, + aws_secret_access_key=creds.secret_key, + region_name="us-east-1", config=Config( + signature_version="s3v4", s3={"addressing_style": "path"}, + ), + ) + signed_form = client.generate_presigned_post( + Bucket="backup-bucket", Key="sdk/form.bin", + Fields={"x-amz-checksum-sha256": checksum}, + Conditions=[ + {"x-amz-checksum-sha256": checksum}, + ["content-length-range", 0, 8 * 1024 * 1024], + ], + ExpiresIn=120, + ) + response = post(service, payload, signed_form["fields"]) + assert response.status_code == 204, response.text + assert service[2].uploads == 1 + assert signed(service, "GET", "/backup-bucket/sdk/form.bin").content == payload + + +def test_browser_post_cannot_claim_unimplemented_encryption(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + blob = b"unencrypted-payload" + fields = make_form(service, blob) + response = service[0].post( + "/backup-bucket", data=fields, + files={"file": ("sensitive.bin", blob)}, + headers={"x-amz-server-side-encryption": "AES256"}, + ) + assert response.status_code == 501 + assert service[2].uploads == 0 diff --git a/tests/s3/test_s3_presigned.py b/tests/s3/test_s3_presigned.py new file mode 100644 index 00000000..8b916501 --- /dev/null +++ b/tests/s3/test_s3_presigned.py @@ -0,0 +1,239 @@ +"""Real boto3-generated short-lived S3 presign regression tests. + +No in-memory signature stubs: botocore produces the wire URL and TestClient +executes it against the production SigV4 verifier and real SQL credential store. +""" +from __future__ import annotations + +import base64 +import hashlib +from datetime import datetime, timedelta, timezone +from urllib.parse import urlsplit + +import boto3 +import pytest +from botocore.config import Config +from starlette.requests import Request + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.sigv4 import SigV4Error, verify_header_signature + + +def sdk(service, access=None, secret=None): + credential = service[1] + return boto3.client( + "s3", endpoint_url="http://localhost:9000", + aws_access_key_id=access or credential.access_key, + aws_secret_access_key=secret or credential.secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", + s3={"addressing_style": "path"}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required"), + ) + + +def request_url(client, method, presigned_url, **kwargs): + uri = urlsplit(presigned_url) + assert uri.netloc == "localhost:9000" + return client.request(method, uri.path + "?" + uri.query, **kwargs) + + +def test_boto3_presigned_get_and_head_obey_scopes(service): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"telegram-presigned-read" * 120 + assert signed(service, "PUT", "/backup-bucket/signed.bin", payload).status_code == 200 + s3 = sdk(service) + get_url = s3.generate_presigned_url( + "get_object", Params={"Bucket": "backup-bucket", "Key": "signed.bin"}, ExpiresIn=120, + ) + result = request_url(client, "GET", get_url) + assert result.status_code == 200, result.text + assert result.content == payload + head_url = s3.generate_presigned_url( + "head_object", Params={"Bucket": "backup-bucket", "Key": "signed.bin"}, ExpiresIn=120, + ) + before = remote.downloads + head = request_url(client, "HEAD", head_url) + assert head.status_code == 200 + assert head.headers["content-length"] == str(len(payload)) + assert remote.downloads == before + + +def test_checksum_bound_presigned_put_prevents_unapproved_bytes(service): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"presigned-telegram-write" * 38 + checksum = base64.b64encode(hashlib.sha256(body).digest()).decode() + url = sdk(service).generate_presigned_url( + "put_object", + Params={"Bucket": "backup-bucket", "Key": "signed-put.bin", + "ChecksumSHA256": checksum}, + ExpiresIn=120, + HttpMethod="PUT", + ) + before = remote.uploads + corrupt = request_url(client, "PUT", url, content=b"modified-payload", + headers={"x-amz-checksum-sha256": checksum}) + assert corrupt.status_code == 400 + assert remote.uploads == before + assert signed(service, "GET", "/backup-bucket/signed-put.bin").status_code == 404 + authorized = request_url(client, "PUT", url, content=body, + headers={"x-amz-checksum-sha256": checksum}) + assert authorized.status_code == 200, authorized.text + assert remote.uploads == before + 1 + assert signed(service, "GET", "/backup-bucket/signed-put.bin").content == body + + +def test_bare_presigned_put_is_refused_not_accepted_unsigned(service): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + url = sdk(service).generate_presigned_url( + "put_object", Params={"Bucket": "backup-bucket", "Key": "unsigned.bin"}, + ExpiresIn=120, HttpMethod="PUT", + ) + before = remote.uploads + denied = request_url(client, "PUT", url, content=b"unbound") + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + assert remote.uploads == before + + +def test_presign_query_path_tampering_duplicate_params_and_revocation(service): + client, credential, _, connect, master = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/unchanged.bin", b"correct").status_code == 200 + url = sdk(service).generate_presigned_url( + "get_object", Params={"Bucket": "backup-bucket", "Key": "unchanged.bin"}, + ExpiresIn=120, + ) + parsed = urlsplit(url) + corrupted_path = parsed.path.replace("unchanged.bin", "spoofed.bin") + bad_path = client.get(corrupted_path + "?" + parsed.query) + assert bad_path.status_code == 403 + assert b"SignatureDoesNotMatch" in bad_path.content + replay_extra = client.get(parsed.path + "?" + parsed.query + "&X-Amz-Expires=120") + assert replay_extra.status_code == 403 + connection = connect() + try: + assert S3CredentialStore(connection, master).revoke(credential.id) + finally: + connection.close() + denied = request_url(client, "GET", url) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + + +def test_presigned_expiration_is_enforced_without_sleep(service): + _, credential, _, connect, master = service + url = sdk(service).generate_presigned_url( + "list_buckets", Params={}, ExpiresIn=3, + ) + parsed = urlsplit(url) + scope = { + "type": "http", "method": "GET", "path": parsed.path, + "raw_path": parsed.path.encode("ascii"), + "query_string": parsed.query.encode("ascii"), + "headers": [(b"host", b"localhost:9000")], + } + request = Request(scope) + db = connect() + try: + store = S3CredentialStore(db, master) + assert verify_header_signature( + request, store.resolve, "s3:ListAllMyBuckets", + now=datetime.now(timezone.utc) + ) == credential.access_key + with pytest.raises(SigV4Error, match="expired") as caught: + verify_header_signature( + request, store.resolve, "s3:ListAllMyBuckets", + now=datetime.now(timezone.utc) + timedelta(seconds=10) + ) + assert caught.value.code == "AccessDenied" + finally: + db.close() + + +def test_presigned_delete_uses_object_scope_and_expires_with_key_revoke(service): + client, _, _, connect, master = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/delete-me.bin", b"still-in-telegram").status_code == 200 + url = sdk(service).generate_presigned_url( + "delete_object", + Params={"Bucket": "backup-bucket", "Key": "delete-me.bin"}, + ExpiresIn=120, + HttpMethod="DELETE", + ) + deleted = request_url(client, "DELETE", url) + assert deleted.status_code == 204 + assert signed(service, "GET", "/backup-bucket/delete-me.bin").status_code == 404 + # A read-only credential may sign a URL, but it must never acquire DELETE. + db = connect() + try: + reader = S3CredentialStore(db, master).create( + "presign-reader", ["s3:GetObject"] + ) + finally: + db.close() + readonly = sdk(service, reader.access_key, reader.secret_key).generate_presigned_url( + "delete_object", + Params={"Bucket": "backup-bucket", "Key": "delete-me.bin"}, + ExpiresIn=120, HttpMethod="DELETE", + ) + rejected = request_url(client, "DELETE", readonly) + assert rejected.status_code == 403 + assert b"AccessDenied" in rejected.content + + +def test_presigned_unicode_key_and_mutated_expiration_fail_closed(service): + client, _, _, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + from urllib.parse import quote + key = "folders/café space %.bin" + body = b"UTF-8-key-and-path" + path = "/backup-bucket/" + quote(key, safe="/") + assert signed(service, "PUT", path, body).status_code == 200 + url = sdk(service).generate_presigned_url( + "get_object", Params={"Bucket": "backup-bucket", "Key": key}, + ExpiresIn=120, + ) + response = request_url(client, "GET", url) + assert response.status_code == 200, response.text + assert response.content == body + parsed = urlsplit(url) + forged = parsed.path + "?" + parsed.query.replace( + "X-Amz-Expires=120", "X-Amz-Expires=600" + ) + invalid = client.get(forged) + assert invalid.status_code == 403 + assert b"SignatureDoesNotMatch" in invalid.content + + +def test_checksum_bound_presigned_create_once_rejects_replay(service): + """Signed If-None-Match prevents reuse of a valid presigned PUT URL. + + A leaked still-valid URL must not silently overwrite the object after + its first successful, checksum-verified Telegram upload. + """ + client, _, remote, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + body = b"one-time-presigned-create" + checksum = base64.b64encode(hashlib.sha256(body).digest()).decode("ascii") + url = sdk(service).generate_presigned_url( + "put_object", + Params={"Bucket": "backup-bucket", "Key": "create-once.bin", + "ChecksumSHA256": checksum, "IfNoneMatch": "*"}, + ExpiresIn=120, + HttpMethod="PUT", + ) + headers = {"x-amz-checksum-sha256": checksum, "If-None-Match": "*"} + first = request_url(client, "PUT", url, content=body, headers=headers) + assert first.status_code == 200, first.text + count = remote.uploads + replay = request_url(client, "PUT", url, content=body, headers=headers) + assert replay.status_code == 412 + assert b"PreconditionFailed" in replay.content + assert remote.uploads == count + assert signed(service, "GET", "/backup-bucket/create-once.bin").content == body diff --git a/tests/s3/test_s3_presigned_complete.py b/tests/s3/test_s3_presigned_complete.py new file mode 100644 index 00000000..9f4b2acb --- /dev/null +++ b/tests/s3/test_s3_presigned_complete.py @@ -0,0 +1,187 @@ +"""Checksum-bound presigned multipart Complete never permits mutable XML bodies.""" +from __future__ import annotations + +import hashlib +from urllib.parse import urlsplit +from xml.etree import ElementTree as ET + +from botocore.auth import S3SigV4QueryAuth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import begin, part +from teledrive.s3.credentials import S3CredentialStore + + +def _url(service, upload_id, payload, *, checksum=True, extra=""): + issued = service[1] + headers = {} + if checksum: + headers["x-amz-content-sha256"] = hashlib.sha256(payload).hexdigest() + request = AWSRequest( + method="POST", + url="http://localhost:9000/backup-bucket/asset.bin?uploadId=" + upload_id + extra, + # Query presigning must not parse XML as botocore JSON/form data. + # SigV4 authenticates method, path/query and the signed XML SHA-256. + data=b"", + headers=headers, + ) + request.context["payload_signing_enabled"] = False + S3SigV4QueryAuth( + Credentials(issued.access_key, issued.secret_key), + "s3", "us-east-1", expires=120, + ).add_auth(request) + parsed = urlsplit(request.url) + return parsed.path + "?" + parsed.query, headers + + +def _body(number, etag): + root = ET.Element("CompleteMultipartUpload") + item = ET.SubElement(root, "Part") + ET.SubElement(item, "PartNumber").text = str(number) + ET.SubElement(item, "ETag").text = etag + return ET.tostring(root) + + +def test_presigned_complete_binds_xml_and_does_not_reupload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + etag = part(service, uid, 1, b"bound-multipart-part") + xml = _body(1, etag) + url, headers = _url(service, uid, xml) + before = service[2].uploads + mutated = service[0].post(url, content=xml + b" ", + headers=headers) + assert mutated.status_code == 400, mutated.text + assert service[2].uploads == before + done = service[0].post(url, content=xml, headers=headers) + assert done.status_code == 200, done.text + assert service[2].uploads == before + assert signed(service, "GET", "/backup-bucket/asset.bin").content == b"bound-multipart-part" + again = service[0].post(url, content=xml, headers=headers) + assert again.status_code == 200 + assert service[2].uploads == before + + +def test_presigned_complete_requires_signed_checksum_and_existing_upload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + etag = part(service, uid, 1, b"keep") + xml = _body(1, etag) + unchecked, headers = _url(service, uid, xml, checksum=False) + before = service[2].uploads + denied = service[0].post(unchecked, content=xml, headers=headers) + assert denied.status_code == 403, denied.text + assert b"AccessDenied" in denied.content + assert service[2].uploads == before + # Other POST query selectors cannot borrow the protected Complete flow. + unsupported, bound_headers = _url(service, uid, xml, extra="&uploads") + bad = service[0].post(unsupported, content=xml, headers=bound_headers) + assert bad.status_code == 501 + assert service[2].uploads == before + + +def test_revoked_presigned_complete_does_not_publish(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + uid = begin(service) + etag = part(service, uid, 1, b"staged") + xml = _body(1, etag) + url, headers = _url(service, uid, xml) + db = service[3]() + try: + assert S3CredentialStore(db, service[4]).revoke(service[1].id) + finally: + db.close() + before = service[2].uploads + denied = service[0].post(url, content=xml, headers=headers) + assert denied.status_code == 403 + assert b"AccessDenied" in denied.content + assert service[2].uploads == before + + +def _presigned_initiate(service, *, checksum=True, extra=""): + issued = service[1] + headers = {} + if checksum: + headers["x-amz-content-sha256"] = hashlib.sha256(b"").hexdigest() + request = AWSRequest( + method="POST", + url="http://localhost:9000/backup-bucket/asset.bin?uploads" + extra, + data=b"", headers=headers, + ) + request.context["payload_signing_enabled"] = False + S3SigV4QueryAuth( + Credentials(issued.access_key, issued.secret_key), + "s3", "us-east-1", expires=120, + ).add_auth(request) + parsed = urlsplit(request.url) + return parsed.path + "?" + parsed.query, headers + + +def _multipart_session_count(service): + from sqlalchemy import func, select + from teledrive.s3.multipart import MultipartStore, uploads + from teledrive.storage.catalog import BucketCatalog + from teledrive.storage.manifest import ManifestStore + db = service[3]() + try: + # The first test may inspect sessions before any multipart endpoint + # has initialized the schema. + MultipartStore(BucketCatalog(ManifestStore(db))) + return db.connection.execute( + select(func.count()).select_from(uploads) + ).scalar_one() + finally: + db.close() + + +def test_presigned_initiate_accepts_exact_empty_body_and_existing_permissions(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + url, headers = _presigned_initiate(service) + before = _multipart_session_count(service) + created = service[0].post(url, content=b"", headers=headers) + assert created.status_code == 200, created.text + assert b"" in created.content + assert _multipart_session_count(service) == before + 1 + assert service[2].uploads == 0 + + +def test_presigned_initiate_rejects_tampered_body_without_sql_session(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + url, headers = _presigned_initiate(service) + before = _multipart_session_count(service) + denied = service[0].post(url, content=b"unexpected-body", headers=headers) + assert denied.status_code == 400 and b"InvalidRequest" in denied.content + assert _multipart_session_count(service) == before + assert service[2].uploads == 0 + + +def test_presigned_initiate_requires_signed_checksum_and_exact_selector(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + before = _multipart_session_count(service) + unsigned_url, missing_headers = _presigned_initiate(service, checksum=False) + missing = service[0].post(unsigned_url, content=b"", headers=missing_headers) + assert missing.status_code == 403 and b"AccessDenied" in missing.content + mixed_url, bound_headers = _presigned_initiate(service, extra="&uploadId=other") + mixed = service[0].post(mixed_url, content=b"", headers=bound_headers) + assert mixed.status_code == 501 and b"NotImplemented" in mixed.content + url, headers = _presigned_initiate(service) + wrong_key = service[0].post(url.replace("asset.bin", "other.bin"), + content=b"", headers=headers) + assert wrong_key.status_code == 403 + assert _multipart_session_count(service) == before + + +def test_revoked_presigned_initiate_never_allocates_session(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + url, headers = _presigned_initiate(service) + before = _multipart_session_count(service) + db = service[3]() + try: + assert S3CredentialStore(db, service[4]).revoke(service[1].id) + finally: + db.close() + response = service[0].post(url, content=b"", headers=headers) + assert response.status_code == 403 and b"AccessDenied" in response.content + assert _multipart_session_count(service) == before diff --git a/tests/s3/test_s3_public_access_block.py b/tests/s3/test_s3_public_access_block.py new file mode 100644 index 00000000..6a44b746 --- /dev/null +++ b/tests/s3/test_s3_public_access_block.py @@ -0,0 +1,126 @@ +"""Public access block maps to the actual enforced private-only contract.""" +from __future__ import annotations + +import hashlib +from xml.etree import ElementTree as ET + +import pytest + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.public_access import public_access_xml + +ALL_TRUE = public_access_xml() + + +def test_private_only_bucket_public_access_controls_and_no_payload_calls(service): + client, _, remote, _, _ = service + assert signed(service, "PUT", "/private-ci").status_code == 200 + path = "/private-ci?publicAccessBlock" + initial = signed(service, "GET", path) + assert initial.status_code == 200, initial.text + root = ET.fromstring(initial.content) + assert len(root) == 4 + assert all(child.text == "true" for child in root) + assert signed(service, "PUT", path, ALL_TRUE).status_code == 200 + assert signed(service, "PUT", path, ALL_TRUE).status_code == 200 + assert signed(service, "DELETE", path).status_code == 501 + assert remote.uploads == 0 and remote.downloads == 0 + assert signed(service, "GET", "/private-ci/file.bin").status_code == 404 + # Anonymous reads remain denied, even after PUT controls. + assert client.get("/private-ci/file.bin").status_code == 403 + + +@pytest.mark.parametrize("bad,status", [ + (b"", 400), + (b'falsetruetruetrue', 501), + (b"]>&x;", 400), + (b"x" * 16385, 400), + (ALL_TRUE.replace(b"true", b""), 400), + (ALL_TRUE.replace(b"true", b"TRUE"), 400), +]) +def test_public_access_cannot_be_disabled_or_partially_configured(service, bad, status): + assert signed(service, "PUT", "/private-ci").status_code == 200 + answer = signed(service, "PUT", "/private-ci?publicAccessBlock", bad) + assert answer.status_code == status, answer.text + assert b"true" in signed(service, "GET", "/private-ci?publicAccessBlock").content + + +def test_private_control_permission_separation(service): + _, _, _, connect, master = service + assert signed(service, "PUT", "/private-ci").status_code == 200 + db = connect() + try: + store = S3CredentialStore(db, master) + reader = store.create("read-public-config", ["s3:GetBucketPublicAccessBlock"]) + writer = store.create("write-public-config", ["s3:PutBucketPublicAccessBlock"]) + other = store.create("unrelated", ["s3:GetObject"]) + finally: + db.close() + path = "/private-ci?publicAccessBlock" + assert signed(service, "GET", path, access=reader.access_key, secret=reader.secret_key).status_code == 200 + assert signed(service, "PUT", path, ALL_TRUE, access=reader.access_key, secret=reader.secret_key).status_code == 403 + assert signed(service, "GET", path, access=writer.access_key, secret=writer.secret_key).status_code == 403 + assert signed(service, "PUT", path, ALL_TRUE, access=writer.access_key, secret=writer.secret_key).status_code == 200 + assert signed(service, "GET", path, access=other.access_key, secret=other.secret_key).status_code == 403 + + +def test_missing_bucket_selector_conflicts_and_signed_checksums(service): + assert signed(service, "GET", "/missing-private?publicAccessBlock").status_code == 404 + assert signed(service, "PUT", "/missing-private?publicAccessBlock", ALL_TRUE).status_code == 404 + assert signed(service, "PUT", "/private-ci").status_code == 200 + for path in ("/private-ci?publicAccessBlock&publicAccessBlock", + "/private-ci?publicAccessBlock&tagging"): + assert signed(service, "GET", path).status_code == 400 + assert signed(service, "GET", "/private-ci/item?publicAccessBlock").status_code == 501 + assert signed(service, "PUT", "/private-ci?publicAccessBlock", ALL_TRUE, + extra_headers={"Content-MD5": "AAAA"}).status_code == 400 + + +def test_actual_boto3_public_access_block_client(service): + import socket + import threading + import time + import uvicorn + import boto3 + from botocore.config import Config + from botocore.exceptions import ClientError + import web.s3 as gateway + + _, credential, remote, _, _ = service + assert signed(service, "PUT", "/private-ci").status_code == 200 + sock = socket.socket() + sock.bind(("127.0.0.1", 0)) + sock.listen(128) + port = sock.getsockname()[1] + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, log_level="error", lifespan="off", + )) + thread = threading.Thread(target=server.run, kwargs={"sockets": [sock]}, daemon=True) + thread.start() + try: + deadline = time.monotonic() + 10 + while not server.started and time.monotonic() < deadline: + time.sleep(0.02) + assert server.started + s3 = boto3.client("s3", endpoint_url=f"http://127.0.0.1:{port}", + region_name="us-east-1", + aws_access_key_id=credential.access_key, + aws_secret_access_key=credential.secret_key, + config=Config(signature_version="s3v4", + s3={"addressing_style": "path"})) + expected = {k: True for k in ( + "BlockPublicAcls", "IgnorePublicAcls", + "BlockPublicPolicy", "RestrictPublicBuckets", + )} + assert s3.get_public_access_block(Bucket="private-ci")["PublicAccessBlockConfiguration"] == expected + ok = s3.put_public_access_block(Bucket="private-ci", PublicAccessBlockConfiguration=expected) + assert ok["ResponseMetadata"]["HTTPStatusCode"] == 200 + with pytest.raises(ClientError): + s3.put_public_access_block(Bucket="private-ci", + PublicAccessBlockConfiguration={ + **expected, "BlockPublicAcls": False, + }) + finally: + server.should_exit = True + thread.join(timeout=10) diff --git a/tests/s3/test_s3_recovery.py b/tests/s3/test_s3_recovery.py new file mode 100644 index 00000000..558f13d4 --- /dev/null +++ b/tests/s3/test_s3_recovery.py @@ -0,0 +1,145 @@ +"""Crash recovery safety contracts for S3 multipart completion. + +These assertions use real SQL transactions and the same authenticated admin +HTTP endpoints deployed by WebUI; no production Telegram messages are touched. +""" +from __future__ import annotations + +import hashlib +import time + +from sqlalchemy import select, update + +from tests.s3.test_s3_admin import admin +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import setup, begin, part, complete +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +from teledrive.s3.multipart import MultipartStore, completion_leases, uploads + + +def _store(url): + db = connect_database(url) + return db, MultipartStore(BucketCatalog(ManifestStore(db))) + + +def test_expired_completion_requires_explicit_admin_execute(admin): + client, url, _ = admin + now = int(time.time()) + db, manager = _store(url) + try: + manager.catalog.create_bucket("recovery-bucket", now) + uid = manager.initiate("recovery-bucket", "data/file.bin", "creator") + token = manager._claim_completion(uid, now - 7200, 120) + assert manager.stale_completions(min_age_seconds=3600)[0]["id"] == uid + finally: + db.close() + + endpoint = f"/api/v1/admin/s3-multipart/{uid}/recover" + auth = ("testadmin", "secure-auth-password") + assert client.post(endpoint, json={"execute": True}).status_code in (401, 403) + assert client.post(endpoint, json={"execute": True}, auth=auth).status_code == 403 + preview = client.get("/api/v1/admin/s3-multipart/stale", auth=auth) + assert preview.status_code == 200 + assert len(preview.json()["uploads"]) == 1 + assert preview.json()["uploads"][0]["id"] == uid + # POST defaults to dry-run: missing execute cannot reset the lease. + check = client.post(endpoint, json={}, auth=auth, headers={"x-teledrive-csrf": "1"}) + assert check.status_code == 200 + assert check.json() == { + "upload_id": uid, "eligible": True, "recovered": False, "dry_run": True + } + db, manager = _store(url) + try: + assert manager._renew_completion(uid, token, now, 120) is False + assert db.connection.execute(select(uploads.c.state).where( + uploads.c.id == uid + )).scalar_one() == "completing" + finally: + db.close() + restored = client.post(endpoint, json={"execute": True}, auth=auth, + headers={"x-teledrive-csrf": "1"}) + assert restored.status_code == 200 + assert restored.json()["recovered"] is True + db, manager = _store(url) + try: + assert manager.require(uid, "recovery-bucket", "data/file.bin", "creator")["state"] == "active" + assert not manager._renew_completion(uid, token, now, 120) + assert not manager._reset_owned_completion(uid, token) + assert manager.stale_completions() == [] + finally: + db.close() + repeat = client.post(endpoint, json={"execute": True}, auth=auth, + headers={"x-teledrive-csrf": "1"}) + assert repeat.status_code == 200 + assert repeat.json()["recovered"] is False + + +def test_no_recovery_for_live_or_legacy_unfenced_completion(admin): + _, url, _ = admin + now = int(time.time()) + db, mpu = _store(url) + try: + mpu.catalog.create_bucket("recovery-bucket", now) + live = mpu.initiate("recovery-bucket", "live", "creator") + live_token = mpu._claim_completion(live, now, 120) + assert mpu.recover_stale_completion(live, dry_run=True) is False + assert mpu.recover_stale_completion(live, dry_run=False) is False + assert mpu._renew_completion(live, live_token, now, 120) + + old = mpu.initiate("recovery-bucket", "legacy", "creator") + db.connection.execute(update(uploads).where(uploads.c.id == old) + .values(state="completing")) + db.commit() + assert mpu.recover_stale_completion( + old, dry_run=False, now=now + 100000 + ) is False + assert db.connection.execute(select(uploads.c.state) + .where(uploads.c.id == old)).scalar_one() == "completing" + finally: + db.close() + + +def test_recovery_fences_old_writer_and_preserves_staged_parts(service): + setup(service) + uid = begin(service) + etag = part(service, uid, 1, b"telegram-survives-recovery") + db = service[3]() + try: + mpu = MultipartStore(BucketCatalog(ManifestStore(db))) + old = int(time.time()) - 7200 + token = mpu._claim_completion(uid, old, 120) + assert mpu.recover_stale_completion(uid, dry_run=True, min_age_seconds=3600) + assert mpu.recover_stale_completion(uid, dry_run=False, min_age_seconds=3600) + assert mpu._renew_completion(uid, token, int(time.time()), 120) is False + assert mpu._reset_owned_completion(uid, token) is False + assert mpu.list_parts(uid, "backup-bucket", "asset.bin", service[1].access_key)[0].etag == etag.strip('"') + finally: + db.close() + done = complete(service, uid, [(1, etag)]) + assert done.status_code == 200, done.text + got = signed(service, "GET", "/backup-bucket/asset.bin") + assert got.status_code == 200 + assert got.content == b"telegram-survives-recovery" + assert service[2].uploads == 1 + + +def test_ownership_token_must_be_live_for_commit(service): + setup(service) + uid = begin(service) + part(service, uid, 1, b"content") + db = service[3]() + try: + mpu = MultipartStore(BucketCatalog(ManifestStore(db))) + token = mpu._claim_completion(uid, int(time.time()) - 7200, 120) + now = int(time.time()) + assert not mpu._renew_completion(uid, token, now, 120) + assert mpu.recover_stale_completion(uid, now=now, min_age_seconds=3600) + assert mpu.recover_stale_completion(uid, now=now, + min_age_seconds=3600, dry_run=False) + # The abandoned uploader must not modify the recovered upload. + assert not mpu._reset_owned_completion(uid, token) + assert mpu.require(uid, "backup-bucket", "asset.bin", service[1].access_key)["state"] == "active" + finally: + db.close() diff --git a/tests/s3/test_s3_route_guard.py b/tests/s3/test_s3_route_guard.py new file mode 100644 index 00000000..67444b50 --- /dev/null +++ b/tests/s3/test_s3_route_guard.py @@ -0,0 +1,219 @@ +"""P2-0 protocol selector audit: unsupported controls must never mutate. + +Tests go through the real SigV4 HTTP app, SQLite and deterministic Telegram +adapter; valid signatures must not turn unsupported AWS methods into success. +""" +from __future__ import annotations + +import json +import re +from collections import Counter +from pathlib import Path +from urllib.parse import urlencode + +import pytest +from starlette.datastructures import QueryParams + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.route_guard import validate_s3_query, S3QueryError + +ROOT = Path(__file__).resolve().parents[2] +CONTROL_SELECTORS = ( + "acl", "policy", "website", "notification", + "notificationConfiguration", "logging", "replication", "encryption", + "object-lock", "retention", "legal-hold", "publicAccessBlock-malformed", + "ownershipControls", "accelerate", "requestPayment", "select", + "select-type", "restore", "torrent", "analytics", "inventory", + "metrics", "intelligent-tiering", "x-unknown-provider-option", +) +EXACT_SUPPORTED = {"location", "cors", "tagging", "versioning", + "publicAccessBlock", "lifecycle"} + + +@pytest.mark.parametrize("method", ["GET", "HEAD", "PUT", "DELETE"]) +@pytest.mark.parametrize("selector", CONTROL_SELECTORS) +def test_unknown_bucket_control_never_falls_back_to_bucket_crud( + service, method, selector, +): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + before = service[2].uploads + reply = signed(service, method, f"/guard-bucket?{selector}") + assert reply.status_code == 501, (method, selector, reply.text) + if method != "HEAD": + assert b"NotImplemented" in reply.content + assert signed(service, "HEAD", "/guard-bucket").status_code == 200 + assert service[2].uploads == before + + +@pytest.mark.parametrize("method", ["GET", "HEAD", "PUT", "DELETE"]) +@pytest.mark.parametrize("selector", CONTROL_SELECTORS) +def test_unknown_object_control_never_falls_back_to_data_or_delete( + service, method, selector, +): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + assert signed(service, "PUT", "/guard-bucket/asset.bin", b"old-data").status_code == 200 + sent = service[2].uploads + reply = signed(service, method, f"/guard-bucket/asset.bin?{selector}", + b"new-data" if method == "PUT" else b"") + assert reply.status_code == 501, (method, selector, reply.text) + assert signed(service, "GET", "/guard-bucket/asset.bin").content == b"old-data" + assert service[2].uploads == sent + + +@pytest.mark.parametrize("method,path", [ + ("DELETE", "/guard-bucket?prefix=backup"), + ("DELETE", "/guard-bucket?max-keys=5"), + ("PUT", "/guard-bucket?max-keys=10"), + ("GET", "/guard-bucket?continuation-token=abc"), + ("GET", "/guard-bucket?marker=x&list-type=2"), + ("GET", "/guard-bucket?tagging&prefix=secret"), + ("GET", "/guard-bucket?uploads&versions"), + ("POST", "/guard-bucket?delete&uploads"), + ("GET", "/guard-bucket/data?response-content-type=text/plain"), + ("PUT", "/guard-bucket/data?prefix=x"), + ("DELETE", "/guard-bucket/data?versioning"), + ("DELETE", "/guard-bucket/data?lifecycle"), + ("GET", "/guard-bucket/data?uploads"), + ("GET", "/guard-bucket/data?uploadId=uid&prefix=x"), + ("POST", "/guard-bucket/data?select&select-type=2"), +]) +def test_conflicting_or_wrong_scope_controls_fail_closed(service, method, path): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + code = signed(service, method, path).status_code + assert code in (400, 501), (method, path, code) + assert signed(service, "HEAD", "/guard-bucket").status_code == 200 + assert service[2].uploads == 0 + + +@pytest.mark.parametrize("path", [ + "/guard-bucket?prefix=x&prefix=y", + "/guard-bucket?list-type=2&list-type=2", + "/guard-bucket?tagging&tagging", + "/guard-bucket?versioning=Enabled", + "/guard-bucket?lifecycle=Enabled", + "/guard-bucket?uploads=1", + "/guard-bucket/data?tagging=1", + "/guard-bucket/data?versionId=a&versionId=b", +]) +def test_duplicate_or_nonempty_subresources_rejected(service, path): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + result = signed(service, "GET", path) + assert result.status_code == 400, result.text + assert service[2].uploads == 0 + + +def test_existing_sdk_query_and_paginated_listings_still_work(service): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + assert signed(service, "PUT", "/guard-bucket/example.bin", b"ok").status_code == 200 + v2 = signed(service, "GET", "/guard-bucket?list-type=2&prefix=example&max-keys=5&x-id=ListObjectsV2") + assert v2.status_code == 200, v2.text + assert b"example.bin" in v2.content + v1 = signed(service, "GET", "/guard-bucket?prefix=example&max-keys=5") + assert v1.status_code == 200, v1.text + assert b"example.bin" in v1.content + + +def test_116_operation_manifest_exactly_matches_reviewed_spec(): + doc = (ROOT / "docs/S3_SPECS.md").read_text() + data = json.loads((ROOT / "teledrive/s3/operation_manifest.json").read_text()) + pattern = ( + r"^\| \[([A-Za-z0-9]+)\]\(" + r"https://docs\.aws\.amazon\.com/AmazonS3/latest/API/API_" + r"([A-Za-z0-9]+)\.html\) \| (✅ Supported|🟡 Partial|❌ Unsupported) \|" + ) + rows = re.findall(pattern, doc, flags=re.MULTILINE) + mapped = {"✅ Supported":"supported", "🟡 Partial":"partial", + "❌ Unsupported":"unsupported"} + expected = {name: mapped[status] for name, api_name, status in rows + if name == api_name} + assert len(rows) == len(expected) == len(data["operations"]) == 116 + actual = {item["name"]: item["status"] for item in data["operations"]} + assert expected == actual + assert Counter(actual.values()) == { + "supported":11, "partial":27, "unsupported":78, + } + assert set(item["workstream"] for item in data["operations"]) >= { + "implemented", "P2-C", "P2-B", "P2-D", "P2-E", "P2-F", "P3-G", "P3-H", + } + assert all(set(item) == {"name","status","workstream"} + for item in data["operations"]) + + +def test_unknown_parameter_is_rejected_before_object_transport(): + # No FastAPI/SQL is needed to prove the guard is independent of secrets. + with pytest.raises(S3QueryError) as denied: + validate_s3_query("DELETE", "source", False, QueryParams("policy")) + assert denied.value.code == "NotImplemented" + + +@pytest.mark.parametrize("method,headers", [ + ("DELETE", {"x-amz-bypass-governance-retention": "true"}), + ("DELETE", {"x-amz-mfa": "123456 123456"}), + ("DELETE", {"x-amz-expected-bucket-owner": "123456789012"}), + ("DELETE", {"x-amz-copy-source": "/guard-bucket/asset.bin"}), + ("DELETE", {"x-amz-tagging": "class=archive"}), + ("HEAD", {"x-amz-request-payer": "requester"}), + ("GET", {"x-amz-server-side-encryption-customer-algorithm": "AES256"}), +]) +def test_unsupported_header_contract_does_not_change_or_expose_object( + service, method, headers, +): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + assert signed(service, "PUT", "/guard-bucket/asset.bin", + b"protected-content").status_code == 200 + before = service[2].uploads + forbidden = signed( + service, method, "/guard-bucket/asset.bin", + extra_headers=headers, + ) + assert forbidden.status_code == 501, (method, headers, forbidden.text) + assert signed(service, "GET", "/guard-bucket/asset.bin").content == b"protected-content" + assert service[2].uploads == before + + +def test_real_sha256_checksum_mode_readback_for_aws_cli_and_sdk(service): + """SDK-requested checksum headers bind the actual immutable object bytes.""" + import base64 + import hashlib + + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + blob = b"checksum-bound-exact-plaintext" * 17 + put = signed(service, "PUT", "/guard-bucket/checksummed", blob) + assert put.status_code == 200, put.text + version = put.headers["x-amz-version-id"] + encoded = base64.b64encode(hashlib.sha256(blob).digest()).decode("ascii") + for method, path in ( + ("GET", "/guard-bucket/checksummed"), + ("HEAD", "/guard-bucket/checksummed"), + ("GET", "/guard-bucket/checksummed?versionId=" + version), + ): + reply = signed(service, method, path, + extra_headers={"x-amz-checksum-mode": "ENABLED"}) + assert reply.status_code == 200, reply.text + assert reply.headers["x-amz-checksum-sha256"] == encoded + assert reply.headers["x-amz-checksum-type"] == "FULL_OBJECT" + if method == "GET": + assert reply.content == blob + assert signed(service, "GET", "/guard-bucket/checksummed").headers.get( + "x-amz-checksum-sha256" + ) is None + + for method, path, mode, expected in ( + ("GET", "/guard-bucket/checksummed", "BAD", 400), + ("PUT", "/guard-bucket/checksummed", "ENABLED", 501), + ("DELETE", "/guard-bucket/checksummed", "ENABLED", 501), + ("GET", "/guard-bucket", "ENABLED", 501), + ): + result = signed(service, method, path, extra_headers={ + "x-amz-checksum-mode": mode, + }) + assert result.status_code == expected, result.text + assert signed(service, "GET", "/guard-bucket/checksummed").content == blob + + +def test_root_prefix_is_intentional_supported_list_buckets_parameter(service): + assert signed(service, "PUT", "/guard-bucket").status_code == 200 + result = signed(service, "GET", "/?prefix=guard-") + assert result.status_code == 200, result.text + assert b"guard-bucket" in result.content + assert service[2].uploads == service[2].downloads == 0 diff --git a/tests/s3/test_s3_sigv4_defense.py b/tests/s3/test_s3_sigv4_defense.py new file mode 100644 index 00000000..7b1736c4 --- /dev/null +++ b/tests/s3/test_s3_sigv4_defense.py @@ -0,0 +1,69 @@ +"""Strict SigV4 wire-query validation in both header and presigned modes.""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest +from starlette.requests import Request + +from teledrive.s3.sigv4 import ( + MAX_QUERY_BYTES, MAX_QUERY_FIELDS, SigV4Error, + _canonical_query, verify_header_signature, +) + +NOW = datetime(2026, 10, 13, 12, 0, tzinfo=timezone.utc) + + +def _request(query: bytes, presigned: bool) -> Request: + headers = [(b"host", b"localhost:9000")] + if not presigned: + auth = ( + "AWS4-HMAC-SHA256 " + "Credential=TDTEST/20261013/us-east-1/s3/aws4_request," + "SignedHeaders=host;x-amz-date,Signature=" + "0" * 64 + ) + headers.extend(( + (b"x-amz-date", b"20261013T120000Z"), + (b"x-amz-content-sha256", b"UNSIGNED-PAYLOAD"), + (b"authorization", auth.encode("ascii")), + )) + return Request({ + "type": "http", "method": "GET", + "scheme": "http", "server": ("localhost", 9000), + "path": "/private/obj.bin", "raw_path": b"/private/obj.bin", + "query_string": query, "headers": headers, + }) + + +@pytest.mark.parametrize("query", ( + b"prefix=valid&bad=\xff", + b"prefix=%FF", + b"prefix=%C3%28", + b"&".join([b"parameter=value"] * (MAX_QUERY_FIELDS + 1)), + b"prefix=" + b"x" * MAX_QUERY_BYTES, +)) +@pytest.mark.parametrize("presigned", [False, True]) +def test_invalid_queries_fail_as_s3_protocol_errors(query, presigned): + if presigned: + query = b"X-Amz-Algorithm=AWS4-HMAC-SHA256&" + query + request = _request(query, presigned) + with pytest.raises(SigV4Error) as error: + verify_header_signature( + request, + lambda *_: pytest.fail("Untrusted query reached credential lookup"), + "s3:GetObject", now=NOW, + ) + assert error.value.code == "InvalidArgument" + + +def test_valid_unicode_and_duplicate_keys_canonicalize_stably(): + assert _canonical_query( + b"prefix=%C3%A9&delimiter=%2F&empty=&prefix=a" + ) == "delimiter=%2F&empty=&prefix=%C3%A9&prefix=a" + + +def test_query_limits_are_inclusive_not_off_by_one(): + wire = b"k=" + b"x" * (MAX_QUERY_BYTES - 2) + assert _canonical_query(wire) == wire.decode("ascii") + pairs = b"&".join([b"k=v"] * MAX_QUERY_FIELDS) + assert _canonical_query(pairs) == pairs.decode("ascii") diff --git a/tests/s3/test_s3_tagging.py b/tests/s3/test_s3_tagging.py new file mode 100644 index 00000000..087bcb5e --- /dev/null +++ b/tests/s3/test_s3_tagging.py @@ -0,0 +1,521 @@ +"""S3 object tagging: signed real HTTP, immutable version metadata and boto3. + +No Telegram payload is reuploaded on tag edits/copies. Exercise actual SQL +transaction boundaries, XML parsing and permission-scoped credentials. +""" +from __future__ import annotations + +import base64 +import hashlib +import socket +import threading +import time +import xml.etree.ElementTree as ET + +import boto3 +import pytest +from botocore.config import Config +from teledrive.s3.credentials import S3CredentialStore +from teledrive.s3.tagging import ObjectTagStore, validate_tags, TaggingError +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore, objects +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_copy import copy + +_NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def tag_xml(entries): + root = ET.Element("Tagging") + set_el = ET.SubElement(root, "TagSet") + for key, value in entries: + item = ET.SubElement(set_el, "Tag") + ET.SubElement(item, "Key").text = key + ET.SubElement(item, "Value").text = value + return ET.tostring(root, encoding="utf-8") + + +def extract_tags(response): + assert response.status_code == 200, response.text + root = ET.fromstring(response.content) + return [(tag.findtext(_NS + "Key"), tag.findtext(_NS + "Value")) + for tag in root.findall(_NS + "TagSet/" + _NS + "Tag")] + + +def create(service, *, key="file.bin", data=b"Telegram-backed-body"): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/" + key, data).status_code == 200 + + +def test_tags_roundtrip_no_telegram_download_or_upload(service): + create(service) + remote = service[2] + uploads, downloads = remote.uploads, remote.downloads + url = "/backup-bucket/file.bin?tagging" + before = signed(service, "GET", url) + assert extract_tags(before) == [] + changed = signed(service, "PUT", url, tag_xml([("project", "TeleDrive"), ("şehir", "İstanbul")])) + assert changed.status_code == 200, changed.text + assert extract_tags(signed(service, "GET", url)) == [("project", "TeleDrive"), ("şehir", "İstanbul")] + assert remote.uploads == uploads and remote.downloads == downloads + cleared = signed(service, "DELETE", url) + assert cleared.status_code == 204 + assert extract_tags(signed(service, "GET", url)) == [] + assert remote.uploads == uploads and remote.downloads == downloads + + +def test_tags_copy_snapshot_isolated_from_source_and_overwrites(service): + create(service, data=b"v1") + url = "/backup-bucket/file.bin?tagging" + assert signed(service, "PUT", url, tag_xml([("stage", "original")])).status_code == 200 + copied = copy(service, "/backup-bucket/file.bin", "/backup-bucket/duplicate") + assert copied.status_code == 200, copied.text + dest = "/backup-bucket/duplicate?tagging" + assert extract_tags(signed(service, "GET", dest)) == [("stage", "original")] + assert signed(service, "PUT", url, tag_xml([("stage", "modified")])).status_code == 200 + assert extract_tags(signed(service, "GET", dest)) == [("stage", "original")] + old_uploads = service[2].uploads + assert signed(service, "PUT", "/backup-bucket/file.bin", b"v2").status_code == 200 + assert service[2].uploads == old_uploads + 1 + assert extract_tags(signed(service, "GET", url)) == [] + assert extract_tags(signed(service, "GET", dest)) == [("stage", "original")] + # Previous immutable version metadata remains available internally. + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + from sqlalchemy import select + recorded = db.connection.execute( + select(objects.c.id, objects.c.sha256).where( + objects.c.object_key == "file.bin" + ) + ).all() + assert len(recorded) == 2 + by_digest = {row.sha256: row.id for row in recorded} + old_version_id = by_digest[hashlib.sha256(b"v1").hexdigest()] + current_version_id = by_digest[hashlib.sha256(b"v2").hexdigest()] + assert catalog.get("backup-bucket", "file.bin").item.id == current_version_id + tagger = ObjectTagStore(catalog) + assert tagger.get(old_version_id) == [("stage", "modified")] + assert tagger.get(current_version_id) == [] + finally: + db.close() + + +def test_tagging_scopes_do_not_leak_through_generic_object_permissions(service): + create(service) + db = service[3]() + try: + credentials = S3CredentialStore(db, service[4]) + reader = credentials.create("read-only-data", ["s3:GetObject"]) + tag_reader = credentials.create("read-tag", ["s3:GetObjectTagging"]) + tag_writer = credentials.create("write-tag", ["s3:PutObjectTagging"]) + finally: + db.close() + path = "/backup-bucket/file.bin?tagging" + denied = signed(service, "GET", path, + access=reader.access_key, secret=reader.secret_key) + assert denied.status_code == 403 + allowed = signed(service, "GET", path, + access=tag_reader.access_key, secret=tag_reader.secret_key) + assert extract_tags(allowed) == [] + denied_put = signed(service, "PUT", path, tag_xml([("k", "v")]), + access=tag_reader.access_key, secret=tag_reader.secret_key) + assert denied_put.status_code == 403 + allowed_put = signed(service, "PUT", path, tag_xml([("k", "v")]), + access=tag_writer.access_key, secret=tag_writer.secret_key) + assert allowed_put.status_code == 200 + denied_delete = signed(service, "DELETE", path, + access=tag_writer.access_key, secret=tag_writer.secret_key) + assert denied_delete.status_code == 403 + assert extract_tags(signed(service, "GET", path)) == [("k", "v")] + + +def test_wrong_digest_and_malformed_xml_never_mutate_tags(service): + create(service) + path = "/backup-bucket/file.bin?tagging" + assert signed(service, "PUT", path, tag_xml([("safe", "old")])).status_code == 200 + malformed = signed(service, "PUT", path, b"") + assert malformed.status_code == 400 + assert b"MalformedXML" in malformed.content + attack = (b']>' + b'&name;x') + hostile = signed(service, "PUT", path, attack) + assert hostile.status_code == 400 + assert extract_tags(signed(service, "GET", path)) == [("safe", "old")] + from botocore.auth import S3SigV4Auth + from botocore.awsrequest import AWSRequest + from botocore.credentials import Credentials + original = tag_xml([("safe", "new")]) + req = AWSRequest( + method="PUT", url="http://localhost:9000" + path, + data=original, headers={"x-amz-content-sha256": hashlib.sha256(original).hexdigest()} + ) + req.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(service[1].access_key, service[1].secret_key), + "s3", "us-east-1").add_auth(req) + changed_body = service[0].put(path, headers=dict(req.headers), content=b"forged body") + assert changed_body.status_code in (400, 403) + assert extract_tags(signed(service, "GET", path)) == [("safe", "old")] + + +def test_limits_duplicate_keys_and_unsupported_version_are_rejected(service): + create(service) + path = "/backup-bucket/file.bin?tagging" + invalid_sets = [ + [("a", "b")] * 2, [(f"k{i}", "v") for i in range(11)], + [("aws:secret", "value")], [("x"*129, "v")], + [("key", "v"*257)], [("", "invalid")] + ] + for entries in invalid_sets: + r = signed(service, "PUT", path, tag_xml(entries)) + assert r.status_code == 400, (entries, r.text) + assert b"InvalidTag" in r.content + assert signed(service, "GET", "/backup-bucket/file.bin?tagging&versionId=old").status_code == 404 + assert signed(service, "GET", "/backup-bucket/file.bin?versionId=old").status_code == 404 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"new", + extra_headers={"x-amz-tagging": "k=v"}).status_code == 200 + assert extract_tags(signed(service, "GET", path)) == [("k", "v")] + + +def test_boto3_tagging_over_actual_http_server(service): + import uvicorn + import web.s3 as gateway + + client, issued, remote, _, _ = service + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/SDK", b"original").status_code == 200 + socket_obj = socket.socket() + socket_obj.bind(("127.0.0.1", 0)) + port = socket_obj.getsockname()[1] + socket_obj.listen(16) + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, lifespan="off", log_level="error" + )) + worker = threading.Thread(target=server.run, kwargs={"sockets":[socket_obj]}, daemon=True) + worker.start() + try: + for _ in range(100): + if server.started: + break + time.sleep(0.05) + assert server.started + sdk = boto3.client("s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=issued.access_key, aws_secret_access_key=issued.secret_key, + region_name="us-east-1", + config=Config( + signature_version="s3v4", + s3={"addressing_style":"path", "payload_signing_enabled":True}, + request_checksum_calculation="when_required", + response_checksum_validation="when_required", + retries={"max_attempts":0}, + ) + ) + before = remote.uploads + put = sdk.put_object_tagging(Bucket="backup-bucket", Key="SDK", + Tagging={"TagSet":[{"Key":"a","Value":"1"},{"Key":"b","Value":"2"}]}) + assert put["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sdk.get_object_tagging(Bucket="backup-bucket", Key="SDK")["TagSet"] == [ + {"Key":"a", "Value":"1"},{"Key":"b", "Value":"2"} + ] + sdk.delete_object_tagging(Bucket="backup-bucket", Key="SDK") + assert sdk.get_object_tagging(Bucket="backup-bucket", Key="SDK")["TagSet"] == [] + sdk.put_bucket_tagging(Bucket="backup-bucket", Tagging={ + "TagSet":[{"Key":"environment","Value":"ci"}] + }) + assert sdk.get_bucket_tagging(Bucket="backup-bucket")["TagSet"] == [ + {"Key":"environment","Value":"ci"} + ] + sdk.delete_bucket_tagging(Bucket="backup-bucket") + inserted = sdk.put_object(Bucket="backup-bucket", Key="sdk-inline", + Body=b"sdk-inline-bytes", Tagging="source=sdk") + assert inserted["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sdk.get_object_tagging(Bucket="backup-bucket", Key="sdk-inline")["TagSet"] == [ + {"Key":"source","Value":"sdk"} + ] + uploaded = remote.uploads + sdk.copy_object(Bucket="backup-bucket", Key="sdk-inline-copy", + CopySource={"Bucket":"backup-bucket", "Key":"sdk-inline"}, + TaggingDirective="REPLACE", Tagging="source=copy") + assert sdk.get_object_tagging(Bucket="backup-bucket", Key="sdk-inline-copy")["TagSet"] == [ + {"Key":"source","Value":"copy"} + ] + assert remote.uploads == uploaded == before + 1 + finally: + server.should_exit = True + worker.join(timeout=10) + socket_obj.close() + assert not worker.is_alive() + + +def test_bucket_tagging_correct_s3_missing_semantics_and_safe_removal(service): + remote = service[2] + assert signed(service, "PUT", "/bucket-with-tags").status_code == 200 + url = "/bucket-with-tags?tagging" + initial = signed(service, "GET", url) + assert initial.status_code == 404 + assert b"NoSuchTagSet" in initial.content + before = (remote.uploads, remote.downloads) + set_tags = signed(service, "PUT", url, tag_xml([("cost", "ops"), ("zone", "eu")])) + assert set_tags.status_code == 200, set_tags.text + assert extract_tags(signed(service, "GET", url)) == [("cost", "ops"), ("zone", "eu")] + assert before == (remote.uploads, remote.downloads) + erased = signed(service, "DELETE", url) + assert erased.status_code == 204 + assert signed(service, "GET", url).status_code == 404 + # Removing a physically empty bucket must not leave a FK-bound tag row. + assert signed(service, "PUT", url, tag_xml([("role","temporary")])).status_code == 200 + assert signed(service, "DELETE", "/bucket-with-tags").status_code == 204 + assert signed(service, "GET", url).status_code == 404 + + +def test_bucket_tagging_scopes_and_checksum_fail_closed(service): + assert signed(service, "PUT", "/bucket-with-tags").status_code == 200 + db = service[3]() + try: + keys = S3CredentialStore(db, service[4]) + data_reader = keys.create("object-read", ["s3:GetObject"]) + tag_reader = keys.create("bucket-tag-read", ["s3:GetBucketTagging"]) + tag_writer = keys.create("bucket-tag-write", ["s3:PutBucketTagging"]) + finally: + db.close() + path = "/bucket-with-tags?tagging" + denied = signed(service, "GET", path, access=data_reader.access_key, + secret=data_reader.secret_key) + assert denied.status_code == 403 + allowed = signed(service, "PUT", path, tag_xml([("account", "1")]), + access=tag_writer.access_key, secret=tag_writer.secret_key) + assert allowed.status_code == 200 + assert extract_tags(signed(service, "GET", path, access=tag_reader.access_key, + secret=tag_reader.secret_key)) == [("account","1")] + assert signed(service, "DELETE", path, access=tag_writer.access_key, + secret=tag_writer.secret_key).status_code == 403 + assert signed(service, "PUT", path, b"").status_code == 400 + assert extract_tags(signed(service, "GET", path)) == [("account","1")] + + +def test_bucket_tagging_is_isolated_from_object_tags(service): + create(service) + path = "/backup-bucket/file.bin?tagging" + assert signed(service, "PUT", path, tag_xml([("object", "only")])).status_code == 200 + assert signed(service, "PUT", "/backup-bucket?tagging", + tag_xml([("bucket", "only")])).status_code == 200 + assert extract_tags(signed(service, "GET", path)) == [("object", "only")] + assert extract_tags(signed(service, "GET", "/backup-bucket?tagging")) == [("bucket", "only")] + assert signed(service, "DELETE", "/backup-bucket?tagging").status_code == 204 + assert extract_tags(signed(service, "GET", path)) == [("object", "only")] + + +def test_signed_inline_tags_are_published_with_new_telegram_object(service): + create(service, data=b"old-blob") + remote = service[2] + response = signed(service, "PUT", "/backup-bucket/file.bin", b"new-blob", + extra_headers={"x-amz-tagging": "role=storage&city=%C4%B0stanbul"}) + assert response.status_code == 200, response.text + assert extract_tags(signed(service, "GET", "/backup-bucket/file.bin?tagging")) == [ + ("city", "İstanbul"), ("role", "storage") + ] + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"new-blob" + assert remote.uploads == 2 + + +def test_invalid_inline_tag_or_missing_tagging_permission_never_uploads(service): + create(service) + remote = service[2] + before = remote.uploads + denied = signed(service, "PUT", "/backup-bucket/file.bin", b"new-blob", + extra_headers={"x-amz-tagging": "k=first&k=duplicate"}) + assert denied.status_code == 400 + assert remote.uploads == before + db = service[3]() + try: + limited = S3CredentialStore(db, service[4]).create( + "put-no-tag-rights", ["s3:PutObject"] + ) + finally: + db.close() + forbidden = signed(service, "PUT", "/backup-bucket/file.bin", b"new-blob", + extra_headers={"x-amz-tagging": "status=private"}, + access=limited.access_key, secret=limited.secret_key) + assert forbidden.status_code == 403 + assert remote.uploads == before + assert signed(service, "GET", "/backup-bucket/file.bin").content == b"Telegram-backed-body" + + +def test_copy_tag_directive_replace_snapshots_new_tags_without_telegram_upload(service): + create(service, data=b"source-body") + assert signed(service, "PUT", "/backup-bucket/file.bin?tagging", + tag_xml([("original", "source")])).status_code == 200 + before = service[2].uploads + replace = copy(service, "/backup-bucket/file.bin", "/backup-bucket/replaced", + **{"x-amz-tagging-directive": "REPLACE", + "x-amz-tagging": "destination=copy"}) + assert replace.status_code == 200, replace.text + assert service[2].uploads == before + assert extract_tags(signed(service, "GET", "/backup-bucket/replaced?tagging")) == [ + ("destination", "copy") + ] + assert extract_tags(signed(service, "GET", "/backup-bucket/file.bin?tagging")) == [ + ("original", "source") + ] + bad = copy(service, "/backup-bucket/file.bin", "/backup-bucket/should-not-exist", + **{"x-amz-tagging": "not-replace=forbidden"}) + assert bad.status_code == 400 + assert signed(service, "HEAD", "/backup-bucket/should-not-exist").status_code == 404 + + +def test_copy_tagging_cannot_bypass_separate_read_write_scopes(service): + create(service) + source = "/backup-bucket/file.bin" + assert signed(service, "PUT", source + "?tagging", + tag_xml([("confidential", "label")])).status_code == 200 + db = service[3]() + try: + limited = S3CredentialStore(db, service[4]).create( + "basic-copy-only", ["s3:GetObject", "s3:PutObject"] + ) + replace_only = S3CredentialStore(db, service[4]).create( + "can-change-tags", ["s3:GetObject", "s3:PutObject", "s3:PutObjectTagging"] + ) + finally: + db.close() + # A client with ordinary CopyObject permissions cannot exfiltrate + # protected source tags by copying the object to its own key. + denied = signed(service, "PUT", "/backup-bucket/no-leak", + extra_headers={"x-amz-copy-source": source}, + access=limited.access_key, secret=limited.secret_key) + assert denied.status_code == 403 + assert signed(service, "HEAD", "/backup-bucket/no-leak").status_code == 404 + # Explicit REPLACE may deliberately strip tags, but only with permission. + replacement = signed(service, "PUT", "/backup-bucket/unlabeled", + extra_headers={"x-amz-copy-source": source, + "x-amz-tagging-directive": "REPLACE"}, + access=replace_only.access_key, secret=replace_only.secret_key) + assert replacement.status_code == 200, replacement.text + assert extract_tags(signed(service, "GET", "/backup-bucket/unlabeled?tagging")) == [] + + +def test_bucket_tagging_full_50_tag_boundary_utf16_and_large_xml(service): + """50 high-entropy UTF-16-valid tags exceed prior 10-tag/16 KiB limits.""" + assert signed(service, "PUT", "/tag-boundary-ci").status_code == 200 + url = "/tag-boundary-ci?tagging" + tags = [(f"key-{n:02d}", "é" * 256) for n in range(50)] + xml = tag_xml(tags) + assert len(xml) > 16 * 1024 + assert len(xml) < 256 * 1024 + first = signed(service, "PUT", url, xml) + assert first.status_code == 200, first.text + expected = sorted(tags) + assert extract_tags(signed(service, "GET", url)) == expected + assert signed(service, "PUT", url, xml).status_code == 200 + assert extract_tags(signed(service, "GET", url)) == expected + # The object path retains the smaller AWS 10-tag maximum. + assert signed(service, "PUT", "/tag-boundary-ci/data", b"unchanged").status_code == 200 + invalid_object = signed(service, "PUT", "/tag-boundary-ci/data?tagging", + tag_xml(tags[:11])) + assert invalid_object.status_code == 400 + assert extract_tags(signed(service, "GET", url)) == expected + assert service[2].uploads == 1 + + +def test_bucket_tagging_rejects_51_tags_and_duplicate_fields_atomically(service): + assert signed(service, "PUT", "/tag-boundary-ci").status_code == 200 + url = "/tag-boundary-ci?tagging" + assert signed(service, "PUT", url, tag_xml([("unchanged", "yes")])).status_code == 200 + attacks = [ + tag_xml([(f"k{n}", str(n)) for n in range(51)]), + tag_xml([("duplicate", "a"), ("duplicate", "b")]), + tag_xml([("aws:reserved", "no")]), + tag_xml([("", "no")]), + tag_xml([("k", "x" * 257)]), + tag_xml([("k" * 129, "v")]), + b'', + b'onetwo', + b']>' + b'&a;x', + ] + for payload in attacks: + rejected = signed(service, "PUT", url, payload) + assert rejected.status_code == 400, (payload[:100], rejected.text) + assert extract_tags(signed(service, "GET", url)) == [("unchanged", "yes")] + assert service[2].uploads == service[2].downloads == 0 + + +def test_bucket_tagging_valid_crc32_and_corrupt_checksum_do_not_update_tags(service): + import zlib + + assert signed(service, "PUT", "/tag-checksum-ci").status_code == 200 + url = "/tag-checksum-ci?tagging" + payload = tag_xml([("checksum", "valid")]) + crc = base64.b64encode((zlib.crc32(payload) & 0xffffffff).to_bytes(4, "big")).decode() + accepted = signed(service, "PUT", url, payload, extra_headers={ + "x-amz-checksum-crc32": crc, "x-amz-sdk-checksum-algorithm": "CRC32", + }) + assert accepted.status_code == 200, accepted.text + rejected = signed(service, "PUT", url, tag_xml([("checksum", "invalid")]), + extra_headers={"x-amz-checksum-crc32": crc, + "x-amz-sdk-checksum-algorithm": "CRC32"}) + assert rejected.status_code == 400, rejected.text + mismatch_md5 = signed(service, "PUT", url, tag_xml([("checksum", "invalid")]), + extra_headers={"Content-MD5": base64.b64encode( + b"0" * 16).decode()}) + assert mismatch_md5.status_code == 400 + assert extract_tags(signed(service, "GET", url)) == [("checksum", "valid")] + + +def test_boto3_full_bucket_tagging_limit_and_replacement_over_http(service): + import uvicorn + import web.s3 as gateway + from botocore.exceptions import ClientError + + _, issued, remote, _, _ = service + assert signed(service, "PUT", "/tag-boundary-ci").status_code == 200 + socket_obj = socket.socket() + socket_obj.bind(("127.0.0.1", 0)) + socket_obj.listen(16) + port = socket_obj.getsockname()[1] + server = uvicorn.Server(uvicorn.Config( + gateway.app, host="127.0.0.1", port=port, + lifespan="off", log_level="error", + )) + worker = threading.Thread(target=server.run, kwargs={"sockets": [socket_obj]}, + daemon=True) + worker.start() + try: + for _ in range(150): + if server.started: + break + time.sleep(0.04) + assert server.started + sdk = boto3.client( + "s3", endpoint_url=f"http://127.0.0.1:{port}", + aws_access_key_id=issued.access_key, aws_secret_access_key=issued.secret_key, + region_name="us-east-1", + config=Config(signature_version="s3v4", + s3={"addressing_style": "path"}, + retries={"max_attempts": 0}), + ) + pairs = [{"Key": f"tag-{n:02d}", "Value": "v" * 256} for n in range(50)] + answer = sdk.put_bucket_tagging( + Bucket="tag-boundary-ci", Tagging={"TagSet": pairs}, + ) + assert answer["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sdk.get_bucket_tagging(Bucket="tag-boundary-ci")["TagSet"] == pairs + with pytest.raises(ClientError): + sdk.put_bucket_tagging( + Bucket="tag-boundary-ci", + Tagging={"TagSet": pairs + [{"Key": "too-many", "Value": "v"}]}, + ) + assert sdk.get_bucket_tagging(Bucket="tag-boundary-ci")["TagSet"] == pairs + assert sdk.put_bucket_tagging( + Bucket="tag-boundary-ci", + Tagging={"TagSet": [{"Key": "replaced", "Value": "yes"}]}, + )["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert sdk.get_bucket_tagging(Bucket="tag-boundary-ci")["TagSet"] == [ + {"Key": "replaced", "Value": "yes"} + ] + assert remote.uploads == remote.downloads == 0 + finally: + server.should_exit = True + worker.join(timeout=10) + socket_obj.close() + assert not worker.is_alive() diff --git a/tests/s3/test_s3_version_copy.py b/tests/s3/test_s3_version_copy.py new file mode 100644 index 00000000..01aa3cbc --- /dev/null +++ b/tests/s3/test_s3_version_copy.py @@ -0,0 +1,65 @@ +"""Signed server-side CopySource historical versions: no Telegram reupload.""" +from __future__ import annotations + +import hashlib + +from tests.s3.test_s3_gateway import service, signed +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +def _version(service, key): + db = service[3]() + try: + return BucketCatalog(ManifestStore(db)).get("backup-bucket", key).item.id + finally: + db.close() + + +def test_copy_old_published_source_after_head_overwrite_and_delete(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source.bin", b"old-version").status_code == 200 + old_id = _version(service, "source.bin") + assert signed(service, "PUT", "/backup-bucket/source.bin", b"new-version").status_code == 200 + uploads = service[2].uploads + result = signed(service, "PUT", "/backup-bucket/copied.bin", b"", + extra_headers={"x-amz-copy-source": + "/backup-bucket/source.bin?versionId=" + old_id}) + assert result.status_code == 200, result.text + assert signed(service, "GET", "/backup-bucket/copied.bin").content == b"old-version" + assert signed(service, "GET", "/backup-bucket/source.bin").content == b"new-version" + assert service[2].uploads == uploads + assert signed(service, "DELETE", "/backup-bucket/source.bin").status_code == 204 + again = signed(service, "PUT", "/backup-bucket/after-delete.bin", b"", + extra_headers={"x-amz-copy-source": + "/backup-bucket/source.bin?versionId=" + old_id}) + assert again.status_code == 200, again.text + assert signed(service, "GET", "/backup-bucket/after-delete.bin").content == b"old-version" + assert service[2].uploads == uploads + + +def test_copy_rejects_unknown_or_wrong_source_version_without_upload(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source.bin", b"current").status_code == 200 + version = _version(service, "source.bin") + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + unpub = cat.manifests.begin("backup-bucket", "source.bin", 0, 123) + cat.manifests.finish(unpub, hashlib.sha256(b"").hexdigest(), 123) + finally: + db.close() + uploads = service[2].uploads + for src, code in ( + ("/backup-bucket/source.bin?versionId=" + unpub, 404), + ("/backup-bucket/other.bin?versionId=" + version, 404), + ("/backup-bucket/source.bin?versionId=unknown", 404), + ("/backup-bucket/source.bin?versionId=" + version + "&versionId=" + version, 400), + ("/backup-bucket/source.bin?partNumber=1", 400), + ("/backup-bucket/source.bin?versionId=", 400), + ): + response = signed(service, "PUT", "/backup-bucket/fail.bin", b"", + extra_headers={"x-amz-copy-source": src}) + assert response.status_code == code, (src, response.text) + assert signed(service, "GET", "/backup-bucket/fail.bin").status_code == 404 + assert service[2].uploads == uploads diff --git a/tests/s3/test_s3_version_listing.py b/tests/s3/test_s3_version_listing.py new file mode 100644 index 00000000..a4a216ba --- /dev/null +++ b/tests/s3/test_s3_version_listing.py @@ -0,0 +1,112 @@ +"""S3 signed ListObjectVersions: published-only scope and stable pagination.""" +from __future__ import annotations + +import xml.etree.ElementTree as ET + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def test_signed_version_listing_excludes_staged_and_enforces_scoped_key(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for key, payload in ( + ("history.bin", b"first"), ("history.bin", b"second"), + ("other.bin", b"third"), + ): + assert signed(service, "PUT", "/backup-bucket/" + key, payload).status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + import hashlib + unpublished = catalog.manifests.begin("backup-bucket", "history.bin", 0, 123) + catalog.manifests.finish(unpublished, hashlib.sha256(b"").hexdigest(), 123) + key = S3CredentialStore(db, service[4]).create( + "versions-only", ["s3:ListBucketVersions"] + ) + finally: + db.close() + listed = signed(service, "GET", "/backup-bucket?versions", + access=key.access_key, secret=key.secret_key) + assert listed.status_code == 200, listed.text + root = ET.fromstring(listed.content) + entries = root.findall(NS + "Version") + assert len(entries) == 3 + assert unpublished not in listed.text + assert [node.findtext(NS + "Key") for node in entries] == [ + "history.bin", "history.bin", "other.bin", + ] + assert [node.findtext(NS + "IsLatest") for node in entries] == [ + "true", "false", "true", + ] + assert signed(service, "GET", "/backup-bucket?list-type=2", + access=key.access_key, secret=key.secret_key).status_code == 403 + assert signed(service, "GET", "/backup-bucket/history.bin", + access=key.access_key, secret=key.secret_key).status_code == 403 + + +def test_version_listing_cursor_prefix_and_invalid_marker(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + for key, data in ( + ("a.bin", b"v1"), ("a.bin", b"v2"), + ("a.bin", b"v3"), ("b.bin", b"last"), + ): + assert signed(service, "PUT", "/backup-bucket/" + key, data).status_code == 200 + got = [] + marker = "" + for _ in range(4): + page = signed(service, "GET", "/backup-bucket?versions&max-keys=1" + marker) + assert page.status_code == 200, page.text + root = ET.fromstring(page.content) + versions = root.findall(NS + "Version") + assert len(versions) == 1 + got.append(versions[0].findtext(NS + "VersionId")) + if root.findtext(NS + "IsTruncated") == "false": + break + marker = ( + "&key-marker=" + root.findtext(NS + "NextKeyMarker") + + "&version-id-marker=" + root.findtext(NS + "NextVersionIdMarker") + ) + assert len(set(got)) == 4 + filtered = signed(service, "GET", "/backup-bucket?versions&prefix=b.") + assert filtered.status_code == 200 + assert [node.findtext(NS + "Key") for node in ET.fromstring( + filtered.content + ).findall(NS + "Version")] == ["b.bin"] + for query in ( + "?versions&max-keys=0", "?versions&version-id-marker=missing", + "?versions&key-marker=a.bin&version-id-marker=missing", + "?versions&versions", "?versions&delimiter=/", + ): + denied = signed(service, "GET", "/backup-bucket" + query) + assert denied.status_code in (400, 501), denied.text + + +def test_version_listing_revocation_and_unlink_preserve_remote_bytes(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/archived.bin", b"retained").status_code == 200 + db = service[3]() + try: + reader = S3CredentialStore(db, service[4]).create( + "versions-revocable", ["s3:ListBucketVersions"] + ) + finally: + db.close() + before = service[2].uploads + assert signed(service, "DELETE", "/backup-bucket/archived.bin").status_code == 204 + listing = signed(service, "GET", "/backup-bucket?versions", + access=reader.access_key, secret=reader.secret_key) + assert listing.status_code == 200 + assert b"archived.bin" in listing.content + db = service[3]() + try: + assert S3CredentialStore(db, service[4]).revoke(reader.id) + finally: + db.close() + assert signed(service, "GET", "/backup-bucket?versions", + access=reader.access_key, secret=reader.secret_key).status_code == 403 + assert service[2].uploads == before diff --git a/tests/s3/test_s3_version_reads.py b/tests/s3/test_s3_version_reads.py new file mode 100644 index 00000000..6a2df390 --- /dev/null +++ b/tests/s3/test_s3_version_reads.py @@ -0,0 +1,58 @@ +"""Signed S3 VersionId historical GET/HEAD never exposes staging manifests.""" +from __future__ import annotations + +from tests.s3.test_s3_gateway import service, signed +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +def test_history_versionid_read_range_and_unlink_preserve_immutable_media(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + old = b"versioned-old-telegram" * 400 + new = b"versioned-new-telegram" * 31 + first = signed(service, "PUT", "/backup-bucket/history.bin", old) + assert first.status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + old_id = catalog.get("backup-bucket", "history.bin").item.id + finally: + db.close() + assert signed(service, "PUT", "/backup-bucket/history.bin", new).status_code == 200 + current = signed(service, "GET", "/backup-bucket/history.bin") + assert current.content == new + prior = signed(service, "GET", f"/backup-bucket/history.bin?versionId={old_id}") + assert prior.status_code == 200, prior.text + assert prior.content == old + assert prior.headers["x-amz-version-id"] == old_id + head = signed(service, "HEAD", f"/backup-bucket/history.bin?versionId={old_id}") + assert head.status_code == 200 + assert head.headers["content-length"] == str(len(old)) + partial = signed(service, "GET", f"/backup-bucket/history.bin?versionId={old_id}", + extra_headers={"Range": "bytes=31-79"}) + assert partial.status_code == 206 + assert partial.content == old[31:80] + assert signed(service, "DELETE", "/backup-bucket/history.bin").status_code == 204 + assert signed(service, "GET", "/backup-bucket/history.bin").status_code == 404 + assert signed(service, "GET", f"/backup-bucket/history.bin?versionId={old_id}").content == old + + +def test_never_serves_ready_unpublished_objects_as_s3_versions(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + db = service[3]() + try: + cat = BucketCatalog(ManifestStore(db)) + staged_id = cat.manifests.begin("backup-bucket", "hidden.bin", 0, 123) + import hashlib + cat.manifests.finish(staged_id, hashlib.sha256(b"").hexdigest(), 123) + finally: + db.close() + unavailable = signed(service, "GET", f"/backup-bucket/hidden.bin?versionId={staged_id}") + assert unavailable.status_code == 404 + assert b"NoSuchVersion" in unavailable.content + cannot_delete = signed(service, "DELETE", f"/backup-bucket/hidden.bin?versionId={staged_id}") + assert cannot_delete.status_code == 404 + assert b"NoSuchVersion" in cannot_delete.content + missing = signed(service, "GET", "/backup-bucket/hidden.bin?versionId=unknown") + assert missing.status_code == 404 diff --git a/tests/s3/test_s3_version_tagging.py b/tests/s3/test_s3_version_tagging.py new file mode 100644 index 00000000..bf4e4dd0 --- /dev/null +++ b/tests/s3/test_s3_version_tagging.py @@ -0,0 +1,98 @@ +"""Version-specific S3 tagging modifies only published version SQL metadata.""" +from __future__ import annotations + +import hashlib +import xml.etree.ElementTree as ET + +from tests.s3.test_s3_gateway import service, signed +from teledrive.s3.credentials import S3CredentialStore +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + +NS = "{http://s3.amazonaws.com/doc/2006-03-01/}" + + +def _tag_xml(key: str, value: str) -> bytes: + root = ET.Element("Tagging") + tags = ET.SubElement(root, "TagSet") + item = ET.SubElement(tags, "Tag") + ET.SubElement(item, "Key").text = key + ET.SubElement(item, "Value").text = value + return ET.tostring(root) + + +def _tags(response): + root = ET.fromstring(response.content) + return [(x.findtext(NS + "Key"), x.findtext(NS + "Value")) + for x in root.findall(NS + "TagSet/" + NS + "Tag")] + + +def test_version_specific_tag_mutations_are_independent_and_retain_telegram(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + path = "/backup-bucket/history.bin" + assert signed(service, "PUT", path, b"old").status_code == 200 + db = service[3]() + try: + old_id = BucketCatalog(ManifestStore(db)).get("backup-bucket", "history.bin").item.id + finally: + db.close() + assert signed(service, "PUT", path, b"new").status_code == 200 + before = service[2].uploads + current = signed(service, "PUT", path + "?tagging", _tag_xml("new", "yes")) + assert current.status_code in (200, 204) + changed = signed(service, "PUT", path + "?tagging&versionId=" + old_id, + _tag_xml("old", "only")) + assert changed.status_code == 200, changed.text + historical = signed(service, "GET", path + "?tagging&versionId=" + old_id) + assert historical.status_code == 200 and _tags(historical) == [("old", "only")] + latest = signed(service, "GET", path + "?tagging") + assert latest.status_code == 200 and _tags(latest) == [("new", "yes")] + deleted = signed(service, "DELETE", path + "?tagging&versionId=" + old_id) + assert deleted.status_code == 204 + assert _tags(signed(service, "GET", path + "?tagging&versionId=" + old_id)) == [] + assert _tags(signed(service, "GET", path + "?tagging")) == [("new", "yes")] + assert signed(service, "GET", path + "?versionId=" + old_id).content == b"old" + assert service[2].uploads == before + + +def test_unpublished_and_foreign_versions_cannot_be_tagged(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + staged = catalog.manifests.begin("backup-bucket", "never.bin", 0, 123) + catalog.manifests.finish(staged, hashlib.sha256(b"").hexdigest(), 123) + finally: + db.close() + for method in ("GET", "DELETE", "PUT"): + path = "/backup-bucket/never.bin?tagging&versionId=" + staged + result = signed(service, method, path, + _tag_xml("bad", "data") if method == "PUT" else b"") + assert result.status_code == 404 + assert b"NoSuchVersion" in result.content + assert signed(service, "GET", "/backup-bucket/never.bin?tagging&versionId=bad&versionId=other").status_code == 400 + + +def test_version_tagging_uses_separate_scope_and_revocation(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/file.bin", b"data").status_code == 200 + db = service[3]() + try: + ver = BucketCatalog(ManifestStore(db)).get("backup-bucket", "file.bin").item.id + reader = S3CredentialStore(db, service[4]).create( + "version-tag-reader", ["s3:GetObjectTagging"] + ) + finally: + db.close() + endpoint = "/backup-bucket/file.bin?tagging&versionId=" + ver + assert signed(service, "GET", endpoint, + access=reader.access_key, secret=reader.secret_key).status_code == 200 + assert signed(service, "PUT", endpoint, _tag_xml("no", "write"), + access=reader.access_key, secret=reader.secret_key).status_code == 403 + db = service[3]() + try: + assert S3CredentialStore(db, service[4]).revoke(reader.id) + finally: + db.close() + assert signed(service, "GET", endpoint, + access=reader.access_key, secret=reader.secret_key).status_code == 403 diff --git a/tests/s3/test_s3_vhost.py b/tests/s3/test_s3_vhost.py new file mode 100644 index 00000000..4343841f --- /dev/null +++ b/tests/s3/test_s3_vhost.py @@ -0,0 +1,108 @@ +"""SigV4 Host-bound S3 virtual-host routing is opt-in and refuses spoofing.""" +from __future__ import annotations + +import hashlib + +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials + +from tests.s3.test_s3_gateway import service, signed + + +def _vhost(service, method, path, payload=b"", *, host="backup-bucket.s3.example.test:9000", + headers=None): + issued = service[1] + request_headers = { + "host": host, + "x-amz-content-sha256": hashlib.sha256(payload).hexdigest(), + } + request_headers.update(headers or {}) + request = AWSRequest( + method=method, url="http://" + host + path, + data=payload, headers=request_headers, + ) + request.context["payload_signing_enabled"] = True + S3SigV4Auth( + Credentials(issued.access_key, issued.secret_key), "s3", "us-east-1" + ).add_auth(request) + return service[0].request( + method, path, content=payload, headers=dict(request.headers), + ) + + +def test_virtual_host_bucket_roundtrip_and_signed_host_is_required(service, monkeypatch): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + monkeypatch.setenv("TELEDRIVE_S3_VHOST_SUFFIX", "s3.example.test") + body = b"virtual-host-telegram-contents" + put = _vhost(service, "PUT", "/nested/file.bin", body) + assert put.status_code == 200, put.text + assert _vhost(service, "GET", "/nested/file.bin").content == body + head = _vhost(service, "HEAD", "/nested/file.bin") + assert head.status_code == 200 + assert head.headers["content-length"] == str(len(body)) + listing = _vhost(service, "GET", "/?list-type=2&prefix=nested") + assert listing.status_code == 200 + assert b"nested/file.bin" in listing.content + + creds = service[1] + original = AWSRequest( + method="GET", url="http://backup-bucket.s3.example.test:9000/nested/file.bin", + headers={"host": "backup-bucket.s3.example.test:9000", + "x-amz-content-sha256": hashlib.sha256(b"").hexdigest()}, + ) + original.context["payload_signing_enabled"] = True + S3SigV4Auth(Credentials(creds.access_key, creds.secret_key), + "s3", "us-east-1").add_auth(original) + tampered = dict(original.headers) + tampered["host"] = "another-bucket.s3.example.test:9000" + rejected = service[0].get("/nested/file.bin", headers=tampered) + assert rejected.status_code == 403 + assert b"SignatureDoesNotMatch" in rejected.content + + +def test_virtual_host_requires_explicit_suffix_and_no_proxy_header_trust(service, monkeypatch): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + monkeypatch.delenv("TELEDRIVE_S3_VHOST_SUFFIX", raising=False) + response = _vhost(service, "GET", "/object.bin") + assert response.status_code == 404 + monkeypatch.setenv("TELEDRIVE_S3_VHOST_SUFFIX", "s3.example.test") + forwarded = signed( + service, "GET", "/backup-bucket?list-type=2", + extra_headers={"X-Forwarded-Host": "another-bucket.s3.example.test:9000"}, + ) + assert forwarded.status_code == 200 + assert _vhost(service, "GET", "/", host="bad_name.s3.example.test:9000").status_code == 400 + + +def test_virtual_host_cors_uses_host_bucket_not_object_key_prefix(service, monkeypatch): + from tests.s3.test_s3_cors import config_xml + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/permissive-bucket").status_code == 200 + assert signed(service, "PUT", "/permissive-bucket?cors", config_xml( + origins=["https://app.example.com"], methods=["GET"], + )).status_code == 200 + monkeypatch.setenv("TELEDRIVE_S3_VHOST_SUFFIX", "s3.example.test") + client = service[0] + # A crafted object key starts with another bucket's name; its permissive + # CORS rule must not be applied to the private virtual-host bucket. + misleading = client.options("/permissive-bucket/private.bin", headers={ + "Host": "backup-bucket.s3.example.test:9000", + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + }) + assert misleading.status_code == 403 + legitimate = client.options("/any-object.bin", headers={ + "Host": "permissive-bucket.s3.example.test:9000", + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + }) + assert legitimate.status_code == 200 + assert legitimate.headers["access-control-allow-origin"] == "https://app.example.com" + invalid = client.options("/", headers={ + "Host": "invalid_name.s3.example.test:9000", + "Origin": "https://app.example.com", + "Access-Control-Request-Method": "GET", + }) + assert invalid.status_code == 400 diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/storage/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/storage/test_gc_reference_planner.py b/tests/storage/test_gc_reference_planner.py new file mode 100644 index 00000000..cecb4267 --- /dev/null +++ b/tests/storage/test_gc_reference_planner.py @@ -0,0 +1,260 @@ +"""GC never authorizes deletion from incomplete SQL, stale writers or version data.""" +from __future__ import annotations + +import time +from hashlib import sha256 + +import pytest +from sqlalchemy import insert +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog, heads, published_versions, removed_versions +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore, object_aliases + + +@pytest.fixture +def cat(tmp_path): + db = connect_database(str(tmp_path / "gc.db")) + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("gc-audit-test", int(time.time())) + try: + yield catalog + finally: + db.close() + + +def _cat(tmp_path): + db = connect_database(str(tmp_path / "gc.db")) + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("gc-audit-test", int(time.time())) + return catalog + + +def old_failure(cat, key="failed"): + old = int(time.time()) - 9 * 86400 + ident = cat.manifests.begin("gc-audit-test", key, 0, old) + cat.manifests.fail(ident, "simulated interruption", old) + return ident + + +def one(cat, ident): + data = StorageGCInventory(cat.manifests).preview_failed_retention( + now=int(time.time()) + ) + return next(x for x in data if x["object_id"] == ident) + + +def test_even_unreferenced_manifest_has_no_deletion_permission(cat): + row = one(cat, old_failure(cat)) + assert row["reference_blocked"] is False + assert row["reference_blockers"] == [] + assert len(row["sql_reference_fingerprint"]) == 64 + for name in ("approved_for_deletion", "deletion_permitted", + "writer_barrier_verified", "independent_payload_backup_verified", + "restore_drill_verified", "two_person_deletion_approval_verified"): + assert row[name] is False + + +@pytest.mark.parametrize("source", [ + "head", "published_version", "removed_version", "writer_fence", + "copy_alias", "multipart_part", "mtproto_send_intent_or_receipt", +]) +def test_reference_classes_always_block_gc(cat, source): + ident = old_failure(cat, source) + now = int(time.time()) + db = cat.db + if source == "head": + db.connection.execute(insert(heads).values( + bucket="gc-audit-test", key_hash=cat._keyhash(source), + object_id=ident, generation=1, updated_at=now, + )) + elif source == "published_version": + db.connection.execute(insert(published_versions).values( + object_id=ident, bucket="gc-audit-test", + key_hash=cat._keyhash(source), committed_ns=now * 10**9, + )) + elif source == "removed_version": + db.connection.execute(insert(removed_versions).values( + object_id=ident, removed_ns=now * 10**9, + )) + elif source == "writer_fence": + cat.manifests.start_writer(ident, "unreconciled-writer", now) + elif source == "multipart_part": + from teledrive.s3.multipart import MultipartStore, parts + upload = MultipartStore(cat).initiate( + "gc-audit-test", "candidate", "ci-owner", now=now, + ) + db.connection.execute(insert(parts).values( + upload_id=upload, number=1, object_id=ident, + etag="a" * 32, size=0, updated_at=now, + )) + elif source == "copy_alias": + child = cat.manifests.begin("gc-audit-test", "alias", 0, now) + db.connection.execute(insert(object_aliases).values( + object_id=child, source_object_id=ident, + )) + else: + from teledrive.storage.user_mode import UserSendJournal + digest = sha256(b"payload").hexdigest() + journal = UserSendJournal(db) + journal.start("test", -100123, ident, 0, digest, 7) + db.commit() + result = one(cat, ident) + assert result["reference_blocked"] is True + assert source in result["reference_blockers"] + assert result["deletion_permitted"] is False + + +def test_transitive_copy_alias_updates_fingerprint(cat): + ident = old_failure(cat) + first = one(cat, ident) + now = int(time.time()) + alias = cat.manifests.begin("gc-audit-test", "child", 0, now) + grandchild = cat.manifests.begin("gc-audit-test", "grandchild", 0, now) + cat.db.connection.execute(insert(object_aliases), [ + {"object_id": alias, "source_object_id": ident}, + {"object_id": grandchild, "source_object_id": alias}, + ]) + cat.db.commit() + later = one(cat, ident) + assert later["copy_alias_descendants"] == 2 + assert later["sql_reference_fingerprint"] != first["sql_reference_fingerprint"] + + +def test_corrupt_alias_cycle_fails_closed(cat): + ident = old_failure(cat) + other = cat.manifests.begin("gc-audit-test", "other", 0, int(time.time())) + cat.db.connection.execute(insert(object_aliases), [ + {"object_id": ident, "source_object_id": other}, + {"object_id": other, "source_object_id": ident}, + ]) + cat.db.commit() + with pytest.raises(RuntimeError, match="alias graph is corrupt"): + one(cat, ident) + + +def test_incomplete_removed_version_schema_is_unsafe(cat): + ident = old_failure(cat) + cat.db.connection.exec_driver_sql("DROP TABLE storage_removed_versions") + cat.db.commit() + with pytest.raises(RuntimeError, match="Incomplete GC reference"): + one(cat, ident) + + +def _active_job(cat, *, state, object_id=None, expected_head_id=None, + bucket="gc-audit-test", key="failed"): + """Insert one durable upload-job row in an active state.""" + from teledrive.storage.jobs import StorageUploadJobs, jobs + StorageUploadJobs(cat) # create_all for the jobs table + now = int(time.time()) + cat.db.connection.execute(insert(jobs).values( + id=f"job-{state}-{object_id or expected_head_id or key}", + bucket=bucket, object_key=key, source_path="/uploads/source.bin", + source_size=7, source_device=1, source_inode=2, source_mtime_ns=3, + expected_generation=0, expected_head_id=expected_head_id, + state=state, object_id=object_id, attempts=0, + lease_token="lease", lease_until=now + 60, + created_at=now, updated_at=now, last_error=None, + )) + cat.db.commit() + + +def test_active_durable_job_with_object_id_blocks_gc(cat): + ident = old_failure(cat) + _active_job(cat, state="running", object_id=ident) + row = one(cat, ident) + assert row["reference_blocked"] is True + assert "durable_job" in row["reference_blockers"] + assert row["deletion_permitted"] is False + + +def test_active_durable_job_for_same_bucket_key_blocks_gc_without_object_id(cat): + ident = old_failure(cat) + _active_job(cat, state="queued", expected_head_id=ident) + row = one(cat, ident) + assert row["reference_blocked"] is True + assert "durable_job" in row["reference_blockers"] + + +def test_finished_job_does_not_mask_missing_head_reference(cat): + ident = old_failure(cat) + _active_job(cat, state="done", object_id=ident) + row = one(cat, ident) + assert row["reference_blocked"] is False + assert "durable_job" not in row["reference_blockers"] + + +def test_active_job_changes_gc_fingerprint(cat): + ident = old_failure(cat) + first = one(cat, ident) + _active_job(cat, state="staged", object_id=ident) + later = one(cat, ident) + assert later["sql_reference_fingerprint"] != first["sql_reference_fingerprint"] + + +def test_inventory_counts_active_upload_jobs(cat): + _active_job(cat, state="queued", key="a") + _active_job(cat, state="running", key="b") + _active_job(cat, state="done", key="c", object_id="done-object") + from teledrive.storage.gc import StorageGCInventory + inventory = StorageGCInventory(cat.manifests).inspect() + assert inventory.active_upload_jobs == 2 + assert inventory.eligible_for_deletion == 0 + + +def test_gc_plan_is_dry_run_with_owner_gates(tmp_path): + catalog = _cat(tmp_path) + ident = old_failure(catalog) + plan = StorageGCInventory(catalog.manifests).plan(ident) + assert plan["dry_run"] is True + assert plan["deletion_permitted"] is False + assert plan["approved_for_deletion"] is False + assert plan["physical_deletion_enabled"] is False + assert plan["remote_history_scan_required"] is True + assert plan["rollback_strategy"] == "new_media_upload_and_fresh_sql_identities" + assert set(plan["owner_gates"]) == { + "independent_payload_backup_verified", "restore_drill_verified", + "writer_barrier_verified", "two_person_deletion_approval_verified", + } + assert len(plan["sql_reference_fingerprint"]) == 64 + # Re-planning the same candidate yields the same fingerprint. + second = StorageGCInventory(catalog.manifests).plan(ident) + assert second["sql_reference_fingerprint"] == plan["sql_reference_fingerprint"] + StorageGCInventory(catalog.manifests).verify_plan(plan) + + +def test_gc_plan_rejects_stale_after_new_reference(tmp_path): + catalog = _cat(tmp_path) + ident = old_failure(catalog) + plan = StorageGCInventory(catalog.manifests).plan(ident) + # A new writer fence appears after planning: the plan is now stale. + catalog.manifests.start_writer(ident, "late-writer", int(time.time())) + catalog.db.commit() + with pytest.raises(RuntimeError, match="stale"): + StorageGCInventory(catalog.manifests).verify_plan(plan) + + +def test_gc_plan_rejects_tampered_deletion_permission(tmp_path): + catalog = _cat(tmp_path) + ident = old_failure(catalog) + plan = StorageGCInventory(catalog.manifests).plan(ident) + plan["deletion_permitted"] = True + with pytest.raises(RuntimeError, match="must not authorize deletion"): + StorageGCInventory(catalog.manifests).verify_plan(plan) + + +def test_gc_plan_rejects_unknown_or_unaged_candidate(tmp_path): + catalog = _cat(tmp_path) + ident = old_failure(catalog) + # Young failure: refuses to plan. + young = catalog.manifests.begin("gc-audit-test", "young", 0, int(time.time())) + catalog.manifests.fail(young, "recent", int(time.time())) + with pytest.raises(ValueError, match="aged|safety"): + StorageGCInventory(catalog.manifests).plan(young) + # Unknown object. + with pytest.raises(FileNotFoundError): + StorageGCInventory(catalog.manifests).plan("does-not-exist") + # Non-failed object. + ready = catalog.manifests.begin("gc-audit-test", "ready", 0, int(time.time())) + with pytest.raises(ValueError, match="failed"): + StorageGCInventory(catalog.manifests).plan(ready) diff --git a/tests/storage/test_legacy_bridge.py b/tests/storage/test_legacy_bridge.py new file mode 100644 index 00000000..01ec578a --- /dev/null +++ b/tests/storage/test_legacy_bridge.py @@ -0,0 +1,220 @@ +"""Legacy folder -> durable object queue is explicit, idempotent and safe.""" +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs, jobs, UploadJobError +from teledrive.storage.legacy_bridge import enqueue_legacy_folder, run_legacy_cutover_cycle +from teledrive.storage.manifest import ManifestStore +from tests.storage.test_storage_engine import MemoryTelegram + + +@pytest.fixture +def bridge(tmp_path): + root = tmp_path / "source" + root.mkdir() + db_path = tmp_path / "legacy-bridge.db" + conn = connect_database(str(db_path)) + cat = BucketCatalog(ManifestStore(conn)) + cat.create_bucket("bridge-ci", int(time.time())) + config = SimpleNamespace( + source_dir=str(root), db_path=str(db_path), + allowed_extensions=[".txt"], max_file_size_mb=1, + ) + try: + yield root, db_path, StorageUploadJobs(cat), config + finally: + conn.close() + + +def test_legacy_folder_bridge_idempotent_across_restart_and_preserves_paths(bridge): + root, db_path, queue, cfg = bridge + nested = root / "documents" + nested.mkdir() + (root / "one.txt").write_bytes(b"one") + (nested / "two.txt").write_bytes(b"two") + (root / "not-included.dat").write_bytes(b"exclude") + (root / "external.txt").symlink_to(root / "one.txt") + + first = enqueue_legacy_folder(cfg, queue, bucket="bridge-ci", prefix="backup") + assert first.queued == 2 + assert first.already_queued == 0 + assert first.excluded == 1 + assert len(first.job_ids) == 2 + records = queue.list_jobs() + assert {r["object_key"] for r in records} == { + "backup/one.txt", "backup/documents/two.txt", + } + assert all(r["state"] == "queued" for r in records) + # No remote bytes are uploaded by a folder scan, even after restart. + conn = connect_database(str(db_path)) + try: + recreated = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + again = enqueue_legacy_folder(cfg, recreated, bucket="bridge-ci", prefix="backup") + assert again.queued == 0 and again.already_queued == 2 + assert {r["id"] for r in recreated.list_jobs()} == set(first.job_ids) + source = root / "one.txt" + source.write_bytes(b"modified") + changed = enqueue_legacy_folder(cfg, recreated, bucket="bridge-ci", prefix="backup") + assert changed.queued == 1 and changed.already_queued == 1 + assert len(recreated.list_jobs()) == 3 + finally: + conn.close() + + +def test_legacy_bridge_rejects_traversal_and_unknown_destination(bridge): + root, _, queue, cfg = bridge + (root / "normal.txt").write_bytes(b"no-send") + for prefix in ("/absolute", "../escape", "foo/../bar", "foo//bar", + "bad\\windows", "bad/"): + with pytest.raises(ValueError): + enqueue_legacy_folder(cfg, queue, bucket="bridge-ci", prefix=prefix) + for limit in (0, 1001): + with pytest.raises(ValueError): + enqueue_legacy_folder(cfg, queue, bucket="bridge-ci", max_files=limit) + with pytest.raises(FileNotFoundError): + enqueue_legacy_folder(cfg, queue, bucket="does-not-exist") + assert queue.list_jobs() == [] + + +def test_legacy_bridge_only_queues_with_explicit_worker(bridge): + root, _, queue, cfg = bridge + (root / "chunked.txt").write_bytes(bytes(range(30))) + report = enqueue_legacy_folder(cfg, queue, bucket="bridge-ci", max_files=1) + assert report.queued == 1 + remote = MemoryTelegram() + assert remote.uploads == 0 + completed = asyncio.run(queue.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=7, + )) + assert completed["state"] == "done" + assert remote.uploads == 5 + assert queue.catalog.get("bridge-ci", "chunked.txt").item.size == 30 + again = enqueue_legacy_folder(cfg, queue, bucket="bridge-ci") + assert again.queued == 0 and again.already_queued == 1 + assert queue.list_jobs()[0]["state"] == "done" + + +def _cutover(bridge, lease, *, bucket="bridge-ci", remote=None, + allowed_root=None): + root, _, queue, cfg = bridge + return asyncio.run(run_legacy_cutover_cycle( + cfg, queue, remote or MemoryTelegram(), lease_db=lease, bucket=bucket, + allowed_root=allowed_root or root, scratch_directory=root, + lease_name="uploader-global", lease_ttl=60, chunk_size=7, + )) + + +def test_cutover_imports_once_and_uploads_exact_source_on_restart(bridge): + root, path, queue, _ = bridge + payload = b"verified-cutover-data" * 9 + (root / "cutover.txt").write_bytes(payload) + remote = MemoryTelegram() + lease = connect_database(str(path)) + try: + first, result = _cutover(bridge, lease, remote=remote) + assert first.queued == 1 and result["state"] == "done" + before = remote.uploads + second, again = _cutover(bridge, lease, remote=remote) + assert second.queued == 0 and second.already_queued == 1 + assert again is None and remote.uploads == before + assert len(queue.list_jobs()) == 1 + assert queue.catalog.get("bridge-ci", "cutover.txt").item.size == len(payload) + assert lease.acquire_lease("uploader-global", "old-uploader", int(time.time()), 60) + lease.release_lease("uploader-global", "old-uploader") + finally: + lease.close() + + +def test_cutover_refuses_concurrent_legacy_uploader_before_import(bridge): + root, path, queue, _ = bridge + (root / "blocked.txt").write_bytes(b"blocked") + legacy = connect_database(str(path)) + lease = connect_database(str(path)) + try: + assert legacy.acquire_lease("uploader-global", "legacy-sender", int(time.time()), 60) + with pytest.raises(UploadJobError, match="holds"): + _cutover(bridge, lease) + assert queue.list_jobs() == [] + legacy.release_lease("uploader-global", "legacy-sender") + imported, result = _cutover(bridge, lease) + assert imported.queued == 1 and result["state"] == "done" + finally: + lease.close() + legacy.close() + + +def test_cutover_checks_root_database_and_bucket_before_telegram(bridge, tmp_path): + root, path, queue, _ = bridge + (root / "private.txt").write_bytes(b"never-send-on-error") + lease = connect_database(str(path)) + unrelated = connect_database(str(tmp_path / "unrelated.db")) + outside = tmp_path / "outside" + outside.mkdir() + try: + with pytest.raises(UploadJobError, match="approved job root"): + _cutover(bridge, lease, allowed_root=outside) + with pytest.raises(UploadJobError, match="same database"): + _cutover(bridge, unrelated) + cfg = bridge[3] + cfg.db_path = str(tmp_path / "other-configured.db") + with pytest.raises(UploadJobError, match="same database"): + _cutover(bridge, lease) + cfg.db_path = str(path) + with pytest.raises(FileNotFoundError): + _cutover(bridge, lease, bucket="not-a-bucket") + assert queue.list_jobs() == [] + assert lease.acquire_lease("uploader-global", "after-exception", int(time.time()), 60) + lease.release_lease("uploader-global", "after-exception") + finally: + lease.close() + unrelated.close() + + +def test_cutover_holds_legacy_lease_during_telegram_send(bridge): + root, path, queue, _ = bridge + (root / "hold.txt").write_bytes(b"lease-fenced-send") + checker = connect_database(str(path)) + lease = connect_database(str(path)) + class FencedTelegram(MemoryTelegram): + async def upload(self, path, *, object_id, ordinal, checksum): + assert not checker.acquire_lease( + "uploader-global", "racing-uploader", int(time.time()), 60 + ) + return await super().upload( + path, object_id=object_id, ordinal=ordinal, checksum=checksum, + ) + try: + _, result = _cutover(bridge, lease, remote=FencedTelegram()) + assert result["state"] == "done" + assert queue.list_jobs()[0]["attempts"] == 1 + finally: + lease.close() + checker.close() + + +def test_cutover_can_publish_after_delete_marker_with_head_cas(bridge): + root, path, queue, _ = bridge + (root / "after.txt").write_bytes(b"version-after-delete") + marker = queue.catalog.delete_marker("bridge-ci", "after.txt") + lease = connect_database(str(path)) + try: + _, done = _cutover(bridge, lease) + assert done["state"] == "done" + assert queue.catalog.get("bridge-ci", "after.txt").item.id != marker + versions, _ = queue.catalog.list_bucket_versions("bridge-ci") + assert any(entry["version_id"] == marker and entry["is_delete_marker"] + for entry in versions) + finally: + lease.close() diff --git a/tests/storage/test_legacy_cutover_acceptance.py b/tests/storage/test_legacy_cutover_acceptance.py new file mode 100644 index 00000000..7cbbfb15 --- /dev/null +++ b/tests/storage/test_legacy_cutover_acceptance.py @@ -0,0 +1,297 @@ +"""Cutover production rehearsal: every eligible source must match published Telegram.""" +from __future__ import annotations + +import asyncio +import hashlib +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.cutover_acceptance import verify_legacy_cutover +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.legacy_bridge import enqueue_legacy_folder +from teledrive.storage.manifest import ManifestStore + + +class DiskTelegram: + def __init__(self): + self.messages = {} + self.uploads = 0 + self.deletions = 0 + + async def upload(self, path, *, object_id, ordinal, checksum): + self.uploads += 1 + self.messages[self.uploads] = path.read_bytes() + return "legacy-test", -100459, self.uploads + + async def download(self, ref, destination): + destination.write_bytes(self.messages[ref.message_id]) + + +def scenario(tmp_path, sizes=(0, 65537, 1048619)): + src = tmp_path / "sources" + src.mkdir() + data = {} + for i, n in enumerate(sizes): + content = hashlib.shake_256(f"legacy:{i}".encode()).digest(n) + name = f"document-{i}.bin" + (src / name).write_bytes(content) + data[name] = content + (src / "excluded.txt").write_bytes(b"not selected") + db = connect_database(str(tmp_path / "cutover.db")) + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("cutover-ci", int(time.time())) + queue = StorageUploadJobs(catalog) + cfg = SimpleNamespace(source_dir=str(src), + allowed_extensions=[".bin"], max_file_size_mb=0) + report = enqueue_legacy_folder( + cfg, queue, bucket="cutover-ci", prefix="archived", max_files=100, + ) + assert report.queued == len(sizes) and report.excluded == 1 + return db, queue, cfg, src, data, DiskTelegram() + + +async def inspect(cfg, queue, remote, src, **kwargs): + return await verify_legacy_cutover( + cfg, queue, remote, bucket="cutover-ci", prefix="archived", + approved_root=src, scratch_directory=src, **kwargs, + ) + + +def test_complete_cutover_acceptance_only_after_all_real_journals_verified(tmp_path): + db, queue, cfg, src, data, remote = scenario(tmp_path) + try: + before = asyncio.run(inspect(cfg, queue, remote, src)) + assert not before["verified"] and before["not_yet_published"] == len(data) + uploads = remote.uploads + assert uploads == 0 + first = asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, chunk_size=32768, + )) + assert first["state"] == "done" + partial = asyncio.run(inspect(cfg, queue, remote, src)) + assert not partial["verified"] + assert partial["verified_files"] == 1 + assert partial["not_yet_published"] == len(data) - 1 + for _ in range(len(data) - 1): + done = asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, chunk_size=32768, + )) + assert done["state"] == "done" + all_done = asyncio.run(inspect(cfg, queue, remote, src)) + assert all_done == { + "verified": True, "eligible_files": len(data), + "verified_files": len(data), "not_yet_published": 0, + "inconsistent": 0, "excluded": 1, "truncated": False, + "telegram_uploads": 0, "telegram_deletions": 0, + "source_modifications": 0, + } + assert queue.list_jobs(limit=20) + for name, body in data.items(): + assert (src / name).read_bytes() == body + uploads_before = remote.uploads + assert asyncio.run(inspect(cfg, queue, remote, src))["verified"] + assert remote.uploads == uploads_before and remote.deletions == 0 + + # Operators cannot reuse an earlier clean report after a source edit. + (src / "document-1.bin").write_bytes(b"source-replaced") + changed = asyncio.run(inspect(cfg, queue, remote, src)) + assert not changed["verified"] and changed["not_yet_published"] >= 1 + finally: + db.close() + + +@pytest.mark.parametrize("kind", ["remote-corruption", "max-files", + "max-bytes", "outside-root"]) +def test_cutover_rejects_incomplete_or_unsafe_state(tmp_path, kind): + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(1024, 8193)) + try: + for _ in range(2): + assert asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, chunk_size=512, + ))["state"] == "done" + assert asyncio.run(inspect(cfg, queue, remote, src))["verified"] + if kind == "remote-corruption": + remote.messages[1] = b"corrupt-telegram" + with pytest.raises(RuntimeError): + asyncio.run(inspect(cfg, queue, remote, src)) + elif kind == "max-files": + report = asyncio.run(inspect(cfg, queue, remote, src, max_files=1)) + assert not report["verified"] and report["truncated"] + elif kind == "max-bytes": + report = asyncio.run(inspect(cfg, queue, remote, src, max_bytes=512)) + assert not report["verified"] and report["inconsistent"] >= 1 + else: + other = tmp_path / "outside" + other.mkdir() + with pytest.raises(ValueError): + asyncio.run(inspect(cfg, queue, remote, other)) + assert remote.deletions == 0 + finally: + db.close() + +def test_cutover_detects_new_source_file_during_remote_telegram_readback(tmp_path): + """A newly arrived file after scan start invalidates old HEAD acceptance.""" + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(65537,)) + try: + completed = asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, + chunk_size=32768, + )) + assert completed["state"] == "done" + uploads_before = remote.uploads + real_download = remote.download + inserted = False + + async def mutate_after_scan(ref, destination): + nonlocal inserted + if not inserted: + inserted = True + (src / "new-arrival.bin").write_bytes(b"arrived during verification") + await real_download(ref, destination) + + remote.download = mutate_after_scan + report = asyncio.run(inspect(cfg, queue, remote, src)) + assert inserted + assert not report["verified"] + assert report["inconsistent"] >= 1 + assert report["telegram_uploads"] == report["telegram_deletions"] == 0 + assert remote.uploads == uploads_before and remote.deletions == 0 + finally: + db.close() + + +def test_legacy_cutover_large_directory_cap_never_reads_telegram(tmp_path): + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(1,)) + try: + for n in range(120): + (src / f"late-{n:04d}.bin").write_bytes(bytes([n % 256])) + result = asyncio.run(inspect( + cfg, queue, remote, src, max_files=100, + )) + assert not result["verified"] and result["truncated"] + assert result["verified_files"] == 0 + assert result["telegram_uploads"] == result["telegram_deletions"] == 0 + assert remote.uploads == 0 and remote.deletions == 0 + finally: + db.close() + + +def test_cutover_detects_late_same_size_edit_even_with_original_mtime(tmp_path): + """A second file's Telegram read must not mask a late edit to the first.""" + import os + + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(65537, 65537)) + try: + for _ in range(2): + assert asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, + chunk_size=32768, + ))["state"] == "done" + assert asyncio.run(inspect(cfg, queue, remote, src))["verified"] + original = src / "document-0.bin" + stamp = original.stat() + source_before = original.read_bytes() + uploads_before = remote.uploads + real_download = remote.download + changed = False + + async def edit_earlier_source(ref, destination): + nonlocal changed + # The first file has three 32KiB chunks. Only mutate it when + # the second file starts its independent Telegram readback. + if not changed and ref.message_id >= 4: + changed = True + original.write_bytes(b"Q" * len(source_before)) + os.utime( + original, ns=(stamp.st_atime_ns, stamp.st_mtime_ns), + ) + await real_download(ref, destination) + + remote.download = edit_earlier_source + report = asyncio.run(inspect(cfg, queue, remote, src)) + assert changed + assert not report["verified"] + assert report["inconsistent"] >= 1 + assert original.stat().st_size == stamp.st_size + assert original.stat().st_mtime_ns == stamp.st_mtime_ns + assert remote.uploads == uploads_before and remote.deletions == 0 + finally: + db.close() + + +def test_cutover_never_follows_last_second_symlink_outside_root( + tmp_path, monkeypatch, +): + """A file swapped after identity check must never be opened via symlink.""" + import teledrive.storage.cutover_acceptance as acceptance + + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(65537,)) + outside = tmp_path / "outside-secret" + outside.write_bytes(b"outside root: forbidden") + try: + assert asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, + chunk_size=32768, + ))["state"] == "done" + original = acceptance._regular_source + path = src / "document-0.bin" + triggered = False + + def swap_before_open(source): + nonlocal triggered + if not triggered: + triggered = True + path.unlink() + path.symlink_to(outside) + return original(source) + + monkeypatch.setattr(acceptance, "_regular_source", swap_before_open) + before_uploads = remote.uploads + report = asyncio.run(inspect(cfg, queue, remote, src)) + assert triggered + assert not report["verified"] + assert report["inconsistent"] >= 1 + assert remote.uploads == before_uploads + assert remote.deletions == 0 + finally: + db.close() + + +def test_cutover_rechecks_published_head_after_telegram_readback(tmp_path): + """A concurrent S3 DELETE cannot leave the cutover report green.""" + db, queue, cfg, src, _, remote = scenario(tmp_path, sizes=(65537,)) + try: + assert asyncio.run(queue.run_next( + remote, allowed_root=src, scratch_directory=src, + chunk_size=32768, + ))["state"] == "done" + assert asyncio.run(inspect(cfg, queue, remote, src))["verified"] + original_download = remote.download + modified = False + before_uploads = remote.uploads + + async def replace_published_head(ref, destination): + nonlocal modified + await original_download(ref, destination) + if not modified: + modified = True + # Test-only adversarial concurrent S3 writer. It changes only + # the SQL HEAD, not the source or previously uploaded chunks. + queue.catalog.remove_current( + "cutover-ci", "archived/document-0.bin", + ) + + remote.download = replace_published_head + report = asyncio.run(inspect(cfg, queue, remote, src)) + assert modified + assert not report["verified"] + assert report["verified_files"] == 1 + assert report["inconsistent"] >= 1 + assert remote.uploads == before_uploads and remote.deletions == 0 + finally: + db.close() diff --git a/tests/storage/test_legacy_import_admin.py b/tests/storage/test_legacy_import_admin.py new file mode 100644 index 00000000..7e41fd2f --- /dev/null +++ b/tests/storage/test_legacy_import_admin.py @@ -0,0 +1,105 @@ +"""Authenticated legacy-folder importer must dry-run before enqueuing.""" +from __future__ import annotations + +import time +import pytest +from pathlib import Path +from types import SimpleNamespace + +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore +import web.app as webapp + + +AUTH = ("testadmin", "secure-auth-password") +CSRF = {"x-teledrive-csrf": "1"} +URL = "/api/v1/admin/storage/jobs/import-legacy" + + +def test_admin_preview_and_execute_legacy_import_without_telegram(admin, tmp_path, monkeypatch): + client, db_path, _ = admin + root = tmp_path / "uploads" + root.mkdir() + folder = root / "legacy" + folder.mkdir() + (folder / "first.txt").write_bytes(b"first") + (folder / "skip.dat").write_bytes(b"excluded") + monkeypatch.setattr(webapp, "JOB_SOURCE_ROOT", str(root)) + cfg = SimpleNamespace(source_dir=str(folder), allowed_extensions=[".txt"], + max_file_size_mb=1) + monkeypatch.setattr(webapp, "effective_config", lambda: cfg) + db = connect_database(db_path) + try: + BucketCatalog(ManifestStore(db)).create_bucket("import-ci", int(time.time())) + finally: + db.close() + body = {"bucket": "import-ci", "prefix": "legacy", "max_files": 10} + assert client.post(URL, json=body).status_code in (401, 403) + assert client.post(URL, auth=AUTH, json=body).status_code == 403 + + preview = client.post(URL, auth=AUTH, headers=CSRF, json=body) + assert preview.status_code == 200, preview.text + assert preview.json()["dry_run"] is True + assert preview.json()["would_queue"] == 1 + assert preview.json()["queued"] == 0 + assert preview.json()["uploads_started"] == 0 + assert client.get("/api/v1/admin/storage/jobs", auth=AUTH).json()["jobs"] == [] + + executed = client.post(URL, auth=AUTH, headers=CSRF, + json={**body, "execute": True}) + assert executed.status_code == 200, executed.text + assert executed.json()["queued"] == executed.json()["would_queue"] == 1 + assert executed.json()["excluded"] == 1 + jobs = client.get("/api/v1/admin/storage/jobs", auth=AUTH).json()["jobs"] + assert len(jobs) == 1 + assert jobs[0]["object_key"] == "legacy/first.txt" + assert jobs[0]["state"] == "queued" + + repeated = client.post(URL, auth=AUTH, headers=CSRF, + json={**body, "execute": True}) + assert repeated.status_code == 200, repeated.text + assert repeated.json()["queued"] == 0 + assert repeated.json()["already_queued"] == 1 + assert len(client.get("/api/v1/admin/storage/jobs", auth=AUTH).json()["jobs"]) == 1 + + +def test_admin_legacy_import_refuses_external_source_and_active_legacy_uploader( + admin, tmp_path, monkeypatch +): + client, db_path, _ = admin + approved = tmp_path / "approved" + approved.mkdir() + external = tmp_path / "outside" + external.mkdir() + (external / "a.txt").write_bytes(b"no-read") + monkeypatch.setattr(webapp, "JOB_SOURCE_ROOT", str(approved)) + monkeypatch.setattr(webapp, "effective_config", lambda: SimpleNamespace( + source_dir=str(external), allowed_extensions=[], max_file_size_mb=0, + )) + db = connect_database(db_path) + try: + BucketCatalog(ManifestStore(db)).create_bucket("import-ci", int(time.time())) + finally: + db.close() + body = {"bucket": "import-ci", "execute": True} + refused = client.post(URL, auth=AUTH, headers=CSRF, json=body) + assert refused.status_code == 403 + assert len(client.get("/api/v1/admin/storage/jobs", auth=AUTH).json()["jobs"]) == 0 + + (approved / "a.txt").write_bytes(b"approved") + monkeypatch.setattr(webapp, "effective_config", lambda: SimpleNamespace( + source_dir=str(approved), allowed_extensions=[], max_file_size_mb=0, + )) + lease = connect_database(db_path) + try: + from teledrive.uploader import LEASE_NAME + assert lease.acquire_lease(LEASE_NAME, "another-live-uploader", int(time.time()), 120) + refused = client.post(URL, auth=AUTH, headers=CSRF, json=body) + assert refused.status_code == 409 + assert client.get("/api/v1/admin/storage/jobs", auth=AUTH).json()["jobs"] == [] + finally: + lease.release_lease(LEASE_NAME, "another-live-uploader") + lease.close() diff --git a/tests/storage/test_metadata_backup.py b/tests/storage/test_metadata_backup.py new file mode 100644 index 00000000..58300fd2 --- /dev/null +++ b/tests/storage/test_metadata_backup.py @@ -0,0 +1,111 @@ +"""Consistent SQLite backup protection, restart and corruption regression tests.""" +from __future__ import annotations + +import os +import sqlite3 +import stat + +import pytest + +from scripts.ops.backup_metadata import BackupError, backup_sqlite + + +def test_online_snapshot_remains_consistent_after_source_changes(tmp_path): + source = tmp_path / "metadata.db" + destination = tmp_path / "backup.db" + with sqlite3.connect(source) as db: + db.execute("CREATE TABLE journal(id INTEGER PRIMARY KEY, value TEXT NOT NULL)") + db.executemany("INSERT INTO journal(value) VALUES(?)", [(str(i),) for i in range(30)]) + db.commit() + digest = backup_sqlite(source, destination) + assert len(digest) == 64 + import hashlib + assert hashlib.sha256(destination.read_bytes()).hexdigest() == digest + with sqlite3.connect(source) as db: + db.execute("INSERT INTO journal(value) VALUES('new data')") + db.commit() + with sqlite3.connect(destination) as backup: + assert backup.execute("PRAGMA integrity_check").fetchone()[0] == "ok" + assert backup.execute("SELECT COUNT(*) FROM journal").fetchone()[0] == 30 + with sqlite3.connect(source) as live: + assert live.execute("SELECT COUNT(*) FROM journal").fetchone()[0] == 31 + if os.name == "posix": + assert stat.S_IMODE(destination.stat().st_mode) == 0o600 + + +def test_never_overwrite_or_replace_live_database(tmp_path): + source = tmp_path / "metadata.db" + with sqlite3.connect(source) as db: + db.execute("CREATE TABLE example(id INTEGER)") + target = tmp_path / "backup.db" + target.write_bytes(b"pre-existing critical backup") + with pytest.raises(FileExistsError): + backup_sqlite(source, target) + assert target.read_bytes() == b"pre-existing critical backup" + with pytest.raises(BackupError, match="live database"): + backup_sqlite(source, source) + assert sqlite3.connect(source).execute("PRAGMA integrity_check").fetchone()[0] == "ok" + + +def test_corrupt_source_and_symlinks_fail_closed_and_clean_temporary_files(tmp_path): + bad = tmp_path / "invalid.db" + bad.write_bytes(b"not sqlite") + output = tmp_path / "not-produced.db" + with pytest.raises(BackupError): + backup_sqlite(bad, output) + assert not output.exists() + assert not list(tmp_path.glob(".teledrive-snapshot-*")) + linked = tmp_path / "source-link.db" + linked.symlink_to(bad) + with pytest.raises(BackupError, match="regular"): + backup_sqlite(linked, output) + assert not output.exists() + +def test_online_metadata_dr_backup_preserves_unknown_mtproto_send_fences(tmp_path): + """A consistent backup must not turn an unknown Telegram send into a retry. + + The backup contains SQL-only metadata; no user media or Telegram messages + are touched. A restored operator cannot silently skip the pre-send fence. + """ + import hashlib + + from teledrive.db import connect_database + from teledrive.storage.user_mode import UserSendJournal, mtproto_send_intents + from sqlalchemy import select + + live_path = tmp_path / "mtproto-intents-live.db" + archive_path = tmp_path / "mtproto-intents-backup.db" + active = connect_database(str(live_path)) + try: + journal = UserSendJournal(active) + digest = hashlib.sha256(b"possibly-accepted").hexdigest() + known = hashlib.sha256(b"received-a-valid-ack").hexdigest() + journal.start("user-dr", -10012345, "unknown-channel-post", 0, + digest, len(b"possibly-accepted")) + journal.start("user-dr", -10012345, "acknowledged-post", 0, + known, len(b"received-a-valid-ack")) + journal.acknowledged("user-dr", -10012345, "acknowledged-post", 0, 17) + assert journal.unresolved_count() == 1 + snapshot_sha = backup_sqlite(live_path, archive_path) + assert len(snapshot_sha) == 64 + finally: + active.close() + + recovered = connect_database(str(archive_path)) + try: + journal = UserSendJournal(recovered) + assert journal.unresolved_count() == 1 + # A restored operator cannot silently skip the pre-send fence: a + # duplicate start for the unknown outcome stays rejected. + with pytest.raises(RuntimeError, match="Unresolved MTProto upload intent"): + journal.start("user-dr", -10012345, "unknown-channel-post", 0, + digest, len(b"possibly-accepted")) + acknowledged = recovered.connection.execute( + select(mtproto_send_intents.c.message_id).where( + mtproto_send_intents.c.object_id == "acknowledged-post", + ) + ).scalar_one() + assert acknowledged == 17 + assert journal.unresolved_count() == 1 + finally: + recovered.close() diff --git a/tests/storage/test_metadata_snapshot_rehearsal.py b/tests/storage/test_metadata_snapshot_rehearsal.py new file mode 100644 index 00000000..4d0a6044 --- /dev/null +++ b/tests/storage/test_metadata_snapshot_rehearsal.py @@ -0,0 +1,109 @@ +"""Staging metadata restore verification is read-only and fail-closed.""" +from __future__ import annotations + +import hashlib +import sqlite3 +import time + +import pytest + +from scripts.ops.backup_metadata import backup_sqlite +from scripts.ops.verify_metadata_backup import InvalidSnapshot, verify_snapshot +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +def seed(path): + db = connect_database(str(path)) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("verify-ci", int(time.time())) + ident = catalog.manifests.begin("verify-ci", "a.bin", 0, int(time.time())) + catalog.manifests.finish(ident, hashlib.sha256(b"").hexdigest(), int(time.time())) + catalog.publish(ident, now=int(time.time())) + finally: + db.close() + + +def test_backup_verification_accepts_valid_snapshot_and_checksum(tmp_path): + live, image = tmp_path / "live.db", tmp_path / "image.db" + seed(live) + receipt = backup_sqlite(live, image) + result = verify_snapshot(image, receipt) + assert result["valid"] and result["objects"] == 1 + assert result["versions"] == result["heads"] == 1 + assert result["physical_deletion_performed"] is False + with pytest.raises(InvalidSnapshot, match="receipt"): + verify_snapshot(image, "0" * 64) + + +def test_backup_verification_rejects_corrupted_heads_and_versions(tmp_path): + live = tmp_path / "live.db" + seed(live) + with sqlite3.connect(live) as db: + db.execute("UPDATE storage_object_heads SET key_hash=?", ("0" * 64,)) + db.commit() + with pytest.raises(InvalidSnapshot, match="HEAD"): + verify_snapshot(live) + second = tmp_path / "second.db" + seed(second) + with sqlite3.connect(second) as db: + db.execute("UPDATE storage_published_versions SET key_hash=?", ("0" * 64,)) + db.commit() + with pytest.raises(InvalidSnapshot, match="Published"): + verify_snapshot(second) + + +def test_snapshot_verification_refuses_symlink_and_non_sqlite(tmp_path): + bad = tmp_path / "bad.db" + bad.write_bytes(b"not a sqlite database") + with pytest.raises(InvalidSnapshot): + verify_snapshot(bad) + alias = tmp_path / "link.db" + alias.symlink_to(bad) + with pytest.raises(InvalidSnapshot, match="regular"): + verify_snapshot(alias) + + + +def test_verifier_includes_removed_versions_and_rejects_head_to_revoked_id(tmp_path): + db_path = tmp_path / "history.db" + seed(db_path) + conn = connect_database(str(db_path)) + try: + cat = BucketCatalog(ManifestStore(conn)) + original = cat.get("verify-ci", "a.bin").item.id + marker = cat.delete_marker("verify-ci", "a.bin") + assert cat.delete_version("verify-ci", "a.bin", marker) + assert cat.is_visible_version("verify-ci", "a.bin", original) + finally: + conn.close() + snapshot = tmp_path / "valid-history.db" + sha = backup_sqlite(db_path, snapshot) + record = verify_snapshot(snapshot, sha) + assert record["removed_versions"] == 1 + assert record["versions"] == 2 + assert record["heads"] == 1 + with sqlite3.connect(snapshot) as raw: + raw.execute( + "UPDATE storage_object_heads SET object_id=? WHERE bucket='verify-ci'", + (marker,), + ) + raw.commit() + with pytest.raises(InvalidSnapshot, match="revoked"): + verify_snapshot(snapshot) + + +def test_verifier_rejects_orphan_removed_version_entry_even_when_sqlite_integrity_ok(tmp_path): + db_path = tmp_path / "orphans.db" + seed(db_path) + with sqlite3.connect(db_path) as raw: + raw.execute( + "INSERT INTO storage_removed_versions(object_id, removed_ns) " + "VALUES(?, ?)", ("unknown-version-id", 12), + ) + raw.commit() + assert raw.execute("PRAGMA integrity_check").fetchone() == ("ok",) + with pytest.raises(InvalidSnapshot, match="published history"): + verify_snapshot(db_path) diff --git a/tests/storage/test_mtproto_multi_credentials.py b/tests/storage/test_mtproto_multi_credentials.py new file mode 100644 index 00000000..7bfc527b --- /dev/null +++ b/tests/storage/test_mtproto_multi_credentials.py @@ -0,0 +1,556 @@ +"""Multiple Telegram api_id/api_hash credential sets with rotation strategies. + +The unified USER transport may hold several authorized sessions posting to +the SAME dedicated channel. round_robin distributes sends across sets; +failover keeps a primary set until a definite flood-wait rejection, then +switches. Ambiguous send outcomes never trigger cross-set retries. +""" +from __future__ import annotations + +import asyncio +import hashlib +from pathlib import Path + +import pytest + +from teledrive.db import connect_database +from teledrive.storage.user_mode import ( + UnifiedUserTransport, UserSendJournal, UserSessionRuntime, +) + + +def _write_session(tmp_path, name, value="fake-session-data"): + target = tmp_path / name + target.write_text(value + "\n", encoding="ascii") + return target + + +class FakeClient: + identity_bot = False + + def __init__(self, *, label, runtime, chat_id=-10093744, peer=-10093744): + self.label = label + self.runtime = runtime + self.chat_id = chat_id + self.peer = peer + self.sent = 0 + + async def connect(self): + self.runtime.connected.append(self.label) + + async def disconnect(self): + pass + + async def is_connected(self): + return True + + async def get_me(self): + from types import SimpleNamespace + return SimpleNamespace(bot=type(self).identity_bot) + + async def iter_dialogs(self, limit): + from types import SimpleNamespace + assert limit == 5000 + from telethon.tl.types import Channel + yield SimpleNamespace(id=self.peer, entity=Channel()) + + async def get_entity(self, target): + from types import SimpleNamespace + return SimpleNamespace() + + async def get_peer_id(self, channel): + return self.chat_id + + async def send_file(self, target, path, caption=None, force_document=False): + self.sent += 1 + from types import SimpleNamespace + return SimpleNamespace(id=1000 + self.sent, chat_id=self.chat_id) + + async def get_messages(self, peer, ids): + raise AssertionError("readback not used here") + + async def download_media(self, message, file): + raise AssertionError("readback not used here") + + +class FakeRuntimeState: + def __init__(self): + self.connected = [] + self.clients = {} + + +def build_runtime(monkeypatch, tmp_path, env, *, bot=False): + state = FakeRuntimeState() + import telethon + import telethon.sessions + import telethon.tl.types + + class Channel: + broadcast = True + admin_rights = None + creator = True + + monkeypatch.setattr(telethon.tl.types, "Channel", Channel) + monkeypatch.setattr(telethon.sessions, "StringSession", lambda s: s) + + def fake_client(session, api_id, api_hash, **kwargs): + label = kwargs.get("device_model", "") + if label.startswith("TeleDrive Storage Client:"): + label = label.split(":", 1)[1] + elif label == "TeleDrive Storage Client": + label = "mtproto-user" + client = FakeClient(label=label, runtime=state) + state.clients[label] = client + return client + + monkeypatch.setattr(telethon, "TelegramClient", fake_client) + for key, value in env.items(): + monkeypatch.setenv(key, value) + FakeClient.identity_bot = bot + return state + + +@pytest.fixture +def runtime_env(tmp_path, monkeypatch): + session_a = _write_session(tmp_path, "session-a.session") + session_b = _write_session(tmp_path, "session-b.session") + + def apply(**overrides): + env = { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_STORAGE_MTPROTO_API_ID": "11111,22222", + "TELEDRIVE_STORAGE_MTPROTO_API_HASH": "a" * 32 + "," + "b" * 32, + "TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE": ( + str(session_a) + "," + str(session_b) + ), + "TELEDRIVE_STORAGE_MTPROTO_ACCOUNT": "user-a,user-b", + "TELEDRIVE_STORAGE_MTPROTO_CHANNEL": "-10093744", + "TELEDRIVE_STORAGE_MTPROTO_STRATEGY": "round_robin", + } + env.update(overrides) + return build_runtime(monkeypatch, tmp_path, env) + + return apply + + +def _config_path(tmp_path, strategy="round_robin"): + session_a = _write_session(tmp_path, "a.session") + session_b = _write_session(tmp_path, "b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + f" strategy: {strategy}\n" + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'a' * 32}\n" + f" session_file: {session_a}\n" + " account: user-a\n" + " - api_id: 22222\n" + f" api_hash: {'b' * 32}\n" + f" session_file: {session_b}\n" + " account: user-b\n", + encoding="ascii", + ) + return config_path + + +def _runtime(db, monkeypatch, tmp_path, strategy="round_robin", **env): + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_CONFIG": str(_config_path(tmp_path, strategy=strategy)), + **env, + }) + runtime = UserSessionRuntime() + return runtime, state + + +def test_runtime_starts_all_credential_sets_and_resolves_one_channel( + monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + runtime, state = _runtime(db, monkeypatch, tmp_path) + asyncio.run(runtime.start()) + assert set(state.connected) == {"user-a", "user-b"} + assert runtime.account == "user-a" + assert runtime.chat_id == -10093744 + # Every set resolves the exact same dedicated channel. + assert set(runtime.transports()) == { + "user-a", "user-b", + } + finally: + db.close() + + +def test_round_robin_rotates_sets_across_uploads(monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + runtime, state = _runtime(db, monkeypatch, tmp_path) + asyncio.run(runtime.start()) + journal = UserSendJournal(db) + transport = UnifiedUserTransport(runtime, journal, + min_upload_interval=0) + source = tmp_path / "body.bin" + source.write_bytes(b"payload-bytes") + checksum = hashlib.sha256(b"payload-bytes").hexdigest() + first = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=0, checksum=checksum)) + second = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=1, checksum=checksum)) + assert first[0] == "user-a" + assert second[0] == "user-b" + assert state.clients["user-a"].sent == 1 + assert state.clients["user-b"].sent == 1 + assert journal.unresolved_count() == 0 + finally: + db.close() + + +def test_failover_switches_set_after_definite_flood_rejection( + monkeypatch, tmp_path): + import telethon.errors + + class Flood(telethon.errors.FloodWaitError): + def __init__(self, seconds=3): + super().__init__(request=None, capture=seconds) + + db = connect_database(str(tmp_path / "state.db")) + try: + runtime, state = _runtime(db, monkeypatch, tmp_path, + strategy="failover") + asyncio.run(runtime.start()) + journal = UserSendJournal(db) + transport = UnifiedUserTransport(runtime, journal, + min_upload_interval=0) + source = tmp_path / "body.bin" + source.write_bytes(b"payload-bytes") + checksum = hashlib.sha256(b"payload-bytes").hexdigest() + + async def flood_once(target, path, caption=None, force_document=False): + if state.clients["user-a"].sent == 0: + state.clients["user-a"].sent += 1 + raise Flood(2) + state.clients["user-a"].sent += 1 + from types import SimpleNamespace + return SimpleNamespace(id=999, chat_id=-10093744) + + state.clients["user-a"].send_file = flood_once + result = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=0, checksum=checksum)) + # The definite rejection cleared the set-A intent; set B performed + # the retry and the receipt is journaled under the set-B label. + assert result[0] == "user-b" + assert journal.unresolved_count() == 0 + # A flood observation was recorded for the rejected set. + from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + assert MtprotoRateGuard(db).total_explicit_flood_observations() == 1 + assert MtprotoRateGuard(db).shared_cooldown_wait( + "user-a", -10093744, + ) == pytest.approx(2, abs=0.2) + # Failover keeps the healthy set for subsequent uploads. + second = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=1, checksum=checksum)) + assert second[0] == "user-b" + finally: + db.close() + + +def test_ambiguous_send_outcome_never_triggers_cross_set_retry( + monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + runtime, state = _runtime(db, monkeypatch, tmp_path) + asyncio.run(runtime.start()) + journal = UserSendJournal(db) + transport = UnifiedUserTransport(runtime, journal, + min_upload_interval=0) + source = tmp_path / "body.bin" + source.write_bytes(b"payload-bytes") + checksum = hashlib.sha256(b"payload-bytes").hexdigest() + + async def ambiguous(target, path, caption=None, force_document=False): + raise OSError("connection lost after send") + + state.clients["user-a"].send_file = ambiguous + with pytest.raises(RuntimeError, match="uncertain"): + asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=0, checksum=checksum)) + # The intent fence for set A is retained; set B was never tried. + assert journal.unresolved_count() == 1 + assert state.clients["user-b"].sent == 0 + finally: + db.close() + + +@pytest.mark.parametrize("config_text,match", [ + # No apis: list at all -> fail closed (there is no env fallback). + ("storage:\n channel: '-10093744'\n", "storage.apis"), + # Invalid strategy value. + ("storage:\n channel: '-10093744'\n strategy: random\n apis: []\n", + "strategy"), +]) +def test_malformed_credential_lists_fail_closed_before_network( + monkeypatch, tmp_path, config_text, match): + db = connect_database(str(tmp_path / "state.db")) + try: + config_path = tmp_path / "config.yaml" + config_path.write_text(config_text, encoding="ascii") + build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + with pytest.raises(RuntimeError, match=match): + asyncio.run(runtime.start()) + finally: + db.close() + + +def test_config_without_apis_has_no_env_fallback(monkeypatch, tmp_path): + """Settings are config-only: stale env credentials are never consulted.""" + db = connect_database(str(tmp_path / "state.db")) + try: + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n channel: '-10093744'\n" + "telegram:\n api_id: 1\n", encoding="ascii") + build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_CONFIG": str(config_path), + "TELEDRIVE_STORAGE_MTPROTO_API_ID": "33333", + "TELEDRIVE_STORAGE_MTPROTO_API_HASH": "c" * 32, + "TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE": str( + _write_session(tmp_path, "env.session")), + }) + runtime = UserSessionRuntime() + with pytest.raises(RuntimeError, match="storage.apis"): + asyncio.run(runtime.start()) + finally: + db.close() + + +def test_single_strategy_uses_only_the_first_set(monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "a.session") + session_b = _write_session(tmp_path, "b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + " strategy: single\n" + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'a' * 32}\n" + f" session_file: {session_a}\n" + " account: user-a\n" + " - api_id: 22222\n" + f" api_hash: {'b' * 32}\n" + f" session_file: {session_b}\n" + " account: user-b\n", + encoding="ascii", + ) + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + asyncio.run(runtime.start()) + journal = UserSendJournal(db) + transport = UnifiedUserTransport(runtime, journal, min_upload_interval=0) + source = tmp_path / "body.bin" + source.write_bytes(b"payload-bytes") + checksum = hashlib.sha256(b"payload-bytes").hexdigest() + first = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=0, checksum=checksum)) + second = asyncio.run(transport.upload( + source, object_id="obj-1", ordinal=1, checksum=checksum)) + assert first[0] == second[0] == "user-a" + assert state.clients["user-b"].sent == 0 + finally: + db.close() + + +def test_parallel_strategy_uploads_chunks_concurrently(monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "a.session") + session_b = _write_session(tmp_path, "b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + " strategy: parallel\n" + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'a' * 32}\n" + f" session_file: {session_a}\n" + " account: user-a\n" + " - api_id: 22222\n" + f" api_hash: {'b' * 32}\n" + f" session_file: {session_b}\n" + " account: user-b\n", + encoding="ascii", + ) + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + asyncio.run(runtime.start()) + assert UnifiedUserTransport(runtime, UserSendJournal(db)).parallel_degree() == 2 + assert runtime.strategy == "parallel" + finally: + db.close() + + +def test_bot_identity_in_any_credential_set_is_rejected( + monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + runtime, state = _runtime(db, monkeypatch, tmp_path) + FakeClient.identity_bot = True + with pytest.raises(RuntimeError, match="USER"): + asyncio.run(runtime.start()) + FakeClient.identity_bot = False + finally: + db.close() + + +def test_config_apis_list_drives_runtime_sets(monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "cfg-a.session") + session_b = _write_session(tmp_path, "cfg-b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + " strategy: round_robin\n" + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'c' * 32}\n" + f" session_file: {session_a}\n" + " account: user-a\n" + " - api_id: 22222\n" + f" api_hash: {'d' * 32}\n" + f" session_file: {session_b}\n" + " account: user-b\n", + encoding="ascii", + ) + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_STORAGE_MTPROTO_CHANNEL": "-10093744", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + asyncio.run(runtime.start()) + assert set(state.connected) == {"user-a", "user-b"} + assert runtime.account == "user-a" + assert runtime.chat_id == -10093744 + assert set(runtime.transports()) == {"user-a", "user-b"} + finally: + db.close() + + +def test_config_apis_list_defaults_account_labels(monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "a.session") + session_b = _write_session(tmp_path, "b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'e' * 32}\n" + f" session_file: {session_a}\n" + " - api_id: 22222\n" + f" api_hash: {'f' * 32}\n" + f" session_file: {session_b}\n", + encoding="ascii", + ) + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_STORAGE_MTPROTO_CHANNEL": "-10093744", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + asyncio.run(runtime.start()) + assert set(state.connected) == {"mtproto-user", "mtproto-user-2"} + finally: + db.close() + + +def test_config_apis_list_takes_precedence_over_env_lists( + monkeypatch, tmp_path): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "cfg-a.session") + session_b = _write_session(tmp_path, "cfg-b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-10093744"\n' + " apis:\n" + " - api_id: 11111\n" + f" api_hash: {'e' * 32}\n" + f" session_file: {session_a}\n" + " - api_id: 22222\n" + f" api_hash: {'f' * 32}\n" + f" session_file: {session_b}\n", + encoding="ascii", + ) + state = build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_STORAGE_MTPROTO_CHANNEL": "-10093744", + "TELEDRIVE_CONFIG": str(config_path), + # Stale env lists must be ignored when the config has apis. + "TELEDRIVE_STORAGE_MTPROTO_API_ID": "99999,88888", + "TELEDRIVE_STORAGE_MTPROTO_API_HASH": "a" * 32 + "," + "b" * 32, + "TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE": "x,y", + "TELEDRIVE_STORAGE_MTPROTO_ACCOUNT": "env-a,env-b", + }) + runtime = UserSessionRuntime() + asyncio.run(runtime.start()) + assert set(state.connected) == {"mtproto-user", "mtproto-user-2"} + finally: + db.close() + + +@pytest.mark.parametrize("apis_text,match", [ + ("storage:\n channel: '-10093744'\n apis:\n - api_id: 0\n api_hash: {h}\n session_file: {a}\n", + "credential"), + ("storage:\n channel: '-10093744'\n apis:\n - api_id: 11111\n api_hash: short\n session_file: {a}\n", + "credential"), + ("storage:\n channel: '-10093744'\n apis:\n - api_id: 11111\n api_hash: {h}\n session_file: /nope.session\n", + "session"), + ("storage:\n channel: '-10093744'\n apis:\n - api_id: 11111\n api_hash: {h}\n session_file: {a}\n account: api-bot\n", + "credential"), + ("storage:\n channel: '-10093744'\n apis:\n - api_id: 11111\n api_hash: {h}\n session_file: {a}\n account: dup\n" + " - api_id: 22222\n api_hash: {h2}\n session_file: {b}\n account: dup\n", + "credential"), +]) +def test_malformed_config_apis_fail_closed(monkeypatch, tmp_path, + apis_text, match): + db = connect_database(str(tmp_path / "state.db")) + try: + session_a = _write_session(tmp_path, "cfg-a.session") + session_b = _write_session(tmp_path, "cfg-b.session") + config_path = tmp_path / "config.yaml" + config_path.write_text( + apis_text.format(h="a" * 32, h2="b" * 32, + a=str(session_a), b=str(session_b)), + encoding="ascii", + ) + build_runtime(monkeypatch, tmp_path, { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + "TELEDRIVE_STORAGE_MTPROTO_CHANNEL": "-10093744", + "TELEDRIVE_CONFIG": str(config_path), + }) + runtime = UserSessionRuntime() + with pytest.raises(RuntimeError, match=match): + asyncio.run(runtime.start()) + finally: + db.close() diff --git a/tests/storage/test_mtproto_rate_guard.py b/tests/storage/test_mtproto_rate_guard.py new file mode 100644 index 00000000..19582069 --- /dev/null +++ b/tests/storage/test_mtproto_rate_guard.py @@ -0,0 +1,240 @@ +"""Cross-worker SQL reservation and explicit flood-wait backoff isolation. + +MTProto-only: the USER transport reserves shared account/channel send slots +in SQL and observes definitive FloodWaitError backoffs, so independent +workers never start parallel sends during a server cooldown. +""" +from __future__ import annotations + +import subprocess +import sys + +import pytest + +from teledrive.db import connect_database +from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + + +def test_shared_mtproto_slots_order_workers_and_separate_accounts(tmp_path): + db_path = str(tmp_path / "shared-slot.db") + db1 = connect_database(db_path) + db2 = connect_database(db_path) + try: + first, second = MtprotoRateGuard(db1), MtprotoRateGuard(db2) + t = 5_000_000 + assert first.reserve_send_slot("user-a", -100111, 1.25, now_ms=t) == 0 + assert second.reserve_send_slot("user-a", -100111, 1.25, now_ms=t) == 1.25 + assert first.reserve_send_slot("user-a", -100111, 1.25, now_ms=t) == 2.5 + assert second.reserve_send_slot("user-b", -100111, 1.25, now_ms=t) == 0 + assert second.reserve_send_slot("user-a", -100112, 1.25, now_ms=t) == 0 + first.impose_shared_cooldown("user-a", -100111, 8, now_ms=t) + assert second.reserve_send_slot("user-a", -100111, 1.25, now_ms=t) == 8 + # A shorter late flood cannot shorten the existing global cooldown. + second.impose_shared_cooldown("user-a", -100111, 2, now_ms=t) + assert first.reserve_send_slot("user-a", -100111, 1.25, now_ms=t) == 9.25 + # Skip a real delay: future timestamps take the next slot immediately. + assert second.reserve_send_slot("user-a", -100111, 1.25, + now_ms=t + 100_000) == 0 + finally: + db2.close() + db1.close() + + +def test_separate_os_processes_allocate_unique_global_send_slots(tmp_path): + db_path = str(tmp_path / "multi-process-slot.db") + db = connect_database(db_path) + try: + guard = MtprotoRateGuard(db) + assert guard.reserve_send_slot("user-a", -100432, 0.75, + now_ms=500000) == 0 + finally: + db.close() + child = r''' +import sys +from teledrive.db import connect_database +from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard +db = connect_database(sys.argv[1]) +try: + guard = MtprotoRateGuard(db) + wait = guard.reserve_send_slot("user-a", -100432, 0.75, now_ms=500000) + print(wait) +finally: + db.close() +''' + processes = [ + subprocess.Popen([sys.executable, "-c", child, db_path], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + for _ in range(4) + ] + results = [] + for process in processes: + stdout, stderr = process.communicate(timeout=30) + assert process.returncode == 0, stderr.decode(errors="replace") + results.append(float(stdout.decode().strip())) + assert sorted(results) == [0.75, 1.5, 2.25, 3.0] + + +@pytest.mark.parametrize("account,chat,seconds", [ + ("", -1001, 1), ("a", 0, 1), ("a", True, 1), + ("a", -1001, -1), ("a", -1001, 61), +]) +def test_invalid_shared_mtproto_send_throttle_never_allocates( + tmp_path, account, chat, seconds): + db = connect_database(str(tmp_path / "bad-slot.db")) + try: + guard = MtprotoRateGuard(db) + with pytest.raises(ValueError): + guard.reserve_send_slot(account, chat, seconds) + finally: + db.close() + + +def test_zero_min_interval_does_not_pace_but_respects_shared_flood(tmp_path): + path = str(tmp_path / "no-throttle.db") + db = connect_database(path) + second = connect_database(path) + try: + guard = MtprotoRateGuard(db) + other = MtprotoRateGuard(second) + now = 70_000 + assert guard.reserve_send_slot("user", -10023, 0, now_ms=now) == 0 + assert other.reserve_send_slot("user", -10023, 0, now_ms=now) == 0 + # Zero pacing still needs a durable slot row for server flood backoff. + guard.impose_shared_cooldown("user", -10023, 5, now_ms=now) + assert other.reserve_send_slot("user", -10023, 0, now_ms=now) == 5 + assert other.shared_cooldown_wait("user", -10023, now_ms=now) == 5 + assert other.reserve_send_slot("other", -10023, 0, now_ms=now) == 0 + assert other.reserve_send_slot("user", -10024, 0, now_ms=now) == 0 + assert other.reserve_send_slot("user", -10023, 0, now_ms=now + 5_000) == 0 + finally: + second.close() + db.close() + + +def test_flood_after_other_workers_reserved_slots_remains_globally_visible(tmp_path): + """A late Telegram flood-wait must fence PREVIOUSLY scheduled sends. + + Without a separate cooldown journal, sleeping workers have a stale + reservation and may wake up/send in parallel during server backoff. + """ + path = str(tmp_path / "late-flood.db") + a = connect_database(path) + b = connect_database(path) + try: + first, second = MtprotoRateGuard(a), MtprotoRateGuard(b) + now = 8_000_000 + assert first.reserve_send_slot("shared", -10077, 1.25, now_ms=now) == 0 + assert second.reserve_send_slot("shared", -10077, 1.25, now_ms=now) == 1.25 + assert first.shared_cooldown_wait("shared", -10077, now_ms=now) == 0 + second.impose_shared_cooldown("shared", -10077, 8, now_ms=now) + # Even the worker which reserved its slot BEFORE this flood must see it. + assert first.shared_cooldown_wait("shared", -10077, now_ms=now) == 8 + assert first.shared_cooldown_wait( + "shared", -10077, now_ms=now + 7_999, + ) == 0.001 + assert first.shared_cooldown_wait( + "shared", -10077, now_ms=now + 8_000, + ) == 0 + assert first.shared_cooldown_wait("other", -10077, now_ms=now) == 0 + assert first.shared_cooldown_wait("shared", -10078, now_ms=now) == 0 + first.impose_shared_cooldown("shared", -10077, 2, now_ms=now) + assert second.shared_cooldown_wait("shared", -10077, now_ms=now) == 8 + first.impose_shared_cooldown("shared", -10077, 12, now_ms=now) + assert second.shared_cooldown_wait("shared", -10077, now_ms=now) == 12 + finally: + b.close() + a.close() + + +def test_late_shared_flood_is_rechecked_before_mtproto_send(tmp_path, monkeypatch): + """A USER upload must inspect the new shared flood after SQL-slot sleeping.""" + import asyncio + import hashlib + import time + from teledrive.storage.user_mode import UnifiedUserTransport, UserSendJournal + + class InspectOnlyUser: + sends = 0 + + async def upload(self, path, *, object_id, ordinal, checksum): + self.sends += 1 + return "user", -100900, self.sends + + db = connect_database(str(tmp_path / "sleep-recheck.db")) + try: + journal = UserSendJournal(db) + from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + rate = MtprotoRateGuard(db) + delegate = InspectOnlyUser() + + class FakeRuntime: + account = "user" + chat_id = -100900 + + def require(self): + return delegate + + transport = UnifiedUserTransport(FakeRuntime(), journal, + rate=rate, min_upload_interval=0.001) + source = tmp_path / "data.bin" + source.write_bytes(b"known-safe-bytes") + cooldown_checks = [] + original = rate.shared_cooldown_wait + + def delayed_check(account, chat_id, **kwargs): + cooldown_checks.append((account, chat_id)) + # Simulate another OS worker receiving a flood-wait AFTER this + # transport finished its old SQL send-slot wait. + return 0.005 if len(cooldown_checks) == 1 else original( + account, chat_id, **kwargs, + ) + + monkeypatch.setattr(rate, "shared_cooldown_wait", delayed_check) + result = asyncio.run(transport.upload( + source, object_id="late-cooldown", ordinal=0, + checksum=hashlib.sha256(source.read_bytes()).hexdigest(), + )) + assert result == ("user", -100900, 1) + assert len(cooldown_checks) >= 2 + assert delegate.sends == 1 + assert journal and rate.total_explicit_flood_observations() == 0 + finally: + db.close() + + +@pytest.mark.parametrize("value", [-1, 0, 1.0, True, 3601]) +def test_invalid_shared_flood_cannot_change_existing_cooldown(tmp_path, value): + db = connect_database(str(tmp_path / "flood-validation.db")) + try: + guard = MtprotoRateGuard(db) + guard.reserve_send_slot("a", -100900, 1.0, now_ms=1000) + guard.impose_shared_cooldown("a", -100900, 10, now_ms=1000) + with pytest.raises(ValueError): + guard.impose_shared_cooldown("a", -100900, value, now_ms=1000) + assert guard.shared_cooldown_wait( + "a", -100900, now_ms=1000, + ) == 10 + finally: + db.close() + + +def test_flood_sql_journal_fault_rolls_back_both_slot_and_cooldown(tmp_path): + """A write failure cannot publish a partial flood-wait observation.""" + path = str(tmp_path / "flood-atomic.db") + db = connect_database(path) + try: + guard = MtprotoRateGuard(db) + t = 50_000 + assert guard.reserve_send_slot("user", -100905, 1.0, now_ms=t) == 0 + db.connection.exec_driver_sql( + "CREATE TRIGGER reject_flood BEFORE INSERT ON " + "storage_mtproto_chat_cooldowns " + "BEGIN SELECT RAISE(ABORT, 'test unavailable cooldown SQL'); END" + ) + db.commit() + with pytest.raises(RuntimeError, match="cooldown"): + guard.impose_shared_cooldown("user", -100905, 4, now_ms=t) + assert guard.shared_cooldown_wait("user", -100905, now_ms=t) == 0 + assert guard.total_explicit_flood_observations() == 0 + finally: + db.close() diff --git a/tests/storage/test_storage_catalog.py b/tests/storage/test_storage_catalog.py new file mode 100644 index 00000000..a38cafb1 --- /dev/null +++ b/tests/storage/test_storage_catalog.py @@ -0,0 +1,142 @@ +"""Real SQL catalog tests for immutable version pointers and bucket behavior.""" +from __future__ import annotations + +from hashlib import sha256 + +import pytest + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +@pytest.fixture +def catalog(tmp_path): + db = connect_database(str(tmp_path / "catalog.sqlite")) + manifests = ManifestStore(db) + try: + yield BucketCatalog(manifests), manifests + finally: + db.close() + + +def ready(manifests, bucket, key, data, now): + ident = manifests.begin(bucket, key, len(data), now) + # Empty files are the only ones that can be published with no remote chunks. + assert data == b"" + manifests.finish(ident, sha256(data).hexdigest(), now + 1) + return ident + + +def test_bucket_crud_and_validation(catalog): + cat, _ = catalog + cat.create_bucket("my-bucket", 1) + cat.create_bucket("other-123", 2) + assert cat.list_buckets() == ["my-bucket", "other-123"] + with pytest.raises(ValueError, match="already exists"): + cat.create_bucket("my-bucket", 3) + for name in ["UPPER", "a", "192.168.1.1", "-bad", "bad..name", + "a" * 64, "bad_.name", "bad.-name"]: + with pytest.raises(ValueError): + cat.create_bucket(name, 4) + cat.delete_bucket("my-bucket") + assert cat.list_buckets() == ["other-123"] + + +def test_atomic_publish_and_overwrite_keeps_old_manifest(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 10) + first = ready(manifests, "backup-store", "folder/foo.txt", b"", 11) + head = cat.publish(first, now=12, expected_generation=0) + assert head.generation == 1 + assert cat.get("backup-store", "folder/foo.txt").item.id == first + second = ready(manifests, "backup-store", "folder/foo.txt", b"", 20) + head = cat.publish(second, now=21, expected_generation=1) + assert head.generation == 2 and head.item.id == second + assert manifests.get(first).id == first + with pytest.raises(ValueError, match="precondition"): + cat.publish(first, now=22, expected_generation=1) + assert cat.get("backup-store", "folder/foo.txt").item.id == second + + +def test_list_pagination_unicode_and_literal_prefix(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 10) + for i, key in enumerate(["prefix%literal", "prefix_a", "prefix_b", + "quotes/\"hello\"", "ümlaut", "prefix/z"]): + item = ready(manifests, "backup-store", key, b"", i + 20) + cat.publish(item, now=i + 20) + first_page = cat.list_objects("backup-store", prefix="prefix", limit=2) + assert [row.item.key for row in first_page] == ["prefix%literal", "prefix/z"] + second_page = cat.list_objects("backup-store", prefix="prefix", + after=first_page[-1].item.key, limit=2) + assert [row.item.key for row in second_page] == ["prefix_a", "prefix_b"] + assert [x.item.key for x in cat.list_objects("backup-store", prefix="prefix%")] == [ + "prefix%literal", + ] + assert [x.item.key for x in cat.list_objects("backup-store", prefix="prefix_")] == [ + "prefix_a", "prefix_b", + ] + + +def test_delete_unlinks_only_current_pointer_and_cannot_destroy_history(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 1) + ident = ready(manifests, "backup-store", "test", b"", 2) + cat.publish(ident, now=3) + assert cat.remove_current("backup-store", "test", expected_generation=1) + assert not cat.remove_current("backup-store", "test") + with pytest.raises(FileNotFoundError): + cat.get("backup-store", "test") + assert manifests.get(ident).id == ident + with pytest.raises(ValueError, match="history"): + cat.delete_bucket("backup-store") + + +def test_uncommitted_object_cannot_be_published(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 1) + ident = manifests.begin("backup-store", "pending", 55, 2) + with pytest.raises(FileNotFoundError): + cat.publish(ident, now=3) + assert cat.list_objects("backup-store") == [] + + +def test_invalid_key_size_fails_before_catalog_write(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 1) + for key in ["", "ü" * 600]: + ident = ready(manifests, "backup-store", key, b"", 2) if key else None + if ident: + with pytest.raises(ValueError, match="UTF-8"): + cat.publish(ident, now=3) + + +def test_generation_guard_prevents_lost_update(catalog): + cat, manifests = catalog + cat.create_bucket("backup-store", 1) + ids = [ready(manifests, "backup-store", "same", b"", i * 10 + 1) for i in range(3)] + assert cat.publish(ids[0], now=5).generation == 1 + assert cat.publish(ids[1], now=6, expected_generation=1).generation == 2 + with pytest.raises(ValueError, match="precondition"): + cat.publish(ids[2], now=7, expected_generation=1) + assert cat.get("backup-store", "same").item.id == ids[1] + + +def test_catalog_metadata_survives_restart(tmp_path): + url = str(tmp_path / "state.db") + one = connect_database(url) + try: + manifests = ManifestStore(one) + cat = BucketCatalog(manifests) + cat.create_bucket("persisted-bucket", 1) + ident = ready(manifests, "persisted-bucket", "some/key", b"", 2) + cat.publish(ident, now=3) + finally: + one.close() + two = connect_database(url) + try: + cat = BucketCatalog(ManifestStore(two)) + assert cat.get("persisted-bucket", "some/key").item.id == ident + finally: + two.close() diff --git a/tests/storage/test_storage_engine.py b/tests/storage/test_storage_engine.py new file mode 100644 index 00000000..2265ebbb --- /dev/null +++ b/tests/storage/test_storage_engine.py @@ -0,0 +1,465 @@ +"""Storage-engine behavior tests: real SQL transactions, fake Telegram I/O. + +These test the actual byte transfer algorithm and manifest persistence, not +merely HTTP response status or mocked function calls. +""" +from __future__ import annotations + +import asyncio +import hashlib +import os +from pathlib import Path + +import pytest +from sqlalchemy import select + +from teledrive.db import connect_database +from teledrive.storage.engine import IntegrityError, TelegramObjectStorage +from teledrive.storage.manifest import ( + ChunkReference, ManifestError, ManifestStore, objects, chunks, +) + + +class MemoryTelegram: + """Deterministic stand-in for Telegram message storage.""" + + def __init__(self): + self.data: dict[tuple[str, int, int], bytes] = {} + self.uploads = 0 + self.downloads = 0 + self.fail_on_upload: int | None = None + self.on_upload = None + self.account = "primary" + + async def upload(self, path: Path, *, object_id: str, ordinal: int, + checksum: str) -> tuple[str, int, int]: + self.uploads += 1 + if self.fail_on_upload == self.uploads: + raise TimeoutError("simulated Telegram disconnect") + message = self.uploads + body = path.read_bytes() + assert hashlib.sha256(body).hexdigest() == checksum + self.data[(self.account, 12345, message)] = body + if self.on_upload: + self.on_upload() + return self.account, 12345, message + + async def download(self, reference: ChunkReference, destination: Path) -> None: + self.downloads += 1 + key = (reference.account, reference.chat_id, reference.message_id) + if key not in self.data: + raise FileNotFoundError("Telegram media missing") + destination.write_bytes(self.data[key]) + + +@pytest.fixture +def store(tmp_path): + conn = connect_database(str(tmp_path / "state.db")) + manifests = ManifestStore(conn) + try: + yield manifests + finally: + conn.close() + + +def execute(coroutine): + return asyncio.run(coroutine) + + +@pytest.mark.parametrize("payload,chunk_size", [ + (b"", 5), (b"a", 5), (b"abcd", 5), + (b"abcde", 5), (b"abcdef", 5), (bytes(range(53)), 5), + (os.urandom(100_003), 4096), +]) +def test_full_telegram_round_trip_and_persisted_manifest(store, tmp_path, payload, chunk_size): + backend = MemoryTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=chunk_size) + source = tmp_path / "original" + source.write_bytes(payload) + item = execute(engine.put_file("backup", "folder/backup.bin", source)) + assert item.size == len(payload) + assert item.sha256 == hashlib.sha256(payload).hexdigest() + assert len(store.parts(item.id)) == (len(payload) + chunk_size - 1) // chunk_size + output = tmp_path / "restored" + # No source file or persistent local content remains for the read path. + source.unlink() + execute(engine.download_to_path(item.id, output)) + assert output.read_bytes() == payload + assert store.get(item.id).key == "folder/backup.bin" + assert len(list(tmp_path.glob("teledrive-*"))) == 0 + + +def test_cross_chunk_range_reads_only_relevant_parts(store, tmp_path): + backend = MemoryTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=4) + source = tmp_path / "file" + source.write_bytes(b"0123456789ABCDEF") + item = execute(engine.put_file("b", "k", source)) + async def read_range(): + result = bytearray() + async for part in engine.iter_bytes(item.id, start=3, end=11, block_size=2): + assert len(part) <= 2 + result.extend(part) + return bytes(result) + assert execute(read_range()) == b"3456789A" + assert backend.downloads == 3 + + +def test_missing_telegram_chunk_never_publishes_partial_destination(store, tmp_path): + backend = MemoryTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=3) + source = tmp_path / "source" + source.write_bytes(b"abcdefghi") + item = execute(engine.put_file("b", "k", source)) + part = store.parts(item.id)[1] + del backend.data[(part.account, part.chat_id, part.message_id)] + output = tmp_path / "output" + output.write_bytes(b"previous-good-value") + with pytest.raises(FileNotFoundError): + execute(engine.download_to_path(item.id, output)) + assert output.read_bytes() == b"previous-good-value" + assert not list(tmp_path.glob(".teledrive-*")) + + +@pytest.mark.parametrize("replacement", [b"xxxx", b"ab", b""]) +def test_tampered_or_truncated_chunk_fails_closed(store, tmp_path, replacement): + backend = MemoryTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=4) + source = tmp_path / "src" + source.write_bytes(b"abcdefgh") + item = execute(engine.put_file("b", "k", source)) + first = store.parts(item.id)[0] + backend.data[(first.account, first.chat_id, first.message_id)] = replacement + with pytest.raises(IntegrityError, match="checksum"): + execute(engine.download_to_path(item.id, tmp_path / "out")) + assert not (tmp_path / "out").exists() + + +def test_transient_corrupt_telegram_read_retries_only_immutable_download( + store, tmp_path +): + class FlakyTelegram(MemoryTelegram): + def __init__(self): + super().__init__() + self.corrupt_once = True + + async def download(self, reference, destination): + await super().download(reference, destination) + if reference.ordinal == 1 and self.corrupt_once: + self.corrupt_once = False + # Same size but different content exercises SHA, not length. + data = destination.read_bytes() + destination.write_bytes(bytes([data[0] ^ 1]) + data[1:]) + + backend = FlakyTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=4) + payload = b"0123456789AB" + source = tmp_path / "original" + source.write_bytes(payload) + obj = execute(engine.put_file("bucket", "key", source)) + initial_uploads = backend.uploads + source.unlink() + restored = tmp_path / "restored" + execute(engine.download_to_path(obj.id, restored)) + assert restored.read_bytes() == payload + assert backend.uploads == initial_uploads # Never resend after a bad read + assert backend.downloads == 4 # Three chunks plus one SHA retry + assert not list(tmp_path.glob(".teledrive-*")) + # A second, independently checked Range read uses the same SQL identity. + async def fetch_range(): + return b"".join([block async for block in engine.iter_bytes( + obj.id, start=3, end=9, block_size=2 + )]) + assert execute(fetch_range()) == payload[3:9] + + +def test_persistent_same_size_remote_corruption_exhausts_bounded_retries( + store, tmp_path +): + backend = MemoryTelegram() + engine = TelegramObjectStorage(store, backend, chunk_size=4) + source = tmp_path / "original" + source.write_bytes(b"abcdefgh") + obj = execute(engine.put_file("bucket", "key", source)) + first = store.parts(obj.id)[0] + key = (first.account, first.chat_id, first.message_id) + backend.data[key] = b"ABCD" # Same length, wrong digest. + output = tmp_path / "existing" + output.write_bytes(b"precious-existing-file") + with pytest.raises(IntegrityError, match="checksum mismatch"): + execute(engine.download_to_path(obj.id, output)) + assert backend.downloads == 3 + assert backend.uploads == 2 # Read retries are never Telegram sends + assert output.read_bytes() == b"precious-existing-file" + assert not list(tmp_path.glob(".teledrive-*")) + + +def test_failed_upload_is_hidden_but_journaled_for_reconciliation(store, tmp_path): + backend = MemoryTelegram() + backend.fail_on_upload = 2 + engine = TelegramObjectStorage(store, backend, chunk_size=3) + source = tmp_path / "src" + source.write_bytes(b"abcdefghi") + with pytest.raises(TimeoutError): + execute(engine.put_file("b", "k", source)) + rows = store.interrupted() + assert len(rows) == 1 and rows[0][1] == "failed" + identifier = rows[0][0] + with pytest.raises(FileNotFoundError): + store.get(identifier) + journal = store.recovery_parts(identifier) + assert len(journal) == 1 + assert journal[0].size == 3 + + +def test_changed_source_does_not_publish_object(store, tmp_path): + backend = MemoryTelegram() + source = tmp_path / "src" + source.write_bytes(b"abcdefgh") + async def edited(): + pass + backend.on_upload = lambda: source.write_bytes(b"ABCDEFGH") + engine = TelegramObjectStorage(store, backend, chunk_size=4) + with pytest.raises(IntegrityError, match="Source changed"): + execute(engine.put_file("b", "k", source)) + assert store.interrupted()[0][1] == "failed" + + +def test_manifest_refuses_incomplete_and_noncontiguous_parts(store): + now = 123 + identifier = store.begin("my-bucket", "my/key", 9, now) + reference = ChunkReference(0, 0, 4, hashlib.sha256(b"abcd").hexdigest(), + "primary", 12345, 99) + store.append(identifier, reference, now) + with pytest.raises(ManifestError, match="incomplete"): + store.finish(identifier, hashlib.sha256(b"abcdefghi").hexdigest(), now) + with pytest.raises(ManifestError, match="order"): + store.append(identifier, ChunkReference(2, 5, 5, reference.sha256, + "primary", 12345, 100), now) + assert store.recovery_parts(identifier) == [reference] + + +def test_object_metadata_is_durable_across_database_restart(tmp_path): + url = str(tmp_path / "metadata.sqlite") + db = connect_database(url) + try: + first = ManifestStore(db) + identifier = first.begin("bucket", "item", 0, 1) + first.finish(identifier, hashlib.sha256(b"").hexdigest(), 2) + finally: + db.close() + db2 = connect_database(url) + try: + second = ManifestStore(db2) + assert second.get(identifier).size == 0 + assert second.parts(identifier) == [] + finally: + db2.close() + + +def test_invalid_range_rejected_without_download(store, tmp_path): + fake = MemoryTelegram() + engine = TelegramObjectStorage(store, fake, chunk_size=2) + path = tmp_path / "x" + path.write_bytes(b"abcd") + obj = execute(engine.put_file("b", "k", path)) + async def consume(): + return [chunk async for chunk in engine.iter_bytes(obj.id, start=3, end=5)] + with pytest.raises(ValueError): + execute(consume()) + assert fake.downloads == 0 + + +def test_symlink_source_rejected_and_never_queued(store, tmp_path): + source = tmp_path / "real" + source.write_bytes(b"secret") + link = tmp_path / "link" + link.symlink_to(source) + engine = TelegramObjectStorage(store, MemoryTelegram(), chunk_size=2) + with pytest.raises(OSError): + execute(engine.put_file("b", "k", link)) + assert store.interrupted() == [] + + +def test_active_writer_lease_cannot_be_abandoned(store): + ident = store.begin("backup", "active", 0, 100) + store.start_writer(ident, "worker-a", 100, ttl=500) + assert store.abandon_stale_pending(now=200, min_age_seconds=60) == [] + assert store.heartbeat_writer(ident, "worker-a", 200, ttl=500) + assert store.abandon_stale_pending(now=500, min_age_seconds=60) == [] + ready = store.finish(ident, hashlib.sha256(b"").hexdigest(), 600, writer_token="worker-a") + assert ready.id == ident + assert store.abandon_stale_pending(now=900, min_age_seconds=60) == [] + + +def test_expired_writer_fenced_before_pending_abandonment(store): + ident = store.begin("backup", "interrupted", 0, 100) + store.start_writer(ident, "crashed-worker", 100, ttl=120) + assert store.abandon_stale_pending(now=161, min_age_seconds=60) == [] + assert store.abandon_stale_pending(now=221, min_age_seconds=60) == [ident] + assert store.interrupted() == [(ident, "failed")] + assert not store.heartbeat_writer(ident, "crashed-worker", 222, ttl=120) + with pytest.raises(ManifestError, match="not pending"): + store.finish(ident, hashlib.sha256(b"").hexdigest(), 222, writer_token="crashed-worker") + with pytest.raises(ManifestError, match="lease lost"): + store.fail(ident, "late worker", 222, writer_token="crashed-worker") + + +def test_stale_legacy_pending_without_writer_fence_is_not_deleted(store): + ident = store.begin("legacy", "old", 0, 100) + assert store.abandon_stale_pending(now=999_999, min_age_seconds=60) == [] + assert store.interrupted() == [(ident, "pending")] + + +def test_writer_tokens_cannot_be_impersonated(store): + ident = store.begin("backup", "owned", 0, 100) + store.start_writer(ident, "secret-a", 100, ttl=200) + assert not store.heartbeat_writer(ident, "secret-b", 110) + with pytest.raises(ManifestError, match="lease lost"): + store.finish(ident, hashlib.sha256(b"").hexdigest(), 110, writer_token="secret-b") + with pytest.raises(ManifestError, match="lease lost"): + store.finish(ident, hashlib.sha256(b"").hexdigest(), 110) + assert store.heartbeat_writer(ident, "secret-a", 150) + assert store.finish(ident, hashlib.sha256(b"").hexdigest(), 151, writer_token="secret-a").id == ident + + +def test_process_killed_after_telegram_send_cannot_publish_later(store, tmp_path): + backend = MemoryTelegram() + source = tmp_path / "source" + source.write_bytes(b"file-payload") + def forced_crash_fence(): + # Simulates an unavailable worker long enough for another process to + # fence its lease, after Telegram accepted the message. + store.abandon_stale_pending(now=int(__import__("time").time()) + 10000, + min_age_seconds=60) + backend.on_upload = forced_crash_fence + engine = TelegramObjectStorage(store, backend, chunk_size=4) + with pytest.raises(ManifestError, match="ownership lost"): + execute(engine.put_file("bucket", "item", source)) + assert len(backend.data) == 1 + assert store.interrupted()[0][1] == "failed" + with pytest.raises(FileNotFoundError): + store.get(store.interrupted()[0][0]) + + +def test_sql_journal_failure_after_telegram_ack_preserves_orphan_and_hides_object( + store, tmp_path, monkeypatch +): + """A Telegram ACK is not a commit; never publish an unjournaled chunk.""" + remote = MemoryTelegram() + source = tmp_path / "source-after-ack.bin" + source.write_bytes(b"test-transient-sql-commit-outage") + engine = TelegramObjectStorage(store, remote, chunk_size=128) + + def fail_append(*args, **kwargs): + raise OSError("simulated SQL journal outage") + + monkeypatch.setattr(store, "append", fail_append) + with pytest.raises(OSError, match="SQL journal outage"): + execute(engine.put_file("backup", "after-ack", source)) + + assert remote.uploads == 1 + assert len(remote.data) == 1, "Telegram orphan must not be deleted without ownership audit" + interrupted = store.interrupted() + assert len(interrupted) == 1 + ident, state = interrupted[0] + assert state == "failed" + assert store.recovery_parts(ident) == [] + with pytest.raises(FileNotFoundError): + store.get(ident) + # A new worker sees exactly the same failed manifest after a DB restart. + reopened = connect_database(str(tmp_path / "state.db")) + try: + fresh = ManifestStore(reopened) + assert fresh.interrupted() == [(ident, "failed")] + with pytest.raises(FileNotFoundError): + fresh.get(ident) + finally: + reopened.close() + + +def test_abandoned_writer_cannot_commit_after_database_restart(store, tmp_path): + """An interrupted worker's token stays fenced across separate SQL sessions.""" + now = 1700000000 + ident = store.begin("backup", "crash-fence", 0, now) + store.start_writer(ident, "original-worker", now, ttl=60) + after_crash = connect_database(str(tmp_path / "state.db")) + try: + challenger = ManifestStore(after_crash) + assert challenger.abandon_stale_pending( + now=now + 3601, min_age_seconds=60 + ) == [ident] + finally: + after_crash.close() + assert not store.heartbeat_writer( + ident, "original-worker", now + 3602, ttl=120 + ) + with pytest.raises(ManifestError): + store.finish( + ident, hashlib.sha256(b"").hexdigest(), + now + 3602, writer_token="original-worker", + ) + assert store.interrupted() == [(ident, "failed")] + + +@pytest.mark.parametrize("size", [ + pytest.param(5 * 1024 * 1024 + 1, id="5MiB-plus-one"), + pytest.param(16 * 1024 * 1024 + 37, id="16MiB-plus-37"), + pytest.param(19 * 1024 * 1024 + 7, id="19MiB-plus-7"), +]) +def test_large_single_telegram_chunk_survives_sql_restart_and_range_checks( + tmp_path, size +): + """Real SQL + independent byte transport at live Bot download boundaries. + + Regression for the 19 MiB + 7 byte live-E2E error: a hardcoded 19 MiB + chunk size accidentally produced two messages, so single-chunk tests + must use the complete payload size and assert the exact journal record. + """ + payload = os.urandom(size) + source = tmp_path / "near-bot-cap.bin" + source.write_bytes(payload) + backend = MemoryTelegram() + db_url = str(tmp_path / "near-cap.db") + db = connect_database(db_url) + try: + engine = TelegramObjectStorage( + ManifestStore(db), backend, chunk_size=size, + scratch_directory=tmp_path, + ) + item = execute(engine.put_file("boundary", "single-chunk", source)) + refs = engine.manifests.parts(item.id) + assert backend.uploads == 1 + assert len(refs) == 1 + assert refs[0].offset == 0 + assert refs[0].ordinal == 0 + assert refs[0].size == size + assert refs[0].sha256 == hashlib.sha256(payload).hexdigest() + finally: + db.close() + source.unlink() + + reopened = connect_database(db_url) + try: + restored = TelegramObjectStorage( + ManifestStore(reopened), backend, chunk_size=size, + scratch_directory=tmp_path, + ) + target = tmp_path / "restored-near-cap.bin" + execute(restored.download_to_path(item.id, target)) + assert target.read_bytes() == payload + + async def read_range(start, end): + return b"".join([ + block async for block in restored.iter_bytes( + item.id, start=start, end=end, + ) + ]) + + for start, end in ( + (0, 1), (size // 2, size // 2 + 29), + (size - 33, size), (size, size), + ): + assert execute(read_range(start, end)) == payload[start:end] + finally: + reopened.close() diff --git a/tests/storage/test_storage_gc_admin.py b/tests/storage/test_storage_gc_admin.py new file mode 100644 index 00000000..054a2a5d --- /dev/null +++ b/tests/storage/test_storage_gc_admin.py @@ -0,0 +1,138 @@ +"""Admin GC audit is read-only and protects copied Telegram data.""" +from __future__ import annotations + +import hashlib +import time + +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.copy import copy_object +from teledrive.storage.manifest import ManifestStore, ChunkReference, chunks +from sqlalchemy import func, insert, select + + +def test_admin_gc_audit_requires_auth_and_never_deletes_referenced_media(admin): + client, url, _ = admin + now = int(time.time()) + conn = connect_database(url) + try: + manifests = ManifestStore(conn) + catalog = BucketCatalog(manifests) + catalog.create_bucket("audit-bucket", now) + identifier = manifests.begin("audit-bucket", "original", 4, now) + manifests.append(identifier, ChunkReference( + ordinal=0, offset=0, size=4, + sha256=hashlib.sha256(b"data").hexdigest(), + account="bot", chat_id=-100123, message_id=101, + ), now) + manifests.finish(identifier, hashlib.sha256(b"data").hexdigest(), now) + catalog.publish(identifier, now=now) + alias = copy_object(catalog, "audit-bucket", "original", + "audit-bucket", "copy", now=now) + catalog.remove_current("audit-bucket", "original") + assert conn.connection.execute(select(func.count()) + .select_from(chunks)).scalar_one() == 1 + finally: + conn.close() + + path = "/api/v1/admin/storage/gc-audit" + assert client.get(path).status_code in (401, 403) + result = client.get(path + "?limit=1", auth=("testadmin", "secure-auth-password")) + assert result.status_code == 200, result.text + record = result.json() + assert record["dry_run"] is True + assert record["physical_deletion_enabled"] is False + assert record["eligible_for_deletion"] == 0 + assert record["total_manifests"] == 2 + assert record["total_telegram_messages"] == 1 + assert record["copy_aliases"] == 1 + assert record["live_object_heads"] == 1 + assert record["protected_chunks"][0]["copied_aliases"] == 1 + assert record["protected_chunks"][0]["live_heads"] == 1 + conn = connect_database(url) + try: + assert conn.connection.execute(select(func.count()) + .select_from(chunks)).scalar_one() == 1 + assert BucketCatalog(ManifestStore(conn)).get( + "audit-bucket", "copy" + ).item.id == alias.object_id + finally: + conn.close() + + +def test_gc_audit_invalid_limits_blocked_by_validation(admin): + client, _, _ = admin + for value in ("0", "1001", "-1", "abc"): + response = client.get( + "/api/v1/admin/storage/gc-audit?limit=" + value, + auth=("testadmin", "secure-auth-password"), + ) + assert response.status_code == 422 + + +def test_admin_gc_audit_counts_unjournaled_receipt_and_ambiguous_mtproto_send(admin): + """MTProto receipts are owned remote media even before a chunk journal exists.""" + from teledrive.storage.user_mode import UserSendJournal + + client, url, _ = admin + now = int(time.time()) + db = connect_database(url) + try: + manifests = ManifestStore(db) + catalog = BucketCatalog(manifests) + catalog.create_bucket("gc-ack-proof", now) + received = manifests.begin("gc-ack-proof", "receipt", 4, now - 86400) + unknown = manifests.begin("gc-ack-proof", "unknown", 4, now - 86400) + manifests.fail(received, "post-ACK SQL crash", now) + manifests.fail(unknown, "lost network response", now) + journal = UserSendJournal(db) + sha = hashlib.sha256(b"data").hexdigest() + peer = -100888777666555 + journal.start("test-ack", peer, received, 0, sha, 4) + journal.acknowledged("test-ack", peer, received, 0, 2001) + journal.start("test-ack", peer, unknown, 0, sha, 4) + assert journal.unresolved_count() == 1 + finally: + db.close() + + path = "/api/v1/admin/storage/gc-audit" + reply = client.get(path, auth=("testadmin", "secure-auth-password")) + assert reply.status_code == 200, reply.text + inventory = reply.json() + assert inventory["total_telegram_messages"] == 1 + assert inventory["unjournaled_mtproto_receipts"] == 1 + assert inventory["unresolved_mtproto_intents"] == 1 + assert inventory["eligible_for_deletion"] == 0 + assert inventory["dry_run"] is True + assert inventory["physical_deletion_enabled"] is False + + +def test_gc_audit_reports_active_upload_jobs_and_never_deletes(admin): + """Queued durable jobs are live references and appear in the admin audit.""" + from teledrive.storage.jobs import StorageUploadJobs, jobs + client, url, _ = admin + now = int(time.time()) + db = connect_database(url) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("gc-jobs-proof", now) + StorageUploadJobs(catalog) + db.connection.execute(insert(jobs).values( + id="queued-job-proof", bucket="gc-jobs-proof", object_key="a.bin", + source_path="/uploads/a.bin", source_size=4, source_device=1, + source_inode=2, source_mtime_ns=3, expected_generation=0, + expected_head_id=None, state="queued", object_id=None, attempts=0, + lease_token=None, lease_until=None, created_at=now, + updated_at=now, last_error=None, + )) + db.commit() + finally: + db.close() + response = client.get("/api/v1/admin/storage/gc-audit", + auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + report = response.json() + assert report["active_upload_jobs"] == 1 + assert report["eligible_for_deletion"] == 0 + assert report["physical_deletion_enabled"] is False diff --git a/tests/storage/test_storage_gc_audit.py b/tests/storage/test_storage_gc_audit.py new file mode 100644 index 00000000..9699d71c --- /dev/null +++ b/tests/storage/test_storage_gc_audit.py @@ -0,0 +1,68 @@ +"""Fail-closed GC reachability proof for real S3 object and multipart traffic.""" +from __future__ import annotations + +import pytest +from sqlalchemy import update + +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_copy import copy +from tests.s3.test_s3_multipart import begin, part +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore, object_aliases + + +def audit(service): + db = service[3]() + try: + return StorageGCInventory(ManifestStore(db)).inspect() + finally: + db.close() + + +def test_copy_alias_protects_original_telegram_chunks_after_source_delete(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source", b"immutable").status_code == 200 + assert copy(service, "backup-bucket/source", "/backup-bucket/clone").status_code == 200 + assert signed(service, "DELETE", "/backup-bucket/source").status_code == 204 + inventory = audit(service) + assert inventory.eligible_for_deletion == 0 + assert inventory.total_telegram_messages == 1 + assert inventory.copy_aliases == 1 + assert inventory.live_object_heads == 1 + assert inventory.protected_chunks[0].copied_aliases == 1 + assert inventory.protected_chunks[0].live_heads == 1 + assert signed(service, "GET", "/backup-bucket/clone").content == b"immutable" + + +def test_deleted_head_retains_history_and_uncommitted_multipart_part(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/old", b"historical-bytes").status_code == 200 + assert signed(service, "DELETE", "/backup-bucket/old").status_code == 204 + uid = begin(service, key="in-progress.bin") + part(service, uid, 1, b"staged-payload", key="in-progress.bin") + inventory = audit(service) + assert inventory.total_telegram_messages == 2 + assert inventory.multipart_parts == 1 + assert inventory.live_object_heads == 0 + assert len(inventory.protected_chunks) == 2 + assert sorted(x.multipart_parts for x in inventory.protected_chunks) == [0, 1] + assert inventory.eligible_for_deletion == 0 + + +def test_corrupt_alias_cycle_cannot_produce_deletion_inventory(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source", b"data").status_code == 200 + assert copy(service, "/backup-bucket/source", "/backup-bucket/clone").status_code == 200 + db = service[3]() + try: + manifests = ManifestStore(db) + cloned = BucketCatalog(manifests).get("backup-bucket", "clone").item + db.connection.execute(update(object_aliases).where( + object_aliases.c.object_id == cloned.id + ).values(source_object_id=cloned.id)) + db.commit() + finally: + db.close() + with pytest.raises(RuntimeError, match="corrupt"): + audit(service) diff --git a/tests/storage/test_storage_gc_retention.py b/tests/storage/test_storage_gc_retention.py new file mode 100644 index 00000000..4f9ef4fd --- /dev/null +++ b/tests/storage/test_storage_gc_retention.py @@ -0,0 +1,107 @@ +"""Retention-aware GC preview never physically deletes Telegram media.""" +from __future__ import annotations +import time +from hashlib import sha256 +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore + + +def test_failed_age_plan_protects_live_pending_and_ready(admin): + client, url, _ = admin + now = int(time.time()) + old = now - 9 * 86400 + db = connect_database(url) + try: + cat = BucketCatalog(ManifestStore(db)) + cat.create_bucket("gc-retained", now) + failed = cat.manifests.begin("gc-retained", "failed", 0, old) + cat.manifests.fail(failed, "interrupted Telegram send", old) + pending = cat.manifests.begin("gc-retained", "pending", 0, old) + ready = cat.manifests.begin("gc-retained", "ready", 0, old) + cat.manifests.finish(ready, sha256(b"").hexdigest(), old) + cat.publish(ready, now=now) + result = StorageGCInventory(cat.manifests).preview_failed_retention(now=now) + assert [x["object_id"] for x in result] == [failed] + assert not result[0]["approved_for_deletion"] + assert result[0]["remote_history_scan_required"] + assert not {pending, ready}.intersection(x["object_id"] for x in result) + finally: + db.close() + endpoint = "/api/v1/admin/storage/gc-candidates" + assert client.get(endpoint).status_code in (401, 403) + assert client.get(endpoint, auth=("testadmin", "secure-auth-password"), + params={"min_age_seconds": 60}).status_code == 422 + response = client.get(endpoint, auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + assert response.json()["dry_run"] is True + assert response.json()["physical_deletion_enabled"] is False + assert response.json()["candidates"][0]["object_id"] == failed + + + +def test_gc_preview_fences_unresolved_send_and_unjournaled_mtproto_receipt(admin): + """Unjournaled real MTProto messages must never become apparent GC candidates.""" + from teledrive.storage.user_mode import UserSendJournal + + _, url, _ = admin + now = int(time.time()) + old = now - 12 * 86400 + db = connect_database(url) + try: + manifest = ManifestStore(db) + cat = BucketCatalog(manifest) + cat.create_bucket("mtproto-gc-receipt", now) + first = manifest.begin("mtproto-gc-receipt", "lost-ack", 4, old) + second = manifest.begin("mtproto-gc-receipt", "received-ack", 4, old) + third = manifest.begin("mtproto-gc-receipt", "pure-failed", 4, old) + for ident in (first, second, third): + manifest.fail(ident, "interrupted send", old) + + journal = UserSendJournal(db) + digest = sha256(b"data").hexdigest() + journal.start("user-gc", -100888777666555, first, 0, digest, 4) + journal.start("user-gc", -100888777666555, second, 0, digest, 4) + journal.acknowledged("user-gc", -100888777666555, second, 0, 42) + inventory = StorageGCInventory(manifest).preview_failed_retention( + now=now, + ) + by_id = {candidate["object_id"]: candidate for candidate in inventory} + assert set(by_id) == {first, second, third} + assert by_id[first]["unresolved_mtproto_intents"] == 1 + assert by_id[first]["reference_blocked"] is True + assert by_id[second]["unresolved_mtproto_intents"] == 0 + assert by_id[second]["reference_blocked"] is True + assert by_id[third]["unresolved_mtproto_intents"] == 0 + assert by_id[third]["reference_blocked"] is False + assert all(not candidate["approved_for_deletion"] + for candidate in inventory) + assert all(candidate["remote_history_scan_required"] + for candidate in inventory) + finally: + db.close() + + +def test_gc_preview_fails_closed_on_incomplete_reference_schema(admin): + """A missing required reference table cannot be interpreted as no reference.""" + client, url, _ = admin + now = int(time.time()) + old = now - 9 * 86400 + db = connect_database(url) + try: + manifests = ManifestStore(db) + cat = BucketCatalog(manifests) + cat.create_bucket("incomplete-gc-schema", now) + ident = manifests.begin("incomplete-gc-schema", "failed", 1, old) + manifests.fail(ident, "old interruption", old) + db.connection.exec_driver_sql( + "DROP TABLE storage_published_versions" + ) + db.commit() + import pytest + with pytest.raises(RuntimeError, match="Incomplete GC reference"): + StorageGCInventory(manifests).preview_failed_retention(now=now) + finally: + db.close() diff --git a/tests/storage/test_storage_jobs.py b/tests/storage/test_storage_jobs.py new file mode 100644 index 00000000..6875cff4 --- /dev/null +++ b/tests/storage/test_storage_jobs.py @@ -0,0 +1,383 @@ +"""Durable upload queue with lease fencing, root confinement and restart retry.""" +from __future__ import annotations + +import asyncio +import os +import time + +import pytest + +from tests.storage.test_storage_engine import MemoryTelegram +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.jobs import StorageUploadJobs, UploadJobError, jobs +from teledrive.storage.manifest import ManifestStore +import web.app as webapp + + +@pytest.fixture +def queue(tmp_path): + source_root = tmp_path / "allowed" + source_root.mkdir() + db_path = str(tmp_path / "metadata.sqlite") + conn = connect_database(db_path) + catalog = BucketCatalog(ManifestStore(conn)) + catalog.create_bucket("job-bucket", int(time.time())) + try: + yield StorageUploadJobs(catalog), source_root, db_path + finally: + conn.close() + + +def test_job_roundtrip_survives_sql_reopen_and_prevents_double_publish(queue): + first, root, path = queue + data = bytes(range(250)) * 17 + source = root / "payload.bin" + source.write_bytes(data) + ident = first.enqueue("job-bucket", "backup/data.bin", source, allowed_root=root) + assert first.list_jobs()[0]["id"] == ident + assert "lease_token" not in first.list_jobs()[0] + assert first.list_jobs()[0]["expected_generation"] == 0 + remote = MemoryTelegram() + one = connect_database(path) + try: + runner = StorageUploadJobs(BucketCatalog(ManifestStore(one))) + result = asyncio.run(runner.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=2048, + )) + assert result["id"] == ident and result["state"] == "done" + assert remote.uploads == 3 + finally: + one.close() + two = connect_database(path) + try: + runner = StorageUploadJobs(BucketCatalog(ManifestStore(two))) + assert runner.run_next is not None + assert asyncio.run(runner.run_next( + remote, allowed_root=root, scratch_directory=root, + )) is None + obj = runner.catalog.get("job-bucket", "backup/data.bin").item + assert obj.id == result["object_id"] + restored = root / "restored.bin" + asyncio.run(TelegramObjectStorage( + runner.catalog.manifests, remote + ).download_to_path(obj.id, restored)) + assert restored.read_bytes() == data + assert runner.list_jobs()[0]["state"] == "done" + finally: + two.close() + + +def test_upload_jobs_reject_outside_root_symlinks_and_changed_source(queue, tmp_path): + manager, root, _ = queue + outside = tmp_path / "not-allowed.bin" + outside.write_bytes(b"secret") + with pytest.raises(UploadJobError, match="outside"): + manager.enqueue("job-bucket", "secret", outside, allowed_root=root) + link = root / "linked.bin" + link.symlink_to(outside) + with pytest.raises(UploadJobError, match="Symlink"): + manager.enqueue("job-bucket", "secret", link, allowed_root=root) + source = root / "upload.bin" + source.write_bytes(b"old") + identifier = manager.enqueue("job-bucket", "new", source, allowed_root=root) + source.write_bytes(b"changed source") + remote = MemoryTelegram() + with pytest.raises(UploadJobError, match="Source changed"): + asyncio.run(manager.run_next(remote, allowed_root=root, scratch_directory=root)) + assert remote.uploads == 0 + assert manager.list_jobs()[0]["state"] == "failed" + assert manager.retry(identifier) + assert not manager.retry(identifier) + assert manager.list_jobs()[0]["state"] == "queued" + + +def test_staged_job_restarts_without_reupload(queue): + manager, root, path = queue + source = root / "stage.bin" + source.write_bytes(b"staged" * 32) + identifier = manager.enqueue("job-bucket", "staged.bin", source, allowed_root=root) + claim = manager.claim_next(now=int(time.time()), ttl=120) + assert claim["id"] == identifier + remote = MemoryTelegram() + engine = TelegramObjectStorage(manager.catalog.manifests, remote, chunk_size=64, + scratch_directory=root) + ready = asyncio.run(engine.put_file("job-bucket", "staged.bin", source)) + assert manager.mark_staged(identifier, claim["lease_token"], ready.id, + now=int(time.time())) + sent_before_restart = remote.uploads + # Simulate a hard-killed runner: the persisted READY manifest and job + # staging record survive, but its owner lease expires. + manager.db.connection.execute( + jobs.update().where(jobs.c.id == identifier).values(lease_until=0) + ) + manager.db.commit() + db = connect_database(path) + try: + resumed = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + outcome = asyncio.run(resumed.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=64, + )) + assert outcome == {"id": identifier, "state": "done", "object_id": ready.id} + assert remote.uploads == sent_before_restart + assert resumed.catalog.get("job-bucket", "staged.bin").item.id == ready.id + assert resumed.list_jobs()[0]["state"] == "done" + finally: + db.close() + + +def test_one_worker_lease_and_head_generation_precondition(queue): + manager, root, path = queue + source = root / "cas.bin" + source.write_bytes(b"delayed") + identifier = manager.enqueue("job-bucket", "cas.bin", source, allowed_root=root) + claim = manager.claim_next(now=int(time.time()), ttl=120) + assert claim and claim["id"] == identifier + assert manager.claim_next() is None + assert not manager.renew(identifier, "not-the-token", now=int(time.time())) + assert manager.renew(identifier, claim["lease_token"], now=int(time.time())) + manager.fail(identifier, claim["lease_token"], "injected failure", now=int(time.time())) + assert manager.retry(identifier) + + # Another writer creates the key after enqueue. The old job cannot + # replace it because expected_generation=0 is checked at final publish. + external = manager.catalog.manifests.begin( + "job-bucket", "cas.bin", 0, int(time.time()) + ) + from hashlib import sha256 + manager.catalog.manifests.finish(external, sha256(b"").hexdigest(), int(time.time())) + manager.catalog.publish(external, now=int(time.time())) + with pytest.raises(ValueError, match="generation precondition"): + asyncio.run(manager.run_next( + MemoryTelegram(), allowed_root=root, + scratch_directory=root, chunk_size=4, + )) + assert manager.catalog.get("job-bucket", "cas.bin").item.id == external + assert manager.list_jobs()[0]["state"] == "failed" + + +def test_inactive_jobs_can_be_cancelled_without_upload_or_losing_history(queue): + manager, root, path = queue + source = root / "cancel.bin" + source.write_bytes(b"safe") + first = manager.enqueue("job-bucket", "cancelled.bin", source, allowed_root=root) + assert manager.cancel(first) is True + assert manager.cancel(first) is False + assert manager.retry(first) is False + assert manager.claim_next() is None + + second = manager.enqueue("job-bucket", "claimed.bin", source, allowed_root=root) + owner = manager.claim_next() + assert owner and owner["id"] == second + assert not manager.cancel(second), "Never cancel an active writer" + assert manager.fail(second, owner["lease_token"], "planned fail", now=int(time.time())) + assert manager.cancel(second) + assert not manager.retry(second) + + reopened = connect_database(path) + try: + jobs_after_restart = StorageUploadJobs(BucketCatalog(ManifestStore(reopened))) + states = {item["id"]: item["state"] for item in jobs_after_restart.list_jobs()} + assert states == {first: "cancelled", second: "cancelled"} + assert jobs_after_restart.claim_next() is None + assert jobs_after_restart.catalog.list_buckets() == ["job-bucket"] + finally: + reopened.close() + + +def test_admin_job_endpoints_require_root_auth_csrf_and_do_not_expose_tokens( + admin, monkeypatch, tmp_path +): + client, db_path, _ = admin + root = tmp_path / "uploads" + root.mkdir() + source = root / "actual.bin" + source.write_bytes(b"safe-bot-upload") + conn = connect_database(db_path) + try: + BucketCatalog(ManifestStore(conn)).create_bucket("job-admin", int(time.time())) + finally: + conn.close() + url = "/api/v1/admin/storage/jobs" + auth = ("testadmin", "secure-auth-password") + assert client.get(url).status_code in (401, 403) + payload = {"bucket": "job-admin", "key": "single.bin", + "source_path": str(source)} + monkeypatch.delenv("TELEDRIVE_JOB_SOURCE_ROOT", raising=False) + assert client.post(url, auth=auth, json=payload).status_code == 403 + assert client.post(url, auth=auth, json=payload, + headers={"x-teledrive-csrf": "1"}).status_code == 503 + monkeypatch.setattr(webapp, "JOB_SOURCE_ROOT", str(root)) + monkeypatch.setattr(webapp, "STORAGE_SCRATCH_DIR", str(tmp_path / "scratch")) + start = client.post(url, auth=auth, json=payload, + headers={"x-teledrive-csrf": "1"}) + assert start.status_code == 201, start.text + entry = client.get(url, auth=auth) + assert entry.status_code == 200 and "lease_token" not in entry.text + remote = MemoryTelegram() + monkeypatch.setattr(webapp, "storage_transport", lambda db: remote) + run = client.post(url + "/run-next", auth=auth, + headers={"x-teledrive-csrf": "1"}) + assert run.status_code == 200, run.text + assert run.json()["job"]["state"] == "done" + assert remote.uploads == 1 + assert client.get(url, auth=auth).json()["jobs"][0]["state"] == "done" + retry = client.post(url + "/" + start.json()["id"] + "/retry", + auth=auth, headers={"x-teledrive-csrf": "1"}) + assert retry.status_code == 409 + + +def test_cancel_job_endpoint_requires_root_and_csrf_and_fences_active_jobs( + admin, monkeypatch, tmp_path +): + client, db_path, _ = admin + root = tmp_path / "allowed-cancel" + root.mkdir() + source = root / "one.bin" + source.write_bytes(b"must-not-be-uploaded") + conn = connect_database(db_path) + try: + BucketCatalog(ManifestStore(conn)).create_bucket("cancel-admin", int(time.time())) + finally: + conn.close() + monkeypatch.setattr(webapp, "JOB_SOURCE_ROOT", str(root)) + auth = ("testadmin", "secure-auth-password") + response = client.post("/api/v1/admin/storage/jobs", + auth=auth, headers={"x-teledrive-csrf": "1"}, + json={"bucket": "cancel-admin", "key": "one.bin", + "source_path": str(source)}) + assert response.status_code == 201, response.text + job_id = response.json()["id"] + url = f"/api/v1/admin/storage/jobs/{job_id}/cancel" + assert client.post(url).status_code in (401, 403) + assert client.post(url, auth=auth).status_code == 403 + success = client.post(url, auth=auth, headers={"x-teledrive-csrf": "1"}) + assert success.status_code == 200, success.text + assert success.json() == {"id": job_id, "state": "cancelled"} + assert client.post(url, auth=auth, + headers={"x-teledrive-csrf": "1"}).status_code == 409 + assert client.get("/api/v1/admin/storage/jobs", auth=auth).json()["jobs"][0]["state"] == "cancelled" + assert client.post("/api/v1/admin/storage/jobs/" + job_id + "/retry", + auth=auth, headers={"x-teledrive-csrf": "1"}).status_code == 409 + + +def _simulate_crash_after_first_journaled_chunk(manager, root, remote): + """Leave the SQL job and object with expired independent worker fences.""" + import hashlib + from sqlalchemy import update + from teledrive.storage.manifest import ChunkReference, writer_fences + + payload = bytes(range(31)) + source = root / "resume-source.bin" + source.write_bytes(payload) + job_id = manager.enqueue("job-bucket", "resumable.bin", source, allowed_root=root) + claim = manager.claim_next() + assert claim["id"] == job_id + now = int(time.time()) + manifests = manager.catalog.manifests + identifier = manifests.begin("job-bucket", "resumable.bin", len(payload), now) + manifests.start_writer(identifier, "crashed-owner", now, ttl=120) + assert manager.mark_pending(job_id, claim["lease_token"], identifier, now=now) + first = root / "sent-before-crash.bin" + first.write_bytes(payload[:5]) + checksum = hashlib.sha256(payload[:5]).hexdigest() + account, chat, message = asyncio.run(remote.upload( + first, object_id=identifier, ordinal=0, checksum=checksum, + )) + part = ChunkReference(0, 0, 5, checksum, account, chat, message) + manifests.append(identifier, part, now, writer_token="crashed-owner") + manager.db.connection.execute( + update(writer_fences).where(writer_fences.c.object_id == identifier) + .values(lease_until=0) + ) + manager.db.connection.execute( + update(jobs).where(jobs.c.id == job_id).values(lease_until=0) + ) + manager.db.commit() + return job_id, identifier, payload, part + + +def test_resume_journaled_chunks_after_crash_and_db_restart(queue): + manager, root, db_path = queue + remote = MemoryTelegram() + job_id, object_id, payload, preserved = _simulate_crash_after_first_journaled_chunk( + manager, root, remote, + ) + original_uploads = remote.uploads + assert original_uploads == 1 + assert manager.catalog.manifests.recovery_parts(object_id) == [preserved] + + new_connection = connect_database(db_path) + try: + runner = StorageUploadJobs(BucketCatalog(ManifestStore(new_connection))) + result = asyncio.run(runner.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=5, + )) + assert result == {"id": job_id, "state": "done", "object_id": object_id} + assert remote.uploads == 1 + (len(payload) - 5 + 4) // 5 + assert remote.downloads == 1, "Old Telegram chunk must be verified" + refs = runner.catalog.manifests.parts(object_id) + assert refs[0] == preserved + assert len(refs) == remote.uploads + assert runner.catalog.get("job-bucket", "resumable.bin").item.id == object_id + restored = root / "resumed-output.bin" + asyncio.run(TelegramObjectStorage( + runner.catalog.manifests, remote + ).download_to_path(object_id, restored)) + assert restored.read_bytes() == payload + assert runner.list_jobs()[0]["state"] == "done" + finally: + new_connection.close() + + +def test_resume_corrupt_remote_fails_closed_and_retry_preserves_orphan(queue): + manager, root, db_path = queue + remote = MemoryTelegram() + job_id, object_id, payload, preserved = _simulate_crash_after_first_journaled_chunk( + manager, root, remote, + ) + remote.data[preserved.account, preserved.chat_id, preserved.message_id] = b"bad!" + reopened = connect_database(db_path) + try: + runner = StorageUploadJobs(BucketCatalog(ManifestStore(reopened))) + from teledrive.storage.engine import IntegrityError + with pytest.raises(IntegrityError): + asyncio.run(runner.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=5, + )) + assert remote.uploads == 1 + assert runner.list_jobs()[0]["state"] == "failed" + assert runner.catalog.manifests.recovery_parts(object_id) == [preserved] + with pytest.raises(FileNotFoundError): + runner.catalog.get("job-bucket", "resumable.bin") + assert runner.retry(job_id) + assert runner.list_jobs()[0]["object_id"] is None + fixed = asyncio.run(runner.run_next( + remote, allowed_root=root, scratch_directory=root, chunk_size=5, + )) + assert fixed["state"] == "done" + assert fixed["object_id"] != object_id + assert (preserved.account, preserved.chat_id, preserved.message_id) in remote.data + restored = root / "after-retry.bin" + asyncio.run(TelegramObjectStorage( + runner.catalog.manifests, remote + ).download_to_path(fixed["object_id"], restored)) + assert restored.read_bytes() == payload + finally: + reopened.close() + + +def test_pending_writer_reclaim_rejects_live_and_unfenced_manifests(queue): + manager, root, _ = queue + now = int(time.time()) + store = manager.catalog.manifests + identifier = store.begin("job-bucket", "guard.bin", 0, now) + store.start_writer(identifier, "current", now, ttl=120) + assert not store.reclaim_writer(identifier, "intruder", now) + assert store.heartbeat_writer(identifier, "current", now) + legacy = store.begin("job-bucket", "legacy-guard.bin", 0, now - 86400) + assert not store.reclaim_writer(legacy, "intruder", now) + assert store.reclaim_writer(identifier, "recovered", now + 121) + assert not store.heartbeat_writer(identifier, "current", now + 122) diff --git a/tests/storage/test_storage_multiworker.py b/tests/storage/test_storage_multiworker.py new file mode 100644 index 00000000..de13a2ec --- /dev/null +++ b/tests/storage/test_storage_multiworker.py @@ -0,0 +1,154 @@ +"""Independent OS workers share one SQL queue without double-claiming jobs. + +No Telegram credentials are used: a disk-backed fake is process-shared and +every reconstructed payload is verified through the real chunk engine. +""" +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +import time +from pathlib import Path + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore + +WORKER = r''' +import asyncio +import secrets +import sys +import time +from pathlib import Path +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore + +class SharedRemote: + def __init__(self, root): + self.root = root + + async def upload(self, source, *, object_id, ordinal, checksum): + mid = secrets.randbits(62) + 1 + (self.root / f"{mid}.bin").write_bytes(source.read_bytes()) + return "process-remote", -100987654321, mid + + async def download(self, chunk, destination): + destination.write_bytes((self.root / f"{chunk.message_id}.bin").read_bytes()) + +async def work(): + path, root, remote, ready, go = map(Path, sys.argv[1:]) + db = connect_database(str(path)) + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(db))) + ready.write_text("ready", encoding="ascii") + for _ in range(500): + if go.exists(): + break + await asyncio.sleep(0.02) + else: + raise RuntimeError("Start signal was not received") + for _ in range(30): + outcome = await manager.run_next( + SharedRemote(remote), allowed_root=root, + scratch_directory=root, chunk_size=37, + ) + if outcome is None: + return + raise RuntimeError("Unexpected excessive worker claims") + finally: + db.close() + +asyncio.run(work()) +''' + + +class SharedRemote: + def __init__(self, directory: Path): + self.directory = directory + + async def download(self, ref, destination): + destination.write_bytes( + (self.directory / f"{ref.message_id}.bin").read_bytes() + ) + + +def test_independent_sqlite_workers_soak_claims_and_restore_all_bytes(tmp_path): + root = tmp_path / "source" + root.mkdir() + remote = tmp_path / "remote" + remote.mkdir() + db_path = tmp_path / "metadata.sqlite" + db = connect_database(str(db_path)) + payloads = {} + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("multi-worker", int(time.time())) + queue = StorageUploadJobs(catalog) + for n in range(12): + content = bytes([n]) * (91 + 13 * n) + file = root / f"file-{n:02d}.bin" + file.write_bytes(content) + payloads[f"objects/{n:02d}.bin"] = content + queue.enqueue( + "multi-worker", f"objects/{n:02d}.bin", file, + allowed_root=root, + ) + finally: + db.close() + + go = tmp_path / "start-workers" + processes = [] + try: + for index in range(3): + ready = tmp_path / f"worker-{index}.ready" + proc = subprocess.Popen( + [sys.executable, "-c", WORKER, + str(db_path), str(root), str(remote), str(ready), str(go)], + cwd=Path(__file__).resolve().parents[2], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + processes.append((proc, ready)) + deadline = time.monotonic() + 22 + while not all(ready.exists() for _, ready in processes): + assert time.monotonic() < deadline, "Workers did not start" + for proc, _ in processes: + assert proc.poll() is None, "Worker exited before start gate" + time.sleep(0.04) + go.write_text("go", encoding="ascii") + for proc, _ in processes: + out, err = proc.communicate(timeout=50) + assert proc.returncode == 0, ( + f"Worker exit {proc.returncode}: {err.decode(errors='replace')[-3500:]}" + ) + finally: + for proc, _ in processes: + if proc.poll() is None: + proc.kill() + proc.communicate(timeout=10) + + reopened = connect_database(str(db_path)) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + states = StorageUploadJobs(catalog).list_jobs(limit=1000) + assert len(states) == 12 + assert all(j["state"] == "done" and j["attempts"] == 1 for j in states) + assert len({j["object_id"] for j in states}) == 12 + expected_remote_files = 0 + for key, data in payloads.items(): + obj = catalog.get("multi-worker", key).item + refs = catalog.manifests.parts(obj.id) + expected_remote_files += len(refs) + restored = tmp_path / (key.replace("/", "-")) + asyncio.run(TelegramObjectStorage( + catalog.manifests, SharedRemote(remote), chunk_size=37 + ).download_to_path(obj.id, restored)) + assert restored.read_bytes() == data + assert len(list(remote.glob("*.bin"))) == expected_remote_files + finally: + reopened.close() diff --git a/tests/storage/test_storage_ops_health.py b/tests/storage/test_storage_ops_health.py new file mode 100644 index 00000000..226cf157 --- /dev/null +++ b/tests/storage/test_storage_ops_health.py @@ -0,0 +1,252 @@ +"""Operator health and durable worker schedule require explicit opt-in.""" +from __future__ import annotations + +import time + +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore + + +def test_operator_storage_health_is_authenticated_and_fail_safe(admin, tmp_path): + client, path, _ = admin + root = tmp_path / "allowed" + root.mkdir() + source = root / "file.bin" + source.write_bytes(b"operator-health") + db = connect_database(path) + try: + cat = BucketCatalog(ManifestStore(db)) + cat.create_bucket("health-ci", int(time.time())) + queue = StorageUploadJobs(cat) + identifier = queue.enqueue( + "health-ci", "file.bin", source, allowed_root=root, + ) + finally: + db.close() + + endpoint = "/api/v1/admin/storage/health" + assert client.get(endpoint).status_code in (401, 403) + response = client.get(endpoint, auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + data = response.json() + assert data["job_states"]["queued"] >= 1 + assert data["unresolved_alert_count"] == 0 + assert data["alerts"] == [] + assert data["stale_pending_sample_count"] == 0 + assert data["telegram_messages_retained"] == 0 + assert data["physical_deletion_enabled"] is False + assert "lease_token" not in response.text + assert "secret_key" not in response.text + assert identifier not in response.text + + +def test_operator_health_alerts_failed_jobs_and_stale_pending_without_secrets(admin, tmp_path): + client, path, _ = admin + now = int(time.time()) + allowed = tmp_path / "alerts" + allowed.mkdir() + source = allowed / "payload.bin" + source.write_bytes(b"metadata-only") + connection = connect_database(path) + try: + catalog = BucketCatalog(ManifestStore(connection)) + catalog.create_bucket("alert-ci", now) + queue = StorageUploadJobs(catalog) + ident = queue.enqueue("alert-ci", "failure.bin", source, allowed_root=allowed, now=now) + worker = queue.claim_next(now=now) + assert worker and worker["id"] == ident + assert queue.fail(ident, worker["lease_token"], "unrecoverable", now=now) + stale = catalog.manifests.begin("alert-ci", "pending.bin", 0, now - 7300) + catalog.manifests.start_writer(stale, "private-writer-token", now-7300, ttl=120) + finally: + connection.close() + denied = client.get("/api/v1/admin/storage/health") + assert denied.status_code in (401, 403) + response = client.get("/api/v1/admin/storage/health", + auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + payload = response.json() + by_code = {entry["code"]: entry for entry in payload["alerts"]} + assert by_code["failed_upload_jobs"]["count"] == 1 + assert by_code["stale_pending_uploads"]["count"] == 1 + assert payload["unresolved_alert_count"] == len(by_code) + assert "private-writer-token" not in response.text + assert ident not in response.text + assert stale not in response.text + assert payload["physical_deletion_enabled"] is False + + +def test_operator_health_counts_unjournaled_mtproto_receipts_without_exposing_ids(admin): + """Only acknowledged sends absent from chunk journals produce a warning.""" + import hashlib + from teledrive.storage.manifest import ChunkReference + from teledrive.storage.user_mode import UserSendJournal + + client, path, _ = admin + db = connect_database(path) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("mtproto-ack-alert", int(time.time())) + identifier = catalog.manifests.begin( + "mtproto-ack-alert", "only-in-receipts.bin", 5, int(time.time()), + ) + digest = hashlib.sha256(b"hello").hexdigest() + journal = UserSendJournal(db) + journal.start("mtproto-ops-ci", -100987654321, identifier, 0, + digest, 5) + journal.acknowledged("mtproto-ops-ci", -100987654321, identifier, 0, + 101) + finally: + db.close() + + url = "/api/v1/admin/storage/health" + response = client.get(url, auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + assert response.json()["unjournaled_mtproto_receipt_count"] == 1 + alerts = {row["code"]: row for row in response.json()["alerts"]} + assert alerts["unjournaled_mtproto_receipts"]["count"] == 1 + assert identifier not in response.text + assert response.json()["physical_deletion_enabled"] is False + + db = connect_database(path) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.manifests.append( + identifier, ChunkReference( + ordinal=0, offset=0, size=5, sha256=digest, + account="mtproto-ops-ci", chat_id=-100987654321, message_id=101, + ), int(time.time()), + ) + finally: + db.close() + resolved = client.get(url, auth=("testadmin", "secure-auth-password")) + assert resolved.status_code == 200, resolved.text + assert resolved.json()["unjournaled_mtproto_receipt_count"] == 0 + assert "unjournaled_mtproto_receipts" not in { + row["code"] for row in resolved.json()["alerts"] + } + +def test_operator_health_warns_on_unresolved_mtproto_send_without_identifiers(admin): + """Ambiguous sends must be visible but session/object ID stay private.""" + import hashlib + from teledrive.storage.user_mode import UserSendJournal + + client, path, _ = admin + db = connect_database(path) + try: + idx = UserSendJournal(db) + payload = b"unknown-channel-acceptance" + idx.start( + "private-operator-user", -10098765432, "private-object-identifier", + 3, hashlib.sha256(payload).hexdigest(), len(payload), + ) + finally: + db.close() + url = "/api/v1/admin/storage/health" + assert client.get(url).status_code in (401, 403) + response = client.get(url, auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + body = response.json() + assert body["unresolved_mtproto_send_count"] == 1 + alerts = {entry["code"]: entry for entry in body["alerts"]} + assert alerts["unresolved_mtproto_sends"]["severity"] == "critical" + assert alerts["unresolved_mtproto_sends"]["count"] == 1 + for secret in ("private-operator-user", "private-object-identifier", + "-10098765432", hashlib.sha256(payload).hexdigest()): + assert secret not in response.text + + # Definitive non-acceptance clears only the corresponding intent. + db = connect_database(path) + try: + UserSendJournal(db).clear_rejected_send( + "private-operator-user", -10098765432, "private-object-identifier", + 3, hashlib.sha256(payload).hexdigest(), len(payload), + ) + finally: + db.close() + again = client.get(url, auth=("testadmin", "secure-auth-password")) + assert again.status_code == 200 + assert again.json()["unresolved_mtproto_send_count"] == 0 + assert "unresolved_mtproto_sends" not in { + entry["code"] for entry in again.json()["alerts"] + } + +def test_operator_unresolved_mtproto_inventory_requires_auth_and_has_no_side_effects(admin): + """Read-only triage explicitly exposes checksum/size to authorized admin only.""" + import hashlib + from teledrive.storage.user_mode import UserSendJournal + + client, path, _ = admin + db = connect_database(path) + try: + index = UserSendJournal(db) + index.start( + "user-a", -100123456, "object-inflight-a", 0, + hashlib.sha256(b"first").hexdigest(), 5, + ) + index.start( + "user-b", -100123457, "object-inflight-b", 1, + hashlib.sha256(b"second").hexdigest(), 6, + ) + finally: + db.close() + endpoint = "/api/v1/admin/storage/mtproto-unresolved" + assert client.get(endpoint).status_code in (401, 403) + unauthorized = client.get(endpoint, auth=("testadmin", "not-correct")) + assert unauthorized.status_code in (401, 403) + for limit in ("0", "1001", "-1", "not-an-integer"): + assert client.get(endpoint + "?limit=" + limit, + auth=("testadmin", "secure-auth-password")).status_code == 422 + response = client.get(endpoint + "?limit=1", + auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + content = response.json() + assert content["total"] == 2 + assert len(content["intents"]) == 1 + assert content["intents"][0]["account"] == "user-a" + assert content["intents"][0]["ordinal"] == 0 + assert content["intents"][0]["size"] == 5 + assert len(content["intents"][0]["sha256"]) == 64 + assert "independently verify" in content["message"].lower() + for forbidden in ("session", "file_id", "secret_key", + "lease_token", "123456:"): + assert forbidden not in response.text + full = client.get(endpoint, auth=("testadmin", "secure-auth-password")) + assert full.status_code == 200 + assert len(full.json()["intents"]) == 2 + assert full.json()["total"] == 2 + # Read-only inventory must not remove/fabricate receipts or send media. + db = connect_database(path) + try: + idx = UserSendJournal(db) + assert idx.unresolved_count() == 2 + finally: + db.close() + + +def test_operator_health_exposes_only_aggregate_explicit_flood_total(admin): + """Cross-host flood evidence has no chat IDs, account aliases, or tokens.""" + from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard + + client, url, _ = admin + conn = connect_database(url) + try: + guard = MtprotoRateGuard(conn) + guard.reserve_send_slot("private-flood-account", -100987654321, 0, + now_ms=1000) + guard.impose_shared_cooldown( + "private-flood-account", -100987654321, 9, now_ms=1000, + ) + assert guard.total_explicit_flood_observations() == 1 + finally: + conn.close() + endpoint = "/api/v1/admin/storage/health" + assert client.get(endpoint).status_code in (401, 403) + response = client.get(endpoint, auth=("testadmin", "secure-auth-password")) + assert response.status_code == 200, response.text + assert response.json()["mtproto_explicit_flood_total"] == 1 + for sensitive in ("private-flood-account", "-100987654321", "session"): + assert sensitive not in response.text diff --git a/tests/storage/test_storage_pending_admin.py b/tests/storage/test_storage_pending_admin.py new file mode 100644 index 00000000..38c1497e --- /dev/null +++ b/tests/storage/test_storage_pending_admin.py @@ -0,0 +1,128 @@ +"""P1 operator-safe PENDING recovery: dry-run, per-object SQL fencing and CSRF.""" +from __future__ import annotations + +import time + +from sqlalchemy import select + +from tests.s3.test_s3_admin import admin +from teledrive.db import connect_database +from teledrive.storage.manifest import ManifestStore, writer_fences + + +def create(url, *, age=7200, ttl=120, leased=True): + now = int(time.time()) - age + db = connect_database(url) + try: + store = ManifestStore(db) + identifier = store.begin("backups", "object.bin", 0, now) + if leased: + store.start_writer(identifier, f"owner-{identifier}", now, ttl=ttl) + return identifier, f"owner-{identifier}" + finally: + db.close() + + +def test_pending_recovery_is_authenticated_csrf_protected_and_dry_run(admin): + client, url, _ = admin + stale, token = create(url) + live, live_token = create(url, age=10, ttl=300) + legacy, _ = create(url, leased=False) + uri = "/api/v1/admin/storage/pending-stale" + auth = ("testadmin", "secure-auth-password") + assert client.get(uri).status_code in (401, 403) + result = client.get(uri, auth=auth) + assert result.status_code == 200, result.text + data = result.json() + assert data["dry_run"] is True + assert data["physical_deletion_enabled"] is False + assert [item["id"] for item in data["uploads"]] == [stale] + assert "token" not in result.text + assert data["uploads"][0]["journaled_chunks"] == 0 + assert client.get(uri, auth=auth, params={"min_age_seconds": 10}).status_code == 422 + + endpoint = f"{uri}/{stale}/abandon" + assert client.post(endpoint, json={"execute": True}).status_code in (401, 403) + assert client.post(endpoint, auth=auth, json={"execute": True}).status_code == 403 + headers = {"X-TeleDrive-CSRF": "1"} + preview = client.post(endpoint, auth=auth, headers=headers, json={}) + assert preview.status_code == 200 + assert preview.json() == { + "object_id": stale, "eligible": True, "abandoned": False, "dry_run": True, + } + db = connect_database(url) + try: + store = ManifestStore(db) + assert dict(store.interrupted())[stale] == "pending" + assert store.heartbeat_writer(stale, token, int(time.time())) is False + finally: + db.close() + + executed = client.post(endpoint, auth=auth, headers=headers, + json={"execute": True}) + assert executed.status_code == 200 + assert executed.json() == { + "object_id": stale, "eligible": True, "abandoned": True, "dry_run": False, + } + retry = client.post(endpoint, auth=auth, headers=headers, json={"execute": True}) + assert retry.json()["abandoned"] is False + db = connect_database(url) + try: + store = ManifestStore(db) + assert dict(store.interrupted()) == { + stale: "failed", live: "pending", legacy: "pending" + } + assert store.heartbeat_writer(live, live_token, int(time.time())) + assert store.heartbeat_writer(stale, token, int(time.time())) is False + assert len(store.stale_pending(now=int(time.time()), limit=100)) == 0 + finally: + db.close() + + +def test_competing_heartbeat_survives_and_second_worker_cannot_reanimate(admin): + client, url, _ = admin + target, old = create(url) + db = connect_database(url) + try: + store = ManifestStore(db) + now = int(time.time()) + # Simulate a second worker renewing a stale lease before admin execute. + db.connection.execute( + writer_fences.update().where(writer_fences.c.object_id == target) + .values(lease_until=now + 200, heartbeat_at=now) + ) + db.commit() + finally: + db.close() + + endpoint = f"/api/v1/admin/storage/pending-stale/{target}/abandon" + out = client.post(endpoint, json={"execute": True}, + auth=("testadmin", "secure-auth-password"), + headers={"X-TeleDrive-CSRF": "1"}) + assert out.status_code == 200 + assert out.json()["abandoned"] is False + db = connect_database(url) + try: + store = ManifestStore(db) + assert dict(store.interrupted())[target] == "pending" + assert store.heartbeat_writer(target, old, int(time.time())) + finally: + db.close() + + +def test_stale_pending_inventory_persists_across_connections(tmp_path): + url = str(tmp_path / "durable.sqlite") + identifier, old = create(url) + first = connect_database(url) + second = connect_database(url) + try: + left, right = ManifestStore(first), ManifestStore(second) + now = int(time.time()) + assert [r["id"] for r in left.stale_pending(now=now)] == [identifier] + first.commit() + assert right.abandon_stale_pending(now=now, object_id=identifier) == [identifier] + assert left.stale_pending(now=now) == [] + assert not left.heartbeat_writer(identifier, old, now) + finally: + first.close() + second.close() diff --git a/tests/storage/test_storage_rebuild.py b/tests/storage/test_storage_rebuild.py new file mode 100644 index 00000000..d084ea15 --- /dev/null +++ b/tests/storage/test_storage_rebuild.py @@ -0,0 +1,233 @@ +"""Rebuild only from original, fully checksummed Telegram chunk evidence.""" +from __future__ import annotations +import asyncio +import hashlib +from uuid import uuid4 +import pytest + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.rebuild import restore_isolated_catalog + + +def sha(data): return hashlib.sha256(data).hexdigest() + + +class TelegramSource: + def __init__(self, chunks): self.chunks = chunks; self.reads = 0 + async def probe(self, chunk, *, object_id): + return "verified" + + async def download(self, ref, destination): + self.reads += 1 + destination.write_bytes(self.chunks[ref.message_id]) + + +def make_plan(parts): + original = b"".join(parts) + return { + "object_id": str(uuid4()), "bucket": "rebuild-test", + "key": "original/object.bin", "size": len(original), + "sha256": sha(original), + "chunks": [ + {"ordinal": i, "account": "original", "chat_id": -100123456, + "message_id": 100+i, "size": len(data), "sha256": sha(data)} + for i, data in enumerate(parts) + ], + } + + +@pytest.mark.parametrize("parts", [[], [b"hello"], [b"one", b"two" * 1000]]) +def test_original_bytes_create_isolated_manifest_and_head(tmp_path, parts): + plan = make_plan(parts) + original = {100+i: p for i,p in enumerate(parts)} + db_path = tmp_path / "recovered.db" + result = asyncio.run(restore_isolated_catalog(plan, TelegramSource(original), db_path)) + assert result["verified"] and result["telegram_deletions"] == 0 + db = connect_database(str(db_path)) + try: + catalog = BucketCatalog(ManifestStore(db)) + item = catalog.get("rebuild-test", "original/object.bin").item + assert item.id == plan["object_id"] and item.sha256 == plan["sha256"] + assert sum(x.size for x in catalog.manifests.parts(item.id)) == plan["size"] + finally: + db.close() + + +def test_original_media_corruption_never_creates_sql_db(tmp_path): + plan = make_plan([b"known", b"good"]) + dest = tmp_path / "recovered.db" + with pytest.raises(ValueError, match="hash/length"): + asyncio.run(restore_isolated_catalog( + plan, TelegramSource({100: b"known", 101: b"bad"}), dest)) + assert not dest.exists() + + +def test_rebuild_never_overwrites_existing_database(tmp_path): + plan = make_plan([b"a"]) + dest = tmp_path / "existing.db" + dest.write_bytes(b"original SQL data") + with pytest.raises(FileExistsError): + asyncio.run(restore_isolated_catalog( + plan, TelegramSource({100:b"a"}), dest)) + assert dest.read_bytes() == b"original SQL data" + + +@pytest.mark.parametrize("mutation", [ + lambda p: p.update(sha256="0"*64), + lambda p: p["chunks"][0].update(ordinal=2), + lambda p: p["chunks"][0].update(size=8), + lambda p: p["chunks"].append(dict(p["chunks"][0])), + lambda p: p.update(bucket="../prod"), +]) +def test_rebuild_rejects_unproven_identity_metadata(tmp_path, mutation): + plan = make_plan([b"hello"]) + mutation(plan) + dest = tmp_path / "recovered.db" + with pytest.raises((ValueError, TypeError)): + asyncio.run(restore_isolated_catalog( + plan, TelegramSource({100:b"hello"}), dest)) + assert not dest.exists() + + +def test_rebuild_rejects_unverified_remote_ownership_even_with_correct_bytes(tmp_path): + plan = make_plan([b"exact-bytes"]) + class WrongOwner(TelegramSource): + async def probe(self, chunk, *, object_id): + return "mismatch" + dest = tmp_path / "blocked.db" + with pytest.raises(ValueError, match="ownership"): + asyncio.run(restore_isolated_catalog(plan, WrongOwner({100:b"exact-bytes"}), dest)) + assert not dest.exists() + + +def test_two_distinct_objects_publish_together_in_one_catalog(tmp_path): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + first = make_plan([b"abc", b"def"]) + second = make_plan([b"final"]) + second["key"] = "original/second.bin" + second["chunks"][0]["message_id"] = 301 + source = TelegramSource({100: b"abc", 101: b"def", 301: b"final"}) + path = tmp_path / "isolated-batch.db" + receipt = asyncio.run(restore_isolated_catalog_batch([first, second], source, path)) + assert receipt["object_count"] == 2 + assert receipt["verified_bytes"] == 11 + assert not receipt["production_database_modified"] + db = connect_database(str(path)) + try: + catalog = BucketCatalog(ManifestStore(db)) + assert catalog.get(first["bucket"], first["key"]).item.id == first["object_id"] + assert catalog.get(second["bucket"], second["key"]).item.id == second["object_id"] + finally: + db.close() + + +def test_invalid_second_object_never_exposes_partial_database(tmp_path): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + first = make_plan([b"good"]) + second = make_plan([b"corrupt"]) + second["key"] = "second" + second["chunks"][0]["message_id"] = 999 + destination = tmp_path / "no-half-catalog.db" + with pytest.raises(ValueError, match="hash/length"): + asyncio.run(restore_isolated_catalog_batch( + [first, second], TelegramSource({100: b"good", 999: b"changed"}), + destination, + )) + assert not destination.exists() + assert not list(tmp_path.glob(".teledrive-rebuild-*")) + + +@pytest.mark.parametrize("kind", ["duplicate-id", "duplicate-head", "duplicate-message", "oversized"]) +def test_invalid_cross_object_metadata_rejected_before_remote_io(tmp_path, kind): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + first = make_plan([b"one"]) + second = make_plan([b"two"]) + second["key"] = "second" + second["chunks"][0]["message_id"] = 301 + if kind == "duplicate-id": + second["object_id"] = first["object_id"] + elif kind == "duplicate-head": + second["key"] = first["key"] + elif kind == "duplicate-message": + second["chunks"][0]["message_id"] = first["chunks"][0]["message_id"] + else: + second["size"] = 1024**3 + remote = TelegramSource({100: b"one", 301: b"two"}) + destination = tmp_path / "invalid.db" + with pytest.raises(ValueError): + asyncio.run(restore_isolated_catalog_batch([first, second], remote, destination)) + assert remote.reads == 0 + assert not destination.exists() + + +def test_batch_must_not_replace_preexisting_database(tmp_path): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + plan = make_plan([b"unchanged"]) + target = tmp_path / "prod.db" + target.write_bytes(b"production metadata preserved") + with pytest.raises(FileExistsError): + asyncio.run(restore_isolated_catalog_batch([plan], TelegramSource({100:b"unchanged"}), target)) + assert target.read_bytes() == b"production metadata preserved" + + +def test_nonmatching_second_remote_owner_fails_the_entire_batch(tmp_path): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + first = make_plan([b"first"]) + second = make_plan([b"second"]) + second["key"] = "other" + second["chunks"][0]["message_id"] = 301 + class OwnerMismatch(TelegramSource): + async def probe(self, chunk, *, object_id): + return "mismatch" if chunk.message_id == 301 else "verified" + target = tmp_path / "blocked.db" + with pytest.raises(ValueError, match="ownership"): + asyncio.run(restore_isolated_catalog_batch( + [first, second], OwnerMismatch({100:b"first", 301:b"second"}), target)) + assert not target.exists() + + +def test_second_sql_publication_error_leaves_no_visible_database(tmp_path, monkeypatch): + """Publishing object one cannot expose half of object two.""" + from teledrive.storage.rebuild import restore_isolated_catalog_batch + first = make_plan([b"one"]) + second = make_plan([b"two"]) + second["key"] = "original/other" + second["chunks"][0]["message_id"] = 301 + original_publish = BucketCatalog.publish + invocations = 0 + + def break_second(self, identifier, *, now, expected_generation=None, + expected_object_id=None): + nonlocal invocations + invocations += 1 + if invocations == 2: + raise RuntimeError("Simulated SQL publishing failure") + return original_publish( + self, identifier, now=now, + expected_generation=expected_generation, + expected_object_id=expected_object_id, + ) + + monkeypatch.setattr(BucketCatalog, "publish", break_second) + destination = tmp_path / "no-partial-publish.db" + with pytest.raises(RuntimeError, match="SQL publishing"): + asyncio.run(restore_isolated_catalog_batch( + [first, second], TelegramSource({100: b"one", 301: b"two"}), + destination, + )) + assert invocations == 2 + assert not destination.exists() + assert not list(tmp_path.glob(".teledrive-rebuild-*")) + + +def test_reconstructed_catalog_is_private_on_disk(tmp_path): + from teledrive.storage.rebuild import restore_isolated_catalog_batch + import stat + plan = make_plan([b"private"]) + destination = tmp_path / "private-rebuild.db" + asyncio.run(restore_isolated_catalog_batch( + [plan], TelegramSource({100: b"private"}), destination, + )) + assert stat.S_IMODE(destination.stat().st_mode) == 0o600 diff --git a/tests/storage/test_storage_recovery.py b/tests/storage/test_storage_recovery.py new file mode 100644 index 00000000..9b008108 --- /dev/null +++ b/tests/storage/test_storage_recovery.py @@ -0,0 +1,391 @@ +"""Behavioral recovery tests: real SQLite journal and fake Telegram history. + +Verify positive ownership before deletion, orphan discovery, partial cleanup, +restart behavior, concurrent leases, and fail-closed incomplete scans. +""" +from __future__ import annotations + +import asyncio +import hashlib +import time +from dataclasses import replace +from pathlib import Path + +import pytest + +from teledrive.db import connect_database +from teledrive.storage.manifest import ManifestStore, ChunkReference +from teledrive.storage.recovery import ( + CleanupReport, Discovery, RecoveryManager, RemoteChunk, UnsafeCleanup, +) +from teledrive.storage.telethon_adapter import TelethonChunkTransport + + +def run(coro): + return asyncio.run(coro) + + +class FakeRecoveryTransport: + def __init__(self, *, complete=True): + self.messages: dict[tuple[str, int, int], RemoteChunk] = {} + self.complete = complete + self.fail_delete = False + self.deleted: list[int] = [] + + async def discover(self, object_id, *, scan_limit): + values = tuple(self.messages.values()) + return Discovery(values[:scan_limit], self.complete and len(values) <= scan_limit, len(values)) + + async def probe(self, chunk, *, object_id): + actual = self.messages.get(chunk.identity) + if actual is None: + return "missing" + return "verified" if actual == chunk else "mismatch" + + async def delete_verified(self, chunk, *, object_id): + if self.fail_delete: + self.fail_delete = False + return False + if await self.probe(chunk, object_id=object_id) != "verified": + return False + del self.messages[chunk.identity] + self.deleted.append(chunk.message_id) + return True + + +@pytest.fixture +def db_store(tmp_path): + connection = connect_database(str(tmp_path / "state.sqlite")) + try: + yield ManifestStore(connection) + finally: + connection.close() + + +def failed_object(store, *, message_id=10, with_journal=True): + # A failed upload older than the required operator safety window. + older = int(time.time()) - 7200 + ident = store.begin("ci", "backups/archive.bin", 12, older) + sha = hashlib.sha256(b"abcdef").hexdigest() + part = ChunkReference(ordinal=0, offset=0, size=6, sha256=sha, + account="primary", chat_id=12345, message_id=message_id) + if with_journal: + store.append(ident, part, older) + store.fail(ident, "simulated timeout", older) + return ident, RemoteChunk("primary", 12345, message_id, 0, sha[:16]) + + +def test_dry_run_detects_recorded_and_unindexed_telegram_messages(db_store): + ident, saved = failed_object(db_store) + remote = FakeRecoveryTransport() + orphan = RemoteChunk("primary", 12345, 11, 1, "a" * 16) + remote.messages[saved.identity] = saved + remote.messages[orphan.identity] = orphan + report = run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident)) + assert report.journaled == 1 + assert report.orphaned == 1 + assert report.discovered == 2 + assert report.deleted == 0 and report.dry_run + assert set(remote.messages) == {saved.identity, orphan.identity} + assert db_store.interrupted()[0][1] == "failed" + + +def test_explicit_cleanup_deletes_only_owned_messages_and_is_durable(db_store): + ident, saved = failed_object(db_store) + remote = FakeRecoveryTransport() + orphan = RemoteChunk("primary", 12345, 19, 1, "b" * 16) + remote.messages = {saved.identity: saved, orphan.identity: orphan} + report = run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + assert report.cleaned and report.deleted == 2 + assert set(remote.deleted) == {10, 19} + assert not remote.messages + with pytest.raises(FileNotFoundError): + db_store.get(ident) + with pytest.raises(UnsafeCleanup, match="Only failed"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + + +def test_truncated_scan_blocks_destructive_cleanup(db_store): + ident, saved = failed_object(db_store) + remote = FakeRecoveryTransport(complete=False) + remote.messages[saved.identity] = saved + with pytest.raises(UnsafeCleanup, match="truncated"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + assert remote.deleted == [] + assert db_store.interrupted()[0][1] == "failed" + + +def test_mismatched_journal_reference_blocks_deletion(db_store): + ident, saved = failed_object(db_store) + remote = FakeRecoveryTransport() + forged = replace(saved, checksum_prefix="0" * 16) + remote.messages[saved.identity] = forged + with pytest.raises(UnsafeCleanup, match="does not belong"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + assert remote.deleted == [] + + +def test_partial_failure_is_retryable_without_double_deleting(db_store): + ident, saved = failed_object(db_store) + remote = FakeRecoveryTransport() + remote.messages[saved.identity] = saved + remote.fail_delete = True + manager = RecoveryManager(db_store, remote, min_age_seconds=3600) + with pytest.raises(UnsafeCleanup, match="not verified"): + run(manager.cleanup(ident, dry_run=False)) + assert len(remote.messages) == 1 + assert db_store.interrupted()[0][1] == "failed" + assert run(manager.cleanup(ident, dry_run=False)).cleaned + assert remote.deleted == [saved.message_id] + + +def test_known_chunk_previously_removed_is_idempotently_cleanable(db_store): + ident, _ = failed_object(db_store) + remote = FakeRecoveryTransport() + report = run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + assert report.cleaned and report.missing == 1 and report.deleted == 0 + + +def test_pending_and_recent_failures_cannot_be_cleaned(db_store): + remote = FakeRecoveryTransport() + fresh = db_store.begin("ci", "pending", 0, int(time.time())) + with pytest.raises(UnsafeCleanup, match="Only failed"): + run(RecoveryManager(db_store, remote).cleanup(fresh, dry_run=False)) + db_store.fail(fresh, "recent", int(time.time())) + with pytest.raises(UnsafeCleanup, match="aged"): + run(RecoveryManager(db_store, remote).cleanup(fresh, dry_run=False)) + + +def test_existing_cleanup_lease_prevents_second_worker(db_store): + ident, chunk = failed_object(db_store) + remote = FakeRecoveryTransport() + now = int(time.time()) + lease = f"storage-gc:{ident}" + assert db_store.db.acquire_lease(lease, "other", now, 300) + try: + with pytest.raises(UnsafeCleanup, match="Another cleanup worker"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + finally: + db_store.db.release_lease(lease, "other") + + +class FakeMessage: + def __init__(self, ident, ordinal, digest, *, mid=12, chat_id=12345, outgoing=True): + self.message = f"teledrive:{ident}:{ordinal}:{digest[:16]}" + self.id = mid + self.chat_id = chat_id + self.out = outgoing + + +class FakeTelethon: + def __init__(self, messages): + self.messages = {m.id: m for m in messages} + self.deletions: list[int] = [] + + async def get_peer_id(self, target): + return 12345 + + async def get_messages(self, target, *, ids): + return self.messages.get(ids) + + async def iter_messages(self, target, *, limit): + for msg in list(self.messages.values())[:limit]: + yield msg + + async def delete_messages(self, target, ids, *, revoke): + for mid in ids: + self.deletions.append(mid) + self.messages.pop(mid, None) + + +def test_telethon_adapter_rejects_spoofed_markers_and_nonowned_messages(db_store): + identifier, _ = failed_object(db_store) + digest = hashlib.sha256(b"abcdef").hexdigest() + correct = FakeMessage(identifier, 0, digest, mid=40) + forged = FakeMessage(identifier, 0, digest, mid=41, outgoing=False) + unrelated = FakeMessage("00000000-0000-0000-0000-000000000000", 0, digest, mid=42) + client = FakeTelethon([correct, forged, unrelated]) + adapter = TelethonChunkTransport({"primary": (client, "me")}) + remote = RemoteChunk("primary", 12345, 40, 0, digest[:16]) + assert run(adapter.probe(remote, object_id=identifier)) == "verified" + discovery = run(adapter.discover(identifier, scan_limit=10)) + assert not discovery.complete # forged channel message makes scan ambiguous + assert discovery.matches == (remote,) + assert run(adapter.delete_verified(remote, object_id=identifier)) + assert client.deletions == [40] + assert 41 in client.messages + with pytest.raises(UnsafeCleanup): + run(RecoveryManager(db_store, adapter, min_age_seconds=3600).cleanup(identifier, dry_run=False)) + + +def test_telethon_adapter_scan_limit_cannot_hide_older_orphans(db_store): + identifier, _ = failed_object(db_store) + digest = hashlib.sha256(b"abcdef").hexdigest() + client = FakeTelethon([ + FakeMessage(identifier, i, digest, mid=100+i) for i in range(3) + ]) + adapter = TelethonChunkTransport({"primary": (client, "me")}) + result = run(adapter.discover(identifier, scan_limit=2)) + assert not result.complete + assert len(result.matches) == 2 + + +def test_unindexed_telegram_upload_from_db_crash_is_discovered(db_store): + """send_file succeeded, append() never committed: caption still locates it.""" + ident, _ = failed_object(db_store, with_journal=False) + remote = FakeRecoveryTransport() + orphan = RemoteChunk("primary", 12345, 500, 0, "c" * 16) + remote.messages[orphan.identity] = orphan + manager = RecoveryManager(db_store, remote, min_age_seconds=3600) + preview = run(manager.cleanup(ident)) + assert preview.journaled == 0 + assert preview.orphaned == 1 + assert remote.messages + report = run(manager.cleanup(ident, dry_run=False)) + assert report.cleaned and report.deleted == 1 + assert not remote.messages + + +def test_ready_object_is_never_a_cleanup_candidate(db_store): + digest = hashlib.sha256(b"").hexdigest() + ident = db_store.begin("ci", "finished", 0, int(time.time()) - 100000) + db_store.finish(ident, digest, int(time.time()) - 100000) + remote = FakeRecoveryTransport() + with pytest.raises(UnsafeCleanup, match="Only failed"): + run(RecoveryManager(db_store, remote).cleanup(ident, dry_run=False)) + assert db_store.get(ident).size == 0 + + +def test_unknown_account_reference_does_not_trigger_deletion(db_store): + ident, known = failed_object(db_store) + remote = FakeRecoveryTransport() + # A different account is not the account recorded for this chunk. + fake = replace(known, account="unexpected") + remote.messages[known.identity] = fake + with pytest.raises(UnsafeCleanup, match="does not belong"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup(ident, dry_run=False)) + assert not remote.deleted + + +def test_physical_cleanup_rejects_published_version_even_if_object_failed(db_store): + """A failed but historically indexed object cannot lose its original bytes.""" + from sqlalchemy import insert + from teledrive.storage.catalog import BucketCatalog, published_versions + + ident, chunk = failed_object(db_store) + BucketCatalog(db_store) + db_store.db.connection.execute(insert(published_versions).values( + object_id=ident, bucket="ci", key_hash="a" * 64, committed_ns=123, + )) + db_store.db.commit() + transport = FakeRecoveryTransport() + transport.messages[chunk.identity] = chunk + preview = run(RecoveryManager(db_store, transport, min_age_seconds=3600).cleanup(ident)) + assert preview.dry_run and preview.deleted == 0 + with pytest.raises(UnsafeCleanup, match="another SQL reference"): + run(RecoveryManager(db_store, transport, min_age_seconds=3600).cleanup( + ident, dry_run=False + )) + assert not transport.deleted and chunk.identity in transport.messages + + +def test_physical_cleanup_rejects_copy_alias_and_unresolved_mtproto_intent(db_store): + from sqlalchemy import insert + from teledrive.storage.manifest import object_aliases + from teledrive.storage.user_mode import UserSendJournal, mtproto_send_intents + + ident, chunk = failed_object(db_store) + copy = db_store.begin("ci", "copy-alias", 0, int(time.time()) - 9000) + db_store.db.connection.execute(insert(object_aliases).values( + object_id=copy, source_object_id=ident, + )) + db_store.db.commit() + transport = FakeRecoveryTransport() + transport.messages[chunk.identity] = chunk + manager = RecoveryManager(db_store, transport, min_age_seconds=3600) + with pytest.raises(UnsafeCleanup, match="another SQL reference"): + run(manager.cleanup(ident, dry_run=False)) + db_store.db.connection.execute(object_aliases.delete().where( + object_aliases.c.object_id == copy + )) + db_store.db.commit() + journal = UserSendJournal(db_store.db) + journal.start("primary", 12345, ident, 0, + hashlib.sha256(b"abcdef").hexdigest(), 6) + with pytest.raises(UnsafeCleanup, match="another SQL reference"): + run(manager.cleanup(ident, dry_run=False)) + assert transport.deleted == [] + + +def test_physical_cleanup_refuses_expired_unreconciled_writer_fence(db_store): + """A lost writer lease does not prove Telegram sends have stopped.""" + identifier, part = failed_object(db_store) + remote = FakeRecoveryTransport() + remote.messages[part.identity] = part + db_store.start_writer(identifier, "not-yet-reconciled", + int(time.time()) - 3600, ttl=120) + with pytest.raises(UnsafeCleanup, match="retained"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup( + identifier, dry_run=False, + )) + assert remote.deleted == [] + assert part.identity in remote.messages + assert db_store.interrupted()[0][1] == "failed" + + +def _queued_job(db_store, *, object_id=None, expected_head_id=None, + bucket="ci", key="backups/archive.bin", index=0): + from sqlalchemy import insert + from teledrive.storage.jobs import StorageUploadJobs, jobs + from teledrive.storage.catalog import BucketCatalog + StorageUploadJobs(BucketCatalog(db_store)) + now = int(time.time()) + db_store.db.connection.execute(insert(jobs).values( + id=f"job-{index}-{object_id or expected_head_id or key}", + bucket=bucket, object_key=key, source_path="/uploads/source.bin", + source_size=6, source_device=1, source_inode=2, source_mtime_ns=3, + expected_generation=0, expected_head_id=expected_head_id, + state="queued", object_id=object_id, attempts=0, + lease_token=None, lease_until=None, created_at=now, + updated_at=now, last_error=None, + )) + db_store.db.commit() + + +def test_physical_cleanup_rejects_active_durable_job_reference(db_store): + """A queued job that can still publish this key must fence physical GC.""" + identifier, part = failed_object(db_store) + remote = FakeRecoveryTransport() + remote.messages[part.identity] = part + manager = RecoveryManager(db_store, remote, min_age_seconds=3600) + for index, kwargs in enumerate(({"object_id": identifier}, + {"expected_head_id": identifier}, + {})): + _queued_job(db_store, index=index, **kwargs) + with pytest.raises(UnsafeCleanup, match="another SQL reference"): + run(manager.cleanup(identifier, dry_run=False)) + assert remote.deleted == [] + assert part.identity in remote.messages + + +def test_physical_cleanup_rejects_mtproto_send_intent_reference(db_store): + """An unresolved USER-MTProto send intent keeps the failed object protected.""" + from teledrive.storage.user_mode import UserSendJournal, mtproto_send_intents + from sqlalchemy import insert + + identifier, part = failed_object(db_store) + remote = FakeRecoveryTransport() + remote.messages[part.identity] = part + UserSendJournal(db_store.db) # create_all for the send-intent table + db_store.db.connection.execute(insert(mtproto_send_intents).values( + account="mtproto-user", chat_id=-100987654321, object_id=identifier, + ordinal=0, sha256=part.checksum_prefix + "0" * 48, size=6, + message_id=None, + )) + db_store.db.commit() + with pytest.raises(UnsafeCleanup, match="another SQL reference"): + run(RecoveryManager(db_store, remote, min_age_seconds=3600).cleanup( + identifier, dry_run=False, + )) + assert remote.deleted == [] + assert part.identity in remote.messages diff --git a/tests/storage/test_storage_sigkill.py b/tests/storage/test_storage_sigkill.py new file mode 100644 index 00000000..3729fd24 --- /dev/null +++ b/tests/storage/test_storage_sigkill.py @@ -0,0 +1,280 @@ +"""Actual SIGKILL/crash of an upload worker with SQLite and Telegram disk stub. + +This runs the real durable-job and storage-engine implementations in another +OS process; after SIGKILL only committed SQL and fake remote media survive. +""" +from __future__ import annotations + +import asyncio +import os +import signal +import subprocess +import sys +import time +from pathlib import Path + +import pytest +from sqlalchemy import update + +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.jobs import StorageUploadJobs, jobs +from teledrive.storage.manifest import ManifestStore, writer_fences + +CHILD = r''' +import asyncio +import os +import signal +import sys +from pathlib import Path +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.jobs import StorageUploadJobs +from teledrive.storage.manifest import ManifestStore + +class DiskTelegram: + async def upload(self, path, *, object_id, ordinal, checksum): + if ordinal == 1: + os.kill(os.getpid(), signal.SIGKILL) + remote = Path(sys.argv[3]) / f"{ordinal + 1}.bin" + remote.write_bytes(path.read_bytes()) + return "disk", -100111, ordinal + 1 + + async def download(self, ref, destination): + destination.write_bytes((Path(sys.argv[3]) / f"{ref.message_id}.bin").read_bytes()) + +async def run(): + db = connect_database(sys.argv[1]) + try: + await StorageUploadJobs(BucketCatalog(ManifestStore(db))).run_next( + DiskTelegram(), allowed_root=Path(sys.argv[2]), + scratch_directory=Path(sys.argv[2]), chunk_size=5, + ) + finally: + db.close() + +asyncio.run(run()) +''' + + +class DiskTelegram: + def __init__(self, directory): + self.directory = directory + self.uploads = 0 + self.downloads = 0 + + async def upload(self, path, *, object_id, ordinal, checksum): + self.uploads += 1 + destination = self.directory / f"{ordinal + 1}.bin" + if destination.exists(): + raise AssertionError("Previously journaled Telegram chunk retransmitted") + destination.write_bytes(path.read_bytes()) + return "disk", -100111, ordinal + 1 + + async def download(self, ref, destination): + self.downloads += 1 + destination.write_bytes( + (self.directory / f"{ref.message_id}.bin").read_bytes() + ) + + +@pytest.mark.skipif(os.name != "posix", reason="SIGKILL is required") +def test_real_sigkill_recovers_only_missing_remote_chunks(tmp_path): + root = tmp_path / "allowed" + root.mkdir() + remote = tmp_path / "remote" + remote.mkdir() + body = bytes(range(19)) + source = root / "source.bin" + source.write_bytes(body) + db_path = tmp_path / "state.db" + conn = connect_database(str(db_path)) + try: + catalog = BucketCatalog(ManifestStore(conn)) + catalog.create_bucket("sigkill-ci", int(time.time())) + job_id = StorageUploadJobs(catalog).enqueue( + "sigkill-ci", "crash.bin", source, allowed_root=root, + ) + finally: + conn.close() + + killed = subprocess.run( + [sys.executable, "-c", CHILD, str(db_path), str(root), str(remote)], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, timeout=25, check=False, + ) + assert killed.returncode == -signal.SIGKILL, killed.stderr.decode(errors="replace") + assert (remote / "1.bin").read_bytes() == body[:5] + assert not (remote / "2.bin").exists() + reopened = connect_database(str(db_path)) + try: + catalog = BucketCatalog(ManifestStore(reopened)) + queue = StorageUploadJobs(catalog) + recorded = queue.list_jobs()[0] + assert recorded["id"] == job_id + assert recorded["state"] == "running" + identifier = recorded["object_id"] + assert identifier + refs = catalog.manifests.recovery_parts(identifier) + assert len(refs) == 1 and refs[0].size == 5 + with pytest.raises(FileNotFoundError): + catalog.get("sigkill-ci", "crash.bin") + # The OS process has genuinely died; advance expired lease time + # without waiting 120 seconds in CI. + reopened.connection.execute( + update(writer_fences).where(writer_fences.c.object_id == identifier) + .values(lease_until=0) + ) + reopened.connection.execute( + update(jobs).where(jobs.c.id == job_id).values(lease_until=0) + ) + reopened.commit() + transport = DiskTelegram(remote) + result = asyncio.run(queue.run_next( + transport, allowed_root=root, scratch_directory=root, chunk_size=5, + )) + assert result == {"id": job_id, "state": "done", "object_id": identifier} + assert transport.uploads == 3 + assert transport.downloads == 1 + restored = root / "restored.bin" + asyncio.run(TelegramObjectStorage( + catalog.manifests, transport, + ).download_to_path(identifier, restored)) + assert restored.read_bytes() == body + assert (remote / "1.bin").read_bytes() == body[:5] + finally: + reopened.close() + +MT_ACK_CRASH_CHILD = r''' +import asyncio +import hashlib +import os +import signal +import sys +from pathlib import Path + +from teledrive.db import connect_database +from teledrive.storage.user_mode import UnifiedUserTransport, UserSendJournal +from teledrive.storage.telethon_adapter import TelethonChunkTransport + +class AcceptedThenKilled: + async def send_file(self, target, path, caption=None, force_document=False): + # Simulate a remote Telegram post that really accepted the bytes. + Path(sys.argv[3]).write_bytes(Path(path).read_bytes()) + os.kill(os.getpid(), signal.SIGKILL) + raise AssertionError("SIGKILL should prevent an ACK from returning") + + async def get_peer_id(self, target): + return -1001234 + + async def get_messages(self, peer, ids): + raise AssertionError("readback is not part of this crash window") + + +class FakeRuntime: + account = "user" + chat_id = -1001234 + + def require(self): + return TelethonChunkTransport({"user": (AcceptedThenKilled(), None)}) + + +async def main(): + db = connect_database(sys.argv[1]) + try: + path = Path(sys.argv[2]) + transport = UnifiedUserTransport(FakeRuntime(), UserSendJournal(db), + min_upload_interval=0) + await transport.upload( + path, object_id="sigkill-user-ack", ordinal=0, + checksum=hashlib.sha256(path.read_bytes()).hexdigest(), + ) + finally: + db.close() + +asyncio.run(main()) +''' + + +@pytest.mark.skipif(os.name != "posix", reason="POSIX SIGKILL required") +def test_actual_sigkill_after_remote_user_acceptance_never_blindly_retries(tmp_path): + """A real child dies after remote acceptance and before its MTProto receipt. + + Recovery sees the previously committed send intent on a new SQL + connection and cannot retransmit that exact object chunk. An operator + may later record an independently proven ACK; no unproven message is + automatically inserted into the manifest or deleted. + """ + import hashlib + from teledrive.storage.user_mode import ( + UnifiedUserTransport, UserSendJournal, mtproto_send_intents, + ) + from teledrive.storage.telethon_adapter import TelethonChunkTransport + from sqlalchemy import select + + path = tmp_path / "mtproto-intents.db" + chunk = tmp_path / "signed-binary.bin" + accepted = tmp_path / "telegram-accepted.bin" + data = bytes(range(256)) * 400 + bytes((0, 255)) * 170 + chunk.write_bytes(data) + sha = hashlib.sha256(data).hexdigest() + seed = connect_database(str(path)) + try: + UserSendJournal(seed) + finally: + seed.close() + + result = subprocess.run( + [sys.executable, "-c", MT_ACK_CRASH_CHILD, + str(path), str(chunk), str(accepted)], + cwd=Path(__file__).resolve().parents[2], + capture_output=True, timeout=30, check=False, + ) + assert result.returncode == -signal.SIGKILL, result.stderr.decode(errors="replace") + assert accepted.read_bytes() == data # remote post definitely occurred + + reopened = connect_database(str(path)) + try: + journal = UserSendJournal(reopened) + assert journal.unresolved_count() == 1 + with pytest.raises(RuntimeError, match="Unresolved MTProto upload intent"): + journal.start("user", -1001234, "sigkill-user-ack", 0, sha, len(data)) + + class UnexpectedResend: + calls = 0 + + async def send_file(self, target, path, caption=None, + force_document=False): + self.calls += 1 + raise AssertionError("Unknown-outcome chunk was resent") + + fake = UnexpectedResend() + + class ResendRuntime: + account = "user" + chat_id = -1001234 + + def require(self): + return TelethonChunkTransport({"user": (fake, None)}) + + transport = UnifiedUserTransport(ResendRuntime(), journal, + min_upload_interval=0) + with pytest.raises(RuntimeError, match="Unresolved MTProto upload intent"): + asyncio.run(transport.upload( + chunk, object_id="sigkill-user-ack", ordinal=0, checksum=sha, + )) + assert fake.calls == 0 + # A separate owner-authorized proof of original message identity + # and byte-for-byte match is required before recovery. + assert hashlib.sha256(accepted.read_bytes()).hexdigest() == sha + journal.acknowledged("user", -1001234, "sigkill-user-ack", 0, 99) + assert journal.unresolved_count() == 0 + recorded = reopened.connection.execute( + select(mtproto_send_intents.c.message_id).where( + mtproto_send_intents.c.object_id == "sigkill-user-ack", + ) + ).scalar_one() + assert recorded == 99 + finally: + reopened.close() diff --git a/tests/storage/test_storage_user_mode.py b/tests/storage/test_storage_user_mode.py new file mode 100644 index 00000000..ecf5bdfb --- /dev/null +++ b/tests/storage/test_storage_user_mode.py @@ -0,0 +1,324 @@ +"""Fake-Telegram signed S3 and SQL safety tests for optional USER MTProto mode.""" +from __future__ import annotations + +import asyncio +import hashlib +from pathlib import Path +from types import SimpleNamespace + +import pytest +from sqlalchemy import func, select + +from tests.s3.test_s3_gateway import service, signed +from teledrive.storage.telethon_adapter import TelethonChunkTransport +from teledrive.storage.user_mode import ( + UserSessionRuntime, UserSendJournal, UnifiedUserTransport, + mtproto_send_intents, transport_mode, +) + + +class FakeUser: + def __init__(self): + self.connected = True + self.posts = {} + self.next_id = 0 + self.fail_after_remote_accept = False + + def is_connected(self): + return self.connected + + async def get_peer_id(self, target): + return -1001234567890 + + async def send_file(self, target, path, caption, force_document): + assert caption.startswith("teledrive:") + assert force_document + self.next_id += 1 + self.posts[self.next_id] = (Path(path).read_bytes(), caption) + if self.fail_after_remote_accept: + raise TimeoutError("simulated lost MTProto ACK") + return SimpleNamespace(id=self.next_id, chat_id=-1001234567890) + + async def get_messages(self, target, ids): + if ids not in self.posts: + return None + return SimpleNamespace( + id=ids, media=True, message=self.posts[ids][1], + chat_id=-1001234567890, out=True, + ) + + async def download_media(self, message, file): + Path(file).write_bytes(self.posts[message.id][0]) + return file + + +def configured_runtime(): + user = FakeUser() + runtime = UserSessionRuntime() + runtime.client = user + runtime.account = "mtproto-user" + runtime.chat_id = -1001234567890 + runtime.transport = TelethonChunkTransport({ + runtime.account: (user, object()), + }) + return runtime, user + + +def test_telethon_is_the_only_transport(monkeypatch): + """Bot mode no longer exists; no env var selects a transport.""" + monkeypatch.setenv("TELEDRIVE_STORAGE_TRANSPORT", "bot") + assert transport_mode() == "telethon" + + +def test_user_session_requires_file_and_disallows_saved_messages(monkeypatch, tmp_path): + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "me"\n' + " apis:\n" + " - api_id: 1234\n" + f" api_hash: {'a' * 32}\n" + f" session_file: {tmp_path / 'missing'}\n", + encoding="ascii", + ) + monkeypatch.setenv("TELEDRIVE_CONFIG", str(config_path)) + with pytest.raises(RuntimeError, match="dedicated channel"): + asyncio.run(UserSessionRuntime().start()) + + +def test_mtp_user_upload_and_s3_readback_with_bot_fallback(service, monkeypatch): + import web.s3 as gateway + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + earlier = signed(service, "PUT", "/backup-bucket/old.bin", b"original-bot-data") + assert earlier.status_code == 200, earlier.text + + runtime, user = configured_runtime() + # A real Telegram channel assigns message IDs across ALL senders. + # The fake Bot and fake user must not independently reuse message ID 1. + user.next_id = service[2].uploads + # Both senders share the same real channel/message sequence. The owner + # USER session can read an old Bot document directly by its message ID. + user.posts[1] = (b"original-bot-data", "old-bot-document") + monkeypatch.setattr( + gateway, "storage_transport", + lambda db: UnifiedUserTransport(runtime, UserSendJournal(db)), + ) + data = b"new-user-mtproto-bytes" * 19 + response = signed(service, "PUT", "/backup-bucket/user.bin", data) + assert response.status_code == 200, response.text + assert user.next_id == 2 + assert signed(service, "GET", "/backup-bucket/user.bin").content == data + assert signed(service, "GET", "/backup-bucket/old.bin").content == b"original-bot-data" + conn = service[3]() + try: + records = conn.connection.execute( + select(mtproto_send_intents.c.account, mtproto_send_intents.c.message_id) + ).all() + assert records == [("mtproto-user", 2)] + finally: + conn.close() + + +def test_user_uncertain_ack_remains_fenced_and_gc_preview_is_blocked(service, monkeypatch): + import web.s3 as gateway + from teledrive.storage.gc_safety import GCReferenceGuard + from teledrive.storage.manifest import ManifestStore, objects + + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + runtime, user = configured_runtime() + user.fail_after_remote_accept = True + def transport(db): + return UnifiedUserTransport(runtime, UserSendJournal(db)) + monkeypatch.setattr(gateway, "storage_transport", transport) + response = signed(service, "PUT", "/backup-bucket/unknown.bin", b"uncertain-send") + assert response.status_code >= 400 + assert len(user.posts) == 1 + assert signed(service, "GET", "/backup-bucket/unknown.bin").status_code == 404 + conn = service[3]() + try: + records = conn.connection.execute(select( + mtproto_send_intents.c.object_id, mtproto_send_intents.c.message_id, + )).all() + assert len(records) == 1 and records[0].message_id is None + guard = GCReferenceGuard(ManifestStore(conn)) + evidence = guard.evidence(records[0].object_id) + assert "mtproto_send_intent_or_receipt" in evidence["reference_blockers"] + assert evidence["deletion_permitted"] is False + finally: + conn.close() + + +def test_wrong_account_has_no_cross_transport_fallback(service): + runtime, _ = configured_runtime() + db = service[3]() + try: + user_transport = UnifiedUserTransport(runtime, UserSendJournal(db)) + ref = SimpleNamespace(account="other-user", chat_id=-1001234567890, message_id=1) + with pytest.raises(FileNotFoundError): + asyncio.run(user_transport.download(ref, Path("/tmp/never-written"))) + finally: + db.close() + + +def test_live_runtime_uses_honest_client_metadata_and_rejects_bot(monkeypatch, tmp_path): + """Telethon app/device identity is explicit; only a user can access history.""" + import telethon + import telethon.sessions + import telethon.tl.types + + observed = [] + fixture_file = tmp_path / "session" + fixture_file.write_text("test-user-session\n", encoding="ascii") + channel = SimpleNamespace(broadcast=True, creator=True) + class FixtureChannel: + broadcast = True + creator = True + + class FakeClient: + bot_identity = False + def __init__(self, session, api_id, api_hash, **options): + observed.append((session, api_id, api_hash, options)) + self.connected = False + async def connect(self): + self.connected = True + async def disconnect(self): + self.connected = False + async def get_me(self): + return SimpleNamespace(bot=type(self).bot_identity) + async def iter_dialogs(self, limit): + assert limit == 5000 + yield SimpleNamespace(id=-1001234567890, entity=FixtureChannel()) + async def get_peer_id(self, target): + return -1001234567890 + def is_connected(self): + return self.connected + + monkeypatch.setattr(telethon, "TelegramClient", FakeClient) + monkeypatch.setattr(telethon.sessions, "StringSession", lambda value: value) + monkeypatch.setattr(telethon.tl.types, "Channel", FixtureChannel) + for key, value in { + "TELEDRIVE_STORAGE_TRANSPORT": "telethon", + }.items(): + monkeypatch.setenv(key, value) + config_path = tmp_path / "config.yaml" + config_path.write_text( + "storage:\n" + ' channel: "-1001234567890"\n' + " apis:\n" + " - api_id: 1234\n" + f" api_hash: {'a' * 32}\n" + f" session_file: {fixture_file}\n", + encoding="ascii", + ) + monkeypatch.setenv("TELEDRIVE_CONFIG", str(config_path)) + async def scenario(): + runtime = UserSessionRuntime() + await runtime.start() + assert runtime.account == "mtproto-user" + assert runtime.chat_id == -1001234567890 + await runtime.stop() + assert runtime.client is None + FakeClient.bot_identity = True + with pytest.raises(RuntimeError, match="USER session"): + await UserSessionRuntime().start() + asyncio.run(scenario()) + assert len(observed) == 2 + session, api_id, api_hash, options = observed[0] + assert session == "test-user-session" + assert api_id == 1234 and api_hash == "a" * 32 + assert options["device_model"] == "TeleDrive Storage Client" + assert options["app_version"].startswith("TeleDrive") + assert options["receive_updates"] is False + assert options["flood_sleep_threshold"] == 0 + + +def test_user_can_read_old_bot_channel_message_without_bot_token(service, monkeypatch): + """An authorized user recovers old bot media by immutable chat/message IDs.""" + import web.s3 as gateway + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + payload = b"bot-written-payload-restored-with-user" + assert signed(service, "PUT", "/backup-bucket/bot-old.bin", payload).status_code == 200 + runtime, user = configured_runtime() + # Telegram history exposes the SAME channel message to the authorized user. + user.posts[1] = (payload, "old-bot-document") + monkeypatch.setattr(gateway, "storage_transport", lambda db: + UnifiedUserTransport(runtime, UserSendJournal(db))) + result = signed(service, "GET", "/backup-bucket/bot-old.bin") + assert result.status_code == 200 and result.content == payload + assert user.next_id == 0, "Bot message was read, never reuploaded" + + +def test_authenticated_admin_can_scan_user_channel_history_without_mutation(monkeypatch): + from fastapi.testclient import TestClient + import web.app as web + from uuid import uuid4 + + runtime, user = configured_runtime() + uid = str(uuid4()) + user.posts[9] = (b"owned", f"teledrive:{uid}:0:{'a'*16}") + class ScannableUser(FakeUser): + async def iter_messages(self, target, *, limit, max_id=None): + for mid in sorted(self.posts, reverse=True): + if max_id is None or mid < max_id: + yield SimpleNamespace( + id=mid, chat_id=-1001234567890, + message=self.posts[mid][1], out=True, media=True, + ) + scannable = ScannableUser() + scannable.posts = user.posts + runtime.client = scannable + runtime.transport = TelethonChunkTransport({ + runtime.account: (scannable, object()), + }) + monkeypatch.setenv("TELEDRIVE_STORAGE_TRANSPORT", "telethon") + monkeypatch.setattr(web, "mtproto_storage_runtime", runtime) + monkeypatch.setattr(web, "WEB_USER", "owner") + monkeypatch.setattr(web, "WEB_PASSWORD", "secure-test-password-1234") + client = TestClient(web.app, base_url="http://localhost") + assert client.get("/api/v1/admin/storage/channel-history").status_code == 401 + response = client.get( + "/api/v1/admin/storage/channel-history?limit=3", + auth=("owner", "secure-test-password-1234"), + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["candidate_object_count"] == 1 + assert body["candidate_objects"][0]["object_id"] == uid + assert body["payload_downloads"] == body["telegram_mutations"] == 0 + assert body["restorable_without_independent_manifest"] is False + assert response.headers["cache-control"] == "no-store" + invalid = client.get( + "/api/v1/admin/storage/channel-history?before_message_id=-10", + auth=("owner", "secure-test-password-1234"), + ) + assert invalid.status_code == 422 + monkeypatch.setenv("TELEDRIVE_STORAGE_TRANSPORT", "bot") + still_enabled = client.get( + "/api/v1/admin/storage/channel-history", + auth=("owner", "secure-test-password-1234"), + ) + # The transport is config-only: stale env vars never change behavior. + assert still_enabled.status_code == 200 + assert list(scannable.posts) == [9] + + +def test_production_storage_uses_only_user_mtproto_even_with_stale_bot_secrets(service, monkeypatch): + from fastapi import HTTPException + import web.app as web + + conn = service[3]() + runtime, _ = configured_runtime() + monkeypatch.setattr(web, "mtproto_storage_runtime", runtime) + monkeypatch.setenv("TELEDRIVE_STORAGE_BOT_TOKEN", "unused-old-credential") + monkeypatch.setenv("TELEDRIVE_STORAGE_BOT_CHAT_ID", "-1001234567890") + try: + transport = web.storage_transport(conn) + assert isinstance(transport, UnifiedUserTransport) + assert not hasattr(transport, "bot_fallback") + runtime.client = None + with pytest.raises(HTTPException) as unavailable: + web.storage_transport(conn) + assert unavailable.value.status_code == 503 + finally: + conn.close() diff --git a/tests/storage/test_storage_versions.py b/tests/storage/test_storage_versions.py new file mode 100644 index 00000000..16d3f36c --- /dev/null +++ b/tests/storage/test_storage_versions.py @@ -0,0 +1,152 @@ +"""Published-version inventory and alias-based restore (no Telegram re-upload).""" +from __future__ import annotations + +import hashlib +import time + +from tests.s3.test_s3_admin import admin +from tests.s3.test_s3_gateway import service, signed +from tests.s3.test_s3_multipart import begin, part, complete +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.copy import copy_object +from teledrive.storage.manifest import ManifestStore + + +def test_catalog_history_records_only_published_versions_and_restore_alias(admin): + client, path, _ = admin + now = int(time.time()) + conn = connect_database(path) + try: + cat = BucketCatalog(ManifestStore(conn)) + cat.create_bucket("published", now) + def empty(name): + ident = cat.manifests.begin("published", name, 0, now) + cat.manifests.finish(ident, hashlib.sha256(b"").hexdigest(), now) + return ident + + one = empty("file.bin") + # Non-published READY manifests must not be advertised as versions. + uncommitted = empty("file.bin") + assert cat.list_versions("published", "file.bin") == [] + cat.publish(one, now=now) + assert [v["version_id"] for v in cat.list_versions( + "published", "file.bin" + )] == [one] + two = empty("file.bin") + cat.publish(two, now=now) + before = cat.list_versions("published", "file.bin") + assert {v["version_id"] for v in before} == {one, two} + assert [v["version_id"] for v in before if v["is_current"]] == [two] + finally: + conn.close() + + url = f"/api/v1/admin/storage/versions/{one}/restore" + auth = ("testadmin", "secure-auth-password") + payload = {"bucket": "published", "key": "file.bin", "expected_generation": 2} + assert client.post(url, json=payload).status_code in (401, 403) + assert client.post(url, auth=auth, json=payload).status_code == 403 + wrong = client.post(url, auth=auth, + headers={"x-teledrive-csrf": "1"}, + json={**payload, "expected_generation": 1}) + assert wrong.status_code == 409 + restored = client.post(url, auth=auth, + headers={"x-teledrive-csrf": "1"}, json=payload) + assert restored.status_code == 200, restored.text + fresh = restored.json() + assert fresh["generation"] == 3 + assert fresh["source_version_id"] == one + assert fresh["id"] not in {one, two} + + conn = connect_database(path) + try: + cat = BucketCatalog(ManifestStore(conn)) + versions = cat.list_versions("published", "file.bin") + assert {v["version_id"] for v in versions} == {one, two, fresh["id"]} + assert [v["version_id"] for v in versions if v["is_current"]] == [fresh["id"]] + assert cat.manifests.parts(fresh["id"]) == [] + assert cat.get("published", "file.bin").item.id == fresh["id"] + assert uncommitted not in {v["version_id"] for v in versions} + assert cat.remove_current("published", "file.bin") + assert cat.list_versions("published", "file.bin") + finally: + conn.close() + + again = client.post(url, auth=auth, + headers={"x-teledrive-csrf": "1"}, + json={"bucket": "published", "key": "file.bin", + "expected_generation": 0}) + assert again.status_code == 200, again.text + + +def test_copy_and_multipart_publish_snapshot_ids_are_recorded(service): + assert signed(service, "PUT", "/backup-bucket").status_code == 200 + assert signed(service, "PUT", "/backup-bucket/source.bin", b"content").status_code == 200 + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + initial = catalog.get("backup-bucket", "source.bin").item.id + result = copy_object( + catalog, "backup-bucket", "source.bin", + "backup-bucket", "copy.bin", + ) + assert {v["version_id"] for v in catalog.list_versions( + "backup-bucket", "copy.bin" + )} == {result.object_id} + assert catalog.list_versions("backup-bucket", "source.bin")[0]["version_id"] == initial + finally: + db.close() + + uid = begin(service, key="multipart.bin") + tag = part(service, uid, 1, b"upload-once", key="multipart.bin") + committed = complete(service, uid, [(1, tag)], key="multipart.bin") + assert committed.status_code == 200, committed.text + db = service[3]() + try: + catalog = BucketCatalog(ManifestStore(db)) + latest = catalog.get("backup-bucket", "multipart.bin").item.id + assert catalog.list_versions("backup-bucket", "multipart.bin")[0]["version_id"] == latest + finally: + db.close() + + +def test_legacy_backfill_only_indexes_live_proven_heads(admin): + client, path, _ = admin + now = int(time.time()) + db = connect_database(path) + try: + from sqlalchemy import delete + from teledrive.storage.catalog import published_versions + cat = BucketCatalog(ManifestStore(db)) + cat.create_bucket("legacy-vers", now) + prior = cat.manifests.begin("legacy-vers", "current", 0, now) + cat.manifests.finish(prior, hashlib.sha256(b"").hexdigest(), now) + cat.publish(prior, now=now) + never_published = cat.manifests.begin("legacy-vers", "staged", 0, now) + cat.manifests.finish(never_published, hashlib.sha256(b"").hexdigest(), now) + db.connection.execute(delete(published_versions).where( + published_versions.c.object_id == prior + )) + db.commit() + assert cat.list_versions("legacy-vers", "current") == [] + finally: + db.close() + + endpoint = "/api/v1/admin/storage/versions/index-current" + auth = ("testadmin", "secure-auth-password") + headers = {"x-teledrive-csrf": "1"} + assert client.post(endpoint, auth=auth, json={"execute":True}).status_code == 403 + preview = client.post(endpoint, auth=auth, headers=headers, json={}) + assert preview.status_code == 200 + assert preview.json()["dry_run"] and prior in preview.json()["ids"] + done = client.post(endpoint, auth=auth, headers=headers, + json={"execute":True}) + assert done.status_code == 200 + assert done.json()["count"] == 1 + db = connect_database(path) + try: + cat = BucketCatalog(ManifestStore(db)) + assert cat.list_versions("legacy-vers", "current")[0]["version_id"] == prior + assert cat.list_versions("legacy-vers", "staged") == [] + finally: + db.close() diff --git a/tests/storage/test_telegram_dr_rehearsal.py b/tests/storage/test_telegram_dr_rehearsal.py new file mode 100644 index 00000000..e81ea850 --- /dev/null +++ b/tests/storage/test_telegram_dr_rehearsal.py @@ -0,0 +1,223 @@ +"""Read-only USER-MTProto DR restores a SQL snapshot and downloads real bytes. + +Uses the production chunk engine plus a disk-backed Telethon stand-in, +and verifies that the archive and live SQL files are never modified. +There is no Bot API transport in the rehearsal path. +""" +from __future__ import annotations + +import asyncio +import hashlib +import os +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts.ops.backup_metadata import backup_sqlite +from scripts.ops.rehearse_telegram_dr import ( + ReadOnlyUserChannelTransport, verify_restored_telegram_object, + verify_original_user_channel, +) +from teledrive.db import connect_database +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.telethon_adapter import TelethonChunkTransport + + +class FakeUserClient: + """Disk-backed stand-in for a connected Telethon user client.""" + + def __init__(self): + self.documents: dict[int, bytes] = {} + self.sent = 0 + self.reads = 0 + self.deletions = 0 + + async def send_file(self, target, path, caption=None, force_document=False): + self.sent += 1 + self.documents[self.sent] = Path(path).read_bytes() + return SimpleNamespace(id=self.sent, chat_id=-10093744) + + async def get_peer_id(self, target): + return -10093744 + + async def get_messages(self, peer, ids): + return SimpleNamespace(id=ids, media=True, chat_id=-10093744) + + async def download_media(self, message, file): + self.reads += 1 + Path(file).write_bytes(self.documents[message.id]) + return str(file) + + async def connect(self): + pass + + async def disconnect(self): + pass + + +def seed(tmp_path, size, *, account="api-bot"): + root = tmp_path / "live" + root.mkdir() + state = root / "manifest.db" + source = root / "body.bin" + payload = os.urandom(size) + source.write_bytes(payload) + remote = FakeUserClient() + db = connect_database(str(state)) + try: + catalog = BucketCatalog(ManifestStore(db)) + catalog.create_bucket("dr-test", int(time.time())) + transport = TelethonChunkTransport({account: (remote, object())}) + stored = asyncio.run(TelegramObjectStorage( + catalog.manifests, transport, chunk_size=64 * 1024, + scratch_directory=root, + ).put_file("dr-test", "restore/sample.bin", source)) + catalog.publish(stored.id, now=int(time.time())) + assert stored.sha256 == hashlib.sha256(payload).hexdigest() + finally: + db.close() + archive = tmp_path / "sealed-archive.db" + checksum = backup_sqlite(state, archive) + return state, archive, checksum, remote, payload + + +def reader_for(remote, *, account, chat_id): + return ReadOnlyUserChannelTransport( + remote, object(), account=account, chat_id=chat_id, + ) + + +@pytest.mark.parametrize("size", [0, 1, 65537, 262147]) +def test_staging_dr_snapshot_and_user_retrieval_are_byte_exact(tmp_path, size): + state, archive, receipt, remote, payload = seed(tmp_path, size) + live_before = hashlib.sha256(state.read_bytes()).hexdigest() + archive_before = archive.read_bytes() + sent_before = remote.sent + reader = reader_for(remote, account="owner-user", chat_id=-10093744) + report = asyncio.run(verify_restored_telegram_object( + archive, receipt, bucket="dr-test", key="restore/sample.bin", + account="api-bot", chat_id=-10093744, transport=reader, + )) + assert report["verified"] and report["archive_sha256"] == receipt + assert report["sha256"] == hashlib.sha256(payload).hexdigest() + assert report["size"] == report["readback_bytes"] == size + assert report["telegram_uploads"] == report["telegram_deletions"] == 0 + assert report["source_database_modified"] is False + assert archive.read_bytes() == archive_before + assert hashlib.sha256(state.read_bytes()).hexdigest() == live_before + assert remote.sent == sent_before and remote.deletions == 0 + assert remote.reads >= (0 if size == 0 else 1) + + +@pytest.mark.parametrize("error", ["tampered-archive", "tampered-remote", + "oversized-budget", "wrong-bucket"]) +def test_staging_dr_denies_corrupt_and_unauthorized_inputs(tmp_path, error): + state, archive, receipt, remote, payload = seed(tmp_path, 65537) + sent_before = remote.sent + reader = reader_for(remote, account="owner-user", chat_id=-10093744) + kwargs = dict(bucket="dr-test", key="restore/sample.bin", + account="api-bot", chat_id=-10093744, transport=reader) + if error == "tampered-archive": + with archive.open("ab") as file: + file.write(b"altered") + elif error == "tampered-remote": + remote.documents[1] = b"modified" * 1024 + elif error == "oversized-budget": + kwargs["max_bytes"] = 65536 + else: + kwargs["bucket"] = "another-bucket" + with pytest.raises((RuntimeError, ValueError, LookupError, FileNotFoundError)): + asyncio.run(verify_restored_telegram_object( + archive, receipt, **kwargs, + )) + assert remote.sent == sent_before and remote.deletions == 0 + assert state.is_file() + + +def test_reader_scope_denies_foreign_account_and_wrong_channel(tmp_path): + state, archive, receipt, remote, _ = seed(tmp_path, 65537, account="owner-user") + for account, channel in [("other-account", -10093744), + ("owner-user", -10093745)]: + reader = reader_for(remote, account=account, chat_id=channel) + with pytest.raises(FileNotFoundError): + asyncio.run(verify_restored_telegram_object( + archive, receipt, bucket="dr-test", key="restore/sample.bin", + account="owner-user", chat_id=-10093744, transport=reader, + )) + assert remote.deletions == 0 + + +def test_reader_exposes_no_mutation_methods(tmp_path): + state, archive, receipt, remote, _ = seed(tmp_path, 1) + reader = reader_for(remote, account="owner-user", chat_id=-10093744) + assert not hasattr(reader, "delete") and not hasattr(reader, "upload") + report = asyncio.run(verify_restored_telegram_object( + archive, receipt, bucket="dr-test", key="restore/sample.bin", + account="api-bot", chat_id=-10093744, transport=reader, + )) + assert report["verified"] and remote.deletions == 0 + + +def test_operator_mtproto_dr_refuses_bot_user_and_wrong_channel(monkeypatch, tmp_path): + import telethon + import telethon.sessions + import telethon.tl.types + + state, archive, receipt, remote, payload = seed(tmp_path, 4097) + session_file = tmp_path / "authorized-user.session" + session_file.write_text("fake-authorized-session\n", encoding="ascii") + + class Channel: + broadcast = True + + class FakeConnectedClient: + identity_bot = False + peer = -10093744 + + def __init__(self, session, api_id, api_hash, **kwargs): + self.options = kwargs + + async def connect(self): + pass + + async def disconnect(self): + pass + + async def get_me(self): + return SimpleNamespace(bot=type(self).identity_bot) + + async def iter_dialogs(self, limit): + assert limit == 5000 + yield SimpleNamespace(id=-10093744, entity=Channel()) + + async def get_peer_id(self, channel): + return type(self).peer + + async def get_messages(self, channel, ids): + return SimpleNamespace(id=ids, media=True) + + async def download_media(self, message, file): + Path(file).write_bytes(remote.documents[message.id]) + return str(file) + + monkeypatch.setattr(telethon, "TelegramClient", FakeConnectedClient) + monkeypatch.setattr(telethon.sessions, "StringSession", lambda s: s) + monkeypatch.setattr(telethon.tl.types, "Channel", Channel) + params = dict(bucket="dr-test", key="restore/sample.bin", + account="api-bot", chat_id=-10093744, + target="-10093744", api_id=12345, + api_hash="a" * 32, session_file=session_file) + result = asyncio.run(verify_original_user_channel(archive, receipt, **params)) + assert result["verified"] and result["sha256"] == hashlib.sha256(payload).hexdigest() + FakeConnectedClient.identity_bot = True + with pytest.raises(ValueError, match="USER"): + asyncio.run(verify_original_user_channel(archive, receipt, **params)) + FakeConnectedClient.identity_bot = False + FakeConnectedClient.peer = -100987654321 + with pytest.raises(ValueError, match="identity mismatch"): + asyncio.run(verify_original_user_channel(archive, receipt, **params)) + assert remote.deletions == 0 diff --git a/tests/storage/test_telegram_history_inventory.py b/tests/storage/test_telegram_history_inventory.py new file mode 100644 index 00000000..779c6b35 --- /dev/null +++ b/tests/storage/test_telegram_history_inventory.py @@ -0,0 +1,243 @@ +"""Read-only Telegram history discovery remains bounded and unverified.""" +from __future__ import annotations + +import asyncio +import stat +from types import SimpleNamespace +from uuid import uuid4 + +import pytest + +from teledrive.storage.history_inventory import ( + inventory_channel, parse_candidate_marker, write_private_report, +) + + +def marker(object_id, ordinal, prefix="a" * 16): + return f"teledrive:{object_id}:{ordinal}:{prefix}" + + +def message(mid, caption, *, out=True, chat_id=-100123, media=True): + return SimpleNamespace( + id=mid, message=caption, out=out, chat_id=chat_id, + media=object() if media else None, + ) + + +class FakeTelegram: + def __init__(self, messages, *, fail_after=None): + self.messages = messages + self.fail_after = fail_after + self.calls = [] + + async def get_peer_id(self, target): + self.calls.append(("get_peer_id", target)) + return -100123 + + async def iter_messages(self, target, *, limit): + self.calls.append(("iter_messages", target, limit)) + for index, item in enumerate(self.messages[:limit]): + if index == self.fail_after: + raise RuntimeError("Lost Telegram history read") + yield item + + +def scan(messages, *, limit=100): + client = FakeTelegram(messages) + report = asyncio.run(inventory_channel( + client, "authorized-channel", account="original", scan_limit=limit, + )) + return report, client + + +def test_valid_markers_are_candidates_not_restorable_manifests(): + ident = str(uuid4()) + report, client = scan([ + message(105, marker(ident, 1, "b" * 16)), + message(104, marker(ident, 0)), + message(103, "ordinary channel post"), + ]) + assert report["scanned_messages"] == 3 + assert report["candidate_object_count"] == 1 + assert report["candidate_chunk_count"] == 2 + assert report["readable_window_exhausted"] and not report["truncated"] + group = report["candidate_objects"][0] + assert [p["ordinal"] for p in group["chunks"]] == [0, 1] + assert [p["message_id"] for p in group["chunks"]] == [104, 105] + assert group["candidate_ordinals_contiguous"] + assert group["bucket"] is None and group["key"] is None + assert group["whole_object_sha256"] is None + assert not group["payload_verified"] and not group["original_mapping_verified"] + assert report["complete_unknown_history_discovered"] is False + assert report["restorable_without_independent_manifest"] is False + assert report["payload_downloads"] == report["telegram_mutations"] == 0 + assert client.calls == [("get_peer_id", "authorized-channel"), + ("iter_messages", "authorized-channel", 101)] + + +def test_wrong_owner_channel_or_media_is_not_a_candidate(): + ident = str(uuid4()) + report, _ = scan([ + message(1, marker(ident, 0), out=False), + message(2, marker(ident, 0), chat_id=-100999), + message(3, marker(ident, 0), media=False), + message(4, marker(ident, 0)), + message(5, marker(ident, 1) + "extra"), + message(6, "other:" + marker(ident, 1)), + ]) + assert report["rejected_marker_messages"] == 3 + assert report["candidate_chunk_count"] == 1 + assert report["candidate_objects"][0]["chunks"][0]["message_id"] == 4 + + +@pytest.mark.parametrize("caption", [ + None, "", "teledrive:no-uuid:0:" + "a" * 16, + "prefix:teledrive:" + str(uuid4()) + ":0:" + "a" * 16, + marker(str(uuid4()).upper(), 0), + marker(str(uuid4()), 0, "x" * 16), + marker(str(uuid4()), 0) + "\n", +]) +def test_ambiguous_markers_are_rejected(caption): + assert parse_candidate_marker(caption) is None + + +def test_bounded_scan_reports_truncation_and_never_guesses_missing_parts(): + ident = str(uuid4()) + report, _ = scan([ + message(9, marker(ident, 7)), + message(8, marker(ident, 0)), + message(7, marker(ident, 1)), + ], limit=2) + assert report["scanned_messages"] == 2 + assert report["truncated"] is True + assert report["readable_window_exhausted"] is False + group = report["candidate_objects"][0] + assert group["candidate_ordinals_contiguous"] is False + assert [p["ordinal"] for p in group["chunks"]] == [0, 7] + + +def test_duplicate_ordinals_are_flagged_not_silently_collapsed(): + ident = str(uuid4()) + report, _ = scan([ + message(9, marker(ident, 0)), + message(8, marker(ident, 0, "b" * 16)), + ]) + group = report["candidate_objects"][0] + assert group["duplicate_ordinals"] is True + assert group["candidate_ordinals_contiguous"] is False + assert len(group["chunks"]) == 2 + + +@pytest.mark.parametrize("limit", [0, -1, 10_001, True, 1.5, "5"]) +def test_invalid_scan_limits_fail_before_network(limit): + client = FakeTelegram([]) + with pytest.raises(ValueError, match="scan_limit"): + asyncio.run(inventory_channel( + client, "authorized-channel", account="original", scan_limit=limit, + )) + assert client.calls == [] + + +def test_failed_remote_iteration_does_not_return_partial_inventory(): + ident = str(uuid4()) + client = FakeTelegram([message(1, marker(ident, 0)), + message(2, marker(ident, 1))], fail_after=1) + with pytest.raises(RuntimeError, match="Lost Telegram"): + asyncio.run(inventory_channel( + client, "authorized-channel", account="original", + )) + + +def test_duplicate_message_identity_fails_closed(): + ident = str(uuid4()) + with pytest.raises(ValueError, match="Duplicate Telegram"): + scan([message(3, marker(ident, 0)), + message(3, marker(ident, 1))]) + + +def test_private_report_uses_exclusive_creation_and_preserves_existing(tmp_path): + report = {"candidate_objects": [{"message_id": 123}]} + path = tmp_path / "inventory.json" + write_private_report(path, report) + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + original = path.read_bytes() + with pytest.raises(FileExistsError): + write_private_report(path, {"different": True}) + assert path.read_bytes() == original + + +def test_private_report_rejects_symlink_and_missing_directory(tmp_path): + target = tmp_path / "existing.json" + target.write_text("untouched", encoding="utf-8") + link = tmp_path / "link.json" + link.symlink_to(target) + with pytest.raises(FileExistsError): + write_private_report(link, {}) + assert target.read_text(encoding="utf-8") == "untouched" + with pytest.raises(FileNotFoundError): + write_private_report(tmp_path / "missing" / "inventory.json", {}) + + +def test_paginated_read_only_history_and_external_markers(): + ident = str(uuid4()) + bot_ident = str(uuid4()) + foreign = message(99, marker(bot_ident, 0), out=False) + telegram = FakeTelegram([ + message(103, "general channel post"), + message(102, marker(ident, 0)), + message(101, "another unrelated post"), + message(100, marker(ident, 1)), + foreign, + ]) + async def first(): + return await inventory_channel( + telegram, "authorized-channel", account="owner", + scan_limit=2, include_external_markers=True, + ) + first_page = asyncio.run(first()) + assert first_page["truncated"] + assert first_page["next_before_message_id"] == 102 + assert first_page["external_marker_candidate_count"] == 0 + assert first_page["candidate_chunk_count"] == 1 + assert telegram.calls[-1] == ("iter_messages", "authorized-channel", 3) + + class PagedTelegram(FakeTelegram): + async def iter_messages(self, target, *, limit, max_id=None): + self.calls.append(("iter_messages", target, limit, max_id)) + for item in self.messages: + if max_id is None or item.id < max_id: + yield item + limit -= 1 + if limit == 0: + break + + paged = PagedTelegram(telegram.messages) + other = asyncio.run(inventory_channel( + paged, "authorized-channel", account="owner", scan_limit=3, + before_message_id=first_page["next_before_message_id"], + include_external_markers=True, + )) + assert other["scanned_messages"] == 3 + assert other["truncated"] is False + assert other["candidate_chunk_count"] == 1 + assert other["external_marker_candidate_count"] == 1 + candidate = other["external_marker_candidates"][0] + assert candidate["object_id"] == bot_ident + assert candidate["message_id"] == 99 + assert candidate["ownership_verified"] is False + assert candidate["payload_verified"] is False + assert candidate["safe_to_publish_or_delete"] is False + assert other["restorable_without_independent_manifest"] is False + assert other["telegram_mutations"] == other["payload_downloads"] == 0 + assert paged.calls[-1] == ("iter_messages", "authorized-channel", 4, 102) + + +@pytest.mark.parametrize("before", [0, -1, "42", True, 1.5]) +def test_invalid_history_cursor_fails_before_network(before): + client = FakeTelegram([]) + with pytest.raises(ValueError, match="continuation"): + asyncio.run(inventory_channel( + client, "authorized-channel", account="owner", + before_message_id=before, + )) + assert client.calls == [] diff --git a/tests/storage/test_telegram_inventory_reconcile.py b/tests/storage/test_telegram_inventory_reconcile.py new file mode 100644 index 00000000..e0c259e7 --- /dev/null +++ b/tests/storage/test_telegram_inventory_reconcile.py @@ -0,0 +1,159 @@ +"""Adversarial offline comparison between Telegram markers and external manifests.""" +from __future__ import annotations + +import copy +import hashlib +from uuid import uuid4 + +import pytest +from teledrive.storage.reconcile_inventory import reconcile_inventory + + +def inputs(parts=(b"abc", b"def")): + identifier = str(uuid4()) + chunks = [{ + "ordinal": i, "account": "ci", "chat_id": -100123, + "message_id": 100 + i, "size": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } for i, data in enumerate(parts)] + plan = { + "object_id": identifier, "bucket": "recovery-test", + "key": "restored/key.bin", "size": sum(len(p) for p in parts), + "sha256": hashlib.sha256(b"".join(parts)).hexdigest(), + "chunks": chunks, + } + inventory = { + "schema_version": 1, + "scope": {"account": "ci", "chat_id": -100123, "scan_limit": 100}, + "candidate_objects": [{ + "object_id": identifier, "chunks": [{ + "ordinal": c["ordinal"], "account": c["account"], + "chat_id": c["chat_id"], "message_id": c["message_id"], + "checksum_prefix": c["sha256"][:16], + } for c in chunks], + }], + "candidate_object_count": 1, "candidate_chunk_count": len(chunks), + "scanned_messages": len(chunks), "truncated": False, + } + return plan, inventory + + +def test_exact_identity_is_only_permission_to_verify_bytes(): + plan, inventory = inputs() + report = reconcile_inventory(inventory, [plan]) + assert report["all_plans_match"] + assert report["ready_for_independent_byte_verification"] + assert report["objects_checked"] == 1 + assert report["matches"][0]["matched_chunk_count"] == 2 + assert not report["remote_ownership_verified"] + assert not report["payload_verified"] + assert not report["historic_versions_reconstructed"] + assert report["telegram_deletions"] == 0 + + +@pytest.mark.parametrize("field,value,raises", [ + ("message_id", 555, False), ("ordinal", 5, False), + ("checksum_prefix", "0" * 16, False), + ("chat_id", -100888, True), ("account", "other", True), +]) +def test_wrong_coordinate_never_matches(field, value, raises): + plan, inventory = inputs() + inventory["candidate_objects"][0]["chunks"][0][field] = value + if raises: + with pytest.raises(ValueError, match="Untrusted"): + reconcile_inventory(inventory, [plan]) + else: + assert not reconcile_inventory(inventory, [plan])["all_plans_match"] + + +def test_missing_or_duplicate_chunk_is_not_accepted(): + plan, inventory = inputs() + inventory["candidate_objects"][0]["chunks"].pop() + inventory["candidate_chunk_count"] = 1 + assert not reconcile_inventory(inventory, [plan])["all_plans_match"] + plan, inventory = inputs() + inventory["candidate_objects"][0]["chunks"][1]["message_id"] = 100 + with pytest.raises(ValueError, match="Duplicate"): + reconcile_inventory(inventory, [plan]) + + +def test_truncated_window_never_proves_unknown_history(): + plan, inventory = inputs() + inventory["truncated"] = True + report = reconcile_inventory(inventory, [plan]) + assert report["all_plans_match"] and report["inventory_window_truncated"] + assert not report["historic_versions_reconstructed"] + + +def test_unmapped_candidates_are_counted_not_named(): + plan, inventory = inputs() + outsider = copy.deepcopy(inventory["candidate_objects"][0]) + outsider["object_id"] = str(uuid4()) + for chunk in outsider["chunks"]: + chunk["message_id"] += 500 + inventory["candidate_objects"].append(outsider) + inventory["candidate_object_count"] = 2 + inventory["candidate_chunk_count"] = 4 + inventory["scanned_messages"] = 4 + report = reconcile_inventory(inventory, [plan]) + assert report["all_plans_match"] + assert report["extra_unmapped_candidates"] == 1 + assert report["automatic_import_performed"] is False + + +@pytest.mark.parametrize("mutation", [ + lambda i: i.update(schema_version=2), + lambda i: i.update(candidate_object_count=99), + lambda i: i.update(scanned_messages=0), + lambda i: i.update(truncated="yes"), + lambda i: i["candidate_objects"].append(i["candidate_objects"][0]), +]) +def test_malformed_inventory_is_rejected(mutation): + plan, inventory = inputs() + mutation(inventory) + with pytest.raises(ValueError): + reconcile_inventory(inventory, [plan]) + + +def test_wrong_manifest_fails_before_remote_access(): + plan, inventory = inputs() + plan["sha256"] = "0" * 16 + with pytest.raises(ValueError, match="independent"): + reconcile_inventory(inventory, [plan]) + + +def test_empty_object_is_not_supported_by_marker_inventory(): + plan, inventory = inputs(parts=()) + inventory["candidate_objects"][0]["chunks"] = [] + inventory["candidate_chunk_count"] = 0 + inventory["scanned_messages"] = 0 + answer = reconcile_inventory(inventory, [plan]) + assert answer["matches"][0]["result"] == "unproven_empty_object" + assert not answer["all_plans_match"] + + +def test_strict_external_manifest_preflight_blocks_cli_before_telegram(tmp_path, monkeypatch): + """A mismatch must never connect to Telegram or create a destination DB.""" + import sys + import json + from scripts.ops import rebuild_from_telegram as cli + + plan, inventory = inputs() + inventory["candidate_objects"][0]["chunks"][0]["message_id"] += 100 + plan_path = tmp_path / "plan.json" + inventory_path = tmp_path / "inventory.json" + destination = tmp_path / "must-not-create.sqlite" + plan_path.write_text(json.dumps(plan), encoding="utf-8") + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + monkeypatch.setenv("TELEDRIVE_REBUILD_API_ID", "123") + monkeypatch.setenv("TELEDRIVE_REBUILD_API_HASH", "ci-fake-api-hash") + monkeypatch.setenv("TELEDRIVE_REBUILD_SESSION", "ci-fake-session") + monkeypatch.setattr(sys, "argv", [ + "rebuild_from_telegram.py", "--plan", str(plan_path), + "--inventory", str(inventory_path), "--new-database", + str(destination), "--account", "ci", "--target", "ci-target", + ]) + with pytest.raises(SystemExit) as exc: + cli.main() + assert exc.value.code == 1 + assert not destination.exists() diff --git a/tests/storage/test_telegram_target_resolution.py b/tests/storage/test_telegram_target_resolution.py new file mode 100644 index 00000000..152086ef --- /dev/null +++ b/tests/storage/test_telegram_target_resolution.py @@ -0,0 +1,93 @@ +"""Offline checks for Telegram E2E destination selection.""" + +import asyncio +from types import SimpleNamespace + +import pytest + +from teledrive.storage.target_resolver import resolve_telegram_target + + +class FakeTelegramClient: + def __init__(self, dialogs=()): + self.dialogs = list(dialogs) + self.entity_requests = [] + + async def iter_dialogs(self): + for dialog in self.dialogs: + yield dialog + + async def get_entity(self, target): + self.entity_requests.append(target) + return ("resolved", target) + + +def channel(title="TeleDrive E2E Storage", *, public=False, is_group=False, is_channel=True): + entity = SimpleNamespace(username="public_username" if public else None) + return SimpleNamespace( + id=-1001234567890, + name=title, + entity=entity, + is_channel=is_channel, + is_group=is_group, + ) + + +def test_private_channel_name_resolves_from_dialogs(): + expected = channel(title="TeleDrive E2E Storage") + client = FakeTelegramClient([expected]) + result = asyncio.run(resolve_telegram_target( + client, "-1009876543210", " teledrive e2e storage " + )) + assert result is expected.entity + assert client.entity_requests == [] + + +def test_private_channel_name_rejects_no_match(): + client = FakeTelegramClient([channel(title="Unrelated")]) + with pytest.raises(ValueError, match="No broadcast channel"): + asyncio.run(resolve_telegram_target(client, None, "TeleDrive E2E Storage")) + assert client.entity_requests == [] + + +def test_private_channel_name_rejects_duplicates(): + client = FakeTelegramClient([channel(), channel()]) + with pytest.raises(ValueError, match="Multiple Telegram broadcast channels"): + asyncio.run(resolve_telegram_target(client, None, "TeleDrive E2E Storage")) + + +@pytest.mark.parametrize("wrong_channel", [ + channel(public=True), + channel(is_group=True), + channel(is_channel=False), +]) +def test_private_channel_name_rejects_public_groups_and_chats(wrong_channel): + client = FakeTelegramClient([wrong_channel]) + with pytest.raises(ValueError, match="No broadcast channel"): + asyncio.run(resolve_telegram_target(client, None, "TeleDrive E2E Storage")) + + +def test_private_channel_id_uses_joined_dialog_entity(): + expected = channel() + client = FakeTelegramClient([expected]) + result = asyncio.run(resolve_telegram_target(client, " -1001234567890 ")) + assert result is expected.entity + assert client.entity_requests == [] + + +def test_unknown_private_channel_fails_closed(): + client = FakeTelegramClient([channel()]) + with pytest.raises(ValueError, match="not in the test account's dialogs"): + asyncio.run(resolve_telegram_target(client, "-1002222222222")) + assert client.entity_requests == [] + + +@pytest.mark.parametrize("value, expected", [ + (None, "me"), + (" ", "me"), + (" @teledrive_e2e ", "@teledrive_e2e"), +]) +def test_non_numeric_target_uses_telethon_entity_resolver(value, expected): + client = FakeTelegramClient() + assert asyncio.run(resolve_telegram_target(client, value)) == ("resolved", expected) + assert client.entity_requests == [expected] diff --git a/tests/web/__init__.py b/tests/web/__init__.py new file mode 100644 index 00000000..b4710d02 --- /dev/null +++ b/tests/web/__init__.py @@ -0,0 +1 @@ +"""TeleDrive regression suite package.""" diff --git a/tests/web/test_api_keys.py b/tests/web/test_api_keys.py new file mode 100644 index 00000000..8aaf8de6 --- /dev/null +++ b/tests/web/test_api_keys.py @@ -0,0 +1,85 @@ +"""API key integration tests against persisted SQLite; no bearer secrets leaked.""" +from __future__ import annotations + +import time + +import pytest + +from teledrive.api_keys import ApiKeyStore +from teledrive.db import connect_database + + +@pytest.fixture +def keys(tmp_path): + connection = connect_database(str(tmp_path / "auth.db")) + try: + yield ApiKeyStore(connection) + finally: + connection.close() + + +def test_credentials_are_hashed_never_listed_and_persisted(keys): + created = keys.create("Backup client", ["objects:write", "objects:read"], now=100) + assert created.value.startswith("td_live_") + assert len(created.value) >= 40 + row = keys.list()[0] + assert row["name"] == "Backup client" + assert row["prefix"] == created.value[:16] + assert "value" not in row and "digest" not in row + assert created.value not in str(keys.list()) + assert keys.authorize(created.value, "objects:read", now=101) + assert keys.authorize(created.value, "objects:write", now=102) + assert not keys.authorize(created.value, "objects:delete", now=103) + assert keys.list()[0]["last_used_at"] == 102 + + +def test_read_only_and_wrong_key_do_not_escalate(keys): + key = keys.create("read", ["objects:read"], now=100) + assert not keys.authorize(key.value, "objects:write", now=101) + assert not keys.authorize(key.value + "x", "objects:read", now=101) + assert not keys.authorize("not-a-token", "objects:read", now=101) + assert keys.authorize(key.value, "objects:read", now=101) + + +def test_expiry_and_revoke_fence_future_requests(keys): + token = keys.create("short", ["objects:read"], now=100, expires_at=150) + assert keys.authorize(token.value, "objects:read", now=149) + assert not keys.authorize(token.value, "objects:read", now=150) + assert keys.revoke(token.id) + assert not keys.revoke(token.id) + assert not keys.authorize(token.value, "objects:read", now=101) + + +def test_rotation_invalidates_old_token_and_preserves_policy(keys): + previous = keys.create("bot", ["buckets:manage"], now=100, expires_at=500) + rotated = keys.rotate(previous.id, now=200) + assert rotated.id != previous.id and rotated.value != previous.value + assert keys.list()[0]["name"] == "bot" + assert not keys.authorize(previous.value, "buckets:manage", now=201) + assert keys.authorize(rotated.value, "buckets:manage", now=201) + assert not keys.authorize(rotated.value, "objects:read", now=201) + with pytest.raises(FileNotFoundError): + keys.rotate(previous.id, now=202) + + +def test_input_validation_rejects_bad_scopes_and_lifetime(keys): + with pytest.raises(ValueError): + keys.create("empty", [], now=100) + with pytest.raises(ValueError): + keys.create("invalid", ["root"], now=100) + with pytest.raises(ValueError): + keys.create("mixed", ["admin:*", "objects:read"], now=100) + with pytest.raises(ValueError): + keys.create("", ["objects:read"], now=100) + with pytest.raises(ValueError): + keys.create("expired", ["objects:read"], now=100, expires_at=100) + with pytest.raises(ValueError): + keys.create("too long", ["objects:read"], now=100, expires_at=100 + 11 * 365 * 86400) + assert keys.list() == [] + + +def test_admin_wildcard_and_scope_distinctions(keys): + key = keys.create("admin service", ["admin:*"], now=100) + for scope in ["buckets:manage", "objects:delete", "keys:manage"]: + assert keys.authorize(key.value, scope, now=110) + assert not keys.authorize("", "objects:read", now=110) diff --git a/tests/web/test_openapi.py b/tests/web/test_openapi.py new file mode 100644 index 00000000..70a211d7 --- /dev/null +++ b/tests/web/test_openapi.py @@ -0,0 +1,51 @@ +"""OpenAPI contract tests independent of Starlette's optional HTTP test client.""" +from __future__ import annotations + +from fastapi import HTTPException +from fastapi.security import HTTPBasicCredentials +from starlette.requests import Request +import pytest + +import web.app as webapp + + +def request(): + return Request({ + "type": "http", "method": "GET", "path": "/openapi.json", + "headers": [(b"host", b"localhost")], + "client": ("127.0.0.1", 1234), + "server": ("127.0.0.1", 8080), + "scheme": "http", "query_string": b"", + }) + + +def test_api_schema_exposes_typed_requests_and_security(): + schema = webapp.app.openapi() + assert schema["info"]["title"] == "TeleDrive API" + paths = schema["paths"] + for endpoint in ["/api/config", "/api/health"]: + assert endpoint in paths + assert "/docs" not in paths + assert "HTTPBasic" in schema["components"]["securitySchemes"] + + +def test_docs_routes_return_official_swagger_redoc_html(): + assert "Swagger" in webapp.swagger_docs(None).body.decode() + assert "ReDoc" in webapp.redoc_docs(None).body.decode() + assert "TeleDrive API" in webapp.openapi_schema(None).body.decode() + + +def test_documentation_requires_http_basic(monkeypatch): + monkeypatch.setattr(webapp, "WEB_USER", "apiadmin") + monkeypatch.setattr(webapp, "WEB_PASSWORD", "strong-password-for-tests") + webapp.AUTH_FAILURES.clear() + with pytest.raises(HTTPException) as exc: + webapp.require_auth(request(), None) + assert exc.value.status_code == 401 + + valid = HTTPBasicCredentials(username="apiadmin", password="strong-password-for-tests") + assert webapp.require_auth(request(), valid) is None + + +def test_health_is_public(): + assert webapp.health() == {"ok": True} diff --git a/tests/web/test_rest_api.py b/tests/web/test_rest_api.py new file mode 100644 index 00000000..4790a1df --- /dev/null +++ b/tests/web/test_rest_api.py @@ -0,0 +1,136 @@ +"""End-to-end HTTP authorization and catalog tests using a real SQLite file.""" +from __future__ import annotations + +import hashlib +import time + +import pytest +from fastapi.testclient import TestClient + +from teledrive.db import connect_database +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.catalog import BucketCatalog +import web.app as webapp + + +@pytest.fixture() +def api(tmp_path, monkeypatch): + path = tmp_path / "rest-api.sqlite" + monkeypatch.setattr(webapp, "db_connect", lambda: connect_database(str(path))) + monkeypatch.setattr(webapp, "WEB_USER", "ci-admin") + monkeypatch.setattr(webapp, "WEB_PASSWORD", "ci-local-password-1234") + webapp.AUTH_FAILURES.clear() + client = TestClient(webapp.app, headers={"host": "localhost"}) + yield client, path + client.close() + + +def admin_headers(): + return {"x-teledrive-csrf": "1"} + + +def create_key(client, scopes): + res = client.post("/api/v1/admin/keys", + auth=("ci-admin", "ci-local-password-1234"), + headers=admin_headers(), json={"name": "integration", "scopes": scopes}) + assert res.status_code == 201, res.text + return res.json() + + +def test_admin_endpoints_require_basic_and_csrf(api): + client, _ = api + assert client.get("/api/v1/admin/keys").status_code == 401 + assert client.post("/api/v1/admin/keys", + auth=("ci-admin", "ci-local-password-1234"), + json={"name": "x", "scopes": ["objects:read"]}).status_code == 403 + assert client.post("/api/v1/admin/keys", + auth=("ci-admin", "ci-local-password-1234"), + headers={"x-teledrive-csrf": "1", "origin": "http://evil.example"}, + json={"name": "x", "scopes": ["objects:read"]}).status_code == 403 + + +def test_key_rotation_revoke_and_metadata_never_discloses_secrets(api): + client, _ = api + old = create_key(client, ["buckets:read"]) + response = client.get("/api/v1/admin/keys", + auth=("ci-admin", "ci-local-password-1234")) + assert response.status_code == 200 + assert old["token"] not in response.text + assert "digest" not in response.text + assert client.get("/api/v1/buckets", + headers={"Authorization": f"Bearer {old['token']}"}).status_code == 200 + rotate = client.post(f"/api/v1/admin/keys/{old['id']}/rotate", + auth=("ci-admin", "ci-local-password-1234"), headers=admin_headers()) + assert rotate.status_code == 200 + new = rotate.json() + assert new["token"] != old["token"] + assert client.get("/api/v1/buckets", + headers={"Authorization": f"Bearer {old['token']}"}).status_code == 403 + assert client.get("/api/v1/buckets", + headers={"Authorization": f"Bearer {new['token']}"}).status_code == 200 + revoked = client.delete(f"/api/v1/admin/keys/{new['id']}", + auth=("ci-admin", "ci-local-password-1234"), headers=admin_headers()) + assert revoked.status_code == 200 + assert client.get("/api/v1/buckets", + headers={"Authorization": f"Bearer {new['token']}"}).status_code == 403 + + +def test_read_only_bearer_cannot_create_or_delete_bucket(api): + client, _ = api + key = create_key(client, ["buckets:read"]) + headers = {"Authorization": f"Bearer {key['token']}"} + assert client.get("/api/v1/buckets", headers=headers).status_code == 200 + assert client.post("/api/v1/buckets", headers=headers, + json={"name": "backups"}).status_code == 403 + assert client.delete("/api/v1/buckets/backups", headers=headers).status_code == 403 + assert client.get("/api/v1/buckets").status_code == 401 + + +def test_bucket_crud_and_object_metadata_scopes(api): + client, path = api + key = create_key(client, ["buckets:read", "buckets:manage", "objects:read", "objects:delete"]) + headers = {"Authorization": f"Bearer {key['token']}"} + assert client.post("/api/v1/buckets", json={"name": "backups"}, headers=headers).status_code == 201 + assert client.get("/api/v1/buckets", headers=headers).json()["buckets"] == ["backups"] + assert client.post("/api/v1/buckets", json={"name": "backups"}, headers=headers).status_code == 409 + + db = connect_database(str(path)) + try: + manifests = ManifestStore(db) + identifier = manifests.begin("backups", "📁/archive.bin", 0, int(time.time())) + manifests.finish(identifier, hashlib.sha256(b"").hexdigest(), int(time.time())) + cat = BucketCatalog(manifests) + cat.publish(identifier, now=int(time.time())) + finally: + db.close() + + result = client.get("/api/v1/buckets/backups/objects", headers=headers).json() + assert [x["key"] for x in result["objects"]] == ["📁/archive.bin"] + assert result["objects"][0]["sha256"] == hashlib.sha256(b"").hexdigest() + assert client.delete("/api/v1/buckets/backups", headers=headers).status_code == 409 + unlink = client.delete("/api/v1/buckets/backups/objects", + params={"key": "📁/archive.bin", "generation": 1}, headers=headers) + assert unlink.status_code == 200 and unlink.json()["unlinked"] + assert client.get("/api/v1/buckets/backups/objects", headers=headers).json()["objects"] == [] + # Historical Telegram data is deliberately not destroyed by unlink. + assert client.delete("/api/v1/buckets/backups", headers=headers).status_code == 409 + + +def test_bearer_token_does_not_unlock_basic_admin(api): + client, _ = api + key = create_key(client, ["admin:*"]) + token = {"Authorization": f"Bearer {key['token']}"} + assert client.get("/api/v1/admin/keys", headers=token).status_code == 401 + assert client.post("/api/v1/admin/keys", headers=token, + json={"name": "escalation", "scopes": ["admin:*"]}).status_code == 401 + + +def test_openapi_documents_rest_key_and_bucket_routes(api): + client, _ = api + response = client.get("/openapi.json", auth=("ci-admin", "ci-local-password-1234")) + assert response.status_code == 200 + paths = response.json()["paths"] + assert "/api/v1/admin/keys" in paths + assert "/api/v1/buckets" in paths + assert "/api/v1/buckets/{bucket}/objects" in paths + assert "HTTPBearer" in response.json()["components"]["securitySchemes"] diff --git a/tests/web/test_rest_transfer.py b/tests/web/test_rest_transfer.py new file mode 100644 index 00000000..eeb8ee5b --- /dev/null +++ b/tests/web/test_rest_transfer.py @@ -0,0 +1,190 @@ +"""HTTP binary transfer contract: actual SQLite metadata and fake USER MTProto bytes.""" +from __future__ import annotations + +import hashlib +import os + +import pytest +from fastapi.testclient import TestClient + +import web.app as webapp +PRODUCTION_TRANSPORT = webapp.storage_transport +from teledrive.db import connect_database + + +class FakeUserTransport: + """Disk-backed USER MTProto stand-in: same upload/download contract.""" + + def __init__(self): + self.parts = {} + self.counter = 0 + self.deletions = 0 + + async def upload(self, path, *, object_id, ordinal, checksum): + self.counter += 1 + identifier = f"test-part-{self.counter}" + self.parts[identifier] = path.read_bytes() + return "mtproto-user", -1001234567890, self.counter + + async def verify_target(self): + pass + + async def download(self, reference, destination): + destination.write_bytes(self.parts[f"test-part-{reference.message_id}"]) + + +@pytest.fixture +def service(tmp_path, monkeypatch): + path = tmp_path / "state.db" + bot = FakeUserTransport() + monkeypatch.setattr(webapp, "db_connect", lambda: connect_database(str(path))) + monkeypatch.setattr( + webapp, "storage_transport", lambda db: bot, + ) + monkeypatch.setattr(webapp, "STORAGE_SCRATCH_DIR", str(tmp_path / "scratch")) + monkeypatch.setattr(webapp, "REST_MAX_UPLOAD_BYTES", 4 * 1024**2) + monkeypatch.setattr(webapp, "WEB_USER", "admin-test") + monkeypatch.setattr(webapp, "WEB_PASSWORD", "some-long-password") + webapp.AUTH_FAILURES.clear() + client = TestClient(webapp.app, headers={"host": "localhost"}) + try: + response = client.post("/api/v1/admin/keys", + auth=("admin-test", "some-long-password"), + headers={"x-teledrive-csrf": "1"}, json={ + "name": "storage-integration", "scopes": [ + "buckets:read", "buckets:manage", "objects:read", + "objects:write", "objects:delete", + ], + }) + assert response.status_code == 201, response.text + token = response.json()["token"] + headers = {"Authorization": f"Bearer {token}"} + res = client.post("/api/v1/buckets", headers=headers, json={"name": "backup-bucket"}) + assert res.status_code == 201, res.text + yield client, headers, bot, tmp_path + finally: + client.close() + + +def put(client, headers, payload, key="data.bin"): + return client.put("/api/v1/buckets/backup-bucket/content", + params={"key": key}, headers={ + **headers, "Content-Type": "application/octet-stream" + }, content=payload) + + +@pytest.mark.parametrize("data", [b"", b"x", os.urandom(4097), os.urandom(1049600)]) +def test_binary_round_trip_persists_bytes_not_in_sql(service, data): + client, headers, bot, root = service + uploaded = put(client, headers, data) + assert uploaded.status_code == 201, uploaded.text + identifier = uploaded.json()["id"] + assert uploaded.json()["sha256"] == hashlib.sha256(data).hexdigest() + assert uploaded.json()["size"] == len(data) + result = client.get("/api/v1/buckets/backup-bucket/content", + params={"key": "data.bin"}, headers=headers) + assert result.status_code == 200, result.text + assert result.content == data + assert result.headers["etag"] == '"' + hashlib.sha256(data).hexdigest() + '"' + assert result.headers["x-teledrive-object-id"] == identifier + assert not list((root / "scratch").glob("api-upload-*")) + # SQLite stores manifests/file_ids, never the file bytes. + if len(data) > 100: + assert data[:100] not in (root / "state.db").read_bytes() + + +def test_http_range_suffix_and_unsatisfiable_ranges(service): + client, headers, _, _ = service + data = b"0123456789abcdef" + assert put(client, headers, data).status_code == 201 + for range_header, expected in [ + ("bytes=3-8", b"345678"), + ("bytes=7-", b"789abcdef"), + ("bytes=-4", b"cdef"), + ]: + result = client.get("/api/v1/buckets/backup-bucket/content", + params={"key": "data.bin"}, headers={**headers, "Range": range_header}) + assert result.status_code == 206 + assert result.content == expected + assert result.headers["content-length"] == str(len(expected)) + for invalid in ["bytes=50-60", "bytes=8-3", "bytes=0-1,4-6", "units=0-1"]: + result = client.get("/api/v1/buckets/backup-bucket/content", + params={"key": "data.bin"}, headers={**headers, "Range": invalid}) + assert result.status_code == 416 + + +def test_auth_and_upload_policy_enforced_before_telegram_send(service): + client, headers, bot, _ = service + args = {"key": "denied"} + url = "/api/v1/buckets/backup-bucket/content" + assert client.put(url, params=args, headers={"Content-Type": "application/octet-stream"}, + content=b"secret").status_code == 401 + assert client.put(url, params=args, headers=headers, content=b"secret").status_code == 415 + assert client.put(url, params=args, headers={ + **headers, "Content-Type": "application/octet-stream", + }, content=b"z" * (4 * 1024**2 + 1)).status_code == 413 + assert bot.counter == 0 + + +def test_missing_key_is_404_and_overwrite_is_versioned(service): + client, headers, bot, _ = service + assert client.get("/api/v1/buckets/backup-bucket/content", + params={"key": "absent"}, headers=headers).status_code == 404 + one = put(client, headers, b"one") + two = put(client, headers, b"two") + assert one.status_code == 201 and two.status_code == 201 + assert one.json()["id"] != two.json()["id"] + assert two.json()["generation"] == 2 + result = client.get("/api/v1/buckets/backup-bucket/content", + params={"key": "data.bin"}, headers=headers) + assert result.content == b"two" + uploads_before_conflict = bot.counter + conflicting = client.put("/api/v1/buckets/backup-bucket/content", + params={"key": "data.bin", "generation": 1}, + headers={**headers, "Content-Type": "application/octet-stream"}, content=b"three") + assert conflicting.status_code == 409 + assert bot.counter == uploads_before_conflict # zero Telegram writes for a stale generation + + +def test_missing_user_session_returns_503_before_upload(service, monkeypatch): + client, headers, bot, _ = service + # The service fixture temporarily patches the transport. Restore the + # production factory and remove its required Telegram configuration. + monkeypatch.setattr(webapp, "storage_transport", PRODUCTION_TRANSPORT) + monkeypatch.delenv("TELEDRIVE_STORAGE_MTPROTO_API_ID", raising=False) + monkeypatch.delenv("TELEDRIVE_STORAGE_MTPROTO_API_HASH", raising=False) + monkeypatch.delenv("TELEDRIVE_STORAGE_MTPROTO_CHANNEL", raising=False) + monkeypatch.delenv("TELEDRIVE_STORAGE_MTPROTO_SESSION_FILE", raising=False) + result = put(client, headers, b"secret", key="unconfigured") + assert result.status_code == 503 + assert bot.counter == 0 + + +def test_head_returns_actual_object_metadata_without_telegram_download(service): + client, headers, bot, _ = service + body = os.urandom(257) + uploaded = put(client, headers, body, key="head-check.bin") + assert uploaded.status_code == 201 + calls_before = bot.counter + result = client.head( + "/api/v1/buckets/backup-bucket/content", + params={"key": "head-check.bin"}, headers=headers, + ) + assert result.status_code == 200 + assert result.content == b"" + assert result.headers["content-length"] == str(len(body)) + assert result.headers["etag"] == '"' + hashlib.sha256(body).hexdigest() + '"' + assert result.headers["x-teledrive-object-id"] == uploaded.json()["id"] + assert bot.counter == calls_before + + absent = client.head( + "/api/v1/buckets/backup-bucket/content", + params={"key": "does-not-exist"}, headers=headers, + ) + assert absent.status_code == 404 + + forbidden = client.head( + "/api/v1/buckets/backup-bucket/content", + params={"key": "head-check.bin"}, + ) + assert forbidden.status_code == 401 diff --git a/tests/web/test_security.py b/tests/web/test_security.py new file mode 100644 index 00000000..c03d46c3 --- /dev/null +++ b/tests/web/test_security.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import os +import tempfile +import time +import unittest +from pathlib import Path + +import yaml + +from teledrive import db +from teledrive import uploader +import web.app as webapp +from fastapi import HTTPException +from starlette.requests import Request + + +class SecurityTests(unittest.TestCase): + def test_python_sources_live_in_packages(self): + root = Path(__file__).resolve().parents[2] + self.assertEqual([], sorted(p.name for p in root.glob("*.py"))) + self.assertTrue((root / "teledrive" / "__init__.py").is_file()) + self.assertTrue((root / "teledrive" / "uploader.py").is_file()) + self.assertTrue((root / "teledrive" / "db.py").is_file()) + + def test_web_config_redacts_and_restores_secrets(self): + raw = { + "telegram": { + "api_hash": "topsecret", + "accounts": [{"name": "primary", "api_hash": "accountsecret"}], + }, + "state": { + "database_url": "postgresql+psycopg://alice:dbsecret@db/teledrive?token=querysecret" + }, + } + redacted = webapp._redacted_config(raw) + rendered = str(redacted) + self.assertNotIn("topsecret", rendered) + self.assertNotIn("accountsecret", rendered) + self.assertNotIn("dbsecret", rendered) + self.assertNotIn("querysecret", rendered) + + restored = webapp._restore_secrets(redacted, raw) + self.assertEqual(restored["telegram"]["api_hash"], "topsecret") + self.assertEqual(restored["telegram"]["accounts"][0]["api_hash"], "accountsecret") + self.assertEqual(restored["state"]["database_url"], raw["state"]["database_url"]) + + def test_state_changes_require_csrf_header_and_same_origin(self): + def request(headers): + return Request({ + "type": "http", + "method": "POST", + "path": "/api/process/start", + "headers": [(k.lower().encode(), v.encode()) for k, v in headers.items()], + "client": ("127.0.0.1", 12345), + "server": ("127.0.0.1", 8080), + "scheme": "http", + "query_string": b"", + }) + + with self.assertRaises(HTTPException): + webapp._same_origin(request({"host": "localhost:8080", "origin": "http://localhost:8080"})) + with self.assertRaises(HTTPException): + webapp._same_origin(request({ + "host": "localhost:8080", + "origin": "http://evil.example", + "x-teledrive-csrf": "1", + })) + webapp._same_origin(request({ + "host": "localhost:8080", + "origin": "http://localhost:8080", + "x-teledrive-csrf": "1", + })) + + def test_uploader_log_redaction(self): + secret = "postgresql+psycopg://alice:supersecret@db.example/teledrive" + scrubbed = uploader._scrub_log_text(secret) + self.assertNotIn("supersecret", scrubbed) + self.assertIn("alice:***@db.example", scrubbed) + + def test_database_url_is_vendor_neutral_and_redacted(self): + url = "oracle+oracledb://alice:supersecret@db.example:1521/?service_name=FREEPDB1" + self.assertEqual(db.normalize_database_url(url), url) + rendered = db.redact_database_url(url) + self.assertNotIn("supersecret", rendered) + self.assertIn("***", rendered) + query_url = "oracle+oracledb://alice:pw@db.example/x?token=querysecret&mode=safe" + query_rendered = db.redact_database_url(query_url) + self.assertNotIn("querysecret", query_rendered) + self.assertIn("mode=safe", query_rendered) + + def test_atomic_file_claim_prevents_second_worker(self): + conn = db.connect_database("sqlite:///:memory:") + try: + now = int(time.time()) + conn.insert_pending("/tmp/example.bin", 10, 1.0, now) + conn.commit() + self.assertTrue(conn.claim_file("/tmp/example.bin", "worker-a", now, 3600)) + self.assertFalse(conn.claim_file("/tmp/example.bin", "worker-b", now, 3600)) + conn.mark_uploaded("/tmp/example.bin", 123, "primary", now) + conn.commit() + finally: + conn.close() + + def test_distributed_lease_is_exclusive(self): + with tempfile.TemporaryDirectory() as td: + url = f"sqlite:///{Path(td) / 'state.db'}" + a = db.connect_database(url) + b = db.connect_database(url) + try: + now = int(time.time()) + self.assertTrue(a.acquire_lease("uploader", "a", now, 60)) + self.assertFalse(b.acquire_lease("uploader", "b", now, 60)) + a.release_lease("uploader", "a") + self.assertTrue(b.acquire_lease("uploader", "b", now + 1, 60)) + finally: + a.close() + b.close() + + def test_symlink_is_not_scanned(self): + with tempfile.TemporaryDirectory() as source, tempfile.TemporaryDirectory() as outside: + good = Path(source) / "good.txt" + good.write_text("ok", encoding="utf-8") + secret = Path(outside) / "secret.txt" + secret.write_text("secret", encoding="utf-8") + link = Path(source) / "link.txt" + link.symlink_to(secret) + files = set(uploader.iter_source_files(source)) + self.assertIn(str(good.resolve()), files) + self.assertNotIn(str(link.absolute()), files) + + def test_caption_template_rejects_attribute_traversal(self): + with self.assertRaises(ValueError): + uploader.build_caption("example.txt", "{name.__class__}") + self.assertEqual(uploader.build_caption("example.txt", "{stem}{ext}"), "example.txt") + + def test_non_regular_file_is_not_scanned(self): + if not hasattr(os, "mkfifo"): + self.skipTest("mkfifo unavailable") + with tempfile.TemporaryDirectory() as source: + fifo = Path(source) / "pipe" + os.mkfifo(fifo) + self.assertNotIn(str(fifo), set(uploader.iter_source_files(source))) + # iter_source_files enumerates paths; scan-level regular-file enforcement is separate. + + def test_allowed_source_root_is_enforced(self): + old = os.environ.get("TELEDRIVE_ALLOWED_SOURCE_ROOTS") + try: + with tempfile.TemporaryDirectory() as allowed, tempfile.TemporaryDirectory() as outside: + os.environ["TELEDRIVE_ALLOWED_SOURCE_ROOTS"] = allowed + with self.assertRaises(ValueError): + uploader._validate_allowed_roots(outside, "TELEDRIVE_ALLOWED_SOURCE_ROOTS") + finally: + if old is None: + os.environ.pop("TELEDRIVE_ALLOWED_SOURCE_ROOTS", None) + else: + os.environ["TELEDRIVE_ALLOWED_SOURCE_ROOTS"] = old + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/web/test_unified_service.py b/tests/web/test_unified_service.py new file mode 100644 index 00000000..a725d40c --- /dev/null +++ b/tests/web/test_unified_service.py @@ -0,0 +1,231 @@ +"""One running TeleDrive instance: REST, SigV4 S3, and local jobs share bytes. + +All writes target the same actual SQLite catalog and fake Telegram Bot +transport. Socket-port dispatch preserves unmodified S3 signed paths. +""" +from __future__ import annotations + +import hashlib +import time + +import os +import pytest +from botocore.auth import S3SigV4Auth +from botocore.awsrequest import AWSRequest +from botocore.credentials import Credentials +from cryptography.fernet import Fernet +from fastapi.testclient import TestClient + +import web.app as web +import web.s3 as s3 +from web.unified import UnifiedApplication, app +from tests.web.test_rest_transfer import FakeUserTransport +from teledrive.db import connect_database +from teledrive.s3.credentials import S3CredentialStore, DEFAULT_PERMISSIONS +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.manifest import ManifestStore + + +@pytest.fixture +def unified(tmp_path, monkeypatch): + path=tmp_path/"unified.sqlite" + telegram=FakeUserTransport() + secret=Fernet.generate_key() + connect=lambda: connect_database(str(path)) + transport=lambda db: telegram + for module in (web,s3): + monkeypatch.setattr(module,"db_connect",connect) + monkeypatch.setattr(module,"storage_transport",transport) + monkeypatch.setattr(s3,"load_master_key",lambda: secret) + config_path = tmp_path / "config.yaml" + config_path.write_text( + "web:\n user: testadmin\n password: long-secure-test-password\n" + " allowed_hosts: localhost,127.0.0.1,[::1]\n" + "storage:\n channel: ''\n apis: []\n scratch_dir: " + str(tmp_path/"scratch") + "\n" + "state:\n database_url: sqlite:////" + str(path) + "\n" + "jobs:\n source_root: " + str(tmp_path/"uploads") + "\n auto_run: false\n", + encoding="ascii", + ) + monkeypatch.setattr(web, "CONFIG_PATH", config_path) + monkeypatch.setattr(web,"WEB_USER","testadmin") + monkeypatch.setattr(web,"WEB_PASSWORD","long-secure-test-password") + # Force the lifespan to read the config file (not the env-default path). + os.environ["TELEDRIVE_CONFIG"] = str(config_path) + _ = config_path + config_path.write_text( + "web:\n user: testadmin\n password: long-secure-test-password\n" + " allowed_hosts: localhost,127.0.0.1,[::1]\n" + "storage:\n channel: ''\n apis: []\n scratch_dir: " + str(tmp_path/"scratch") + "\n" + "state:\n database_url: sqlite:////" + str(path) + "\n" + "jobs:\n source_root: " + str(tmp_path/"uploads") + "\n auto_run: false\n", + encoding="ascii", + ) + web.AUTH_FAILURES.clear() + web.CONFIG_PATH = config_path + monkeypatch.setattr(web, "AUTH_DIR", tmp_path / "auth") + monkeypatch.setattr(web, "STORAGE_SCRATCH_DIR", str(tmp_path/"scratch")) + monkeypatch.setattr(web, "REST_MAX_UPLOAD_BYTES", 4*1024*1024) + root=tmp_path/"uploads" + root.mkdir() + monkeypatch.setattr(web, "JOB_SOURCE_ROOT", str(root)) + db=connect() + try: + creds=S3CredentialStore(db,secret).create("unified",list(DEFAULT_PERMISSIONS)) + finally: + db.close() + web_client=TestClient(app,base_url="http://localhost:8080") + s3_client=TestClient(app,base_url="http://localhost:9000") + try: + result=web_client.post("/api/v1/admin/keys", + auth=("testadmin","long-secure-test-password"), + headers={"X-TeleDrive-CSRF":"1"}, + json={"name":"unified-rest","scopes":[ + "buckets:read","buckets:manage","objects:read", + "objects:write","objects:delete", + ]}) + assert result.status_code==201,result.text + auth={"Authorization":"Bearer "+result.json()["token"]} + bucket=web_client.post("/api/v1/buckets", + headers=auth,json={"name":"unified-bucket"}) + assert bucket.status_code==201,bucket.text + yield web_client,s3_client,auth,creds,telegram,root,connect + finally: + web_client.close() + s3_client.close() + + +def signed(s3_client,creds,method,path,content=b""): + sha=hashlib.sha256(content).hexdigest() + headers={"x-amz-content-sha256":sha} + aws=AWSRequest(method=method,url="http://localhost:9000"+path, + data=content,headers=headers) + S3SigV4Auth(Credentials(creds.access_key,creds.secret_key), + "s3","us-east-1").add_auth(aws) + return s3_client.request(method,path,content=content,headers=dict(aws.headers)) + + +def test_single_process_listener_separation_and_health(unified): + ui,gateway,*_=unified + assert ui.get("/api/health").json()=={"ok":True} + assert gateway.get("/__health").json()=={"ok":True} + assert ui.get("/__health").status_code==404 + assert gateway.get("/api/health").status_code!=200 + assert ui.get("/api/config").status_code==401 + assert gateway.get("/").status_code in (400,403) + with pytest.raises(ValueError): + UnifiedApplication(web_port=8080,s3_port=8080) + + +def test_rest_s3_and_local_file_publish_one_catalog_and_shared_generations(unified): + ui,gateway,auth,creds,bot,root,connect=unified + rest_data=b"written-via-REST-"*17 + rest=ui.put("/api/v1/buckets/unified-bucket/content", + params={"key":"rest.bin"},headers={**auth,"Content-Type":"application/octet-stream"}, + content=rest_data) + assert rest.status_code==201,rest.text + s3_read=signed(gateway,creds,"GET","/unified-bucket/rest.bin") + assert s3_read.status_code==200,s3_read.text + assert s3_read.content==rest_data + + s3_data=b"written-via-S3-"*39 + remote=signed(gateway,creds,"PUT","/unified-bucket/s3.bin",s3_data) + assert remote.status_code==200,remote.text + local=ui.get("/api/v1/buckets/unified-bucket/content", + params={"key":"s3.bin"},headers=auth) + assert local.status_code==200,local.text + assert local.content==s3_data + + original=root/"folder-upload.bin" + folder_data=b"local-queue-same-storage"*19 + original.write_bytes(folder_data) + enq=ui.post("/api/v1/admin/storage/jobs", + auth=("testadmin","long-secure-test-password"), + headers={"X-TeleDrive-CSRF":"1"}, + json={"bucket":"unified-bucket","key":"folder/file.bin", + "source_path":str(original)}) + assert enq.status_code==201,enq.text + processed=ui.post("/api/v1/admin/storage/jobs/run-next", + auth=("testadmin","long-secure-test-password"), + headers={"X-TeleDrive-CSRF":"1"}) + assert processed.status_code==200,processed.text + assert processed.json()["job"]["state"]=="done" + folder_read=signed(gateway,creds,"GET","/unified-bucket/folder/file.bin") + assert folder_read.status_code==200,folder_read.text + assert folder_read.content==folder_data + + # The next S3 write updates the EXACT same REST head/generation. + replacement=b"s3-overwriting-rest" + overwrite=signed(gateway,creds,"PUT","/unified-bucket/rest.bin",replacement) + assert overwrite.status_code==200,overwrite.text + after=ui.get("/api/v1/buckets/unified-bucket/content", + params={"key":"rest.bin"},headers=auth) + assert after.content==replacement + assert after.headers["etag"]=='"'+hashlib.sha256(replacement).hexdigest()+'"' + db=connect() + try: + catalog=BucketCatalog(ManifestStore(db)) + assert catalog.get("unified-bucket","rest.bin").generation==2 + assert catalog.get("unified-bucket","s3.bin").item.sha256==hashlib.sha256(s3_data).hexdigest() + assert catalog.get("unified-bucket","folder/file.bin").item.sha256==hashlib.sha256(folder_data).hexdigest() + assert len(catalog.list_buckets())==1 + finally: + db.close() + assert bot.counter>=4 + + +def test_single_service_legacy_folder_cutover_is_visible_from_rest_and_s3(unified): + """One fenced legacy scan publishes through the shared durable job engine.""" + import asyncio + from types import SimpleNamespace + + from teledrive.storage.jobs import StorageUploadJobs + from teledrive.storage.legacy_bridge import run_legacy_cutover_cycle + + ui, gateway, auth, creds, bot, root, connect = unified + payload = b"single-process-legacy-ingestion" * 23 + folder = root / "archive" + folder.mkdir() + (folder / "document.bin").write_bytes(payload) + manager_db = connect() + lease_db = connect() + try: + cfg = SimpleNamespace( + source_dir=str(root), db_path=manager_db.url, + allowed_extensions=[".bin"], max_file_size_mb=1, + ) + queue = StorageUploadJobs(BucketCatalog(ManifestStore(manager_db))) + transport = web.storage_transport(manager_db) + async def cycle(): + return await run_legacy_cutover_cycle( + cfg, queue, transport, lease_db=lease_db, + bucket="unified-bucket", allowed_root=root, + scratch_directory=root, lease_name="uploader-global", + lease_ttl=60, chunk_size=512, prefix="legacy", max_files=10, + ) + before = bot.counter + first, completed = asyncio.run(cycle()) + assert first.queued == 1 and completed["state"] == "done" + assert bot.counter > before + published = queue.catalog.get( + "unified-bucket", "legacy/archive/document.bin", + ) + assert published.item.size == len(payload) + count = bot.counter + second, result = asyncio.run(cycle()) + assert second.queued == 0 and second.already_queued == 1 + assert result is None and bot.counter == count + # The original Telethon direct-send path was never invoked. + remote = signed( + gateway, creds, "GET", "/unified-bucket/legacy/archive/document.bin", + ) + assert remote.status_code == 200 and remote.content == payload + local = ui.get( + "/api/v1/buckets/unified-bucket/content", + params={"key": "legacy/archive/document.bin"}, + headers=auth, + ) + assert local.status_code == 200 and local.content == payload + assert local.headers["etag"] == remote.headers["etag"] + finally: + lease_db.close() + manager_db.close() diff --git a/uploader.py b/uploader.py deleted file mode 100644 index 9e29998c..00000000 --- a/uploader.py +++ /dev/null @@ -1,588 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import asyncio -from contextlib import contextmanager -import logging -import os -import random -import shutil -import sqlite3 -import tempfile -import time -from dataclasses import dataclass -from pathlib import Path -from typing import Iterable, Optional, Union - -import yaml -from telethon import TelegramClient -from telethon.errors import FloodWaitError, RPCError, SessionPasswordNeededError - - -# Telegram's free-account limit is 4,000 parts. Telethon uses 512 KiB parts, -# so keep split files below the resulting ~2 GiB boundary with some headroom. -TELEGRAM_FREE_FILE_LIMIT = 1_900_000_000 -SPLIT_BUFFER_SIZE = 1024 * 1024 - - -@dataclass -class AppConfig: - api_id: int - api_hash: str - phone: Optional[str] - target: Union[str, int] - session_path: str - source_dir: str - allowed_extensions: list[str] - max_file_size_mb: int - sleep_min_seconds: int - sleep_max_seconds: int - max_files_per_run: int - max_files_per_day: int - retry_attempts: int - backoff_base_seconds: int - floodwait_buffer_seconds: int - caption_template: str - send_mode: str - db_path: str - log_path: str - log_level: str - - -def load_config(path: str) -> AppConfig: - config_dir = Path(path).expanduser().resolve().parent - - def config_path(value: object) -> str: - candidate = Path(str(value)).expanduser() - return str(candidate if candidate.is_absolute() else config_dir / candidate) - - with open(path, "r", encoding="utf-8") as f: - raw = yaml.safe_load(f) - - tg = raw["telegram"] - up = raw["upload"] - st = raw["state"] - lg = raw["logging"] - - return AppConfig( - api_id=int(tg["api_id"]), - api_hash=str(tg["api_hash"]), - phone=tg.get("phone"), - target=tg["target"], - session_path=config_path(tg["session_path"]), - source_dir=config_path(up["source_dir"]), - allowed_extensions=[str(x).lower() for x in up.get("allowed_extensions", [])], - max_file_size_mb=int(up.get("max_file_size_mb", 0)), - sleep_min_seconds=int(up["sleep_min_seconds"]), - sleep_max_seconds=int(up["sleep_max_seconds"]), - max_files_per_run=int(up["max_files_per_run"]), - max_files_per_day=int(up["max_files_per_day"]), - retry_attempts=int(up.get("retry_attempts", 5)), - backoff_base_seconds=int(up.get("backoff_base_seconds", 10)), - floodwait_buffer_seconds=int(up.get("floodwait_buffer_seconds", 5)), - caption_template=str(up.get("caption_template", "{name}")), - send_mode=str(up.get("send_mode", "document")).lower(), - db_path=config_path(st["db_path"]), - log_path=config_path(lg["log_path"]), - log_level=str(lg.get("level", "INFO")), - ) - - -def ensure_parent(path: str) -> None: - Path(path).parent.mkdir(parents=True, exist_ok=True) - - -def setup_logging(cfg: AppConfig) -> None: - ensure_parent(cfg.log_path) - level = getattr(logging, cfg.log_level.upper(), logging.INFO) - logging.basicConfig( - level=level, - format="%(asctime)s [%(levelname)s] %(message)s", - handlers=[logging.FileHandler(cfg.log_path, encoding="utf-8"), logging.StreamHandler()], - ) - - -def connect_db(db_path: str) -> sqlite3.Connection: - ensure_parent(db_path) - conn = sqlite3.connect(db_path, timeout=30) - conn.execute("PRAGMA busy_timeout=30000") - conn.execute("PRAGMA journal_mode=WAL;") - conn.execute( - """ - CREATE TABLE IF NOT EXISTS files ( - path TEXT PRIMARY KEY, - size INTEGER NOT NULL, - mtime REAL NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - tg_message_id INTEGER, - attempts INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - first_seen_ts INTEGER NOT NULL, - last_update_ts INTEGER NOT NULL, - uploaded_ts INTEGER - ); - """ - ) - conn.execute("CREATE INDEX IF NOT EXISTS idx_files_status ON files(status);") - conn.commit() - return conn - - -@contextmanager -def uploader_lock(db_path: str): - """Prevent overlapping uploader processes using an exclusive lock file.""" - lock_path = Path(db_path).with_suffix(Path(db_path).suffix + ".lock") - ensure_parent(str(lock_path)) - handle = None - try: - try: - handle = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) - except FileExistsError: - try: - pid = int(lock_path.read_text(encoding="utf-8").strip()) - os.kill(pid, 0) - except (OSError, ValueError): - lock_path.unlink(missing_ok=True) - handle = os.open(str(lock_path), os.O_CREAT | os.O_EXCL | os.O_WRONLY) - else: - raise RuntimeError(f"Another uploader run is already active (pid {pid})") - os.write(handle, str(os.getpid()).encode("ascii")) - yield - finally: - if handle is not None: - os.close(handle) - lock_path.unlink(missing_ok=True) - - -def iter_source_files(source_dir: str) -> Iterable[str]: - for root, _, files in os.walk(source_dir): - for name in files: - yield os.path.abspath(os.path.join(root, name)) - - -def file_allowed(path: str, cfg: AppConfig) -> bool: - if cfg.allowed_extensions: - ext = os.path.splitext(path)[1].lower() - if ext not in cfg.allowed_extensions: - return False - if cfg.max_file_size_mb > 0: - max_bytes = cfg.max_file_size_mb * 1024 * 1024 - try: - if os.path.getsize(path) > max_bytes: - return False - except OSError: - return False - return True - - -def scan_and_queue(conn: sqlite3.Connection, cfg: AppConfig) -> tuple[int, int, int]: - now = int(time.time()) - inserted = 0 - updated = 0 - unchanged = 0 - - for path in iter_source_files(cfg.source_dir): - if not file_allowed(path, cfg): - continue - try: - stat = os.stat(path) - except OSError: - continue - size = int(stat.st_size) - mtime = float(stat.st_mtime) - - row = conn.execute("SELECT size, mtime, status FROM files WHERE path = ?", (path,)).fetchone() - if row is None: - conn.execute( - """ - INSERT INTO files(path, size, mtime, status, first_seen_ts, last_update_ts) - VALUES(?, ?, ?, 'pending', ?, ?) - """, - (path, size, mtime, now, now), - ) - inserted += 1 - continue - - old_size, old_mtime, old_status = int(row[0]), float(row[1]), row[2] - if old_size == size and abs(old_mtime - mtime) < 1e-6: - unchanged += 1 - continue - - # File changed on disk; always re-queue safely. - new_status = "pending" - conn.execute( - """ - UPDATE files - SET size = ?, mtime = ?, status = ?, tg_message_id = NULL, - uploaded_ts = NULL, last_error = NULL, last_update_ts = ? - WHERE path = ? - """, - (size, mtime, new_status, now, path), - ) - updated += 1 - - conn.commit() - return inserted, updated, unchanged - - -def uploaded_today(conn: sqlite3.Connection) -> int: - row = conn.execute( - """ - SELECT COUNT(*) - FROM files - WHERE status = 'uploaded' - AND date(uploaded_ts, 'unixepoch', 'localtime') = date('now', 'localtime') - """ - ).fetchone() - return int(row[0] if row else 0) - - -def fetch_pending(conn: sqlite3.Connection, limit: int) -> list[sqlite3.Row]: - conn.row_factory = sqlite3.Row - rows = conn.execute( - """ - SELECT path, size, mtime, attempts - FROM files - WHERE status IN ('pending', 'failed') - ORDER BY first_seen_ts ASC - LIMIT ? - """, - (limit,), - ).fetchall() - return rows - - -def mark_failed(conn: sqlite3.Connection, path: str, err: str, attempts_inc: int = 1) -> None: - conn.execute( - """ - UPDATE files - SET status = 'failed', - attempts = attempts + ?, - last_error = ?, - last_update_ts = ? - WHERE path = ? - """, - (attempts_inc, err[:1000], int(time.time()), path), - ) - conn.commit() - - -def mark_uploaded(conn: sqlite3.Connection, path: str, msg_id: Optional[int]) -> None: - now = int(time.time()) - conn.execute( - """ - UPDATE files - SET status = 'uploaded', - tg_message_id = ?, - attempts = attempts + 1, - last_error = NULL, - uploaded_ts = ?, - last_update_ts = ? - WHERE path = ? - """, - (msg_id, now, now, path), - ) - conn.commit() - - -def build_caption(path: str, template: str) -> str: - name = os.path.basename(path) - stem, ext = os.path.splitext(name) - return template.format(name=name, stem=stem, ext=ext)[:1024] - - -def wait_for_auth_value(auth_dir: Optional[str], name: str, timeout: int = 600) -> str: - if not auth_dir: - return input(f"Please enter the {name}: ") - print(f"Please enter the {name}:", flush=True) - path = Path(auth_dir) / f"{name.replace(' ', '_')}.txt" - deadline = time.time() + timeout - while time.time() < deadline: - try: - value = path.read_text(encoding="utf-8").strip() - except FileNotFoundError: - value = "" - if value: - path.unlink(missing_ok=True) - return value - time.sleep(0.25) - raise TimeoutError(f"Timed out waiting for Telegram {name}") - - -async def ensure_client(cfg: AppConfig, login_code: Optional[str] = None, login_password: Optional[str] = None, auth_dir: Optional[str] = None) -> TelegramClient: - ensure_parent(cfg.session_path) - client = TelegramClient(cfg.session_path, cfg.api_id, cfg.api_hash) - if cfg.phone: - await client.connect() - if not await client.is_user_authorized(): - logging.info("Requesting a new Telegram login code for %s", cfg.phone) - sent = await client.send_code_request(cfg.phone) - logging.info("Telegram login code requested. Check your Telegram app for the code.") - code = login_code or wait_for_auth_value(auth_dir, "code") - try: - await client.sign_in(cfg.phone, code=code, phone_code_hash=sent.phone_code_hash) - except SessionPasswordNeededError: - logging.info("Telegram 2-step verification is required.") - password = login_password or wait_for_auth_value(auth_dir, "password") - await client.sign_in(password=password) - logging.info("Telegram authentication complete.") - else: - await client.start() - return client - - -def resolve_target(target: Union[str, int]) -> Union[str, int]: - if isinstance(target, int): - return target - if isinstance(target, str): - s = target.strip() - if s.startswith("-") and s[1:].isdigit(): - return int(s) - if s.isdigit(): - return int(s) - return s - return str(target) - - -async def upload_one(client: TelegramClient, target, path: str, caption: str, send_mode: str): - force_document = send_mode == "document" - supports_streaming = send_mode == "media" - return await client.send_file( - entity=target, - file=path, - caption=caption, - force_document=force_document, - supports_streaming=supports_streaming, - ) - - -def split_for_telegram(path: str, max_bytes: int = TELEGRAM_FREE_FILE_LIMIT) -> tuple[str, list[str]]: - """Create raw, lossless parts for a file that exceeds Telegram's free limit.""" - source = Path(path) - total_size = source.stat().st_size - part_count = (total_size + max_bytes - 1) // max_bytes - temp_dir = tempfile.mkdtemp(prefix=".teledrive-parts-", dir=str(source.parent)) - width = max(2, len(str(part_count))) - part_paths: list[str] = [] - - try: - with source.open("rb") as source_file: - for part_number in range(1, part_count + 1): - part_path = Path(temp_dir) / f"{source.stem}.part{part_number:0{width}d}{source.suffix}" - remaining = min(max_bytes, total_size - (part_number - 1) * max_bytes) - with part_path.open("wb") as part_file: - while remaining: - block = source_file.read(min(SPLIT_BUFFER_SIZE, remaining)) - if not block: - raise OSError(f"Unexpected end of file while splitting {path}") - part_file.write(block) - remaining -= len(block) - part_paths.append(str(part_path)) - except Exception: - shutil.rmtree(temp_dir, ignore_errors=True) - raise - - logging.info("Split oversized file into %s parts: %s", len(part_paths), path) - return temp_dir, part_paths - - -def is_file_parts_invalid(error: BaseException) -> bool: - message = str(error).lower() - return "file_parts_invalid" in message or "number of file parts is invalid" in message - - -async def process_uploads(cfg: AppConfig, conn: sqlite3.Connection, login_code: Optional[str] = None, login_password: Optional[str] = None, auth_dir: Optional[str] = None) -> None: - client = await ensure_client(cfg, login_code=login_code, login_password=login_password, auth_dir=auth_dir) - async with client: - target = await client.get_entity(resolve_target(cfg.target)) - today_count = uploaded_today(conn) - if today_count >= cfg.max_files_per_day: - logging.info( - "Daily limit reached (%s/%s). Exiting.", - today_count, - cfg.max_files_per_day, - ) - return - - pending = fetch_pending(conn, limit=cfg.max_files_per_run * 3) - if not pending: - logging.info("No pending files.") - return - - done_this_run = 0 - for row in pending: - if done_this_run >= cfg.max_files_per_run: - break - if today_count >= cfg.max_files_per_day: - logging.info("Reached daily cap during run.") - break - - path = row["path"] - expected_size = int(row["size"]) - expected_mtime = float(row["mtime"]) - - if not os.path.exists(path): - mark_failed(conn, path, "File missing on disk") - logging.warning("Missing file: %s", path) - continue - - try: - st = os.stat(path) - except OSError as e: - mark_failed(conn, path, f"os.stat failed: {e}") - logging.warning("Stat failed for %s: %s", path, e) - continue - - # Avoid uploading files that changed after scanning. - if int(st.st_size) != expected_size or abs(float(st.st_mtime) - expected_mtime) > 1e-6: - mark_failed(conn, path, "File changed since scan; will requeue on next scan") - logging.warning("Changed file skipped (will requeue): %s", path) - continue - - caption = build_caption(path, cfg.caption_template) - ok = False - split_dir: Optional[str] = None - for attempt in range(1, cfg.retry_attempts + 1): - try: - upload_paths = [path] - if expected_size >= TELEGRAM_FREE_FILE_LIMIT: - if split_dir is None: - split_dir, upload_paths = split_for_telegram(path) - else: - upload_paths = sorted(str(p) for p in Path(split_dir).iterdir()) - - msg_id = None - for part_number, upload_path in enumerate(upload_paths, start=1): - part_caption = caption - send_mode = cfg.send_mode - if len(upload_paths) > 1: - part_caption = f"{caption} [part {part_number}/{len(upload_paths)}]" - # Raw byte parts are not independently playable media. - send_mode = "document" - msg = await upload_one(client, target, upload_path, part_caption, send_mode) - msg_id = getattr(msg, "id", None) - - assert msg_id is not None - mark_uploaded(conn, path, msg_id) - done_this_run += 1 - today_count += 1 - ok = True - logging.info( - "Uploaded (%s/%s today, %s this run): %s", - today_count, - cfg.max_files_per_day, - done_this_run, - path, - ) - break - except FloodWaitError as e: - wait_s = int(getattr(e, "seconds", 0)) + cfg.floodwait_buffer_seconds - mark_failed(conn, path, f"FloodWait: {wait_s}s", attempts_inc=0) - logging.warning("FloodWait for %ss while uploading %s", wait_s, path) - await asyncio.sleep(wait_s) - break - except (RPCError, OSError, TimeoutError) as e: - if is_file_parts_invalid(e): - mark_failed(conn, path, f"Too large for Telegram free-account limit: {e}") - logging.error("Skipped oversized file %s: %s", path, e) - break - if attempt >= cfg.retry_attempts: - mark_failed(conn, path, f"{type(e).__name__}: {e}") - logging.error("Permanent failure %s: %s", path, e) - else: - backoff = cfg.backoff_base_seconds * (2 ** (attempt - 1)) - logging.warning( - "Retry %s/%s for %s after %ss due to: %s", - attempt, - cfg.retry_attempts, - path, - backoff, - e, - ) - await asyncio.sleep(backoff) - except Exception as e: - mark_failed(conn, path, f"Unexpected: {type(e).__name__}: {e}") - logging.exception("Unexpected error for %s: %s", path, e) - break - - if split_dir is not None: - shutil.rmtree(split_dir, ignore_errors=True) - - if ok: - sleep_s = random.randint(cfg.sleep_min_seconds, cfg.sleep_max_seconds) - logging.info("Sleeping %ss", sleep_s) - await asyncio.sleep(sleep_s) - - -def validate_config(cfg: AppConfig) -> None: - if cfg.sleep_min_seconds <= 0 or cfg.sleep_max_seconds <= 0: - raise ValueError("Sleep values must be > 0") - if cfg.sleep_min_seconds > cfg.sleep_max_seconds: - raise ValueError("sleep_min_seconds cannot be greater than sleep_max_seconds") - if cfg.max_files_per_run <= 0: - raise ValueError("max_files_per_run must be > 0") - if cfg.max_files_per_day <= 0: - raise ValueError("max_files_per_day must be > 0") - if cfg.retry_attempts <= 0: - raise ValueError("retry_attempts must be > 0") - if cfg.backoff_base_seconds < 0 or cfg.floodwait_buffer_seconds < 0: - raise ValueError("Backoff and floodwait buffer values cannot be negative") - if cfg.max_file_size_mb < 0: - raise ValueError("max_file_size_mb cannot be negative") - if cfg.send_mode not in {"document", "media", "auto"}: - raise ValueError("send_mode must be document, media, or auto") - if not cfg.api_hash.strip(): - raise ValueError("api_hash must not be empty") - if not str(cfg.target).strip(): - raise ValueError("target must not be empty") - try: - build_caption("example.txt", cfg.caption_template) - except (KeyError, ValueError, IndexError) as exc: - raise ValueError("caption_template may only use {name}, {stem}, and {ext}") from exc - if not os.path.isdir(cfg.source_dir): - raise ValueError(f"source_dir not found: {cfg.source_dir}") - - -def parse_args() -> argparse.Namespace: - p = argparse.ArgumentParser(description="Safe Telegram auto uploader") - p.add_argument("--config", required=True, help="Path to YAML config") - p.add_argument("--scan-only", action="store_true", help="Only scan and queue files, then exit") - p.add_argument("--run-once", action="store_true", help="Run one upload batch and exit") - p.add_argument("--no-scan", action="store_true", help="Skip scan phase before upload") - p.add_argument("--login-code", help=argparse.SUPPRESS) - p.add_argument("--login-password", help=argparse.SUPPRESS) - p.add_argument("--auth-dir", help=argparse.SUPPRESS) - return p.parse_args() - - -async def async_main() -> int: - args = parse_args() - cfg = load_config(args.config) - setup_logging(cfg) - validate_config(cfg) - conn = connect_db(cfg.db_path) - - with uploader_lock(cfg.db_path): - if not args.no_scan: - inserted, updated, unchanged = scan_and_queue(conn, cfg) - logging.info("Scan complete. inserted=%s updated=%s unchanged=%s", inserted, updated, unchanged) - - if args.scan_only and not args.run_once: - return 0 - - await process_uploads(cfg, conn, login_code=args.login_code, login_password=args.login_password, auth_dir=args.auth_dir) - return 0 - - -def main() -> int: - try: - return asyncio.run(async_main()) - except KeyboardInterrupt: - logging.warning("Interrupted by user") - return 130 - except Exception as e: - logging.exception("Fatal error: %s", e) - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/web/__init__.py b/web/__init__.py new file mode 100644 index 00000000..e7cbaa0d --- /dev/null +++ b/web/__init__.py @@ -0,0 +1 @@ +"""TeleDrive browser administration package.""" diff --git a/web/app.py b/web/app.py new file mode 100644 index 00000000..8b1e19ae --- /dev/null +++ b/web/app.py @@ -0,0 +1,1606 @@ +from __future__ import annotations + +import asyncio +import hmac +import copy +import sys +import tempfile +import os +import re +import time +from collections import deque +from contextlib import asynccontextmanager +from pathlib import Path +from urllib.parse import urlsplit +from typing import Any + +import yaml +from fastapi import Body, Depends, FastAPI, HTTPException, Query, Request, status +from fastapi.responses import FileResponse, JSONResponse, StreamingResponse, Response +from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html +from fastapi.openapi.utils import get_openapi +from pydantic import BaseModel, Field +from typing import Literal +from fastapi.security import HTTPBasic, HTTPBasicCredentials, HTTPBearer, HTTPAuthorizationCredentials +from fastapi.staticfiles import StaticFiles + +from teledrive import uploader +from teledrive.db import redact_database_url +from teledrive.api_keys import ApiKeyStore +from teledrive.s3.credentials import S3CredentialStore, load_master_key, DEFAULT_PERMISSIONS +from teledrive.s3.multipart import MultipartStore +from teledrive.storage.gc import StorageGCInventory +from teledrive.storage.manifest import ManifestStore +from teledrive.storage.jobs import StorageUploadJobs, UploadJobError, jobs as storage_job_rows +from teledrive.storage.catalog import BucketCatalog +from teledrive.storage.copy import copy_object, CopyConflict +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.telethon_adapter import RECOMMENDED_CHUNK_SIZE +from teledrive.storage.user_mode import ( + UserSessionRuntime, UserSendJournal, UnifiedUserTransport, transport_mode, +) +from teledrive.storage.history_inventory import inventory_channel +from teledrive.storage.mtproto_rate_guard import MtprotoRateGuard +mtproto_storage_runtime = UserSessionRuntime() + +BASE_DIR = Path(__file__).resolve().parent.parent +CONFIG_PATH = Path(os.environ.get("TELEDRIVE_CONFIG", "config/config.yaml")) +STATIC_DIR = Path(__file__).resolve().parent / "static" +REDACTED = "__REDACTED__" + +# Service settings come ONLY from config/config.yaml (owner policy); the +# TELEDRIVE_CONFIG env var just points at the file. Tests override the +# module attributes below directly. +WEB_USER = "admin" +WEB_PASSWORD = "" +WEB_PASSWORD_FILE = "" +ALLOW_INSECURE = False +MAX_REQUEST_BYTES = 1048576 +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "[::1]"] +AUTH_DIR = Path("/data/auth") +STORAGE_SCRATCH_DIR = "/data/tmp" +REST_MAX_UPLOAD_BYTES = 2 * 1024**3 +JOB_SOURCE_ROOT = "/uploads" +JOB_AUTO_RUN = True +ALLOWED_SQLITE_ROOTS = ("/data",) +AUTH_FAILURE_WINDOW_SECONDS = 300 +AUTH_FAILURE_LIMIT = 5 +SSL_CERTFILE = "" +SSL_KEYFILE = "" +DB_EXTRA_PACKAGES: tuple[str, ...] = () + + +def settings_bind_address() -> str: + from teledrive.settings import load_settings + try: + return load_settings(uploader.load_config(str(CONFIG_PATH)), + str(CONFIG_PATH)).bind_address + except Exception: + return "0.0.0.0" # nosec B104 — in-container bind, host mapping loopback + + +def settings_web_port() -> int: + from teledrive.settings import load_settings + try: + return load_settings(uploader.load_config(str(CONFIG_PATH)), + str(CONFIG_PATH)).web_port + except Exception: + return 8080 + + +def settings_s3_port() -> int: + from teledrive.settings import load_settings + try: + return load_settings(uploader.load_config(str(CONFIG_PATH)), + str(CONFIG_PATH)).s3_port + except Exception: + return 9000 + + +def _reload_service_settings() -> None: + """Read service settings from the deployment config into module attrs.""" + global WEB_USER, WEB_PASSWORD, WEB_PASSWORD_FILE, ALLOW_INSECURE + global MAX_REQUEST_BYTES, ALLOWED_HOSTS, AUTH_DIR, STORAGE_SCRATCH_DIR + global REST_MAX_UPLOAD_BYTES, JOB_SOURCE_ROOT, JOB_AUTO_RUN + global ALLOWED_SQLITE_ROOTS, AUTH_FAILURE_WINDOW_SECONDS, AUTH_FAILURE_LIMIT + global SSL_CERTFILE, SSL_KEYFILE, DB_EXTRA_PACKAGES + + from teledrive.settings import load_settings + cfg = uploader.load_config(str(CONFIG_PATH)) + settings = load_settings(cfg, str(CONFIG_PATH)) + WEB_USER = settings.web_user + WEB_PASSWORD = settings.web_password + WEB_PASSWORD_FILE = settings.web_password_file + ALLOW_INSECURE = settings.allow_insecure + MAX_REQUEST_BYTES = settings.max_request_bytes + ALLOWED_HOSTS = list(settings.allowed_hosts) + AUTH_DIR = Path(settings.auth_dir) + STORAGE_SCRATCH_DIR = settings.storage_scratch_dir + REST_MAX_UPLOAD_BYTES = settings.rest_max_upload_bytes + JOB_SOURCE_ROOT = settings.job_source_root + JOB_AUTO_RUN = settings.job_auto_run + ALLOWED_SQLITE_ROOTS = settings.allowed_sqlite_roots + AUTH_FAILURE_WINDOW_SECONDS = settings.auth_failure_window_seconds + AUTH_FAILURE_LIMIT = settings.auth_failure_limit + SSL_CERTFILE = settings.ssl_certfile + SSL_KEYFILE = settings.ssl_keyfile + DB_EXTRA_PACKAGES = settings.db_extra_packages + # Legacy uploader path allowlists read os.environ internally; seed them + # from the config so the operator never configures them via env. + os.environ["TELEDRIVE_ALLOWED_SOURCE_ROOTS"] = settings.allowed_source_roots + os.environ["TELEDRIVE_ALLOWED_SESSION_ROOTS"] = settings.allowed_session_roots + os.environ["TELEDRIVE_ALLOWED_LOG_ROOTS"] = settings.allowed_log_roots + os.environ["TELEDRIVE_ALLOWED_SQLITE_ROOTS"] = ",".join( + settings.allowed_sqlite_roots) + os.environ["TELEDRIVE_SPLIT_DIR"] = settings.split_dir + os.environ["TELEDRIVE_SPLIT_MIN_FREE_BYTES"] = str( + settings.split_min_free_bytes) + os.environ["TELEDRIVE_LOCK_DIR"] = settings.lock_dir + os.environ["TELEDRIVE_S3_MASTER_KEY_FILE"] = settings.s3_master_key_file +LOG_BUFFER: deque[str] = deque(maxlen=1500) +PROMPT_RE = re.compile(r"Please enter the (code|password) for account '([^']+)':", re.I) +_QUERY_SECRET_RE = re.compile( + r"(?i)([?&](?:password|passwd|pwd|token|access_token|secret|api_key|apikey)=)[^&#\s]+" +) + +security = HTTPBasic(auto_error=False) +bearer_auth = HTTPBearer(auto_error=False) +AUTH_FAILURES: dict[str, deque[float]] = {} + + +def _secure_mkdir(path: Path) -> None: + if path.is_symlink(): + raise RuntimeError(f"Refusing to use symlinked security-sensitive directory: {path}") + path.mkdir(parents=True, exist_ok=True) + flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(str(path), flags) + try: + os.fchmod(fd, 0o700) + finally: + os.close(fd) + except OSError: + # Permission hardening may be unavailable on some non-POSIX hosts. + pass + + +def _secure_write(path: Path, content: str) -> None: + _secure_mkdir(path.parent) + flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(str(path), flags, 0o600) + os.fchmod(fd, 0o600) + with os.fdopen(fd, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + + +def _scrub_text(value: str) -> str: + value = re.sub(r"([A-Za-z][A-Za-z0-9+.-]*://[^:/@\s]+:)[^@\s]+(@)", r"\1***\2", value) + return _QUERY_SECRET_RE.sub(r"\1***", value) + + +def _same_origin(request: Request) -> None: + if request.headers.get("x-teledrive-csrf") != "1": + raise HTTPException(403, "Missing CSRF protection header") + origin = request.headers.get("origin") + if not origin: + return + host = request.headers.get("host", "").lower() + try: + origin_host = urlsplit(origin).netloc.lower() + except Exception: + raise HTTPException(403, "Invalid Origin header") + if not host or origin_host != host: + raise HTTPException(403, "Cross-origin state change rejected") + + +def require_auth(request: Request, credentials: HTTPBasicCredentials | None = Depends(security)) -> None: + if not WEB_PASSWORD: + return + + client = request.client.host if request.client else "unknown" + now = time.monotonic() + failures = AUTH_FAILURES.setdefault(client, deque()) + while failures and now - failures[0] > AUTH_FAILURE_WINDOW_SECONDS: + failures.popleft() + if len(failures) >= AUTH_FAILURE_LIMIT: + raise HTTPException(status_code=429, detail="Too many authentication failures") + + valid = credentials is not None and ( + hmac.compare_digest(credentials.username, WEB_USER) + and hmac.compare_digest(credentials.password, WEB_PASSWORD) + ) + if not valid: + failures.append(now) + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + headers={"WWW-Authenticate": 'Basic realm="TeleDrive"'}, + ) + + AUTH_FAILURES.pop(client, None) + + +def default_config() -> dict[str, Any]: + return { + "web": { + "user": "admin", + "password": "", + "allowed_hosts": "localhost,127.0.0.1,[::1]", + "allow_insecure": False, + "bind_address": "0.0.0.0", # nosec B104 + "port": 8080, + "s3_port": 9000, + "rest_max_upload_bytes": 2147483648, + "auth_failure_limit": 5, + "auth_failure_window_seconds": 300, + "ssl_certfile": "", + "ssl_keyfile": "", + }, + "storage": { + "channel": "", + "strategy": "round_robin", + "scratch_dir": "/data/tmp", + "apis": [], + }, + "jobs": { + "source_root": "/uploads", + "auto_run": True, + }, + "upload": { + "source_dir": "/uploads", + "allowed_extensions": [], + "max_file_size_mb": 0, + "sleep_min_seconds": 60, + "sleep_max_seconds": 90, + "max_files_per_run": 12, + "max_files_per_day": 100, + "retry_attempts": 5, + "backoff_base_seconds": 10, + "floodwait_buffer_seconds": 5, + "caption_template": "{name}", + "send_mode": "document", + }, + "state": {"database_url": "sqlite:////data/state.db"}, + "logging": {"log_path": "/data/uploader.log", "level": "INFO"}, + "app": {"auto_run": False, "run_interval_minutes": 60}, + } + + +def ensure_config() -> None: + _secure_mkdir(CONFIG_PATH.parent) + if CONFIG_PATH.is_symlink(): + raise RuntimeError("Refusing to use a symlinked TeleDrive config file") + _secure_mkdir(AUTH_DIR) + _secure_mkdir(Path("/data/sessions")) + _secure_mkdir(Path("/data/tmp")) + Path("/uploads").mkdir(parents=True, exist_ok=True) + if not CONFIG_PATH.exists(): + _secure_write(CONFIG_PATH, yaml.safe_dump(default_config(), sort_keys=False)) + else: + flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) + try: + fd = os.open(str(CONFIG_PATH), flags) + try: + try: + os.fchmod(fd, 0o600) + except PermissionError: + # A mounted config may be owned by another uid; accept it + # only when it is already private. + if os.fstat(fd).st_mode & 0o077: + raise + finally: + os.close(fd) + except OSError as exc: + raise RuntimeError("Unable to safely open TeleDrive config file") from exc + + +def read_raw_config() -> dict[str, Any]: + ensure_config() + return yaml.safe_load(CONFIG_PATH.read_text(encoding="utf-8")) or {} + + +def _redacted_config(raw: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(raw) + tg = result.get("telegram", {}) + if tg.get("api_hash"): + tg["api_hash"] = REDACTED + for account in tg.get("accounts", []) or []: + if isinstance(account, dict) and account.get("api_hash"): + account["api_hash"] = REDACTED + state_cfg = result.get("state", {}) + db_url = state_cfg.get("database_url") or state_cfg.get("db_path") + if db_url: + state_cfg["database_url"] = redact_database_url(str(db_url)) + state_cfg.pop("db_path", None) + return result + + +def _restore_secrets(candidate: dict[str, Any], current: dict[str, Any]) -> dict[str, Any]: + result = copy.deepcopy(candidate) + new_tg = result.setdefault("telegram", {}) + old_tg = current.get("telegram", {}) + if new_tg.get("api_hash") == REDACTED: + new_tg["api_hash"] = old_tg.get("api_hash", "") + + old_accounts = { + str(a.get("name")): a + for a in (old_tg.get("accounts") or []) + if isinstance(a, dict) and a.get("name") + } + for account in new_tg.get("accounts", []) or []: + if not isinstance(account, dict): + continue + if account.get("api_hash") == REDACTED: + account["api_hash"] = old_accounts.get(str(account.get("name")), {}).get("api_hash", "") + + new_state = result.setdefault("state", {}) + old_state = current.get("state", {}) + old_url = old_state.get("database_url") or old_state.get("db_path") + submitted = new_state.get("database_url") + if old_url and submitted == redact_database_url(str(old_url)): + new_state["database_url"] = old_url + return result + + +def effective_config(): + ensure_config() + return uploader.load_config(str(CONFIG_PATH)) + + +def validate_and_write_config(raw: dict[str, Any]) -> None: + current = read_raw_config() + merged = _restore_secrets(raw, current) + tmp = CONFIG_PATH.with_suffix(CONFIG_PATH.suffix + ".tmp") + _secure_write(tmp, yaml.safe_dump(merged, sort_keys=False)) + try: + cfg = uploader.load_config(str(tmp)) + uploader.validate_config(cfg) + conn = uploader.connect_db(cfg.db_path) + conn.close() + except ValueError: + tmp.unlink(missing_ok=True) + raise + except Exception as exc: + tmp.unlink(missing_ok=True) + raise RuntimeError("Database connection/initialization failed") from exc + os.replace(tmp, CONFIG_PATH) + _reload_service_settings() + + +def db_connect(): + cfg = effective_config() + uploader.validate_storage_paths(cfg) + return uploader.connect_db(cfg.db_path) + + +async def upload_job_scheduler_loop() -> None: + """Opt-in durable job worker; SQL leases prevent duplicate claims. + + Disabled unless TELEDRIVE_JOB_AUTO_RUN=1. Every cycle gets its own + connection; failures retain job metadata for explicit operator retry. + """ + while True: + conn = None + try: + if not JOB_AUTO_RUN: + await asyncio.sleep(10) + continue + root = _storage_job_root() + conn = db_connect() + queue = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + scratch = Path(STORAGE_SCRATCH_DIR) + _secure_mkdir(scratch) + result = await queue.run_next( + storage_transport(conn), allowed_root=root, + scratch_directory=scratch, chunk_size=RECOMMENDED_CHUNK_SIZE, + ) + await asyncio.sleep(2 if result else 10) + except asyncio.CancelledError: + raise + except Exception as exc: + # Do not log source paths, credentials or Telegram tokens. + LOG_BUFFER.append( + "[web] durable upload job scheduler: " + type(exc).__name__ + ) + await asyncio.sleep(20) + finally: + if conn is not None: + conn.close() + + +async def lifecycle_scheduler_loop() -> None: + """Apply bounded SQL-only lifecycle rules hourly, in this one process.""" + while True: + connection = None + try: + from teledrive.s3.lifecycle import BucketLifecycleStore + connection = db_connect() + cat = BucketCatalog(ManifestStore(connection)) + BucketLifecycleStore(cat).apply_due(now=int(time.time()), limit=100) + except asyncio.CancelledError: + raise + except Exception as exc: + LOG_BUFFER.append("[web] lifecycle scheduler: " + type(exc).__name__) + finally: + if connection is not None: + connection.close() + await asyncio.sleep(3600) + + +@asynccontextmanager +async def lifespan(_: FastAPI): + global upload_job_scheduler_task, lifecycle_scheduler_task + os.umask(0o077) + ensure_config() + _reload_service_settings() + transport_mode() # Enforce Telethon USER mode at startup; Bot mode no longer exists. + if not WEB_PASSWORD and not ALLOW_INSECURE: + raise RuntimeError( + "config.yaml web.password is required (or web.allow_insecure for " + "explicitly trusted local deployments only)." + ) + if WEB_PASSWORD and len(WEB_PASSWORD) < 16: + raise RuntimeError("config.yaml web.password must be at least 16 characters") + if DB_EXTRA_PACKAGES: + install_db_extra_packages(DB_EXTRA_PACKAGES, "/data/site-packages") + # Permit a new unconfigured installation to serve health/configuration; + # writes fail closed until a USER session connects. + try: + await mtproto_storage_runtime.start() + except Exception as exc: + LOG_BUFFER.append(_scrub_text(f"[web] user session warning: {exc}")) + try: + conn = db_connect() + conn.close() + except Exception as exc: + LOG_BUFFER.append(_scrub_text(f"[web] startup database warning: {exc}")) + upload_job_scheduler_task = asyncio.create_task(upload_job_scheduler_loop()) + lifecycle_scheduler_task = asyncio.create_task(lifecycle_scheduler_loop()) + yield + if upload_job_scheduler_task: + upload_job_scheduler_task.cancel() + await asyncio.gather(upload_job_scheduler_task, return_exceptions=True) + if lifecycle_scheduler_task: + lifecycle_scheduler_task.cancel() + await asyncio.gather(lifecycle_scheduler_task, return_exceptions=True) + await mtproto_storage_runtime.stop() + + +app = FastAPI( + title="TeleDrive API", + version="2.1.0", + description=( + "Authenticated management API for Telegram upload accounts, queue/history, " + "application configuration, execution and Telegram login challenges. " + "Use HTTP Basic credentials and set X-TeleDrive-CSRF: 1 for state-changing requests. " + "Protect access with HTTPS or a trusted VPN." + ), + openapi_tags=[ + {"name": "System", "description": "Service state and health"}, + {"name": "Configuration", "description": "Settings, Telegram accounts, and database URL"}, + {"name": "Uploads", "description": "Queue/history and uploader lifecycle"}, + {"name": "Authentication", "description": "Telegram code and two-factor password handoff"}, + {"name": "API keys", "description": "Root-admin API key creation, rotation and revocation"}, + {"name": "Object catalog", "description": "Bearer-scoped bucket and object metadata"}, + ], + lifespan=lifespan, + docs_url=None, + redoc_url=None, + openapi_url=None, +) +@app.middleware("http") +async def _trusted_host_middleware(request: Request, call_next): + # ALLOWED_HOSTS reloads from config.yaml at runtime; resolve it per + # request instead of capturing the import-time default. + host = (request.headers.get("host") or "").split(":")[0] + if host and host not in ALLOWED_HOSTS: + raise HTTPException(400, "Invalid host header") + return await call_next(request) + + +@app.middleware("http") +async def security_middleware(request: Request, call_next): + binary_upload = (request.method == "PUT" and + re.fullmatch(r"/api/v1/buckets/[A-Za-z0-9.-]+/content", request.url.path)) + if request.method in {"POST", "PUT", "PATCH"} and not binary_upload: + # Size-bound browser JSON requests. Cache on the ORIGINAL request: + # Starlette's call_next() replays _CachedRequest's cached body. + # Constructing a new Request here silently discards POST/PUT payloads. + length = request.headers.get("content-length") + if length: + try: + if int(length) > MAX_REQUEST_BYTES: + return JSONResponse(status_code=413, content={"detail": "Request body too large"}) + except ValueError: + return JSONResponse(status_code=400, content={"detail": "Invalid Content-Length"}) + body = bytearray() + async for chunk in request.stream(): + body.extend(chunk) + if len(body) > MAX_REQUEST_BYTES: + return JSONResponse(status_code=413, content={"detail": "Request body too large"}) + request._body = bytes(body) + + response = await call_next(request) + response.headers["X-Content-Type-Options"] = "nosniff" + response.headers["X-Frame-Options"] = "DENY" + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Permissions-Policy"] = "camera=(), microphone=(), geolocation=()" + if request.url.path in {"/docs", "/redoc"}: + # FastAPI's official interactive documentation assets are served from the CDN. + response.headers["Content-Security-Policy"] = ( + "default-src 'self'; base-uri 'none'; frame-ancestors 'none'; " + "object-src 'none'; connect-src 'self'; img-src 'self' data: https://fastapi.tiangolo.com; " + "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " + "script-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net;" + ) + else: + response.headers["Content-Security-Policy"] = ( + "default-src 'self'; base-uri 'none'; frame-ancestors 'none'; " + "form-action 'self'; object-src 'none'; connect-src 'self'; img-src 'self' data:; " + "style-src 'self' 'unsafe-inline'; script-src 'self'" + ) + if request.url.path in {"/", "/docs", "/redoc", "/openapi.json"} or request.url.path.startswith("/api/"): + response.headers["Cache-Control"] = "no-store" + return response + + +class StartRequest(BaseModel): + mode: Literal["run-once", "scan-only", "connect"] = Field( + default="run-once", description="Run upload batch, scan only, or connect accounts." + ) + + +class TelegramAuthRequest(BaseModel): + account: str = Field(min_length=1, max_length=64, examples=["primary"]) + kind: Literal["code", "password"] + value: str = Field(min_length=1, max_length=1024, description="Telegram code or 2FA password; never logged.") + + +class OKResponse(BaseModel): + ok: bool = True + + +class KeyCreateRequest(BaseModel): + name: str = Field(min_length=1, max_length=120) + scopes: list[str] = Field(min_length=1, max_length=20) + expires_at: int | None = Field(default=None, description="Optional Unix expiry time") + + +class S3KeyCreateRequest(BaseModel): + name: str = Field(min_length=1, max_length=120) + permissions: list[str] = Field(default_factory=lambda: list(DEFAULT_PERMISSIONS)) + expires_at: int | None = None + + +class S3CompletionRecoveryRequest(BaseModel): + execute: bool = False + min_age_seconds: int = Field(default=3600, ge=60, le=30 * 86400) + + +class PendingUploadRecoveryRequest(BaseModel): + execute: bool = False + min_age_seconds: int = Field(default=3600, ge=60, le=30 * 86400) + + +class StorageUploadJobRequest(BaseModel): + bucket: str = Field(min_length=3, max_length=63) + key: str = Field(min_length=1, max_length=1024) + source_path: str = Field(min_length=1, max_length=4096) + + +class LegacyFolderImportRequest(BaseModel): + bucket: str = Field(min_length=3, max_length=63) + prefix: str = Field(default="", max_length=512) + max_files: int = Field(default=100, ge=1, le=1000) + execute: bool = False + + +class VersionRestoreRequest(BaseModel): + bucket: str = Field(min_length=3, max_length=63) + key: str = Field(min_length=1, max_length=1024) + expected_generation: int | None = Field(default=None, ge=0) + + +class VersionIndexRequest(BaseModel): + execute: bool = False + limit: int = Field(default=100, ge=1, le=1000) + + +class BucketCreateRequest(BaseModel): + name: str = Field(min_length=3, max_length=63) + +def storage_transport(conn): + """Build the live USER-MTProto transport for REST/S3/local-job payload I/O. + + A preauthorized Telethon user session must be connected. There is no Bot + API transport; the unified runtime is Telethon USER MTProto only. + """ + try: + transport_mode() + mtproto_storage_runtime.require() + return UnifiedUserTransport( + mtproto_storage_runtime, UserSendJournal(conn), + ) + except RuntimeError: + raise HTTPException(503, "Authorized Telethon USER session is required") from None + + +def _byte_range(value: str | None, size: int) -> tuple[int, int, bool]: + """HTTP Range with inclusive input -> half-open output; 416 on invalid input.""" + if value is None: + return 0, size, False + match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip()) + if match is None or not (match.group(1) or match.group(2)): + raise HTTPException(416, "Unsupported byte range", headers={"Content-Range": f"bytes */{size}"}) + first, last = match.groups() + if size == 0: + raise HTTPException(416, "Empty object has no satisfiable range", + headers={"Content-Range": "bytes */0"}) + if not first: + count = int(last) + if count <= 0: + raise HTTPException(416, "Invalid suffix range", headers={"Content-Range": f"bytes */{size}"}) + start, end = max(0, size - count), size + else: + start = int(first) + end = min(size, int(last) + 1) if last else size + if start >= size or end <= start: + raise HTTPException(416, "Unsatisfiable range", headers={"Content-Range": f"bytes */{size}"}) + return start, end, True + + + +def require_api_scope(required: str): + """HTTP bearer authorization, independent of the browser Basic/CSRF flow.""" + def check(credentials: HTTPAuthorizationCredentials | None = Depends(bearer_auth)) -> None: + if credentials is None or credentials.scheme.lower() != "bearer": + raise HTTPException(401, "API Bearer token required", + headers={"WWW-Authenticate": "Bearer"}) + conn = db_connect() + try: + authorized = ApiKeyStore(conn).authorize(credentials.credentials, required) + finally: + conn.close() + if not authorized: + raise HTTPException(403, "API token is invalid, expired or lacks permission") + return check + + + + +app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") + + +@app.get("/openapi.json", tags=["System"], include_in_schema=False) +def openapi_schema(_: None = Depends(require_auth)): + return JSONResponse(app.openapi()) + + +@app.get("/docs", tags=["System"], include_in_schema=False) +def swagger_docs(_: None = Depends(require_auth)): + return get_swagger_ui_html(openapi_url="/openapi.json", title="TeleDrive API · Swagger UI") + + +@app.get("/redoc", tags=["System"], include_in_schema=False) +def redoc_docs(_: None = Depends(require_auth)): + return get_redoc_html(openapi_url="/openapi.json", title="TeleDrive API · ReDoc") + + +@app.get("/", include_in_schema=False) +def index(_: None = Depends(require_auth)): + return FileResponse(STATIC_DIR / "index.html") + + +@app.get("/api/config", tags=["Configuration"], summary="Read configuration (secrets redacted)") +def get_config(_: None = Depends(require_auth)): + return _redacted_config(read_raw_config()) + + +@app.put("/api/config", tags=["Configuration"], response_model=OKResponse, summary="Validate and save configuration") +def put_config(request: Request, payload: dict[str, Any] = Body(...), _: None = Depends(require_auth)): + _same_origin(request) + try: + validate_and_write_config(payload) + except ValueError as exc: + raise HTTPException(400, _scrub_text(str(exc))) from exc + except Exception as exc: + raise HTTPException(400, "Configuration/database validation failed") from exc + return {"ok": True} + + +@app.get("/api/v1/admin/keys", tags=["API keys"], summary="List API key metadata; never returns secrets") +def list_api_keys(_: None = Depends(require_auth)): + conn = db_connect() + try: + return {"keys": ApiKeyStore(conn).list()} + finally: + conn.close() + + +@app.post("/api/v1/admin/keys", tags=["API keys"], status_code=201, summary="Issue a scoped API key (secret returned once)") +def issue_api_key(request: Request, payload: KeyCreateRequest, _: None = Depends(require_auth)): + _same_origin(request) + conn = db_connect() + try: + try: + key = ApiKeyStore(conn).create(payload.name, payload.scopes, expires_at=payload.expires_at) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return {"id": key.id, "token": key.value, "prefix": key.prefix, "scopes": key.scopes} + finally: + conn.close() + + +@app.delete("/api/v1/admin/keys/{identifier}", tags=["API keys"], summary="Revoke an API key immediately") +def revoke_api_key(identifier: str, request: Request, _: None = Depends(require_auth)): + _same_origin(request) + conn = db_connect() + try: + if not ApiKeyStore(conn).revoke(identifier): + raise HTTPException(404, "API key not found or already revoked") + return {"ok": True} + finally: + conn.close() + + +@app.post("/api/v1/admin/keys/{identifier}/rotate", tags=["API keys"], summary="Rotate an API key; secret returned once") +def rotate_api_key(identifier: str, request: Request, _: None = Depends(require_auth)): + _same_origin(request) + conn = db_connect() + try: + try: + key = ApiKeyStore(conn).rotate(identifier) + except FileNotFoundError as exc: + raise HTTPException(404, str(exc)) from exc + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"id": key.id, "token": key.value, "prefix": key.prefix, "scopes": key.scopes} + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/gc-candidates", tags=["Storage recovery"], + summary="Retention-aware dry-run FAILED journal planning, never deletion") +def list_storage_gc_candidates( + min_age_seconds: int = Query(default=7*86400, ge=86400, le=365*86400), + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + try: + candidates = StorageGCInventory(catalog.manifests).preview_failed_retention( + now=int(time.time()), min_age_seconds=min_age_seconds, limit=limit + ) + except RuntimeError as exc: + raise HTTPException(409, "Reference graph is inconsistent; GC blocked") from exc + return { + "candidates": candidates, + "dry_run": True, "physical_deletion_enabled": False, + "remote_history_scan_required": True, + } + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/gc-audit", tags=["API keys"], + summary="Audit Telegram object references before any physical cleanup") +def get_storage_gc_audit( + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + """Root-admin-only, non-destructive inventory; never authorizes remote deletion.""" + from dataclasses import asdict + + conn = db_connect() + try: + # Also ensure the bucket/head metadata tables exist on a new DB. + catalog = BucketCatalog(ManifestStore(conn)) + try: + inventory = StorageGCInventory(catalog.manifests).inspect() + except RuntimeError as exc: + raise HTTPException(409, "Reference graph is inconsistent; GC blocked") from exc + return { + "total_manifests": inventory.total_manifests, + "total_telegram_messages": inventory.total_telegram_messages, + "unjournaled_mtproto_receipts": inventory.unjournaled_mtproto_receipts, + "unresolved_mtproto_intents": inventory.unresolved_mtproto_intents, + "active_upload_jobs": inventory.active_upload_jobs, + "live_object_heads": inventory.live_object_heads, + "copy_aliases": inventory.copy_aliases, + "multipart_parts": inventory.multipart_parts, + "eligible_for_deletion": 0, + "dry_run": True, + "physical_deletion_enabled": False, + "protected_chunks": [ + asdict(row) for row in inventory.protected_chunks[:limit] + ], + "truncated": len(inventory.protected_chunks) > limit, + } + finally: + conn.close() + + +def _storage_job_root() -> Path: + """Durable jobs may read only an operator-configured local source root.""" + raw = JOB_SOURCE_ROOT.strip() + if not raw: + raise HTTPException(503, "Configure TELEDRIVE_JOB_SOURCE_ROOT for local upload jobs") + try: + root = Path(raw).resolve(strict=True) + except OSError as exc: + raise HTTPException(503, "Upload source root does not exist") from exc + if not root.is_dir(): + raise HTTPException(503, "Upload source root must be a directory") + return root + + +@app.post("/api/v1/admin/storage/versions/index-current", + tags=["Storage versions"], + summary="Dry-run or explicitly index only verified historical HEAD references") +def index_legacy_current_versions( + request: Request, payload: VersionIndexRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + try: + registered = catalog.index_legacy_current_heads( + dry_run=not payload.execute, limit=payload.limit, + ) + except RuntimeError as exc: + raise HTTPException(409, str(exc)) from exc + return {"ids": registered, "count": len(registered), + "dry_run": not payload.execute, + "indexed": bool(payload.execute and registered)} + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/versions", tags=["Storage versions"], + summary="Admin listing of published historical versions for restore") +def admin_list_published_versions( + bucket: str = Query(..., min_length=3, max_length=63), + key: str = Query(..., min_length=1, max_length=1024), + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + if bucket not in catalog.list_buckets(): + raise HTTPException(404, "Bucket not found") + try: + current = catalog.get(bucket, key) + generation = current.generation + except FileNotFoundError: + generation = 0 + return {"bucket": bucket, "key": key, "generation": generation, + "versions": catalog.list_versions(bucket, key, limit=limit)} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/versions/{version_id}/restore", + tags=["Storage versions"], + summary="Restore a previously published immutable version without Telegram uploads") +def restore_published_version( + version_id: str, request: Request, payload: VersionRestoreRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + try: + head = catalog.get(payload.bucket, payload.key) + generation, current_id = head.generation, head.item.id + except FileNotFoundError: + generation, current_id = 0, None + if payload.expected_generation is not None and ( + payload.expected_generation != generation + ): + raise HTTPException(409, "Generation has changed since restore preview") + try: + restored = copy_object( + catalog, payload.bucket, payload.key, + payload.bucket, payload.key, + source_version_id=version_id, + expected_destination_generation=generation, + expected_destination_object_id=current_id, + ) + except FileNotFoundError as exc: + raise HTTPException(404, "Published version does not exist") from exc + except (CopyConflict, ValueError) as exc: + raise HTTPException(409, str(exc)) from exc + return { + "id": restored.object_id, "source_version_id": version_id, + "etag": restored.etag, "size": restored.size, + "generation": catalog.get(payload.bucket, payload.key).generation, + } + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/health", tags=["Storage jobs"], + summary="Read storage jobs, stale leases and protected-media metrics") +def storage_operations_health(_: None = Depends(require_auth)): + from sqlalchemy import func, select + + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + queue = StorageUploadJobs(catalog) + counts = conn.connection.execute( + select(storage_job_rows.c.state, func.count()) + .group_by(storage_job_rows.c.state) + ).all() + now = int(time.time()) + pending = catalog.manifests.stale_pending(now=now, limit=1000) + completing = MultipartStore(catalog).stale_completions(now=now, limit=1000) + try: + gc = StorageGCInventory(catalog.manifests).inspect() + except RuntimeError as exc: + raise HTTPException(409, "Reference graph is inconsistent; GC blocked") from exc + states = {state: int(count) for state, count in counts} + alerts = [] + if gc.unresolved_mtproto_intents: + alerts.append({ + "code": "unresolved_mtproto_sends", "severity": "critical", + "count": gc.unresolved_mtproto_intents, + "message": "MTProto sends have unknown acceptance; do not retry until owner-verified reconciliation", + }) + if gc.unjournaled_mtproto_receipts: + alerts.append({ + "code": "unjournaled_mtproto_receipts", "severity": "warning", + "count": gc.unjournaled_mtproto_receipts, + "message": "MTProto send receipts not yet linked to a chunk journal; review persistent samples", + }) + if states.get("failed", 0): + alerts.append({ + "code": "failed_upload_jobs", "severity": "warning", + "count": states["failed"], + "message": "Failed upload jobs require explicit operator retry or cancellation", + }) + if pending: + alerts.append({ + "code": "stale_pending_uploads", "severity": "warning", + "count": len(pending), + "message": "Expired PENDING writers need dry-run recovery review", + }) + if completing: + alerts.append({ + "code": "stale_multipart_completions", "severity": "warning", + "count": len(completing), + "message": "Stale multipart completion leases need dry-run review", + }) + truncated = len(pending) >= 1000 or len(completing) >= 1000 + if truncated: + alerts.append({ + "code": "sample_limit_reached", "severity": "critical", + "count": None, + "message": "Recovery sample limit reached; inventory may be incomplete", + }) + return { + "job_states": states, + "alerts": alerts, + "unresolved_alert_count": len(alerts), + "stale_pending_sample_count": len(pending), + "stale_multipart_sample_count": len(completing), + "unjournaled_mtproto_receipt_count": gc.unjournaled_mtproto_receipts, + "unresolved_mtproto_send_count": gc.unresolved_mtproto_intents, + "mtproto_explicit_flood_total": MtprotoRateGuard( + conn + ).total_explicit_flood_observations(), + "samples_truncated": truncated, + "telegram_messages_retained": gc.total_telegram_messages, + "live_object_heads": gc.live_object_heads, + "copy_aliases": gc.copy_aliases, + "physical_deletion_enabled": False, + } + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/mtproto-unresolved", tags=["Storage jobs"], + summary="Inspect unresolved USER-MTProto send intents without modifying Telegram") +def list_unresolved_mtproto_sends( + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + # Operator-only diagnostic: return immutable source identity for + # independent channel proof. Never accept unverified receipt IDs or + # perform a retry/deletion as a side effect of inspecting this list. + from sqlalchemy import func, select + from teledrive.storage.user_mode import mtproto_send_intents + + conn = db_connect() + try: + rows = conn.connection.execute( + select( + mtproto_send_intents.c.account, mtproto_send_intents.c.chat_id, + mtproto_send_intents.c.object_id, mtproto_send_intents.c.ordinal, + mtproto_send_intents.c.sha256, mtproto_send_intents.c.size, + ).where(mtproto_send_intents.c.message_id.is_(None)) + .order_by(mtproto_send_intents.c.object_id, + mtproto_send_intents.c.ordinal) + .limit(limit) + ).all() + total = conn.connection.execute( + select(func.count()).select_from(mtproto_send_intents).where( + mtproto_send_intents.c.message_id.is_(None) + ) + ).scalar_one() + return { + "total": int(total), + "intents": [ + { + "account": account, "chat_id": int(chat_id), + "object_id": object_id, "ordinal": int(ordinal), + "sha256": sha256, "size": int(size), + } + for account, chat_id, object_id, ordinal, sha256, size in rows + ], + "message": ( + "Possible remotely accepted USER-MTProto sends. Independently " + "verify the original channel message ID and full Telegram " + "bytes before recording any recovery receipt. No automatic retry." + ), + } + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/jobs", tags=["Storage jobs"], + summary="Inspect durable local-source upload jobs (no worker tokens)") +def list_storage_upload_jobs( + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + conn = db_connect() + try: + return {"jobs": StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + .list_jobs(limit=limit)} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/jobs", tags=["Storage jobs"], + status_code=201, summary="Queue an allowed-root file for Telegram upload") +def create_storage_upload_job( + request: Request, payload: StorageUploadJobRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + root = _storage_job_root() + conn = db_connect() + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + try: + job_id = manager.enqueue( + payload.bucket, payload.key, Path(payload.source_path), + allowed_root=root, + ) + except FileNotFoundError as exc: + raise HTTPException(404, str(exc)) from exc + except (UploadJobError, ValueError, OSError) as exc: + raise HTTPException(422, str(exc)) from exc + return {"id": job_id, "state": "queued"} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/jobs/import-legacy", + tags=["Storage jobs"], summary="Preview or queue legacy folder files without uploading") +def import_legacy_folder_jobs( + request: Request, payload: LegacyFolderImportRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + from teledrive.storage.legacy_bridge import enqueue_legacy_folder + approved_root = _storage_job_root() + cfg = effective_config() + try: + source_root = Path(cfg.source_dir).resolve(strict=True) + except OSError as exc: + raise HTTPException(503, "Legacy source folder unavailable") from exc + if not source_root.is_relative_to(approved_root): + raise HTTPException(403, "Legacy source is outside the configured job root") + conn = db_connect() + try: + now = int(time.time()) + holder = "legacy-bridge-" + os.urandom(16).hex() + if not conn.acquire_lease( + uploader.LEASE_NAME, holder, now, uploader.LEASE_TTL_SECONDS + ): + raise HTTPException(409, "Legacy uploader currently holds a lease") + try: + last_renew = time.monotonic() + + def heartbeat(): + nonlocal last_renew + if time.monotonic() - last_renew < 10: + return + if not conn.renew_lease( + uploader.LEASE_NAME, holder, + int(time.time()), uploader.LEASE_TTL_SECONDS + ): + raise RuntimeError("Legacy uploader lease lost during import") + last_renew = time.monotonic() + + manager = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + try: + report = enqueue_legacy_folder( + cfg, manager, bucket=payload.bucket, prefix=payload.prefix, + max_files=payload.max_files, + dry_run=not payload.execute, lease_heartbeat=heartbeat, + ) + except FileNotFoundError as exc: + raise HTTPException(404, "Destination object bucket not found") from exc + except (UploadJobError, ValueError, OSError) as exc: + raise HTTPException(422, str(exc)) from exc + return { + "dry_run": not payload.execute, + "queued": report.queued if payload.execute else 0, + "would_queue": report.queued, + "already_queued": report.already_queued, + "excluded": report.excluded, + "job_ids": list(report.job_ids) if payload.execute else [], + "uploads_started": 0, + } + finally: + conn.release_lease(uploader.LEASE_NAME, holder) + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/jobs/run-next", tags=["Storage jobs"], + summary="Process one queued upload with SQL lease, checksums and CAS") +async def run_next_storage_upload_job( + request: Request, _: None = Depends(require_auth), +): + _same_origin(request) + root = _storage_job_root() + conn = db_connect() + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + transport = storage_transport(conn) + scratch = Path(STORAGE_SCRATCH_DIR) + _secure_mkdir(scratch) + try: + result = await manager.run_next( + transport, allowed_root=root, scratch_directory=scratch, + chunk_size=RECOMMENDED_CHUNK_SIZE, + ) + except (UploadJobError, ValueError) as exc: + raise HTTPException(409, str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(503, "Telegram upload failed; job retained for retry") from exc + return {"job": result} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/jobs/{job_id}/retry", tags=["Storage jobs"], + summary="Explicitly retry a failed job; never remove Telegram messages") +def retry_storage_upload_job( + job_id: str, request: Request, _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + if not manager.retry(job_id): + raise HTTPException(409, "Job is not failed or does not exist") + return {"id": job_id, "state": "queued"} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/jobs/{job_id}/cancel", tags=["Storage jobs"], + summary="Cancel only inactive jobs; preserve all Telegram media") +def cancel_storage_upload_job( + job_id: str, request: Request, _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + manager = StorageUploadJobs(BucketCatalog(ManifestStore(conn))) + if not manager.cancel(job_id): + raise HTTPException(409, "Job is active, completed, cancelled or does not exist") + return {"id": job_id, "state": "cancelled"} + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/pending-stale", tags=["Storage recovery"], + summary="Preview aged PENDING upload leases; no Telegram deletion") +def list_stale_pending_objects( + min_age_seconds: int = Query(default=3600, ge=60, le=30 * 86400), + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + conn = db_connect() + try: + items = ManifestStore(conn).stale_pending( + now=int(time.time()), min_age_seconds=min_age_seconds, limit=limit + ) + return {"uploads": items, "dry_run": True, "physical_deletion_enabled": False} + finally: + conn.close() + + +@app.post("/api/v1/admin/storage/pending-stale/{object_id}/abandon", + tags=["Storage recovery"], + summary="Explicitly fence an aged PENDING writer and retain Telegram journal") +def abandon_stale_pending_object( + object_id: str, request: Request, + payload: PendingUploadRecoveryRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + manifests = ManifestStore(conn) + now = int(time.time()) + # A preview never mutates the writer's lease, even after a restart. + if not payload.execute: + matches = manifests.stale_pending( + now=now, min_age_seconds=payload.min_age_seconds, limit=1000 + ) + return {"object_id": object_id, + "eligible": any(item["id"] == object_id for item in matches), + "abandoned": False, "dry_run": True} + # Only this object can be transitioned. The SQL conditional update + # ensures a heartbeat, completed object or another admin wins safely. + affected = manifests.abandon_stale_pending( + now=now, min_age_seconds=payload.min_age_seconds, + limit=1, object_id=object_id, + ) + return {"object_id": object_id, "eligible": bool(affected), + "abandoned": bool(affected), "dry_run": False} + finally: + conn.close() + + +@app.get("/api/v1/admin/storage/channel-history", tags=["Storage history"], + summary="Bounded read-only user-MTProto channel history (unverified candidates)") +async def inspect_user_channel_history( + limit: int = Query(default=100, ge=1, le=1000), + before_message_id: int | None = Query(default=None, ge=1), + include_external_markers: bool = Query(default=False), + _: None = Depends(require_auth), +): + """Never download media, change catalog mappings or delete channel posts. + + Other senders' markers are NOT trusted ownership/manifest evidence. + The endpoint deliberately only runs with the already-connected USER + transport; only an authorized user session can enumerate channel history. + """ + try: + runtime = mtproto_storage_runtime + runtime.require() + target = runtime.transport.accounts[runtime.account][1] + return await inventory_channel( + runtime.client, target, account=runtime.account, + scan_limit=limit, before_message_id=before_message_id, + include_external_markers=include_external_markers, + ) + except (ValueError, RuntimeError, OSError): + raise HTTPException(503, "Authorized channel history inspection unavailable") from None + + +@app.get("/api/v1/admin/s3-multipart/stale", tags=["API keys"], + summary="Preview expired multipart completion leases (never deletes chunks)") +def list_stale_s3_completions( + min_age_seconds: int = Query(default=3600, ge=60, le=30 * 86400), + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_auth), +): + conn = db_connect() + try: + mpu = MultipartStore(BucketCatalog(ManifestStore(conn))) + return {"uploads": mpu.stale_completions( + min_age_seconds=min_age_seconds, limit=limit + )} + finally: + conn.close() + + +@app.post("/api/v1/admin/s3-multipart/{upload_id}/recover", tags=["API keys"], + summary="Dry-run or explicitly recover an expired multipart completion lease") +def recover_stale_s3_completion( + request: Request, upload_id: str, payload: S3CompletionRecoveryRequest, + _: None = Depends(require_auth), +): + _same_origin(request) + conn = db_connect() + try: + mpu = MultipartStore(BucketCatalog(ManifestStore(conn))) + eligible = mpu.recover_stale_completion( + upload_id, dry_run=not payload.execute, + min_age_seconds=payload.min_age_seconds, + ) + return {"upload_id": upload_id, "eligible": eligible, + "recovered": bool(eligible and payload.execute), + "dry_run": not payload.execute} + finally: + conn.close() + + +@app.get("/api/v1/admin/s3-credentials", tags=["API keys"], + summary="List S3 access key metadata (secrets never returned)") +def list_s3_credentials(_: None = Depends(require_auth)): + conn = db_connect() + try: + try: + return {"credentials": S3CredentialStore(conn, load_master_key()).list()} + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + finally: + conn.close() + + +@app.post("/api/v1/admin/s3-credentials", tags=["API keys"], status_code=201, + summary="Issue encrypted S3 access credentials (secret displayed once)") +def issue_s3_credentials(request: Request, payload: S3KeyCreateRequest, + _: None = Depends(require_auth)): + _same_origin(request) + conn = db_connect() + try: + try: + key = S3CredentialStore(conn, load_master_key()).create( + payload.name, payload.permissions, expires_at=payload.expires_at + ) + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return {"id": key.id, "access_key": key.access_key, + "secret_key": key.secret_key, "permissions": key.permissions} + finally: + conn.close() + + +@app.delete("/api/v1/admin/s3-credentials/{identifier}", tags=["API keys"], + summary="Revoke an S3 access key immediately") +def revoke_s3_credentials(request: Request, identifier: str, + _: None = Depends(require_auth)): + _same_origin(request) + conn = db_connect() + try: + try: + changed = S3CredentialStore(conn, load_master_key()).revoke(identifier) + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + if not changed: + raise HTTPException(404, "S3 key not found or already revoked") + return {"ok": True} + finally: + conn.close() + + +@app.get("/api/v1/buckets", tags=["Object catalog"], summary="List Telegram-backed storage buckets") +def list_storage_buckets(_: None = Depends(require_api_scope("buckets:read"))): + conn = db_connect() + try: + return {"buckets": BucketCatalog(ManifestStore(conn)).list_buckets()} + finally: + conn.close() + + +@app.post("/api/v1/buckets", tags=["Object catalog"], status_code=201, summary="Create a bucket") +def create_storage_bucket(payload: BucketCreateRequest, + _: None = Depends(require_api_scope("buckets:manage"))): + conn = db_connect() + try: + try: + BucketCatalog(ManifestStore(conn)).create_bucket(payload.name, int(time.time())) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"name": payload.name} + finally: + conn.close() + + +@app.delete("/api/v1/buckets/{bucket}", tags=["Object catalog"], summary="Remove a physically empty bucket") +def remove_storage_bucket(bucket: str, _: None = Depends(require_api_scope("buckets:manage"))): + conn = db_connect() + try: + try: + BucketCatalog(ManifestStore(conn)).delete_bucket(bucket) + except FileNotFoundError as exc: + raise HTTPException(404, str(exc)) from exc + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"ok": True} + finally: + conn.close() + + +@app.get("/api/v1/buckets/{bucket}/versions", tags=["Object catalog"], + summary="List proven published immutable versions of a key") +def list_published_object_versions( + bucket: str, key: str = Query(..., min_length=1, max_length=1024), + limit: int = Query(default=100, ge=1, le=1000), + _: None = Depends(require_api_scope("objects:read")), +): + conn = db_connect() + try: + catalog = BucketCatalog(ManifestStore(conn)) + if bucket not in catalog.list_buckets(): + raise HTTPException(404, "Bucket not found") + return {"bucket": bucket, "key": key, + "versions": catalog.list_versions(bucket, key, limit=limit)} + finally: + conn.close() + + +@app.get("/api/v1/buckets/{bucket}/objects", tags=["Object catalog"], summary="List current object metadata") +def list_current_objects(bucket: str, + prefix: str = Query("", max_length=1024), + after: str | None = Query(None, max_length=1024), + limit: int = Query(100, ge=1, le=1000), + _: None = Depends(require_api_scope("objects:read"))): + conn = db_connect() + try: + items = BucketCatalog(ManifestStore(conn)).list_objects(bucket, prefix=prefix, after=after, limit=limit) + return {"objects": [ + {"id": x.item.id, "key": x.item.key, "bucket": x.item.bucket, + "size": x.item.size, "sha256": x.item.sha256, + "created_at": x.item.created_at, "generation": x.generation} + for x in items + ]} + finally: + conn.close() + + +@app.delete("/api/v1/buckets/{bucket}/objects", tags=["Object catalog"], summary="Unlink a current object (Telegram data retained)") +def unlink_current_object(bucket: str, key: str = Query(..., min_length=1, max_length=1024), + generation: int | None = Query(None, ge=1), + _: None = Depends(require_api_scope("objects:delete"))): + conn = db_connect() + try: + try: + deleted = BucketCatalog(ManifestStore(conn)).remove_current( + bucket, key, expected_generation=generation + ) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"ok": True, "unlinked": deleted} + finally: + conn.close() + + +@app.put("/api/v1/buckets/{bucket}/content", tags=["Uploads"], + summary="Upload raw binary content to Telegram (synchronous commit)", status_code=201) +async def put_object_content( + bucket: str, request: Request, key: str = Query(..., min_length=1, max_length=1024), + generation: int | None = Query(None, ge=0), + _: None = Depends(require_api_scope("objects:write")), +): + if request.headers.get("content-type", "").split(";")[0].strip().lower() != "application/octet-stream": + raise HTTPException(415, "Content-Type must be application/octet-stream") + max_size = REST_MAX_UPLOAD_BYTES + if max_size < 1: + raise HTTPException(503, "REST uploads are disabled by policy") + stage_root = Path(STORAGE_SCRATCH_DIR) + _secure_mkdir(stage_root) + conn = db_connect() + staged = None + try: + manifests = ManifestStore(conn) + catalog = BucketCatalog(manifests) + if bucket not in catalog.list_buckets(): + raise HTTPException(404, "Bucket not found") + if generation is not None: + try: + current_generation = catalog.get(bucket, key).generation + except FileNotFoundError: + current_generation = 0 + if current_generation != generation: + # Reject stale conditional writes BEFORE staging/uploading any + # bytes to Telegram. Recheck atomically at final publication. + raise HTTPException(409, "Object generation precondition failed") + transport = storage_transport(conn) + # Verify Telegram access BEFORE consuming a potentially large body. + await transport.verify_target() + fd, pathname = tempfile.mkstemp(dir=stage_root, prefix="api-upload-") + staged = Path(pathname) + transferred = 0 + with os.fdopen(fd, "wb") as sink: + async for data in request.stream(): + transferred += len(data) + if transferred > max_size: + raise HTTPException(413, "Upload exceeds configured limit") + sink.write(data) + sink.flush() + os.fsync(sink.fileno()) + engine = TelegramObjectStorage(manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, + scratch_directory=stage_root) + result = await engine.put_file(bucket, key, staged) + try: + head = catalog.publish(result.id, now=int(time.time()), + expected_generation=generation) + except ValueError as exc: + raise HTTPException(409, str(exc)) from exc + return {"id": result.id, "bucket": bucket, "key": key, + "size": result.size, "sha256": result.sha256, + "generation": head.generation} + finally: + if staged is not None: + staged.unlink(missing_ok=True) + conn.close() + + +@app.head("/api/v1/buckets/{bucket}/content", tags=["Uploads"], + summary="Read object headers without downloading Telegram bytes") +def head_object_content( + bucket: str, key: str = Query(..., min_length=1, max_length=1024), + _: None = Depends(require_api_scope("objects:read")), +): + conn = db_connect() + try: + try: + item = BucketCatalog(ManifestStore(conn)).get(bucket, key).item + except FileNotFoundError as exc: + raise HTTPException(404, "Object not found") from exc + return Response(status_code=200, headers={ + "ETag": '"' + item.sha256 + '"', + "Content-Length": str(item.size), + "Accept-Ranges": "bytes", + "X-TeleDrive-Object-ID": item.id, + "Content-Type": "application/octet-stream", + }) + finally: + conn.close() + + +@app.get("/api/v1/buckets/{bucket}/content", tags=["Uploads"], + summary="Stream verified object bytes from Telegram; supports HTTP Range") +async def get_object_content( + bucket: str, request: Request, key: str = Query(..., min_length=1, max_length=1024), + _: None = Depends(require_api_scope("objects:read")), +): + conn = db_connect() + try: + manifests = ManifestStore(conn) + catalog = BucketCatalog(manifests) + try: + item = catalog.get(bucket, key).item + except FileNotFoundError as exc: + raise HTTPException(404, "Object not found") from exc + start, end, partial = _byte_range(request.headers.get("range"), item.size) + transport = storage_transport(conn) + root = Path(STORAGE_SCRATCH_DIR) + _secure_mkdir(root) + engine = TelegramObjectStorage(manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, + scratch_directory=root) + except BaseException: + conn.close() + raise + + async def stream(): + try: + async for piece in engine.iter_bytes(item.id, start=start, end=end): + yield piece + finally: + conn.close() + + headers = {"ETag": '"' + item.sha256 + '"', "Accept-Ranges": "bytes", + "Content-Length": str(end-start), "X-TeleDrive-Object-ID": item.id} + if partial: + headers["Content-Range"] = f"bytes {start}-{end-1}/{item.size}" + return StreamingResponse(stream(), status_code=206 if partial else 200, + media_type="application/octet-stream", headers=headers) + + +@app.get("/api/health", tags=["System"], response_model=OKResponse, summary="Liveness health check (unauthenticated)") +def health(): + return {"ok": True} diff --git a/web/s3.py b/web/s3.py new file mode 100644 index 00000000..6c9e5e7a --- /dev/null +++ b/web/s3.py @@ -0,0 +1,1982 @@ +"""Telegram-backed path-style S3 gateway (initial core subset). + +Run as a separate ASGI process at port 9000. Shares SQL metadata and Telegram +Bot API credentials with REST; never stores object bodies in another S3 service. +Unimplemented S3 operations explicitly return NotImplemented (501). +""" +from __future__ import annotations + +import base64 +import binascii +import hashlib +import hmac +import json +import os +import re +import tempfile +import time +import zlib +from pathlib import Path +from urllib.parse import unquote, parse_qsl, quote, urlencode, urlsplit, urlunsplit +from xml.etree import ElementTree as ET +from defusedxml import ElementTree as SafeET +from defusedxml.common import DefusedXmlException + +from fastapi import FastAPI, Request +from fastapi.responses import Response, StreamingResponse +from sqlalchemy import select + +from teledrive.s3.credentials import S3CredentialStore, load_master_key +from teledrive.s3.route_guard import validate_s3_query, S3QueryError +from teledrive.s3.bucket_listing import list_buckets_xml, BucketListingError +from teledrive.s3.checksums import crc32c, crc64nvme +from teledrive.s3.sigv4 import SigV4Error, verify_header_signature +from teledrive.s3.post_policy import parse_form, verify_post_policy, FORM_LIMIT +from teledrive.s3.multipart import MultipartStore, MultipartError, MAX_PARTS +from teledrive.s3.batch_delete import BatchDeleteError, MAX_XML_BYTES, parse_batch, verify_body +from teledrive.s3.conditions import http_last_modified, read_precondition +from teledrive.s3.write_conditions import check_write_precondition, WritePreconditionError +from teledrive.s3.tagging import ObjectTagStore, BucketTagStore, TaggingError, validate_tags +from teledrive.s3.cors import (BucketCORSStore, CORSError, MAX_CORS_XML_BYTES, + parse_cors_xml, cors_xml, matching_rule, response_cors_headers) +from teledrive.s3.versioning import (VersioningError, MAX_VERSIONING_XML_BYTES, + parse_versioning_xml, versioning_xml) +from teledrive.s3.public_access import (PublicAccessError, MAX_PUBLIC_ACCESS_XML_BYTES, + parse_public_access_xml, public_access_xml) +from teledrive.s3.lifecycle import (LifecycleError, MAX_XML as MAX_LIFECYCLE_XML, + BucketLifecycleStore, parse_lifecycle_xml, lifecycle_xml) +from teledrive.s3.object_metadata import ( + ObjectMetadataStore, ObjectMetadataError, parse_object_metadata, signed_header_names, + STANDARD_HEADERS, +) +from teledrive.storage.catalog import BucketCatalog, published_versions +from teledrive.storage.copy import copy_object, CopyConflict +from teledrive.storage.engine import TelegramObjectStorage +from teledrive.storage.manifest import ManifestStore, objects +from teledrive.storage.telethon_adapter import RECOMMENDED_CHUNK_SIZE +import web.app as _webapp +from web.app import db_connect, storage_transport, _secure_mkdir + +NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/" +app = FastAPI(title="TeleDrive S3 (initial compatibility subset)", docs_url=None, + redoc_url=None, openapi_url=None) + + +def _xml(tag: str, pairs: list[tuple[str, str]] | None = None) -> bytes: + root = ET.Element(tag, xmlns=NAMESPACE) + for name, value in pairs or []: + ET.SubElement(root, name).text = str(value) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def _error(code: str, message: str, status: int) -> Response: + # S3 error XML is *not* in the successful-response namespace. + # Boto3/other AWS SDK error parsers otherwise yield Error.Code == "". + root = ET.Element("Error") + ET.SubElement(root, "Code").text = code + ET.SubElement(root, "Message").text = message + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), + media_type="application/xml", status_code=status) + + +def _headers(item, etag: str | None = None, + metadata: dict[str, str] | None = None, + *, checksum_mode: bool = False) -> dict[str, str]: + headers = { + "ETag": '"' + (etag or item.sha256) + '"', + "Content-Length": str(item.size), + "Accept-Ranges": "bytes", + "Content-Type": "application/octet-stream", + "Last-Modified": http_last_modified(item.created_at), + "x-amz-version-id": item.id, + } + if checksum_mode: + # The immutable manifest stores the digest of the FULL plaintext + # object, independently of ETag/part layout. This is a real SHA-256, + # not an invented checksum or AWS multipart checksum composition. + headers["x-amz-checksum-sha256"] = base64.b64encode( + bytes.fromhex(item.sha256) + ).decode("ascii") + headers["x-amz-checksum-type"] = "FULL_OBJECT" + for name, value in (metadata or {}).items(): + # HTTP header names are case-insensitive. Do not emit both + # Content-Type (the default) and content-type (persisted metadata): + # SDK clients otherwise interpret them as a comma-joined MIME type. + if name.lower() == "content-type": + headers["Content-Type"] = value + else: + headers[name] = value + return headers + + +def _reject_unimplemented_write_options(request: Request) -> None: + """Fail closed for dangerous S3 semantics we cannot yet guarantee. + + In particular, acknowledging encryption, retention, ACL or a storage + class without implementing its semantics would mislead backup clients. + Do not stage bytes or contact Telegram when any option is present. + """ + rejected = { + "x-amz-grant-read", "x-amz-grant-write", + "x-amz-grant-full-control", "x-amz-grant-read-acp", + "x-amz-grant-write-acp", + "x-amz-server-side-encryption", + "x-amz-server-side-encryption-aws-kms-key-id", + "x-amz-server-side-encryption-context", + "x-amz-server-side-encryption-customer-algorithm", + "x-amz-server-side-encryption-customer-key", + "x-amz-server-side-encryption-customer-key-md5", + "x-amz-bucket-key-enabled", + "x-amz-website-redirect-location", "x-amz-object-lock-mode", + "x-amz-object-lock-retain-until-date", "x-amz-object-lock-legal-hold", + "x-amz-request-payer", "x-amz-trailer", + "x-amz-bypass-governance-retention", "x-amz-mfa", + "x-amz-expected-bucket-owner", + "x-amz-decoded-content-length", + } + # The S3 gateway is permanently private and has exactly one storage + # tier, so these explicit default values do not change actual behavior. + # Reject every other canned ACL / storage-class rather than pretending + # to provide public-read, Glacier, Standard-IA or replicated durability. + defaults = { + "x-amz-acl": "private", + "x-amz-storage-class": "STANDARD", + } + signed = signed_header_names(request) + for name, expected in defaults.items(): + actual = request.headers.get(name) + if actual is None: + continue + if name not in signed: + raise SigV4Error("AccessDenied", "S3 access mode must be signed") + if actual != expected: + raise SigV4Error("NotImplemented", "Unsupported S3 ACL or storage tier") + if (any(name in request.headers for name in rejected) + or request.headers.get("content-encoding", "").lower() == "aws-chunked"): + raise SigV4Error( + "NotImplemented", + "S3 encryption, retention, ACL, storage class or aws-chunked semantics unsupported", + ) + + +def _verify_upload_sha256( + request: Request, digest: bytes, *, md5_digest: bytes | None = None, + sha1_digest: bytes | None = None, + crc64nvme_value: int | None = None, +) -> None: + """Verify *every* supplied checksum, not only the first matching digest. + + Signature verification and upload checksum validation are independent. + A valid SHA-256 content digest must never permit a contradictory + Content-MD5, SHA-1 or x-amz-checksum-sha256 to go unreported. + """ + actual = base64.b64encode(digest).decode("ascii") + sha256_header = request.headers.get("x-amz-checksum-sha256") + if sha256_header is not None and not hmac.compare_digest(sha256_header, actual): + raise SigV4Error("BadDigest", "SHA-256 checksum mismatch") + md5_header = request.headers.get("content-md5") + if md5_header is not None: + if md5_digest is None or not hmac.compare_digest( + md5_header, base64.b64encode(md5_digest).decode("ascii") + ): + raise SigV4Error("BadDigest", "Content-MD5 checksum mismatch") + crc64_header = request.headers.get("x-amz-checksum-crc64nvme") + if crc64_header is not None: + if crc64nvme_value is None: + # Never acknowledge an unverified CRC64NVME declaration. + raise SigV4Error("NotImplemented", "CRC64NVME not verified on this operation") + actual_crc64 = base64.b64encode( + crc64nvme_value.to_bytes(8, "big") + ).decode("ascii") + if not hmac.compare_digest(crc64_header, actual_crc64): + raise SigV4Error("BadDigest", "CRC64NVME checksum mismatch") + sha1_header = request.headers.get("x-amz-checksum-sha1") + if sha1_header is not None: + if sha1_digest is None or not hmac.compare_digest( + sha1_header, base64.b64encode(sha1_digest).decode("ascii") + ): + raise SigV4Error("BadDigest", "SHA-1 checksum mismatch") + declared = request.headers.get("x-amz-sdk-checksum-algorithm") + if declared is not None: + allowed = {"SHA1": "x-amz-checksum-sha1", "SHA256": "x-amz-checksum-sha256", + "CRC32": "x-amz-checksum-crc32", + "CRC32C": "x-amz-checksum-crc32c", + "CRC64NVME": "x-amz-checksum-crc64nvme"} + if declared not in allowed: + raise SigV4Error("NotImplemented", "Unsupported SDK checksum algorithm") + if request.headers.get(allowed[declared]) is None: + raise SigV4Error("BadDigest", "Declared SDK checksum is missing") + + hexdigest = digest.hex() + signed_content = request.headers.get("x-amz-content-sha256", "") + presigned = "X-Amz-Algorithm" in request.query_params + if signed_content == hexdigest: + return + if sha256_header is not None and hmac.compare_digest(sha256_header, actual): + return + if (not presigned and signed_content == "UNSIGNED-PAYLOAD" + and md5_header is not None and md5_digest is not None): + # Header-mode SigV4 has already authenticated the signed MD5 header. + if "content-md5" in signed_header_names(request): + return + raise SigV4Error("XAmzContentSHA256Mismatch", "Signed upload checksum mismatch") + + +def _verify_bucket_control_payload(request: Request, body: bytes) -> None: + """Exact signed checksum policy for small S3 bucket-control XML payloads.""" + import zlib + for header, actual in ( + ("x-amz-checksum-crc32", zlib.crc32(body) & 0xffffffff), + ("x-amz-checksum-crc32c", crc32c(body)), + ): + declared = request.headers.get(header) + if declared is not None and not hmac.compare_digest( + declared, base64.b64encode(actual.to_bytes(4, "big")).decode(), + ): + raise SigV4Error("BadDigest", "Bucket XML checksum mismatch") + _verify_upload_sha256( + request, hashlib.sha256(body).digest(), + md5_digest=hashlib.md5(body, usedforsecurity=False).digest(), + sha1_digest=hashlib.sha1(body, usedforsecurity=False).digest(), + crc64nvme_value=crc64nvme(body), + ) + + +def _initial_tags(request: Request, owner: str, credentials: S3CredentialStore): + """Parse only SigV4-bound x-amz-tagging; never let an unsigned header mutate metadata.""" + raw = request.headers.get("x-amz-tagging") + if raw is None: + return None + if len(raw.encode("utf-8")) > 8192: + raise TaggingError("InvalidTag", "Tag header is too large") + if "X-Amz-Algorithm" in request.query_params: + signed = request.query_params.get("X-Amz-SignedHeaders", "").split(";") + else: + auth = request.headers.get("authorization", "") + signed = auth.partition("SignedHeaders=")[2].split(",", 1)[0].split(";") + if "x-amz-tagging" not in signed: + raise SigV4Error("AccessDenied", "x-amz-tagging must be signed") + if credentials.resolve(owner, "s3:PutObjectTagging") is None: + raise SigV4Error("AccessDenied", "PutObjectTagging permission required") + return _parse_initial_tags(raw) + + +def _parse_initial_tags(raw: str) -> list[tuple[str, str]]: + """Strictly decode and validate either signed header or POST form tags.""" + import re + if len(raw.encode("utf-8")) > 8192: + raise TaggingError("InvalidTag", "Tag payload is too large") + if re.search(r"%(?![0-9A-Fa-f]{2})", raw): + raise TaggingError("InvalidTag", "Malformed tag URL encoding") + try: + pairs = parse_qsl(raw, keep_blank_values=True, strict_parsing=True, + encoding="utf-8", errors="strict", max_num_fields=10) + except (UnicodeError, ValueError) as exc: + raise TaggingError("InvalidTag", "Malformed tagging header") from exc + return validate_tags(pairs) + + +def _tag_xml(tags: list[tuple[str, str]]) -> bytes: + root = ET.Element("Tagging", xmlns=NAMESPACE) + tagset = ET.SubElement(root, "TagSet") + for key, value in tags: + tag = ET.SubElement(tagset, "Tag") + ET.SubElement(tag, "Key").text = key + ET.SubElement(tag, "Value").text = value + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def _parse_tag_xml(body: bytes, *, bucket: bool = False) -> list[tuple[str, str]]: + if len(body) > (256 * 1024 if bucket else 16 * 1024): + raise TaggingError("MalformedXML", "Tagging XML exceeds maximum size") + try: + root = SafeET.fromstring(body) + except (ET.ParseError, DefusedXmlException) as exc: + raise TaggingError("MalformedXML", "Tagging XML is malformed") from exc + def tag_name(node): + # Both bare AWS S3 XML and canonical 2006-03-01 namespace are + # accepted; arbitrary XML namespaces must not bypass validation. + name = node.tag.rsplit("}", 1)[-1] + if node.tag not in {name, f"{{{NAMESPACE}}}{name}"}: + raise TaggingError("MalformedXML", "Unexpected tagging XML namespace") + return name + if (tag_name(root) != "Tagging" or root.attrib + or len(root) != 1 or tag_name(root[0]) != "TagSet" + or root[0].attrib): + raise TaggingError("MalformedXML", "Unexpected tagging XML structure") + entries = [] + for tag in root[0]: + if tag_name(tag) != "Tag" or tag.attrib or len(tag) != 2: + raise TaggingError("MalformedXML", "Invalid Tag element") + children = {tag_name(child): child for child in tag} + if (set(children) != {"Key", "Value"} or + any(child.attrib or len(child) for child in tag)): + raise TaggingError("MalformedXML", "Invalid Tag key/value fields") + entries.append((children["Key"].text or "", children["Value"].text or "")) + return validate_tags(entries, max_tags=50 if bucket else 10) + + +async def _tag_request_tags(request: Request, *, bucket: bool = False) -> list[tuple[str, str]]: + body = bytearray() + async for block in request.stream(): + body.extend(block) + if len(body) > (256 * 1024 if bucket else 16 * 1024): + raise TaggingError("MalformedXML", "Tagging XML exceeds maximum size") + if bucket: + _verify_bucket_control_payload(request, bytes(body)) + else: + _verify_upload_sha256( + request, hashlib.sha256(body).digest(), crc64nvme_value=crc64nvme(body), + ) + md5_header = request.headers.get("content-md5") + if md5_header and not hmac.compare_digest( + md5_header, + base64.b64encode(hashlib.md5(body, usedforsecurity=False).digest()).decode("ascii") + ): + raise TaggingError("BadDigest", "Content-MD5 checksum mismatch") + return _parse_tag_xml(bytes(body), bucket=bucket) + + +def _range(value: str | None, size: int) -> tuple[int, int, bool]: + if value is None: + return 0, size, False + import re + match = re.fullmatch(r"bytes=(\d*)-(\d*)", value.strip()) + if not match or not (match.group(1) or match.group(2)) or size <= 0: + raise SigV4Error("InvalidRange", "Unsupported byte range") + first, last = match.groups() + if first: + start = int(first) + end = min(int(last) + 1, size) if last else size + else: + amount = int(last) + start, end = max(size - amount, 0), size + if start >= size or start >= end or (not first and int(last) <= 0): + raise SigV4Error("InvalidRange", "Unsatisfiable byte range") + return start, end, True + + +def _cursor_encode(master: bytes, bucket: str, prefix: str, delimiter: str, + after: str, skip_prefix: str) -> str: + payload = json.dumps( + {"bucket": bucket, "prefix": prefix, "delimiter": delimiter, + "after": after, "skip_prefix": skip_prefix}, + separators=(",", ":"), sort_keys=True, + ).encode() + encoded = base64.urlsafe_b64encode(payload).decode().rstrip("=") + signature = hmac.new(master, encoded.encode(), hashlib.sha256).hexdigest() + return f"{encoded}.{signature}" + + +def _cursor_decode(token: str, master: bytes, bucket: str, + prefix: str, delimiter: str) -> tuple[str, str]: + try: + encoded, signature = token.split(".", 1) + if len(token) > 4096 or not hmac.compare_digest( + signature, hmac.new(master, encoded.encode(), hashlib.sha256).hexdigest() + ): + raise ValueError("Signature mismatch") + parsed = json.loads(base64.urlsafe_b64decode(encoded + "=" * (-len(encoded) % 4))) + if (not isinstance(parsed, dict) or parsed.get("bucket") != bucket + or parsed.get("prefix") != prefix or parsed.get("delimiter") != delimiter + or not isinstance(parsed.get("after"), str) + or not isinstance(parsed.get("skip_prefix"), str)): + raise ValueError("Cursor does not match request") + return parsed["after"], parsed["skip_prefix"] + except (ValueError, UnicodeError, TypeError, KeyError, binascii.Error) as exc: + raise SigV4Error("InvalidArgument", "Invalid continuation token") from exc + + +def _page(catalog: BucketCatalog, bucket: str, prefix: str, + delimiter: str, after: str, limit: int, + skip_prefix: str) -> tuple[list, list[str], str, str, bool]: + """Page over raw keys without splitting a common-prefix across pages. + + The last emitted prefix is carried in the signed continuation cursor + so a folder containing >1000 objects is not shown repeatedly. + """ + keys = [] + common = [] + seen = {skip_prefix} if skip_prefix else set() + position = after + last_consumed = after + while True: + batch = catalog.list_objects(bucket, prefix=prefix, after=position, limit=1000) + if not batch: + return keys, common, last_consumed, skip_prefix, False + for item in batch: + key = item.item.key + tail = key[len(prefix):] + common_name = "" + if delimiter and delimiter in tail: + common_name = prefix + tail.split(delimiter, 1)[0] + delimiter + if common_name and common_name in seen: + position = key + last_consumed = key + continue + if len(keys) + len(common) >= limit: + return keys, common, last_consumed, skip_prefix, True + if common_name: + common.append(common_name) + seen.add(common_name) + skip_prefix = common_name + else: + keys.append(item) + skip_prefix = "" + position = key + last_consumed = key + if len(batch) < 1000: + return keys, common, last_consumed, skip_prefix, False + + +def _list_xml(bucket: str, keys: list, common: list[str], prefix: str, + limit: int, after: str, delimiter: str, + more: bool, next_token: str | None, etag_for=None, *, + version_one: bool = False, next_marker: str | None = None, + encoding_type: str | None = None) -> bytes: + root = ET.Element("ListBucketResult", xmlns=NAMESPACE) + # S3 EncodingType=url percent-encodes UTF-8 keys *inside XML* so names + # containing XML control characters remain safely representable. + encoded = lambda value: quote(value, safe="") if encoding_type == "url" else value + fields = [("Name", bucket), ("Prefix", encoded(prefix)), + ("MaxKeys", limit), ("IsTruncated", "true" if more else "false")] + if not version_one: + fields.append(("KeyCount", len(keys) + len(common))) + for field, val in fields: + ET.SubElement(root, field).text = str(val) + if delimiter: + ET.SubElement(root, "Delimiter").text = encoded(delimiter) + if encoding_type: + ET.SubElement(root, "EncodingType").text = encoding_type + if version_one: + ET.SubElement(root, "Marker").text = encoded(after) + elif after: + ET.SubElement(root, "StartAfter").text = encoded(after) + for item in keys: + c = ET.SubElement(root, "Contents") + for field, value in ( + ("Key", encoded(item.item.key)), + ("LastModified", time.strftime("%Y-%m-%dT%H:%M:%S.000Z", time.gmtime(item.item.created_at))), + ("ETag", '"' + (etag_for(item.item) if etag_for else item.item.sha256) + '"'), + ("Size", item.item.size), + ("StorageClass", "STANDARD"), + ): + ET.SubElement(c, field).text = str(value) + for name in common: + ET.SubElement(ET.SubElement(root, "CommonPrefixes"), "Prefix").text = encoded(name) + if next_token: + ET.SubElement(root, "NextContinuationToken").text = next_token + if version_one and next_marker: + ET.SubElement(root, "NextMarker").text = encoded(next_marker) + return ET.tostring(root, encoding="utf-8", xml_declaration=True) + + +def _virtual_host_bucket(request: Request) -> str | None: + """Use Host only when a public DNS suffix was explicitly configured. + + SigV4 must authenticate the exact Host afterwards; never trust proxy + forwarding headers for bucket selection. The default remains path-style. + """ + suffix = os.environ.get("TELEDRIVE_S3_VHOST_SUFFIX", "").strip().lower().strip(".") + if not suffix: + return None + label = r"[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?" + if len(suffix) > 253 or re.fullmatch(label + r"(?:\." + label + r")+", suffix) is None: + raise SigV4Error("InvalidArgument", "Invalid virtual-host DNS suffix") + authority = request.headers.get("host", "").lower() + if authority.count(":") == 1 and authority.rsplit(":", 1)[1].isdigit(): + hostname = authority.rsplit(":", 1)[0] + else: + hostname = authority + ending = "." + suffix + if not hostname.endswith(ending): + return None + bucket = hostname[:-len(ending)] + try: + BucketCatalog.validate_bucket(bucket) + except ValueError as exc: + raise SigV4Error("InvalidArgument", "Invalid virtual-host bucket") from exc + return bucket + + +async def _handle(request: Request): + db = None + retain = False + try: + raw_path = request.scope.get("raw_path", b"/").decode("ascii") + logical = unquote(raw_path.lstrip("/")) + virtual_bucket = _virtual_host_bucket(request) + if virtual_bucket is not None: + bucket, separator, key = virtual_bucket, "/" if logical else "", logical + else: + bucket, separator, key = logical.partition("/") + validate_s3_query(request.method, bucket, bool(separator), request.query_params) + if not bucket: + if request.method != "GET": + return _error("MethodNotAllowed", "Use GET to list buckets", 405) + action = "s3:ListAllMyBuckets" + elif not separator: + action = { + "GET": "s3:ListBucket", "HEAD": "s3:ListBucket", + "PUT": "s3:CreateBucket", "DELETE": "s3:DeleteBucket", + }.get(request.method) + else: + action = { + "GET": "s3:GetObject", "HEAD": "s3:GetObject", + "PUT": "s3:PutObject", "DELETE": "s3:DeleteObject", + }.get(request.method) + form_upload = bool(bucket and not separator and request.method == "POST" + and not request.query_params) + multi_delete = bool(bucket and not separator and request.method == "POST" + and "delete" in request.query_params) + if "delete" in request.query_params and not multi_delete: + return _error("NotImplemented", "Unsupported DeleteObjects route", 501) + if request.method in ("PUT", "DELETE") or ( + request.method == "POST" and separator and request.query_params.get("uploadId") + ): + if any(name in request.headers for name in ("if-modified-since", "if-unmodified-since")): + return _error("NotImplemented", "Conditional write dates are unsupported", 501) + has_write_cond = any(name in request.headers for name in ("if-match", "if-none-match")) + # SigV4 presigned URLs carry authentication query fields, not + # operation selectors. Permit these to use the same signed CAS. + auth_only_query = all(name.startswith("X-Amz-") + for name in request.query_params) + simple_object = bool(request.method in ("PUT", "DELETE") + and separator and auth_only_query + and not request.headers.get("x-amz-copy-source")) + multipart_completion = bool( + request.method == "POST" and separator + and request.query_params.get("uploadId") + and all(k == "uploadId" or k.startswith("X-Amz-") + for k in request.query_params) + ) + if has_write_cond and not (simple_object or multipart_completion): + return _error("NotImplemented", "Unsupported conditional S3 operation", 501) + is_version_read = bool( + separator and request.method in ("GET", "HEAD") + and "versionId" in request.query_params + and all( + key_name == "versionId" + or (key_name == "attributes" and "attributes" in request.query_params + and request.method == "GET") + or key_name.startswith("X-Amz-") + for key_name in request.query_params + ) + ) + if is_version_read and len(request.query_params.getlist("versionId")) != 1: + return _error("InvalidArgument", "Ambiguous version ID", 400) + is_version_delete = bool( + separator and request.method == "DELETE" + and "versionId" in request.query_params + and "tagging" not in request.query_params + ) + if is_version_delete and ( + len(request.query_params.getlist("versionId")) != 1 + or any(k != "versionId" and not k.startswith("X-Amz-") + for k in request.query_params) + ): + return _error("InvalidArgument", "Ambiguous VersionId deletion selector", 400) + is_version_listing = bool( + bucket and not separator and request.method == "GET" + and "versions" in request.query_params + ) + if is_version_listing and any( + item not in {"versions", "prefix", "max-keys", "key-marker", + "version-id-marker"} and not item.startswith("X-Amz-") + for item in request.query_params + ): + return _error("NotImplemented", "Unsupported ListObjectVersions parameter", 501) + mpu_bucket_list = bool(bucket and not separator and "uploads" in request.query_params + and request.method == "GET") + mpu_start = bool(separator and "uploads" in request.query_params and request.method == "POST") + mpu_id = request.query_params.get("uploadId") if separator else None + mpu_part = bool(mpu_id and request.method == "PUT" and "partNumber" in request.query_params) + mpu_complete = bool(mpu_id and request.method == "POST") + mpu_abort = bool(mpu_id and request.method == "DELETE") + mpu_list = bool(mpu_id and request.method == "GET") + is_object_attributes = bool(separator and "attributes" in request.query_params) + is_object_tagging = bool(separator and "tagging" in request.query_params) + is_version_tagging = bool(is_object_tagging and "versionId" in request.query_params) + if is_version_tagging and ( + len(request.query_params.getlist("versionId")) != 1 + or any(name not in {"tagging", "versionId"} and not name.startswith("X-Amz-") + for name in request.query_params) + ): + return _error("InvalidArgument", "Ambiguous versioned tagging selector", 400) + is_bucket_tagging = bool(bucket and not separator and "tagging" in request.query_params) + is_bucket_cors = bool(bucket and not separator and "cors" in request.query_params) + is_bucket_versioning = bool(bucket and not separator and "versioning" in request.query_params) + is_public_access_block = bool(bucket and not separator and "publicAccessBlock" in request.query_params) + is_bucket_lifecycle = bool(bucket and not separator and "lifecycle" in request.query_params) + is_bucket_policy_status = bool(bucket and not separator and "policyStatus" in request.query_params) + if is_bucket_lifecycle and ( + len(request.query_params.getlist("lifecycle")) != 1 + or request.query_params.get("lifecycle") != "" + or any(k != "lifecycle" and not k.startswith("X-Amz-") + for k in request.query_params) + ): + return _error("InvalidArgument", "Ambiguous lifecycle selector", 400) + if is_public_access_block and ( + len(request.query_params.getlist("publicAccessBlock")) != 1 + or request.query_params.get("publicAccessBlock") != "" + or any(name != "publicAccessBlock" and not name.startswith("X-Amz-") + for name in request.query_params) + ): + return _error("InvalidArgument", "Invalid public access control selector", 400) + if is_bucket_versioning and ( + len(request.query_params.getlist("versioning")) != 1 + or request.query_params.get("versioning") != "" + or any(name != "versioning" and not name.startswith("X-Amz-") + for name in request.query_params) + ): + return _error("InvalidArgument", "Invalid bucket versioning selector", 400) + if (is_object_tagging or is_bucket_tagging) and any( + k in request.query_params for k in ("uploads", "uploadId") + ): + return _error("NotImplemented", "Tagging with multipart selectors unsupported", 501) + if is_bucket_tagging and "versionId" in request.query_params: + return _error("NotImplemented", "Bucket tags cannot select object versions", 501) + if is_object_attributes: + if request.method != "GET" or any( + p in request.query_params for p in ("uploadId", "uploads", "tagging") + ): + return _error("NotImplemented", "Unsupported GetObjectAttributes variant", 501) + action = "s3:GetObjectAttributes" + elif form_upload: + action = "s3:PutObject" + elif multi_delete: + if any(p in request.query_params for p in ("tagging", "uploads", "uploadId", "versionId")): + return _error("NotImplemented", "Conflicting DeleteObjects controls", 501) + action = "s3:DeleteObject" + elif is_object_tagging: + action = { + "GET": "s3:GetObjectTagging", + "PUT": "s3:PutObjectTagging", + "DELETE": "s3:DeleteObjectTagging", + }.get(request.method) + elif is_bucket_tagging: + action = { + "GET": "s3:GetBucketTagging", + "PUT": "s3:PutBucketTagging", + "DELETE": "s3:DeleteBucketTagging", + }.get(request.method) + elif is_bucket_cors: + action = { + "GET": "s3:GetBucketCORS", + "PUT": "s3:PutBucketCORS", + "DELETE": "s3:PutBucketCORS", + }.get(request.method) + elif is_bucket_versioning: + action = { + "GET": "s3:GetBucketVersioning", + "PUT": "s3:PutBucketVersioning", + }.get(request.method) + elif is_public_access_block: + action = { + "GET": "s3:GetBucketPublicAccessBlock", + "PUT": "s3:PutBucketPublicAccessBlock", + }.get(request.method) + elif is_bucket_lifecycle: + action = { + "GET": "s3:GetLifecycleConfiguration", + "PUT": "s3:PutLifecycleConfiguration", + "DELETE": "s3:PutLifecycleConfiguration", + }.get(request.method) + elif is_bucket_policy_status: + action = "s3:GetBucketPolicyStatus" if request.method == "GET" else None + elif bucket and not separator and "location" in request.query_params: + action = "s3:GetBucketLocation" if request.method == "GET" else None + elif mpu_start or mpu_part or mpu_complete: + action = "s3:PutObject" + elif is_version_listing: + action = "s3:ListBucketVersions" + elif mpu_bucket_list: + action = "s3:ListBucket" + elif mpu_abort: + action = "s3:DeleteObject" + elif mpu_list: + action = "s3:GetObject" + elif "uploads" in request.query_params or "uploadId" in request.query_params: + return _error("NotImplemented", "This multipart operation is unsupported", 501) + # A selector on an object path must never silently degrade to a + # regular object operation (particularly an unexpected DELETE). + if "lifecycle" in request.query_params and not is_bucket_lifecycle: + return _error("NotImplemented", "Lifecycle controls require bucket route", 501) + if "publicAccessBlock" in request.query_params and not is_public_access_block: + return _error("NotImplemented", "Public access control is bucket-only", 501) + if "versioning" in request.query_params and not is_bucket_versioning: + return _error("NotImplemented", "Bucket versioning is a bucket-only operation", 501) + if action is None or (separator and "partNumber" in request.query_params and not mpu_part): + return _error("NotImplemented", "This S3 operation is not supported yet", 501) + # Unknown control-plane query operations must not be misinterpreted as + # ordinary object reads/writes (e.g. ?tagging, ?versioning, ?acl). + unsupported = { + "acl", "versions", "policy", + "notification", "replication", + "retention", "legal-hold", "logging", + "encryption", "object-lock", "versionId", + } + if any(parameter in request.query_params and ( + (parameter != "versionId" or not (is_version_read or is_version_tagging or is_version_delete)) + and (parameter != "versions" or not is_version_listing) + ) for parameter in unsupported): + return _error("NotImplemented", "This S3 extension is not implemented", 501) + + master = load_master_key() + db = db_connect() + store = S3CredentialStore(db, master) + if form_upload: + if request.headers.get("authorization"): + raise SigV4Error("AccessDenied", "Ambiguous form and header authentication") + _reject_unimplemented_write_options(request) + max_upload = _webapp.REST_MAX_UPLOAD_BYTES + form_limit = min(FORM_LIMIT, max_upload) + if form_limit < 1: + return _error("EntityTooLarge", "Form upload size is disabled", 413) + form_bytes = bytearray() + async for block in request.stream(): + form_bytes.extend(block) + if len(form_bytes) > form_limit: + return _error("EntityTooLarge", "Browser upload form too large", 413) + post_fields, payload = parse_form( + request.headers.get("content-type", ""), bytes(form_bytes), + ) + post_key = verify_post_policy( + post_fields, payload, bucket, store.resolve, max_bytes=form_limit, + ) + # Signed form fields are validated using the same bounded + # metadata contract as ordinary S3 PUT. Browser-controlled MIME, + # cache rules and x-amz-meta-* values must not be silently + # accepted without durable storage. + post_headers = { + name.lower(): value for name, value in post_fields.items() + if name.lower() in STANDARD_HEADERS + or name.startswith("x-amz-meta-") + } + post_metadata = parse_object_metadata( + post_headers, set(post_headers), + ) + # Tagging is a separate S3 capability, not included in generic + # PutObject credentials. Enforce BOTH privileges before sending + # even a single byte, then publish tags in the HEAD transaction. + form_tags = None + if "x-amz-tagging" in post_fields: + credential_access = post_fields["x-amz-credential"].split("/", 1)[0] + if store.resolve(credential_access, "s3:PutObjectTagging") is None: + raise SigV4Error("AccessDenied", "PutObjectTagging permission required") + form_tags = _parse_initial_tags(post_fields["x-amz-tagging"]) + post_catalog = BucketCatalog(ManifestStore(db)) + if bucket not in post_catalog.list_buckets(): + return _error("NoSuchBucket", "Upload bucket does not exist", 404) + directory = Path(_webapp.STORAGE_SCRATCH_DIR) + _secure_mkdir(directory) + fd, temp_name = tempfile.mkstemp(prefix="s3-form-", dir=directory) + staged = Path(temp_name) + try: + with os.fdopen(fd, "wb") as dest: + dest.write(payload) + dest.flush() + os.fsync(dest.fileno()) + transport = storage_transport(db) + await transport.verify_target() + obj = await TelegramObjectStorage( + post_catalog.manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, + scratch_directory=directory, + ).put_file(bucket, post_key, staged) + ObjectMetadataStore(post_catalog).attach_uncommitted( + obj.id, post_metadata, + ) + if form_tags is not None: + ObjectTagStore(post_catalog).attach_uncommitted( + obj.id, form_tags, + ) + post_catalog.publish(obj.id, now=int(time.time())) + etag = '"' + obj.sha256 + '"' + response_headers = {"ETag": etag, "x-amz-version-id": obj.id} + # A redirect is an explicit signed browser instruction. The + # gateway never contacts the target and does not disclose + # credentials or original payload in the callback parameters. + redirect = post_fields.get("success_action_redirect") + if redirect is not None: + parsed_redirect = urlsplit(redirect) # verified HTTPS URL + parameters = urlencode({ + "bucket": bucket, "key": post_key, "etag": etag, + }) + redirect_query = (parsed_redirect.query + "&" + parameters + if parsed_redirect.query else parameters) + location = urlunsplit(( + parsed_redirect.scheme, parsed_redirect.netloc, + parsed_redirect.path, redirect_query, "", + )) + return Response(status_code=303, headers={ + **response_headers, "Location": location, + }) + # The requested response status is itself an exact signed + # policy condition; no unsigned status override is accepted. + status = int(post_fields.get("success_action_status", "204")) + if status == 201: + # Quote untrusted key text before placing it in Location. + # The XML serializer escapes the same key in the body. + location = (str(request.base_url).rstrip("/") + + "/" + quote(bucket, safe="") + + "/" + quote(post_key, safe="/")) + response_headers["Location"] = location + return Response(_xml("PostResponse", [ + ("Location", location), ("Bucket", bucket), + ("Key", post_key), ("ETag", etag), + ]), media_type="application/xml", status_code=201, + headers=response_headers) + return Response(status_code=status, headers=response_headers) + finally: + staged.unlink(missing_ok=True) + owner = verify_header_signature(request, store.resolve, action) + # Every signed S3 method must reject unsupported retention, SSE, + # requester-payer and governance semantics, including GET/HEAD/DELETE. + # Ignoring those headers can turn a supposedly protected delete into + # an ordinary tombstone or pretend to verify a requested checksum. + _reject_unimplemented_write_options(request) + if not is_object_attributes and any( + name in request.headers for name in + ("x-amz-max-parts", "x-amz-part-number-marker") + ): + raise SigV4Error( + "NotImplemented", "Multipart attributes headers require GetObjectAttributes" + ) + checksum_mode = request.headers.get("x-amz-checksum-mode") + if checksum_mode is not None: + # AWS CLI/botocore send ENABLED by default for standard downloads. + # Only object GET/HEAD can use the manifest's full-object digest. + if (request.method not in {"GET", "HEAD"} or not separator + or is_object_attributes or is_object_tagging + or mpu_list or mpu_abort or mpu_part or mpu_complete): + raise SigV4Error( + "NotImplemented", "ChecksumMode unsupported on this S3 operation", + ) + if checksum_mode != "ENABLED": + raise SigV4Error("InvalidArgument", "Unknown checksum mode") + if request.method == "DELETE" and any( + name in request.headers for name in ("x-amz-copy-source", "x-amz-tagging") + ): + raise SigV4Error("NotImplemented", "Unsupported S3 delete headers") + inline_tags = _initial_tags(request, owner, store) if ( + separator and request.method == "PUT" and not is_object_tagging + ) else None + catalog = BucketCatalog(ManifestStore(db)) + + if not bucket: + return Response( + list_buckets_xml(catalog, request.query_params, master, owner), + media_type="application/xml", + ) + + if not separator: + if multi_delete: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "The specified bucket does not exist", 404) + if request.headers.get("x-amz-mfa") or request.headers.get( + "x-amz-bypass-governance-retention" + ): + return _error("NotImplemented", "Retention/MFA Delete unsupported", 501) + body = bytearray() + async for block in request.stream(): + body.extend(block) + if len(body) > MAX_XML_BYTES: + raise BatchDeleteError("MalformedXML", "DeleteObjects XML exceeds 1 MiB") + payload = bytes(body) + verify_body(payload, request.headers) + batch = parse_batch(payload) + result = ET.Element("DeleteResult", xmlns=NAMESPACE) + for key_to_delete in batch.keys: + try: + marker = catalog.delete_marker(bucket, key_to_delete) + except ValueError: + problem = ET.SubElement(result, "Error") + ET.SubElement(problem, "Key").text = key_to_delete + ET.SubElement(problem, "Code").text = "InvalidObjectName" + ET.SubElement(problem, "Message").text = "Invalid object name" + continue + if not batch.quiet: + deleted = ET.SubElement(result, "Deleted") + ET.SubElement(deleted, "Key").text = key_to_delete + ET.SubElement(deleted, "VersionId").text = marker + ET.SubElement(deleted, "DeleteMarker").text = "true" + ET.SubElement(deleted, "DeleteMarkerVersionId").text = marker + return Response(ET.tostring(result, encoding="utf-8", xml_declaration=True), + media_type="application/xml", status_code=200) + if is_bucket_policy_status: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket does not exist", 404) + # Public grants and bucket policies cannot be enabled through + # this private-only gateway. This is NOT a bucket policy engine. + return Response(_xml("PolicyStatus", [("IsPublic", "false")]), + media_type="application/xml") + if is_bucket_lifecycle: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket not found", 404) + rules = BucketLifecycleStore(catalog) + if request.method == "GET": + value = rules.get(bucket) + if value is None: + return _error("NoSuchLifecycleConfiguration", "No lifecycle configuration", 404) + return Response(lifecycle_xml(value), media_type="application/xml") + if request.method == "PUT": + data = bytearray() + async for block in request.stream(): + data.extend(block) + if len(data) > MAX_LIFECYCLE_XML: + raise LifecycleError("MalformedXML", "Lifecycle XML exceeds 32 KiB") + _verify_bucket_control_payload(request, bytes(data)) + rules.replace(bucket, parse_lifecycle_xml(bytes(data))) + return Response(status_code=200) + if request.method == "DELETE": + async for block in request.stream(): + if block: + raise LifecycleError("InvalidRequest", "DeleteBucketLifecycle cannot have body") + rules.replace(bucket, None) + return Response(status_code=204) + return _error("NotImplemented", "Unsupported lifecycle method", 501) + if is_public_access_block: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket does not exist", 404) + if request.method == "GET": + return Response(public_access_xml(), media_type="application/xml") + if request.method == "PUT": + payload = bytearray() + async for block in request.stream(): + payload.extend(block) + if len(payload) > MAX_PUBLIC_ACCESS_XML_BYTES: + raise PublicAccessError("MalformedXML", + "Public access XML exceeds 16 KiB") + _verify_bucket_control_payload(request, bytes(payload)) + parse_public_access_xml(bytes(payload)) + return Response(status_code=200) + return _error("NotImplemented", "Unsupported public access method", 501) + if is_bucket_versioning: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket does not exist", 404) + if request.method == "GET": + return Response(versioning_xml(), media_type="application/xml") + if request.method == "PUT": + payload = bytearray() + async for block in request.stream(): + payload.extend(block) + if len(payload) > MAX_VERSIONING_XML_BYTES: + raise VersioningError("MalformedXML", + "Versioning XML exceeds 16 KiB") + _verify_bucket_control_payload(request, bytes(payload)) + parse_versioning_xml(bytes(payload)) + return Response(status_code=200) + return _error("NotImplemented", "Unsupported bucket versioning method", 501) + if is_bucket_cors: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket does not exist", 404) + cors_store = BucketCORSStore(catalog) + if request.method == "GET": + rules = cors_store.get(bucket) + if rules is None: + return _error("NoSuchCORSConfiguration", "No CORS configuration", 404) + return Response(cors_xml(rules), media_type="application/xml") + if request.method == "PUT": + body = bytearray() + async for block in request.stream(): + body.extend(block) + if len(body) > MAX_CORS_XML_BYTES: + raise CORSError("MalformedXML", "CORS XML exceeds 64 KiB") + _verify_upload_sha256( + request, hashlib.sha256(body).digest(), + crc64nvme_value=crc64nvme(body), + ) + md5_header = request.headers.get("content-md5") + if md5_header and not hmac.compare_digest( + md5_header, + base64.b64encode(hashlib.md5(body, usedforsecurity=False).digest()).decode() + ): + raise CORSError("BadDigest", "Content-MD5 mismatch") + cors_store.replace(bucket, parse_cors_xml(bytes(body))) + return Response(status_code=200) + if request.method == "DELETE": + async for block in request.stream(): + if block: + raise CORSError("InvalidRequest", "DeleteBucketCors must not have a body") + cors_store.replace(bucket, []) + return Response(status_code=204) + return _error("NotImplemented", "Unsupported bucket CORS operation", 501) + if is_bucket_tagging: + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "Bucket does not exist", 404) + tagstore = BucketTagStore(catalog) + if request.method == "GET": + values = tagstore.get(bucket) + if not values: + return _error("NoSuchTagSet", "No bucket tag set exists", 404) + return Response(_tag_xml(values), media_type="application/xml") + if request.method == "PUT": + tagstore.replace(bucket, await _tag_request_tags(request, bucket=True)) + return Response(status_code=200) + if request.method == "DELETE": + async for block in request.stream(): + if block: + raise TaggingError("InvalidRequest", "DeleteBucketTagging must have no body") + tagstore.replace(bucket, []) + return Response(status_code=204) + return _error("NotImplemented", "Unsupported bucket tagging method", 501) + if request.method == "PUT": + catalog.create_bucket(bucket, int(time.time())) + return Response(status_code=200) + if request.method == "DELETE": + try: + catalog.delete_bucket(bucket) + except ValueError as exc: + raise MultipartError("BucketNotEmpty", str(exc), 409) from exc + return Response(status_code=204) + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "The specified bucket does not exist", 404) + if request.method == "HEAD": + return Response(status_code=200) + if "location" in request.query_params: + if request.method != "GET": + return _error("MethodNotAllowed", "GetBucketLocation requires GET", 405) + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "The specified bucket does not exist", 404) + # AWS represents us-east-1 as an empty LocationConstraint. + root = ET.Element("LocationConstraint", xmlns=NAMESPACE) + return Response( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + media_type="application/xml", + ) + if is_version_listing: + if len(request.query_params.getlist("versions")) != 1 or ( + request.query_params.get("versions") not in ("",) + ): + return _error("InvalidArgument", "Invalid versions selector", 400) + for marker in ("prefix", "max-keys", "key-marker", "version-id-marker"): + if len(request.query_params.getlist(marker)) > 1: + return _error("InvalidArgument", "Ambiguous version listing parameter", 400) + try: + maximum = int(request.query_params.get("max-keys", "1000")) + key_marker = request.query_params.get("key-marker", "") + version_marker = request.query_params.get("version-id-marker", "") + prefix = request.query_params.get("prefix", "") + entries, has_more = catalog.list_bucket_versions( + bucket, prefix=prefix, key_marker=key_marker, + version_marker=version_marker, limit=maximum, + ) + except ValueError: + return _error("InvalidArgument", "Invalid version list cursor or limit", 400) + root = ET.Element("ListVersionsResult", xmlns=NAMESPACE) + for label, value in ( + ("Name", bucket), ("Prefix", prefix), + ("KeyMarker", key_marker), ("VersionIdMarker", version_marker), + ("MaxKeys", maximum), ("IsTruncated", str(has_more).lower()), + ): + ET.SubElement(root, label).text = str(value) + if has_more and entries: + ET.SubElement(root, "NextKeyMarker").text = entries[-1]["key"] + ET.SubElement(root, "NextVersionIdMarker").text = entries[-1]["version_id"] + mpu = MultipartStore(catalog) + for entry in entries: + element = ET.SubElement( + root, "DeleteMarker" if entry["is_delete_marker"] else "Version" + ) + fields = [ + ("Key", entry["key"]), ("VersionId", entry["version_id"]), + ("IsLatest", str(entry["is_current"]).lower()), + ("LastModified", time.strftime( + "%Y-%m-%dT%H:%M:%S.000Z", + time.gmtime(entry["committed_ns"] // 1_000_000_000), + )), + ] + if not entry["is_delete_marker"]: + fields += [ + ("ETag", '"' + mpu.object_etag( + entry["version_id"], entry["sha256"] + ) + '"'), + ("Size", entry["size"]), ("StorageClass", "STANDARD"), + ] + for label, value in fields: + ET.SubElement(element, label).text = str(value) + return Response( + ET.tostring(root, encoding="utf-8", xml_declaration=True), + media_type="application/xml", + ) + if mpu_bucket_list: + query = request.query_params + prefix = query.get("prefix", "") + delimiter = query.get("delimiter", "") + encoding = query.get("encoding-type") + if encoding not in (None, "url"): + return _error("InvalidArgument", "Only encoding-type=url is supported", 400) + if len(prefix.encode("utf-8")) > 1024 or len(delimiter.encode("utf-8")) > 16: + return _error("InvalidArgument", "Multipart prefix/delimiter too long", 400) + try: + limit = int(query.get("max-uploads", "1000")) + except ValueError: + return _error("InvalidArgument", "Invalid max-uploads", 400) + if not 1 <= limit <= 1000: + return _error("InvalidArgument", "max-uploads must be 1..1000", 400) + marker_key = query.get("key-marker", "") + marker_id = query.get("upload-id-marker", "") + if marker_id and not marker_key: + return _error("InvalidArgument", "Upload-id marker requires key marker", 400) + multipart = MultipartStore(catalog) + if delimiter: + entries, more, next_marker = multipart.list_uploads_grouped( + bucket, owner, prefix=prefix, delimiter=delimiter, + after_key=marker_key, after_id=marker_id, limit=limit, + ) + else: + records, more = multipart.list_uploads( + bucket, owner, prefix=prefix, after_key=marker_key, + after_id=marker_id, limit=limit, + ) + entries = [("upload", record) for record in records] + next_marker = ((records[-1].object_key, records[-1].id) + if records else ("", "")) + + encode = (lambda value: quote(value, safe="/") if encoding == "url" else value) + root = ET.Element("ListMultipartUploadsResult", xmlns=NAMESPACE) + for label, val in ( + ("Bucket", bucket), ("KeyMarker", encode(marker_key)), + ("UploadIdMarker", marker_id), ("Prefix", encode(prefix)), + ("MaxUploads", limit), ("IsTruncated", str(more).lower()), + ): + ET.SubElement(root, label).text = str(val) + if delimiter: + ET.SubElement(root, "Delimiter").text = encode(delimiter) + if encoding: + ET.SubElement(root, "EncodingType").text = encoding + if more: + ET.SubElement(root, "NextKeyMarker").text = encode(next_marker[0]) + ET.SubElement(root, "NextUploadIdMarker").text = next_marker[1] + for kind, value in entries: + if kind == "prefix": + ET.SubElement( + ET.SubElement(root, "CommonPrefixes"), "Prefix" + ).text = encode(value) + continue + item = ET.SubElement(root, "Upload") + for label, contents in ( + ("Key", encode(value.object_key)), ("UploadId", value.id), + ("StorageClass", "STANDARD"), + ("Initiated", time.strftime("%Y-%m-%dT%H:%M:%S.000Z", + time.gmtime(value.created_at))), + ): + ET.SubElement(item, label).text = str(contents) + return Response(ET.tostring(root, encoding="utf-8", xml_declaration=True), + media_type="application/xml") + list_type = request.query_params.get("list-type") + if list_type not in (None, "2"): + return _error("NotImplemented", "Only ListObjectsV1/V2 are supported", 501) + version_one = list_type is None + prefix = request.query_params.get("prefix", "") + delimiter = request.query_params.get("delimiter", "") + encoding_type = request.query_params.get("encoding-type") + if encoding_type not in (None, "url"): + return _error("InvalidArgument", "Unsupported list encoding-type", 400) + if len(prefix.encode("utf-8")) > 1024 or len(delimiter.encode("utf-8")) > 16: + return _error("InvalidArgument", "Prefix or delimiter is too large", 400) + if version_one: + if any(p in request.query_params for p in ("continuation-token", "start-after")): + return _error("InvalidArgument", "V2 cursor is not a V1 marker", 400) + after = request.query_params.get("marker", "") + # V1 NextMarker may denote a CommonPrefix. Skip every + # descendant of that prefix instead of returning it again. + skip_prefix = after if delimiter and after.endswith(delimiter) else "" + token = None + else: + if "marker" in request.query_params: + return _error("InvalidArgument", "V1 marker cannot be used with V2", 400) + token = request.query_params.get("continuation-token") + if token: + after, skip_prefix = _cursor_decode(token, master, bucket, prefix, delimiter) + else: + after = request.query_params.get("start-after", "") + skip_prefix = "" + try: + limit = int(request.query_params.get("max-keys", "1000")) + except ValueError: + return _error("InvalidArgument", "Invalid max-keys", 400) + if limit < 1 or limit > 1000: + return _error("InvalidArgument", "max-keys must be between 1 and 1000", 400) + keys, common, last, skipped, more = _page( + catalog, bucket, prefix, delimiter, after, limit, skip_prefix + ) + next_token = (_cursor_encode(master, bucket, prefix, delimiter, last, skipped) + if more and not version_one else None) + next_marker = (max([item.item.key for item in keys] + common) + if more and version_one and (keys or common) else None) + tags = MultipartStore(catalog) + return Response(_list_xml(bucket, keys, common, prefix, limit, + after if version_one or not token else "", + delimiter, more, next_token, + lambda obj: tags.object_etag(obj.id, obj.sha256), + version_one=version_one, next_marker=next_marker, + encoding_type=encoding_type), + media_type="application/xml") + + if bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "The specified bucket does not exist", 404) + if not key: + return _error("InvalidObjectName", "Object key is empty", 400) + + if is_object_tagging: + if is_version_tagging: + selected_id = request.query_params.get("versionId", "") + if not selected_id or len(selected_id) > 80: + return _error("InvalidArgument", "Invalid object VersionId", 400) + if not catalog.is_visible_version(bucket, key, selected_id): + return _error("NoSuchVersion", "Object version is not published", 404) + try: + version_item = catalog.manifests.get(selected_id) + except FileNotFoundError: + return _error("NoSuchVersion", "Object version is not ready", 404) + if version_item.bucket != bucket or version_item.key != key: + return _error("NoSuchVersion", "Version ownership mismatch", 404) + target_id = selected_id + else: + try: + target_id = catalog.get(bucket, key).item.id + except FileNotFoundError: + return _error("NoSuchKey", "Object does not exist", 404) + tags = ObjectTagStore(catalog) + if request.method == "GET": + return Response(_tag_xml(tags.get(target_id)), + media_type="application/xml", status_code=200, + headers={"x-amz-version-id": target_id}) + if request.method == "PUT": + parsed = await _tag_request_tags(request) + if is_version_tagging: + tags.replace_published(bucket, key, target_id, parsed) + else: + tags.replace_current(bucket, key, parsed) + return Response(status_code=200, headers={"x-amz-version-id": target_id}) + if request.method == "DELETE": + async for block in request.stream(): + if block: + raise TaggingError("InvalidRequest", "DeleteObjectTagging must have no body") + if is_version_tagging: + tags.replace_published(bucket, key, target_id, []) + else: + tags.replace_current(bucket, key, []) + return Response(status_code=204, headers={"x-amz-version-id": target_id}) + return _error("NotImplemented", "Unsupported object tagging method", 501) + + if mpu_start or mpu_part or mpu_complete or mpu_abort or mpu_list: + mpu = MultipartStore(catalog) + if mpu_start: + # Initiate is metadata-only. Reject any supplied body before + # allocating a durable multipart session, including if a + # leaked presigned URL is replayed with altered content. + async for block in request.stream(): + if block: + raise MultipartError( + "InvalidRequest", "CreateMultipartUpload requires an empty body" + ) + http_properties = parse_object_metadata( + request.headers, signed_header_names(request), + ) + upload_id = mpu.initiate( + bucket, key, owner, http_metadata=http_properties, + ) + return Response(_xml("InitiateMultipartUploadResult", [ + ("Bucket", bucket), ("Key", key), ("UploadId", upload_id), + ]), media_type="application/xml", status_code=200) + upload_id = mpu_id + if not upload_id: + raise MultipartError("NoSuchUpload", "Missing UploadId", 404) + if mpu_abort: + mpu.abort(upload_id, bucket, key, owner) + return Response(status_code=204) + if mpu_list: + try: + raw_marker = request.query_params.get("part-number-marker", "0") + raw_limit = request.query_params.get("max-parts", "1000") + if (re.fullmatch(r"[0-9]{1,5}", raw_marker) is None + or re.fullmatch(r"[0-9]{1,4}", raw_limit) is None): + raise ValueError("Invalid decimal") + marker = int(raw_marker) + limit = int(raw_limit) + except ValueError: + raise MultipartError("InvalidArgument", "Invalid ListParts pagination") + selected_parts, more = mpu.list_parts_page( + upload_id, bucket, key, owner, marker=marker, limit=limit, + ) + result = ET.Element("ListPartsResult", xmlns=NAMESPACE) + for title, val in ( + ("Bucket", bucket), ("Key", key), ("UploadId", upload_id), + ("PartNumberMarker", marker), ("MaxParts", limit), + ("StorageClass", "STANDARD"), + ("IsTruncated", str(more).lower()), + ): + ET.SubElement(result, title).text = str(val) + if more: + ET.SubElement(result, "NextPartNumberMarker").text = str( + selected_parts[-1].number + ) + for part in selected_parts: + row = ET.SubElement(result, "Part") + ET.SubElement(row, "PartNumber").text = str(part.number) + ET.SubElement(row, "ETag").text = '"' + part.etag + '"' + ET.SubElement(row, "LastModified").text = time.strftime( + "%Y-%m-%dT%H:%M:%S.000Z", time.gmtime(part.updated_at), + ) + ET.SubElement(row, "Size").text = str(part.size) + return Response(ET.tostring(result, encoding="utf-8", xml_declaration=True), + media_type="application/xml") + if mpu_complete: + mpu.require(upload_id, bucket, key, owner, allow_completed=True) + expected_completion_head = check_write_precondition( + request, catalog, bucket, key, + ) + content = bytearray() + async for block in request.stream(): + content.extend(block) + if len(content) > 1024 * 1024: + raise MultipartError("MalformedXML", "Complete XML exceeds 1 MiB") + if hashlib.sha256(content).hexdigest() != request.headers.get("x-amz-content-sha256"): + raise MultipartError("XAmzContentSHA256Mismatch", "Completion XML checksum mismatch") + if b" max_bytes: + raise MultipartError("EntityTooLarge", "Part exceeds configured upload maximum", 413) + sha.update(block) + md5.update(block) + sha1.update(block) + crc = zlib.crc32(block, crc) + crc32c_value = crc32c(block, crc32c_value) + crc64_value = crc64nvme(block, crc64_value) + output.write(block) + output.flush() + os.fsync(output.fileno()) + _verify_upload_sha256( + request, sha.digest(), md5_digest=md5.digest(), + sha1_digest=sha1.digest(), crc64nvme_value=crc64_value, + ) + checksum_crc = request.headers.get("x-amz-checksum-crc32") + if checksum_crc and checksum_crc != base64.b64encode( + (crc & 0xffffffff).to_bytes(4, "big") + ).decode(): + raise MultipartError("BadDigest", "CRC32 checksum mismatch") + declared_crc32c = request.headers.get("x-amz-checksum-crc32c") + if declared_crc32c is not None and not hmac.compare_digest( + declared_crc32c, + base64.b64encode(crc32c_value.to_bytes(4, "big")).decode("ascii"), + ): + raise MultipartError("BadDigest", "CRC32C checksum mismatch") + transport = storage_transport(db) + await transport.verify_target() + engine = TelegramObjectStorage( + catalog.manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, scratch_directory=scratch, + ) + staged = await engine.put_file(bucket, key, stage) + tag = md5.hexdigest() + mpu.record_part(upload_id, bucket, key, owner, + number=number, object_id=staged.id, etag=tag) + return Response(status_code=200, headers={"ETag": '"' + tag + '"'}) + finally: + stage.unlink(missing_ok=True) + + if is_version_delete: + if any(h in request.headers for h in ("if-match", "if-none-match")): + return _error("NotImplemented", "Conditional VersionId delete unsupported", 501) + version_id = request.query_params["versionId"] + if not version_id or len(version_id) > 80: + return _error("InvalidArgument", "Invalid version ID", 400) + is_marker = db.connection.execute(select(objects.c.status).where( + objects.c.id == version_id, + objects.c.bucket == bucket, + objects.c.object_key == key, + )).scalar_one_or_none() == "delete-marker" + if not catalog.delete_version(bucket, key, version_id): + return _error("NoSuchVersion", "Version does not exist", 404) + headers = {"x-amz-version-id": version_id} + if is_marker: + headers["x-amz-delete-marker"] = "true" + return Response(status_code=204, headers=headers) + + expected_head = None + if request.method in ("PUT", "DELETE") and not request.headers.get("x-amz-copy-source"): + expected_head = check_write_precondition( + request, catalog, bucket, key, delete=request.method == "DELETE" + ) + + if request.method == "PUT" and request.headers.get("x-amz-copy-source") is not None: + # The source path must be signed. Otherwise a valid PUT signature + # could be replayed to copy a different private source object. + if "X-Amz-Algorithm" in request.query_params: + raise SigV4Error("NotImplemented", "Presigned CopyObject unsupported") + authorization = request.headers.get("authorization", "") + signed_headers = authorization.partition("SignedHeaders=")[2].split(",", 1)[0].split(";") + source_header = request.headers["x-amz-copy-source"] + if ("x-amz-copy-source" not in signed_headers + or store.resolve(owner, "s3:GetObject") is None): + raise SigV4Error("AccessDenied", "Signed source and GetObject scope required") + qualifiers = ( + "x-amz-copy-source-if-match", "x-amz-copy-source-if-none-match", + ) + for condition in qualifiers: + if request.headers.get(condition) is not None and condition not in signed_headers: + raise SigV4Error("AccessDenied", "Copy precondition must be signed") + unsupported_headers = ( + "x-amz-copy-source-if-modified-since", + "x-amz-copy-source-if-unmodified-since", + "x-amz-copy-source-range", + "x-amz-server-side-encryption", "x-amz-storage-class", + ) + if any(request.headers.get(name) is not None for name in unsupported_headers): + return _error("NotImplemented", "CopyObject option is unsupported", 501) + if (request.headers.get("x-amz-metadata-directive") is not None + and "x-amz-metadata-directive" not in signed_headers): + raise SigV4Error( + "AccessDenied", "Copy metadata directive must be signed" + ) + metadata_directive = request.headers.get( + "x-amz-metadata-directive", "COPY" + ) + if metadata_directive not in {"COPY", "REPLACE"}: + raise ObjectMetadataError("InvalidArgument", "Invalid metadata directive") + copy_properties = parse_object_metadata( + request.headers, signed_header_names(request), + ) + if metadata_directive == "COPY" and copy_properties: + raise ObjectMetadataError( + "InvalidRequest", "Object metadata requires MetadataDirective=REPLACE" + ) + tag_directive = request.headers.get("x-amz-tagging-directive", "COPY") + if tag_directive not in {"COPY", "REPLACE"}: + raise TaggingError("InvalidArgument", "Invalid CopyObject tagging directive") + if tag_directive == "COPY" and inline_tags is not None: + raise TaggingError("InvalidRequest", "x-amz-tagging requires TaggingDirective=REPLACE") + if (request.headers.get("x-amz-tagging-directive") is not None + and "x-amz-tagging-directive" not in signed_headers): + raise SigV4Error("AccessDenied", "Copy tagging directive must be signed") + # Copying nonempty source tags is a separate protected read/write + # operation; GetObject/PutObject alone must not bypass tag scopes. + if tag_directive == "REPLACE": + if store.resolve(owner, "s3:PutObjectTagging") is None: + raise SigV4Error("AccessDenied", "Copy tag replacement needs PutObjectTagging") + + if (request.headers.get("x-amz-metadata-directive") is not None + and "x-amz-metadata-directive" not in signed_headers): + raise SigV4Error("AccessDenied", "Metadata directive must be signed") + # A signed CopySource may name a specific historical version. + # Never treat arbitrary query selectors or duplicate VersionIds as + # part of the object key; URI escapes are decoded only afterwards. + if source_header.startswith("//") or "://" in source_header: + return _error("InvalidArgument", "Copy source must be a local bucket/object path", 400) + original_header, divider, source_query = source_header.partition("?") + source_version_id = None + if divider: + try: + qualifiers = parse_qsl( + source_query, keep_blank_values=True, strict_parsing=True, + encoding="utf-8", errors="strict", max_num_fields=2, + ) + except (UnicodeError, ValueError): + return _error("InvalidArgument", "Malformed CopySource version selector", 400) + if len(qualifiers) != 1 or qualifiers[0][0] != "versionId": + return _error("InvalidArgument", "Only a single source VersionId is supported", 400) + source_version_id = qualifiers[0][1] + if not source_version_id or len(source_version_id) > 80: + return _error("InvalidArgument", "Invalid CopySource version ID", 400) + original_path = unquote( + original_header[1:] if original_header.startswith("/") else original_header + ) + src_bucket, slash, src_key = original_path.partition("/") + if not slash or not src_bucket or not src_key: + return _error("InvalidArgument", "Invalid CopySource bucket/object key", 400) + if src_bucket not in catalog.list_buckets(): + return _error("NoSuchBucket", "CopySource bucket does not exist", 404) + if source_version_id is not None: + if not catalog.is_visible_version(src_bucket, src_key, source_version_id): + return _error("NoSuchVersion", "CopySource version is not published", 404) + try: + source_item = catalog.manifests.get(source_version_id) + except FileNotFoundError: + return _error("NoSuchVersion", "CopySource version not readable", 404) + if source_item.bucket != src_bucket or source_item.key != src_key: + return _error("NoSuchVersion", "CopySource key mismatch", 404) + else: + try: + source_item = catalog.get(src_bucket, src_key).item + except FileNotFoundError: + return _error("NoSuchKey", "CopySource object does not exist", 404) + if tag_directive == "COPY" and ObjectTagStore(catalog).get(source_item.id): + if (store.resolve(owner, "s3:GetObjectTagging") is None + or store.resolve(owner, "s3:PutObjectTagging") is None): + raise SigV4Error("AccessDenied", "Copying source tags needs tag read/write scopes") + existing_tag = MultipartStore(catalog).object_etag(source_item.id, source_item.sha256) + if_match = request.headers.get("x-amz-copy-source-if-match") + if_none_match = request.headers.get("x-amz-copy-source-if-none-match") + quoted_tag = '"' + existing_tag + '"' + if (if_match is not None and if_match not in (quoted_tag, "*") + or if_none_match is not None and if_none_match in (quoted_tag, "*")): + return _error("PreconditionFailed", "CopySource ETag precondition failed", 412) + # CopyObject is a metadata-only operation and has no upload body. + # Consume streaming requests safely rather than trusting Content-Length. + async for block in request.stream(): + if block: + return _error("InvalidRequest", "CopyObject request must have no body", 400) + try: + result = copy_object( + catalog, src_bucket, src_key, bucket, key, + expected_source_id=source_item.id, + source_version_id=source_version_id, + replacement_tags=(inline_tags or []) if tag_directive == "REPLACE" else None, + replacement_metadata=copy_properties if metadata_directive == "REPLACE" else None, + ) + except CopyConflict: + return _error("OperationAborted", "Concurrent source/destination mutation", 409) + copied = ET.Element("CopyObjectResult", xmlns=NAMESPACE) + ET.SubElement(copied, "LastModified").text = time.strftime( + "%Y-%m-%dT%H:%M:%S.000Z", time.gmtime() + ) + ET.SubElement(copied, "ETag").text = '"' + result.etag + '"' + return Response(ET.tostring(copied, encoding="utf-8", xml_declaration=True), + media_type="application/xml", status_code=200) + + if request.method == "DELETE": + try: + marker = catalog.delete_marker( + bucket, key, + expected_generation=expected_head.generation if expected_head else None, + expected_object_id=expected_head.object_id if expected_head else None, + ) + except ValueError as exc: + if expected_head is not None: + raise WritePreconditionError( + "PreconditionFailed", "Object changed during conditional DELETE", 412 + ) from exc + raise + return Response(status_code=204, headers={ + "x-amz-delete-marker": "true", "x-amz-version-id": marker, + }) + + if request.method == "PUT": + root = Path(_webapp.STORAGE_SCRATCH_DIR) + _secure_mkdir(root) + max_bytes = _webapp.REST_MAX_UPLOAD_BYTES + transport = storage_transport(db) + await transport.verify_target() + fd, name = tempfile.mkstemp(prefix="s3-upload-", dir=root) + stage = Path(name) + digest = hashlib.sha256() + md5 = hashlib.md5(usedforsecurity=False) + sha1 = hashlib.sha1(usedforsecurity=False) + crc = 0 + crc32c_value = 0 + crc64_value = 0 + length = 0 + try: + with os.fdopen(fd, "wb") as dest: + async for block in request.stream(): + length += len(block) + if max_bytes < 1 or length > max_bytes: + return _error("EntityTooLarge", "Upload exceeds configured maximum", 413) + digest.update(block) + md5.update(block) + sha1.update(block) + crc = zlib.crc32(block, crc) + crc32c_value = crc32c(block, crc32c_value) + crc64_value = crc64nvme(block, crc64_value) + dest.write(block) + dest.flush() + os.fsync(dest.fileno()) + _verify_upload_sha256( + request, digest.digest(), md5_digest=md5.digest(), + sha1_digest=sha1.digest(), crc64nvme_value=crc64_value, + ) + checksum_crc = request.headers.get("x-amz-checksum-crc32") + if checksum_crc and checksum_crc != base64.b64encode((crc & 0xffffffff).to_bytes(4, "big")).decode(): + return _error("BadDigest", "CRC32 checksum mismatch", 400) + declared_crc32c = request.headers.get("x-amz-checksum-crc32c") + if declared_crc32c is not None and not hmac.compare_digest( + declared_crc32c, + base64.b64encode(crc32c_value.to_bytes(4, "big")).decode("ascii"), + ): + return _error("BadDigest", "CRC32C checksum mismatch", 400) + engine = TelegramObjectStorage( + catalog.manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, scratch_directory=root, + ) + # Metadata, tags and the head pointer share one SQL + # transaction. A failed metadata write cannot publish a + # partially described object. + http_properties = parse_object_metadata( + request.headers, signed_header_names(request), + ) + http_store = ObjectMetadataStore(catalog) + tag_store = ObjectTagStore(catalog) if inline_tags is not None else None + item = await engine.put_file(bucket, key, stage) + http_store.attach_uncommitted(item.id, http_properties) + if tag_store is not None: + tag_store.attach_uncommitted(item.id, inline_tags) + try: + catalog.publish( + item.id, now=int(time.time()), + expected_generation=expected_head.generation if expected_head else None, + expected_object_id=expected_head.object_id if expected_head else None, + ) + except ValueError as exc: + if expected_head is not None: + raise WritePreconditionError( + "PreconditionFailed", "Object changed during conditional PUT", 412 + ) from exc + raise + return Response(status_code=200, headers={ + "ETag": '"' + item.sha256 + '"', + "x-amz-version-id": item.id, + }) + finally: + stage.unlink(missing_ok=True) + + if is_version_read: + version_id = request.query_params["versionId"] + if not version_id or len(version_id) > 80: + return _error("InvalidArgument", "Invalid version ID", 400) + if not catalog.is_visible_version(bucket, key, version_id): + return _error("NoSuchVersion", "Object version is not published", 404) + status_row = db.connection.execute(select( + objects.c.status, objects.c.created_at + ).where(objects.c.id == version_id)).one_or_none() + if status_row is not None and status_row[0] == "delete-marker": + response = _error( + "MethodNotAllowed", "Requested version is a delete marker", 405, + ) + response.headers.update({ + "x-amz-delete-marker": "true", "x-amz-version-id": version_id, + "Last-Modified": http_last_modified(int(status_row[1])), + }) + return response + try: + item = catalog.manifests.get(version_id) + except FileNotFoundError: + return _error("NoSuchVersion", "Object version is not readable", 404) + if item.bucket != bucket or item.key != key: + return _error("NoSuchVersion", "Version ownership mismatch", 404) + else: + try: + item = catalog.get(bucket, key).item + except FileNotFoundError: + marker = catalog.current_marker(bucket, key) + if marker is not None: + result = _error("NoSuchKey", "Current version is a delete marker", 404) + result.headers["x-amz-delete-marker"] = "true" + result.headers["x-amz-version-id"] = marker + return result + return _error("NoSuchKey", "The specified key does not exist", 404) + item_etag = MultipartStore(catalog).object_etag(item.id, item.sha256) + if is_object_attributes: + raw = request.headers.get("x-amz-object-attributes", "") + signed = signed_header_names(request) + if "x-amz-object-attributes" not in signed: + raise SigV4Error("AccessDenied", "Object attributes header must be signed") + if len(request.headers.getlist("x-amz-object-attributes")) != 1: + return _error("InvalidArgument", "Duplicate object attributes header", 400) + requested = raw.split(",") + if (not requested or any(not name for name in requested) + or len(set(requested)) != len(requested)): + return _error("InvalidArgument", "Invalid object attributes list", 400) + permitted = {"ETag", "ObjectSize", "Checksum", "StorageClass", "ObjectParts"} + if any(name not in permitted for name in requested): + return _error("NotImplemented", "Requested object attributes are unsupported", 501) + + options = {} + for name, default, low, high in ( + ("x-amz-max-parts", 1000, 1, 1000), + ("x-amz-part-number-marker", 0, 0, MAX_PARTS), + ): + values = request.headers.getlist(name) + if values: + if name not in signed: + raise SigV4Error("AccessDenied", "Object parts pagination must be signed") + if (len(values) != 1 + or re.fullmatch(r"[0-9]{1,5}", values[0]) is None + or not low <= int(values[0]) <= high): + return _error("InvalidArgument", "Invalid object parts pagination", 400) + if "ObjectParts" not in requested: + return _error("InvalidArgument", "Part pagination requires ObjectParts", 400) + options[name] = int(values[0]) + else: + options[name] = default + result = ET.Element("GetObjectAttributesOutput", xmlns=NAMESPACE) + if "ETag" in requested: + ET.SubElement(result, "ETag").text = '"' + item_etag + '"' + if "ObjectSize" in requested: + ET.SubElement(result, "ObjectSize").text = str(item.size) + if "Checksum" in requested: + checksum = ET.SubElement(result, "Checksum") + # Immutable object SHA-256 covers the full plaintext bytes, + # independent of multipart ETag and physical Telegram chunks. + ET.SubElement(checksum, "ChecksumSHA256").text = base64.b64encode( + bytes.fromhex(item.sha256) + ).decode("ascii") + ET.SubElement(checksum, "ChecksumType").text = "FULL_OBJECT" + if "StorageClass" in requested: + # TeleDrive exposes one immutable STANDARD-like tier; archived + # or replica-backed AWS storage classes do not exist here. + ET.SubElement(result, "StorageClass").text = "STANDARD" + if "ObjectParts" in requested: + layout = MultipartStore(catalog).object_parts_for_version(item.id, item.size) + # AWS omits ObjectParts for a plain single PUT. This must not + # fabricate logical parts out of Telegram's internal chunks. + if layout is not None: + marker = options["x-amz-part-number-marker"] + limit = options["x-amz-max-parts"] + remaining = [part for part in layout if part[0] > marker] + page = remaining[:limit] + listing = ET.SubElement(result, "ObjectParts") + for name, value in ( + ("PartsCount", len(layout)), + ("PartNumberMarker", marker), + ("MaxParts", limit), + ("IsTruncated", str(len(remaining) > len(page)).lower()), + ): + ET.SubElement(listing, name).text = str(value) + if len(remaining) > len(page): + ET.SubElement(listing, "NextPartNumberMarker").text = str(page[-1][0]) + for number, size, digest, _etag in page: + part_node = ET.SubElement(listing, "Part") + ET.SubElement(part_node, "PartNumber").text = str(number) + ET.SubElement(part_node, "Size").text = str(size) + ET.SubElement(part_node, "ChecksumSHA256").text = base64.b64encode( + bytes.fromhex(digest) + ).decode("ascii") + return Response( + ET.tostring(result, encoding="utf-8", xml_declaration=True), + media_type="application/xml", status_code=200, + headers={ + "x-amz-version-id": item.id, + "Last-Modified": http_last_modified(item.created_at), + }, + ) + http_properties = ObjectMetadataStore(catalog).get(item.id) + precondition = read_precondition(request, item_etag, item.created_at) + if precondition == 304: + return Response(status_code=304, headers={ + "ETag": '"' + item_etag + '"', + "Last-Modified": http_last_modified(item.created_at), + }) + if precondition == 412: + return _error("PreconditionFailed", "Object precondition failed", 412) + if request.method == "HEAD": + return Response(status_code=200, headers=_headers( + item, item_etag, http_properties, + checksum_mode=(checksum_mode == "ENABLED"), + )) + start, end, partial = _range(request.headers.get("range"), item.size) + transport = storage_transport(db) + scratch = Path(_webapp.STORAGE_SCRATCH_DIR) + _secure_mkdir(scratch) + engine = TelegramObjectStorage( + catalog.manifests, transport, + chunk_size=RECOMMENDED_CHUNK_SIZE, scratch_directory=scratch, + ) + + async def stream(): + try: + async for piece in engine.iter_bytes(item.id, start=start, end=end): + yield piece + finally: + db.close() + + retain = True + headers = _headers(item, item_etag, http_properties, + checksum_mode=(checksum_mode == "ENABLED")) + headers["Content-Length"] = str(end - start) + if partial: + headers["Content-Range"] = f"bytes {start}-{end-1}/{item.size}" + return StreamingResponse(stream(), status_code=206 if partial else 200, headers=headers) + except WritePreconditionError as exc: + response = _error(exc.code, str(exc), exc.status) + if request.method == "PUT": + # Conditional writes can fail before their request body is read. + # Never reuse an HTTP/1.1 connection with an unread PUT payload: + # boto3 otherwise receives an invalid next-request 400 from h11. + response.headers["Connection"] = "close" + return response + except BatchDeleteError as exc: + return _error(exc.code, str(exc), exc.status) + except S3QueryError as exc: + return _error(exc.code, str(exc), exc.status) + except BucketListingError as exc: + return _error(exc.code, str(exc), exc.status) + except LifecycleError as exc: + return _error(exc.code, str(exc), exc.status) + except PublicAccessError as exc: + return _error(exc.code, str(exc), exc.status) + except VersioningError as exc: + return _error(exc.code, str(exc), exc.status) + except CORSError as exc: + return _error(exc.code, str(exc), exc.status) + except TaggingError as exc: + return _error(exc.code, str(exc), exc.status) + except ObjectMetadataError as exc: + return _error(exc.code, str(exc), exc.status) + except MultipartError as exc: + return _error(exc.code, str(exc), exc.status) + except SigV4Error as exc: + return _error(exc.code, exc.message, { + "RequestTimeTooSkewed": 403, "NotImplemented": 501, + "InvalidArgument": 400, "InvalidRequest": 400, + "EntityTooLarge": 413, + "XAmzContentSHA256Mismatch": 400, "BadDigest": 400, + "InvalidRange": 416, + }.get(exc.code, 403)) + except FileNotFoundError: + return _error("NoSuchKey", "The requested resource does not exist", 404) + except ValueError: + return _error("InvalidArgument", "Invalid request parameters", 400) + except RuntimeError: + return _error("ServiceUnavailable", "S3 storage backend is unavailable", 503) + finally: + if db is not None and not retain: + db.close() + + +@app.middleware("http") +async def _route_encoded_control_keys(request: Request, call_next): + """Preserve S3 opaque keys and enforce bucket CORS independently of SigV4. + + A browser OPTIONS preflight has no AWS signature but cannot read/write + anything. Actual S3 methods still require the normal signed authorization. + """ + path = request.scope.get("path", "") + origin = request.headers.get("origin", "") + wants_cors = bool(origin and path != "/__health" and path.startswith("/")) + if wants_cors: + logical = unquote((request.scope.get("raw_path") or b"/").decode("ascii").lstrip("/")) + try: + virtual_bucket = _virtual_host_bucket(request) + except SigV4Error as exc: + return _error(exc.code, exc.message, 400) + bucket = virtual_bucket if virtual_bucket is not None else logical.split("/", 1)[0] + if bucket: + connection = None + try: + connection = db_connect() + catalog = BucketCatalog(ManifestStore(connection)) + if bucket in catalog.list_buckets(): + rules = BucketCORSStore(catalog).get(bucket) or [] + else: + rules = [] + is_preflight = request.method == "OPTIONS" + cors_method = (request.headers.get("access-control-request-method", "") + if is_preflight else request.method) + headers_string = (request.headers.get("access-control-request-headers", "") + if is_preflight else "") + requested = ([h.strip() for h in headers_string.split(",") if h.strip()] + if headers_string else []) + # Cross-origin browser clients may send many headers; bound + # their CORS preflight processing without trusting the Origin. + if (len(headers_string) > 2048 or len(requested) > 32 + or any(not h.isascii() for h in requested)): + return _error("AccessForbidden", "CORS preflight rejected", 403) + matched = matching_rule(rules, origin, cors_method, requested) + if is_preflight: + if not matched: + return _error("AccessForbidden", "CORS preflight rejected", 403) + return Response(status_code=200, headers=response_cors_headers( + matched, origin, preflight=True, requested_headers=requested + )) + except Exception: + if request.method == "OPTIONS": + return _error("ServiceUnavailable", "CORS lookup unavailable", 503) + raise + finally: + if connection is not None: + connection.close() + # Do not bypass signature validation for actual HTTP methods. + if path != "/__health" and any(ord(ch) < 32 for ch in path): + response = await _handle(request) + else: + response = await call_next(request) + if matched: + for name, value in response_cors_headers( + matched, origin, preflight=False + ).items(): + response.headers[name] = value + return response + if request.method == "OPTIONS": + return _error("AccessForbidden", "CORS preflight rejected", 403) + if path != "/__health" and any(ord(character) < 32 for character in path): + return await _handle(request) + return await call_next(request) + + +@app.get("/__health", include_in_schema=False) +def health(): + # Container-local liveness only; no credentials or Telegram calls. + return {"ok": True} + + +@app.api_route("/", methods=["GET", "PUT", "POST", "DELETE", "HEAD"]) +async def root(request: Request): + return await _handle(request) + + +@app.api_route("/{path:path}", methods=["GET", "PUT", "POST", "DELETE", "HEAD"]) +async def path_route(request: Request, path: str): + return await _handle(request) diff --git a/web/static/app.js b/web/static/app.js new file mode 100644 index 00000000..511f9500 --- /dev/null +++ b/web/static/app.js @@ -0,0 +1,371 @@ +const $=q=>document.querySelector(q), $$=q=>[...document.querySelectorAll(q)]; +let config=null, authPrompt=null; +async function api(url,opt={}){const r=await fetch(url,{headers:{'Content-Type':'application/json','X-TeleDrive-CSRF':'1',...(opt.headers||{})},...opt});if(!r.ok)throw new Error(await r.text()||r.statusText);return r.json()} +function toast(msg,bad=false){const e=$('#toast');e.textContent=msg;e.className='toast'+(bad?' bad':'');e.style.display='block';setTimeout(()=>e.style.display='none',3500)} +function fmtTs(v){return v?new Date(v*1000).toLocaleString():'-'} +function esc(v){return String(v??'').replace(/[&<>"']/g,c=>({'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]))} + +$$('.tab').forEach(b=>b.onclick=()=>{$$('.tab').forEach(x=>x.classList.remove('active'));$$('.page').forEach(x=>x.classList.remove('active'));b.classList.add('active');$('#'+b.dataset.page).classList.add('active');if(b.dataset.page==='queue')loadFiles('queue');if(b.dataset.page==='history')loadFiles('history');if(b.dataset.page==='settings')loadSettings();if(b.dataset.page==='api-management'){loadKeys();loadS3Keys();loadPendingRecovery();loadStorageJobs();}if(b.dataset.page==='logs')loadLogs()}); + +async function refreshDashboard(){ +try{ +const d=await api('/api/dashboard');const q=d.queue||{}; +$('#mPending').textContent=q.pending||0;$('#mFailed').textContent=q.failed||0;$('#mUploaded').textContent=q.uploaded||0;$('#mToday').textContent=q.uploaded_today||0; +$('#strategyView').textContent=d.strategy||'-';$('#dbView').textContent=d.database_url||'-'; +const p=d.process||{};$('#processView').textContent=p.running?('running · '+(p.mode||'')):'idle';$('#subtitle').textContent=(d.strategy||'single')+' · '+(p.running?'running':'idle'); +authPrompt=p.auth_prompt||null;const ap=$('#authPrompt');if(authPrompt){ap.classList.add('show');$('#authTitle').textContent=authPrompt.account+' is waiting for Telegram '+authPrompt.kind;$('#authValue').type=authPrompt.kind==='password'?'password':'text'}else{ap.classList.remove('show');$('#authValue').value=''} +$('#accounts').innerHTML=(d.accounts||[]).map(a=>'
'+esc(a.name)+''+esc(a.state)+'
enabled: '+a.enabled+' · uploaded: '+(a.uploaded_count||0)+'
target: '+esc(a.target)+' · last success: '+fmtTs(a.last_success_ts)+'
'+(a.last_error?'
'+esc(a.last_error)+'
':'')+'
').join(''); +}catch(e){toast(e.message,true)} +} +async function loadFiles(kind){try{const rows=await api('/api/files?kind='+kind+'&limit=200');if(kind==='queue')$('#queueRows').innerHTML=rows.map(r=>''+esc(r.path)+''+esc(r.status)+''+r.attempts+''+esc(r.tg_account||'-')+''+esc(r.last_error||'')+'').join('');else $('#historyRows').innerHTML=rows.map(r=>''+esc(r.path)+''+esc(r.tg_account||'-')+''+esc(r.tg_message_id||'-')+''+fmtTs(r.uploaded_ts)+'').join('')}catch(e){toast(e.message,true)}} +async function loadLogs(){try{const x=await api('/api/logs?limit=500');$('#logBox').textContent=(x.lines||[]).join('\n');$('#logBox').scrollTop=$('#logBox').scrollHeight}catch(e){toast(e.message,true)}} + +function accountEditor(a={}){return ''} +function wireEditors(){$$('.ae-remove').forEach(b=>b.onclick=()=>b.closest('.account-edit').remove())} +$('#addAccount').onclick=()=>{$('#accountEditors').insertAdjacentHTML('beforeend',accountEditor({name:'account'+($$('.account-edit').length+1)}));wireEditors()} + +async function loadSettings(){try{ +config=await api('/api/config');const tg=config.telegram||{},up=config.upload||{},st=config.state||{},app=config.app||{}; +const storage=config.storage||{},jobs=config.jobs||{}; +$('#storageChannel').value=storage.channel||'';$('#storageStrategy').value=storage.strategy||'round_robin';$('#storageScratch').value=storage.scratch_dir||'/data/tmp'; +$('#apiEditors').innerHTML=(storage.apis||[]).map(apiEditor).join(''); +wireApiEditors() + +$('#strategy').value=tg.strategy||'single';$('#apiId').value=tg.api_id||'';$('#apiHash').value=tg.api_hash||'';$('#tgTarget').value=tg.target??'me'; +$('#sourceDir').value=up.source_dir||'/uploads';$('#sendMode').value=up.send_mode||'document';$('#perRun').value=up.max_files_per_run??12;$('#perDay').value=up.max_files_per_day??100;$('#sleepMin').value=up.sleep_min_seconds??60;$('#sleepMax').value=up.sleep_max_seconds??90;$('#retryAttempts').value=up.retry_attempts??5;$('#backoff').value=up.backoff_base_seconds??10;$('#floodBuffer').value=up.floodwait_buffer_seconds??5;$('#maxFileSize').value=up.max_file_size_mb??0;$('#captionTemplate').value=up.caption_template||'{name}';$('#extensions').value=(up.allowed_extensions||[]).join(','); +$('#databaseUrl').value=st.database_url||st.db_path||'sqlite:////data/state.db';$('#autoRun').checked=!!app.auto_run;$('#runInterval').value=app.run_interval_minutes??60; +$('#accountEditors').innerHTML=(tg.accounts||[]).map(accountEditor).join('');wireEditors() +}catch(e){toast(e.message,true)}} + +function apiEditor(entry={}){ +const e=document.createElement('div');e.className='account-edit api-edit'; +e.innerHTML=`
+
+
+
+
+
`; +e.querySelector('.ae-remove').onclick=()=>{e.remove()}; +return e} +function wireApiEditors(){ +$('#addApi').onclick=()=>{const wrap=document.createElement('div');wrap.className='api-edit';wrap.innerHTML='
';wrap.querySelector('.ae-remove').onclick=()=>wrap.remove();$('#apiEditors').appendChild(wrap)}; +$$('#apiEditors .ae-remove').forEach(b=>b.onclick=()=>b.closest('.api-edit').remove())} + +$('#saveSettings').onclick=async()=>{try{ +if(!config)await loadSettings();const tg=config.telegram=config.telegram||{},up=config.upload=config.upload||{},st=config.state=config.state||{},app=config.app=config.app||{},storage=config.storage=config.storage||{}; +storage.channel=$('#storageChannel').value.trim();storage.strategy=$('#storageStrategy').value;storage.scratch_dir=$('#storageScratch').value.trim()||'/data/tmp'; +storage.apis=$$('#apiEditors .api-edit').map(e=>{const o={api_id:Number(e.querySelector('.ae-api-id').value||0),api_hash:e.querySelector('.ae-api-hash').value.trim(),session_file:e.querySelector('.ae-session').value.trim()};const name=e.querySelector('.ae-name').value.trim();if(name)o.account=name;return o}).filter(o=>o.api_id&&o.api_hash&&o.session_file); +tg.strategy=$('#strategy').value;tg.api_id=Number($('#apiId').value||0);tg.api_hash=$('#apiHash').value.trim();tg.target=$('#tgTarget').value.trim()||'me'; +tg.accounts=$$('.account-edit').map(e=>{const o={name:e.querySelector('.ae-name').value.trim(),enabled:e.querySelector('.ae-enabled').checked,phone:e.querySelector('.ae-phone').value.trim(),session_path:e.querySelector('.ae-session').value.trim()};const target=e.querySelector('.ae-target').value.trim(),id=e.querySelector('.ae-api-id').value.trim(),hash=e.querySelector('.ae-api-hash').value.trim();if(target)o.target=target;if(id)o.api_id=Number(id);if(hash)o.api_hash=hash;return o}); +up.source_dir=$('#sourceDir').value.trim();up.send_mode=$('#sendMode').value;up.max_files_per_run=Number($('#perRun').value);up.max_files_per_day=Number($('#perDay').value);up.sleep_min_seconds=Number($('#sleepMin').value);up.sleep_max_seconds=Number($('#sleepMax').value);up.retry_attempts=Number($('#retryAttempts').value);up.backoff_base_seconds=Number($('#backoff').value);up.floodwait_buffer_seconds=Number($('#floodBuffer').value);up.max_file_size_mb=Number($('#maxFileSize').value);up.caption_template=$('#captionTemplate').value;up.allowed_extensions=$('#extensions').value.split(',').map(x=>x.trim()).filter(Boolean); +st.database_url=$('#databaseUrl').value.trim()||'sqlite:////data/state.db';delete st.db_path;app.auto_run=$('#autoRun').checked;app.run_interval_minutes=Number($('#runInterval').value); +await api('/api/config',{method:'PUT',body:JSON.stringify(config)});toast('Settings saved');await refreshDashboard() +}catch(e){toast(e.message,true)}}; + +$('#runBtn').onclick=()=>api('/api/process/start',{method:'POST',body:JSON.stringify({mode:'run-once'})}).then(refreshDashboard).catch(e=>toast(e.message,true)); +$('#scanBtn').onclick=()=>api('/api/process/start',{method:'POST',body:JSON.stringify({mode:'scan-only'})}).then(refreshDashboard).catch(e=>toast(e.message,true)); +$('#stopBtn').onclick=()=>api('/api/process/stop',{method:'POST'}).then(refreshDashboard).catch(e=>toast(e.message,true)); +$('#authSubmit').onclick=async()=>{if(!authPrompt)return;try{await api('/api/auth',{method:'POST',body:JSON.stringify({account:authPrompt.account,kind:authPrompt.kind,value:$('#authValue').value})});$('#authValue').value='';toast('Auth value submitted')}catch(e){toast(e.message,true)}}; + +async function loadKeys(){ + try { + const result=await api('/api/v1/admin/keys'); + const rows=(result.keys||[]).map(k=>{ + const state=k.revoked?'Revoked':(k.expires_at&&k.expires_at<=Date.now()/1000?'Expired':'Active'); + return ''+esc(k.name)+''+esc(k.prefix)+'…'+esc((k.scopes||[]).join(', '))+''+fmtTs(k.created_at)+'
'+fmtTs(k.last_used_at)+''+state+'
'; + }).join(''); + $('#apiKeysRows').innerHTML=rows; + $$('.key-revoke').forEach(b=>b.onclick=async()=>{ + if(!confirm('Revoke this API key immediately? Existing clients will lose access.'))return; + try{await api('/api/v1/admin/keys/'+encodeURIComponent(b.dataset.id),{method:'DELETE'});toast('Key revoked');await loadKeys()}catch(e){toast(e.message,true)} + }); + $$('.key-rotate').forEach(b=>b.onclick=async()=>{ + if(!confirm('Rotate this key now? The old token will stop working immediately.'))return; + try{const result=await api('/api/v1/admin/keys/'+encodeURIComponent(b.dataset.id)+'/rotate',{method:'POST'});revealKey(result.token);toast('Key rotated');await loadKeys()}catch(e){toast(e.message,true)} + }); + }catch(e){toast(e.message,true)} +} + +function revealKey(value){ + $('#createdKeySecret').value=value; + $('#createdKeyCard').style.display='block'; +} +$('#createKey').onclick=async()=>{ + const scopes=$('#keyScopes input[type=checkbox]:checked').map(e=>e.value); + const expiry=$('#keyExpiry').value; + try{ + const response=await api('/api/v1/admin/keys',{method:'POST',body:JSON.stringify({ + name:$('#keyName').value.trim(),scopes, + expires_at:expiry?Math.floor(new Date(expiry).getTime()/1000):null + })}); + revealKey(response.token); + $('#keyName').value=''; + toast('Key created — copy the secret now'); + await loadKeys(); + }catch(e){toast(e.message,true)} +}; +$('#refreshKeys').onclick=loadKeys; +$('#dismissKey').onclick=()=>{$('#createdKeySecret').value='';$('#createdKeyCard').style.display='none'}; +$('#copyKey').onclick=async()=>{ + try{await navigator.clipboard.writeText($('#createdKeySecret').value);toast('Key copied')}catch(e){toast('Clipboard access denied; select and copy the key manually',true)} +}; +$('#keyScopes input[type=checkbox]').forEach(field=>field.onchange=()=>{ + const admin=$('#keyScopes input[value="admin:*"]'); + if(field===admin&&admin.checked){$('#keyScopes input[type=checkbox]').forEach(x=>{if(x!==admin)x.checked=false})} + else if(field.checked){admin.checked=false} +}); + + +async function loadS3Keys(){ + try { + const data=await api('/api/v1/admin/s3-credentials'); + $('#s3KeysRows').innerHTML=(data.credentials||[]).map(k=>{ + const state=k.revoked?'Revoked':(k.expires_at&&k.expires_at<=Date.now()/1000?'Expired':'Active'); + return ''+esc(k.name)+''+esc(k.access_key)+''+esc((k.permissions||[]).join(', '))+''+fmtTs(k.created_at)+''+state+''; + }).join(''); + $('.s3-revoke').forEach(b=>b.onclick=async()=>{ + if(!confirm('Revoke this S3 credential immediately? Connected clients will be denied.'))return; + try{await api('/api/v1/admin/s3-credentials/'+encodeURIComponent(b.dataset.id),{method:'DELETE'});toast('S3 key revoked');await loadS3Keys()}catch(e){toast(e.message,true)} + }); + }catch(e){$('#s3KeysRows').textContent='S3 key storage unavailable: '+e.message} +} +async function loadS3Recovery(){ + const target=$('#s3RecoveryRows'); + try { + const response=await api('/api/v1/admin/s3-multipart/stale?min_age_seconds=3600'); + const uploads=response.uploads||[]; + target.innerHTML=uploads.length?uploads.map(u=> + ''+esc(u.bucket)+''+esc(u.object_key)+'' + +''+esc(u.id)+''+fmtTs(u.lease_until)+'' + +'' + ).join(''):'No eligible stale completions'; + $('.s3-recover').forEach(button=>button.onclick=async()=>{ + const uploadId=button.dataset.id; + const url='/api/v1/admin/s3-multipart/'+encodeURIComponent(uploadId)+'/recover'; + button.disabled=true; + try { + const preview=await api(url,{method:'POST',body:JSON.stringify({execute:false,min_age_seconds:3600})}); + if(!preview.eligible){toast('Upload no longer eligible for recovery',true);await loadS3Recovery();return} + if(!confirm('Recover this expired multipart completion? No Telegram messages will be deleted. The S3 client must retry Complete explicitly.'))return; + const done=await api(url,{method:'POST',body:JSON.stringify({execute:true,min_age_seconds:3600})}); + toast(done.recovered?'Upload safely returned to active':'Recovery eligibility changed; no action taken',!done.recovered); + await loadS3Recovery(); + }catch(e){toast(e.message,true)} + finally{button.disabled=false} + }); + }catch(e){target.textContent='Cannot scan stale multipart uploads: '+e.message} +} +$('#refreshS3Recovery').onclick=loadS3Recovery; + +async function loadPendingRecovery(){ + const target=$('#pendingRecoveryRows'); + try{ + const response=await api('/api/v1/admin/storage/pending-stale?min_age_seconds=3600'); + const uploads=response.uploads||[]; + target.innerHTML=uploads.length?uploads.map(u=> + ''+esc(u.bucket)+''+esc(u.key)+'' + +''+esc(u.id)+''+Number(u.journaled_chunks||0)+'' + +''+fmtTs(u.lease_until)+'' + +'' + ).join(''):'No eligible stale PENDING uploads'; + $('.pending-abandon').forEach(button=>button.onclick=async()=>{ + const objectId=button.dataset.id; + const url='/api/v1/admin/storage/pending-stale/'+encodeURIComponent(objectId)+'/abandon'; + button.disabled=true; + try{ + const preview=await api(url,{method:'POST',body:JSON.stringify({execute:false,min_age_seconds:3600})}); + if(!preview.eligible){toast('Writer heartbeat or state changed; no action taken',true);await loadPendingRecovery();return} + if(!confirm('Fence only this expired upload and mark its metadata FAILED? Telegram messages will NOT be deleted. It may need subsequent reconciliation.'))return; + const executed=await api(url,{method:'POST',body:JSON.stringify({execute:true,min_age_seconds:3600})}); + toast(executed.abandoned?'Old writer fenced. Telegram journal preserved.':'Writer state changed; no action taken',!executed.abandoned); + await loadPendingRecovery(); + }catch(e){toast(e.message,true)} + finally{button.disabled=false} + }); + }catch(e){target.textContent='Cannot inspect PENDING uploads: '+e.message} +} +$('#refreshPendingRecovery').onclick=loadPendingRecovery; + +async function loadObjectVersions(){ + const bucket=$('#historyVersionBucket').value.trim(); + const key=$('#historyVersionKey').value; + const target=$('#objectVersionsRows'); + const summary=$('#objectVersionGeneration'); + if(!bucket||!key){toast('Bucket and key are required',true);return} + try{ + const d=await api('/api/v1/admin/storage/versions?bucket='+ + encodeURIComponent(bucket)+'&key='+encodeURIComponent(key)); + summary.textContent='Current generation: '+d.generation; + const versions=d.versions||[]; + target.innerHTML=versions.length?versions.map(v=> + ''+esc(v.version_id)+''+(v.is_current?' current':'')+'' + +''+Number(v.size)+''+esc(v.sha256)+'' + +''+fmtTs(Math.floor(v.committed_ns/1000000000))+'' + +'' + ).join(''):'No indexed published versions for this key'; + $('.object-version-restore').forEach(button=>button.onclick=async()=>{ + if(!confirm('Restore this historical version as a new immutable HEAD? Telegram media is referenced, never reuploaded or deleted.'))return; + button.disabled=true; + try{ + const result=await api('/api/v1/admin/storage/versions/'+ + encodeURIComponent(button.dataset.id)+'/restore',{ + method:'POST',body:JSON.stringify({ + bucket,key,expected_generation:d.generation + }), + }); + toast('Version restored: '+result.id); + await loadObjectVersions(); + }catch(e){toast(e.message,true)} + finally{button.disabled=false} + }); + }catch(e){target.textContent='History unavailable: '+e.message;summary.textContent=''} +} +$('#loadObjectVersions').onclick=loadObjectVersions; +$('#indexCurrentVersions').onclick=async()=>{ + const endpoint='/api/v1/admin/storage/versions/index-current'; + try{ + const preview=await api(endpoint,{method:'POST',body:JSON.stringify({execute:false,limit:100})}); + if(!preview.count){toast('No existing published HEADs need indexing');return} + if(!confirm('Index '+preview.count+' pre-upgrade current HEAD(s) as known published versions? Unpublished old manifests will stay hidden; Telegram media is not touched.'))return; + const done=await api(endpoint,{method:'POST',body:JSON.stringify({execute:true,limit:100})}); + toast('Indexed '+done.count+' existing HEAD(s)'); + if($('#historyVersionBucket').value&&$('#historyVersionKey').value)await loadObjectVersions(); + }catch(e){toast(e.message,true)} +}; + +async function loadStorageJobs(){ + const target=$('#storageJobsRows'); + try{ + const result=await api('/api/v1/admin/storage/jobs?limit=100'); + const items=result.jobs||[]; + target.innerHTML=items.length?items.map(j=> + ''+esc(j.bucket)+' / '+esc(j.object_key)+'' + +''+esc(j.source_path)+'' + +''+esc(j.state)+''+Number(j.attempts||0)+'' + +''+fmtTs(j.updated_at)+'' + +''+(j.state==='failed'?' ':'' )+(j.state==='queued'||j.state==='failed'?'':(j.last_error?esc(j.last_error):'-'))+'' + ).join(''):'No upload jobs'; + $('.storage-job-retry').forEach(button=>button.onclick=async()=>{ + if(!confirm('Requeue this failed job? The original source must still be unchanged. Existing Telegram media will not be deleted.'))return; + try{ + await api('/api/v1/admin/storage/jobs/'+encodeURIComponent(button.dataset.id)+'/retry',{method:'POST'}); + toast('Job queued for retry');await loadStorageJobs(); + }catch(e){toast(e.message,true)} + }); + $('.storage-job-cancel').forEach(button=>button.onclick=async()=>{ + if(!confirm('Cancel this idle upload job? No Telegram message will be deleted.'))return; + try{ + await api('/api/v1/admin/storage/jobs/'+encodeURIComponent(button.dataset.id)+'/cancel',{method:'POST'}); + toast('Idle job cancelled');await loadStorageJobs(); + }catch(e){toast(e.message,true)} + }); + }catch(e){target.textContent='Cannot list upload jobs: '+e.message} +} +$('#refreshStorageJobs').onclick=loadStorageJobs; +$('#refreshStorageHealth').onclick=async()=>{ + const target=$('#storageHealthSummary'); + try{ + const h=await api('/api/v1/admin/storage/health'); + target.textContent=[ + 'Job states: '+JSON.stringify(h.job_states), + 'Unresolved alerts: '+h.unresolved_alert_count, + ...(h.alerts||[]).map(a=>'['+a.severity.toUpperCase()+'] '+a.code+': '+a.count+' — '+a.message), + 'Stale PENDING uploads: '+h.stale_pending_sample_count, + 'Stale multipart completions: '+h.stale_multipart_sample_count, + 'Telegram messages retained: '+h.telegram_messages_retained, + 'Live object heads: '+h.live_object_heads, + 'Copy aliases: '+h.copy_aliases, + 'Physical GC enabled: '+(h.physical_deletion_enabled?'YES':'no'), + 'Samples truncated: '+(h.samples_truncated?'yes':'no'), + ].join('\n'); + }catch(e){target.textContent='Health check failed: '+e.message} +}; +$('#queueStorageJob').onclick=async()=>{ + const bucket=$('#storageJobBucket').value.trim(); + const key=$('#storageJobKey').value; + const source_path=$('#storageJobSource').value.trim(); + if(!bucket||!key||!source_path){toast('Bucket, key and source path are required',true);return} + try{ + const result=await api('/api/v1/admin/storage/jobs',{ + method:'POST',body:JSON.stringify({bucket,key,source_path}), + }); + toast('Local file queued: '+result.id); + await loadStorageJobs(); + }catch(e){toast(e.message,true)} +}; +async function scanLegacyFolder(execute){ + const bucket=$('#legacyImportBucket').value.trim(); + const prefix=$('#legacyImportPrefix').value.trim(); + const max_files=Number($('#legacyImportLimit').value); + if(!bucket||!Number.isInteger(max_files)||max_files<1||max_files>1000){ + toast('Valid bucket and 1..1000 file limit required',true);return + } + if(execute&&!confirm('Queue eligible legacy files into durable storage jobs? No Telegram upload starts now. Ensure the legacy uploader is stopped.'))return; + const target=$('#legacyImportReport'); + try{ + const report=await api('/api/v1/admin/storage/jobs/import-legacy',{ + method:'POST',body:JSON.stringify({bucket,prefix,max_files,execute}), + }); + target.textContent=JSON.stringify(report,null,2); + if(execute){toast('Queued '+report.queued+' durable jobs');await loadStorageJobs()} + }catch(e){target.textContent='Legacy import failed: '+e.message;toast(e.message,true)} +} +$('#previewLegacyImport').onclick=()=>scanLegacyFolder(false); +$('#executeLegacyImport').onclick=()=>scanLegacyFolder(true); +$('#runStorageJob').onclick=async()=>{ + if(!confirm('Process the next upload job and publish its Telegram object if the generation precondition still matches?'))return; + const button=$('#runStorageJob'); + button.disabled=true; + try{ + const result=await api('/api/v1/admin/storage/jobs/run-next',{method:'POST'}); + toast(result.job?'Upload job '+result.job.state:'No queued upload jobs'); + await loadStorageJobs(); + }catch(e){toast(e.message,true);await loadStorageJobs()} + finally{button.disabled=false} +}; + +$('#runGCAudit').onclick=async()=>{ + const target=$('#gcAuditSummary'); + target.textContent='Auditing references…'; + try{ + const d=await api('/api/v1/admin/storage/gc-audit?limit=100'); + target.textContent=[ + 'Telegram messages: '+d.total_telegram_messages, + 'Stored manifests: '+d.total_manifests, + 'Current object heads: '+d.live_object_heads, + 'CopyObject aliases: '+d.copy_aliases, + 'Multipart part references: '+d.multipart_parts, + 'Deletion-eligible: '+d.eligible_for_deletion, + 'Protected chunk owners shown: '+d.protected_chunks.length+(d.truncated?' (list truncated)':''), + 'Physical deletion disabled: '+(!d.physical_deletion_enabled?'yes':'NO') + ].join('\n'); + }catch(e){target.textContent='Audit failed: '+e.message;toast(e.message,true)} +}; +function hideS3Secret(){ + $('#s3NewSecret').value='';$('#s3NewAccess').value='';$('#createdS3Card').style.display='none'; +} +$('#createS3Key').onclick=async()=>{ + hideS3Secret(); + const permissions=$('#s3Permissions input[type=checkbox]:checked').map(x=>x.value); + const expiry=$('#s3KeyExpiry').value; + try{ + const k=await api('/api/v1/admin/s3-credentials',{method:'POST',body:JSON.stringify({ + name:$('#s3KeyName').value.trim(),permissions, + expires_at:expiry?Math.floor(new Date(expiry).getTime()/1000):null + })}); + $('#s3NewAccess').value=k.access_key; + $('#s3NewSecret').value=k.secret_key; + $('#createdS3Card').style.display='block'; + $('#s3KeyName').value=''; + toast('S3 credential created; copy the secret now'); + await loadS3Keys(); + }catch(e){toast(e.message,true)} +}; +$('#refreshS3Keys').onclick=loadS3Keys; +$('#hideS3Credentials').onclick=hideS3Secret; +$('#copyS3Credentials').onclick=async()=>{ + const text='AWS_ACCESS_KEY_ID='+$('#s3NewAccess').value+'\nAWS_SECRET_ACCESS_KEY='+$('#s3NewSecret').value; + try{await navigator.clipboard.writeText(text);toast('S3 keys copied')}catch(e){toast('Clipboard unavailable; select and copy manually',true)} +}; + +refreshDashboard();setInterval(refreshDashboard,3000); diff --git a/web/static/index.html b/web/static/index.html new file mode 100644 index 00000000..08555f85 --- /dev/null +++ b/web/static/index.html @@ -0,0 +1,238 @@ + + + + + +TeleDrive + + + +

TeleDrive Web UI

Loading…
API Docs ↗
+
+
+ + +
+
+
Pending0
+
Failed0
+
Uploaded0
+
Today0
+
+
Strategy
-
Database
-
Uploader
-
+

Accounts

+
+ +
PathStatusAttemptsAccountError
+
PathAccountMessage IDUploaded
+ +
+
+

Storage · Telethon USER MTProto

+

One or more API credential sets sharing the same dedicated channel. Sessions are pre-authorized files; phone/2FA never appears here.

+
+
+
+
+
+

API credential sets

+
+ +
+

Telegram (legacy uploader)

+
+
+
+
+
+
+

Accounts

+
+ +

Upload

+
+
+
+
+
+
+
+
+
+
+
+
+
+ +

Database

+ +
No vendor whitelist. Any synchronous SQLAlchemy dialect works when its dialect/DBAPI driver is installed in the image.
+
+ +

Automation

+
+
+
+
+
+ +
+
+

REST API access keys

+

Keys are stored hashed. Secret values are shown once and cannot be recovered. Keep admin keys private; give integrations only the scopes they need.

+
+
+
+
+
+ +
+ + + + + + + + + + + + + +
+
+ +
+
+

Existing keys

+
NameKey prefixPermissionsCreated / last usedStatusActions
+
+
+

S3-compatible gateway (initial core)

+

Telegram is the only payload backend. S3 credentials are encrypted at rest, scoped independently of REST keys, and displayed once. Configure the S3 master key before issuing credentials.

+
+
+
+
+
+ + + + + + + + + + + + + + + + + +
+
+ +
+

Existing S3 credentials

+
NameAccess Key IDPermissionsCreatedStatusActions
+

Local S3 endpoint: http://127.0.0.1:9000 (enable with Docker Compose profile s3). Supports signed path-style bucket/object operations, multipart uploads and checksum-bound presigned URLs. Advanced S3 operations are still being developed.

+
+
+

S3 multipart recovery

+

Review abandoned completing uploads before recovery. Only leases expired for at least one hour are eligible. Recovery returns an upload to active for a client retry; it never deletes Telegram messages.

+
+
BucketObject keyUpload IDLease expiredAction
+
+
+

Immutable object history and restore

+

Inspect known published versions of a key and restore an immutable snapshot. A restore creates a new version pointing at the original Telegram chunks; no Telegram payload is copied or deleted. Versions published before the history index was installed might not appear.

+
+
+
+
+
+
Version IDBytesSHA-256PublishedAction
+
+
+

Durable Telegram upload jobs

+

Queue an existing local file under the operator-configured TELEDRIVE_JOB_SOURCE_ROOT. The source remains on disk until uploaded to Telegram. Jobs survive SQL restart, use expiring worker leases and never overwrite a concurrently changed object head. Retrying a failed upload can create new Telegram chunks; historical media remains protected.

+
+
+
+
+
+
+ + + + +
+

Legacy folder → durable job queue

+

Explicitly import files from the configured legacy source folder, preserving path layout and filters. Requires the legacy source to be within TELEDRIVE_JOB_SOURCE_ROOT. Preview before importing. No Telegram upload starts during import; stop the old uploader before using the durable worker.

+
+
+
+
+
+
+ + +
+
Not scanned
+
+
Not checked
+
Bucket / keySourceStatusAttemptsUpdatedAction
+
+
+

Abandoned PENDING upload reconciliation

+

Inspect uploads whose SQL writer lease expired. Only an explicitly selected expired writer can be marked failed; Telegram messages and chunk journals are retained. A live lease, a READY object or a legacy upload without writer token cannot be touched.

+
+
BucketObject keyObject IDChunksExpiredAction
+
+
+

Telegram storage reference audit

+

Non-destructive inventory covering live objects, historical manifests, copied-object aliases and multipart parts. No Telegram messages can be deleted from this screen.

+
+
Not scanned
+
+

API endpoints

+

/api/v1/buckets · /api/v1/buckets/{bucket}/objects

+

Use Authorization: Bearer <token> for scoped endpoints. Administrative key management still requires the WebUI Basic login and CSRF header. The Telegram binary upload and S3 protocol endpoints are under development.

+OpenAPI / Swagger documentation ↗ +
+
+ +
+
+
+ + + \ No newline at end of file diff --git a/web/unified.py b/web/unified.py new file mode 100644 index 00000000..aa0f83e6 --- /dev/null +++ b/web/unified.py @@ -0,0 +1,73 @@ +"""Single-process Web UI + path-style S3, routed by socket-local TCP port.""" +import argparse +import socket +import uvicorn +from web.app import app as web_app +from web.s3 import app as s3_app + +class UnifiedApplication: + def __init__(self, web_port=8080, s3_port=9000): + if not all(1 <= value <= 65535 for value in (web_port,s3_port)) or web_port == s3_port: + raise ValueError("Distinct valid Web and S3 ports required") + self.web_port, self.s3_port = web_port, s3_port + + async def __call__(self, scope, receive, send): + if scope["type"] == "lifespan": + await web_app(scope, receive, send) + return + if scope["type"] not in ("http","websocket"): + raise RuntimeError("Unsupported ASGI scope") + server = scope.get("server") + if not server or len(server) < 2: + raise RuntimeError("Missing local listener identity") + if server[1] == self.web_port: + await web_app(scope, receive, send) + elif server[1] == self.s3_port: + await s3_app(scope, receive, send) + elif scope["type"] == "http": + from starlette.responses import PlainTextResponse + await PlainTextResponse("Unknown TeleDrive listener",status_code=421)(scope,receive,send) + else: + await send({"type":"websocket.close","code":1008}) + +app = UnifiedApplication() + +def _bind(host, port): + sock=socket.socket(socket.AF_INET,socket.SOCK_STREAM) + try: + sock.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) + sock.bind((host,port)) + sock.listen(2048) + sock.set_inheritable(False) + return sock + except BaseException: + sock.close() + raise + +def main(): + import web.app as webapp + + parser=argparse.ArgumentParser() + parser.add_argument("--host",default=None) + parser.add_argument("--web-port",type=int,default=None) + parser.add_argument("--s3-port",type=int,default=None) + args=parser.parse_args() + host = args.host or webapp.settings_bind_address() + web_port = args.web_port or webapp.settings_web_port() + s3_port = args.s3_port or webapp.settings_s3_port() + service=UnifiedApplication(web_port,s3_port) + listeners=[] + try: + listeners=[_bind(host,service.web_port),_bind(host,service.s3_port)] + uvicorn.Server(uvicorn.Config( + service,host=host,port=service.web_port, + workers=1,lifespan="on",server_header=False, + ssl_certfile=webapp.SSL_CERTFILE or None, + ssl_keyfile=webapp.SSL_KEYFILE or None, + )).run(sockets=listeners) + finally: + for listener in listeners: + listener.close() + +if __name__=="__main__": + main() diff --git a/website/.gitignore b/website/.gitignore deleted file mode 100644 index 16d54bb1..00000000 --- a/website/.gitignore +++ /dev/null @@ -1,24 +0,0 @@ -# build output -dist/ -# generated types -.astro/ - -# dependencies -node_modules/ - -# logs -npm-debug.log* -yarn-debug.log* -yarn-error.log* -pnpm-debug.log* - - -# environment variables -.env -.env.production - -# macOS-specific files -.DS_Store - -# jetbrains setting folder -.idea/ diff --git a/website/.vscode/extensions.json b/website/.vscode/extensions.json deleted file mode 100644 index 22a15055..00000000 --- a/website/.vscode/extensions.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "recommendations": ["astro-build.astro-vscode"], - "unwantedRecommendations": [] -} diff --git a/website/.vscode/launch.json b/website/.vscode/launch.json deleted file mode 100644 index d6422097..00000000 --- a/website/.vscode/launch.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "version": "0.2.0", - "configurations": [ - { - "command": "./node_modules/.bin/astro dev", - "name": "Development server", - "request": "launch", - "type": "node-terminal" - } - ] -} diff --git a/website/AGENTS.md b/website/AGENTS.md deleted file mode 100644 index c5ff4297..00000000 --- a/website/AGENTS.md +++ /dev/null @@ -1,22 +0,0 @@ -## Development - -When starting the dev server, use background mode: - -``` -astro dev --background -``` - -Manage the background server with `astro dev stop`, `astro dev status`, and `astro dev logs`. - -## Documentation - -Full documentation: https://docs.astro.build - -Consult these guides before working on related tasks: - -- [Adding pages, dynamic routes, or middleware](https://docs.astro.build/en/guides/routing/) -- [Working with Astro components](https://docs.astro.build/en/basics/astro-components/) -- [Using React, Vue, Svelte, or other framework components](https://docs.astro.build/en/guides/framework-components/) -- [Adding or managing content](https://docs.astro.build/en/guides/content-collections/) -- [Adding styles or using Tailwind](https://docs.astro.build/en/guides/styling/) -- [Supporting multiple languages](https://docs.astro.build/en/guides/internationalization/) diff --git a/website/CLAUDE.md b/website/CLAUDE.md deleted file mode 120000 index 47dc3e3d..00000000 --- a/website/CLAUDE.md +++ /dev/null @@ -1 +0,0 @@ -AGENTS.md \ No newline at end of file diff --git a/website/README.md b/website/README.md deleted file mode 100644 index 87b813ae..00000000 --- a/website/README.md +++ /dev/null @@ -1,43 +0,0 @@ -# Astro Starter Kit: Minimal - -```sh -npm create astro@latest -- --template minimal -``` - -> 🧑‍🚀 **Seasoned astronaut?** Delete this file. Have fun! - -## 🚀 Project Structure - -Inside of your Astro project, you'll see the following folders and files: - -```text -/ -├── public/ -├── src/ -│ └── pages/ -│ └── index.astro -└── package.json -``` - -Astro looks for `.astro` or `.md` files in the `src/pages/` directory. Each page is exposed as a route based on its file name. - -There's nothing special about `src/components/`, but that's where we like to put any Astro/React/Vue/Svelte/Preact components. - -Any static assets, like images, can be placed in the `public/` directory. - -## 🧞 Commands - -All commands are run from the root of the project, from a terminal: - -| Command | Action | -| :------------------------ | :----------------------------------------------- | -| `npm install` | Installs dependencies | -| `npm run dev` | Starts local dev server at `localhost:4321` | -| `npm run build` | Build your production site to `./dist/` | -| `npm run preview` | Preview your build locally, before deploying | -| `npm run astro ...` | Run CLI commands like `astro add`, `astro check` | -| `npm run astro -- --help` | Get help using the Astro CLI | - -## 👀 Want to learn more? - -Feel free to check [our documentation](https://docs.astro.build) or jump into our [Discord server](https://astro.build/chat). diff --git a/website/astro.config.mjs b/website/astro.config.mjs deleted file mode 100644 index e762ba5c..00000000 --- a/website/astro.config.mjs +++ /dev/null @@ -1,5 +0,0 @@ -// @ts-check -import { defineConfig } from 'astro/config'; - -// https://astro.build/config -export default defineConfig({}); diff --git a/website/package-lock.json b/website/package-lock.json deleted file mode 100644 index 0ba5ff8b..00000000 --- a/website/package-lock.json +++ /dev/null @@ -1,4096 +0,0 @@ -{ - "name": "website", - "version": "0.0.1", - "lockfileVersion": 3, - "requires": true, - "packages": { - "": { - "name": "website", - "version": "0.0.1", - "dependencies": { - "astro": "^7.2.9" - }, - "engines": { - "node": ">=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding/-/compiler-binding-0.4.0.tgz", - "integrity": "sha512-x2RjDUuWfwLNtc3mjAdSRInwqh/rqbLar9cm/5FOMbHvmYZB7yfKewzSclAxWjIZsypJDXv1lhaP2WG+P8TK3g==", - "license": "MIT", - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@astrojs/compiler-binding-darwin-arm64": "0.4.0", - "@astrojs/compiler-binding-darwin-x64": "0.4.0", - "@astrojs/compiler-binding-linux-arm64-gnu": "0.4.0", - "@astrojs/compiler-binding-linux-arm64-musl": "0.4.0", - "@astrojs/compiler-binding-linux-x64-gnu": "0.4.0", - "@astrojs/compiler-binding-linux-x64-musl": "0.4.0", - "@astrojs/compiler-binding-wasm32-wasi": "0.4.0", - "@astrojs/compiler-binding-win32-arm64-msvc": "0.4.0", - "@astrojs/compiler-binding-win32-x64-msvc": "0.4.0" - } - }, - "node_modules/@astrojs/compiler-binding-darwin-arm64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-darwin-arm64/-/compiler-binding-darwin-arm64-0.4.0.tgz", - "integrity": "sha512-ZVUwHundaQyFNjE6uoa0usaC0WOCitDCLS/4mdb4rOiJXwVUuKJBMxI5WMzXLWmamsXtK/Z//ifLXvV5Yeh4Hw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-darwin-x64": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-darwin-x64/-/compiler-binding-darwin-x64-0.4.0.tgz", - "integrity": "sha512-FI6G8AY8u6fR1SI/QRR5yGMwtvZwP34CDmZpZ5HwJGa50UM1VISTLhqkhV4a476pmgd25X1Aur2dqw6hUnrlKA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-linux-arm64-gnu": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-linux-arm64-gnu/-/compiler-binding-linux-arm64-gnu-0.4.0.tgz", - "integrity": "sha512-lB9gLFJK7m82EnjaU8nlRBEfcwGNeHidW3sSjODTUjMNaoewVuUz9fwwdY5M4jiSXIqWLH3yl6TX8FTDKA74Sw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-linux-arm64-musl": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-linux-arm64-musl/-/compiler-binding-linux-arm64-musl-0.4.0.tgz", - "integrity": "sha512-HPbvWqbxFxyaoQJhLxCaSjtYBx9KBo7JGVzEFZCmMl968a2PsSH0UfiODYgYPXofTOIsIH2aoCcrHXML0IA3ig==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-linux-x64-gnu": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-linux-x64-gnu/-/compiler-binding-linux-x64-gnu-0.4.0.tgz", - "integrity": "sha512-tQKolMxoJ/+0AmLWm1PmJ/i+z3i10ZU1bNuVjEDulCf48azEMtUNjTZgHJ5MPtpYRNc7dlETr8QujUfduzoC7Q==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-linux-x64-musl": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-linux-x64-musl/-/compiler-binding-linux-x64-musl-0.4.0.tgz", - "integrity": "sha512-5v5YymudsxMHp3NBLCS8BUlu5CRqeLtWD9cKS/4nIhIEHCbpz9okmVV6I0HWqmBAPhWYcDa3vw/vltYPrOQCTA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-wasm32-wasi": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-wasm32-wasi/-/compiler-binding-wasm32-wasi-0.4.0.tgz", - "integrity": "sha512-m/phuH3x3PREvv1OnkM44NoPh4MatUadix1fB1u5SvMLCyDTUZykDJbKnWf1cjnYmHdlB8HcjTjl6JrCqAIXcw==", - "cpu": [ - "wasm32" - ], - "license": "MIT", - "optional": true, - "dependencies": { - "@napi-rs/wasm-runtime": "^1.2.2" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@astrojs/compiler-binding-win32-arm64-msvc": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-win32-arm64-msvc/-/compiler-binding-win32-arm64-msvc-0.4.0.tgz", - "integrity": "sha512-B9zYf3okEY83kM8gydlpH2BHP00w4ifxPqlYlWrgTwuD6wnkrJDCwBlgy1q31cERjCJRXN1lrE2VmkLvFjv/6g==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-binding-win32-x64-msvc": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-binding-win32-x64-msvc/-/compiler-binding-win32-x64-msvc-0.4.0.tgz", - "integrity": "sha512-zB0Nrv0dGc0zZWPGDRmmETTPhDRqyZjAjk+gWMlVrJX5U89obpB3VUUE1ZiHxOCN5LQojeLK6O8L/dnoHolvNQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@astrojs/compiler-rs": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/@astrojs/compiler-rs/-/compiler-rs-0.4.0.tgz", - "integrity": "sha512-koVikeon1kreEy+/JzLQRy3vzHHQVOjycs4degg4vFufKApZOwMZvSSAEztYNhmcQVfNVsVZZI4cEge3cexAbQ==", - "license": "MIT", - "dependencies": { - "@astrojs/compiler-binding": "0.4.0" - }, - "engines": { - "node": ">=22.12.0" - } - }, - "node_modules/@astrojs/internal-helpers": { - "version": "0.10.4", - "resolved": "https://registry.npmjs.org/@astrojs/internal-helpers/-/internal-helpers-0.10.4.tgz", - "integrity": "sha512-nozZSy/mKYLqe4YrqbKtdOszedAfXYCtw3wZ0d+CAjz4GqQ4L9rl1ltIL5BlgwmYVinJg/RZ0MgGuWOdlyRZlA==", - "license": "MIT", - "dependencies": { - "@types/hast": "^3.0.4", - "@types/mdast": "^4.0.4", - "js-yaml": "^4.3.0", - "picomatch": "^4.0.4", - "retext-smartypants": "^6.2.0", - "shiki": "^4.0.2", - "smol-toml": "^1.6.0", - "unified": "^11.0.5" - } - }, - "node_modules/@astrojs/markdown-satteri": { - "version": "0.3.8", - "resolved": "https://registry.npmjs.org/@astrojs/markdown-satteri/-/markdown-satteri-0.3.8.tgz", - "integrity": "sha512-n8ItpFTCmlDsVR5+rwDmehSf+jFCYLWmZiisZNGuF7xILqYhsVeBfcha4qgS2Seq3fAc9Tm58ZVrvqFQ6RQgRQ==", - "license": "MIT", - "dependencies": { - "@astrojs/internal-helpers": "0.10.4", - "@astrojs/prism": "4.0.2", - "github-slugger": "^2.0.0", - "satteri": "^0.10.3" - } - }, - "node_modules/@astrojs/prism": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@astrojs/prism/-/prism-4.0.2.tgz", - "integrity": "sha512-KTivpmnz6lDsC6o9H4+DNm2SrE/GHzw8cNAvEJwAvUT+eoaEnn/4NtbDNfRRaxaJHdp15gf+tfHAWiXR4wB3BA==", - "license": "MIT", - "dependencies": { - "prismjs": "^1.30.0" - }, - "engines": { - "node": ">=22.12.0" - } - }, - "node_modules/@astrojs/telemetry": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/@astrojs/telemetry/-/telemetry-3.3.3.tgz", - "integrity": "sha512-C1TLn5sPJr0x4vk56piHWKbnqlEB8BKyte5Y45V02U+D7BGO5eMqZDH5aPjnkXQWJggvmsTXxH03QMZ9NgWLzQ==", - "license": "MIT", - "dependencies": { - "ci-info": "^4.4.0", - "dset": "^3.1.4", - "is-docker": "^4.0.0", - "package-manager-detector": "^1.6.0" - }, - "engines": { - "node": "18.20.8 || ^20.3.0 || >=22.0.0" - } - }, - "node_modules/@babel/helper-string-parser": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", - "integrity": "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-validator-identifier": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", - "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", - "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", - "integrity": "sha512-E8lTAYNB1KW+FH+VGJuZM1ioAx2E6oVlvQFRrf5P8ZZmsiJXYAD9vTFV7yyEURNzgh1dFqMZuO6tUwcARbqFCA==", - "license": "MIT", - "dependencies": { - "@babel/types": "^7.29.8" - }, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/types": { - "version": "7.29.8", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", - "integrity": "sha512-Vj1jF3cPfxg7OAfoI7QnVKLoILlm2JF9pnVHrX8qx7AHMiYWT+NDAA7jChlNgRS4WTLc/fD1lXLmPixluj+3Gg==", - "license": "MIT", - "dependencies": { - "@babel/helper-string-parser": "^7.29.7", - "@babel/helper-validator-identifier": "^7.29.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@bruits/satteri-darwin-arm64": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-darwin-arm64/-/satteri-darwin-arm64-0.10.5.tgz", - "integrity": "sha512-27KTVl4TJkVahMy/ohyA7qd4938G5UNneFUz/PsScYfpIhj0IVAS23mpcJXdPF44sa6nva198lmV/cKIb2YPyA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@bruits/satteri-darwin-x64": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-darwin-x64/-/satteri-darwin-x64-0.10.5.tgz", - "integrity": "sha512-IjnLe3nKspq6qaeqGgjT7MT8VrTV74yWRlaag7ZdNsI8TDAYZ0iPxMCo+9KQZHUk5EyVB+reBI/PFWL5KuFw9Q==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ] - }, - "node_modules/@bruits/satteri-linux-arm64-gnu": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-linux-arm64-gnu/-/satteri-linux-arm64-gnu-0.10.5.tgz", - "integrity": "sha512-glkYXZCJywjP13v67eAyAMSJdF+ncvEbYvgi/wOtffL9tQ27lr/zsyzUfgs+ovjJ9d8JNQKiXeiArJcX8PJL9w==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@bruits/satteri-linux-arm64-musl": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-linux-arm64-musl/-/satteri-linux-arm64-musl-0.10.5.tgz", - "integrity": "sha512-yWdgG1g17Nh2QyGVlFUxGRa3FEFwiMcpZEyMNWkbM3deC94cmVc+/i9OuyFpdKuWo3GkgoCtYVOoxk1uCnCZIA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@bruits/satteri-linux-x64-gnu": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-linux-x64-gnu/-/satteri-linux-x64-gnu-0.10.5.tgz", - "integrity": "sha512-FVaLoPT1fBgGl0J+AYebyyXJYBachGl8Oyyrf1lye4RTqCB4S0Gwkj1uM9RJyThUOvx5VUmAT1CnNh1SFHA+kw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@bruits/satteri-linux-x64-musl": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-linux-x64-musl/-/satteri-linux-x64-musl-0.10.5.tgz", - "integrity": "sha512-EHpVAx2bqW3GINHTKkljtxVfQmVDGWIuwOYOP5YghTj+0PkBa2o8oKPRtQ9Kbsr1Fye8jtUcDjhwj2jMNugZKg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ] - }, - "node_modules/@bruits/satteri-wasm32-wasi": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-wasm32-wasi/-/satteri-wasm32-wasi-0.10.5.tgz", - "integrity": "sha512-ypz8c/Zmipxp4IoeDa228Gstv6TLzVmNs3yC6wKCoNSOjx1iwpgzu87Y3hTkXFdwChVGU85qeUDuOIarGUZQLw==", - "cpu": [ - "wasm32" - ], - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/core": "1.11.1", - "@emnapi/runtime": "1.11.1", - "@napi-rs/wasm-runtime": "^1.2.3" - }, - "engines": { - "node": ">=14.0.0" - } - }, - "node_modules/@bruits/satteri-wasm32-wasi/node_modules/@emnapi/core": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", - "integrity": "sha512-RSvbQmHzdKzNsLYa/wHrbc3KN4sYLKAdPZxqiM2HATqv/SBk2/ENSHpvXGaLOMcsAyz0poEGqkmmKYG3OWiJEQ==", - "license": "MIT", - "optional": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.2", - "tslib": "^2.4.0" - } - }, - "node_modules/@bruits/satteri-wasm32-wasi/node_modules/@emnapi/runtime": { - "version": "1.11.1", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.1.tgz", - "integrity": "sha512-vgj7R3y3Wgx24IQaGPA/R6YFXLHVMOZ0uVEyIQPaWs+rd1AzfEMXlAC22FYwO1XkKR6NPsq7mUandH8oIRdZFw==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@bruits/satteri-wasm32-wasi/node_modules/@emnapi/wasi-threads": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.2.tgz", - "integrity": "sha512-c95qOXkHdydNKhscBTebqEC1CVAZpyqOfVfBzQ1qgzyl3gfeldUjIggDbIZgDKsHLgnsM+igH7TJ/eAasaVuMA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@bruits/satteri-win32-arm64-msvc": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-win32-arm64-msvc/-/satteri-win32-arm64-msvc-0.10.5.tgz", - "integrity": "sha512-siTV88nb0LRqNpkL2gXboqCwVdq95sLtzMHS1/3eONV2gLbB3NAK46wmSMvCO/yquBvI2lvaFIfd8P12ecsxBw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@bruits/satteri-win32-x64-msvc": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@bruits/satteri-win32-x64-msvc/-/satteri-win32-x64-msvc-0.10.5.tgz", - "integrity": "sha512-C3IfPvfvMXmlzBxaMPKFS1XiuV9pu2mC7YqkPk7PSvTgPZ8gbdASIpHpztDLvTTQjqZ0z1Ol8tK5X+V6XXC0wQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ] - }, - "node_modules/@capsizecss/unpack": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/@capsizecss/unpack/-/unpack-4.0.1.tgz", - "integrity": "sha512-CuNiSqg7+e1cO/GjffyMOm5Tt2jUF9CWHHnvQ/UkqvtkGfHdgwEC0wpmq7fkN3gxwpRnrAN0WzO3vREKmNolMQ==", - "license": "MIT", - "dependencies": { - "fontkitten": "^1.0.3" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/@clack/core": { - "version": "1.4.3", - "resolved": "https://registry.npmjs.org/@clack/core/-/core-1.4.3.tgz", - "integrity": "sha512-/kr3UWNtdJfxZtPgDqUOmG2pvwlmcLGheex5yiZKdwbzZJxhV+HMNR9QNmyY5cGwTNV6LrR7Jtp+KjhUAP1qBQ==", - "license": "MIT", - "dependencies": { - "fast-wrap-ansi": "^0.2.0", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 20.12.0" - } - }, - "node_modules/@clack/prompts": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/@clack/prompts/-/prompts-1.7.0.tgz", - "integrity": "sha512-y7/yvZ2TPAnR9+jnc00klvNNLkJiXFFrQA/hlLCcxA9a2A4zQIOimyFQ9XfwYKiGD1fb5GY8vbKIIgO8d5Tb2A==", - "license": "MIT", - "dependencies": { - "@clack/core": "1.4.3", - "fast-string-width": "^3.0.2", - "fast-wrap-ansi": "^0.2.0", - "sisteransi": "^1.0.5" - }, - "engines": { - "node": ">= 20.12.0" - } - }, - "node_modules/@emnapi/core": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.3.tgz", - "integrity": "sha512-zLpS5asjEb7lq8jYLq37N6XKaE41DIexlY1rF/z4/tIl3wo13Sqm28fRyfIsKZD+NZ8mM5RoKkpW/rBcuoSZSg==", - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "@emnapi/wasi-threads": "1.2.3", - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@emnapi/wasi-threads": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@emnapi/wasi-threads/-/wasi-threads-1.2.3.tgz", - "integrity": "sha512-ELEBe8PsLvvJ6QMr0zLt8ffvOHW/dc1m3CEzNMg7aJUv3bMaoDtw2TXyDAwkYBuroxxuHEwhRTLJSe5sya547g==", - "license": "MIT", - "optional": true, - "peer": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@esbuild/aix-ppc64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", - "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", - "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", - "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/android-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", - "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", - "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/darwin-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", - "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", - "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/freebsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", - "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", - "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", - "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ia32": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", - "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-loong64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", - "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", - "cpu": [ - "loong64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-mips64el": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", - "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", - "cpu": [ - "mips64el" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-ppc64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", - "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-riscv64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", - "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", - "cpu": [ - "riscv64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-s390x": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", - "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", - "cpu": [ - "s390x" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/linux-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", - "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", - "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/netbsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", - "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", - "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openbsd-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", - "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/openharmony-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", - "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/sunos-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", - "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-arm64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", - "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-ia32": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", - "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", - "cpu": [ - "ia32" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@esbuild/win32-x64": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", - "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/colour": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@img/colour/-/colour-1.1.0.tgz", - "integrity": "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ==", - "license": "MIT", - "optional": true, - "engines": { - "node": ">=18" - } - }, - "node_modules/@img/sharp-darwin-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.35.4.tgz", - "integrity": "sha512-Uhfl4V4lhP2nbUVF9+hyH1+luj86f1gUFeo8ALYxFoULoU+G87D43BfeMP8XHsk9boxAnCY/bf2EHwhA7MuGsA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-darwin-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.35.4.tgz", - "integrity": "sha512-hWniXY3bG5qKpkKrAwPe4y+VTPmf086YQAnkxWh7uA1YrlRouWGa0M0Mxj3ZjnXFkv7/TD1bTy9lGUK26vRvWw==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-darwin-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-freebsd-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-freebsd-wasm32/-/sharp-freebsd-wasm32-0.35.4.tgz", - "integrity": "sha512-lIsKw/BU+kjB4eZjxrYrZmwOJYi3Ajrv66iAlBmUPyKc3HpnloevB1g3wxGD9P/5BbQ1brBGl65VRRrCvQDEqA==", - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.3.3.tgz", - "integrity": "sha512-suTBPTDGrI9WodccaDdwZItTSaBYASlBk1NSfElSHrUfzu3szG6lvIF58+WiFvnfzuK8ZBFS5zE00PxqxnRiPg==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-darwin-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.3.3.tgz", - "integrity": "sha512-FVJZ5mITMobmXIz/hPDTw0EintTW5H3WfrxwLqEqjiIihlu+hVRyGrFQ60xl0Lxn7Bt3zdpevPaQi0HEzqz9fw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "darwin" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.3.3.tgz", - "integrity": "sha512-3rbU4vqXXc3hY/OiXdl52xZvT0F1yEngWfvqudtPJg/KkyiaQw2DRsFrNzpmLvfavbwOq3qXn36GP8obHRULQA==", - "cpu": [ - "arm" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.3.3.tgz", - "integrity": "sha512-0DaL0A6Xu6sQSQFwe4iVCrKWU2cCTItnRsYsCdxAMm9NF6twAA9BKnoqy4hqz4+azQ0JHuA26qiUKsf1XJ/v5A==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-ppc64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-ppc64/-/sharp-libvips-linux-ppc64-1.3.3.tgz", - "integrity": "sha512-cdn1OvUBwsXhbC0zSzJnNzf5MZ/mTrobawDvNXBTxe8VtqKAm0sRuEY2Evzovb/w9JMk4TvRxqt1mekSuJz64w==", - "cpu": [ - "ppc64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-riscv64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-riscv64/-/sharp-libvips-linux-riscv64-1.3.3.tgz", - "integrity": "sha512-HjPVx7yKz+0lqdhDlTw1tt90wamBoxhiXpvl1XZpJLiHH4RCJ5yDTqH+VlYPv2fwFs89JFw4c1IexYOcQUi4IQ==", - "cpu": [ - "riscv64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-s390x": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.3.3.tgz", - "integrity": "sha512-neWLh+3yCNThxnfy3c4BbVBeGgt9aftno+XbT56iK28RgeDs3UOFWviLWlUu0bArYVYJaFDK+RRohbicUNCm8Q==", - "cpu": [ - "s390x" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linux-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.3.3.tgz", - "integrity": "sha512-4vKmvAst9nrowcqquKFAyZJUDolUaIp8uRiN0mWFguJ1IplC9/pitXtlnnlU4aa/eJw3J7i67V+pwUL+wZGdsA==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-arm64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.3.3.tgz", - "integrity": "sha512-Y9kQaLMuNoB0bPYOOdcZMaseNrFpPodIWWMrx+CZyydf2xn68j9WYc6sWWRrDwNkzCQjKYfc68L7jKjGlHMibw==", - "cpu": [ - "arm64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-libvips-linuxmusl-x64": { - "version": "1.3.3", - "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.3.3.tgz", - "integrity": "sha512-fj8Mv0HHfD1Rr+4I68+3agJynxDWtBFgicTbSOb9Bke6pIwzGcJ+RX/yHjmiEGFMCavY/dxvem7MyNaJF+wDiw==", - "cpu": [ - "x64" - ], - "license": "LGPL-3.0-or-later", - "optional": true, - "os": [ - "linux" - ], - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-linux-arm": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.35.4.tgz", - "integrity": "sha512-7OAS8gI0EReKGVN2HssHlM6umJgxF5VI3xN0p9FA91p/YO+ou5hiNghLdZ5BEHztwaaK5+bLKRf8x/o2L2nk9A==", - "cpu": [ - "arm" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.35.4.tgz", - "integrity": "sha512-De4jpEnAU8Hd5oT0j1G3uL4ZvTuipVMn7YC6vPaJhy6/7EwEae0SVAoBrUMYQbkLGDm85taVWwuPc1a44LTzCQ==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-ppc64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-ppc64/-/sharp-linux-ppc64-0.35.4.tgz", - "integrity": "sha512-2oYZJeIl4kCcMGk4ouZVjnkCtFrpQFlNEtJ6GbxzhHQchwH0NH/qEb9ykmOl29dqwMq+JhFdZn+1ak2FKhI9fQ==", - "cpu": [ - "ppc64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-ppc64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-riscv64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-riscv64/-/sharp-linux-riscv64-0.35.4.tgz", - "integrity": "sha512-cPbNChoRURAWdebDIHSenxRpgEdy7JkPydSnUxRm9VvKD7m0/xVaR/8Fzlu81pk5nHEvHH87UZUA7cTtwnbJSA==", - "cpu": [ - "riscv64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-riscv64": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-s390x": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.35.4.tgz", - "integrity": "sha512-RY0JFY8Fd6RonCBtHz+DvadaPkXDSI1AUn6yWL9TipqkZ1vY8w8evqdgyDFnkm4/K1ve1TvZiaePP5oSd4+WVQ==", - "cpu": [ - "s390x" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-s390x": "1.3.3" - } - }, - "node_modules/@img/sharp-linux-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.35.4.tgz", - "integrity": "sha512-9qvvEAuk8k89TfWUoX2htWjbAMX8p+NxCppjpcg5k6xMsjhBQPTsoIh36h9Qde4WRuGpJeYnOjdosDn/cnv+OA==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linux-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.35.4.tgz", - "integrity": "sha512-KB5jxpfWQTr0nc3xdHtWChdbifHrBGsd2SM62Eyxrl8afikm+f5qGBU75SJIZBT/S1MC8XyacdlXBMSWq6OURA==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3" - } - }, - "node_modules/@img/sharp-linuxmusl-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.35.4.tgz", - "integrity": "sha512-f+eZJZIQNEEd26RPSW+76chwOf1XtA2Y/O+5ocVyLliHkeih3e+jhLVBdNTd2rS3IbNXK8+ug93Vf5ZXtF5Lxg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-libvips-linuxmusl-x64": "1.3.3" - } - }, - "node_modules/@img/sharp-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.35.4.tgz", - "integrity": "sha512-zQnl4Kwp7Q6NHsENtU2T/00Zi+w3AQNwz3+UaTyVBy2FpXrzXzGjndpK61onhZjRtRpQXxCTeqw19bVyXOh7jA==", - "license": "Apache-2.0 AND LGPL-3.0-or-later AND MIT", - "optional": true, - "dependencies": { - "@emnapi/runtime": "^1.11.3" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-webcontainers-wasm32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-webcontainers-wasm32/-/sharp-webcontainers-wasm32-0.35.4.tgz", - "integrity": "sha512-ESfNkywmCfPNyaZjxooddJQiQ+l/nTpGEOGthxiLnIHXC/CmcBixnfwUleX9mCz9ovrUUvKMap/pm8RYbzfwaA==", - "cpu": [ - "wasm32" - ], - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@img/sharp-wasm32": "0.35.4" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-arm64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-arm64/-/sharp-win32-arm64-0.35.4.tgz", - "integrity": "sha512-iNdlBX9gLVvqe2I3uIJSIKTq6wckP/DYxZtcqxm09x5Gi24DnFBmPAWZmr60ZyYMG0xlzo6goG3670ar+RXvRw==", - "cpu": [ - "arm64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-ia32": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.35.4.tgz", - "integrity": "sha512-kqRsbaa5CS6KHlpxnN7WhE6vAAugXyZButpRdvDWetlv6Qv4N9WTcrWzF7tXfB9T7MsoadqdI8hmwLq6UlLvtw==", - "cpu": [ - "ia32" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@img/sharp-win32-x64": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.35.4.tgz", - "integrity": "sha512-XtmnYhBcrORsJ4XJngyzr/EWP0hRZLAZRFaApdKuviyqF78+ylxh2y06ZmtULAMOnObJ3ucpN0AcwSWnMowTRg==", - "cpu": [ - "x64" - ], - "license": "Apache-2.0 AND LGPL-3.0-or-later", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - } - }, - "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", - "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", - "license": "MIT" - }, - "node_modules/@napi-rs/wasm-runtime": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.3.tgz", - "integrity": "sha512-UMduMbqO5s5zF2NkNacMT/yK5Y5QiKvWr2+50bzIIxFDwVJ2h49b+oyjaCGPhJxd2/gC2x39EHv/gHVuu36x2Q==", - "license": "MIT", - "optional": true, - "dependencies": { - "@tybys/wasm-util": "^0.10.3" - }, - "engines": { - "node": "^20.19.0 || ^22.13.0 || >=23.5.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/Brooooooklyn" - }, - "peerDependencies": { - "@emnapi/core": "^1.7.1 || ^2.0.0-alpha.4", - "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.4" - } - }, - "node_modules/@oslojs/encoding": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", - "integrity": "sha512-70wQhgYmndg4GCPxPPxPGevRKqTIJ2Nh4OkiMWmDAVYsTQ+Ta7Sq+rPevXyXGdzr30/qZBnyOalCszoMxlyldQ==", - "license": "MIT" - }, - "node_modules/@oxc-project/types": { - "version": "0.147.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.147.0.tgz", - "integrity": "sha512-IJ3s6ltHLp45S0bh7phkX+gJO7A1Wuz2EaqpAhb8WjqDwbzMiWKHhyyT42tskaWjEYXtHtVCPpnBJVT9+dcRLg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/Boshen" - } - }, - "node_modules/@rolldown/binding-android-arm-eabi": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.6.tgz", - "integrity": "sha512-b+jTcARdTiFLI6jB4a5XjTm0RWd6KcRfQj/I2356fxUZemiho9zQLxo0RtCuMDAyKcLo6cEltkgbQp6d1+sjjQ==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.6.tgz", - "integrity": "sha512-lkWU8ZJaRk9q3CIEY1Tc7vIFALp3Xw5NfGJo2hQg5oIqNgxWi1zI+IiDEK3r70BF5Dzol1tcXsnzsRc8NLhG+Q==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.6.tgz", - "integrity": "sha512-dgR56NYnvAszm7Ob1B2/Vn0e8bUQYZH2UjVaMMtMVOCKFSfjhfLmuA/9+O+F+ajUdG6B/bSssrKW6JJYASa8jA==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.6.tgz", - "integrity": "sha512-vpVxFvUCFioJqug7OTvqptkc4yb8UX0AwfDmJpaR/0sWz+BUmqSVAf7c8JkUgnN8YLspb4a/N6NhTyMAmdyQ7Q==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.6.tgz", - "integrity": "sha512-h1wG6Y6K3JlRswxsI64qQJqBAy4vrLuHgRbc8CZMGSWTOFRY6ghMApM1NKzB2I0n5xV1fjkE18SuVl2QpLeNpA==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.6.tgz", - "integrity": "sha512-tbCiqub0q2MVWJKgF5PoAlNWCtQydiOYSLIkd8sByqK/6MMYLJRcSXSYodqYtd0O+Fw7QaVmKKlS4oL94YRZ0w==", - "cpu": [ - "arm" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.6.tgz", - "integrity": "sha512-oxK9+baEBPhZG5HB4URY+uU04zJWeZlH6Tb9rB5DK4DF9XR1uXNLXt5Q5ZsugTKayNCNLhkcwz/ye74hRI98dg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.6.tgz", - "integrity": "sha512-muWCk27FVBEZtv0MsK8gnfSmgczA8KQ0uRVJbTABKhkRfQc38aUrcb7fhi3BNiyseFmgcRsoMfQsSNJ+DbZdSw==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.6.tgz", - "integrity": "sha512-eWDoSfU7Co2qj3vgB3Dt4lj1mG6CoWbcJQkRMP3XJplyCMtuaq3LHvPFjS9QIPvMGWVadJC04Xiy0IdcVPtnwQ==", - "cpu": [ - "ppc64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.6.tgz", - "integrity": "sha512-2bWNjRSIayvupRKxXUY2tWG9fYdoUlTqWywHRvE8Eq3GvuQ+f2HeIkve697fIt+IQs/PV8yFsdWuhp1aJ1PdnA==", - "cpu": [ - "s390x" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.6.tgz", - "integrity": "sha512-KekI0gS0wLxe1UBSQSjenBVwou/JkcQPDzBPICGZjxUv9k3RteHDPBQaiOicZUFKRIH2wKEimGwVpnJsbPzu7w==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.6.tgz", - "integrity": "sha512-TvtPnfVr+HtyGiDmPK4VWmlNm7QhNNAcK5Q9A7aOXsI8545yCyaoMaicXrFZ72JzeYjaUVk7yT243zT0jzjFKQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.6.tgz", - "integrity": "sha512-iOo0VEay2XFhaCcH0sps5XIimkSuOnNaZrf6+ZkoSOQBJPKNU48RkmJv0/lSpipexu5P+ouFgafe5IGr/DiQfg==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "openharmony" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.6.tgz", - "integrity": "sha512-y5NTmmasMS455JlOCO4ZM9krIchv3Mvm1crL1iUPGOPgEzSkves9n0SdC5Sjz6+qWDFhd8/JpfWMH8NSWNHe+A==", - "cpu": [ - "arm64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.6.tgz", - "integrity": "sha512-np8iZSLfXlAD4kWhiyq/u0Yt8oZDtRQ8lGhQaCXo2rl37KNjeU0GjJuwr4P3oeZ++ROfofsKNBqR5LTO8aXyWQ==", - "cpu": [ - "x64" - ], - "license": "MIT", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/@rolldown/pluginutils": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@rolldown/pluginutils/-/pluginutils-1.0.1.tgz", - "integrity": "sha512-2j9bGt5Jh8hj+vPtgzPtl72j0yRxHAyumoo6TNfAjsLB04UtpSvPbPcDcBMxz7n+9CYB0c1GxQFxYRg2jimqGw==", - "license": "MIT" - }, - "node_modules/@shikijs/core": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/core/-/core-4.4.3.tgz", - "integrity": "sha512-QCR4q2ZO/ILJEuwiBMel4wdcTDb1JGwfjKTxPDF6x8ixOaluPrVqIn06C99AcRPhmYlBR56d/Fb+GN58GzExpg==", - "license": "MIT", - "dependencies": { - "@shikijs/primitive": "4.4.3", - "@shikijs/types": "4.4.3", - "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.5", - "hast-util-to-html": "^9.0.5" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/engine-javascript": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/engine-javascript/-/engine-javascript-4.4.3.tgz", - "integrity": "sha512-FbOjFJp9VLdo1Wevs10BBtVxiTWwNLqZh5Gkhjgda/ioL15YOgeSl9n+6XMa3qRlPQzfhFNe641SrynFHYG0nQ==", - "license": "MIT", - "dependencies": { - "@shikijs/types": "4.4.3", - "@shikijs/vscode-textmate": "^10.0.2", - "oniguruma-to-es": "^4.3.6" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/engine-oniguruma": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/engine-oniguruma/-/engine-oniguruma-4.4.3.tgz", - "integrity": "sha512-EcOQkxdxGQrc1Row/cC2c96/v1dbZqGnEVu1qTuT/MJmp6+cXCvQussowVmCv5Tqr3KuY3c7IbM6HTW3LJ1k9w==", - "license": "MIT", - "dependencies": { - "@shikijs/types": "4.4.3", - "@shikijs/vscode-textmate": "^10.0.2" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/langs": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/langs/-/langs-4.4.3.tgz", - "integrity": "sha512-ePic0yfAJGOF83D5wBHK/00EjK65oahBYxFk5epgq33WRv7X9UuxLEV8PtR0szC0z8dl7INIpIodB99JRFlR+A==", - "license": "MIT", - "dependencies": { - "@shikijs/types": "4.4.3" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/primitive": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/primitive/-/primitive-4.4.3.tgz", - "integrity": "sha512-m0wBeLDQDeIxRdUmrCPdQqfuUamDwRL5isCfYbguKD6NiaKpVbsv+3J81DyIKgNW5h4WAIIr8T4EkgQrBBxvaQ==", - "license": "MIT", - "dependencies": { - "@shikijs/types": "4.4.3", - "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.5" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/themes": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/themes/-/themes-4.4.3.tgz", - "integrity": "sha512-w8UHjeUnIR965KMWJHUPXOc2mNJUnK3vpVLYLvw5IYU2mnTTJ89E24OrJDBNiJDQ0qzb0tc4l7mrIXx5cFeIyw==", - "license": "MIT", - "dependencies": { - "@shikijs/types": "4.4.3" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/types": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/@shikijs/types/-/types-4.4.3.tgz", - "integrity": "sha512-UEJxmRR++MAGR6hugn0vgVS2W/6lWAts84FFSrnlH9sP0LNol7E5+NQ792pH8liWUhyMyjhTgSUH3k7iD7tc5g==", - "license": "MIT", - "dependencies": { - "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.5" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/@shikijs/vscode-textmate": { - "version": "10.0.2", - "resolved": "https://registry.npmjs.org/@shikijs/vscode-textmate/-/vscode-textmate-10.0.2.tgz", - "integrity": "sha512-83yeghZ2xxin3Nj8z1NMd/NCuca+gsYXswywDy5bHvwlWL8tpTQmzGeUuHd9FC3E/SBEMvzJRwWEOz5gGes9Qg==", - "license": "MIT" - }, - "node_modules/@tybys/wasm-util": { - "version": "0.10.3", - "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", - "integrity": "sha512-F3fo1MYrRJYL3zER0OUOmkutjr1Vp23m7OsSgp7nq4SP6OqX6C/56XFIPAl5bt3zaBRjmW7SGz3u/6LwFpYcOg==", - "license": "MIT", - "optional": true, - "dependencies": { - "tslib": "^2.4.0" - } - }, - "node_modules/@types/estree": { - "version": "1.0.9", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", - "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", - "license": "MIT" - }, - "node_modules/@types/estree-jsx": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/@types/estree-jsx/-/estree-jsx-1.0.5.tgz", - "integrity": "sha512-52CcUVNFyfb1A2ALocQw/Dd1BQFNmSdkuC3BkZ6iqhdMfQz7JWOFRuJFloOzjk+6WijU56m9oKXFAXc7o3Towg==", - "license": "MIT", - "dependencies": { - "@types/estree": "*" - } - }, - "node_modules/@types/hast": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@types/hast/-/hast-3.0.5.tgz", - "integrity": "sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==", - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/mdast": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-4.0.4.tgz", - "integrity": "sha512-kGaNbPh1k7AFzgpud/gMdvIm5xuECykRR+JnWKQno9TAXVa6WIVCGTPvYGekIDL4uwCZQSYbUxNBSb1aUo79oA==", - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/nlcst": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/@types/nlcst/-/nlcst-2.0.3.tgz", - "integrity": "sha512-vSYNSDe6Ix3q+6Z7ri9lyWqgGhJTmzRjZRqyq15N0Z/1/UnVsno9G/N40NBijoYx2seFDIl0+B2mgAb9mezUCA==", - "license": "MIT", - "dependencies": { - "@types/unist": "*" - } - }, - "node_modules/@types/unist": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", - "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", - "license": "MIT" - }, - "node_modules/@ungap/structured-clone": { - "version": "1.4.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", - "integrity": "sha512-1mEZtMKPM09vDmQt5y7YvmN2+DFTP7Tg0EWXdic8/C6VRnpb33e4ghisCIE3WZjsE2N8mf+QV1Zqh7ZFYLWInQ==", - "license": "ISC" - }, - "node_modules/am-i-vibing": { - "version": "0.4.0", - "resolved": "https://registry.npmjs.org/am-i-vibing/-/am-i-vibing-0.4.0.tgz", - "integrity": "sha512-MxT4XZL7pzLHpuvhDKdMaQHMGGkJDLluKBLsbstn+8wv9sWcFT6h+0ve9qkml95amVTZtZV83gQe2hY+ojgHLg==", - "license": "MIT", - "dependencies": { - "process-ancestry": "^0.1.0" - }, - "bin": { - "am-i-vibing": "dist/cli.mjs" - } - }, - "node_modules/anymatch": { - "version": "3.1.3", - "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", - "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", - "license": "ISC", - "dependencies": { - "normalize-path": "^3.0.0", - "picomatch": "^2.0.4" - }, - "engines": { - "node": ">= 8" - } - }, - "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/argparse": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", - "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "license": "Python-2.0" - }, - "node_modules/aria-query": { - "version": "5.3.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.2.tgz", - "integrity": "sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==", - "license": "Apache-2.0", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/astro": { - "version": "7.2.9", - "resolved": "https://registry.npmjs.org/astro/-/astro-7.2.9.tgz", - "integrity": "sha512-o5nZFo/bieF6rp4x9sQSLhI7GO7Bahpld38Y4LvX27nFoxNiuttjI+Hea81w5ksORLHgdPUlQpU9obPz5QRa8g==", - "license": "MIT", - "dependencies": { - "@astrojs/compiler-rs": "^0.4.0", - "@astrojs/internal-helpers": "0.10.4", - "@astrojs/markdown-satteri": "0.3.8", - "@astrojs/telemetry": "3.3.3", - "@capsizecss/unpack": "^4.0.0", - "@clack/prompts": "^1.1.0", - "@oslojs/encoding": "^1.1.0", - "am-i-vibing": "^0.4.0", - "aria-query": "^5.3.2", - "axobject-query": "^4.1.0", - "ci-info": "^4.4.0", - "clsx": "^2.1.1", - "common-ancestor-path": "^2.0.0", - "cookie": "^2.0.1", - "devalue": "^5.8.1", - "diff": "^9.0.0", - "dset": "^3.1.4", - "es-module-lexer": "^2.0.0", - "esbuild": "^0.28.0", - "find-proc": "0.1.0", - "flattie": "^1.1.1", - "fontace": "~0.4.1", - "get-tsconfig": "5.0.0-beta.4", - "github-slugger": "^2.0.0", - "html-escaper": "3.0.3", - "http-cache-semantics": "^4.2.0", - "js-yaml": "^4.3.0", - "jsonc-parser": "^3.3.1", - "magic-string": "^1.0.0", - "magicast": "^0.5.2", - "mrmime": "^2.0.1", - "neotraverse": "^1.0.1", - "obug": "^2.1.1", - "p-limit": "^7.3.0", - "p-queue": "^9.1.0", - "package-manager-detector": "^1.6.0", - "piccolore": "^0.1.3", - "picomatch": "^4.0.4", - "semver": "^7.7.4", - "shiki": "^4.0.2", - "smol-toml": "^1.6.0", - "svgo": "^4.0.1", - "tinyclip": "^0.1.12", - "tinyexec": "^1.0.4", - "tinyglobby": "^0.2.15", - "ultrahtml": "^1.6.0", - "unifont": "~0.7.5", - "unstorage": "^1.17.5", - "vite": "^8.0.13", - "vitefu": "^1.1.2", - "xxhash-wasm": "^1.1.0", - "yargs-parser": "^22.0.0", - "zod": "^4.3.6" - }, - "bin": { - "astro": "bin/astro.mjs" - }, - "engines": { - "node": ">=22.12.0", - "npm": ">=9.6.5", - "pnpm": ">=7.1.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/astrodotbuild" - }, - "optionalDependencies": { - "sharp": "^0.35.4" - }, - "peerDependencies": { - "@astrojs/markdown-remark": "7.2.4" - }, - "peerDependenciesMeta": { - "@astrojs/markdown-remark": { - "optional": true - } - } - }, - "node_modules/axobject-query": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-4.1.0.tgz", - "integrity": "sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==", - "license": "Apache-2.0", - "engines": { - "node": ">= 0.4" - } - }, - "node_modules/bail": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/bail/-/bail-2.0.2.tgz", - "integrity": "sha512-0xO6mYd7JB2YesxDKplafRpsiOzPt9V02ddPCLbY1xYGPOX24NTyN50qnUxgCPcSoYMhKpAuBTjQoRZCAkUDRw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/boolbase": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", - "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", - "license": "ISC" - }, - "node_modules/ccount": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz", - "integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-html4": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/character-entities-html4/-/character-entities-html4-2.1.0.tgz", - "integrity": "sha512-1v7fgQRj6hnSwFpq1Eu0ynr/CDEw0rXo2B61qXrLNdHZmPKgb7fqS1a2JwF0rISo9q77jDI8VMEHoApn8qDoZA==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/character-entities-legacy": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/character-entities-legacy/-/character-entities-legacy-3.0.0.tgz", - "integrity": "sha512-RpPp0asT/6ufRm//AJVwpViZbGM/MkjQFxJccQRHmISF/22NBtsHqAWmL+/pmkPWoIUJdWyeVleTl1wydHATVQ==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/chokidar": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-5.0.0.tgz", - "integrity": "sha512-TQMmc3w+5AxjpL8iIiwebF73dRDF4fBIieAqGn9RGCWaEVwQ6Fb2cGe31Yns0RRIzii5goJ1Y7xbMwo1TxMplw==", - "license": "MIT", - "dependencies": { - "readdirp": "^5.0.0" - }, - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/ci-info": { - "version": "4.4.0", - "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", - "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/sibiraj-s" - } - ], - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/clsx": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", - "integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/comma-separated-tokens": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/comma-separated-tokens/-/comma-separated-tokens-2.0.3.tgz", - "integrity": "sha512-Fu4hJdvzeylCfQPp9SGWidpzrMs7tTrlu6Vb8XGaRGck8QSNZJJp538Wrb60Lax4fPwR64ViY468OIUTbRlGZg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/commander": { - "version": "11.1.0", - "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", - "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", - "license": "MIT", - "engines": { - "node": ">=16" - } - }, - "node_modules/common-ancestor-path": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/common-ancestor-path/-/common-ancestor-path-2.0.0.tgz", - "integrity": "sha512-dnN3ibLeoRf2HNC+OlCiNc5d2zxbLJXOtiZUudNFSXZrNSydxcCsSpRzXwfu7BBWCIfHPw+xTayeBvJCP/D8Ng==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">= 18" - } - }, - "node_modules/cookie": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-2.0.1.tgz", - "integrity": "sha512-yuToqVvRrj6pfDXREyQAAv8SkAEk/8GS3jQRTiUMm66TVtBYmqQeoEjL2Lmq8Rpo6271vH76InTChTitEAm65w==", - "license": "MIT", - "engines": { - "node": ">=22" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/express" - } - }, - "node_modules/cookie-es": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/cookie-es/-/cookie-es-1.2.3.tgz", - "integrity": "sha512-lXVyvUvrNXblMqzIRrxHb57UUVmqsSWlxqt3XIjCkUP0wDAf6uicO6KMbEgYrMNtEvWgWHwe42CKxPu9MYAnWw==", - "license": "MIT" - }, - "node_modules/crossws": { - "version": "0.3.5", - "resolved": "https://registry.npmjs.org/crossws/-/crossws-0.3.5.tgz", - "integrity": "sha512-ojKiDvcmByhwa8YYqbQI/hg7MEU0NC03+pSdEq4ZUnZR9xXpwk7E43SMNGkn+JxJGPFtNvQ48+vV2p+P1ml5PA==", - "license": "MIT", - "dependencies": { - "uncrypto": "^0.1.3" - } - }, - "node_modules/css-select": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/css-select/-/css-select-6.0.0.tgz", - "integrity": "sha512-rZZVSLle8v0+EY8QAkDWrKhpgt6SA5OtHsgBnsj6ZaLb5dmDVOWUDtQitd9ydxxvEjhewNudS6eTVU7uOyzvXw==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0", - "css-what": "^7.0.0", - "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "nth-check": "^2.1.1" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/css-tree": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz", - "integrity": "sha512-X7sjQzceUhu1u7Y/ylrRZFU2FS6LRiFVp6rKLPg23y3x3c3DOKAwuXGDp+PAGjh6CSnCjYeAul8pcT8bAl+lSA==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.27.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0" - } - }, - "node_modules/css-what": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/css-what/-/css-what-7.0.0.tgz", - "integrity": "sha512-wD5oz5xibMOPHzy13CyGmogB3phdvcDaB5t0W/Nr5Z2O/agcB8YwOz6e2Lsp10pNDzBoDO9nVa3RGs/2BttpHQ==", - "license": "BSD-2-Clause", - "engines": { - "node": ">= 6" - }, - "funding": { - "url": "https://github.com/sponsors/fb55" - } - }, - "node_modules/csso": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/csso/-/csso-5.0.5.tgz", - "integrity": "sha512-0LrrStPOdJj+SPCCrGhzryycLjwcgUSHBtxNA8aIDxf0GLsRh1cKYhB00Gd1lDOS4yGH69+SNn13+TWbVHETFQ==", - "license": "MIT", - "dependencies": { - "css-tree": "~2.2.0" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/css-tree": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/css-tree/-/css-tree-2.2.1.tgz", - "integrity": "sha512-OA0mILzGc1kCOCSJerOeqDxDQ4HOh+G8NbOJFOTgOCzpw7fCBubk0fEyxp8AgOL/jvLgYA/uV0cMbe43ElF1JA==", - "license": "MIT", - "dependencies": { - "mdn-data": "2.0.28", - "source-map-js": "^1.0.1" - }, - "engines": { - "node": "^10 || ^12.20.0 || ^14.13.0 || >=15.0.0", - "npm": ">=7.0.0" - } - }, - "node_modules/csso/node_modules/mdn-data": { - "version": "2.0.28", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.0.28.tgz", - "integrity": "sha512-aylIc7Z9y4yzHYAJNuESG3hfhC+0Ibp/MAMiaOZgNv4pmEdFyfZhhhny4MNiAfWdBQ1RQ2mfDWmM1x8SvGyp8g==", - "license": "CC0-1.0" - }, - "node_modules/defu": { - "version": "6.1.7", - "resolved": "https://registry.npmjs.org/defu/-/defu-6.1.7.tgz", - "integrity": "sha512-7z22QmUWiQ/2d0KkdYmANbRUVABpZ9SNYyH5vx6PZ+nE5bcC0l7uFvEfHlyld/HcGBFTL536ClDt3DEcSlEJAQ==", - "license": "MIT" - }, - "node_modules/dequal": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", - "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/destr": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/destr/-/destr-2.0.5.tgz", - "integrity": "sha512-ugFTXCtDZunbzasqBxrK93Ik/DRYsO6S/fedkWEMKqt04xZ4csmnmwGDBAb07QWNaGMAmnTIemsYZCksjATwsA==", - "license": "MIT" - }, - "node_modules/detect-libc": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", - "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", - "license": "Apache-2.0", - "engines": { - "node": ">=8" - } - }, - "node_modules/devalue": { - "version": "5.9.2", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.9.2.tgz", - "integrity": "sha512-po4PAY5c53tw5XMocSnf8A/5OHhbbUftpr93aEN6BBoAdntUmK7vu7wOATqvt7cXO7m1Cl4gMVn6p7n6n4mj0w==", - "license": "MIT" - }, - "node_modules/devlop": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/devlop/-/devlop-1.1.0.tgz", - "integrity": "sha512-RWmIqhcFf1lRYBvNmr7qTNuyCt/7/ns2jbpp1+PalgE/rDQcBT0fioSMUpJ93irlUhC5hrg4cYqe6U+0ImW0rA==", - "license": "MIT", - "dependencies": { - "dequal": "^2.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/diff": { - "version": "9.0.0", - "resolved": "https://registry.npmjs.org/diff/-/diff-9.0.0.tgz", - "integrity": "sha512-svtcdpS8CgJyqAjEQIXdb3OjhFVVYjzGAPO8WGCmRbrml64SPw/jJD4GoE98aR7r25A0XcgrK3F02yw9R/vhQw==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.3.1" - } - }, - "node_modules/dom-serializer": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", - "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", - "license": "MIT", - "dependencies": { - "domelementtype": "^2.3.0", - "domhandler": "^5.0.2", - "entities": "^4.2.0" - }, - "funding": { - "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" - } - }, - "node_modules/domelementtype": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", - "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/fb55" - } - ], - "license": "BSD-2-Clause" - }, - "node_modules/domhandler": { - "version": "5.0.3", - "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", - "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", - "license": "BSD-2-Clause", - "dependencies": { - "domelementtype": "^2.3.0" - }, - "engines": { - "node": ">= 4" - }, - "funding": { - "url": "https://github.com/fb55/domhandler?sponsor=1" - } - }, - "node_modules/domutils": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", - "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", - "license": "BSD-2-Clause", - "dependencies": { - "dom-serializer": "^2.0.0", - "domelementtype": "^2.3.0", - "domhandler": "^5.0.3" - }, - "funding": { - "url": "https://github.com/fb55/domutils?sponsor=1" - } - }, - "node_modules/dset": { - "version": "3.1.4", - "resolved": "https://registry.npmjs.org/dset/-/dset-3.1.4.tgz", - "integrity": "sha512-2QF/g9/zTaPDc3BjNcVTGoBbXBgYfMTTceLaYcFJ/W9kggFUkhxD/hMEeuLKbugyef9SqAx8cpgwlIP/jinUTA==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, - "node_modules/entities": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", - "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" - } - }, - "node_modules/es-module-lexer": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz", - "integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==", - "license": "MIT" - }, - "node_modules/esbuild": { - "version": "0.28.2", - "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", - "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", - "hasInstallScript": true, - "license": "MIT", - "bin": { - "esbuild": "bin/esbuild" - }, - "engines": { - "node": ">=18" - }, - "optionalDependencies": { - "@esbuild/aix-ppc64": "0.28.2", - "@esbuild/android-arm": "0.28.2", - "@esbuild/android-arm64": "0.28.2", - "@esbuild/android-x64": "0.28.2", - "@esbuild/darwin-arm64": "0.28.2", - "@esbuild/darwin-x64": "0.28.2", - "@esbuild/freebsd-arm64": "0.28.2", - "@esbuild/freebsd-x64": "0.28.2", - "@esbuild/linux-arm": "0.28.2", - "@esbuild/linux-arm64": "0.28.2", - "@esbuild/linux-ia32": "0.28.2", - "@esbuild/linux-loong64": "0.28.2", - "@esbuild/linux-mips64el": "0.28.2", - "@esbuild/linux-ppc64": "0.28.2", - "@esbuild/linux-riscv64": "0.28.2", - "@esbuild/linux-s390x": "0.28.2", - "@esbuild/linux-x64": "0.28.2", - "@esbuild/netbsd-arm64": "0.28.2", - "@esbuild/netbsd-x64": "0.28.2", - "@esbuild/openbsd-arm64": "0.28.2", - "@esbuild/openbsd-x64": "0.28.2", - "@esbuild/openharmony-arm64": "0.28.2", - "@esbuild/sunos-x64": "0.28.2", - "@esbuild/win32-arm64": "0.28.2", - "@esbuild/win32-ia32": "0.28.2", - "@esbuild/win32-x64": "0.28.2" - } - }, - "node_modules/eventemitter3": { - "version": "5.0.4", - "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", - "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", - "license": "MIT" - }, - "node_modules/extend": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", - "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", - "license": "MIT" - }, - "node_modules/fast-string-truncated-width": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/fast-string-truncated-width/-/fast-string-truncated-width-3.0.3.tgz", - "integrity": "sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==", - "license": "MIT" - }, - "node_modules/fast-string-width": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/fast-string-width/-/fast-string-width-3.0.2.tgz", - "integrity": "sha512-gX8LrtNEI5hq8DVUfRQMbr5lpaS4nMIWV+7XEbXk2b8kiQIizgnlr12B4dA3ZEx3308ze0O4Q1R+cHts8kyUJg==", - "license": "MIT", - "dependencies": { - "fast-string-truncated-width": "^3.0.2" - } - }, - "node_modules/fast-wrap-ansi": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/fast-wrap-ansi/-/fast-wrap-ansi-0.2.2.tgz", - "integrity": "sha512-7F2Fl+TjRSenLqlU3UjSH0iyqopqoZIu7eZVpEirP2g1GtWa2G/ecEmBdgz31+Mxr+ELclgg6sokpSFIQiZ02Q==", - "license": "MIT", - "dependencies": { - "fast-string-width": "^3.0.2" - } - }, - "node_modules/fdir": { - "version": "6.5.0", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz", - "integrity": "sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==", - "license": "MIT", - "engines": { - "node": ">=12.0.0" - }, - "peerDependencies": { - "picomatch": "^3 || ^4" - }, - "peerDependenciesMeta": { - "picomatch": { - "optional": true - } - } - }, - "node_modules/find-proc": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/find-proc/-/find-proc-0.1.0.tgz", - "integrity": "sha512-OaOpEYv2PiQ7SQ5LIrl+deA1XaWcxEjnpM6VuWXTUvn+teIXxeFTLDmu18/zDQpFmHN4o3oDBX+BT0AGwEhemg==", - "license": "MIT", - "engines": { - "node": "^20.19.0 || >=22.12.0" - } - }, - "node_modules/flattie": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/flattie/-/flattie-1.1.1.tgz", - "integrity": "sha512-9UbaD6XdAL97+k/n+N7JwX46K/M6Zc6KcFYskrYL8wbBV/Uyk0CTAMY0VT+qiK5PM7AIc9aTWYtq65U7T+aCNQ==", - "license": "MIT", - "engines": { - "node": ">=8" - } - }, - "node_modules/fontace": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/fontace/-/fontace-0.4.1.tgz", - "integrity": "sha512-lDMvbAzSnHmbYMTEld5qdtvNH2/pWpICOqpean9IgC7vUbUJc3k+k5Dokp85CegamqQpFbXf0rAVkbzpyTA8aw==", - "license": "MIT", - "dependencies": { - "fontkitten": "^1.0.2" - } - }, - "node_modules/fontkitten": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/fontkitten/-/fontkitten-1.0.3.tgz", - "integrity": "sha512-Wp1zXWPVUPBmfoa3Cqc9ctaKuzKAV6uLstRqlR56kSjplf5uAce+qeyYym7F+PHbGTk+tCEdkCW6RD7DX/gBZw==", - "license": "MIT", - "dependencies": { - "tiny-inflate": "^1.0.3" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, - "node_modules/get-tsconfig": { - "version": "5.0.0-beta.4", - "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-5.0.0-beta.4.tgz", - "integrity": "sha512-7nF7C9fIPFEMHgEMEfgIlO9wDdZ8CyHw27rWciFZfHvHDReIiPhsYuzPRXsfvBCqFy1l8RRyyWV7QLM+ZhUJsQ==", - "license": "MIT", - "dependencies": { - "resolve-pkg-maps": "^1.0.0" - }, - "engines": { - "node": ">=20.20.0" - }, - "funding": { - "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" - } - }, - "node_modules/github-slugger": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/github-slugger/-/github-slugger-2.0.0.tgz", - "integrity": "sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==", - "license": "ISC" - }, - "node_modules/h3": { - "version": "1.15.11", - "resolved": "https://registry.npmjs.org/h3/-/h3-1.15.11.tgz", - "integrity": "sha512-L3THSe2MPeBwgIZVSH5zLdBBU90TOxarvhK9d04IDY2AmVS8j2Jz2LIWtwsGOU3lu2I5jCN7FNvVfY2+XyF+mg==", - "license": "MIT", - "dependencies": { - "cookie-es": "^1.2.3", - "crossws": "^0.3.5", - "defu": "^6.1.6", - "destr": "^2.0.5", - "iron-webcrypto": "^1.2.1", - "node-mock-http": "^1.0.4", - "radix3": "^1.1.2", - "ufo": "^1.6.3", - "uncrypto": "^0.1.3" - } - }, - "node_modules/hast-util-to-html": { - "version": "9.0.5", - "resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz", - "integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==", - "license": "MIT", - "dependencies": { - "@types/hast": "^3.0.0", - "@types/unist": "^3.0.0", - "ccount": "^2.0.0", - "comma-separated-tokens": "^2.0.0", - "hast-util-whitespace": "^3.0.0", - "html-void-elements": "^3.0.0", - "mdast-util-to-hast": "^13.0.0", - "property-information": "^7.0.0", - "space-separated-tokens": "^2.0.0", - "stringify-entities": "^4.0.0", - "zwitch": "^2.0.4" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/hast-util-whitespace": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", - "integrity": "sha512-88JUN06ipLwsnv+dVn+OIYOvAuvBMy/Qoi6O7mQHxdPXpjy+Cd6xRkWwux7DKO+4sYILtLBRIKgsdpS2gQc7qw==", - "license": "MIT", - "dependencies": { - "@types/hast": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/html-escaper": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/html-escaper/-/html-escaper-3.0.3.tgz", - "integrity": "sha512-RuMffC89BOWQoY0WKGpIhn5gX3iI54O6nRA0yC124NYVtzjmFWBIiFd8M0x+ZdX0P9R4lADg1mgP8C7PxGOWuQ==", - "license": "MIT" - }, - "node_modules/html-void-elements": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", - "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/http-cache-semantics": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.2.0.tgz", - "integrity": "sha512-dTxcvPXqPvXBQpq5dUr6mEMJX4oIEFv6bwom3FDwKRDsuIjjJGANqhBuoAn9c1RQJIdAKav33ED65E2ys+87QQ==", - "license": "BSD-2-Clause" - }, - "node_modules/iron-webcrypto": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/iron-webcrypto/-/iron-webcrypto-1.2.1.tgz", - "integrity": "sha512-feOM6FaSr6rEABp/eDfVseKyTMDt+KGpeB35SkVn9Tyn0CqvVsY3EwI0v5i8nMHyJnzCIQf7nsy3p41TPkJZhg==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/brc-dd" - } - }, - "node_modules/is-docker": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-4.0.0.tgz", - "integrity": "sha512-LHE+wROyG/Y/0ZnbktRCoTix2c1RhgWaZraMZ8o1Q7zCh0VSrICJQO5oqIIISrcSBtrXv0o233w1IYwsWCjTzA==", - "license": "MIT", - "bin": { - "is-docker": "cli.js" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/is-plain-obj": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-4.1.0.tgz", - "integrity": "sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/js-yaml": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz", - "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/puzrin" - }, - { - "type": "github", - "url": "https://github.com/sponsors/nodeca" - } - ], - "license": "MIT", - "dependencies": { - "argparse": "^2.0.1" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/jsonc-parser": { - "version": "3.3.1", - "resolved": "https://registry.npmjs.org/jsonc-parser/-/jsonc-parser-3.3.1.tgz", - "integrity": "sha512-HUgH65KyejrUFPvHFPbqOY0rsFip3Bo5wb4ngvdi1EpCYWUQDC5V+Y7mZws+DLkr4M//zQJoanu1SP+87Dv1oQ==", - "license": "MIT" - }, - "node_modules/lightningcss": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", - "integrity": "sha512-WkUDrojuJs0xkgGf2udWxa3yGBRxPtxUkB79i6aCZLRgc7PM8fZe9TosfPDcvEpQZbuFASnHYmRLBLUbmLOIIA==", - "license": "MPL-2.0", - "dependencies": { - "detect-libc": "^2.0.3" - }, - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "optionalDependencies": { - "lightningcss-android-arm64": "1.33.0", - "lightningcss-darwin-arm64": "1.33.0", - "lightningcss-darwin-x64": "1.33.0", - "lightningcss-freebsd-x64": "1.33.0", - "lightningcss-linux-arm-gnueabihf": "1.33.0", - "lightningcss-linux-arm64-gnu": "1.33.0", - "lightningcss-linux-arm64-musl": "1.33.0", - "lightningcss-linux-x64-gnu": "1.33.0", - "lightningcss-linux-x64-musl": "1.33.0", - "lightningcss-win32-arm64-msvc": "1.33.0", - "lightningcss-win32-x64-msvc": "1.33.0" - } - }, - "node_modules/lightningcss-android-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-android-arm64/-/lightningcss-android-arm64-1.33.0.tgz", - "integrity": "sha512-gEpRTalKdosp4Bb8qWtc2iOgE5SeIHlpS1up9bFq2wAyYhl1UdTObYiHe98zEM9SQvSoqQZ1IQD0JNpg3Ml5pg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "android" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-arm64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-arm64/-/lightningcss-darwin-arm64-1.33.0.tgz", - "integrity": "sha512-Sciaz8eenNTKn9b3t7+xr0ipTp9YxKQY4npwQ3mrRuL0BAVHBLyZxofhaKBAVtzmtRZ/zTyo0/to4B1uWG/Djg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-darwin-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-darwin-x64/-/lightningcss-darwin-x64-1.33.0.tgz", - "integrity": "sha512-Z5UPAxzrjlWNNyGy6i65cJzzvgJ5D3T6wMvs+gWpY9d7qRhANrxqAp6LhxIgZhWEw18RfJTGcRxjuLIBr+m8XQ==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-freebsd-x64": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-freebsd-x64/-/lightningcss-freebsd-x64-1.33.0.tgz", - "integrity": "sha512-QQM/Ti/hQajJwCY+RiWuCZ9sdtI/XQk7nDK5vC8kkdwixezOlDgvDx7+RT+QjK6FcFT4MpsuoBnHIo/O3StRRg==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm-gnueabihf": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm-gnueabihf/-/lightningcss-linux-arm-gnueabihf-1.33.0.tgz", - "integrity": "sha512-N7FVBe6iS24MlM6R/4RBTxGhQheZGs7tiQ9U32UtF75NzP5Q7xWPRqLBCKxlRQRk3rY1jCIPLzx7WzOhuUIRLQ==", - "cpu": [ - "arm" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-gnu/-/lightningcss-linux-arm64-gnu-1.33.0.tgz", - "integrity": "sha512-j2v/itmy4HlNxlc6voKXYgBqNi0Ng2LShg4z7GufpEgs05P+2suBVyi9I6YHq5uoVFx9ETin3eCEhLVyXGQnKg==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-arm64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-arm64-musl/-/lightningcss-linux-arm64-musl-1.33.0.tgz", - "integrity": "sha512-yiO5ROMuYQgXbC60yjZU5CYSFZGKXL0HFATXt9mHJn1+zW55oCtMI9NfcVhYLMFDL7gV7oBPon/EmMMGg2OvtQ==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-gnu": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-gnu/-/lightningcss-linux-x64-gnu-1.33.0.tgz", - "integrity": "sha512-ar+Ju7LmcN0Jo4FpL4hpFybwNG9/3A/Br5KW2n2jyODg3MEZXaDYADdemoNS+BDNfMgKvylJLj4S5tyRActuAg==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-linux-x64-musl": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-linux-x64-musl/-/lightningcss-linux-x64-musl-1.33.0.tgz", - "integrity": "sha512-RYiYbkokw0trfKqqzfF55lginwEPrD3OJDfTuJzFs1MK6iFnDenaz1fqLLtX4ITG3OktJQXOeTaw1awrBAlZPw==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-arm64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-arm64-msvc/-/lightningcss-win32-arm64-msvc-1.33.0.tgz", - "integrity": "sha512-1K+MPfLSFVpphzpdbfkhlWk6wBrTObBzS2T6db10PNOZgR9GoVsAWzwNyuhUYYbTp23j+4RrncfujZ4uAzXvwA==", - "cpu": [ - "arm64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lightningcss-win32-x64-msvc": { - "version": "1.33.0", - "resolved": "https://registry.npmjs.org/lightningcss-win32-x64-msvc/-/lightningcss-win32-x64-msvc-1.33.0.tgz", - "integrity": "sha512-OlEICDx/Xl0FqSp4bry8zFnCvGpig3Gl4gCquvYwHuqJKEC1+n9NgDniFvqHGmMv1ZkqDJrDqKKSykTDX+ehuA==", - "cpu": [ - "x64" - ], - "license": "MPL-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 12.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - } - }, - "node_modules/lru-cache": { - "version": "11.5.2", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-11.5.2.tgz", - "integrity": "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g==", - "license": "BlueOak-1.0.0", - "engines": { - "node": "20 || >=22" - } - }, - "node_modules/magic-string": { - "version": "1.2.3", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.2.3.tgz", - "integrity": "sha512-Bpb0W2TbLKOZ7vJnOUnVRGq3WL2p+ISV29M6hYPL1AFCpyKZpdr5ytiXoTSSxRVhg8YW7f65+6gbG8WG6PCa/g==", - "license": "MIT", - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.5.5" - } - }, - "node_modules/magicast": { - "version": "0.5.4", - "resolved": "https://registry.npmjs.org/magicast/-/magicast-0.5.4.tgz", - "integrity": "sha512-llBEhWm1SacoRwgHUoQJYtwp4PBLF4faQi5TCpIGyGs9n4y5+juI0tDgyKIfpqxckRHaHzouUEph3THklWh03w==", - "license": "MIT", - "dependencies": { - "@babel/parser": "^7.29.7", - "@babel/types": "^7.29.7", - "source-map-js": "^1.2.1" - } - }, - "node_modules/mdast-util-to-hast": { - "version": "13.2.1", - "resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-13.2.1.tgz", - "integrity": "sha512-cctsq2wp5vTsLIcaymblUriiTcZd0CwWtCbLvrOzYCDZoWyMNV8sZ7krj09FSnsiJi3WVsHLM4k6Dq/yaPyCXA==", - "license": "MIT", - "dependencies": { - "@types/hast": "^3.0.0", - "@types/mdast": "^4.0.0", - "@ungap/structured-clone": "^1.0.0", - "devlop": "^1.0.0", - "micromark-util-sanitize-uri": "^2.0.0", - "trim-lines": "^3.0.0", - "unist-util-position": "^5.0.0", - "unist-util-visit": "^5.0.0", - "vfile": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/mdn-data": { - "version": "2.27.1", - "resolved": "https://registry.npmjs.org/mdn-data/-/mdn-data-2.27.1.tgz", - "integrity": "sha512-9Yubnt3e8A0OKwxYSXyhLymGW4sCufcLG6VdiDdUGVkPhpqLxlvP5vl1983gQjJl3tqbrM731mjaZaP68AgosQ==", - "license": "CC0-1.0" - }, - "node_modules/micromark-util-character": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/micromark-util-character/-/micromark-util-character-2.1.1.tgz", - "integrity": "sha512-wv8tdUTJ3thSFFFJKtpYKOYiGP2+v96Hvk4Tu8KpCAsTMs6yi+nVmGh1syvSCsaxz45J6Jbw+9DD6g97+NV67Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-symbol": "^2.0.0", - "micromark-util-types": "^2.0.0" - } - }, - "node_modules/micromark-util-encode": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-encode/-/micromark-util-encode-2.0.1.tgz", - "integrity": "sha512-c3cVx2y4KqUnwopcO9b/SCdo2O67LwJJ/UyqGfbigahfegL9myoEFoDYZgkT7f36T0bLrM9hZTAaAyH+PCAXjw==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromark-util-sanitize-uri": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-sanitize-uri/-/micromark-util-sanitize-uri-2.0.1.tgz", - "integrity": "sha512-9N9IomZ/YuGGZZmQec1MbgxtlgougxTodVwDzzEouPKo3qFWvymFHWcnDi2vzV1ff6kas9ucW+o3yzJK9YB1AQ==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT", - "dependencies": { - "micromark-util-character": "^2.0.0", - "micromark-util-encode": "^2.0.0", - "micromark-util-symbol": "^2.0.0" - } - }, - "node_modules/micromark-util-symbol": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/micromark-util-symbol/-/micromark-util-symbol-2.0.1.tgz", - "integrity": "sha512-vs5t8Apaud9N28kgCrRUdEed4UJ+wWNvicHLPxCa9ENlYuAY31M0ETy5y1vA33YoNPDFTghEbnh6efaE8h4x0Q==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/micromark-util-types": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/micromark-util-types/-/micromark-util-types-2.0.2.tgz", - "integrity": "sha512-Yw0ECSpJoViF1qTU4DC6NwtC4aWGt1EkzaQB8KPPyCRR8z9TWeV0HbEFGTO+ZY1wB22zmxnJqhPyTpOVCpeHTA==", - "funding": [ - { - "type": "GitHub Sponsors", - "url": "https://github.com/sponsors/unifiedjs" - }, - { - "type": "OpenCollective", - "url": "https://opencollective.com/unified" - } - ], - "license": "MIT" - }, - "node_modules/mrmime": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.1.tgz", - "integrity": "sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==", - "license": "MIT", - "engines": { - "node": ">=10" - } - }, - "node_modules/nanoid": { - "version": "3.3.18", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.18.tgz", - "integrity": "sha512-DTg4MJbGMWkfi6VZFdNt2/caMbQy4Ou+Op/hJQvGEWcnVfoA1QA+xzRKAzw9jD6+GVOOeYr/mIcuDSdug6F6+w==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, - "node_modules/neotraverse": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/neotraverse/-/neotraverse-1.0.1.tgz", - "integrity": "sha512-WmmLty1YWwJl9yZi77v2dVIV6X2kuYV8YYBI/G3LWGKdGHmHUvL1z7FW0iDvEvGAwNEoc5x1tOOOyDnf5jJw/w==", - "license": "MIT", - "engines": { - "node": ">= 10" - } - }, - "node_modules/nlcst-to-string": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/nlcst-to-string/-/nlcst-to-string-4.0.0.tgz", - "integrity": "sha512-YKLBCcUYKAg0FNlOBT6aI91qFmSiFKiluk655WzPF+DDMA02qIyy8uiRqI8QXtcFpEvll12LpL5MXqEmAZ+dcA==", - "license": "MIT", - "dependencies": { - "@types/nlcst": "^2.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/node-fetch-native": { - "version": "1.6.7", - "resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz", - "integrity": "sha512-g9yhqoedzIUm0nTnTqAQvueMPVOuIY16bqgAJJC8XOOubYFNwz6IER9qs0Gq2Xd0+CecCKFjtdDTMA4u4xG06Q==", - "license": "MIT" - }, - "node_modules/node-mock-http": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/node-mock-http/-/node-mock-http-1.0.5.tgz", - "integrity": "sha512-KQyt/wLjG3TAc7DOUhpqWzgd4ERxR80JOlTK5VE5R1S12IaPVN5qkj4klBce9HPG1Njuup4Sb5bljaT34lIyjw==", - "license": "MIT" - }, - "node_modules/normalize-path": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", - "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", - "license": "MIT", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/nth-check": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", - "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", - "license": "BSD-2-Clause", - "dependencies": { - "boolbase": "^1.0.0" - }, - "funding": { - "url": "https://github.com/fb55/nth-check?sponsor=1" - } - }, - "node_modules/obug": { - "version": "2.1.4", - "resolved": "https://registry.npmjs.org/obug/-/obug-2.1.4.tgz", - "integrity": "sha512-4a+OsYv9UktOJKE+l1A4OufDgdRF9PifWj+tJnHURo/P+WOxpG4GzUFL9qCalmWauao6ogiG+QvnCovwPoyAWA==", - "funding": [ - "https://github.com/sponsors/sxzz", - "https://opencollective.com/debug" - ], - "license": "MIT", - "engines": { - "node": ">=12.20.0" - } - }, - "node_modules/ofetch": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/ofetch/-/ofetch-1.5.1.tgz", - "integrity": "sha512-2W4oUZlVaqAPAil6FUg/difl6YhqhUR7x2eZY4bQCko22UXg3hptq9KLQdqFClV+Wu85UX7hNtdGTngi/1BxcA==", - "license": "MIT", - "dependencies": { - "destr": "^2.0.5", - "node-fetch-native": "^1.6.7", - "ufo": "^1.6.1" - } - }, - "node_modules/ohash": { - "version": "2.0.12", - "resolved": "https://registry.npmjs.org/ohash/-/ohash-2.0.12.tgz", - "integrity": "sha512-65S/5gk9YSsaRjcyf7Nfa6h/d3E8/1gslpXfI4W7Dxn/oap8IKRuNT5VXkLQ1YFKIEg4apRY4Pj6aiwFzrDdmw==", - "license": "MIT" - }, - "node_modules/oniguruma-parser": { - "version": "0.12.2", - "resolved": "https://registry.npmjs.org/oniguruma-parser/-/oniguruma-parser-0.12.2.tgz", - "integrity": "sha512-6HVa5oIrgMC6aA6WF6XyyqbhRPJrKR02L20+2+zpDtO5QAzGHAUGw5TKQvwi5vctNnRHkJYmjAhRVQF2EKdTQw==", - "license": "MIT" - }, - "node_modules/oniguruma-to-es": { - "version": "4.3.6", - "resolved": "https://registry.npmjs.org/oniguruma-to-es/-/oniguruma-to-es-4.3.6.tgz", - "integrity": "sha512-csuQ9x3Yr0cEIs/Zgx/OEt9iBw9vqIunAPQkx19R/fiMq2oGVTgcMqO/V3Ybqefr1TBvosI6jU539ksaBULJyA==", - "license": "MIT", - "dependencies": { - "oniguruma-parser": "^0.12.2", - "regex": "^6.1.0", - "regex-recursion": "^6.0.2" - } - }, - "node_modules/p-limit": { - "version": "7.3.1", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-7.3.1.tgz", - "integrity": "sha512-0trZaiG7Y7kN/Egy9a8j47t9osC0Tch4PaIWd9yGF6bvmlk7muExRvGNYb8sXBwEKMoNKsbNN9P8EefuQekE4Q==", - "license": "MIT", - "dependencies": { - "yocto-queue": "^1.2.1" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-queue": { - "version": "9.3.3", - "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.3.tgz", - "integrity": "sha512-NXAOdnEe5FsZJfT4oK84lE1Y5cFFdWlRuOo5tww8DyNMxyRXwn39fIkUtNLKppcPC+UYU/bXujNCUGDv01y7CA==", - "license": "MIT", - "dependencies": { - "eventemitter3": "^5.0.4", - "p-timeout": "^7.0.0" - }, - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-timeout": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", - "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", - "license": "MIT", - "engines": { - "node": ">=20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/package-manager-detector": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/package-manager-detector/-/package-manager-detector-1.8.0.tgz", - "integrity": "sha512-yQA4H19AmPEoMUeavPMDIe1higySl/gH/yaQrkT/s07Qp+7pp2hYz30N3z2l5BkjVkF9Ow6o0wjJamm2y7Sn0A==", - "license": "MIT" - }, - "node_modules/piccolore": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/piccolore/-/piccolore-0.1.3.tgz", - "integrity": "sha512-o8bTeDWjE086iwKrROaDf31K0qC/BENdm15/uH9usSC/uZjJOKb2YGiVHfLY4GhwsERiPI1jmwI2XrA7ACOxVw==", - "license": "ISC" - }, - "node_modules/picocolors": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", - "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", - "license": "ISC" - }, - "node_modules/picomatch": { - "version": "4.0.7", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.7.tgz", - "integrity": "sha512-qcJu88Q2IWqJsDD529JKMdwGm/dvInW4HvQnRwiH9JtihJvzGOscDtHE3x1pBKeUOTysQ8kVmLnJ2kJu7yhcGA==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, - "node_modules/postcss": { - "version": "8.5.26", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.26.tgz", - "integrity": "sha512-u82N74LFzG8ca+dD8puPnplTXoGH4fTPpVGuIbt36G3qvNlkvfD0lEAZSxaly3KX8TS/L1A1gsCEmvKmBcVbkQ==", - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "license": "MIT", - "dependencies": { - "nanoid": "^3.3.17", - "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } - }, - "node_modules/prismjs": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/prismjs/-/prismjs-1.30.0.tgz", - "integrity": "sha512-DEvV2ZF2r2/63V+tK8hQvrR2ZGn10srHbXviTlcv7Kpzw8jWiNTqbVgjO3IY8RxrrOUF8VPMQQFysYYYv0YZxw==", - "license": "MIT", - "engines": { - "node": ">=6" - } - }, - "node_modules/process-ancestry": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/process-ancestry/-/process-ancestry-0.1.0.tgz", - "integrity": "sha512-tGqJW/UnclpYASFcM6Xh8D8l/BMtaQ9+CSG0vlJSJTcdMM4lDRv4c6H0Pdcsfted+bVczdYSfk2fdukg2gQkZg==", - "license": "MIT", - "engines": { - "node": ">=18.0.0" - } - }, - "node_modules/property-information": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/property-information/-/property-information-7.2.0.tgz", - "integrity": "sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/radix3": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/radix3/-/radix3-1.1.2.tgz", - "integrity": "sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==", - "license": "MIT" - }, - "node_modules/readdirp": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-5.1.1.tgz", - "integrity": "sha512-Kko+Y5XQ6fM+Ce3dq3m9YGxnacYZYl9cA1wZjaF3Vbry2L3i1qVg8+CAgNPsXRArPMUMCaOR7oa9Nqntc43JKA==", - "license": "MIT", - "engines": { - "node": ">= 20.19.0" - }, - "funding": { - "type": "individual", - "url": "https://paulmillr.com/funding/" - } - }, - "node_modules/regex": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/regex/-/regex-6.1.0.tgz", - "integrity": "sha512-6VwtthbV4o/7+OaAF9I5L5V3llLEsoPyq9P1JVXkedTP33c7MfCG0/5NOPcSJn0TzXcG9YUrR0gQSWioew3LDg==", - "license": "MIT", - "dependencies": { - "regex-utilities": "^2.3.0" - } - }, - "node_modules/regex-recursion": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/regex-recursion/-/regex-recursion-6.0.2.tgz", - "integrity": "sha512-0YCaSCq2VRIebiaUviZNs0cBz1kg5kVS2UKUfNIx8YVs1cN3AV7NTctO5FOKBA+UT2BPJIWZauYHPqJODG50cg==", - "license": "MIT", - "dependencies": { - "regex-utilities": "^2.3.0" - } - }, - "node_modules/regex-utilities": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/regex-utilities/-/regex-utilities-2.3.0.tgz", - "integrity": "sha512-8VhliFJAWRaUiVvREIiW2NXXTmHs4vMNnSzuJVhscgmGav3g9VDxLrQndI3dZZVVdp0ZO/5v0xmX516/7M9cng==", - "license": "MIT" - }, - "node_modules/resolve-pkg-maps": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", - "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", - "license": "MIT", - "funding": { - "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" - } - }, - "node_modules/retext-smartypants": { - "version": "6.2.0", - "resolved": "https://registry.npmjs.org/retext-smartypants/-/retext-smartypants-6.2.0.tgz", - "integrity": "sha512-kk0jOU7+zGv//kfjXEBjdIryL1Acl4i9XNkHxtM7Tm5lFiCog576fjNC9hjoR7LTKQ0DsPWy09JummSsH1uqfQ==", - "license": "MIT", - "dependencies": { - "@types/nlcst": "^2.0.0", - "nlcst-to-string": "^4.0.0", - "unist-util-visit": "^5.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/rolldown": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.6.tgz", - "integrity": "sha512-vMM4q3aixf46GiF1Kok8jDPFsEpXgFWGjUHXNkNHNm+Y2adXAG2dbX91jkti3i0ZRsOlcmbuzAz1poObSHCmUA==", - "license": "MIT", - "dependencies": { - "@oxc-project/types": "=0.147.0", - "@rolldown/pluginutils": "^1.0.0" - }, - "bin": { - "rolldown": "bin/cli.mjs" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "optionalDependencies": { - "@rolldown/binding-android-arm-eabi": "1.2.6", - "@rolldown/binding-android-arm64": "1.2.6", - "@rolldown/binding-darwin-arm64": "1.2.6", - "@rolldown/binding-darwin-x64": "1.2.6", - "@rolldown/binding-freebsd-x64": "1.2.6", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.6", - "@rolldown/binding-linux-arm64-gnu": "1.2.6", - "@rolldown/binding-linux-arm64-musl": "1.2.6", - "@rolldown/binding-linux-ppc64-gnu": "1.2.6", - "@rolldown/binding-linux-s390x-gnu": "1.2.6", - "@rolldown/binding-linux-x64-gnu": "1.2.6", - "@rolldown/binding-linux-x64-musl": "1.2.6", - "@rolldown/binding-openharmony-arm64": "1.2.6", - "@rolldown/binding-win32-arm64-msvc": "1.2.6", - "@rolldown/binding-win32-x64-msvc": "1.2.6" - } - }, - "node_modules/satteri": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/satteri/-/satteri-0.10.5.tgz", - "integrity": "sha512-Ao1LKpAEa9Wdg0otgbVKViZHEq9ebdXe4DMrp3s9vQAU0HNIuHnFEuMuOcm0ZIXyV0Yzxj91NvhLpvXZJO/5ZQ==", - "license": "MIT", - "dependencies": { - "@types/estree-jsx": "^1.0.5", - "@types/hast": "^3.0.5", - "@types/mdast": "^4.0.4", - "@types/unist": "^3.0.3" - }, - "optionalDependencies": { - "@bruits/satteri-darwin-arm64": "0.10.5", - "@bruits/satteri-darwin-x64": "0.10.5", - "@bruits/satteri-linux-arm64-gnu": "0.10.5", - "@bruits/satteri-linux-arm64-musl": "0.10.5", - "@bruits/satteri-linux-x64-gnu": "0.10.5", - "@bruits/satteri-linux-x64-musl": "0.10.5", - "@bruits/satteri-wasm32-wasi": "0.10.5", - "@bruits/satteri-win32-arm64-msvc": "0.10.5", - "@bruits/satteri-win32-x64-msvc": "0.10.5" - } - }, - "node_modules/sax": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.1.tgz", - "integrity": "sha512-42tBVwLWnaQvW5zc4HbZrTuWccECCZfBi92FDuwtqxasH+JbPB3/FOKb1m222K42R4WxuxzzMsTswfzgtSu64Q==", - "license": "BlueOak-1.0.0", - "engines": { - "node": ">=11.0.0" - } - }, - "node_modules/semver": { - "version": "7.8.5", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", - "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", - "license": "ISC", - "bin": { - "semver": "bin/semver.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/sharp": { - "version": "0.35.4", - "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.35.4.tgz", - "integrity": "sha512-n++8XWcj+jCOr2IOl7h8LbKnGBDY4aPbmprMONBNFdn0ImXqpGVv5zliDs0V9HbmbCQLpbuo2ej9rAoOQTvMDA==", - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "@img/colour": "^1.1.0", - "detect-libc": "^2.1.2", - "semver": "^7.8.5" - }, - "engines": { - "node": ">=20.9.0" - }, - "funding": { - "url": "https://opencollective.com/libvips" - }, - "optionalDependencies": { - "@img/sharp-darwin-arm64": "0.35.4", - "@img/sharp-darwin-x64": "0.35.4", - "@img/sharp-freebsd-wasm32": "0.35.4", - "@img/sharp-libvips-darwin-arm64": "1.3.3", - "@img/sharp-libvips-darwin-x64": "1.3.3", - "@img/sharp-libvips-linux-arm": "1.3.3", - "@img/sharp-libvips-linux-arm64": "1.3.3", - "@img/sharp-libvips-linux-ppc64": "1.3.3", - "@img/sharp-libvips-linux-riscv64": "1.3.3", - "@img/sharp-libvips-linux-s390x": "1.3.3", - "@img/sharp-libvips-linux-x64": "1.3.3", - "@img/sharp-libvips-linuxmusl-arm64": "1.3.3", - "@img/sharp-libvips-linuxmusl-x64": "1.3.3", - "@img/sharp-linux-arm": "0.35.4", - "@img/sharp-linux-arm64": "0.35.4", - "@img/sharp-linux-ppc64": "0.35.4", - "@img/sharp-linux-riscv64": "0.35.4", - "@img/sharp-linux-s390x": "0.35.4", - "@img/sharp-linux-x64": "0.35.4", - "@img/sharp-linuxmusl-arm64": "0.35.4", - "@img/sharp-linuxmusl-x64": "0.35.4", - "@img/sharp-webcontainers-wasm32": "0.35.4", - "@img/sharp-win32-arm64": "0.35.4", - "@img/sharp-win32-ia32": "0.35.4", - "@img/sharp-win32-x64": "0.35.4" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - } - } - }, - "node_modules/shiki": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/shiki/-/shiki-4.4.3.tgz", - "integrity": "sha512-Mb/GvXPHBAXdgGIcnfU5L3ldpn1XcxrGkPHwqgRx17/I2XRfqlFKk2vGkHWINn1kdXvzJZeuO3is6I9KLPFm0g==", - "license": "MIT", - "dependencies": { - "@shikijs/core": "4.4.3", - "@shikijs/engine-javascript": "4.4.3", - "@shikijs/engine-oniguruma": "4.4.3", - "@shikijs/langs": "4.4.3", - "@shikijs/themes": "4.4.3", - "@shikijs/types": "4.4.3", - "@shikijs/vscode-textmate": "^10.0.2", - "@types/hast": "^3.0.5" - }, - "engines": { - "node": ">=20" - } - }, - "node_modules/sisteransi": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz", - "integrity": "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==", - "license": "MIT" - }, - "node_modules/smol-toml": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/smol-toml/-/smol-toml-1.8.0.tgz", - "integrity": "sha512-kCZr2V3ch9i00x8zXRhjUNVcjG9ijES5dDudkXvUVCT5QlJNQWElSJdZqyPemffHoLNUYwOcou0Fy+ojN0uHSQ==", - "license": "BSD-3-Clause", - "engines": { - "node": ">= 18" - }, - "funding": { - "url": "https://github.com/sponsors/cyyynthia" - } - }, - "node_modules/source-map-js": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.1.tgz", - "integrity": "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==", - "license": "BSD-3-Clause", - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/space-separated-tokens": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/space-separated-tokens/-/space-separated-tokens-2.0.2.tgz", - "integrity": "sha512-PEGlAwrG8yXGXRjW32fGbg66JAlOAwbObuqVoJpv/mRgoWDQfgH1wDPvtzWyUSNAXBGSk8h755YDbbcEy3SH2Q==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/stringify-entities": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/stringify-entities/-/stringify-entities-4.0.4.tgz", - "integrity": "sha512-IwfBptatlO+QCJUo19AqvrPNqlVMpW9YEL2LIVY+Rpv2qsjCGxaDLNRgeGsQWJhfItebuJhsGSLjaBbNSQ+ieg==", - "license": "MIT", - "dependencies": { - "character-entities-html4": "^2.0.0", - "character-entities-legacy": "^3.0.0" - }, - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/svgo": { - "version": "4.1.0", - "resolved": "https://registry.npmjs.org/svgo/-/svgo-4.1.0.tgz", - "integrity": "sha512-bkxnTg1kSU0guhIBmibA6UUhrQmPVA1XsQLN+ylCd+UWzbnLkySOcXpyk1mrl05f+pcaCx2eHb+sp6BgMZWX+Q==", - "license": "MIT", - "dependencies": { - "commander": "^11.1.0", - "css-select": "^6.0.0", - "css-tree": "^3.0.1", - "css-what": "^7.0.0", - "csso": "^5.0.5", - "picocolors": "^1.1.1", - "sax": "1.6.1" - }, - "bin": { - "svgo": "bin/svgo.js" - }, - "engines": { - "node": ">=16" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/svgo" - } - }, - "node_modules/tiny-inflate": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/tiny-inflate/-/tiny-inflate-1.0.3.tgz", - "integrity": "sha512-pkY1fj1cKHb2seWDy0B16HeWyczlJA9/WW3u3c4z/NiWDsO3DOU5D7nhTLE9CF0yXv/QZFY7sEJmj24dK+Rrqw==", - "license": "MIT" - }, - "node_modules/tinyclip": { - "version": "0.1.15", - "resolved": "https://registry.npmjs.org/tinyclip/-/tinyclip-0.1.15.tgz", - "integrity": "sha512-uo33abH+Ays0xYaDysoBt494Hb3hsEczMpcC0MwFl773pazORx4fmvKhclhR1wonUbB6vvpRsvVMwnhfqeMc+A==", - "license": "MIT", - "engines": { - "node": "^16.14.0 || >= 17.3.0" - } - }, - "node_modules/tinyexec": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.0.tgz", - "integrity": "sha512-QKAl9m8gWWGHV8jZcPeym6j+XULi6tOf1mT83WYJ4Lk2ytW/uwAWkrP0uFsdoYMdueVJ0qs26wZ+23xeB4ibNQ==", - "license": "MIT", - "engines": { - "node": ">=18" - } - }, - "node_modules/tinyglobby": { - "version": "0.2.17", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz", - "integrity": "sha512-wXR/dYpcqKmfWpEdZjiKJOwCNFndD0DMnrW/cYjVGttEkBfVgcLFHoNrlj47mjOVic9yyNu65alsgF4NQyTa2g==", - "license": "MIT", - "dependencies": { - "fdir": "^6.5.0", - "picomatch": "^4.0.4" - }, - "engines": { - "node": ">=12.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/SuperchupuDev" - } - }, - "node_modules/trim-lines": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/trim-lines/-/trim-lines-3.0.1.tgz", - "integrity": "sha512-kRj8B+YHZCc9kQYdWfJB2/oUl9rA99qbowYYBtr4ui4mZyAQ2JpvVBd/6U2YloATfqBhBTSMhTpgBHtU0Mf3Rg==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/trough": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz", - "integrity": "sha512-tmMpK00BjZiUyVyvrBK7knerNgmgvcV/KLVyuma/SC+TQN167GrMRciANTz09+k3zW8L8t60jWO1GpfkZdjTaw==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "license": "0BSD", - "optional": true - }, - "node_modules/ufo": { - "version": "1.6.4", - "resolved": "https://registry.npmjs.org/ufo/-/ufo-1.6.4.tgz", - "integrity": "sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==", - "license": "MIT" - }, - "node_modules/ultrahtml": { - "version": "1.7.0", - "resolved": "https://registry.npmjs.org/ultrahtml/-/ultrahtml-1.7.0.tgz", - "integrity": "sha512-2xRd0VHoAQE4M+vF/DvFFB7pUV0ZxTW1TLi7lHQWnF/Sb5TPeEUV/l+hxcNnGO00ZXGnR0voCMmYRKQf+rvJ2g==", - "license": "MIT" - }, - "node_modules/uncrypto": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/uncrypto/-/uncrypto-0.1.3.tgz", - "integrity": "sha512-Ql87qFHB3s/De2ClA9e0gsnS6zXG27SkTiSJwjCc9MebbfapQfuPzumMIUMi38ezPZVNFcHI9sUIepeQfw8J8Q==", - "license": "MIT" - }, - "node_modules/undici": { - "version": "8.10.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-8.10.0.tgz", - "integrity": "sha512-HvltHd7avK13QIw/oLe4qoOLyoVSoafqJ2jYOrtMRBkbYT31eiBQ8O0ehRKZiEZCMEyLFQNIADpgCWC5fALvYQ==", - "license": "MIT", - "engines": { - "node": ">=22.19.0" - } - }, - "node_modules/unified": { - "version": "11.0.5", - "resolved": "https://registry.npmjs.org/unified/-/unified-11.0.5.tgz", - "integrity": "sha512-xKvGhPWw3k84Qjh8bI3ZeJjqnyadK+GEFtazSfZv/rKeTkTjOJho6mFqh2SM96iIcZokxiOpg78GazTSg8+KHA==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "bail": "^2.0.0", - "devlop": "^1.0.0", - "extend": "^3.0.0", - "is-plain-obj": "^4.0.0", - "trough": "^2.0.0", - "vfile": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unifont": { - "version": "0.7.5", - "resolved": "https://registry.npmjs.org/unifont/-/unifont-0.7.5.tgz", - "integrity": "sha512-ULe/Cs+ZIsq+dcFofNkhqielCrUJnb5mr+Yc4EBM2VlL+6OZR6+cjtI2mT1bJvRBrVncqHAbLURxmPLcCXzWMg==", - "license": "MIT", - "dependencies": { - "css-tree": "^3.1.0", - "ohash": "^2.0.11", - "undici": "^8.0.0" - } - }, - "node_modules/unist-util-is": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz", - "integrity": "sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-position": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/unist-util-position/-/unist-util-position-5.0.0.tgz", - "integrity": "sha512-fucsC7HjXvkB5R3kTCO7kUjRdrS0BJt3M/FPxmHMBOm8JQi2BsHAHFsy27E0EolP8rp0NzXsJ+jNPyDWvOJZPA==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-stringify-position": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/unist-util-stringify-position/-/unist-util-stringify-position-4.0.0.tgz", - "integrity": "sha512-0ASV06AAoKCDkS2+xw5RXJywruurpbC4JZSm7nr7MOt1ojAzvyyaO+UxZf18j8FCF6kmzCZKcAgN/yu2gm2XgQ==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-visit": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/unist-util-visit/-/unist-util-visit-5.1.0.tgz", - "integrity": "sha512-m+vIdyeCOpdr/QeQCu2EzxX/ohgS8KbnPDgFni4dQsfSCtpz8UqDyY5GjRru8PDKuYn7Fq19j1CQ+nJSsGKOzg==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-is": "^6.0.0", - "unist-util-visit-parents": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unist-util-visit-parents": { - "version": "6.0.2", - "resolved": "https://registry.npmjs.org/unist-util-visit-parents/-/unist-util-visit-parents-6.0.2.tgz", - "integrity": "sha512-goh1s1TBrqSqukSc8wrjwWhL0hiJxgA8m4kFxGlQ+8FYQ3C/m11FcTs4YYem7V664AhHVvgoQLk890Ssdsr2IQ==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-is": "^6.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/unstorage": { - "version": "1.17.5", - "resolved": "https://registry.npmjs.org/unstorage/-/unstorage-1.17.5.tgz", - "integrity": "sha512-0i3iqvRfx29hkNntHyQvJTpf5W9dQ9ZadSoRU8+xVlhVtT7jAX57fazYO9EHvcRCfBCyi5YRya7XCDOsbTgkPg==", - "license": "MIT", - "dependencies": { - "anymatch": "^3.1.3", - "chokidar": "^5.0.0", - "destr": "^2.0.5", - "h3": "^1.15.10", - "lru-cache": "^11.2.7", - "node-fetch-native": "^1.6.7", - "ofetch": "^1.5.1", - "ufo": "^1.6.3" - }, - "peerDependencies": { - "@azure/app-configuration": "^1.8.0", - "@azure/cosmos": "^4.2.0", - "@azure/data-tables": "^13.3.0", - "@azure/identity": "^4.6.0", - "@azure/keyvault-secrets": "^4.9.0", - "@azure/storage-blob": "^12.26.0", - "@capacitor/preferences": "^6 || ^7 || ^8", - "@deno/kv": ">=0.9.0", - "@netlify/blobs": "^6.5.0 || ^7.0.0 || ^8.1.0 || ^9.0.0 || ^10.0.0", - "@planetscale/database": "^1.19.0", - "@upstash/redis": "^1.34.3", - "@vercel/blob": ">=0.27.1", - "@vercel/functions": "^2.2.12 || ^3.0.0", - "@vercel/kv": "^1 || ^2 || ^3", - "aws4fetch": "^1.0.20", - "db0": ">=0.2.1", - "idb-keyval": "^6.2.1", - "ioredis": "^5.4.2", - "uploadthing": "^7.4.4" - }, - "peerDependenciesMeta": { - "@azure/app-configuration": { - "optional": true - }, - "@azure/cosmos": { - "optional": true - }, - "@azure/data-tables": { - "optional": true - }, - "@azure/identity": { - "optional": true - }, - "@azure/keyvault-secrets": { - "optional": true - }, - "@azure/storage-blob": { - "optional": true - }, - "@capacitor/preferences": { - "optional": true - }, - "@deno/kv": { - "optional": true - }, - "@netlify/blobs": { - "optional": true - }, - "@planetscale/database": { - "optional": true - }, - "@upstash/redis": { - "optional": true - }, - "@vercel/blob": { - "optional": true - }, - "@vercel/functions": { - "optional": true - }, - "@vercel/kv": { - "optional": true - }, - "aws4fetch": { - "optional": true - }, - "db0": { - "optional": true - }, - "idb-keyval": { - "optional": true - }, - "ioredis": { - "optional": true - }, - "uploadthing": { - "optional": true - } - } - }, - "node_modules/vfile": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/vfile/-/vfile-6.0.3.tgz", - "integrity": "sha512-KzIbH/9tXat2u30jf+smMwFCsno4wHVdNmzFyL+T/L3UGqqk6JKfVqOFOZEpZSHADH1k40ab6NUIXZq422ov3Q==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "vfile-message": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/vfile-message": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", - "integrity": "sha512-QTHzsGd1EhbZs4AsQ20JX1rC3cOlt/IWJruk893DfLRr57lcnOeMaWG4K0JrRta4mIJZKth2Au3mM3u03/JWKw==", - "license": "MIT", - "dependencies": { - "@types/unist": "^3.0.0", - "unist-util-stringify-position": "^4.0.0" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/unified" - } - }, - "node_modules/vite": { - "version": "8.2.2", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.2.2.tgz", - "integrity": "sha512-cFKLV/PRgAUlIRm5WjMjJ86jrftzpqcgH+Us+DS8mI3CDNiH30Whrz8uHL3+MOLPAgqbMBAqWdAHAphOAM+z/Q==", - "license": "MIT", - "dependencies": { - "lightningcss": "^1.33.0", - "picomatch": "^4.0.5", - "postcss": "^8.5.26", - "rolldown": "~1.2.4", - "tinyglobby": "^0.2.17" - }, - "bin": { - "vite": "bin/vite.js" - }, - "engines": { - "node": "^20.19.0 || >=22.12.0" - }, - "funding": { - "url": "https://github.com/vitejs/vite?sponsor=1" - }, - "optionalDependencies": { - "fsevents": "~2.3.3" - }, - "peerDependencies": { - "@types/node": "^20.19.0 || >=22.12.0", - "@vitejs/devtools": "^0.4.0 || ^0.5.0", - "esbuild": "^0.27.0 || ^0.28.0", - "jiti": ">=1.21.0", - "less": "^4.0.0", - "sass": "^1.70.0", - "sass-embedded": "^1.70.0", - "stylus": ">=0.54.8", - "sugarss": "^5.0.0", - "terser": "^5.16.0", - "tsx": "^4.8.1", - "yaml": "^2.4.2" - }, - "peerDependenciesMeta": { - "@types/node": { - "optional": true - }, - "@vitejs/devtools": { - "optional": true - }, - "esbuild": { - "optional": true - }, - "jiti": { - "optional": true - }, - "less": { - "optional": true - }, - "sass": { - "optional": true - }, - "sass-embedded": { - "optional": true - }, - "stylus": { - "optional": true - }, - "sugarss": { - "optional": true - }, - "terser": { - "optional": true - }, - "tsx": { - "optional": true - }, - "yaml": { - "optional": true - } - } - }, - "node_modules/vitefu": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/vitefu/-/vitefu-1.1.3.tgz", - "integrity": "sha512-ub4okH7Z5KLjb6hDyjqrGXqWtWvoYdU3IGm/NorpgHncKoLTCfRIbvlhBm7r0YstIaQRYlp4yEbFqDcKSzXSSg==", - "license": "MIT", - "workspaces": [ - "tests/deps/*", - "tests/projects/*", - "tests/projects/workspace/packages/*" - ], - "peerDependencies": { - "vite": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0" - }, - "peerDependenciesMeta": { - "vite": { - "optional": true - } - } - }, - "node_modules/xxhash-wasm": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", - "integrity": "sha512-147y/6YNh+tlp6nd/2pWq38i9h6mz/EuQ6njIrmW8D1BS5nCqs0P6DG+m6zTGnNz5I+uhZ0SHxBs9BsPrwcKDA==", - "license": "MIT" - }, - "node_modules/yargs-parser": { - "version": "22.0.0", - "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-22.0.0.tgz", - "integrity": "sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==", - "license": "ISC", - "engines": { - "node": "^20.19.0 || ^22.12.0 || >=23" - } - }, - "node_modules/yocto-queue": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", - "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", - "license": "MIT", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/zod": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.5.4.tgz", - "integrity": "sha512-sC95tT5iHHH9gtpj6A81kh+NEaRAUFN+qlUPDUbRfOMvNf5QCBqsb3WgvnpVtK5Y+4UfA6KqufotuTvMGiTlsA==", - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/colinhacks" - } - }, - "node_modules/zwitch": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz", - "integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==", - "license": "MIT", - "funding": { - "type": "github", - "url": "https://github.com/sponsors/wooorm" - } - } - } -} diff --git a/website/package.json b/website/package.json deleted file mode 100644 index dcd6a10d..00000000 --- a/website/package.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "name": "website", - "type": "module", - "version": "0.0.1", - "engines": { - "node": ">=22.12.0" - }, - "scripts": { - "dev": "astro dev", - "build": "astro build", - "preview": "astro preview", - "astro": "astro" - }, - "dependencies": { - "astro": "^7.2.9" - }, - "allowScripts": { - "esbuild": true - } -} \ No newline at end of file diff --git a/website/public/favicon.ico b/website/public/favicon.ico deleted file mode 100644 index 7f48a94d..00000000 Binary files a/website/public/favicon.ico and /dev/null differ diff --git a/website/public/favicon.svg b/website/public/favicon.svg deleted file mode 100644 index f157bd1c..00000000 --- a/website/public/favicon.svg +++ /dev/null @@ -1,9 +0,0 @@ - - - - diff --git a/website/public/images/dashboard.png b/website/public/images/dashboard.png deleted file mode 100644 index 1487b95c..00000000 Binary files a/website/public/images/dashboard.png and /dev/null differ diff --git a/website/public/images/settings.png b/website/public/images/settings.png deleted file mode 100644 index 915c14cc..00000000 Binary files a/website/public/images/settings.png and /dev/null differ diff --git a/website/public/images/upload-queue.png b/website/public/images/upload-queue.png deleted file mode 100644 index c75cb0ff..00000000 Binary files a/website/public/images/upload-queue.png and /dev/null differ diff --git a/website/public/knowledge.md b/website/public/knowledge.md deleted file mode 100644 index 4a0d6842..00000000 --- a/website/public/knowledge.md +++ /dev/null @@ -1,37 +0,0 @@ -# TeleDrive Knowledge Base - -## Overview -TeleDrive is a safe, resumable, and autonomous personal file uploader and backup client for Windows and Linux. It turns Telegram into a private, unlimited cloud storage drive by uploading files directly to your personal Saved Messages or any private Telegram Channel. - -## Key Features -- **Zero Third-Party Cloud Storage Costs**: Uses your own Telegram account storage via the official MTProto API. -- **Flood-Wait Shield**: Intelligent rate-limiting protection. When Telegram returns a FLOOD_WAIT penalty, TeleDrive automatically pauses and waits for the exact required cooldown plus safety jitter before resuming. -- **Human-like Jitter Delays**: Configurable randomized delays (e.g., 20s - 60s) between file uploads to keep account activity natural and safe. -- **ACID-Compliant SQLite Queue**: Scanned files, hashes, sizes, and upload progress are persisted locally in SQLite (`uploader.db`). If your computer restarts, updates, or loses power, TeleDrive resumes right where it left off without duplicate uploads. -- **Standalone Desktop App**: Built with PySide6 (Qt). Windows users can install via `.exe` installer with zero Python dependencies needed. Linux x86_64 tarball package is also available. -- **Open Source**: MIT Licensed repository on GitHub at `https://github.com/amjadlle/TeleDrive`. - -## Installation & Setup -### Windows -1. Download `TeleDrive-Setup-1.0.6.exe` from GitHub releases or run: - ```powershell - irm "https://github.com/amjadlle/TeleDrive/releases/download/v1.0.6/TeleDrive-Setup-1.0.6.exe" -OutFile "$env:TEMP\TeleDrive-Setup.exe"; Start-Process "$env:TEMP\TeleDrive-Setup.exe" - ``` -2. Launch TeleDrive and enter your Telegram `api_id` and `api_hash` from [my.telegram.org](https://my.telegram.org). -3. Authenticate with your phone number and SMS/Telegram OTP code (2FA password supported). -4. Select a local folder to scan and click **Run Batch** or **Start Loop**. - -### Linux -Download the `TeleDrive-linux-x86_64.tar.gz` package, extract, and execute `TeleDrive`. - -## Frequently Asked Questions (FAQ) -- **Do I need a Telegram Bot / BotFather token?** No. TeleDrive logs in as your regular user account via MTProto API to access unlimited cloud storage. -- **Can I upload to a channel?** Yes. Enter `@channel_username` or `-100xxxxxxxxxx` ID as the target. -- **Is 2-Step Verification (2FA) supported?** Yes. TeleDrive securely handles cloud passwords during login. -- **Where are my credentials saved?** Stored 100% locally on your machine in the app configuration folder. - -## Creator Info -- **Creator**: Amjad P A (@amjadlle) -- **Portfolio**: [amjad.mapki.in](https://amjad.mapki.in) -- **GitHub**: [github.com/amjadlle/TeleDrive](https://github.com/amjadlle/TeleDrive) -- **Email**: hire.amjad@gmail.com diff --git a/website/public/logo.png b/website/public/logo.png deleted file mode 100644 index 62a90a5b..00000000 Binary files a/website/public/logo.png and /dev/null differ diff --git a/website/public/og-image.png b/website/public/og-image.png deleted file mode 100644 index 7123af2e..00000000 Binary files a/website/public/og-image.png and /dev/null differ diff --git a/website/src/components/Author.astro b/website/src/components/Author.astro deleted file mode 100644 index d8163d4a..00000000 --- a/website/src/components/Author.astro +++ /dev/null @@ -1,298 +0,0 @@ ---- -const socials = [ - { - name: "GitHub", - url: "https://github.com/amjadlle", - icon: `` - }, - { - name: "YouTube", - url: "https://www.youtube.com/@reputedculprit", - icon: `` - }, - { - name: "LinkedIn", - url: "https://www.linkedin.com/in/amjadlle", - icon: `` - }, - { - name: "X (Twitter)", - url: "https://x.com/amjadlle", - icon: `` - }, - { - name: "Threads", - url: "https://www.threads.net/@amjadlle", - icon: `` - }, - { - name: "Instagram", - url: "https://www.instagram.com/amjadlle", - icon: `` - }, - { - name: "Reddit", - url: "https://www.reddit.com/user/reputed_culprit/", - icon: `` - } -]; ---- - -
-
-
-
- -
-
- Amjad P A -
- -
-
Creator & Developer
-

Amjad P A

-

@amjadlle

-
-
- - -
-

- Passionate software engineer and systems builder. Creator of TeleDrive, exploring autonomous backup architectures, cloud toolings, and high-performance modern web experiences. -

- - -
- - ☕ - Buy Me a Coffee - - - - - - - - hire.amjad@gmail.com - - - - - - - - - Portfolio - -
- - -
- {socials.map((social) => ( - - ))} -
-
-
-
-
-
- - diff --git a/website/src/components/Faq.astro b/website/src/components/Faq.astro deleted file mode 100644 index 11e7673c..00000000 --- a/website/src/components/Faq.astro +++ /dev/null @@ -1,132 +0,0 @@ ---- -const faqs = [ - { - q: "Is it safe for my Telegram account to upload files?", - a: "Yes. TeleDrive uses official Telethon client sessions via Telegram's standard MTProto protocol. Unlike aggressive mass-uploaders, TeleDrive is deliberately designed with slow, randomized delays (e.g. 20-60s) and respects every FLOOD_WAIT signal with extra buffers to prevent rate-limit penalties." - }, - { - q: "Where are my Telegram session keys and database stored?", - a: "Everything is stored 100% locally on your machine. On Windows: %LOCALAPPDATA%\\Telegram Auto Upload, and on Linux: ~/.local/share/Telegram Auto Upload. TeleDrive contains zero analytics, zero external API pings, and zero telemetry." - }, - { - q: "Do I need Python installed to run TeleDrive?", - a: "No! The Windows release comes with a standalone per-user installer (.exe) that bundles everything. The Linux release includes a portable x86_64 pre-compiled package. Python is only needed if you wish to hack on the source code." - }, - { - q: "What happens if my computer restarts during an upload?", - a: "TeleDrive uses a persistent SQLite database. Every scanned file and upload state is tracked with ACID transactions. When you reboot and restart TeleDrive, it immediately resumes exactly where it left off without duplicate re-uploads." - }, - { - q: "Can I upload to a private Channel instead of Saved Messages?", - a: "Yes. Set the target to a public username such as @my_backup_channel or a numeric ID such as -1001234567890. The logged-in account must be a member and have permission to post. For a private channel or supergroup, copy a Telegram Desktop message link and convert t.me/c/1234567890/... to -1001234567890." - }, - { - q: "Do I need to create a bot or get a BotFather token?", - a: "No. TeleDrive currently signs in as your normal Telegram user account through the MTProto API. Get api_id and api_hash from my.telegram.org, then complete the normal phone-code login. BotFather tokens are not used by the current application." - }, - { - q: "Is Two-Step Verification (2FA) supported?", - a: "Yes. When logging in for the first time, TeleDrive prompts you in the desktop app for your phone code and, if enabled, your 2FA cloud password securely." - } -]; ---- - -
-
-
- Frequently Asked Questions -

Everything You Need to Know

-

- Detailed answers on security, storage locations, account limits, and platform compatibility. -

-
- -
- {faqs.map((item, idx) => ( -
- - {item.q} - - - - -
-

{item.a}

-
-
- ))} -
-
-
- - diff --git a/website/src/components/Features.astro b/website/src/components/Features.astro deleted file mode 100644 index a86c0014..00000000 --- a/website/src/components/Features.astro +++ /dev/null @@ -1,163 +0,0 @@ ---- ---- - -
-
-
- Core Architecture -

Built for Safety, Built for Scale

-

- Unlike generic upload scripts that get accounts restricted, TeleDrive was engineered from day one to treat Telegram API limits with extreme care. -

-
- -
- -
-
- -
-

Adaptive Flood-Wait Shield

-

- When Telegram returns FLOOD_WAIT_X, TeleDrive honors the exact cooldown plus an extra configurable safety buffer (e.g. +30s) instead of failing or hammering the server. -

-
Dynamic Server Backoff
-
- - -
-
- -
-

Resumable SQLite Engine

-

- All scanned files, queue states, and batch audit logs are committed to an ACID-compliant local SQLite database. Restart your PC anytime without losing progress. -

-
Zero State Loss
-
- - -
-
- -
-

Randomized Jitter Delays

-

- Avoid robotic request signatures. Configurable minimum and maximum intervals (e.g., 20–60 sec) ensure uploads mimic natural human pacing. -

-
Anti-Bot Signature
-
- - -
-
- -
-

Direct to Saved Messages

-

- Upload directly to your private "Saved Messages" or a private backup channel. No third-party relay servers, no telemetry, and no middleware subscriptions. -

-
100% Peer-to-Peer
-
- - -
-
- -
-

Zero Dependency Binary

-

- Available as a standalone Windows .exe installer and portable Linux x86_64 bundle. No Python setup, pip packages, or virtual environments needed. -

-
Self-Contained Executable
-
- - -
-
- -
-

Automatic Loop & Scheduler

-

- Set TeleDrive on autopilot with hourly or daily batch intervals. Single-process locking ensures multiple instances never clash or overlap. -

-
Background Daemon
-
-
-
-
- - diff --git a/website/src/components/Footer.astro b/website/src/components/Footer.astro deleted file mode 100644 index fa3c35f4..00000000 --- a/website/src/components/Footer.astro +++ /dev/null @@ -1,204 +0,0 @@ ---- ---- - - - - diff --git a/website/src/components/Hero.astro b/website/src/components/Hero.astro deleted file mode 100644 index 704f61d3..00000000 --- a/website/src/components/Hero.astro +++ /dev/null @@ -1,563 +0,0 @@ ---- ---- - -
-
- -
-
- - TeleDrive v1.0.6 is Live • Windows & Linux -
-
- - -

- Turn Telegram into your
- Private, Unlimited Cloud -

- - -

- A safe, resumable, and autonomous personal file uploader for Windows and Linux. - Gradually scans local folders, queues files in SQLite, and uploads directly to your private Saved Messages or Channel with intelligent flood-wait protection. -

- - - - - -
-
-
- - - -
-
- - -
-
⚡ Instant One-Liner
-
- -
-
- C:\> -
- mkdir "%TEMP%\TeleDrive" 2>nul & curl.exe -fL --progress-bar -o "%TEMP%\TeleDrive\TeleDrive-Setup-1.0.6.exe" "https://github.com/amjadlle/TeleDrive/releases/download/v1.0.6/TeleDrive-Setup-1.0.6.exe" & if errorlevel 1 (echo Download failed. & exit /b 1) else (start "" "%TEMP%\TeleDrive\TeleDrive-Setup-1.0.6.exe") -
-
- - - -
-
- - -
-
-
- -
-
- -
-
TeleDrive Desktop — Production Client
-
- - Uploader Engine Active -
-
-
-
- - Click to view details -
- TeleDrive Desktop Dashboard -
- -
-
- - -
-
- - Zero Third-Party Cloud -
-
- - No Python Required (.exe) -
-
- - Automated Loop Scheduling -
-
- - MIT Open Source -
-
-
-
- - - - diff --git a/website/src/components/InstallGuide.astro b/website/src/components/InstallGuide.astro deleted file mode 100644 index 09a691d2..00000000 --- a/website/src/components/InstallGuide.astro +++ /dev/null @@ -1,111 +0,0 @@ ---- -const steps = [ - { - step: "01", - title: "Obtain Telegram API Keys", - desc: "Log in to my.telegram.org and create an application under API Development Tools. Copy the api_id and api_hash. A BotFather token is not needed.", - code: "https://my.telegram.org/apps" - }, - { - step: "02", - title: "Configure Target & Timing", - desc: "In Settings, enter your keys and choose me, a public @username, or a private channel/group ID. Start with Saved Messages for a safe test.", - code: "Target: me / @channel / -100..." - }, - { - step: "03", - title: "Recursive Folder Scan", - desc: "Click 'Scan Folder'. TeleDrive recursively indexes all files into its local SQLite database, generating deduplicated file hashes.", - code: "SQLite queue indexed instantly" - }, - { - step: "04", - title: "Launch Autonomous Backup", - desc: "Click 'Run Batch' or toggle the 'Loop Scheduler'. TeleDrive uploads file-by-file with randomized delays, automatically logging every event.", - code: "Auto-resumes across reboots" - } -]; ---- - -
-
-
- Quick Start -

Up and Running in 4 Steps

-

- No technical configuration hurdles. Get your personal automated cloud backup running in minutes. -

-
- -
- {steps.map((item) => ( -
-
{item.step}
-

{item.title}

-

{item.desc}

-
- {item.code} -
-
- ))} -
-
-
- - diff --git a/website/src/components/Navbar.astro b/website/src/components/Navbar.astro deleted file mode 100644 index 140db0e1..00000000 --- a/website/src/components/Navbar.astro +++ /dev/null @@ -1,288 +0,0 @@ ---- ---- - -
- -
- - diff --git a/website/src/components/SettingsGuide.astro b/website/src/components/SettingsGuide.astro deleted file mode 100644 index 4514c875..00000000 --- a/website/src/components/SettingsGuide.astro +++ /dev/null @@ -1,219 +0,0 @@ ---- -const settings = [ - { - name: "Files per run", - value: "20", - desc: "Maximum files processed in one continuous execution batch.", - tag: "Batch Control" - }, - { - name: "Files per day", - value: "500", - desc: "Hard daily cap across all automatic and manual batches combined.", - tag: "Daily Quota" - }, - { - name: "Delay between uploads (min / max)", - value: "20s – 60s", - desc: "TeleDrive selects a random wait time within this window after each file.", - tag: "Jitter Interval" - }, - { - name: "Delay between automatic runs", - value: "60 min", - desc: "Rest period before the scheduler initiates the next queue scan and upload.", - tag: "Scheduler" - }, - { - name: "Retry attempts per file", - value: "3", - desc: "Number of automated retry attempts after transient network disconnects.", - tag: "Resilience" - }, - { - name: "Retry backoff base", - value: "30 sec", - desc: "Exponential delay scale (30s, 60s, 120s) between failed retries.", - tag: "Backoff" - }, - { - name: "Flood-wait buffer", - value: "+30 sec", - desc: "Extra cushion added to server-mandated cooldowns to guarantee safety.", - tag: "Protection" - }, - { - name: "Send files as", - value: "Document", - desc: "Preserves raw bit-for-bit file integrity without Telegram compression.", - tag: "Storage Format" - } -]; ---- - -
-
-
- Reference Guide -

Recommended Safety Presets

-

- Field-tested default configurations crafted for private personal backup archives without triggering Telegram spam filters. -

-
- -
- - - - - - - - - - - {settings.map((item) => ( - - - - - - - ))} - -
Setting NameRecommended ValueOperational PurposeType
- {item.name} - - {item.value} - {item.desc} - {item.tag} -
-
- -
-
- -
-
-

Why Telegram limits matter

-

- Telegram dynamically enforces sliding-window rate limits based on account age and destination. Always keep one TeleDrive process running at a time, upload directly to your private me (Saved Messages) or private channel, and avoid using the same phone number for bulk chat messaging while an archive batch is active. -

-
-
-
-
- - diff --git a/website/src/components/Showcase.astro b/website/src/components/Showcase.astro deleted file mode 100644 index 933d3218..00000000 --- a/website/src/components/Showcase.astro +++ /dev/null @@ -1,460 +0,0 @@ ---- ---- - -
-
-
- Visual Walkthrough -

Engineered for Complete Transparency

-

- Explore the PySide6 desktop client interface. Click any screenshot below to inspect high-resolution details in full view. -

-
- - -
- - - - - -
- - -
- -
-
- -
-
- - Click to view in Full HD -
- TeleDrive Dashboard Screenshot -
- - -
-
-
Batch Progress
-
1 uploaded • 4 remaining
-
Real-time percentage progress bar
-
-
-
One-Click Actions
-
Run Batch • Scan • Loop
-
Immediate control buttons
-
-
-
Live Activity Audit
-
[INFO] Scan complete
-
Color-coded terminal log console
-
-
-
-
- - - - - - -
-
- - - -
- - - - diff --git a/website/src/components/TimingSimulator.astro b/website/src/components/TimingSimulator.astro deleted file mode 100644 index 3899001a..00000000 --- a/website/src/components/TimingSimulator.astro +++ /dev/null @@ -1,466 +0,0 @@ ---- ---- - -
-
-
- Interactive Calculator -

Timing & Safety Simulator

-

- Calculate your backup throughput and simulate Telegram rate-limit safety scores before running your first batch. -

-
- -
-
- Quick Presets: - - - -
- -
- -
-
-
- - 20 files -
- -
Conservative starting point: 10–25 files/run
-
- -
-
- - 40 sec -
- -
Randomized jitter window (20–60s)
-
- -
-
- - 60 min -
- -
Wait time before TeleDrive begins the next batch
-
-
- - -
-
-
- 🛡️ High Safety - 98 / 100 -
- -
-
-
-

- Excellent. Natural upload jitter prevents Telegram flood-waits and protects your session. -

-
- -
-
-
~13 mins
-
Single Batch Duration
-
- -
-
~390 files
-
Daily 24h Capacity
-
- -
-
~16 files/hr
-
Sustained Pace
-
- -
-
Low Risk
-
Flood-Wait Probability
-
-
- -
- 💡 TeleDrive Tip: Uploading directly to Saved Messages has the highest rate tolerance compared to public groups. -
-
-
-
-
-
- - - - diff --git a/website/src/layouts/Layout.astro b/website/src/layouts/Layout.astro deleted file mode 100644 index 5af5f19c..00000000 --- a/website/src/layouts/Layout.astro +++ /dev/null @@ -1,86 +0,0 @@ ---- -import '../styles/global.css'; - -interface Props { - title?: string; - description?: string; -} - -const { - title = "TeleDrive — Safe, Resumable Telegram Cloud Backup", - description = "Turn Telegram into your personal, safe, and unlimited cloud backup. Standalone desktop app with SQLite queue, flood-wait protection, and automated scheduling for Windows & Linux.", -} = Astro.props; - -const canonicalUrl = "https://teledrive.mapki.in"; -const socialImage = "https://teledrive.mapki.in/og-image.png"; ---- - - - - - - - - - - - - {title} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/website/src/pages/index.astro b/website/src/pages/index.astro deleted file mode 100644 index 6033af46..00000000 --- a/website/src/pages/index.astro +++ /dev/null @@ -1,28 +0,0 @@ ---- -import Layout from '../layouts/Layout.astro'; -import Navbar from '../components/Navbar.astro'; -import Hero from '../components/Hero.astro'; -import Showcase from '../components/Showcase.astro'; -import Features from '../components/Features.astro'; -import TimingSimulator from '../components/TimingSimulator.astro'; -import SettingsGuide from '../components/SettingsGuide.astro'; -import InstallGuide from '../components/InstallGuide.astro'; -import Author from '../components/Author.astro'; -import Faq from '../components/Faq.astro'; -import Footer from '../components/Footer.astro'; ---- - - - -
- - - - - - - - -
-