diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..5246f89f1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,71 @@ +# Git +.git +.gitignore +.gitattributes + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +env/ +venv/ +ENV/ +.venv +*.egg-info/ +dist/ +build/ +*.egg + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Docker (don't copy docker files into the image) +Dockerfile +docker-compose*.yml +.dockerignore + +# Database lock files +*.db +*.sqlite +*.sqlite3 +.db_initialized + +# Logs +logs/ +*.log + +# Test files +.coverage +htmlcov/ +.pytest_cache/ +.tox/ + +# CI/CD +.github/ +.travis.yml +.circleci/ + +# Temporary files +tmp/ +temp/ +*.tmp + +# Secrets and credentials (prevent accidental inclusion) +*.pem +*.key +service-account.json +gcp-key.json +secret_key +secret_csrf +config.py +.env diff --git a/.gitattributes b/.gitattributes index 0b2229721..1e62db730 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ -*.bat text eol=crlf \ No newline at end of file +*.bat text eol=crlf +*.sh text eol=lf \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 906401e51..e27484631 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -10,3 +10,11 @@ updates: reviewers: - canihavesomecoffee - thealphadollar +- package-ecosystem: npm + directory: "/web" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + minor-and-patch: + update-types: ["minor", "patch"] diff --git a/.github/workflows/ci-vm-scripts.yml b/.github/workflows/ci-vm-scripts.yml new file mode 100644 index 000000000..128e9ee7c --- /dev/null +++ b/.github/workflows/ci-vm-scripts.yml @@ -0,0 +1,37 @@ +name: CI-VM scripts + +# Static analysis for the CI-VM runner scripts (the scripts that run on the +# throwaway test VMs). Runs only when those scripts change. shellcheck lints the +# bash runners; PSScriptAnalyzer lints the PowerShell startup script. + +on: + pull_request: + paths: + - 'install/ci-vm/**' + - '.github/workflows/ci-vm-scripts.yml' + push: + paths: + - 'install/ci-vm/**' + - '.github/workflows/ci-vm-scripts.yml' + +jobs: + shellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install shellcheck + run: sudo apt-get update && sudo apt-get install -y shellcheck + - name: shellcheck (bash runner scripts) + run: shellcheck --severity=error install/ci-vm/ci-linux/ci/runCI install/ci-vm/ci-linux/startup-script.sh + + psscriptanalyzer: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + - name: PSScriptAnalyzer (startup-script.ps1) + shell: pwsh + run: | + Set-PSRepository PSGallery -InstallationPolicy Trusted + Install-Module -Name PSScriptAnalyzer -Force -Scope CurrentUser + $issues = Invoke-ScriptAnalyzer -Path install/ci-vm/ci-windows/startup-script.ps1 -Severity Error + if ($issues) { $issues | Format-Table -AutoSize; exit 1 } diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e70be681e..da599c03d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,19 +15,25 @@ jobs: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9] + python-version: ['3.12', '3.13', '3.14', '3.15'] + include: + - python-version: '3.15' + experimental: true + + continue-on-error: ${{ matrix.experimental || false }} steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | sudo apt-get update - sudo apt-get install libvirt-dev + sudo apt-get install libvirt-dev libxml2-dev libxslt-dev python -m pip install --upgrade pip if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi if [ -f requirements.txt ]; then pip install -r requirements.txt; fi @@ -48,14 +54,4 @@ jobs: pycodestyle ./ --config=./.pycodestylerc - name: Test with nose run: | - nosetests --nocapture --with-cov --cov-config .coveragerc - - name: Temporary fix for codecov - run: | - coverage xml -i - - name: Upload to codecov - uses: codecov/codecov-action@v2 - with: - flags: unittests - name: sample-platform - fail_ci_if_error: true - files: ./coverage.xml + nose2 diff --git a/.github/workflows/sp-deployment-pipeline.yml b/.github/workflows/sp-deployment-pipeline.yml new file mode 100644 index 000000000..df04668f1 --- /dev/null +++ b/.github/workflows/sp-deployment-pipeline.yml @@ -0,0 +1,218 @@ +name: Deploy sample platform + +on: + workflow_dispatch: + workflow_run: + workflows: [ "Run tests and code checks" ] + types: [ completed ] + branches: + - "master" + +env: + DEPLOY_BRANCH: master + INSTALL_FOLDER: /var/www/sample-platform + SAMPLE_REPOSITORY: /repository + +jobs: + deploy: + runs-on: ubuntu-latest + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + permissions: + id-token: write + contents: read + + steps: + - name: Pre-deployment checks + uses: appleboy/ssh-action@823bd89e131d8d508129f9443cad5855e9ba96f0 # v1.2.4 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + command_timeout: 2m + envs: INSTALL_FOLDER,SAMPLE_REPOSITORY,DEPLOY_BRANCH + script: | + set -e + echo "=== Pre-deployment checks ===" + cd $INSTALL_FOLDER + + # Check if deployment scripts exist (for backwards compatibility) + if [ -f "install/deploy/pre_deploy.sh" ]; then + sudo INSTALL_FOLDER="$INSTALL_FOLDER" \ + SAMPLE_REPOSITORY="$SAMPLE_REPOSITORY" \ + DEPLOY_BRANCH="$DEPLOY_BRANCH" \ + bash install/deploy/pre_deploy.sh + else + echo "Deployment scripts not found, using legacy validation" + # Basic validation + test -f config.py || { echo "ERROR: config.py not found"; exit 1; } + # Save current commit for potential manual rollback + git rev-parse HEAD > /tmp/previous_commit.txt + echo "Current commit saved: $(cat /tmp/previous_commit.txt)" + fi + + - name: Deploy application + uses: appleboy/ssh-action@823bd89e131d8d508129f9443cad5855e9ba96f0 # v1.2.4 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + command_timeout: 10m + envs: INSTALL_FOLDER,SAMPLE_REPOSITORY,DEPLOY_BRANCH + script: | + set -e + echo "=== Deploying application ===" + cd $INSTALL_FOLDER + + # Check if deployment scripts exist + if [ -f "install/deploy/deploy.sh" ]; then + sudo INSTALL_FOLDER="$INSTALL_FOLDER" \ + SAMPLE_REPOSITORY="$SAMPLE_REPOSITORY" \ + DEPLOY_BRANCH="$DEPLOY_BRANCH" \ + bash install/deploy/deploy.sh + else + echo "Using legacy deployment" + # Legacy deployment (will be removed after scripts are merged) + sudo git restore . + sudo git checkout $DEPLOY_BRANCH + sudo git fetch origin $DEPLOY_BRANCH + sudo git reset --hard origin/$DEPLOY_BRANCH + sudo git clean -f -d + sudo git pull origin $DEPLOY_BRANCH + + sudo python3 -m pip install -r requirements.txt --break-system-packages + sudo FLASK_APP=./run.py flask db upgrade + + sudo cp "install/ci-vm/ci-linux/ci/bootstrap" "${SAMPLE_REPOSITORY}/TestData/ci-linux/bootstrap" 2>/dev/null || true + sudo cp "install/ci-vm/ci-linux/ci/runCI" "${SAMPLE_REPOSITORY}/TestData/ci-linux/runCI" 2>/dev/null || true + sudo cp "install/ci-vm/ci-windows/ci/runCI.bat" "${SAMPLE_REPOSITORY}/TestData/ci-windows/runCI.bat" 2>/dev/null || true + + sudo systemctl reload platform + fi + + # The console is a static build, so it is compiled here and only the + # output is shipped; the VM needs no Node toolchain. This runs after + # the code deploy so the target directory exists. + - name: Build web console + uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/package-lock.json + + - name: Compile console + working-directory: web + run: | + npm ci --ignore-scripts + npm run build:app + + - name: Ship console + uses: appleboy/scp-action@917f8b81dfc1ccd331fef9e2d61bdc6c8be94634 # v0.1.7 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + # Lands at /web/app/, which is what the nginx + # root expects: the directory name has to match the /app/ URL. + source: "web/dist/**" + target: ${{ env.INSTALL_FOLDER }}/web/app + strip_components: 2 + overwrite: true + + - name: Verify deployment + id: health_check + uses: appleboy/ssh-action@823bd89e131d8d508129f9443cad5855e9ba96f0 # v1.2.4 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + command_timeout: 2m + envs: INSTALL_FOLDER + script: | + echo "=== Verifying deployment ===" + cd $INSTALL_FOLDER + + # Check if deployment scripts exist + if [ -f "install/deploy/post_deploy.sh" ]; then + sudo INSTALL_FOLDER="$INSTALL_FOLDER" bash install/deploy/post_deploy.sh + else + echo "Using legacy health check" + # Legacy health check - just verify service is running + sleep 5 + if systemctl is-active --quiet platform; then + echo "Platform service is running" + # Try to hit the homepage + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/ 2>/dev/null || echo "000") + if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 400 ]; then + echo "Homepage responding with HTTP $HTTP_CODE" + exit 0 + else + echo "ERROR: Homepage returned HTTP $HTTP_CODE" + exit 1 + fi + else + echo "ERROR: Platform service is not running" + systemctl status platform || true + exit 1 + fi + fi + + - name: Rollback on failure + if: failure() && steps.health_check.outcome == 'failure' + uses: appleboy/ssh-action@823bd89e131d8d508129f9443cad5855e9ba96f0 # v1.2.4 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + command_timeout: 5m + envs: INSTALL_FOLDER,SAMPLE_REPOSITORY + script: | + echo "=== ROLLBACK INITIATED ===" + cd $INSTALL_FOLDER + + # Check if deployment scripts exist + if [ -f "install/deploy/rollback.sh" ]; then + sudo INSTALL_FOLDER="$INSTALL_FOLDER" \ + SAMPLE_REPOSITORY="$SAMPLE_REPOSITORY" \ + bash install/deploy/rollback.sh + else + echo "Using legacy rollback" + # Legacy rollback + if [ -f "/tmp/previous_commit.txt" ]; then + PREV_COMMIT=$(cat /tmp/previous_commit.txt) + echo "Rolling back to commit: $PREV_COMMIT" + sudo git checkout "$PREV_COMMIT" + sudo python3 -m pip install -r requirements.txt --break-system-packages + sudo systemctl reload platform + echo "Rollback complete" + else + echo "ERROR: No previous commit saved, cannot rollback" + echo "MANUAL INTERVENTION REQUIRED" + fi + fi + + - name: Report deployment status + if: always() + uses: appleboy/ssh-action@823bd89e131d8d508129f9443cad5855e9ba96f0 # v1.2.4 + with: + host: ${{ vars.PLATFORM_DOMAIN }} + username: ${{ vars.SSH_USER }} + key: ${{ secrets.SSH_KEY_PRIVATE }} + port: 22 + command_timeout: 30s + envs: INSTALL_FOLDER + script: | + echo "=== Deployment Summary ===" + cd $INSTALL_FOLDER + echo "Current commit: $(git rev-parse HEAD)" + echo "Branch: $(git branch --show-current)" + echo "Service status: $(systemctl is-active platform 2>/dev/null || echo 'unknown')" + + # Cleanup lock file if it exists + rm -f /tmp/sp-deploy.lock diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml new file mode 100644 index 000000000..1e9970ca3 --- /dev/null +++ b/.github/workflows/web.yml @@ -0,0 +1,34 @@ +name: Web console + +# Only runs when the console changes; the Python checks in main.yml are +# unaffected by anything under web/. +on: + push: + branches: [master] + paths: ['web/**', '.github/workflows/web.yml'] + pull_request: + paths: ['web/**', '.github/workflows/web.yml'] + +defaults: + run: + working-directory: web + +jobs: + web: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + cache: npm + cache-dependency-path: web/package-lock.json + # --ignore-scripts: nothing here needs a lifecycle script, and not + # running them stops a compromised dependency executing during + # install. + - run: npm ci --ignore-scripts + - run: npm run lint + # The local binary, not npx, which would fetch on demand. + - run: ./node_modules/.bin/tsc -b + - run: npm run build:app + - run: npm audit --audit-level=high diff --git a/.gitignore b/.gitignore index 8722e2714..4f8ca2d8d 100644 --- a/.gitignore +++ b/.gitignore @@ -17,16 +17,19 @@ instance/ # Log files install/PlatformInstall_* -logs/*.log +logs/*.log* # Sensitive information secret_csrf secret_key config.py -install/ci-vm/ci-linux/ci/variables +parse.py +service-account.json +install/ci-vm/ci-windows/rclone.conf -# Build commit -build_commit.py +# Build svgs +static/img/status/build-linux.svg +static/img/status/build-windows.svg # Gunicorn gunicorn.pid @@ -37,6 +40,7 @@ gunicorn.pid # Coverage data coverage_report/ .coverage +coverage.xml # virtualenv venv/ @@ -46,3 +50,6 @@ venv/ monkeytype.sqlite3 .pytype/ .env + +# Test related data +temp/ diff --git a/.pycodestylerc b/.pycodestylerc index f438ba4a6..162bcd630 100644 --- a/.pycodestylerc +++ b/.pycodestylerc @@ -1,5 +1,5 @@ [pycodestyle] count = True max-line-length = 120 -exclude=TestDiff.py,migrations +exclude=test_diff.py,migrations,venv*,.venv*,parse.py,config.py ignore = E701 diff --git a/DOCKER.md b/DOCKER.md new file mode 100644 index 000000000..61eb35750 --- /dev/null +++ b/DOCKER.md @@ -0,0 +1,82 @@ +# Running the Sample Platform with Docker + +A two-container development stack: MySQL 8 and the Flask application served by +Gunicorn. It gives contributors a working platform without installing MySQL, +Python, or the native libraries the app depends on. + +## Prerequisites + +- Docker Engine 24+ with the Compose plugin (`docker compose`). + +## Quick start (empty database) + +```sh +cp env.example .env # then edit the passwords +docker compose up --build +``` + +On first start the database initialises, the app waits for it, builds the +schema, loads the bundled fixture set (categories, samples, regression tests) +and serves on . Later starts only apply migrations +newer than the database, and never re-seed. + +Create an administrator so you can sign in: + +```sh +docker compose exec backend \ + python install/init_db.py "$SQLALCHEMY_DATABASE_URI" admin admin@example.com admin +``` + +## Starting from a database dump + +To develop against real data, load a `mysqldump` on the database's **first** +start by dropping it into the init directory. Create `docker-compose.override.yml`: + +```yaml +services: + db: + volumes: + - /absolute/path/to/dump.sql:/docker-entrypoint-initdb.d/01-dump.sql:ro +``` + +Then `docker compose up --build`. MySQL imports the dump before the app starts; +the app applies any migrations newer than the dump on top of it. The import +only runs while the `db_data` volume is empty — `docker compose down -v` first +to reload a different dump. + +## Live reload + +The image is self-contained (code is copied in, not mounted). For an +edit-refresh loop, mount the package you are working on and enable Gunicorn's +reloader in the override file: + +```yaml +services: + backend: + environment: + GUNICORN_RELOAD: "1" + volumes: + - ./mod_sample:/app/mod_sample + - ./templates:/app/templates +``` + +## Common commands + +| Task | Command | +|---|---| +| Start | `docker compose up --build` | +| Stop | `docker compose down` | +| Reset the database | `docker compose down -v` | +| App logs | `docker compose logs -f backend` | +| A shell in the app | `docker compose exec backend bash` | +| A MySQL shell | `docker compose exec db mysql -u root -p` | + +## Notes + +- The database port is not published to the host. Reach MySQL through + `docker compose exec`, or add a `ports` mapping in an override if you need a + local client. +- The image generates throwaway secret keys and GCP credentials at build time + so the app can boot offline. They are not suitable for production. +- Storage falls back to the local `/repository` volume; no GCS bucket is + required for development. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..77fe8d74b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,72 @@ +FROM python:3.12-slim-bookworm + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + FLASK_APP=run.py + +# git for the build commit run.py reads, libmagic for upload sniffing, +# mediainfo for sample metadata. No compiler: everything has a wheel. +RUN apt-get update && apt-get install -y --no-install-recommends \ + git libmagic1 mediainfo \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +# Dependencies first so application edits don't invalidate this layer. +# Pinned, and wheels only, so nothing runs a setup script during the build. +# cryptography is used by the credential generator further down. +COPY requirements.txt . +RUN pip install --only-binary :all: --upgrade \ + pip==26.2 setuptools==83.0.0 wheel==0.47.0 \ + && pip install --only-binary :all: \ + cryptography==50.0.0 gunicorn==26.0.0 \ + && pip install --only-binary :all: -r requirements.txt + +# Listed by name instead of "COPY . ." so a secret_key, config.py or +# service-account.json left in a working tree can't end up in the image. +COPY run.py config_parser.py database.py decorators.py exceptions.py \ + log_configuration.py mailer.py utility.py ./ +COPY mod_api/ ./mod_api/ +COPY mod_auth/ ./mod_auth/ +COPY mod_ci/ ./mod_ci/ +COPY mod_customized/ ./mod_customized/ +COPY mod_health/ ./mod_health/ +COPY mod_home/ ./mod_home/ +COPY mod_regression/ ./mod_regression/ +COPY mod_sample/ ./mod_sample/ +COPY mod_test/ ./mod_test/ +COPY mod_upload/ ./mod_upload/ +COPY install/ ./install/ +COPY migrations/ ./migrations/ +COPY static/ ./static/ +COPY templates/ ./templates/ + +# The app reads its settings from config.py; this variant takes them from +# the environment. +COPY config.docker.py config.py + +# Done once here instead of on every start: unprivileged user, secret keys, +# throwaway GCP credentials, the repository tree (named volumes inherit this +# layout on first mount) and a git repo so run.py can resolve a commit. +RUN useradd --create-home --uid 1001 appuser \ + && python install/generate_dev_credentials.py \ + && head -c 32 /dev/urandom > secret_key \ + && head -c 32 /dev/urandom > secret_csrf \ + && mkdir -p logs \ + /repository/ci-tests /repository/unsafe-ccextractor /repository/TempFiles \ + /repository/LogFiles /repository/TestResults /repository/TestFiles/media \ + /repository/QueuedFiles /repository/TestData/ci-linux \ + /repository/TestData/ci-windows /repository/vm_data \ + && git init -q . \ + && git -c user.email=dev@local -c user.name=docker add -A \ + && git -c user.email=dev@local -c user.name=docker commit -qm "container image" \ + && chown -R appuser:appuser /app /repository + +COPY docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh +RUN sed -i 's/\r$//' /usr/local/bin/docker-entrypoint.sh \ + && chmod +x /usr/local/bin/docker-entrypoint.sh + +USER appuser +EXPOSE 5000 +ENTRYPOINT ["docker-entrypoint.sh"] diff --git a/README.md b/README.md index aec8c2da1..6849f237a 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ This repository contains the code for a platform that manages a test suite bot, sample upload and more. This platform allows for a unified place to report errors, submit samples, view existing samples and more. It was -originally developed during GSoC 2015 and rewritten during the 2016 edition. It was further improved and worked upon during GSoC 2017, 2018 and 2019. +originally developed during GSoC 2015 and rewritten during the 2016 edition. It was further improved and worked upon during GSoC 2017, 2018, 2019 and 2022. To see the live version of the platform, simply go to [CCExtractor Submission Platform](https://sampleplatform.ccextractor.org/). @@ -32,7 +32,7 @@ regression tests useless. In these cases, the tests can give false negative. For result files by the following command. ```shell -python manage.py update /path/to/ccextractor/executable +python3 manage.py update /path/to/ccextractor/executable ``` NOTE: Please backup old results and be sure about updating them as there is no method to go back. @@ -95,11 +95,26 @@ For creating a virtual environment, we use [virtualenv](https://pypi.org/project ```bash virtualenv venv # create a virtual environment source venv/bin/activate # activate the virtual environment -pip install -r requirements.txt # install dependencies -pip install -r test-requirements.txt # install test dependencies -TESTING=True nosetests --with-cov --cov-config .coveragerc +pip install -r requirements.txt # install dependencies +pip install -r test-requirements.txt # install test dependencies +TESTING=True nose2 ``` +## Migrating platform between machines + +In case you want to replicate/migrate a platform instance with all the data, samples, regression tests.etc., follow the following steps: +- Install platform on the new instance, using the [installation guide](install/installation.md). +- Now transfer the contents of the previous GCS bucket to the new GCS bucket and export the SQL database of the previous platform instance into a file using the following command: + ``` + mysqldump -u PLATFORM_USER_USERNAME -p PLATFORM_DATABASE_NAME > sample_platform.sql + ``` + PLATFORM_USER_USERNAME and PLATFORM_DATABASE_NAME values are details for the SQL database of the previous platform instance. +- Now import the database using the `sample_platform.sql` file into the new instance using the following command: + ``` + mysql -u NEW_PLATFORM_USER_USERNAME -p NEW_PLATFORM_DATABASE_NAME < sample_platform.sql + ``` + NEW_PLATFORM_USER_USERNAME and NEW_PLATFORM_DATABASE_NAME values are details for the SQL database of the new platform instance. + ## Etiquettes We follow certain etiquettes which include docstrings, annotation, import sorting etc. diff --git a/bootstrap_gunicorn.py b/bootstrap_gunicorn.py index f38959ee0..30fd9d4f3 100644 --- a/bootstrap_gunicorn.py +++ b/bootstrap_gunicorn.py @@ -4,13 +4,14 @@ from os import path current_dir = path.dirname(path.abspath(__file__)) +TIMEOUT = 120 # In seconds -# Arguments to start gunicorn args = [ - "gunicorn", "-w", "4", "--daemon", "--pid", "gunicorn.pid", "-b", "unix:sampleplatform.sock", "-m", "007", + "gunicorn", "-w", "4", "-b", "unix:sampleplatform.sock", "-m", "007", "-g", "www-data", "-u", "www-data", f"--chdir={current_dir}", "--log-level", "debug", + "--timeout", f"{TIMEOUT}", "--access-logfile", f"{current_dir}/logs/access.log", "--capture-output", "--log-file", f"{current_dir}/logs/error.log", "run:app" ] -subprocess.Popen(args) +subprocess.run(args) diff --git a/config.docker.py b/config.docker.py new file mode 100644 index 000000000..0feb840cc --- /dev/null +++ b/config.docker.py @@ -0,0 +1,60 @@ +"""Container configuration for the Sample Platform. + +Copied to ``config.py`` inside the image at build time. Every value is read +from the environment (see ``env.example``) with local-development defaults, +so the repository never carries a real secret. Override every placeholder +before pointing this at anything but a throwaway database. +""" +import os + + +def _int(name: str, default: int) -> int: + try: + return int(os.environ[name]) + except (KeyError, ValueError): + return default + + +APPLICATION_ROOT = None +CSRF_ENABLED = True + +DATABASE_URI = os.environ.get( + "SQLALCHEMY_DATABASE_URI", + "mysql+pymysql://sample_platform:sample_platform@db/sample_platform?charset=utf8mb4", +) +SERVER_NAME = os.environ.get("SERVER_NAME", "localhost:5000") +SESSION_COOKIE_PATH = "/" + +INSTALL_FOLDER = os.environ.get("INSTALL_FOLDER", "/app") +SAMPLE_REPOSITORY = os.environ.get("SAMPLE_REPOSITORY", "/repository") + +HMAC_KEY = os.environ.get("HMAC_KEY", "dev-hmac-key") +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "") +GITHUB_OWNER = os.environ.get("GITHUB_OWNER", "CCExtractor") +GITHUB_REPOSITORY = os.environ.get("GITHUB_REPOSITORY", "ccextractor") +GITHUB_CI_KEY = os.environ.get("GITHUB_CI_KEY", "") +GITHUB_CLIENT_ID = os.environ.get("GITHUB_CLIENT_ID", "") +GITHUB_CLIENT_KEY = os.environ.get("GITHUB_CLIENT_KEY", "") + +EMAIL_DOMAIN = os.environ.get("EMAIL_DOMAIN", "") +EMAIL_API_KEY = os.environ.get("EMAIL_API_KEY", "") + +FTP_PORT = _int("FTP_PORT", 21) +MAX_CONTENT_LENGTH = 512 * 1024 * 1024 +MIN_PWD_LEN = 10 +MAX_PWD_LEN = 500 + +# GCP / Cloud Storage. The build generates a throwaway service account so the +# storage client can initialise offline; file serving falls back to local disk. +SCOPES = ["https://www.googleapis.com/auth/cloud-platform"] +SERVICE_ACCOUNT_FILE = os.environ.get("SERVICE_ACCOUNT_FILE", "service-account.json") +ZONE = "us-west4-b" +PROJECT_NAME = "ccextractor-sampleplatform" +MACHINE_TYPE = f"zones/{ZONE}/machineTypes/n1-standard-1" +WINDOWS_INSTANCE_PROJECT_NAME = "windows-cloud" +WINDOWS_INSTANCE_FAMILY_NAME = "windows-2019" +LINUX_INSTANCE_PROJECT_NAME = "ubuntu-os-cloud" +LINUX_INSTANCE_FAMILY_NAME = "ubuntu-minimal-2404-lts-amd64" +GCP_INSTANCE_MAX_RUNTIME = 120 +GCS_BUCKET_NAME = os.environ.get("GCS_BUCKET_NAME", "sample-platform-dev") +GCS_SIGNED_URL_EXPIRY_LIMIT = 720 diff --git a/config_sample.py b/config_sample.py index 58e0dd65d..99f609cc1 100755 --- a/config_sample.py +++ b/config_sample.py @@ -5,7 +5,6 @@ APPLICATION_ROOT = None CSRF_ENABLED = True DATABASE_URI = 'mysql+pymysql://root:@localhost:3306/test?charset=utf8' -GITHUB_BOT = '' GITHUB_TOKEN = '' GITHUB_OWNER = 'CCExtractor' GITHUB_REPOSITORY = 'ccextractor' @@ -13,17 +12,31 @@ EMAIL_DOMAIN = '' EMAIL_API_KEY = '' HMAC_KEY = '' -GITHUB_DEPLOY_KEY = '' GITHUB_CI_KEY = '' GITHUB_CLIENT_ID = '' -GITHUB_CLIENT_SECRET = '' +GITHUB_CLIENT_KEY = '' INSTALL_FOLDER = '/path/to/installation' -KVM_LINUX_NAME = '' -KVM_WINDOWS_NAME = '' -KVM_MAX_RUNTIME = 120 # In minutes SAMPLE_REPOSITORY = '/path/to/samples' SESSION_COOKIE_PATH = '/' +# Where the web console is served, if one is. Password reset links point +# there when it is set, and at these pages when it is not. +CONSOLE_URL = '' FTP_PORT = 21 MAX_CONTENT_LENGTH = 512 * 1024 * 1024 MIN_PWD_LEN = 10 MAX_PWD_LEN = 500 + + +# GCP SPECIFIC CONFIG +SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] +SERVICE_ACCOUNT_FILE = 'service-account.json' +ZONE = "us-west4-b" +PROJECT_NAME = "ccextractor-sampleplatform" +MACHINE_TYPE = f"zones/{ZONE}/machineTypes/n1-standard-1" +WINDOWS_INSTANCE_PROJECT_NAME = "windows-cloud" +WINDOWS_INSTANCE_FAMILY_NAME = "windows-2019" +LINUX_INSTANCE_PROJECT_NAME = "ubuntu-os-cloud" +LINUX_INSTANCE_FAMILY_NAME = "ubuntu-minimal-2404-lts-amd64" +GCP_INSTANCE_MAX_RUNTIME = 120 # In minutes +GCS_BUCKET_NAME = 'spdev' +GCS_SIGNED_URL_EXPIRY_LIMIT = 720 # In minutes diff --git a/database.py b/database.py index 204778b41..8248e2d7e 100755 --- a/database.py +++ b/database.py @@ -4,15 +4,15 @@ import re import traceback from abc import ABCMeta -from typing import Any, Dict, Iterator, Tuple, Type, Union +from typing import Any, Dict, Iterator, Optional, Tuple, Type, Union from sqlalchemy import create_engine -from sqlalchemy.dialects.sqlite.pysqlite import SQLiteDialect_pysqlite +from sqlalchemy.engine import Dialect from sqlalchemy.exc import SQLAlchemyError -from sqlalchemy.ext.declarative import DeclarativeMeta, declarative_base -from sqlalchemy.orm import scoped_session, sessionmaker -from sqlalchemy.sql.schema import Column, Table -from sqlalchemy.sql.sqltypes import Enum, SchemaType, TypeDecorator +from sqlalchemy.orm import (DeclarativeBase, DeclarativeMeta, scoped_session, + sessionmaker) +from sqlalchemy.pool import StaticPool +from sqlalchemy.sql.sqltypes import String, TypeDecorator from exceptions import EnumParsingException, FailedToSpawnDBSession @@ -23,7 +23,12 @@ class DeclarativeABCMeta(DeclarativeMeta, ABCMeta): pass -Base = declarative_base(metaclass=DeclarativeMeta) +class Base(DeclarativeBase): + """Base class for all models.""" + + pass + + Base.query = None db_engine = None @@ -43,9 +48,18 @@ def create_session(db_string: str, drop_tables: bool = False) -> scoped_session: global db_engine, Base try: - # In testing, we want to maintain same memory variable - if db_engine is None or 'TESTING' not in os.environ or os.environ['TESTING'] == 'False': - db_engine = create_engine(db_string, convert_unicode=True) + # Only create engine if it doesn't exist + # For SQLite in-memory, we must reuse the engine to share the database + if db_engine is None: + # For SQLite in-memory databases, use StaticPool to share connection + if db_string == 'sqlite:///:memory:': + db_engine = create_engine( + db_string, + connect_args={"check_same_thread": False}, + poolclass=StaticPool + ) + else: + db_engine = create_engine(db_string) db_session = scoped_session(sessionmaker(bind=db_engine)) Base.query = db_session.query_property() @@ -107,7 +121,9 @@ def __init__(self, classname: str, bases: Union[Tuple[Type[DeclEnum]], Tuple[Typ self._reg: Dict self._reg = reg = self._reg.copy() for k, v in dict_.items(): - if isinstance(v, tuple): + # Skip dunder attributes (Python 3.13+ adds __static_attributes__ tuple) + # and validate tuple has exactly 2 elements (value, description) + if isinstance(v, tuple) and not k.startswith('_') and len(v) == 2: sym = reg[v[0]] = EnumSymbol(self, k, *v) setattr(self, k, sym) return type.__init__(self, classname, bases, dict_) @@ -160,30 +176,27 @@ def db_type(cls) -> DeclEnumType: return DeclEnumType(cls) -class DeclEnumType(SchemaType, TypeDecorator): +class DeclEnumType(TypeDecorator): """Declarative enumeration type.""" + cache_ok = True + impl = String(50) + def __init__(self, enum: Any) -> None: self.enum = enum - self.impl = Enum( - *enum.values(), - name="ck{0}".format(re.sub('([A-Z])', lambda m: "_" + m.group(1).lower(), enum.__name__)) - ) - - def _set_table(self, table: Column, column: Table) -> None: - self.impl._set_table(table, column) + super().__init__() - def copy(self) -> DeclEnumType: + def copy(self, **kwargs: Any) -> DeclEnumType: """Get enumeration type of self.""" return DeclEnumType(self.enum) - def process_bind_param(self, value: EnumSymbol, dialect: SQLiteDialect_pysqlite) -> str: + def process_bind_param(self, value: Optional[EnumSymbol], dialect: Dialect) -> Optional[str]: """Get process bind parameter.""" if value is None: return None return value.value - def process_result_value(self, value: str, dialect: SQLiteDialect_pysqlite) -> EnumSymbol: + def process_result_value(self, value: Optional[str], dialect: Dialect) -> Optional[EnumSymbol]: """Get process result value.""" if value is None: return None diff --git a/decorators.py b/decorators.py index 76e454ee1..673b63f03 100644 --- a/decorators.py +++ b/decorators.py @@ -4,6 +4,7 @@ from functools import wraps from typing import Any, Callable, Dict, List, Optional, Union +import git from flask import g, render_template, request from database import EnumSymbol @@ -93,11 +94,7 @@ def decorated_function(*args, **kwargs): ctx['applicationName'] = 'CCExtractor CI platform' ctx['applicationVersion'] = getattr(g, 'version', 'Unknown') ctx['currentYear'] = date.today().strftime('%Y') - try: - from build_commit import build_commit - except ImportError: - build_commit = 'Unknown' - ctx['build_commit'] = build_commit + ctx['build_commit'] = getattr(g, 'build_commit', "Unknown") user = getattr(g, 'user', None) ctx['user'] = user # Create menu entries diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..27d2a05b4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,50 @@ +services: + db: + image: mysql:8.0 + environment: + MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:?set MYSQL_ROOT_PASSWORD in .env} + MYSQL_DATABASE: ${MYSQL_DATABASE:-sample_platform} + MYSQL_USER: ${MYSQL_USER:-sample_platform} + MYSQL_PASSWORD: ${MYSQL_PASSWORD:?set MYSQL_PASSWORD in .env} + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + volumes: + - db_data:/var/lib/mysql + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "127.0.0.1", "-u", "root", "-p${MYSQL_ROOT_PASSWORD}"] + interval: 5s + timeout: 5s + retries: 20 + networks: + - sample_platform + + backend: + build: . + depends_on: + db: + condition: service_healthy + environment: + SQLALCHEMY_DATABASE_URI: mysql+pymysql://${MYSQL_USER:-sample_platform}:${MYSQL_PASSWORD}@db/${MYSQL_DATABASE:-sample_platform}?charset=utf8mb4 + DB_HOST: db + DB_PORT: "3306" + DB_USER: ${MYSQL_USER:-sample_platform} + DB_PASSWORD: ${MYSQL_PASSWORD} + SERVER_NAME: ${SERVER_NAME:-localhost:5000} + HMAC_KEY: ${HMAC_KEY:-dev-hmac-key} + GCS_BUCKET_NAME: ${GCS_BUCKET_NAME:-sample-platform-dev} + GITHUB_OWNER: ${GITHUB_OWNER:-CCExtractor} + GITHUB_REPOSITORY: ${GITHUB_REPOSITORY:-ccextractor} + DEBUG: ${DEBUG:-True} + ports: + - "${APP_PORT:-5000}:5000" + volumes: + - repository_data:/repository + networks: + - sample_platform + restart: unless-stopped + +networks: + sample_platform: + +volumes: + db_data: + repository_data: diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100644 index 000000000..3fb0160b6 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,21 @@ +#!/bin/sh +set -e + +# /app is a git repo created at build time; run.py reads a build commit from +# it. safe.directory guards against ownership mismatches under a bind mount. +git config --global --add safe.directory /app 2>/dev/null || true + +python install/wait_for_db.py + +# Creates the schema on an empty database, applies pending migrations on an +# existing one. See install/init_schema.py for why it is not just an upgrade. +python install/init_schema.py + +exec gunicorn \ + --workers "${GUNICORN_WORKERS:-3}" \ + --bind 0.0.0.0:5000 \ + --timeout 120 \ + --access-logfile - \ + --error-logfile - \ + ${GUNICORN_RELOAD:+--reload} \ + run:app diff --git a/env.example b/env.example new file mode 100644 index 000000000..1a2433a4e --- /dev/null +++ b/env.example @@ -0,0 +1,19 @@ +# Sample Platform — Docker environment. Copy to .env and edit before use. + +# MySQL. The root password stays with the database container and is never +# passed to the application, which connects only as the user below. +MYSQL_ROOT_PASSWORD=change-me-root +MYSQL_DATABASE=sample_platform +MYSQL_USER=sample_platform +MYSQL_PASSWORD=change-me-app + +# Host port for the web app (the container always listens on 5000). +APP_PORT=5000 + +# Application settings. +SERVER_NAME=localhost:5000 +HMAC_KEY=change-me +GCS_BUCKET_NAME=sample-platform-dev +GITHUB_OWNER=CCExtractor +GITHUB_REPOSITORY=ccextractor +DEBUG=True diff --git a/exceptions.py b/exceptions.py index 658af56a7..16e4780fd 100644 --- a/exceptions.py +++ b/exceptions.py @@ -3,10 +3,13 @@ class QueuedSampleNotFoundException(Exception): - """Custom exception handler for queued sample not found.""" + """Raised when a queued sample cannot be found by the given ID.""" - def __init__(self, message: str) -> None: - Exception.__init__(self) + def __init__(self, message="Queued sample not found."): + # By providing a default, all existing raise sites that call + # this exception without arguments will still work, and new + # calls can optionally provide a custom message. + super().__init__(message) self.message = message diff --git a/install/ci-vm/ci-linux/ci/runCI b/install/ci-vm/ci-linux/ci/runCI index c5498a8f7..e5c2bcbc4 100644 --- a/install/ci-vm/ci-linux/ci/runCI +++ b/install/ci-vm/ci-linux/ci/runCI @@ -12,79 +12,163 @@ if [ ! -f "$DIR/variables" ]; then sudo shutdown -h now fi -# Source variables -. "$DIR/variables" - -# Add cargo to path -PATH="/root/.cargo/bin:$PATH" - -# Check if required files & folders are present -if [ ! -f ${reportURLFile} ]; then - # No report URL file defined - sudo shutdown -h now -fi -if [ ! -d ${srcDir} ]; then - # No source dir defined - sudo shutdown -h now -fi - -reportURL=$(cat ${reportURLFile}) -userAgent="CCX/CI_BOT" -logFile="${reportFolder}/log.html" - # Functions for re-use in various stages of the test progress -# Post status to the server +# Post status to the server with retry logic function postStatus { - echo "Posting ${1} - ${2} to the server:" >> "${logFile}" - curl -s -A "${userAgent}" --data "type=progress&status=$1&message=$2" -w "\n" "${reportURL}" >> "${logFile}" + local status="$1" + local message="$2" + local max_retries=3 + local retry_delay=5 + local attempt=1 + + echo "Posting ${status} - ${message} to the server:" >> "${logFile}" + + while [ $attempt -le $max_retries ]; do + local http_code + http_code=$(curl -s -A "${userAgent}" --data "type=progress&status=${status}&message=${message}" \ + -w "%{http_code}" -o /tmp/curl_response.txt "${reportURL}" 2>/dev/null) + local curl_exit=$? + + if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + echo "Status posted successfully (HTTP ${http_code})" >> "${logFile}" + cat /tmp/curl_response.txt >> "${logFile}" 2>/dev/null + echo "" >> "${logFile}" + sleep 5 + return 0 + fi + + echo "Attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}" + attempt=$((attempt + 1)) + + if [ $attempt -le $max_retries ]; then + echo "Retrying in ${retry_delay} seconds..." >> "${logFile}" + sleep $retry_delay + retry_delay=$((retry_delay * 2)) + fi + done + + echo "ERROR: Failed to post status after ${max_retries} attempts" >> "${logFile}" + return 1 } -# Send the log file to the server so it can be used +# Send the log file to the server with retry logic function sendLogFile { + local max_retries=3 + local retry_delay=5 + local attempt=1 + echo "Sending log to the server:" >> "${logFile}" - curl -s -A "${userAgent}" --form "type=logupload" --form "file=@${logFile}" -w "\n" "${reportURL}" - sleep 5 + + while [ $attempt -le $max_retries ]; do + local http_code + http_code=$(curl -s -A "${userAgent}" --form "type=logupload" --form "file=@${logFile}" \ + -w "%{http_code}" -o /tmp/curl_log_response.txt "${reportURL}" 2>/dev/null) + local curl_exit=$? + + if [ $curl_exit -eq 0 ] && [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then + echo "Log uploaded successfully (HTTP ${http_code})" >> "${logFile}" + sleep 5 + return 0 + fi + + echo "Log upload attempt ${attempt}/${max_retries} failed (curl exit: ${curl_exit}, HTTP: ${http_code})" >> "${logFile}" + attempt=$((attempt + 1)) + + if [ $attempt -le $max_retries ]; then + echo "Retrying log upload in ${retry_delay} seconds..." >> "${logFile}" + sleep $retry_delay + retry_delay=$((retry_delay * 2)) + fi + done + + echo "ERROR: Failed to upload log after ${max_retries} attempts" >> "${logFile}" + return 1 } # Exit script and post abort status function haltAndCatchFire { - postStatus "canceled" $1 >> "${logFile}" - sendLogFile - sudo shutdown -h now + sendLogFile + postStatus "canceled" $1 >> "${logFile}" + sudo shutdown -h now } # Fail when the exit status is not equal to 0 function executeCommand { - #echo "$@" - "$@" >> "${logFile}" - local status=$? - if [ ${status} -ne 0 ]; then - haltAndCatchFire "" # No message needed as we post before anyway - fi + #echo "$@" + "$@" >> "${logFile}" + local status=$? + if [ ${status} -ne 0 ]; then + haltAndCatchFire "" # No message needed as we post before anyway + fi +} + +# Send a build artifact to the server (best-effort: a failed upload must never +# abort the run, so this always returns 0). Reuses the reportURL/curl pattern. +function sendArtifact { + local artifactType="$1" + local filePath="$2" + if [ ! -f "${filePath}" ]; then + echo "Artifact ${artifactType}: no file at '${filePath}', skipping" >> "${logFile}" + return 0 + fi + echo "Uploading ${artifactType} artifact (${filePath})" >> "${logFile}" + # --fail surfaces HTTP >= 400 (e.g. 413 on an oversized coredump) as a + # curl error so failed uploads are visible in the log; still best-effort. + curl -s --fail -A "${userAgent}" --form "type=artifactupload" --form "artifact_type=${artifactType}" \ + --form "file=@${filePath}" "${reportURL}" >> "${logFile}" 2>&1 + local curlStatus=$? + if [ ${curlStatus} -ne 0 ]; then + echo "Artifact ${artifactType}: upload failed (curl exit ${curlStatus})" >> "${logFile}" + fi + return 0 } -# Truncate log in case there was anything left from a previous session -echo "" > "${logFile}" - -postStatus "preparation" "Copy testsuite to local folder" -executeCommand cp -r ${suiteSrcDir}/* ${suiteDstDir} - -postStatus "preparation" "Copy build artifact to local folder" -executeCommand cp -r ${srcDir}/* ${dstDir} -executeCommand cd ${dstDir} - -postStatus "building" "Checking for CCExtractor build artifact" -if [ -x "ccextractor" ]; then - chmod 700 ccextractor - # Run testSuite - postStatus "testing" "Running tests" - executeCommand cd ${suiteDstDir} - executeCommand ${tester} --entries "${testFile}" --executable "${dstDir}/ccextractor" --tempfolder "${tempFolder}" --timeout 3000 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" - postStatus "completed" "Ran all tests" - sendLogFile - # Shut down - sudo shutdown -h now +# Source variables +. "$DIR/variables" + +# Add cargo to path +PATH="/root/.cargo/bin:$PATH" + +reportURL=$(curl http://metadata/computeMetadata/v1/instance/attributes/reportURL -H "Metadata-Flavor: Google") +userAgent="CCX/CI_BOT" +logFile="${reportFolder}/log.html" + +postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" + +if [ -e "${dstDir}/ccextractor" ]; then + cp $dstDir/* ./ + chmod 700 ccextractor + chmod +x ${tester} + # Log binary version for verification (commit SHA will be visible in output) + echo "=== CCExtractor Binary Version ===" >> "${logFile}" + ./ccextractor --version >> "${logFile}" 2>&1 + echo "=== End Version Info ===" >> "${logFile}" + postStatus "testing" "Running tests" + executeCommand cd ${suiteDstDir} + + # Enable core dumps and capture the test run's combined stdout/stderr so + # both can be uploaded as artifacts (the API serves them per run). + ulimit -c unlimited 2>/dev/null + echo "core.%p" | sudo tee /proc/sys/kernel/core_pattern >/dev/null 2>&1 + combinedLog="${reportFolder}/combined_stdout.log" + ${tester} --debug --entries "${testFile}" --executable "ccextractor" --tempfolder "${tempFolder}" --timeout 600 --reportfolder "${reportFolder}" --resultfolder "${resultFolder}" --samplefolder "${sampleFolder}" --method Server --url "${reportURL}" > "${combinedLog}" 2>&1 + testerStatus=$? + cat "${combinedLog}" >> "${logFile}" + + # Upload artifacts before any failure-halt so crash data survives. + sendArtifact "binary" "${dstDir}/ccextractor" + sendArtifact "combined_stdout" "${combinedLog}" + sendArtifact "coredump" "$(ls -1t core.* 2>/dev/null | head -n1)" + + if [ ${testerStatus} -ne 0 ]; then + haltAndCatchFire "" + fi + + sendLogFile + postStatus "completed" "Ran all tests" + + sudo shutdown -h now else - haltAndCatchFire "build" + haltAndCatchFire "artifact" fi diff --git a/install/ci-vm/ci-linux/ci/variables b/install/ci-vm/ci-linux/ci/variables new file mode 100644 index 000000000..9f611d68e --- /dev/null +++ b/install/ci-vm/ci-linux/ci/variables @@ -0,0 +1,25 @@ +#!/bin/bash + +# This script contains the variables which are used during CI on VM instances. +# Copy/rename this file in case you want to change the default variables. + +# The URL to check if the VM is in maintenance +maintenanceURL="http://foo.bar/maintenance/linux" +# The runCI file URL +runCIFile="/repository/TestData/ci-linux/runCI" +# Directory where the test suite is located +suiteDstDir="/repository" +# Shell script to launch the test suite +tester="/repository/ccextractortester" +# Location of the samples +sampleFolder="/repository/TestFiles" +# Location of the result files +resultFolder="/repository/TestResults" +# Testfile location +testFile="/repository/vm_data/ci-tests/TestAll.xml" +# The folder that will be used to store the results in +reportFolder="/repository/reports" +# The folder that will be used to temporarily store the result files in +tempFolder="/repository/TempFiles" +# Directory where ccextractor executable is located +dstDir="/repository/vm_data/unsafe-ccextractor" diff --git a/install/ci-vm/ci-linux/ci/variables-sample b/install/ci-vm/ci-linux/ci/variables-sample deleted file mode 100644 index c15f887de..000000000 --- a/install/ci-vm/ci-linux/ci/variables-sample +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash - -# This script contains the variables which we don't want to post on GitHub -# Copy/rename this to variables, and fill this in. - -# The URL to check if the VM is in maintenance -maintenanceURL="http://foo.bar/maintenance/linux" -# The URL with the runCI blob -runCIURL="https://raw.githubusercontent.com/CCExtractor/sample-platform/master/install/ci-vm/ci-linux/ci/runCI" -# File that contains the URL to report to -reportURLFile="/path/to/token/file" -# runCI location -runCIFile="/path/to/runCI/file" -# Directory that contains the sources of the version to test -srcDir="/path/to/ccextractor/repository" -# Directory to copy the files to -dstDir="/new/path/to/ccextractor/repository" -# Directory where the test suite is located -suiteSrcDir="/path/to/test/suite" -# Directory where the test suite is located -suiteDstDir="/path/to/test/suite" -# Shell script to launch the test suite -tester="./ccextractortester" -# Location of the samples -sampleFolder="/path/to/test/files" -# Location of the result files -resultFolder="/path/to/result/files" -# Testfile location -testFile="/path/to/testfile.xml" -# The folder that will be used to store the results in -reportFolder="/path/to/folder/" -# The folder that will be used to temporarily store the result files in -tempFolder="/path/to/temp/folder/" diff --git a/install/ci-vm/ci-linux/startup-script.sh b/install/ci-vm/ci-linux/startup-script.sh new file mode 100644 index 000000000..4401e741f --- /dev/null +++ b/install/ci-vm/ci-linux/startup-script.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v3.2.0/gcsfuse_3.2.0_amd64.deb +dpkg --install gcsfuse_3.2.0_amd64.deb +rm gcsfuse_3.2.0_amd64.deb + +apt install -y gnupg ca-certificates +gpg --homedir /tmp --no-default-keyring --keyring /usr/share/keyrings/mono-official-archive-keyring.gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF +echo "deb [signed-by=/usr/share/keyrings/mono-official-archive-keyring.gpg] https://download.mono-project.com/repo/ubuntu stable-focal main" | sudo tee /etc/apt/sources.list.d/mono-official-stable.list +sudo apt update +apt install -y mono-complete libtesseract-dev libgpac-dev tesseract-ocr-eng fonts-noto-core + +mkdir repository +cd repository + +# Use gcsfuse and import required files +# TempFiles is used by the tester (--tempfolder) and must exist +mkdir temp TestFiles TestResults TempFiles vm_data reports + +gcs_bucket=$(curl http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google") + +vm_name=$(curl http://metadata.google.internal/computeMetadata/v1/instance/hostname -H "Metadata-Flavor: Google") +vm_name=(${vm_name//./ }) + +echo "${gcs_bucket} /repository/temp gcsfuse rw,noatime,async,_netdev,noexec,user,implicit_dirs,allow_other,only_dir=TestData/ci-linux 0 0" | sudo tee -a /etc/fstab +echo "${gcs_bucket} /repository/vm_data gcsfuse rw,noatime,async,_netdev,noexec,user,implicit_dirs,allow_other,only_dir=vm_data/${vm_name} 0 0" | sudo tee -a /etc/fstab +echo "${gcs_bucket} /repository/TestFiles gcsfuse rw,noatime,async,_netdev,noexec,user,implicit_dirs,allow_other,only_dir=TestFiles,cache_dir=/tmp,file_cache_max_size_mb=3000,file_cache_cache_file_for_range_read=true,metadata_cache_ttl_secs=-1 0 0" | sudo tee -a /etc/fstab +echo "${gcs_bucket} /repository/TestResults gcsfuse rw,noatime,async,_netdev,noexec,user,implicit_dirs,allow_other,only_dir=TestResults 0 0" | sudo tee -a /etc/fstab + +mount temp +mount vm_data +mount TestFiles +mount TestResults + +# Give gcsfuse mounts time to become ready +sleep 10 + +cp temp/* ./ + +chmod +x bootstrap +./bootstrap diff --git a/install/ci-vm/ci-linux/testsuite b/install/ci-vm/ci-linux/testsuite deleted file mode 100644 index a9160faf5..000000000 --- a/install/ci-vm/ci-linux/testsuite +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -# /etc/init.d/testsuite -# -# Starts the test suite when booting -case "${1}" in - start) - echo "Starting test suite daemon..." - cd /usr/src/ci/ - ./bootstrap - ;; - *) - echo "Usage: /etc/init.d/testsuite {start}" - exit 1 - ;; -esac - -exit 0 \ No newline at end of file diff --git a/install/ci-vm/ci-windows/ci/runCI.bat b/install/ci-vm/ci-windows/ci/runCI.bat index 99fd0be42..4a8bc1b8d 100644 --- a/install/ci-vm/ci-windows/ci/runCI.bat +++ b/install/ci-vm/ci-windows/ci/runCI.bat @@ -10,45 +10,51 @@ if NOT EXIST "variables.bat" ( echo Loading variables.bat rem Source variables call %~dp0\variables.bat -if "%reportURLFile%"=="" ( - rem No report URL file defined - shutdown -s -t 0 - exit -) -if "%srcDir%"=="" ( - rem No source dir defined - shutdown -s -t 0 - exit -) -SET /P reportURL=<%reportURLFile% +for /F %%R in ('curl http://metadata/computeMetadata/v1/instance/attributes/reportURL -H "Metadata-Flavor: Google"') do SET reportURL=%%R SET userAgent="CCX/CI_BOT" SET logFile="%reportFolder%/log.html" -echo Copy files over to local disk -call :postStatus "preparation" "Copy testsuite to local folder" -rem robocopy returns a non-zero exit code even on success (https://ss64.com/nt/robocopy-exit.html), so we cannot use executeCommand -call robocopy %suiteSrcDir% %suiteDstDir% /e /MIR >> "%logFile%" - -call :postStatus "preparation" "Copy build artifact to local folder" -call robocopy %srcDir% %dstDir% /e /MIR /XD %srcDir%\.git >> "%logFile%" -call :executeCommand cd %dstDir% +rem NB: no outer ">> %logFile%" here. postStatus already appends to %logFile% +rem internally; an outer redirect to the same file self-locks on Windows +rem (the inner append cannot open a file the outer redirect holds open). +call :postStatus "preparation" "Loaded variables, created log file and checking for CCExtractor build artifact" -call :postStatus "building" "Checking for CCExtractor build artifact" -if EXIST "ccextractorwinfull.exe" ( - rem Run testsuite +echo Checking for CCExtractor build artifact +if EXIST "%dstDir%\ccextractorwinfull.exe" ( echo Run tests + copy "%dstDir%\*" . + rem Log binary version for verification (commit SHA will be visible in output) + echo === CCExtractor Binary Version === >> "%logFile%" + ccextractorwinfull.exe --version >> "%logFile%" 2>&1 + echo === End Version Info === >> "%logFile%" call :postStatus "testing" "Running tests" call :executeCommand cd %suiteDstDir% - call :executeCommand "%tester%" --entries "%testFile%" --executable "%dstDir%\ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 3000 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" - call :postStatus "completed" "Ran all tests" + + rem Capture the test run's combined stdout/stderr for upload as an artifact. + rem NB: "||" instead of "if errorlevel 1" - it fires at runtime on ANY + rem nonzero exit code, including the negative NTSTATUS codes a crash + rem returns, e.g. 0xC0000005, and works inside this parenthesized block + rem where %ERRORLEVEL% would expand too early. + set "testerFailed=" + "%tester%" --debug True --entries "%testFile%" --executable "ccextractorwinfull.exe" --tempfolder "%tempFolder%" --timeout 600 --reportfolder "%reportFolder%" --resultfolder "%resultFolder%" --samplefolder "%sampleFolder%" --method Server --url "%reportURL%" > "%reportFolder%/combined_stdout.log" 2>&1 || set "testerFailed=1" + type "%reportFolder%/combined_stdout.log" >> "%logFile%" + + rem Upload artifacts (best-effort; never aborts the run). Windows skips coredump for v1. + call :sendArtifact "binary" "%dstDir%\ccextractorwinfull.exe" + call :sendArtifact "combined_stdout" "%reportFolder%/combined_stdout.log" + + if defined testerFailed call :haltAndCatchFire "" + + call :sendLogFile + echo Done running tests - rem Shut down - timeout 5 + call :postStatus "completed" "Ran all tests" + shutdown -s -t 0 exit ) else ( - call :haltAndCatchFire "build" + call :haltAndCatchFire "artifact" ) echo End EXIT %ERRORLEVEL% @@ -66,20 +72,110 @@ IF %status% NEQ 0 ( ) EXIT /B 0 -rem Post status to the server +rem Post status to the server with retry logic :postStatus -echo "Posting status %~1 (message: %~2) to the server" -curl -s -A "%userAgent%" --data "type=progress&status=%~1&message=%~2" -w "\n" "%reportURL%" >> "%logFile%" -EXIT /B 0 +setlocal EnableDelayedExpansion +set "ps_status=%~1" +set "ps_message=%~2" +set ps_attempt=1 +set ps_max_retries=3 +set ps_retry_delay=5 + +:postStatusRetry +echo "Posting status %ps_status% (message: %ps_message%) to the server (attempt !ps_attempt!/%ps_max_retries%)" +echo Posting status %ps_status% - %ps_message% (attempt !ps_attempt!/%ps_max_retries%) >> "%logFile%" + +curl -s -A "%userAgent%" --data "type=progress&status=%ps_status%&message=%ps_message%" -w "%%{http_code}" -o "%TEMP%\curl_response.txt" "%reportURL%" > "%TEMP%\http_code.txt" 2>&1 +set ps_curl_exit=%ERRORLEVEL% + +set /p ps_http_code=<"%TEMP%\http_code.txt" + +if %ps_curl_exit% EQU 0 ( + if !ps_http_code! GEQ 200 if !ps_http_code! LSS 300 ( + echo Status posted successfully (HTTP !ps_http_code!) >> "%logFile%" + type "%TEMP%\curl_response.txt" >> "%logFile%" 2>nul + echo. >> "%logFile%" + timeout 5 + endlocal + EXIT /B 0 + ) +) + +echo Attempt !ps_attempt!/%ps_max_retries% failed (curl exit: %ps_curl_exit%, HTTP: !ps_http_code!) >> "%logFile%" +set /A ps_attempt+=1 + +if !ps_attempt! LEQ %ps_max_retries% ( + echo Retrying in %ps_retry_delay% seconds... >> "%logFile%" + timeout %ps_retry_delay% + set /A ps_retry_delay*=2 + goto postStatusRetry +) + +echo ERROR: Failed to post status after %ps_max_retries% attempts >> "%logFile%" +endlocal +EXIT /B 1 rem Exit script and post abort status :haltAndCatchFire echo "Halt and catch fire (reason: %~1)" -call :postStatus "canceled" "%~1" -timeout 5 echo Post log -curl -s -A "%userAgent%" --form "type=logupload" --form "file=@%logFile%" -w "\n" "%reportURL%" -rem Shut down, but only in 10 seconds, to give the time to finish the post status -timeout 10 +call :sendLogFile +call :postStatus "canceled" "%~1" shutdown -s -t 0 EXIT 0 + +rem Send log file to server with retry logic +:sendLogFile +setlocal EnableDelayedExpansion +set sl_attempt=1 +set sl_max_retries=3 +set sl_retry_delay=5 + +:sendLogFileRetry +echo Sending log to server (attempt !sl_attempt!/%sl_max_retries%) >> "%logFile%" + +curl -s -A "%userAgent%" --form "type=logupload" --form "file=@%logFile%" -w "%%{http_code}" -o "%TEMP%\curl_log_response.txt" "%reportURL%" > "%TEMP%\log_http_code.txt" 2>&1 +set sl_curl_exit=%ERRORLEVEL% + +set /p sl_http_code=<"%TEMP%\log_http_code.txt" + +if %sl_curl_exit% EQU 0 ( + if !sl_http_code! GEQ 200 if !sl_http_code! LSS 300 ( + echo Log uploaded successfully (HTTP !sl_http_code!) >> "%logFile%" + timeout 5 + endlocal + EXIT /B 0 + ) +) + +echo Log upload attempt !sl_attempt!/%sl_max_retries% failed (curl exit: %sl_curl_exit%, HTTP: !sl_http_code!) >> "%logFile%" +set /A sl_attempt+=1 + +if !sl_attempt! LEQ %sl_max_retries% ( + echo Retrying log upload in %sl_retry_delay% seconds... >> "%logFile%" + timeout %sl_retry_delay% + set /A sl_retry_delay*=2 + goto sendLogFileRetry +) + +echo ERROR: Failed to upload log after %sl_max_retries% attempts >> "%logFile%" +endlocal +EXIT /B 1 + +rem Send a build artifact to the server (best-effort; never aborts the run) +:sendArtifact +setlocal +set "sa_type=%~1" +set "sa_file=%~2" +if NOT EXIST "%sa_file%" ( + echo Artifact %sa_type%: no file at "%sa_file%", skipping >> "%logFile%" + endlocal + EXIT /B 0 +) +echo Uploading %sa_type% artifact (%sa_file%) >> "%logFile%" +rem --fail makes HTTP >= 400 (e.g. 413 on an oversized file) a curl error so +rem the failure is at least visible in the log; the upload stays best-effort. +curl -s --fail -A "%userAgent%" --form "type=artifactupload" --form "artifact_type=%sa_type%" --form "file=@%sa_file%" "%reportURL%" >> "%logFile%" 2>&1 +if errorlevel 1 echo Artifact %sa_type%: upload failed with curl exit code %ERRORLEVEL% >> "%logFile%" +endlocal +EXIT /B 0 diff --git a/install/ci-vm/ci-windows/ci/variables.bat b/install/ci-vm/ci-windows/ci/variables.bat index ca376340e..1d4ab6e6e 100644 --- a/install/ci-vm/ci-windows/ci/variables.bat +++ b/install/ci-vm/ci-windows/ci/variables.bat @@ -1,28 +1,22 @@ :: batch file for variables -:: This script contains the variables which we don't want to post on GitHub -:: Copy/rename this to variables, and fill this in. +:: This script contains the variables which are used during CI on VM instances. +:: Copy/rename this file in case you want to change the default variables. @echo off -:: File that contains the URL to report to -set reportURLFile="/path/to/token/file" -:: Directory that contains the sources of the version to test -set srcDir="/path/to/ccextractor/repository" -:: Directory to copy the files to -set dstDir="/new/path/to/ccextractor/repository" +:: Directory where ccextractor executable is located +set dstDir=.\vm_data\unsafe-ccextractor :: Directory where the test suite is located -set suiteSrcDir="/path/to/test/suite" -:: Directory where the test suite is located -set suiteDstDir="/path/to/test/suite" +set suiteDstDir=. :: Shell script to launch the test suite -set tester="./ccextractortester" +set tester=.\ccextractortester :: Location of the samples -set sampleFolder="/path/to/test/files" +set sampleFolder=.\TestFiles :: Location of the result files -set resultFolder="/path/to/result/files" +set resultFolder=.\TestResults :: Testfile location -set testFile="/path/to/testfile.xml" +set testFile=.\vm_data\ci-tests\TestAll.xml :: The folder that will be used to store the results in -set reportFolder="/path/to/folder/" -:: The folder that will be used to temporarily store the result files in -set tempFolder="/path/to/temp/folder/" +set reportFolder=.\reports +:: Temporary folder for ccextractortester output files +set tempFolder=.\TempFiles diff --git a/install/ci-vm/ci-windows/rclone_sample.conf b/install/ci-vm/ci-windows/rclone_sample.conf new file mode 100644 index 000000000..a5503129d --- /dev/null +++ b/install/ci-vm/ci-windows/rclone_sample.conf @@ -0,0 +1,9 @@ +[GCS_BUCKET_NAME] +type = google cloud storage +project_number = GCP_PROJECT_NUMBER +service_account_file = .\service-account.json +object_acl = projectPrivate +bucket_acl = projectPrivate +location = GCS_BUCKET_LOCATION +storage_class = GCS_BUCKET_LOCATION_TYPE +bucket_policy_only = true diff --git a/install/ci-vm/ci-windows/startup-script.ps1 b/install/ci-vm/ci-windows/startup-script.ps1 new file mode 100644 index 000000000..c35c50d7e --- /dev/null +++ b/install/ci-vm/ci-windows/startup-script.ps1 @@ -0,0 +1,39 @@ +Set-ExecutionPolicy Bypass -Scope Process -Force; [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.ServicePointManager]::SecurityProtocol -bor 3072; iex ((New-Object System.Net.WebClient).DownloadString('https://community.chocolatey.org/install.ps1')) +choco install winfsp tesseract -y + +cd C:\Windows\Temp + +curl.exe https://downloads.rclone.org/v1.69.3/rclone-v1.69.3-windows-amd64.zip --output rclone.zip +Expand-Archive -Path rclone.zip -DestinationPath .\ +New-Item -Path '.\repository' -ItemType Directory +Copy-Item -Path .\rclone-v1.69.3-windows-amd64\rclone.exe -Destination .\repository\ +Add-MpPreference -ExclusionProcess 'C:\Windows\Temp\repository\rclone.exe' + +cd repository +New-Item -Path '.\reports' -ItemType Directory + +$gcs_bucket = curl.exe http://metadata/computeMetadata/v1/instance/attributes/bucket -H "Metadata-Flavor: Google" +$env:mount_path = $gcs_bucket, $gcs_bucket -join ":" + +$env:vm_name = curl.exe http://metadata.google.internal/computeMetadata/v1/instance/hostname -H "Metadata-Flavor: Google" +$env:vm_name = ($env:vm_name -split "\.")[0] + +curl.exe http://metadata/computeMetadata/v1/instance/attributes/rclone_conf -H "Metadata-Flavor: Google" > rclone.conf +(Get-Content -path .\rclone.conf) | Set-Content -Encoding UTF8 -Path .\rclone.conf + +curl.exe http://metadata/computeMetadata/v1/instance/attributes/service_account -H "Metadata-Flavor: Google" > service-account.json +(Get-Content -path .\service-account.json) | Set-Content -Encoding ASCII -Path .\service-account.json + +start powershell {.\rclone.exe mount $env:mount_path\TestFiles .\TestFiles --config=".\rclone.conf" --vfs-cache-mode full --vfs-cache-max-size 5G --no-modtime -o FileInfoTimeout=-1 --vfs-read-wait 0 --no-console --read-only} +Start-Sleep -Seconds 5 + +start powershell {.\rclone.exe mount $env:mount_path\TestData\ci-windows .\temp --config=".\rclone.conf" --no-console --read-only} +Start-Sleep -Seconds 5 + +start powershell {.\rclone.exe mount $env:mount_path\vm_data\$env:vm_name .\vm_data --config=".\rclone.conf" --no-console --read-only} +Start-Sleep -Seconds 5 + +Copy-Item -Path "temp\*" -Destination "." +.\rclone.exe copy $env:mount_path\TestResults .\TestResults --config=".\rclone.conf" + +powershell -command "Start-Process runCI.bat -Verb runas" diff --git a/install/ci-vm/ci-windows/startup.bat b/install/ci-vm/ci-windows/startup.bat deleted file mode 100644 index 20b20f5ea..000000000 --- a/install/ci-vm/ci-windows/startup.bat +++ /dev/null @@ -1,11 +0,0 @@ -rem Place this script in the startup folder (e.g. C:\Users\[Username]\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup) -rem Depending on the setup, it will need to be modified. -rem E.g In the case of the live server, we use a network mount, so we need to add a "net use ..." command. - -timeout 5 -net use Z: \\path\to\network\mount -timeout 5 - -rem Modify this path to the path of your runCI.bat file -cd C:\Users\ci\Documents\ci -call runCI.bat \ No newline at end of file diff --git a/install/ci-vm/installation.md b/install/ci-vm/installation.md index 4aea72cd1..644b38dc8 100644 --- a/install/ci-vm/installation.md +++ b/install/ci-vm/installation.md @@ -1,87 +1,67 @@ -# Installation guide for various platforms +# Installation guide configuring GCP for the platform -This guide will give some hints and tips for configuring the VM's that will -run the CI setup. +This guide will give instructions for configuring and setting up the project on the Google Cloud Platform (GCP) before the installation of the platform. -Currently the guide covers Linux & Windows. +Currently, the platform uses Linux and Windows GCP instances for running regression tests. -To reduce the required size of the VM's, as well as not having to update the -VM template each time a sample changes, a network mount will be used to make -the data accessible towards the VM's. Setting up this network mount is not -covered in this guide. +## Setting up a Google Cloud Account +Apart from the central platform server, the platform utilises Infrastructure as a Service (IaaS) through the Google Cloud Platform. -## Linux +For this create a GCP account or log in to an existing account [here](https://console.cloud.google.com/). -### Prerequisites +## Creating a Google Cloud Project +The next step is to create a Google Cloud Project that the platform would use to manage instances for running regression tests and maintain a GCS bucket for samples, variables, tester, etc. -* Linux KVM with an distro of your choice -* CI user automatically logs in -* CI user has sudo rights to perform the `shutdown` command -* CCExtractor compiles successfully -* curl +For creating and managing projects, refer to the official documentation [here](https://cloud.google.com/resource-manager/docs/creating-managing-projects). -### Adding the necessary services and files +Now create a project, with the project name and project id of your choice. -#### runCI +## Creating a Google Cloud Storage bucket -* Copy over the `runCI` and `variables-sample` to a directory of choice. The -next step assumes it's located in `/usr/src/ci/`, if this is not the case, -please **modify** the necessary files in the next step. -* Rename `variables-sample` to `variables`, and fill in variables -* Chmod both files so they can be executed. +Working on Google Cloud Platform is not free, therefore it is recommended to either get a [free 90-day trial](https://console.cloud.google.com/freetrial) or [enable billing](https://console.cloud.google.com/billing) for the GCP account to get access the full set of services and increased usage limits. -#### Network folder +Now create a GCS bucket with the following settings (refer to the official documentation [here](https://cloud.google.com/storage/docs/creating-buckets)): +- Location: Choose as per your preference and budget, however it is advised to keep the bucket and sample-platform VM instance in the same region for faster access. +- Storage Class: Standard +- Access Control: Uniform, and enable `Prevent Public Access` +- Protection Tools: None -In case of the network mount, add the mount so that it's automatically -connecting at start-up. E.g., for Debian/derivatives this can be added to -`/etc/fstab`: +Note: Sometimes while creating an instance through the platform one might receive an error(ZONE_RESOURCE_POOL_EXHAUSTED or ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS), this is an issue faced quite often, hence it is advised to choose region and zone wisely. Here is a [stackoverflow link](https://stackoverflow.com/questions/65884360/cannot-start-gce-vm-instance-the-zone-does-not-have-enough-resources) for the same error. -``` -//127.0.0.1/repo /repo cifs user=,pass= 0 0 -``` +Note: We will provide downloads of the samples to users when requested via [Signed URLs](https://cloud.google.com/storage/docs/access-control#signed_urls_query_string_authentication). -#### Startup service +## Creating a Service Account -* Copy `testsuite` to `/etc/init.d/testsuite` -* Chmod the file so it can be executed +Now, create a service account with sufficient permissions (at least "Google Batch Service Agent") that would be used by the platform, VM instances to mount and read-write the bucket contents and also manage VM instances. -## Windows +If you are not the owner of the GCP project you are working on, make sure you have sufficient permissions for creating and managing service accounts; if not, request the project owner for the same. -### Prerequisites +- Create a service account [here](https://cloud.google.com/iam/docs/service-accounts-create) +- Choose the service account name as per your choice, but at least provide the role of "Google Batch Service Agent" to the account. -* Windows KVM with a Windows version of your choice -* CI user automatically logs in +You might also want to understand roles in GCP, you can find the official documentation [here](https://cloud.google.com/iam/docs/understanding-roles). -### Compiling CCExtractor on Windows +Now navigate to the "keys" section of the service account created, create a new key and download the JSON file. -Depending on the Windows version you'll have to install different compilers. +Note: This is a secret key and access to the key would give access to your GCP project. -For Windows 10, you'll need VS2015 (platform 140), **with** XP support -enabled for C++. You'll also need the VS2015 build tools. +## Provide access of the GCS bucket to the service account -### Adding the necessary services and files +Note: If you have provided the "editor" role to the service account (not recommended), skip this step. -#### Installing cURL +If you are not the owner of the GCP project you are working on, make sure you have the "Service Account Admin" role; if not, request the project owner for the same. -You can download cURL from https://curl.haxx.se/download.html, and to make -it callable through `curl` from the command line, this tutorial can be -followed: -http://callejoabel.blogspot.in/2013/09/making-curl-work-on-windows-7.html +- Go to [cloud storage page](https://console.cloud.google.com/storage/browser), and select the bucket to be used for the platform. +- Now go to the "Permissions" tab and check that the service account you created has "Storage Legacy Bucket Owner" and "Storage Legacy Object Owner" permissions, if not add these permissions by creating a new principal: + - Give the new principal name as the email of the service account created (can be found in the IAM & Admin -> Service Accounts section). + - Select the roles mentioned above and click on "Save". -#### runCI +## Mounting the Cloud Storage Bucket -* Copy over the `runCI.bat` and `variables-sample.bat` to a directory of -choice. The next step assumes it's located in `C:\Users\ci\Documents\ci`, -if this is not the case, please **modify** the necessary files in the next -step. -* Rename `variables-sample.bat` to `variables.bat`, and fill in variables +To reduce the required size of the VM instances, a network mount will be used to make the data accessible to the VM instances. -#### Network folder +This network mount is done using: +- [Google Cloud Storage FUSE](https://cloud.google.com/storage/docs/gcs-fuse) for linux VM instances. +- [Rclone](https://rclone.org/) and [WinFsp](https://winfsp.dev/) for windows VM instances. -In case of the network mount, it's best to have it connecting when the -startup service is running. - -#### Startup service - -* Copy `startup.bat` to -`C:\Users\ci\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup` +These tools are downloaded and installed automatically using the startup scripts on the VM instances. diff --git a/install/deploy/README.md b/install/deploy/README.md new file mode 100644 index 000000000..de25e6d24 --- /dev/null +++ b/install/deploy/README.md @@ -0,0 +1,144 @@ +# Deployment Scripts + +These scripts implement safe, automated deployment with automatic rollback on failure. + +## Overview + +``` +┌─────────────────────────────────────────────────────────────┐ +│ DEPLOYMENT FLOW │ +├─────────────────────────────────────────────────────────────┤ +│ pre_deploy.sh → deploy.sh → post_deploy.sh │ +│ ↓ ↓ ↓ │ +│ Validate env Update code Health check │ +│ Save state Run migrations Verify app │ +│ ↓ ↓ ↓ │ +│ ┌────┴────┐ │ +│ ↓ ↓ │ +│ SUCCESS FAILURE │ +│ ↓ ↓ │ +│ Done rollback.sh │ +└─────────────────────────────────────────────────────────────┘ +``` + +## Scripts + +### pre_deploy.sh +Validates the environment before deployment: +- Checks for concurrent deployments (lock file) +- Verifies install folder and config.py exist +- Tests database connection +- Saves current commit and migration version for rollback +- Checks disk space + +### deploy.sh +Performs the actual deployment: +- Fetches and updates code from git +- Installs/updates Python dependencies +- Runs database migrations +- Copies CI scripts to test data directory +- Reloads the platform service + +### post_deploy.sh +Verifies deployment was successful: +- Waits for application to start +- Checks `/health` endpoint (with fallback to `/`) +- Retries up to 6 times with 5-second delay +- Returns success (0) or failure (1) + +### rollback.sh +Restores previous working state: +- Restores previous git commit +- Downgrades database migrations if needed +- Reinstalls dependencies +- Reloads service +- Verifies rollback was successful + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `INSTALL_FOLDER` | `/var/www/sample-platform` | Application directory | +| `SAMPLE_REPOSITORY` | `/repository` | Test data repository | +| `DEPLOY_BRANCH` | `master` | Git branch to deploy | +| `HEALTH_URL` | `http://127.0.0.1/health` | Health check endpoint | +| `FALLBACK_URL` | `http://127.0.0.1/` | Fallback check URL | +| `MAX_RETRIES` | `6` | Health check retry count | +| `RETRY_DELAY` | `5` | Seconds between retries | + +## Manual Usage + +```bash +# Set environment +export INSTALL_FOLDER="/var/www/sample-platform" +export SAMPLE_REPOSITORY="/repository" + +# Run deployment +cd $INSTALL_FOLDER +sudo bash install/deploy/pre_deploy.sh && \ +sudo bash install/deploy/deploy.sh && \ +sudo bash install/deploy/post_deploy.sh || \ +sudo bash install/deploy/rollback.sh +``` + +## GitHub Actions Integration + +These scripts are designed to be called from the GitHub Actions workflow: + +```yaml +- name: Pre-deployment checks + run: sudo bash install/deploy/pre_deploy.sh + +- name: Deploy + run: sudo bash install/deploy/deploy.sh + +- name: Verify deployment + run: sudo bash install/deploy/post_deploy.sh + +- name: Rollback on failure + if: failure() + run: sudo bash install/deploy/rollback.sh +``` + +## Files Created During Deployment + +| File | Purpose | +|------|---------| +| `/tmp/sp-deploy.lock` | Prevents concurrent deployments | +| `/tmp/sp-deploy-backup-dir.txt` | Points to backup directory | +| `/tmp/sp-deploy-YYYYMMDD-HHMMSS/` | Backup directory with rollback info | +| `/tmp/health_response.json` | Last health check response | + +## Troubleshooting + +### Deployment stuck +Check for stale lock file: +```bash +ls -la /tmp/sp-deploy.lock +rm /tmp/sp-deploy.lock # If stale (>10 minutes old) +``` + +### Health check failing +```bash +# Check application logs +tail -f /var/www/sample-platform/logs/error.log + +# Check service status +systemctl status platform + +# Test health endpoint manually +curl -v http://127.0.0.1/health +``` + +### Rollback failed +```bash +# Check current state +cd /var/www/sample-platform +git log --oneline -5 +FLASK_APP=./run.py flask db current + +# Manual rollback +git checkout +FLASK_APP=./run.py flask db downgrade +systemctl restart platform +``` diff --git a/install/deploy/deploy.sh b/install/deploy/deploy.sh new file mode 100755 index 000000000..b3e25ab4a --- /dev/null +++ b/install/deploy/deploy.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# Main deployment script +# Exit codes: 0 = success, 1 = failure (triggers rollback) +# +# This script performs the actual deployment steps: +# 1. Pull latest code from git +# 2. Install/update Python dependencies +# 3. Run database migrations +# 4. Copy CI scripts +# 5. Reload the application service + +set -e + +INSTALL_FOLDER="${INSTALL_FOLDER:-/var/www/sample-platform}" +SAMPLE_REPOSITORY="${SAMPLE_REPOSITORY:-/repository}" +DEPLOY_BRANCH="${DEPLOY_BRANCH:-master}" + +echo "=== Starting deployment ===" +echo "Timestamp: $(date -Iseconds)" +echo "Branch: $DEPLOY_BRANCH" +echo "Install folder: $INSTALL_FOLDER" + +cd "$INSTALL_FOLDER" + +# Step 1: Fetch latest code +echo "" +echo "--- Step 1: Fetching latest code ---" +git fetch origin "$DEPLOY_BRANCH" +echo "✓ Fetched latest from origin/$DEPLOY_BRANCH" + +# Step 2: Check for local changes and handle them +if ! git diff --quiet; then + echo "WARNING: Local changes detected, stashing..." + git stash push -m "pre-deploy-$(date +%Y%m%d-%H%M%S)" || true +fi + +# Step 3: Update to latest code +echo "" +echo "--- Step 2: Updating code ---" +git checkout "$DEPLOY_BRANCH" +git reset --hard "origin/$DEPLOY_BRANCH" +NEW_COMMIT=$(git rev-parse HEAD) +echo "✓ Updated to commit: $NEW_COMMIT" + +# Step 4: Update dependencies +echo "" +echo "--- Step 3: Updating dependencies ---" +python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check --break-system-packages +echo "✓ Dependencies updated" + +# Step 5: Run database migrations +echo "" +echo "--- Step 4: Running database migrations ---" +FLASK_APP=./run.py python3 -m flask db upgrade +echo "✓ Database migrations complete" + +# Step 6: Copy CI scripts (if directories exist) +echo "" +echo "--- Step 5: Updating CI scripts ---" +if [ -d "${SAMPLE_REPOSITORY}/TestData" ]; then + if [ -f "install/ci-vm/ci-linux/ci/bootstrap" ]; then + mkdir -p "${SAMPLE_REPOSITORY}/TestData/ci-linux" + cp "install/ci-vm/ci-linux/ci/bootstrap" "${SAMPLE_REPOSITORY}/TestData/ci-linux/bootstrap" + echo "✓ Copied ci-linux/bootstrap" + fi + if [ -f "install/ci-vm/ci-linux/ci/runCI" ]; then + mkdir -p "${SAMPLE_REPOSITORY}/TestData/ci-linux" + cp "install/ci-vm/ci-linux/ci/runCI" "${SAMPLE_REPOSITORY}/TestData/ci-linux/runCI" + echo "✓ Copied ci-linux/runCI" + fi + if [ -f "install/ci-vm/ci-windows/ci/runCI.bat" ]; then + mkdir -p "${SAMPLE_REPOSITORY}/TestData/ci-windows" + cp "install/ci-vm/ci-windows/ci/runCI.bat" "${SAMPLE_REPOSITORY}/TestData/ci-windows/runCI.bat" + echo "✓ Copied ci-windows/runCI.bat" + fi +else + echo "⚠ TestData directory not found, skipping CI script copy" +fi + +# Step 7: Reload application +echo "" +echo "--- Step 6: Reloading application ---" +if systemctl is-active --quiet platform; then + systemctl reload platform + echo "✓ Platform service reloaded" +else + echo "⚠ Platform service not running, attempting start..." + systemctl start platform + echo "✓ Platform service started" +fi + +echo "" +echo "=== Deployment complete ===" +echo "Deployed commit: $NEW_COMMIT" +echo "Waiting for application to start..." +exit 0 diff --git a/install/deploy/post_deploy.sh b/install/deploy/post_deploy.sh new file mode 100755 index 000000000..ba2283d26 --- /dev/null +++ b/install/deploy/post_deploy.sh @@ -0,0 +1,75 @@ +#!/bin/bash +# Post-deployment health check +# Exit codes: 0 = healthy, 1 = unhealthy (triggers rollback) +# +# This script verifies the deployment was successful by checking: +# 1. The /health endpoint (if available) +# 2. Fallback: the root endpoint responds with 2xx/3xx + +set -e + +# Configuration +HEALTH_URL="${HEALTH_URL:-http://127.0.0.1/health}" +FALLBACK_URL="${FALLBACK_URL:-http://127.0.0.1/}" +MAX_RETRIES="${MAX_RETRIES:-6}" +RETRY_DELAY="${RETRY_DELAY:-5}" + +echo "=== Post-deployment health check ===" +echo "Timestamp: $(date -Iseconds)" +echo "Health URL: $HEALTH_URL" +echo "Max retries: $MAX_RETRIES" +echo "Retry delay: ${RETRY_DELAY}s" + +# Wait for application to start +echo "" +echo "Waiting for application to initialize..." +sleep 3 + +# Check health endpoint with retries +echo "" +echo "--- Health check ---" +for i in $(seq 1 $MAX_RETRIES); do + echo "Attempt $i/$MAX_RETRIES..." + + # Try the /health endpoint first (use -L to follow HTTP->HTTPS redirects) + HTTP_CODE=$(curl -sL -o /tmp/health_response.json -w "%{http_code}" "$HEALTH_URL" 2>/dev/null || echo "000") + + if [ "$HTTP_CODE" = "200" ]; then + echo "✓ Health check passed (HTTP $HTTP_CODE)" + echo "" + echo "Response:" + cat /tmp/health_response.json 2>/dev/null | python3 -m json.tool 2>/dev/null || cat /tmp/health_response.json + echo "" + echo "=== Deployment verified successfully ===" + exit 0 + elif [ "$HTTP_CODE" = "404" ]; then + # Health endpoint doesn't exist, try fallback (use -L to follow redirects) + echo "Health endpoint not found, trying fallback URL..." + HTTP_CODE=$(curl -sL -o /dev/null -w "%{http_code}" "$FALLBACK_URL" 2>/dev/null || echo "000") + if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 400 ]; then + echo "✓ Fallback check passed (HTTP $HTTP_CODE)" + echo "" + echo "=== Deployment verified successfully (using fallback) ===" + exit 0 + fi + elif [ "$HTTP_CODE" = "503" ]; then + echo "Health check returned unhealthy (HTTP 503)" + echo "Response:" + cat /tmp/health_response.json 2>/dev/null || echo "(no response body)" + fi + + if [ "$i" -lt "$MAX_RETRIES" ]; then + echo "Health check returned HTTP $HTTP_CODE, retrying in ${RETRY_DELAY}s..." + sleep "$RETRY_DELAY" + fi +done + +echo "" +echo "=== HEALTH CHECK FAILED ===" +echo "Health check failed after $MAX_RETRIES attempts" +echo "Last HTTP code: $HTTP_CODE" +echo "Last response:" +cat /tmp/health_response.json 2>/dev/null || echo "(no response)" +echo "" +echo "Deployment verification FAILED - rollback recommended" +exit 1 diff --git a/install/deploy/pre_deploy.sh b/install/deploy/pre_deploy.sh new file mode 100755 index 000000000..9f8412994 --- /dev/null +++ b/install/deploy/pre_deploy.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Pre-deployment checks - run before any deployment actions +# Exit codes: 0 = success, 1 = failure +# +# This script validates the environment before deployment and stores +# information needed for potential rollback. + +set -e + +INSTALL_FOLDER="${INSTALL_FOLDER:-/var/www/sample-platform}" +BACKUP_DIR="/tmp/sp-deploy-$(date +%Y%m%d-%H%M%S)" +LOCK_FILE="/tmp/sp-deploy.lock" + +echo "=== Pre-deployment checks ===" +echo "Timestamp: $(date -Iseconds)" + +# Check for concurrent deployments +if [ -f "$LOCK_FILE" ]; then + LOCK_AGE=$(($(date +%s) - $(stat -c %Y "$LOCK_FILE" 2>/dev/null || echo 0))) + if [ "$LOCK_AGE" -lt 600 ]; then # 10 minute lock + echo "ERROR: Deployment already in progress (lock age: ${LOCK_AGE}s)" + echo "If this is stale, remove: $LOCK_FILE" + exit 1 + fi + echo "WARNING: Stale lock file found (age: ${LOCK_AGE}s), removing..." + rm -f "$LOCK_FILE" +fi + +# Create lock file +echo "$$" > "$LOCK_FILE" +trap 'rm -f "$LOCK_FILE"' EXIT + +# Create backup directory +mkdir -p "$BACKUP_DIR" +echo "Backup directory: $BACKUP_DIR" + +# Check 1: Verify we're in the right directory +if [ ! -f "$INSTALL_FOLDER/run.py" ]; then + echo "ERROR: run.py not found in $INSTALL_FOLDER" + exit 1 +fi +echo "✓ Install folder verified: $INSTALL_FOLDER" + +cd "$INSTALL_FOLDER" + +# Check 2: Verify config.py exists +if [ ! -f "$INSTALL_FOLDER/config.py" ]; then + echo "ERROR: config.py not found" + exit 1 +fi +echo "✓ config.py exists" + +# Check 3: Test database connection +echo "Testing database connection..." +python3 -c " +import sys +sys.path.insert(0, '$INSTALL_FOLDER') +try: + from config_parser import parse_config + config = parse_config('config') + from sqlalchemy import create_engine, text + engine = create_engine(config['DATABASE_URI']) + conn = engine.connect() + conn.execute(text('SELECT 1')) + conn.close() + print('Database connection OK') +except Exception as e: + print(f'Database connection FAILED: {e}') + sys.exit(1) +" || { + echo "ERROR: Database connection failed" + exit 1 +} +echo "✓ Database connection OK" + +# Check 4: Store current state for rollback +CURRENT_COMMIT=$(git rev-parse HEAD) +echo "$CURRENT_COMMIT" > "$BACKUP_DIR/previous_commit.txt" +echo "✓ Current commit saved: $CURRENT_COMMIT" + +# Check 5: Store current migration version +FLASK_APP=./run.py python3 -m flask db current 2>/dev/null > "$BACKUP_DIR/previous_migration.txt" || true +if [ -s "$BACKUP_DIR/previous_migration.txt" ]; then + echo "✓ Migration version saved: $(cat $BACKUP_DIR/previous_migration.txt | head -1)" +else + echo "✓ Migration version saved: (none or could not determine)" +fi + +# Check 6: Check disk space (need at least 500MB free) +FREE_SPACE_KB=$(df -k "$INSTALL_FOLDER" | tail -1 | awk '{print $4}') +FREE_SPACE_MB=$((FREE_SPACE_KB / 1024)) +if [ "$FREE_SPACE_MB" -lt 500 ]; then + echo "ERROR: Less than 500MB free disk space (${FREE_SPACE_MB}MB available)" + exit 1 +fi +echo "✓ Disk space OK (${FREE_SPACE_MB}MB free)" + +# Check 7: Verify git repository is clean (no uncommitted changes) +if ! git diff --quiet 2>/dev/null; then + echo "WARNING: Uncommitted changes detected in repository" + git status --short +fi + +# Check 8: Verify logs directory ownership +LOGS_DIR="$INSTALL_FOLDER/logs" +WEB_USER="${WEB_USER:-www-data}" +if [ -d "$LOGS_DIR" ]; then + LOGS_OWNER=$(stat -c '%U' "$LOGS_DIR" 2>/dev/null || echo "unknown") + if [ "$LOGS_OWNER" != "$WEB_USER" ]; then + echo "WARNING: Logs directory owned by '$LOGS_OWNER', should be '$WEB_USER'" + echo "Fixing ownership..." + chown -R "$WEB_USER:$WEB_USER" "$LOGS_DIR" 2>/dev/null || { + echo "ERROR: Failed to fix logs ownership. Run manually:" + echo " sudo chown -R $WEB_USER:$WEB_USER $LOGS_DIR" + exit 1 + } + echo "✓ Logs directory ownership fixed" + else + echo "✓ Logs directory ownership OK ($WEB_USER)" + fi +else + echo "Creating logs directory with correct ownership..." + mkdir -p "$LOGS_DIR" + chown "$WEB_USER:$WEB_USER" "$LOGS_DIR" 2>/dev/null || { + echo "WARNING: Could not set logs ownership (run as root)" + } + echo "✓ Logs directory created" +fi + +# Export backup directory for other scripts +echo "$BACKUP_DIR" > /tmp/sp-deploy-backup-dir.txt + +echo "" +echo "=== Pre-deployment checks PASSED ===" +echo "Backup directory: $BACKUP_DIR" +exit 0 diff --git a/install/deploy/rollback.sh b/install/deploy/rollback.sh new file mode 100755 index 000000000..0dd32c047 --- /dev/null +++ b/install/deploy/rollback.sh @@ -0,0 +1,164 @@ +#!/bin/bash +# Rollback script - restore previous working state +# Called automatically when post_deploy.sh fails +# +# This script attempts to restore the previous state: +# 1. Restore previous git commit +# 2. Downgrade database migrations (if applicable) +# 3. Reinstall previous dependencies +# 4. Reload application +# 5. Verify rollback was successful + +set -e + +INSTALL_FOLDER="${INSTALL_FOLDER:-/var/www/sample-platform}" +BACKUP_DIR_FILE="/tmp/sp-deploy-backup-dir.txt" + +echo "=== ROLLBACK INITIATED ===" +echo "Timestamp: $(date -Iseconds)" +echo "" + +# Get backup directory +if [ -f "$BACKUP_DIR_FILE" ]; then + BACKUP_DIR=$(cat "$BACKUP_DIR_FILE") + echo "Backup directory: $BACKUP_DIR" +else + echo "ERROR: Backup directory file not found: $BACKUP_DIR_FILE" + echo "Cannot determine previous state for rollback" + echo "MANUAL INTERVENTION REQUIRED" + exit 1 +fi + +cd "$INSTALL_FOLDER" + +# Step 1: Get previous commit +echo "" +echo "--- Step 1: Restoring previous code ---" +if [ -f "$BACKUP_DIR/previous_commit.txt" ]; then + PREVIOUS_COMMIT=$(cat "$BACKUP_DIR/previous_commit.txt") + echo "Rolling back to commit: $PREVIOUS_COMMIT" + + git fetch origin + git checkout "$PREVIOUS_COMMIT" + echo "✓ Code rolled back to $PREVIOUS_COMMIT" +else + echo "WARNING: No previous commit recorded" + echo "Attempting to use git reflog..." + PREVIOUS_COMMIT=$(git reflog | head -2 | tail -1 | awk '{print $1}') + if [ -n "$PREVIOUS_COMMIT" ]; then + git checkout "$PREVIOUS_COMMIT" + echo "✓ Code rolled back to $PREVIOUS_COMMIT (from reflog)" + else + echo "ERROR: Could not determine previous commit" + echo "MANUAL INTERVENTION REQUIRED" + exit 1 + fi +fi + +# Step 2: Downgrade database migration if needed +echo "" +echo "--- Step 2: Checking database migrations ---" +if [ -f "$BACKUP_DIR/previous_migration.txt" ]; then + PREVIOUS_MIGRATION=$(cat "$BACKUP_DIR/previous_migration.txt" | grep -oP '[a-f0-9]{12}' | head -1 || true) + if [ -n "$PREVIOUS_MIGRATION" ]; then + CURRENT_MIGRATION=$(FLASK_APP=./run.py python3 -m flask db current 2>/dev/null | grep -oP '[a-f0-9]{12}' | head -1 || true) + if [ "$CURRENT_MIGRATION" != "$PREVIOUS_MIGRATION" ]; then + echo "Rolling back database to migration: $PREVIOUS_MIGRATION" + FLASK_APP=./run.py python3 -m flask db downgrade "$PREVIOUS_MIGRATION" || { + echo "WARNING: Database downgrade failed" + echo "Current migration: $CURRENT_MIGRATION" + echo "Target migration: $PREVIOUS_MIGRATION" + echo "May need manual database intervention" + } + echo "✓ Database migration rolled back" + else + echo "✓ Database migration unchanged (already at $PREVIOUS_MIGRATION)" + fi + else + echo "✓ No specific migration version to restore" + fi +else + echo "✓ No previous migration info, skipping database rollback" +fi + +# Step 3: Reinstall dependencies for rolled-back version +echo "" +echo "--- Step 3: Reinstalling dependencies ---" +python3 -m pip install -r requirements.txt --quiet --disable-pip-version-check --break-system-packages +echo "✓ Dependencies reinstalled" + +# Step 4: Reload application +echo "" +echo "--- Step 4: Reloading application ---" +if systemctl is-active --quiet platform; then + systemctl reload platform + echo "✓ Platform service reloaded" +else + systemctl start platform || { + echo "WARNING: Could not start platform service" + echo "Check service status manually: systemctl status platform" + } +fi + +# Step 5: Verify rollback worked +echo "" +echo "--- Step 5: Verifying rollback ---" +sleep 3 + +# Try health endpoint first, fallback to root +HEALTH_URL="${HEALTH_URL:-http://127.0.0.1/health}" +FALLBACK_URL="${FALLBACK_URL:-http://127.0.0.1/}" + +HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$HEALTH_URL" 2>/dev/null || echo "000") + +if [ "$HTTP_CODE" = "200" ]; then + echo "✓ Health check passed (HTTP $HTTP_CODE)" + ROLLBACK_SUCCESS=true +elif [ "$HTTP_CODE" = "404" ]; then + # Health endpoint doesn't exist, try fallback + HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$FALLBACK_URL" 2>/dev/null || echo "000") + if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 400 ]; then + echo "✓ Fallback check passed (HTTP $HTTP_CODE)" + ROLLBACK_SUCCESS=true + else + ROLLBACK_SUCCESS=false + fi +else + ROLLBACK_SUCCESS=false +fi + +echo "" +echo "=== ROLLBACK COMPLETE ===" +echo "" +echo "Summary:" +echo " Restored commit: ${PREVIOUS_COMMIT:-unknown}" +echo " Health status: HTTP $HTTP_CODE" + +if [ "$ROLLBACK_SUCCESS" = true ]; then + echo " Result: SUCCESS" + echo "" + echo "Application has been rolled back to the previous version." +else + echo " Result: VERIFICATION FAILED" + echo "" + echo "WARNING: Rollback completed but application health check failed." + echo "MANUAL INTERVENTION MAY BE REQUIRED" + echo "" + echo "Troubleshooting steps:" + echo " 1. Check logs: tail -f $INSTALL_FOLDER/logs/error.log" + echo " 2. Check service: systemctl status platform" + echo " 3. Check nginx: tail -f /var/log/nginx/error.log" +fi + +echo "" +echo "Please investigate the failed deployment before retrying." + +# Clean up lock file if it exists +rm -f /tmp/sp-deploy.lock + +# Exit with appropriate code +if [ "$ROLLBACK_SUCCESS" = true ]; then + exit 0 +else + exit 1 +fi diff --git a/install/generate_dev_credentials.py b/install/generate_dev_credentials.py new file mode 100644 index 000000000..b9a245405 --- /dev/null +++ b/install/generate_dev_credentials.py @@ -0,0 +1,44 @@ +"""Write a throwaway Google service-account file for local development. + +run.py builds a storage client at import time, which needs a syntactically +valid service-account JSON even when there is no GCP project behind it. The +key generated here authenticates nothing. +""" +import json +from pathlib import Path + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +# Matches SERVICE_ACCOUNT_FILE in config.docker.py. Not a command line +# argument: nothing needs to pick the location, and accepting one would only +# be a way to write outside the application directory. +TARGET = Path(__file__).resolve().parent.parent / 'service-account.json' + + +def main() -> None: + """Write the credentials file unless one is already present.""" + if TARGET.exists(): + return + + key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ).decode() + + TARGET.write_text(json.dumps({ + "type": "service_account", + "project_id": "sample-platform-dev", + "private_key_id": "dev", + "private_key": private_key, + "client_email": "dev@sample-platform-dev.iam.gserviceaccount.com", + "client_id": "0", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + }, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/install/init_db.py b/install/init_db.py index b8bdf1de7..8a6421075 100644 --- a/install/init_db.py +++ b/install/init_db.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import sys from os import path diff --git a/install/init_schema.py b/install/init_schema.py new file mode 100644 index 000000000..0754d8f26 --- /dev/null +++ b/install/init_schema.py @@ -0,0 +1,66 @@ +"""Bring the database schema up to date, creating it on the first start. + +The migration chain cannot be replayed against an empty database: early +revisions drop foreign keys by the names MySQL generates automatically +(``regression_test_ibfk_1`` and friends), and those exist only when the +schema was first built from the models rather than from the migrations. A +fresh database is therefore created from the models and stamped at the +current head, which is what the application does anyway on its first +request. A database that already carries an ``alembic_version`` table -- an +existing environment, or one restored from a dump -- only gets the pending +migrations applied. + +Creating the tables is deliberately skipped in that second case: doing both +would let ``create_all`` add a table that a pending migration still expects +to create itself, which then fails. + +A fresh database is also seeded, because several pages read rows they assume +are always present -- the home page dereferences the ``last_commit`` entry +without a null check -- and an empty schema alone therefore serves a 500. +""" +import subprocess +import sys +from os import path + +from flask_migrate import stamp, upgrade +from sqlalchemy import create_engine, inspect + +# Need to append server root path to ensure we can import the necessary files. +ROOT = path.dirname(path.dirname(path.abspath(__file__))) +sys.path.append(ROOT) + + +def main() -> int: + """Create or migrate the schema, depending on what is already there.""" + from database import create_session + from run import app, config + + uri = config['DATABASE_URI'] + engine = create_engine(uri) + try: + established = inspect(engine).has_table('alembic_version') + finally: + engine.dispose() + + with app.app_context(): + if established: + print('existing database: applying pending migrations', flush=True) + upgrade() + else: + print('fresh database: creating schema from the models', flush=True) + create_session(uri) + stamp() + + if not established: + # Run out of process: sample_db seeds on import and reads the URI from + # argv, so it cannot simply be called. + print('seeding development data', flush=True) + subprocess.check_call( + [sys.executable, path.join(ROOT, 'install', 'sample_db.py'), uri]) + + print('schema is up to date', flush=True) + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/install/install.sh b/install/install.sh old mode 100755 new mode 100644 index 8bda0bfe2..6a8a904f5 --- a/install/install.sh +++ b/install/install.sh @@ -26,31 +26,47 @@ echo "-------------------------------" echo "" echo "* Updating package list" apt-get update >> "$install_log" 2>&1 -echo "* Installing nginx, python, pip, kvm, libvirt and virt-manager" -apt-get -q -y install nginx python3 python-is-python3 python3-libvirt libxslt1-dev python3-libxml2 python3-pip qemu-system-x86 libvirt-daemon-system libvirt-clients virt-manager mediainfo >> "$install_log" 2>&1 -for file in /etc/init.d/mysql* -do - if [ ! -f "$file" ]; then - mysql_user_resp="Y" - while [ "$mysql_user_resp" != "N" ]; do - echo "* Installing MySQL (root password will be empty!)" - apt-get install -y mysql-server >> "$install_log" 2>&1 - if [ $? -ne 0 ]; then - read -e -r -p "MySQL installation failed! Do you want to try again? [Y for yes | N for No | Q to quit installation] " -i "Y" mysql_user_resp - if [ "$mysql_user_resp" = "Q" ]; then - exit 1 - fi - fi - if [ -f "$file" ]; then - break - fi - done +echo "* Installing nginx, python3, pip, mediainfo and gunicorn" +add-apt-repository ppa:deadsnakes/ppa -y >> "$install_log" 2>&1 +apt-get -q -y install python3.9 nginx python3.9-distutils python3-pip mediainfo gunicorn3 >> "$install_log" 2>&1 +update-alternatives --install /usr/bin/python python /usr/bin/python3 1 >> "$install_log" 2>&1 +update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3 1 >> "$install_log" 2>&1 +rm -f /etc/nginx/sites-available/default +rm -f /etc/nginx/sites-enabled/default +if [ ! -f /etc/init.d/mysql* ]; then + echo "* Installing MySQL (root password will be empty!)" + apt-get install -y mysql-server >> "$install_log" 2>&1 + if [ ! -f /etc/init.d/mysql* ]; then + echo "Failed to install MySQL! Please check the installation log!" + exit -1 fi -done +fi + +RED='\033[0;31m' +NC='\033[0m' + +userInput () { + # Utility function to take user inputs + # $1 -> Variable Name + # $2 -> Variable details + # $3 -> Default Value (pass "" if none) + # $4 -> Variable importance description (if variable is set to not required) + # $5 -> Whether variable is required or not (pass 1 if required) + while : ; do + read -e -r -p "$(tput bold)$2$(tput sgr0) " -i "$3" newinput + [[ $newinput == "" && $5 == "1" ]] || break + echo "This is a required parameter, cannot be empty!" + done + printf -v $1 "$newinput" + if [[ $newinput == "" && $4 != "" ]]; then + echo -e "${RED}WARNING${NC}: ${2}\b not set, ${4} might not work as intended." + fi +} + echo "* Update pip, setuptools and wheel" -python -m pip install --upgrade pip setuptools wheel >> "$install_log" 2>&1 +python3 -m pip install --upgrade pip setuptools wheel --break-system-packages >> "$install_log" 2>&1 echo "* Installing pip dependencies" -pip install -r "${root_dir}/requirements.txt" >> "$install_log" 2>&1 +python3 -m pip install -r "${root_dir}/requirements.txt" --break-system-packages >> "$install_log" 2>&1 echo "" echo "-------------------------------" echo "| Configuration |" @@ -58,87 +74,102 @@ echo "-------------------------------" echo "" echo "In order to configure the platform, we need some information from you. Please reply to the following questions:" echo "" -read -s -e -r -p "Password of the 'root' user of MySQL: " -i "" db_root_password +read -s -e -r -p "$(tput bold)Password of the 'root' user of MySQL:$(tput sgr0) " -i "" db_root_password echo "" # Verify password supress_warning=$(mysql_config_editor set --login-path=root_login --host=localhost --user=root --password "${db_root_password}") >> "$install_log" 2>&1 -while ! mysql --login-path=root_login -e ";" ; do - read -s -e -r -p "Invalid password, please retry: " -i "" db_root_password +while ! MYSQL_PWD=$db_root_password mysql --login-path=root_login -e ";" ; do + read -s -e -r -p "$(tput bold)Invalid password, please retry:$(tput sgr0) " -i "" db_root_password echo "" supress_warning=$(mysql_config_editor set --login-path=root_login --host=localhost --user=root --password "${db_root_password}") >> "$install_log" 2>&1 done - -read -e -r -p "Database name for storing data: " -i "sample_platform" db_name -mysql -u root --password="${db_root_password}" -e "CREATE DATABASE IF NOT EXISTS ${db_name};" >> "$install_log" 2>&1 +userInput db_name "Database name for storing data:" "sample_platform" "" 1 +MYSQL_PWD=$db_root_password mysql --login-path=root_login -e "CREATE DATABASE IF NOT EXISTS ${db_name};" >> "$install_log" 2>&1 # Check if DB exists -db_exists=$(mysql --login-path=root_login -se"USE ${db_name};" 2>&1) +db_exists=$(MYSQL_PWD=$db_root_password mysql --login-path=root_login -se"USE ${db_name};" 2>&1) if [ ! "${db_exists}" == "" ]; then echo "Failed to create the database! Please check the installation log!" exit -1 fi -read -e -r -p "Username to connect to ${db_name}: " -i "sample_platform" db_user +userInput db_user "Username to connect to ${db_name}:" "sample_platform" "" 1 # Check if user exists -db_user_exists=$(mysql --login-path=root_login -sse "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '${db_user}')") +db_user_exists=$(MYSQL_PWD=$db_root_password mysql --login-path=root_login -sse "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '${db_user}')") if [ "${db_user_exists}" = 0 ]; then rand_pass=$(< /dev/urandom tr -dc 'a-zA-Z0-9' | fold -w 16 | head -n 1) - read -e -r -p "Password for ${db_user} (will be created): " -i "${rand_pass}" db_user_password + userInput db_user_password "Password for ${db_user} (will be created):" $rand_pass "" # Attempt to create the user - mysql --login-path=root_login -e "CREATE USER '${db_user}'@'localhost' IDENTIFIED BY '${db_user_password}';" >> "$install_log" 2>&1 - db_user_exists=$(mysql --login-path=root_login -sse "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '$db_user')") + MYSQL_PWD=$db_root_password mysql --login-path=root_login -e "CREATE USER '${db_user}'@'localhost' IDENTIFIED BY '${db_user_password}';" >> "$install_log" 2>&1 + db_user_exists=$(MYSQL_PWD=$db_root_password mysql --login-path=root_login -sse "SELECT EXISTS(SELECT 1 FROM mysql.user WHERE user = '$db_user')") if [ "${db_user_exists}" = 0 ]; then echo "Failed to create the user! Please check the installation log!" exit -1 fi else - read -s -e -r -p "Password for ${db_user}: " db_user_password - supress_warning=$(mysql_config_editor set --login-path=check_login --host=localhost --user="${db_user}" --password "${db_root_password}") >> "$install_log" 2>&1 + read -s -e -r -p "$(tput bold)Password for ${db_user}:$(tput sgr0) " db_user_password # Check if we have access - while ! mysql --login-path=check_login -e ";" ; do - read -s -e -r -p "Invalid password, please retry: " -i "" db_user_password - supress_warning=$(mysql_config_editor set --login-path=check_login --host=localhost --user="${db_user}" --password "${db_root_password}") >> "$install_log" 2>&1 + while ! MYSQL_PWD=$db_user_password mysql -u $db_user -e ";" ; do + read -s -e -r -p "$(tput bold)Invalid password, please retry:$(tput sgr0) " -i "" db_user_password done fi supress_warning=$(mysql_config_editor set --login-path=user_login --host=localhost --user="${db_user}" --password "${db_user_password}") >> "$install_log" 2>&1 # Grant user access to database -mysql --login-path=root_login -e "GRANT ALL ON ${db_name}.* TO '${db_user}'@localhost;" >> "$install_log" 2>&1 +MYSQL_PWD=$db_root_password mysql --login-path=root_login -e "GRANT ALL ON ${db_name}.* TO '${db_user}'@localhost;" >> "$install_log" 2>&1 # Check if user has access -db_access=$(mysql --login-path=user_login -se "USE ${db_name};" 2>&1) +db_access=$(MYSQL_PWD=$db_root_password mysql --login-path=user_login -se "USE ${db_name};" 2>&1) if [ ! "${db_access}" == "" ]; then echo "Failed to grant user access to database! Please check the installation log!" exit -1 fi -read -r -p "Do you want to install a sample database? (y/n) :" sample_response +userInput sample_response "Do you want to install a sample database? (Enter 'y' for yes):" # Request information for generating the config.py file echo "" echo "For the following questions, press enter to leave a field blank." -read -e -r -p "(Sub)domain this will be running on? " -i "" config_server_name -read -e -r -p "Application root (if not a whole (sub)domain, enter the path. None if whole (sub)domain): " -i "None" config_application_root -read -e -r -p "GitHub Token (Generate here : https://help.github.com/articles/creating-an-access-token-for-command-line-use/): " -i "" github_token -read -e -r -p "GitHub Owner Name : " -i "CCExtractor" github_owner_name -read -e -r -p "GitHub repository : " -i "ccextractor" github_repository -read -e -r -p "Email Domain : " -i "${config_server_name}" email_domain -read -e -r -p "Email API key (Generate one here https://www.mailgun.com/) : " -i "" email_api_key +userInput config_server_name "(Sub)domain this will be running on?" "127.0.0.1" "" 1 +userInput config_application_root "Application root (if not a whole (sub)domain, enter the path. None if whole (sub)domain):" "None" "" 1 +echo "You can get details about creating a Personal-Access-Token, https://help.github.com/articles/creating-an-access-token-for-command-line-use/)" +userInput github_token "GitHub Token:" "" "Creating GitHub issues, updating comments on Pull Requests" 0 +userInput github_owner_name "GitHub Owner Name:" "CCExtractor" "" 1 +userInput github_repository "GitHub repository:" "ccextractor" "" 1 +userInput email_domain "Email Domain:" $config_server_name "" 1 +echo "You can generate your own Email API key here, https://www.mailgun.com/)" +userInput email_api_key "Email API key:" "" "Authentication, Email notification related functions" 0 hmac_key=$(head -80 /dev/urandom | LC_ALL=c tr -dc 'a-zA-Z0-9' | fold -w 32 | head -n 1) -read -e -r -p "GitHub Automated Deploy Webhook Secret (More info : https://developer.github.com/webhooks/) : " -i "" github_deploy_key -read -e -r -p "GitHub CI Webhook Secret (More info: https://developer.github.com/webhooks/) : " -i "" github_ci_key -read -e -r -p "KVM Linux Name: " -i "" kvm_linux_name -read -e -r -p "KVM Windows Name: " -i "" kvm_windows_name -read -e -r -p "KVM Max Runtime (In minutes): " -i "120" kvm_max_runtime -read -e -r -p "FTP Server IP/Domain name :" -i "" server_name -read -e -r -p "FTP port: " -i "21" ftp_port -read -e -r -p "Max HTTP sample size (in bytes) : " -i "536870912" max_content_length -read -e -r -p "Minimum password length : " -i "10" min_pwd_len -read -e -r -p "Maximum password length : " -i "500" max_pwd_len +echo "You can get details about creating WEBHOOK_SECRET here, https://developer.github.com/webhooks/" +userInput github_ci_key "GitHub CI Webhook Secret:" "" "CI related functions (mod_ci module)" 0 +echo "Get the GitHub client ID and client secret for the platform, by registering the platform here, https://github.com/settings/applications/new" +echo "Enter /account/github_callback endpoint as the callback URL, you may opt to skip GitHub client ID, client secret for now, and later add it manually" +userInput github_client_id "GitHub Client ID:" "" "GitHub User Authentication" 0 +userInput github_client_secret_key "GitHub Client Secret:" "" "GitHub User Authentication" 0 +userInput server_name "FTP Server IP/Domain name:" $config_server_name "" 1 +userInput ftp_port "FTP port:" "21" "" 1 +userInput max_content_length "Max HTTP sample size (in bytes):" "536870912" "" 1 +userInput min_pwd_len "Minimum password length:" "10" "" 1 +userInput max_pwd_len "Maximum password length:" "500" "" 1 + +userInput gcp_service_account_file "GCP service account file name (to be placed at root of project):" "service-account.json" "" 1 +userInput sample_platform_project_name "GCP Project Name for the platform:" "" "" 1 +userInput sample_platform_project_number "GCP Project Number for the platform:" "" "" 1 +userInput gcp_instance_zone_name "Zone name for GCP Instances:" "us-central1-a" "" 1 +userInput gcp_instance_machine_type "Machine type for GCP Instances:" "n1-standard-1" "" 1 +userInput windows_instance_project_name "Windows GCP instance project name:" "windows-cloud" "" 1 +userInput windows_instance_family_name "Windows GCP instance family name:" "windows-2019" "" 1 +userInput linux_instance_project_name "Linux GCP instance project name:" "ubuntu-os-cloud" "" 1 +userInput linux_instance_family_name "Linux GCP instance family name: " "ubuntu-minimal-2404-lts-amd64" "" 1 +userInput gcp_instance_max_runtime "GCP Instance Max Runtime (In minutes):" "120" "" 1 +userInput gcs_bucket_name "Google Cloud Storage bucket name:" "" "" 1 +userInput gcs_bucket_location "Google Cloud Storage bucket location:" "us-central1" "" 1 +userInput gcs_bucket_location_type "Google Cloud Storage bucket location type:" "REGIONAL" "" 1 +userInput signed_url_expiry_time "Signed Download URLs expiry time (In minutes):" "720" "" 1 echo "" echo "In the following lines, enter the path " -read -e -r -p " To SSL certificate: " -i "/etc/letsencrypt/live/${config_server_name}/fullchain.pem" config_ssl_cert -read -e -r -p " To SSL key: " -i "/etc/letsencrypt/live/${config_server_name}/privkey.pem" config_ssl_key -read -e -r -p " To the root directory containing all files (Samples, reports etc.) : " -i "/repository" sample_repository +userInput config_ssl_cert " To SSL certificate:" "/etc/letsencrypt/live/${config_server_name}/fullchain.pem" "" 1 +userInput config_ssl_key " To SSL key:" "/etc/letsencrypt/live/${config_server_name}/privkey.pem" "" 1 +userInput sample_repository " To the root directory containing all files (Samples, reports etc.):" "/repository" "" 1 echo "Setting up the directories.." @@ -152,39 +183,34 @@ echo "Setting up the directories.." mkdir -p "${sample_repository}/TestFiles" mkdir -p "${sample_repository}/TestFiles/media" mkdir -p "${sample_repository}/QueuedFiles" + mkdir -p "${sample_repository}/TestData/ci-linux" + mkdir -p "${sample_repository}/TestData/ci-windows" } >> "$install_log" 2>&1 config_db_uri="mysql+pymysql://${db_user}:${db_user_password}@localhost:3306/${db_name}" # Request info for creating admin account -echo "" -echo "We need some information for the admin account" -read -e -r -p "Admin username: " -i "admin" admin_name +userInput admin_name "Admin username:" "admin" "We need some information for the admin account" 1 +userInput admin_email "Admin email:" "" "" 1 -while [ -z $admin_email ];do - echo "Enter Admin email ( It can't be empty! )" - read -e -r -p "Admin email: " admin_email -done - -while [ -z $admin_password ];do - echo "Enter Admin password (size of password >1) " +echo "Enter Admin password (size of password >= 1)" +while [[ $admin_password == "" ]];do read -s -e -r -p "Admin password: " admin_password - echo " " + echo "" read -s -e -r -p "Confirm admin password: " confirm_admin_password - -done -while [ $admin_password != $confirm_admin_password ]; do - echo "Entered passwords did not match! Retrying..." - read -s -e -r -p "Admin password: " admin_password - echo "" - read -s -e -r -p "Confirm admin password: " confirm_admin_password + echo "" + if [[ $admin_password != $confirm_admin_password ]]; then + echo "Entered passwords did not match! Retrying..." + admin_password="" + fi done echo "Creating admin account: " -python "${dir}/init_db.py" "${config_db_uri}" "${admin_name}" "${admin_email}" "${admin_password}" +python3 "${root_dir}/install/init_db.py" "${config_db_uri}" "${admin_name}" "${admin_email}" "${admin_password}" # Create sample database if user wanted to if [ "${sample_response}" == 'y' ]; then echo "Creating sample database.." - cp -r sample_files/* "${sample_repository}/TestFiles" - python "${dir}/sample_db.py" "${config_db_uri}" + sample_dir="${dir}/sample_files/*" + cp -r $sample_dir "${sample_repository}/TestFiles" + python3 "${dir}/sample_db.py" "${config_db_uri}" fi echo "" echo "-------------------------------" @@ -208,42 +234,71 @@ SERVER_NAME = '${server_name}' EMAIL_DOMAIN = '${email_domain}' EMAIL_API_KEY = '${email_api_key}' HMAC_KEY = '${hmac_key}' -GITHUB_DEPLOY_KEY = '${github_deploy_key}' GITHUB_CI_KEY = '${github_ci_key}' +GITHUB_CLIENT_ID = '${github_client_id}' +GITHUB_CLIENT_KEY = '${github_client_secret_key}' INSTALL_FOLDER = '${root_dir}' -KVM_LINUX_NAME = '${kvm_linux_name}' -KVM_WINDOWS_NAME = '${kvm_windows_name}' -KVM_MAX_RUNTIME = $kvm_max_runtime # In minutes SAMPLE_REPOSITORY = '${sample_repository}' SESSION_COOKIE_PATH = '/' +# Where the web console is served. Recovery emails link here instead of the +# classic pages when it is set; leave empty to keep the old behaviour. +CONSOLE_URL = 'https://${config_server_name}/app' FTP_PORT = $ftp_port MAX_CONTENT_LENGTH = $max_content_length MIN_PWD_LEN = $min_pwd_len MAX_PWD_LEN = $max_pwd_len -" > "${dir}/../config.py" + + +# GCP SPECIFIC CONFIG +SCOPES = ['https://www.googleapis.com/auth/cloud-platform'] +SERVICE_ACCOUNT_FILE = '${gcp_service_account_file}' +ZONE = '${gcp_instance_zone_name}' +PROJECT_NAME = '${sample_platform_project_name}' +MACHINE_TYPE = f'zones/{ZONE}/machineTypes/${gcp_instance_machine_type}' +WINDOWS_INSTANCE_PROJECT_NAME = '${windows_instance_project_name}' +WINDOWS_INSTANCE_FAMILY_NAME = '${windows_instance_family_name}' +LINUX_INSTANCE_PROJECT_NAME = '${linux_instance_project_name}' +LINUX_INSTANCE_FAMILY_NAME = '${linux_instance_family_name}' +GCP_INSTANCE_MAX_RUNTIME = $gcp_instance_max_runtime # In minutes +GCS_BUCKET_NAME = '${gcs_bucket_name}' +GCS_SIGNED_URL_EXPIRY_LIMIT = $signed_url_expiry_time # In minutes" > "${dir}/../config.py" # Ensure the files are executable by www-data chown -R www-data:www-data "${root_dir}" "${sample_repository}" echo "* Creating startup script" { - cp "${dir}/platform" /etc/init.d/platform - sed -i "s#BASE_DIR#${root_dir}#g" /etc/init.d/platform - chmod 755 /etc/init.d/platform - update-rc.d platform defaults + rm -f /etc/init.d/platform + update-rc.d platform remove || true + cp "${dir}/platform.service" /etc/systemd/system/platform.service + sed -i "s|#BASE_DIR#|${root_dir}|g" /etc/systemd/system/platform.service + systemctl daemon-reload + systemctl enable platform.service +} >> "$install_log" 2>&1 +echo "* Creating RClone config file" + +{ + cp "${root_dir}/install/ci-vm/ci-windows/rclone_sample.conf" "${root_dir}/install/ci-vm/ci-windows/rclone.conf" + sed -i "s#GCS_BUCKET_NAME#${gcs_bucket_name}#g" "${root_dir}/install/ci-vm/ci-windows/rclone.conf" + sed -i "s#GCP_PROJECT_NUMBER#${sample_platform_project_number}#g" "${root_dir}/install/ci-vm/ci-windows/rclone.conf" + sed -i "s#GCS_BUCKET_LOCATION_TYPE#${gcs_bucket_location_type}#g" "${root_dir}/install/ci-vm/ci-windows/rclone.conf" + sed -i "s#GCS_BUCKET_LOCATION#${gcs_bucket_location}#g" "${root_dir}/install/ci-vm/ci-windows/rclone.conf" } >> "$install_log" 2>&1 echo "* Creating Nginx config" { cp "${dir}/nginx.conf" /etc/nginx/sites-available/platform sed -i "s/NGINX_HOST/${config_server_name}/g" /etc/nginx/sites-available/platform - sed -i "s#SAMPLE_DIR#${sample_repository}/TestFiles/media#g" /etc/nginx/sites-available/platform - sed -i "s#LOGFILE_DIR#${sample_repository}/LogFiles#g" /etc/nginx/sites-available/platform - sed -i "s#RESULT_DIR#${sample_repository}/TestResults#g" /etc/nginx/sites-available/platform sed -i "s#NGINX_CERT#${config_ssl_cert}#g" /etc/nginx/sites-available/platform sed -i "s#NGINX_KEY#${config_ssl_key}#g" /etc/nginx/sites-available/platform sed -i "s#NGINX_DIR#${root_dir}#g" /etc/nginx/sites-available/platform ln -s /etc/nginx/sites-available/platform /etc/nginx/sites-enabled/platform } >> "$install_log" 2>&1 +echo "* Moving variables and runCI files" + +{ + cp $root_dir/install/ci-vm/ci-windows/ci/* "${sample_repository}/TestData/ci-windows/" + cp $root_dir/install/ci-vm/ci-linux/ci/* "${sample_repository}/TestData/ci-linux/" +} >> "$install_log" 2>&1 echo "* Reloading nginx" service nginx reload >> "$install_log" 2>&1 echo "" diff --git a/install/installation.md b/install/installation.md index 816c17b58..4464febf0 100644 --- a/install/installation.md +++ b/install/installation.md @@ -5,52 +5,140 @@ * Nginx (Other possible when modifying the sample download section) * Python 3 (Flask and other dependencies) * MySQL -* Pure-FTPD with mysql +* Pure-FTPD with mysql (optional, only needed for FTP file uploads) + +## Configuring Google Cloud Platform + +To configure the GCP for the platform, see [the installation guide](ci-vm/installation.md). ## Automated install -Automated install only works for the platform section, **not** for the KVM -functionality. To install the VM's for KVM, see -[the installation guide](ci-vm/installation.md). +Automated install only works for the platform section; make sure to have configured the GCP before continuing the platform installation. + +## Creation of sample platform server instance + +For deployment of the platform on a Google Cloud VM instance, one would require to create an instance and configure some network firewall settings. If you are deploying the platform on a local instance, you can skip these two steps. + +1. Creating a VM instance + + - Open the Google Cloud console, navigate to Compute Engine -> VM instances section, and click on "Create Instance". + - The following are the details of the VM instance to be entered (most of the default configuration below can be changed as per the requirements): + - Region: us-central1 (Iowa) + - Zone: us-central1-a + - Machine Family: General Purpose + - Series: E2 + - Machine Type: e2-small + - Boot Disk + - For Linux: + - OS: Ubuntu + - Version: Ubuntu 22.04 LTS (x86) + - Boot type disk: Balanced persistent disk + - Size: 10GB + - For Windows: + - OS: Windows Server + - Version: +Windows Server 2019 Datacenter + - Boot type disk: Balanced persistent disk + - Size: 50GB + - Navigate to Security and choose the service account as the service account you just created for the platform. + - Navigate to Network and select the "Allow HTTP traffic" and "Allow HTTPS traffic" checkboxes. + - Under Network Interfaces -> default, reserve a new static external IPv4 address for the platform. + +2. Setting up firewall settings + + To allow access to the platform through an external IPv4 address just created, there are some firewall configurations to be made: + - Navigate to VPC network -> Firewall and click on "Create Firewall Rule". + - Set the rule name as "default-allow-https" and enter the following details for this rule: + - Priority: 1000 + - Direction of Traffic: Ingress + - Action on the match: Allow + - Target type: Specified target tags -> Target Tags: "https-server" + - Source Filter: IPv4 ranges + - Source IPv4 ranges: 0.0.0.0/0 + - Protocols and ports -> Specified protocols and ports -> TCP -> Port: 443 (Nginx server is configured on this port) + - Now click on "Save" + - Now create another firewall rule for HTTP as "default-allow-http" with the following changes in the above configuration: + - Target type: Specified target tags -> Target Tags: "http-server" + - Protocols and ports -> Specified protocols and ports -> TCP -> Port: 80 + ### Linux Clone the latest sample-platform repository from https://github.com/CCExtractor/sample-platform. Note that root (or sudo) is required for both installation and running the program. -The `sample-repository` directory needs to be accessible by `www-data`. The +The `sample-platform` directory needs to be accessible by `www-data`, the recommended directory is thus `/var/www/`. ``` cd /var/www/ sudo git clone https://github.com/CCExtractor/sample-platform.git ``` +Place the service account key file that you generated earlier at the root of the sample-platform folder. + +#### Mounting the bucket + +Mounting on Linux OS can be done using [Google Cloud Storage FUSE](https://cloud.google.com/storage/docs/gcs-fuse). + +Steps: +- Install gcsfuse using [official documentation](https://cloud.google.com/storage/docs/cloud-storage-fuse/install) or using the following script + ``` + curl -L -O https://github.com/GoogleCloudPlatform/gcsfuse/releases/download/v0.39.2/gcsfuse_0.39.2_amd64.deb + sudo dpkg --install gcsfuse_0.39.2_amd64.deb + rm gcsfuse_0.39.2_amd64.deb + ``` +- Now, there are multiple ways to mount the bucket, official documentation [here](https://cloud.google.com/storage/docs/cloud-storage-fuse/mount-bucket). + + For Ubuntu and derivatives, assuming `/repository` to be the location of samples to be configured, an entry can be added to `/etc/fstab` file, replace _GCS_BUCKET_NAME_ with the name of the bucket created for the platform: + ``` + echo "GCS_BUCKET_NAME /repository gcsfuse rw,gid=33,noatime,async,_netdev,noexec,user,implicit_dirs,allow_other,file_mode=774,dir_mode=775 0 0" | sudo tee -a /etc/fstab + ``` + +- Now run the following command as root to mount the bucket: + ``` + sudo mkdir /repository + sudo mount /repository + ``` + +Note that **this directory needs to be accessible by the `www-data` user**, you can verify if the mount was successful by running `sudo -u www-data ls /repository` -Next, navigate to the `install` folder and run `install.sh` with root +#### Troubleshooting: Mounting of Bucket + +In case you get "permission denied" for `/repository`, you can check for the following reasons: +1. Check if the service account created has access to the GCS bucket. +2. Check the output of `sudo mount /repository` command. +3. Check the directory permissions for `/repository` + + +#### MySQL installation +The platform has been tested for MySQL v8.0 and Python 3.7 to 3.9. + +It is recommended to install python and MySQL beforehand to avoid any inconvenience. Here is the [installation link](https://www.digitalocean.com/community/tutorials/how-to-install-mysql-on-ubuntu-22-04) of MySQL on Ubuntu 22.04. + +#### Installing The Platform +Next, navigate to the `install` folder and run `install.sh` with root permissions. ``` cd sample-platform/install/ sudo ./install.sh -``` +``` -The `install.sh` will begin downloading and updating all the necessary -dependencies. Once done, it'll ask to enter some details in order to set up -the sample-platform. After filling in these the platform should be ready for -use. +The `install.sh` will begin downloading and updating all the necessary dependencies. Once done, it'll ask to enter some details in order to set up the sample-platform. After filling in these details, the platform should be ready for use. -Please read the below troubleshooting notes in case of any error or doubt. +When the domain is asked during installation, enter the domain name that will run the platform. E.g., if the platform will run locally, enter `localhost` as the server name. ### Windows -* Install cygwin (http://cygwin.com/install.html). When cygwin asks which - packages to install, select Python, MySql, virt-manager and openssh. If you - already have cygwin installed, you must run its setup file to install the new packages. Make sure the dropdown menu is set to Full, so you can all packages. To select one, click skip and it will change to the version number of the package. Use the end of [this](https://www.davidbaumgold.com/tutorials/set-up-python-windows/) tutorial for help on getting cygwin to recognize python. -* Start a terminal session once installation is complete. -* Install Nginx (see http://nginx.org/en/docs/windows.html) -* Install XMing XServer and setup Putty for ssh connections. -* Virt-manager can call ssh to make the connection to KVM Server and should be -able to run virsh and send commands to it. +* Install cygwin (http://cygwin.com/install.html). When cygwin asks which packages to install, select Python, MySql, and google-api-client. If you already have cygwin installed, you must run its setup file to install the new packages. Make sure the dropdown menu is set to Full, so you can see all packages. To select one, click skip and it will change to the version number of the package. Use the end of [this](https://www.davidbaumgold.com/tutorials/set-up-python-windows/) tutorial for help on getting cygwin to recognize python. +* Install [WinFsp](https://winfsp.dev/) and [Rclone](https://rclone.org/) from their official websites. +* Now rclone is a command line program, follow the [official documentation](https://rclone.org/googlecloudstorage/) to mount the google cloud storage bucket, using the service account key file. +* By default home directory of Cygwin is `C:\cygwin\home\\` (this can be obtained by running `cygpath -w ~` from cygwin terminal), and assuming `\repository` to be the location of samples to be configured, mount the bucket using rclone at `C:\cygwin\home\\repository` through the following command using command prompt: + ``` + rclone mount GCS_BUCKET_NAME MOUNT_LOCATION --no-console + ``` +* Start a terminal session once installation is complete. +* Install Nginx (see http://nginx.org/en/docs/windows.html) * Follow the steps from the Linux installation from within the Cygwin terminal to complete the platform's installation. @@ -68,55 +156,72 @@ or platform configuration.** it's **recommended** to use a valid certificate. [Let's Encrypt](https://letsencrypt.org/) offers free certificates. For local testing, a self-signed certificate can be enough. -* When the server name is asked during installation, enter the domain name -that will run the platform. E.g., if the platform will run locally, enter -`localhost` as the server name. * In case of a `502 Bad Gateway` response, the platform didn't start correctly. Manually running `bootstrap_gunicorn.py` (as root!) can help to determine what goes wrong. The snippet below shows how this can be done: ``` cd /var/www/sample-platform/ -sudo python bootstrap_gunicorn.py +sudo python3 bootstrap_gunicorn.py ``` * If `gunicorn` boots up successfully, most relevant logs will be stored in the `logs` directory. Otherwise they'll likely be in `syslog`. -* If it shows the error regarding the `libvirt`, then there is missing `fftw3.h` file. Try the following: -``` -sudo apt-get install libvirt-dev -sudo apt-get install libfftw3-dev -sudo apt-get install libsndfile1-dev -``` - * If any issue still persists, follow the mentioned steps to debug and troubleshoot your issue: 1. Firstly check the Platform Installation log file in the install folder. Check for any errors, which may have been caused during platform installation on your system, and then try to resolve them accordingly. 2. Next check for nginx status by `service nginx status` command, if it is not active, check nginx error log file, possibly in `/var/log/nginx/error.log` file. 3. Next check for platform status by `service platform status` command, if it is not `active(running)` then check for platform logs in the `logs` directory of your project. - 4. In case of any gunicorn error try manually running `/etc/init.d/platform start` command and recheck the platform status. + 4. In case of any gunicorn error try manually running `sudo systemctl start platform.service` command and recheck the platform status. + +### Setting Up The Bucket + +After the completion of the automated installation of the platform, the following folder structure is created in the 'SAMPLE_REPOSITORY' set during installation: +- `LogFiles/` - Directory containing log files of the tests completed +- `QueuedFiles/` - Directory containing files related to queued samples +- `README` - A readme file related to SSL certificates required by the platform +- `TempFiles/` - Directory containing temporary files +- `TestData/` - Directory containing files required for starting a test - runCI files, variables file, tester +- `TestFiles/` - Directory containing regression test samples +- `TestResults/` - Directory containing regression test results +- `vm_data/` - Directory containing test-specific subfolders, each folder containing files required for testing to be passed to the VM instance, test files and CCExtractor build artefact. + +Now for tests to run, we need to download the [CCExtractor testsuite](https://github.com/CCExtractor/ccx_testsuite) release file, extract and put it in the `TestData/ci-linux` and `TestData/ci-windows` folders. + +You also need to create a shell script named `ccextractortester` and place it in both `ci-linux` and `ci-windows`. This script is meant to launch the testsuite binary, here is what it should look like for Linux: +```sh +#!/bin/bash +exec mono CCExtractorTester.exe "$@" +``` -## Nginx configuration for X-Accel-Redirect +## Setting up GitHub webhooks -To serve files without any scripting language overhead, the X-Accel-Redirect -feature of Nginx is used. To enable it, a special section (as seen below) -needs to be added to the nginx configuration file: +Now that the server is running, you can queue new tests either manually (via `/custom/`) or automatically through GitHub Actions. -``` -location /protected/ { - internal; - alias /path/to/storage/of/samples/; # Trailing slash is important! -} -``` +To queue a test whenever a new commit/PR is made, you need to create a GitHub [webhook](https://docs.github.com/en/webhooks/about-webhooks) on the ccextractor repository (or fork of it): +- Set the payload URL to `https:///start-ci` +- Set the content type to JSON. +- Enter the same secret that you used during installation (`GITHUB_CI_KEY`) +- Select the Push, PR and Issue events as triggers. + +## Setting up cron job to run tests + +To run the new tests that are being queued up, a cron job is required. +The file `mod_ci/cron.py` is to be run in periodic intervals. To setup a cron job follow the steps below: +1. Open your terminal and enter the command `sudo crontab -e`. +2. To setup a cron job that runs this file every 10 minutes, append this at the bottom of the file + ``` + */10 * * * * python3 /var/www/sample-platform/mod_ci/cron.py > /var/www/sample-platform/logs/cron.log 2>&1 + ``` + Change the `/var/www/sample-plaform` directory, if you have installed the platform in a different directory. -More info on this directive is available at the -[Nginx wiki](http://wiki.nginx.org/NginxXSendfile). +## GCS configuration to serve file downloads using Signed URLs -Other web servers can be configured too (see this excellent -[SO](http://stackoverflow.com/a/3731639) answer), but will require a small -modification in the relevant section of the `serve_file_download` definition -in `mod_sample/controllers.py` which is responsible for handling the download -requests. +To serve file downloads directly from the private GCS bucket, Signed download URLs have been used. + +The `serve_file_download` function in the `utility.py` file implements the generation of a signed URL for the file to be downloaded that would expire after a configured time limit (maximum limit: 7 days) and redirects the client to the URL. + +For more information about Signed URLs, you can refer to the [official documentation](https://cloud.google.com/storage/docs/access-control/signed-urls). ## File upload size for HTTP @@ -195,24 +300,40 @@ Next, a file called `ChrootEveryone` must be created, so that the individual users are jailed: ``` -echo "yes" > /etc/pure-ftpd/conf/ChrootEveryone +echo "yes" | sudo tee /etc/pure-ftpd/conf/ChrootEveryone ``` The same needs to be done for `CreateHomeDir` and `CallUploadScript`: ``` -echo "yes" > /etc/pure-ftpd/conf/CreateHomeDir -echo "yes" > /etc/pure-ftpd/conf/CallUploadScript +echo "yes" | sudo tee /etc/pure-ftpd/conf/CreateHomeDir +echo "yes" | sudo tee /etc/pure-ftpd/conf/CallUploadScript +``` + +Set `PassivePortRange` for pure-ftpd: +``` +echo "30000 50000" | sudo tee /etc/pure-ftpd/conf/PassivePortRange ``` Also `/etc/default/pure-ftpd-common` needs some modification: ``` UPLOADSCRIPT=/path/to/cron/progress_ftp_upload.py -UPLOADUID=2015 # User that owns the upload.sh script -UPLOADGID=2015 # Group that owns the upload.sh script +UPLOADUID=33 # User that owns the upload.sh script +UPLOADGID=33 # Group that owns the upload.sh script ``` +You can get the UPLOADUID and UPLOADGID using the following command: +``` +ls -l /path/to/cron/progress_ftp_upload.py +``` + +Since we provide bucket access only to group `www-data`, we will now add `ftpuser` to the group: +``` +sudo usermod -a -G www-data ftpuser +``` +In manual installation, you may change `www-data` to the group you provided the bucket access. + When necessary, an appropriate value in the Umask file (`/etc/pure-ftpd/conf/Umask`) should be set as well. @@ -221,15 +342,57 @@ After this you Pure-FTPD can be restarted using Note: if there is no output saying: `Restarting ftp upload handler: pure-uploadscript.`, the uploadscript will -need to be started. This can be done using the next command (assuming 1000 is +need to be started. This can be done using the next command (assuming 33 is the `gid` and `uid` of the user which was specified earlier): ``` -sudo pure-uploadscript -u 2015 -g 2015 -B -r /home/path/to/src/cron/progress_ftp_upload.py -sudo chown 2015:2015 /home/path/to/src/cron/progress_ftp_upload.py - +sudo pure-uploadscript -u 33 -g 33 -B -r /home/path/to/src/cron/progress_ftp_upload.py +sudo chown 33:33 /home/path/to/src/cron/progress_ftp_upload.py ``` To check if the upload script is running, the next command can help: `ps aux | grep pure-uploadscript`. If it still doesn't work, rebooting the server might help as well. + +### Configure GCP Firewall for Pure-FTP +NOTE: These steps need to be performed only if the platform is being installed on a GCP VM instance. + +Since pure-ftpd server runs over TCP port 21 and we allow passive TCP port range as 30000-50000, we should now allow incoming communication requests through these ports to the platform server: + +- Navigate to VPC network -> Firewall and click on "Create Firewall Rule". +- Set the name as "default-allow-ftp-ingress" and enter the following details for this rule: + - Priority: 1000 + - Direction of Traffic: Ingress + - Action on the match: Allow + - Target type: "Allow instances in the network" + - Source Filter: IPv4 ranges + - Source IPv4 ranges: 0.0.0.0/0 + - Protocols and ports -> Specified protocols and ports -> TCP -> Port: 21,30000-50000 + - Now click on "Save" + +### Setup Automated Deployments for Platform via GitHub +To setup automated deployments via GitHub workflows, follow these steps: +- Ensure GitHub Actions is enabled in the repository (It is disabled for forks by default). +- Create two new GitHub repository variables, from the Settings tab + - Navigate to "Secrets and variables" -> "Actions" -> "Variables". + - Click on "New repository variable" and setup the following variables: + - `PLATFORM_DOMAIN`: The domain your platform is to be deployed, for example: `sampleplatform.ccextractor.org`. + - `SSH_USER`: The user that has root access and would be used to SSH into the server. + + (You can see a list of users in your system by running the command ```less /etc/passwd```) + + - Now get the SSH private and public(`.pub`) keys by running the following command locally: + ``` + ssh-keygen -t ed25519 -C "your_email@example.com" + ``` + Note: If you are using a legacy system that doesn't support the Ed25519 algorithm, use: + ``` + ssh-keygen -t rsa -b 4096 -C "your_email@example.com" + ``` + - Now SSH into the VM instance where the platform is to be deployed, open the file `/home//.ssh/authorized_keys` and append the contents of the public key to the end of the file. + - Now the last step is to add the SSH private key to the GitHub secrets + - Navigate to "Secrets and variables" -> "Actions" -> "Secrets". + - Click on "New repository secret" and setup the following variable: + - `SSH_KEY_PRIVATE`: Save the contents of the private SSH key file created in the last step as this secret. + - Also checkout the variables `INSTALL_FOLDER` and `SAMPLE_REPOSITORY` in the [deployment pipeline](/.github/workflows/sp-deployment-pipeline.yml) in case you have configured values other than default. + diff --git a/install/nginx.conf b/install/nginx.conf index f97627aa0..3158c6421 100644 --- a/install/nginx.conf +++ b/install/nginx.conf @@ -33,28 +33,42 @@ server { ssl_prefer_server_ciphers on; ssl_ecdh_curve secp384r1; + # Samples are media files and the API accepts up to MAX_CONTENT_LENGTH + # (512 MB). Without this nginx applies its 1 MB default and rejects an + # upload with its own 413 before the application ever sees it. + client_max_body_size 512m; + location ^~ /static/ { # Serve static files with Nginx include /etc/nginx/mime.types; root NGINX_DIR; } - location /media-download/ { - # Download file acceleration - internal; - alias SAMPLE_DIR; + # Web console: a static build served straight off disk. Everything it + # talks to is under /api on this same host, so no CORS is involved. + # + location = /app { + return 301 /app/; } - location /logfile-download/ { - # Download file acceleration - internal; - alias LOGFILE_DIR; + # The directory is named to match the URL so plain root resolution + # applies. "alias" with try_files appends the whole URI to the alias + # rather than the remainder, which silently breaks the SPA fallback. + location ^~ /app/ { + include /etc/nginx/mime.types; + root NGINX_DIR/web; + try_files $uri $uri/ /app/index.html; + # index.html names the hashed bundles, so it must be revalidated or a + # browser keeps loading the previous build after a deploy. "expires" + # rather than add_header: any add_header here would stop the server + # block's HSTS header being inherited. + expires -1; } - location /regression-download/ { - # Download file acceleration - internal; - alias RESULT_DIR; + location ^~ /app/assets/ { + include /etc/nginx/mime.types; + root NGINX_DIR/web; + expires max; # hashed filenames } location / { @@ -63,7 +77,6 @@ server { location @proxy_to_app { include proxy_params; - proxy_set_header X-Sendfile-Type X-Accel-Redirect; proxy_pass http://unix:NGINX_DIR/sampleplatform.sock; } } \ No newline at end of file diff --git a/install/platform b/install/platform deleted file mode 100644 index 21c161f8e..000000000 --- a/install/platform +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/bash -# /etc/init.d/platform -# -# Carry out specific functions when asked to by the system -case "${1}" in - start) - echo "Starting Platform daemon..." - cd BASE_DIR - python bootstrap_gunicorn.py - ;; - stop) - echo "Stopping Platform daemon..." - pid=`cat "BASE_DIR/gunicorn.pid"` - kill "${pid}" - ;; - reload) - echo "Reloading Platform daemon..." - pid=`cat "BASE_DIR/gunicorn.pid"` - kill -HUP "${pid}" - ;; - *) - echo "Usage: /etc/init.d/platform {start|stop|reload}" - exit 1 - ;; -esac - -exit 0 \ No newline at end of file diff --git a/install/platform.service b/install/platform.service new file mode 100644 index 000000000..06b2c4ac2 --- /dev/null +++ b/install/platform.service @@ -0,0 +1,19 @@ +[Unit] +Description=CCExtractor Sample Platform (gunicorn) +After=network.target mysql.service +Wants=mysql.service + +[Service] +Type=simple +User=www-data +Group=www-data +WorkingDirectory=#BASE_DIR# +ExecStart=/usr/bin/gunicorn -w 4 -b unix:#BASE_DIR#/sampleplatform.sock -m 007 --timeout 120 --log-level debug --access-logfile #BASE_DIR#/logs/access.log --capture-output --log-file #BASE_DIR#/logs/error.log run:app +ExecReload=/bin/kill -s HUP $MAINPID +Restart=on-failure +RestartSec=5 +KillMode=control-group +TimeoutStopSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/install/sample_db.py b/install/sample_db.py index 5adab8541..1ef328777 100644 --- a/install/sample_db.py +++ b/install/sample_db.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import sys from os import path @@ -11,10 +11,12 @@ def run(): from database import create_session from mod_auth.models import User + from mod_customized.models import CustomizedTest from mod_home.models import CCExtractorVersion, GeneralData from mod_regression.models import (Category, InputType, OutputType, RegressionTest, RegressionTestOutput) from mod_sample.models import Sample + from mod_test.models import Test from mod_upload.models import Upload db = create_session(sys.argv[1]) diff --git a/install/wait_for_db.py b/install/wait_for_db.py new file mode 100644 index 000000000..634b14711 --- /dev/null +++ b/install/wait_for_db.py @@ -0,0 +1,39 @@ +"""Block until the database accepts connections, then exit. + +The Docker entrypoint runs this before starting the app so migrations and +the web server never race the database container. Connection details come +from the environment; see ``env.example``. +""" +import os +import sys +import time + +import pymysql + + +def main() -> int: + host = os.environ.get("DB_HOST", "db") + port = int(os.environ.get("DB_PORT", "3306")) + user = os.environ.get("DB_USER", "sample_platform") + password = os.environ.get("DB_PASSWORD", "sample_platform") + timeout = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) + + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + pymysql.connect( + host=host, port=port, user=user, password=password, + connect_timeout=3, + ).close() + print(f"database reachable at {host}:{port}", flush=True) + return 0 + except pymysql.Error as exc: + print(f"waiting for database at {host}:{port} ({exc})", flush=True) + time.sleep(2) + + print(f"gave up waiting for database after {timeout}s", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/log_configuration.py b/log_configuration.py index 4d8a0cdcb..b554c3d88 100644 --- a/log_configuration.py +++ b/log_configuration.py @@ -3,9 +3,10 @@ import logging import logging.handlers import os +import sys from logging import Logger, StreamHandler from logging.handlers import RotatingFileHandler -from typing import Union +from typing import Optional, Union class LogConfiguration: @@ -19,21 +20,47 @@ def __init__(self, folder: str, filename: str, debug: bool = False) -> None: self._consoleLogger.setLevel(logging.DEBUG) else: self._consoleLogger.setLevel(logging.INFO) - # create a file handler - path = os.path.join(folder, 'logs', f'{filename}.log') - self._fileLogger = logging.handlers.RotatingFileHandler(path, maxBytes=1024 * 1024, backupCount=20) - self._fileLogger.setLevel(logging.DEBUG) - # create a logging format - formatter = logging.Formatter('[%(name)s][%(levelname)s][%(asctime)s] %(message)s') - self._fileLogger.setFormatter(formatter) + + # create a file handler with permission error handling + self._fileLogger: Optional[RotatingFileHandler] = None + log_dir = os.path.join(folder, 'logs') + path = os.path.join(log_dir, f'{filename}.log') + + try: + # Ensure logs directory exists + os.makedirs(log_dir, exist_ok=True) + + self._fileLogger = logging.handlers.RotatingFileHandler( + path, maxBytes=1024 * 1024, backupCount=20 + ) + self._fileLogger.setLevel(logging.DEBUG) + # create a logging format + formatter = logging.Formatter('[%(name)s][%(levelname)s][%(asctime)s] %(message)s') + self._fileLogger.setFormatter(formatter) + except PermissionError as e: + # Log file owned by different user (e.g., root vs www-data) + # Fall back to console-only logging rather than crashing + print( + f"[WARNING] Cannot write to log file {path}: {e}. " + f"Falling back to console-only logging. " + f"Fix: sudo chown www-data:www-data {log_dir} -R", + file=sys.stderr + ) + except OSError as e: + # Other filesystem errors (disk full, etc.) + print( + f"[WARNING] Cannot create log file {path}: {e}. " + f"Falling back to console-only logging.", + file=sys.stderr + ) @property - def file_logger(self) -> RotatingFileHandler: + def file_logger(self) -> Optional[RotatingFileHandler]: """ Get file logger. - :return: file logger - :rtype: logging.handlers.RotatingFileHandler + :return: file logger or None if file logging unavailable + :rtype: Optional[logging.handlers.RotatingFileHandler] """ return self._fileLogger @@ -59,7 +86,8 @@ def create_logger(self, name: str) -> Logger: logger = logging.getLogger(name) logger.setLevel(logging.DEBUG) # add the handlers to the logger - logger.addHandler(self.file_logger) - logger.addHandler(self.console_logger) + if self._fileLogger is not None: + logger.addHandler(self._fileLogger) + logger.addHandler(self._consoleLogger) return logger diff --git a/manage.py b/manage.py index d3063df8f..c06ca21de 100644 --- a/manage.py +++ b/manage.py @@ -1,41 +1,34 @@ #!/usr/local/bin/python3 -"""Root module to manage flask script commands.""" -from flask_script import Command, Manager +"""Root module to manage flask CLI commands.""" +import click from exceptions import CCExtractorEndedWithNonZero, MissingPathToCCExtractor from mod_regression.update_regression import update_expected_results from run import app -manager = Manager(app) - -@manager.add_command -class UpdateResults(Command): +@app.cli.command('update') +@click.argument('path_to_ccex') +def update_results(path_to_ccex): """ Update results for the present samples with new ccextractor version. - Pass path to CCExtractor binary as the first argument. Example, `python manage.py update /path/to/ccextractor` + Pass path to CCExtractor binary as the argument. + Example: flask update /path/to/ccextractor """ + if not path_to_ccex: + click.echo('path to ccextractor is missing') + raise MissingPathToCCExtractor - name = 'update' - capture_all_args = True - - def run(self, remaining): - """Driver function for update command.""" - if len(remaining) == 0: - print('path to ccextractor is missing') - raise MissingPathToCCExtractor - - path_to_ccex = remaining[0] - print(f'path to ccextractor: {path_to_ccex}') + click.echo(f'path to ccextractor: {path_to_ccex}') - if not update_expected_results(path_to_ccex): - print('update function errored') - raise CCExtractorEndedWithNonZero + if not update_expected_results(path_to_ccex): + click.echo('update function errored') + raise CCExtractorEndedWithNonZero - print('update function finished') - return 0 + click.echo('update function finished') + return 0 if __name__ == '__main__': - manager.run() + app.cli() diff --git a/migrations/env.py b/migrations/env.py index bada96d40..571cab1b7 100644 --- a/migrations/env.py +++ b/migrations/env.py @@ -22,9 +22,9 @@ fileConfig(config.config_file_name) logger = logging.getLogger('alembic.env') -config.set_main_option( - 'sqlalchemy.url', current_app.config.get( - 'SQLALCHEMY_DATABASE_URI').replace('%', '%%')) +db_uri = current_app.config.get('SQLALCHEMY_DATABASE_URI') +if db_uri is not None: + config.set_main_option('sqlalchemy.url', db_uri.replace('%', '%%')) target_metadata = current_app.extensions['migrate'].db.metadata # other values from the config, defined by the needs of env.py, diff --git a/migrations/versions/2e0d2e02a721_.py b/migrations/versions/2e0d2e02a721_.py new file mode 100644 index 000000000..f5a42ed05 --- /dev/null +++ b/migrations/versions/2e0d2e02a721_.py @@ -0,0 +1,69 @@ +""" Upgrade: Rename 'kvm' table to 'gcp_instance', remove CCExtractor build step + +Revision ID: 2e0d2e02a721 +Revises: 6b1274f61edd +Create Date: 2022-09-23 11:03:38.783561 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '2e0d2e02a721' +down_revision = '6b1274f61edd' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('kvm', 'timestamp_build_finished') + op.alter_column('regression_test', 'expected_rc', + existing_type=mysql.INTEGER(), + nullable=True, + existing_server_default=sa.text("'0'")) + op.drop_constraint('regression_test_ibfk_1', 'regression_test', type_='foreignkey') + op.create_foreign_key('regression_test_fk', 'regression_test', 'sample', ['sample_id'], ['id'], onupdate='CASCADE', ondelete='CASCADE') + op.alter_column('test_result', 'runtime', + existing_type=mysql.INTEGER(), + nullable=True) + op.alter_column('test_result', 'exit_code', + existing_type=mysql.INTEGER(), + nullable=True) + op.alter_column('test_result', 'expected_rc', + existing_type=mysql.INTEGER(), + nullable=True, + existing_server_default=sa.text("'0'")) + op.drop_constraint('upload_ibfk_1', 'upload', type_='foreignkey') + op.create_foreign_key('upload_fk', 'upload', 'user', ['user_id'], ['id'], onupdate='CASCADE', ondelete='RESTRICT') + op.execute('RENAME TABLE kvm TO gcp_instance') + op.execute("DELETE FROM test_progress WHERE status NOT IN ('completed', 'testing', 'preparation', 'canceled')") + op.execute("ALTER TABLE test_progress MODIFY COLUMN status ENUM('completed', 'testing', 'preparation', 'canceled') NOT NULL") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.execute("ALTER TABLE test_progress MODIFY COLUMN status ENUM('completed', 'testing', 'preparation', 'canceled', 'building') NOT NULL") + op.execute('RENAME TABLE gcp_instance TO kvm') + op.drop_constraint('upload_fk', 'upload', type_='foreignkey') + op.create_foreign_key('upload_ibfk_1', 'upload', 'user', ['user_id'], ['id'], ondelete='RESTRICT') + op.alter_column('test_result', 'expected_rc', + existing_type=mysql.INTEGER(), + nullable=False, + existing_server_default=sa.text("'0'")) + op.alter_column('test_result', 'exit_code', + existing_type=mysql.INTEGER(), + nullable=False) + op.alter_column('test_result', 'runtime', + existing_type=mysql.INTEGER(), + nullable=False) + op.drop_constraint('regression_test_fk', 'regression_test', type_='foreignkey') + op.create_foreign_key('regression_test_ibfk_1', 'regression_test', 'sample', ['sample_id'], ['id'], onupdate='CASCADE') + op.alter_column('regression_test', 'expected_rc', + existing_type=mysql.INTEGER(), + nullable=False, + existing_server_default=sa.text("'0'")) + op.add_column('kvm', sa.Column('timestamp_build_finished', mysql.DATETIME(), nullable=True)) + # ### end Alembic commands ### diff --git a/migrations/versions/7793881905c5_.py b/migrations/versions/7793881905c5_.py new file mode 100644 index 000000000..cb80985ca --- /dev/null +++ b/migrations/versions/7793881905c5_.py @@ -0,0 +1,89 @@ +"""split last_passed_on into two + +Revision ID: 7793881905c5 +Revises: b3ed927671bd +Create Date: 2025-07-24 09:51:55.814649 + +""" + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = "7793881905c5" +down_revision = "b3ed927671bd" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("regression_test", schema=None) as batch_op: + batch_op.add_column( + sa.Column("last_passed_on_windows", sa.Integer(), nullable=True) + ) + batch_op.add_column( + sa.Column("last_passed_on_linux", sa.Integer(), nullable=True) + ) + batch_op.drop_constraint("regression_test_ibfk_2", type_="foreignkey") + batch_op.drop_constraint("regression_test_ibfk_1", type_="foreignkey") + batch_op.create_foreign_key( + None, + "test", + ["last_passed_on_windows"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + None, + "test", + ["last_passed_on_linux"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + batch_op.create_foreign_key( + None, + "sample", + ["sample_id"], + ["id"], + onupdate="CASCADE", + ondelete="CASCADE", + ) + batch_op.drop_column("last_passed_on") + + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table("regression_test", schema=None) as batch_op: + batch_op.add_column( + sa.Column( + "last_passed_on", mysql.INTEGER(), autoincrement=False, nullable=True + ) + ) + batch_op.drop_constraint(None, type_="foreignkey") + batch_op.drop_constraint(None, type_="foreignkey") + batch_op.drop_constraint(None, type_="foreignkey") + batch_op.create_foreign_key( + "regression_test_ibfk_1", + "sample", + ["sample_id"], + ["id"], + onupdate="CASCADE", + ) + batch_op.create_foreign_key( + "regression_test_ibfk_2", + "test", + ["last_passed_on"], + ["id"], + onupdate="CASCADE", + ondelete="SET NULL", + ) + batch_op.drop_column("last_passed_on_linux") + batch_op.drop_column("last_passed_on_windows") + + # ### end Alembic commands ### diff --git a/migrations/versions/a5183973c3e9_.py b/migrations/versions/a5183973c3e9_.py new file mode 100644 index 000000000..fb1a160d6 --- /dev/null +++ b/migrations/versions/a5183973c3e9_.py @@ -0,0 +1,23 @@ +"""Change SQLAlchemy Boolean bit(1) to bool + +Revision ID: a5183973c3e9 +Revises: 2e0d2e02a721 +Create Date: 2023-08-10 15:51:13.537000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'a5183973c3e9' +down_revision = '2e0d2e02a721' +branch_labels = None +depends_on = None + + +def upgrade(): + op.alter_column('regression_test', 'active', existing_type=sa.dialects.mysql.BIT(1), type_=sa.Boolean(), nullable=False) + + +def downgrade(): + op.alter_column('regression_test', 'active', existing_type=sa.Boolean(), type_=sa.dialects.mysql.BIT(1), nullable=False) diff --git a/migrations/versions/b3ed927671bd_.py b/migrations/versions/b3ed927671bd_.py new file mode 100644 index 000000000..936dd9339 --- /dev/null +++ b/migrations/versions/b3ed927671bd_.py @@ -0,0 +1,34 @@ +"""Add last_passed_on and description fields in regression_test + +Revision ID: b3ed927671bd +Revises: a5183973c3e9 +Create Date: 2023-08-17 00:41:01.237549 + +""" +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = 'b3ed927671bd' +down_revision = 'a5183973c3e9' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('regression_test', schema=None) as batch_op: + batch_op.add_column(sa.Column('last_passed_on', sa.Integer(), nullable=True)) + batch_op.create_foreign_key('regression_test_ibfk_2', 'test', ['last_passed_on'], ['id'], onupdate='CASCADE', ondelete='SET NULL') + batch_op.add_column(sa.Column('description', sa.String(length=1024), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + with op.batch_alter_table('regression_test', schema=None) as batch_op: + batch_op.drop_constraint('regression_test_ibfk_2', type_='foreignkey') + batch_op.drop_column('last_passed_on') + batch_op.drop_column('description') + # ### end Alembic commands ### diff --git a/migrations/versions/c8f3a2b1d4e5_.py b/migrations/versions/c8f3a2b1d4e5_.py new file mode 100644 index 000000000..436d06e09 --- /dev/null +++ b/migrations/versions/c8f3a2b1d4e5_.py @@ -0,0 +1,34 @@ +"""Add pending_deletion table for VM deletion tracking + +Revision ID: c8f3a2b1d4e5 +Revises: 7793881905c5 +Create Date: 2025-12-30 12:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'c8f3a2b1d4e5' +down_revision = '7793881905c5' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('pending_deletion', + sa.Column('vm_name', sa.String(length=64), nullable=False), + sa.Column('operation_name', sa.String(length=128), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=False), + sa.Column('retry_count', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('vm_name'), + mysql_engine='InnoDB' + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('pending_deletion') + # ### end Alembic commands ### diff --git a/migrations/versions/d4f8e2a1b3c7_.py b/migrations/versions/d4f8e2a1b3c7_.py new file mode 100644 index 000000000..e84d0302e --- /dev/null +++ b/migrations/versions/d4f8e2a1b3c7_.py @@ -0,0 +1,44 @@ +"""Add api_token table for scoped API token auth. + +Revision ID: d4f8e2a1b3c7 +Revises: c8f3a2b1d4e5 +Create Date: 2026-06-11 03:00:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'd4f8e2a1b3c7' +down_revision = 'c8f3a2b1d4e5' +branch_labels = None +depends_on = None + + +def upgrade(): + """Apply the migration.""" + op.add_column('user', sa.Column('github_login', sa.String(length=255), nullable=True)) + op.create_table( + 'api_token', + sa.Column('id', sa.Integer(), nullable=False, autoincrement=True), + sa.Column('user_id', sa.Integer(), nullable=False), + sa.Column('token_name', sa.String(length=50), nullable=False), + sa.Column('token_hash', sa.String(length=255), nullable=False), + sa.Column('token_prefix', sa.String(length=16), nullable=False), + sa.Column('scopes_json', sa.Text(), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('expires_at', sa.DateTime(timezone=True), nullable=False), + sa.Column('revoked_at', sa.DateTime(timezone=True), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['user_id'], ['user.id'], onupdate='CASCADE', ondelete='CASCADE'), + sa.UniqueConstraint('user_id', 'token_name', name='uq_user_token_name'), + mysql_engine='InnoDB' + ) + op.create_index('ix_api_token_token_prefix', 'api_token', ['token_prefix']) + + +def downgrade(): + """Revert the migration.""" + op.drop_index('ix_api_token_token_prefix', table_name='api_token') + op.drop_table('api_token') + op.drop_column('user', 'github_login') diff --git a/migrations/versions/e7c9b4a2d1f0_add_test_built_commit.py b/migrations/versions/e7c9b4a2d1f0_add_test_built_commit.py new file mode 100644 index 000000000..1720127b2 --- /dev/null +++ b/migrations/versions/e7c9b4a2d1f0_add_test_built_commit.py @@ -0,0 +1,25 @@ +"""Record the SHA that CI actually built alongside the PR head. + +Revision ID: e7c9b4a2d1f0 +Revises: d4f8e2a1b3c7 +Create Date: 2026-09-08 16:50:00.000000 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = 'e7c9b4a2d1f0' +down_revision = 'd4f8e2a1b3c7' +branch_labels = None +depends_on = None + + +def upgrade(): + """Apply the migration.""" + op.add_column('test', sa.Column('built_commit', sa.String(length=64), nullable=True)) + + +def downgrade(): + """Revert the migration.""" + op.drop_column('test', 'built_commit') diff --git a/mod_api/__init__.py b/mod_api/__init__.py new file mode 100644 index 000000000..74b2d4fa3 --- /dev/null +++ b/mod_api/__init__.py @@ -0,0 +1,46 @@ +""" +mod_api: JSON REST API blueprint for the CCExtractor CI platform. + +Registered at /api/v1. All endpoints return structured JSON, use scoped +Bearer token auth, and enforce per-client rate limiting. +""" + +from flask import Blueprint + +mod_api = Blueprint('api', __name__) + +# Middleware imports +from mod_api.middleware import auth # noqa: E402 +from mod_api.middleware import error_handler # noqa: E402 +from mod_api.middleware import rate_limit # noqa: E402 +from mod_api.middleware import security # noqa: E402 + +# Explicitly register before_request hooks in the exact order they should run +mod_api.before_request(auth.authenticate_request) +mod_api.before_request(rate_limit.check_rate_limit) +mod_api.before_request(auth.enforce_auth_error) + +# Explicitly register after_request hooks. +# NOTE: Flask executes after_request hooks in REVERSE registration order. +# Registration: security → rate_limit → (convert is app-level, see below) +# Execution: rate_limit → security +# This means rate-limit headers are added first, then security headers layer +# on top — both on the same response object. +mod_api.after_request(security.add_security_headers) +mod_api.after_request(rate_limit.add_rate_limit_headers) + +# Registered as after_app_request so it fires for ALL requests (including +# routing-level 404s/405s that never enter the blueprint). +mod_api.after_app_request(error_handler.convert_api_errors_to_json) + +# Route modules +from mod_api.routes import auth as auth_routes # noqa: E402, F401 +from mod_api.routes import \ + errors_logs as errors_logs_routes # noqa: E402, F401 +from mod_api.routes import \ + regression_tests as regression_tests_routes # noqa: E402, F401 +from mod_api.routes import results as results_routes # noqa: E402, F401 +from mod_api.routes import runs as runs_routes # noqa: E402, F401 +from mod_api.routes import samples as samples_routes # noqa: E402, F401 +from mod_api.routes import system as system_routes # noqa: E402, F401 +from mod_api.routes import uploads as uploads_routes # noqa: E402, F401 diff --git a/mod_api/middleware/__init__.py b/mod_api/middleware/__init__.py new file mode 100644 index 000000000..860b3ce01 --- /dev/null +++ b/mod_api/middleware/__init__.py @@ -0,0 +1 @@ +"""mod_api.middleware: auth, rate limiting, validation, and error handling.""" diff --git a/mod_api/middleware/auth.py b/mod_api/middleware/auth.py new file mode 100644 index 000000000..31e53ec74 --- /dev/null +++ b/mod_api/middleware/auth.py @@ -0,0 +1,151 @@ +""" +Bearer token authentication and scope/role enforcement for API routes. + +Runs as a before_request hook on the api blueprint. Public endpoints +(token creation, health check) are exempted. On success, the authenticated +user and token are stored in flask.g for downstream handlers. + +HTTP semantics: + 401 = token missing, expired, revoked, or invalid + 403 = valid token but insufficient scope or role +""" + +import functools +from typing import Any, List + +from flask import g, request + +from mod_api.middleware.error_handler import make_error_response +from mod_api.models.api_token import TOKEN_PREFIX, ApiToken + +_AUTH_FAILED_MSG = 'Bearer token is missing, expired, or invalid.' + +# These endpoints bypass auth entirely. +_PUBLIC_ENDPOINTS = frozenset([ + 'api.create_token', # POST /auth/tokens (uses email/password body) + 'api.system_health', # GET /system/health (uptime monitoring) + # Account recovery: nobody holds a token at this point, which is the + # whole reason for asking. These are limited per IP instead. + 'api.signup', + 'api.request_password_reset', + 'api.complete_password_reset', +]) + + +def _unauthorized(): + """Shorthand for a 401 response with the standard auth failure message.""" + return make_error_response( + 'unauthorized', _AUTH_FAILED_MSG, http_status=401) + + +def authenticate_request(): + """Validate Bearer token and attach user context to the request. + + If auth fails, sets g.auth_error instead of returning immediately, + so that subsequent hooks (like rate limiting) still run. + """ + if request.endpoint in _PUBLIC_ENDPOINTS: + g.api_user = None + g.api_token = None + return + + auth_header = request.headers.get('Authorization', '') + if not auth_header: + g.auth_error = _unauthorized() + return + + parts = auth_header.split(' ', 1) + # Auth scheme names are case-insensitive (RFC 7235 section 2.1). + if len(parts) != 2 or parts[0].lower() != 'bearer': + g.auth_error = _unauthorized() + return + + token_value = parts[1].strip() + if not token_value or not token_value.startswith(TOKEN_PREFIX): + g.auth_error = _unauthorized() + return + + # Look up by prefix, then verify the full hash against each candidate. + prefix = ApiToken.extract_prefix(token_value) + candidates = ApiToken.query.filter_by(token_prefix=prefix).all() + + if not candidates: + g.auth_error = _unauthorized() + return + + matched_token = None + for candidate in candidates: + if ApiToken.verify_token(token_value, candidate.token_hash): + matched_token = candidate + break + + if matched_token is None: + g.auth_error = _unauthorized() + return + + if not matched_token.is_valid: + g.auth_error = _unauthorized() + return + + g.api_token = matched_token + g.api_user = matched_token.user + + +def enforce_auth_error(): + """Return any stored auth errors after rate limiting.""" + if hasattr(g, 'auth_error') and g.auth_error is not None: + return g.auth_error + + +def require_scope(*scopes: str): + """Reject the request if the token lacks any of the ``scopes``.""" + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + token = getattr(g, 'api_token', None) + if token is None: + return _unauthorized() + + missing_scopes = [s for s in scopes if not token.has_scope(s)] + if missing_scopes: + return make_error_response( + 'forbidden', + 'Token lacks the required scopes for this operation.', + details={ + 'required_scopes': list(scopes), + 'missing_scopes': missing_scopes, + 'token_scopes': token.scopes, + }, + http_status=403, + ) + return f(*args, **kwargs) + return decorated_function + return decorator + + +def require_roles(roles: List[Any]): + """Reject the request if the user's role is not in ``roles``. + + Takes Role members (not strings), matching check_access_rights in + mod_auth. Typed loosely because DeclEnum members are EnumSymbol + instances at runtime but plain tuples to the type checker. + """ + def decorator(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + user = getattr(g, 'api_user', None) + if user is None: + return _unauthorized() + if user.role not in roles: + return make_error_response( + 'forbidden', + 'Your role does not have permission for this operation.', + details={ + 'required_roles': [role.value for role in roles], + 'user_role': user.role.value, + }, + http_status=403, + ) + return f(*args, **kwargs) + return decorated_function + return decorator diff --git a/mod_api/middleware/error_handler.py b/mod_api/middleware/error_handler.py new file mode 100644 index 000000000..33b50842e --- /dev/null +++ b/mod_api/middleware/error_handler.py @@ -0,0 +1,160 @@ +"""Structured JSON error responses for API routes.""" + +from flask import current_app, jsonify, request +from marshmallow import ValidationError as MarshmallowValidationError +from sqlalchemy.exc import SQLAlchemyError + +from mod_api import mod_api + +_API_PREFIX = '/api/v1' + + +def make_error_response(code, message, details=None, http_status=400): + """Build a JSON error response conforming to the ErrorResponse schema.""" + body = { + 'code': code, + 'message': str(message)[:500], + 'details': details if details is not None else {}, + } + response = jsonify(body) + response.status_code = http_status + return response + + +@mod_api.errorhandler(400) +def handle_400(error): + """Bad request.""" + return make_error_response( + 'validation_error', + getattr(error, 'description', 'Bad request.'), + http_status=400, + ) + + +@mod_api.errorhandler(401) +def handle_401(error): + """Unauthorized.""" + return make_error_response( + 'unauthorized', + 'Bearer token is missing, expired, or invalid.', + http_status=401, + ) + + +@mod_api.errorhandler(403) +def handle_403(error): + """Forbidden.""" + return make_error_response( + 'forbidden', + 'Token does not have the required scope for this operation.', + http_status=403, + ) + + +@mod_api.errorhandler(404) +def handle_404(error): + """Not found.""" + return make_error_response( + 'not_found', + getattr(error, 'description', 'Resource not found.'), + http_status=404, + ) + + +@mod_api.errorhandler(405) +def handle_405(error): + """Handle method-not-allowed errors for API routes.""" + resp = make_error_response( + 'method_not_allowed', + 'Method not allowed.', + http_status=405, + ) + if hasattr(error, 'valid_methods') and error.valid_methods: + resp.headers['Allow'] = ', '.join(error.valid_methods) + return resp + + +@mod_api.errorhandler(422) +def handle_422(error): + """Unprocessable entity.""" + return make_error_response( + 'unprocessable', + getattr( + error, + 'description', + 'Request is valid JSON but semantically invalid.'), + http_status=422, + ) + + +@mod_api.errorhandler(429) +def handle_429(error): + """Rate limited. + + This is only a fallback for 429s raised outside the rate-limit + middleware. The live limiter (mod_api.middleware.rate_limit) returns + accurate per-bucket limit/retry_after/window values and the + Retry-After header; we deliberately don't hardcode numbers here that + would be wrong for the auth (5/15m) and write (20/min) buckets. + """ + return make_error_response( + 'rate_limited', + 'Rate limit exceeded.', + http_status=429, + ) + + +@mod_api.errorhandler(500) +def handle_500(error): + """Handle unexpected server errors for API routes.""" + current_app.logger.exception(error) + return make_error_response( + 'internal_error', + 'An unexpected error occurred.', + http_status=500, + ) + + +@mod_api.errorhandler(MarshmallowValidationError) +def handle_marshmallow_validation_error(error): + """Catch schema validation failures and return them as 400.""" + return make_error_response( + 'validation_error', + 'Request failed schema validation.', + details={'fields': error.messages}, + http_status=400, + ) + + +@mod_api.errorhandler(SQLAlchemyError) +def handle_sqlalchemy_error(error): + """Log database errors.""" + current_app.logger.exception(error) + return make_error_response( + 'internal_error', + 'An unexpected database error occurred.', + http_status=500, + ) + + +def convert_api_errors_to_json(response): + """Catch routing errors that were handled by global app handlers and convert them to JSON.""" + if request.path.startswith(_API_PREFIX): + if response.status_code >= 500 and not response.is_json: + new_resp = make_error_response( + 'internal_error', 'An unexpected error occurred.', http_status=response.status_code + ) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + if response.status_code == 404 and not response.is_json: + new_resp = make_error_response('not_found', 'Resource not found.', http_status=404) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + if response.status_code == 405 and not response.is_json: + new_resp = make_error_response('method_not_allowed', 'Method not allowed.', http_status=405) + response.data = new_resp.data + response.mimetype = new_resp.mimetype + return response + return response diff --git a/mod_api/middleware/rate_limit.py b/mod_api/middleware/rate_limit.py new file mode 100644 index 000000000..1b13f2652 --- /dev/null +++ b/mod_api/middleware/rate_limit.py @@ -0,0 +1,154 @@ +""" +Per-client fixed-window rate limiting for API endpoints. + +Limits: + POST /auth/tokens 5 req / 15 min (keyed by IP) + POST/DELETE/PUT/PATCH 20 req / min (keyed by token) + GET 120 req / min (keyed by token) + +Includes X-RateLimit-* headers on every response. + +Note: This is a fixed-window implementation (counter resets when the +window expires). For true sliding-window behavior, consider migrating +to Redis with a sorted-set approach. State is per-process, so multiple +Gunicorn workers enforce limits independently. +""" + +import threading +import time + +from flask import current_app, g, request + +from mod_api.middleware.error_handler import make_error_response + +_rate_limit_store = {} # key -> {'count': int, 'window_start': float} +_rate_limit_lock = threading.Lock() +_eviction_counter = 0 +_EVICTION_INTERVAL = 100 # run cleanup every N requests +_MAX_ENTRIES = 10000 # hard limit on stored keys to prevent memory exhaustion + + +def _evict_stale_entries(): + """Prune entries older than 15 min to bound memory usage.""" + global _eviction_counter + with _rate_limit_lock: + _eviction_counter += 1 + if _eviction_counter < _EVICTION_INTERVAL: + return + _eviction_counter = 0 + now = time.time() + stale_keys = [ + key for key, entry in _rate_limit_store.items() + if (now - entry['window_start']) > 900 + ] + for key in stale_keys: + del _rate_limit_store[key] + + if len(_rate_limit_store) > _MAX_ENTRIES: + # Sort by window_start (oldest first) and evict until we are at 90% capacity + sorted_keys = sorted( + _rate_limit_store.keys(), + key=lambda k: _rate_limit_store[k]['window_start'] + ) + keys_to_remove = len(_rate_limit_store) - int(_MAX_ENTRIES * 0.9) + for key in sorted_keys[:keys_to_remove]: + del _rate_limit_store[key] + + +def _get_client_ip(): + """Extract the real client IP (ProxyFix handles X-Forwarded-For securely).""" + return request.remote_addr + + +# Endpoints reached without a token, so there is nothing to key on but the +# address. All of them either hand out credentials or send mail to an address +# the caller names, which is worth rationing tightly. +_UNAUTHENTICATED_ENDPOINTS = frozenset([ + 'api.create_token', + 'api.signup', + 'api.request_password_reset', + 'api.complete_password_reset', +]) + + +def _get_rate_limit_key(): + """Build the rate-limit bucket key for this request.""" + if request.endpoint in _UNAUTHENTICATED_ENDPOINTS: + return f'ip:{_get_client_ip()}' + token = getattr(g, 'api_token', None) + if token: + return f'token:{token.id}' + return f'ip:{_get_client_ip()}' + + +def _get_limits(): + """Return (max_requests, window_seconds) for the current endpoint.""" + if request.endpoint in _UNAUTHENTICATED_ENDPOINTS: + return 5, 900 + if request.method in ('POST', 'DELETE', 'PUT', 'PATCH'): + return 20, 60 + return 120, 60 + + +def check_rate_limit(): + """Apply rate limits based on client IP or API token.""" + if current_app.config.get('TESTING'): + return + + _evict_stale_entries() + + key = _get_rate_limit_key() + max_requests, window_seconds = _get_limits() + now = time.time() + + with _rate_limit_lock: + entry = _rate_limit_store.get(key) + + if entry is None or (now - entry['window_start']) >= window_seconds: + _rate_limit_store[key] = {'count': 1, 'window_start': now} + else: + entry['count'] += 1 + if entry['count'] > max_requests: + reset_at = int(entry['window_start'] + window_seconds) + retry_after = max(1, reset_at - int(now)) + + response = make_error_response( + 'rate_limited', + f'Rate limit exceeded. Retry after {retry_after} seconds.', + details={ + 'retry_after': retry_after, + 'limit': max_requests, + 'window': f'{window_seconds}s', + }, + http_status=429, + ) + response.headers['Retry-After'] = str(retry_after) + response.headers['X-RateLimit-Limit'] = str(max_requests) + response.headers['X-RateLimit-Remaining'] = '0' + response.headers['X-RateLimit-Reset'] = str(reset_at) + return response + + +def add_rate_limit_headers(response): + """Inject X-RateLimit-* headers based on the current window.""" + if current_app.config.get('TESTING') or response.status_code == 429: + return response + + key = _get_rate_limit_key() + max_requests, window_seconds = _get_limits() + now = time.time() + + with _rate_limit_lock: + entry = _rate_limit_store.get(key) + if entry: + remaining = max(0, max_requests - entry['count']) + reset_at = int(entry['window_start'] + window_seconds) + else: + remaining = max_requests + reset_at = int(now + window_seconds) + + response.headers['X-RateLimit-Limit'] = str(max_requests) + response.headers['X-RateLimit-Remaining'] = str(remaining) + response.headers['X-RateLimit-Reset'] = str(reset_at) + + return response diff --git a/mod_api/middleware/security.py b/mod_api/middleware/security.py new file mode 100644 index 000000000..c639b006c --- /dev/null +++ b/mod_api/middleware/security.py @@ -0,0 +1,10 @@ +"""Security headers middleware for API responses.""" + + +def add_security_headers(response): + """Attach security headers to all API responses.""" + response.headers['Strict-Transport-Security'] = 'max-age=31536000; includeSubDomains' + response.headers['Content-Security-Policy'] = "default-src 'none'; frame-ancestors 'none'" + response.headers['X-Content-Type-Options'] = 'nosniff' + response.headers['X-Frame-Options'] = 'DENY' + return response diff --git a/mod_api/middleware/validation.py b/mod_api/middleware/validation.py new file mode 100644 index 000000000..7922db568 --- /dev/null +++ b/mod_api/middleware/validation.py @@ -0,0 +1,309 @@ +""" +Request validation decorators for bodies, query params, and path IDs. + +All of these return 400 with field-level details on failure, so route +handlers can assume clean input. +""" + +from datetime import datetime, timezone +from functools import wraps + +from flask import request +from marshmallow import ValidationError as MarshmallowValidationError + +from mod_api.middleware.error_handler import make_error_response + +# Whitelist of allowed sort params. +ALLOWED_RUN_SORTS = frozenset([ + 'created_at', '-created_at', + 'run_id', '-run_id', +]) + + +def validate_body(schema_class): + """Validate the JSON body with a schema, pass result as ``validated_data``.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + content_type = request.content_type or '' + if content_type.split(';')[0].strip() != 'application/json': + return make_error_response( + 'validation_error', + 'Content-Type must be application/json.', + http_status=415, + ) + json_data = request.get_json(silent=True) + if json_data is None: + return make_error_response( + 'validation_error', + 'Request body must be valid JSON.', + http_status=400, + ) + schema = schema_class() + try: + validated = schema.load(json_data) + except MarshmallowValidationError as e: + return make_error_response( + 'validation_error', + 'Request failed schema validation.', + details={'fields': e.messages}, + http_status=400, + ) + kwargs['validated_data'] = validated + return f(*args, **kwargs) + return decorated + return decorator + + +def validate_offset_pagination(default_limit=50): + """Extract and validate ``limit`` and ``offset`` query params.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if 'cursor' in request.args: + return make_error_response( + 'validation_error', + 'Cannot mix cursor and offset pagination.', + details={'fields': { + 'cursor': 'Cannot specify cursor when using offset pagination.'}}, + http_status=400, + ) + + try: + limit = int(request.args.get('limit', default_limit)) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'limit must be an integer.', + details={'fields': { + 'limit': 'Must be an integer between 1 and 100.'}}, + http_status=400, + ) + + try: + offset = int(request.args.get('offset', 0)) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'offset must be a non-negative integer.', + details={'fields': { + 'offset': 'Must be a non-negative integer.'}}, + http_status=400, + ) + + if limit < 1 or limit > 100: + return make_error_response( + 'validation_error', + 'limit must be between 1 and 100.', + details={'fields': {'limit': 'Must be between 1 and 100.'}}, + http_status=400, + ) + + if offset < 0: + return make_error_response( + 'validation_error', + 'offset must be non-negative.', + details={'fields': {'offset': 'Must be >= 0.'}}, + http_status=400, + ) + + if offset > 2147483647: + return make_error_response( + 'validation_error', + 'offset is too large.', + details={'fields': {'offset': 'Must be <= 2147483647.'}}, + http_status=400, + ) + + kwargs['limit'] = limit + kwargs['offset'] = offset + return f(*args, **kwargs) + return decorated + return decorator + + +def _parse_limit(default_limit): + try: + limit = int(request.args.get('limit', default_limit)) + except (ValueError, TypeError): + return None, make_error_response( + 'validation_error', + 'limit must be an integer.', + details={'fields': {'limit': 'Must be an integer between 1 and 100.'}}, + http_status=400, + ) + + if limit < 1 or limit > 100: + return None, make_error_response( + 'validation_error', + 'limit must be between 1 and 100.', + details={'fields': {'limit': 'Must be between 1 and 100.'}}, + http_status=400, + ) + return limit, None + + +def _parse_cursor(): + cursor = request.args.get('cursor') + if cursor is None: + return None, None + try: + cursor = int(cursor) + except (ValueError, TypeError): + return None, make_error_response( + 'validation_error', + 'cursor must be an integer.', + details={'fields': {'cursor': 'Must be an integer.'}}, + http_status=400, + ) + if cursor < 0: + return None, make_error_response( + 'validation_error', + 'cursor must be non-negative.', + details={'fields': {'cursor': 'Must be >= 0.'}}, + http_status=400, + ) + if cursor > 10_000_000: + return None, make_error_response( + 'validation_error', + 'cursor out of range.', + details={'fields': {'cursor': 'Must be <= 10000000.'}}, + http_status=400, + ) + return cursor, None + + +def validate_cursor_pagination(default_limit=50): + """Extract and validate ``limit`` and ``cursor`` query params.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + if 'offset' in request.args: + return make_error_response( + 'validation_error', + 'Cannot mix cursor and offset pagination.', + details={'fields': { + 'offset': 'Cannot specify offset when using cursor pagination.'}}, + http_status=400, + ) + + limit, err = _parse_limit(default_limit) + if err: + return err + + cursor, err = _parse_cursor() + if err: + return err + + kwargs['limit'] = limit + kwargs['cursor'] = cursor + return f(*args, **kwargs) + return decorated + return decorator + + +def validate_path_id(param_name): + """Ensure a URL path parameter is a positive integer.""" + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + value = kwargs.get(param_name) + try: + int_value = int(value) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + f'{param_name} must be a positive integer.', + details={ + 'fields': { + param_name: 'Must be a positive integer.'}}, + http_status=400, + ) + if int_value < 1 or int_value > 2147483647: + return make_error_response( + 'validation_error', + f'{param_name} must be between 1 and 2147483647.', + details={ + 'fields': { + param_name: 'Must be between 1 and 2147483647. Out of bounds IDs are rejected.' + } + }, + http_status=400, + ) + kwargs[param_name] = int_value + return f(*args, **kwargs) + return decorated + return decorator + + +def _parse_iso8601_date(param_name, param_str): + if not param_str: + return None, None + try: + dt = datetime.fromisoformat(param_str.replace('Z', '+00:00')) + except ValueError: + return None, make_error_response( + 'validation_error', + f'{param_name} must be a valid ISO 8601 datetime.', + details={'fields': {param_name: 'Invalid ISO 8601 format.'}}, + http_status=400, + ) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=timezone.utc) + return dt, None + + +def validate_date_range(f): + """Parse date query params and reject inverted ranges.""" + @wraps(f) + def decorated(*args, **kwargs): + created_after_str = request.args.get('created_after') + created_before_str = request.args.get('created_before') + + created_after, err = _parse_iso8601_date('created_after', created_after_str) + if err: + return err + + created_before, err = _parse_iso8601_date('created_before', created_before_str) + if err: + return err + + if created_after and created_before and created_after > created_before: + return make_error_response( + 'validation_error', + 'created_after cannot be later than created_before.', + details={'fields': { + 'created_after': 'Cannot be after created_before.'}}, + http_status=400, + ) + + kwargs['created_after'] = created_after + kwargs['created_before'] = created_before + return f(*args, **kwargs) + return decorated + + +def validate_sort(allowed=None): + """Validate the ``sort`` query param against a whitelist.""" + if allowed is None: + allowed = ALLOWED_RUN_SORTS + + def decorator(f): + @wraps(f) + def decorated(*args, **kwargs): + sort = request.args.get('sort', '-created_at') + if sort not in allowed: + return make_error_response( + 'validation_error', + f'sort must be one of: {", ".join(sorted(allowed))}', + details={ + 'fields': { + 'sort': f'Must be one of: {sorted(allowed)}' + } + }, + http_status=400, + ) + kwargs['sort'] = sort + return f(*args, **kwargs) + return decorated + return decorator diff --git a/mod_api/models/__init__.py b/mod_api/models/__init__.py new file mode 100644 index 000000000..dcb36537a --- /dev/null +++ b/mod_api/models/__init__.py @@ -0,0 +1 @@ +"""mod_api.models: database models for the API module.""" diff --git a/mod_api/models/api_token.py b/mod_api/models/api_token.py new file mode 100644 index 000000000..996c21c8e --- /dev/null +++ b/mod_api/models/api_token.py @@ -0,0 +1,179 @@ +""" +ApiToken model: server-side storage for scoped API tokens. + +Tokens are opaque strings prefixed with 'spci_'. Only the SHA-256 hash +is persisted; the plaintext is returned exactly once at creation time. +A fast hash with a constant-time compare is sufficient here because the +tokens are 256-bit random secrets — a slow password KDF (argon2/bcrypt) +buys nothing against brute force on high-entropy values. +""" + +import hashlib +import hmac +import json +import secrets +from datetime import datetime, timedelta, timezone +from typing import List + +from sqlalchemy import (Column, DateTime, ForeignKey, Integer, String, Text, + UniqueConstraint) +from sqlalchemy.orm import relationship, validates + +from database import Base + + +class Scope: + """The scopes a token can carry. + + Reference these constants instead of writing the strings inline, so a + rename stays a single edit. Plain strings rather than a DeclEnum + because scopes cross the wire: they arrive in request bodies, are + stored as a JSON array, and are compared against what the client sent. + """ + + RUNS_READ = 'runs:read' + RUNS_WRITE = 'runs:write' + RESULTS_READ = 'results:read' + BASELINES_WRITE = 'baselines:write' + SYSTEM_READ = 'system:read' + # Kept apart from system:read so a monitoring token can watch the + # platform without being able to reconfigure it. + SYSTEM_WRITE = 'system:write' + TOKENS_MANAGE = 'tokens:manage' + + +VALID_SCOPES = frozenset([ + Scope.RUNS_READ, + Scope.RUNS_WRITE, + Scope.RESULTS_READ, + Scope.BASELINES_WRITE, + Scope.SYSTEM_READ, + Scope.SYSTEM_WRITE, + Scope.TOKENS_MANAGE, +]) + +DEFAULT_SCOPES = [Scope.RUNS_READ, Scope.RESULTS_READ] + +TOKEN_PREFIX = 'spci_' +TOKEN_BYTE_LENGTH = 32 + + +class ApiToken(Base): + """Scoped API token bound to a user account.""" + + __tablename__ = 'api_token' + __table_args__ = ( + UniqueConstraint('user_id', 'token_name', name='uq_user_token_name'), + {'mysql_engine': 'InnoDB'}, + ) + + id = Column(Integer, primary_key=True) + user_id = Column( + Integer, + ForeignKey('user.id', onupdate='CASCADE', ondelete='CASCADE'), + nullable=False, + ) + user = relationship('User', uselist=False) + token_name = Column(String(50), nullable=False) + token_hash = Column(String(255), nullable=False) + token_prefix = Column(String(16), nullable=False, index=True) + scopes_json = Column(Text(), nullable=False) + created_at = Column(DateTime(timezone=True), nullable=False) + expires_at = Column(DateTime(timezone=True), nullable=False) + revoked_at = Column(DateTime(timezone=True), nullable=True) + + @validates('scopes_json') + def validate_scopes_json(self, key, value): + """Ensure scopes_json only contains known scopes.""" + try: + scopes = json.loads(value) + except json.JSONDecodeError: + raise ValueError("scopes_json must be a valid JSON string") + + if not isinstance(scopes, list): + raise ValueError("scopes_json must be a JSON array") + + for scope in scopes: + if scope not in VALID_SCOPES: + raise ValueError(f"Unknown scope: {scope}") + return value + + def __init__( + self, + user_id: int, + token_name: str, + token_hash: str, + token_prefix: str, + scopes: List[str], + expires_in_days: int = 7, + ) -> None: + self.user_id = user_id + self.token_name = token_name + self.token_hash = token_hash + self.token_prefix = token_prefix + self.scopes_json = json.dumps(scopes) + self.created_at = datetime.now(timezone.utc) + self.expires_at = self.created_at + timedelta(days=expires_in_days) + + def __repr__(self) -> str: + """Return a debug representation of the token.""" + return f'' + + @property + def scopes(self) -> List[str]: + """Parse the JSON scopes column into a list.""" + return json.loads(self.scopes_json) + + @property + def is_expired(self) -> bool: + """Check whether this token has passed its expiration time.""" + now = datetime.now(timezone.utc) + expires = self.expires_at + if expires is None: + return True + # MySQL DATETIME columns don't preserve tzinfo; treat naive as UTC. + if expires.tzinfo is None: + expires = expires.replace(tzinfo=timezone.utc) + return bool(now > expires) + + @property + def is_revoked(self) -> bool: + """Check whether this token has been explicitly revoked.""" + return bool(self.revoked_at is not None) + + @property + def is_valid(self) -> bool: + """Return True if the token is neither expired nor revoked.""" + return not self.is_expired and not self.is_revoked + + def has_scope(self, scope: str) -> bool: + """Return True if the token grants the given scope.""" + return scope in self.scopes + + def revoke(self) -> None: + """Mark this token as revoked with the current timestamp.""" + self.revoked_at = datetime.now(timezone.utc) + + @staticmethod + def generate_token() -> str: + """Create a new random token string with the spci_ prefix.""" + random_bytes = secrets.token_urlsafe(TOKEN_BYTE_LENGTH) + return f'{TOKEN_PREFIX}{random_bytes}' + + @staticmethod + def hash_token(plaintext: str) -> str: + """Hash a token securely using SHA-256.""" + return hashlib.sha256(plaintext.encode('utf-8')).hexdigest() + + @staticmethod + def verify_token(plaintext: str, token_hash: str) -> bool: + """Verify a token against its SHA-256 hash using constant-time comparison.""" + if not plaintext or not token_hash: + return False + expected_hash = ApiToken.hash_token(plaintext) + return hmac.compare_digest(expected_hash, token_hash) + + @staticmethod + def extract_prefix(token: str) -> str: + """Return the first 16 chars used for DB lookup.""" + return token[:16] if len(token) >= 16 else token diff --git a/mod_api/routes/__init__.py b/mod_api/routes/__init__.py new file mode 100644 index 000000000..eac65b967 --- /dev/null +++ b/mod_api/routes/__init__.py @@ -0,0 +1 @@ +"""mod_api.routes — Endpoint handlers for the API.""" diff --git a/mod_api/routes/auth.py b/mod_api/routes/auth.py new file mode 100644 index 000000000..583abe06f --- /dev/null +++ b/mod_api/routes/auth.py @@ -0,0 +1,684 @@ +""" +Token lifecycle, caller identity, account and admin user management. + +POST /auth/tokens Authenticate with email/password, get a token +GET /auth/tokens List tokens (admin-only; ?all=true for all users) +DELETE /auth/tokens/current Revoke the token you're currently using +DELETE /auth/tokens/{id} Revoke a specific token by ID +GET /auth/me Identity, role and scopes behind the token +PATCH /auth/me Change your own name, email or password +POST /auth/signup Send a registration link +POST /auth/password-reset Send a password reset link +POST /auth/password-reset/complete + Set a new password from a reset link +GET /auth/me/ftp-credentials + Your own FTP details for the ingest server +GET /auth/me/github Whether your account is connected to GitHub +DELETE /auth/me/github Forget this platform's copy of that connection +GET /users List platform users (admin) +GET /users/{id} One platform user (admin) +PATCH /users/{id} Change a user's role (admin) +POST /users/{id}/deactivate Anonymise an account (admin, or your own) +POST /users/{id}/password-reset + Send someone a reset link (admin, or your own) + +Signup and reset send the same emails as the classic pages and reuse their +signed links, so accounts are still created and passwords still set by one +implementation rather than two. +""" + +import hmac +import time + +from flask import g, request, url_for +from passlib.apps import custom_app_context as pwd_context +from sqlalchemy.exc import IntegrityError + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import DEFAULT_SCOPES, ApiToken, Scope +from mod_api.schemas.auth import (AccountUpdateSchema, ApiTokenItemSchema, + AuthTokenSchema, EmailOnlySchema, + PasswordResetCompleteSchema, + RoleUpdateSchema, TokenCreateRequestSchema) +from mod_api.utils import paginated_response, single_response +from mod_auth.models import Role, User + +_DUMMY_HASH = pwd_context.hash('__dummy__') + +# Signup and reset links last a day, matching the classic pages. +_LINK_TTL = 86400 + + +def _send(to, subject, text): + """Send one email, logging rather than failing when the mailer says no.""" + if not g.mailer.send_simple_message( + {'to': to, 'subject': subject, 'text': text}): + g.log.error(f'could not send "{subject}" to {to}') + + +def _send_reset_link(user): + """ + Email a password reset link to one user. + + The signature and the template are the classic ones, so a link from + here and a link from the classic page are interchangeable. What cannot + be reused is mod_auth's own send_reset_email: it builds the URL with a + relative endpoint, which resolves against whichever blueprint is + handling the request and so cannot be built from inside this one. + + Where CONSOLE_URL names a web console, the link points there instead, + so somebody who started in the console is not handed to a second site + to finish. The console posts the same three values back to + /auth/password-reset/complete, so the link means the same thing either + way. Unset, which is every existing install, nothing changes. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + expires = int(time.time()) + _LINK_TTL + mac = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), + f'{user.id}|{expires}|{user.password}') + console = app.config.get('CONSOLE_URL', '') + if console: + url = (f"{console.rstrip('/')}/reset" + f'?uid={user.id}&expires={expires}&mac={mac}') + else: + url = url_for('auth.complete_reset', uid=user.id, expires=expires, + mac=mac, _external=True) + template = app.jinja_env.get_or_select_template('email/recovery_link.txt') + _send(user.email, + 'CCExtractor CI platform password recovery instructions', + template.render(url=url, name=user.name)) + + +def _invalid_reset_link(): + """ + Build the single reply every bad reset link gets. + + Expired, unknown user and bad signature deliberately share one message: + telling them apart would confirm which accounts exist. + """ + return make_error_response( + 'invalid_link', + 'This reset link is invalid or has expired. Request a new one.', + http_status=400) + + +@mod_api.route('/auth/tokens', methods=['POST']) +@validate_body(TokenCreateRequestSchema) +def create_token(validated_data=None): + """ + Authenticate with email + password and issue a scoped API token. + + The plaintext token value is returned exactly once in this response. + It's never stored or logged — only the SHA-256 hash is persisted + (see ApiToken: the token is a 256-bit random secret, so a fast hash + with constant-time compare is sufficient). + """ + email = validated_data['email'] + password = validated_data['password'] + token_name = validated_data['token_name'] + expires_in_days = validated_data.get('expires_in_days', 7) + scopes = validated_data.get('scopes') or DEFAULT_SCOPES + + user = User.query.filter_by(email=email).first() + + # Hash password even if user is not found to prevent timing attacks + if user is None: + try: + pwd_context.verify(password, _DUMMY_HASH) + except Exception: + pass + return make_error_response( + 'invalid_credentials', + 'Invalid email or password.', + http_status=401, + ) + + if not user.is_password_valid(password): + return make_error_response( + 'invalid_credentials', + 'Invalid email or password.', + http_status=401, + ) + + # Check role limitations + # Note: Plain 'user' role deliberately cannot request tokens:manage. They + # can create tokens with runs:write but cannot list them. They must revoke + # either the current token or by ID. + allowed_scopes = { + Scope.RUNS_READ, Scope.RUNS_WRITE, Scope.RESULTS_READ, + Scope.SYSTEM_READ, + } + if user.is_admin: + allowed_scopes.add(Scope.TOKENS_MANAGE) + allowed_scopes.add(Scope.BASELINES_WRITE) + allowed_scopes.add(Scope.SYSTEM_WRITE) + + invalid_scopes = set(scopes) - allowed_scopes + if invalid_scopes: + return make_error_response( + 'forbidden', + f'Your current role ({user.role.value}) does not permit requesting ' + f'the following scopes: {", ".join(invalid_scopes)}.', + http_status=403, + ) + + plaintext = ApiToken.generate_token() + token_hash = ApiToken.hash_token(plaintext) + token_prefix = ApiToken.extract_prefix(plaintext) + + api_token = ApiToken( + user_id=user.id, + token_name=token_name, + token_hash=token_hash, + token_prefix=token_prefix, + scopes=scopes, + expires_in_days=expires_in_days, + ) + g.db.add(api_token) + + try: + g.db.commit() + except IntegrityError as e: + g.db.rollback() + error_msg = str(e).lower() + if 'uq_user_token_name' in error_msg or 'api_token.user_id, api_token.token_name' in error_msg: + # Names stay reserved even after revocation (the unique + # constraint spans revoked rows, kept for audit history), + # so "revoke and retry" would not free the name. + return make_error_response( + 'validation_error', + f'Token name "{token_name}" already exists for this user. ' + 'Names remain reserved after revocation; choose a new name.', + details={'fields': { + 'token_name': 'Already in use (including by revoked ' + 'tokens). Choose a different name.'}}, + http_status=400, + ) + raise + + return single_response( + { + 'token': plaintext, + 'token_type': 'bearer', + 'token_name': token_name, + 'scopes': scopes, + 'expires_at': api_token.expires_at, + }, + schema=AuthTokenSchema(), + http_status=201, + ) + + +@mod_api.route('/auth/tokens/current', methods=['DELETE']) +def revoke_current_token(): + """Revoke whatever token is in the Authorization header right now. + + Note: This endpoint is intentionally scope-free. Any valid token + is allowed to revoke itself regardless of its scopes. + """ + token = getattr(g, 'api_token', None) + if token is None: + return make_error_response( + 'unauthorized', + 'No token found in the current request.', + http_status=401, + ) + token.revoke() + g.db.add(token) + g.db.commit() + return '', 204 + + +@mod_api.route('/auth/tokens', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_offset_pagination() +def list_tokens(limit=50, offset=0): + """ + List API tokens, paginated. Admin-only. + + tokens:manage is an admin-only scope (see create_token), so the + require_roles guard above already rejects everyone else with 403. + Lists the caller's own tokens by default; pass ?all=true to list + every token in the system. + """ + want_all = request.args.get('all', 'false').lower() == 'true' + + if want_all: + query = ApiToken.query.order_by(ApiToken.created_at.desc()) + else: + query = ApiToken.query.filter_by( + user_id=g.api_user.id, + ).order_by(ApiToken.created_at.desc()) + + total = query.count() + tokens = query.offset(offset).limit(limit).all() + schema = ApiTokenItemSchema(many=True) + + return paginated_response(tokens, total, limit, offset, schema=schema) + + +@mod_api.route('/auth/tokens/', methods=['DELETE']) +@validate_path_id('token_id') +def revoke_specific_token(token_id): + """ + Revoke a token by its numeric ID. + + Non-admins can only revoke their own tokens. Admins can revoke anyone's. + Already-revoked tokens are silently accepted (idempotent). + + Deliberately requires no extra scope: scopes gate data access, while + revocation is self-service credential hygiene. Any valid token may + revoke tokens belonging to its own user — plain users cannot obtain + tokens:manage (see create_token), yet must be able to clean up their + own credentials. + """ + is_admin = g.api_user.is_admin + token = ApiToken.query.filter_by(id=token_id).first() + + # Non-admins get a uniform 404 for both "doesn't exist" and "belongs to + # another user" to prevent token-ID enumeration. + is_own = token is not None and token.user_id == g.api_user.id + if not token or (not is_admin and not is_own): + return make_error_response('not_found', 'Token not found.', http_status=404) + + # Reaching here means the caller is either the owner or an admin (any other + # caller was already given a 404 above), so the revocation is authorized. + if not token.is_revoked: + token.revoke() + g.db.add(token) + g.db.commit() + + return '', 204 + + +@mod_api.route('/auth/me', methods=['GET']) +def get_current_user(): + """ + Return the identity, role and scopes behind the calling token. + + Needs no extra scope: it reports on the caller itself and discloses + nothing another endpoint would withhold. + """ + return single_response({ + 'user_id': g.api_user.id, + 'name': g.api_user.name, + 'email': g.api_user.email, + 'role': g.api_user.role.value, + 'scopes': g.api_token.scopes if g.api_token else [], + }) + + +def _serialize_user(user): + """Public shape of a user; omits password hash and GitHub token.""" + return { + 'user_id': user.id, + 'name': user.name, + 'email': user.email, + 'role': user.role.value, + 'github_linked': bool(user.github_login), + 'github_login': user.github_login, + } + + +@mod_api.route('/users', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_offset_pagination() +def list_users(limit=50, offset=0): + """List platform users, oldest first.""" + query = User.query.order_by(User.id.asc()) + total = query.count() + users = query.offset(offset).limit(limit).all() + return paginated_response( + [_serialize_user(user) for user in users], total, limit, offset) + + +@mod_api.route('/users/', methods=['PATCH']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_path_id('user_id') +@validate_body(RoleUpdateSchema) +def update_user_role(user_id, validated_data=None): + """ + Change a user's role. + + Admins cannot change their own role: demoting the last admin here + would leave nobody able to undo it. + """ + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + + if user.id == g.api_user.id: + return make_error_response( + 'forbidden', 'You cannot change your own role.', http_status=403) + + previous_role = user.role.value + user.role = Role.from_string(validated_data['role']) + g.db.commit() + + g.log.info(f'user {user.id} role {previous_role} -> {user.role.value} ' + f'by admin {g.api_user.id}') + return single_response(_serialize_user(user)) + + +@mod_api.route('/auth/signup', methods=['POST']) +@validate_body(EmailOnlySchema) +def signup(validated_data=None): + """ + Send a registration link to an email address. + + Answers the same way whether or not the address is already registered, + because a different reply here would tell an anonymous caller who has an + account. The link lands on the classic completion page, which is where + the account is actually created: two places able to mint accounts is a + surface worth not having. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + email = validated_data['email'] + existing = User.query.filter_by(email=email).first() + + if existing is None: + expires = int(time.time()) + _LINK_TTL + mac = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), f'{email}|{expires}') + url = url_for('auth.complete_signup', email=email, expires=expires, + mac=mac, _external=True) + template = 'email/registration_email.txt' + message = app.jinja_env.get_or_select_template(template).render(url=url) + else: + url = url_for('auth.reset', _external=True) + template = 'email/registration_existing.txt' + message = app.jinja_env.get_or_select_template(template).render( + url=url, name=existing.name) + + _send(email, 'CCExtractor CI platform registration', message) + return single_response({'sent': True}, http_status=202) + + +@mod_api.route('/auth/password-reset', methods=['POST']) +@validate_body(EmailOnlySchema) +def request_password_reset(validated_data=None): + """ + Send a password reset link. + + Silent about whether the address is registered, for the same reason + signup is. + """ + user = User.query.filter_by(email=validated_data['email']).first() + if user is not None: + _send_reset_link(user) + return single_response({'sent': True}, http_status=202) + + +@mod_api.route('/auth/password-reset/complete', methods=['POST']) +@validate_body(PasswordResetCompleteSchema) +def complete_password_reset(validated_data=None): + """ + Set a new password using a link from the reset email. + + The signature covers the current password hash, so a link stops working + the moment it is used or the password changes by any other route. The + caller is not signed in here: they still have to authenticate, which + proves the new password arrived intact. + """ + from mod_auth.controllers import generate_hmac_hash + from run import app + + data = validated_data + if int(time.time()) > data['expires']: + return _invalid_reset_link() + + user = User.query.filter_by(id=data['user_id']).first() + if user is None: + return _invalid_reset_link() + + expected = generate_hmac_hash( + app.config.get('HMAC_KEY', ''), + f"{data['user_id']}|{data['expires']}|{user.password}") + if not hmac.compare_digest(expected, data['mac']): + return _invalid_reset_link() + + user.password = User.generate_hash(data['password']) + g.db.commit() + + template = app.jinja_env.get_or_select_template('email/password_reset.txt') + _send(user.email, 'CCExtractor CI platform password reset', + template.render(name=user.name)) + + g.log.info(f'password reset completed via API for user {user.id}') + return single_response({'user_id': user.id, 'password_changed': True}) + + +@mod_api.route('/auth/me', methods=['PATCH']) +@validate_body(AccountUpdateSchema) +def update_account(validated_data=None): + """ + Change your own name, email or password. + + Touching the email or the password needs the current one as well: the + bearer token proves the request came from a signed-in session, not that + the person sending it knows the account's own credentials. + """ + data = validated_data + if not data or set(data) == {'current_password'}: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + user = g.api_user + sensitive = {'email', 'new_password'} & set(data) + if sensitive: + current = data.get('current_password') + if not current or not user.is_password_valid(current): + return make_error_response( + 'forbidden', + 'current_password is required and must match to change ' + 'your email or password.', + http_status=403) + + if 'email' in data and data['email'] != user.email: + if User.query.filter_by(email=data['email']).first() is not None: + return make_error_response( + 'conflict', 'That email is already in use.', http_status=409) + user.email = data['email'] + + if 'name' in data: + user.name = data['name'] + if 'new_password' in data: + user.password = User.generate_hash(data['new_password']) + + try: + g.db.commit() + except IntegrityError: + # name and email are both unique, so a race lands here. + g.db.rollback() + return make_error_response( + 'conflict', 'That name or email is already in use.', + http_status=409) + + g.log.info(f'user {user.id} updated their own account: ' + f'{sorted(k for k in data if k != "current_password")}') + return single_response(_serialize_user(user)) + + +@mod_api.route('/users//deactivate', methods=['POST']) +@validate_path_id('user_id') +def deactivate_user(user_id): + """ + Anonymise an account and lock it out. + + The row stays so the uploads and runs it owns keep an author, which is + why this scrubs the identity instead of deleting. Open to admins and to + the account's owner, matching the classic page. + + Scope-free for the same reason revoking your own token is: closing your + own account cannot depend on tokens:manage, which no role below admin is + ever allowed to hold. Ownership is checked below instead. + + The account's own tokens are revoked as part of this. Scrambling the + password only stops new ones being minted, and an admin reaching for + this because somebody is abusing the platform means to end the access + they already have, not to leave it running for up to thirty days. + + Tokens belonging to the caller are untouched unless they are the same + account, so an admin doing this to somebody else keeps working. + """ + caller = g.api_user + if caller.role != Role.admin and caller.id != int(user_id): + return make_error_response( + 'forbidden', 'You can only deactivate your own account.', + http_status=403) + + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + + user.name = f'Anonymous {user.id}' + user.email = f'unknown{user.id}@ccextractor.org' + user.password = User.generate_hash(User.create_random_password(16)) + + revoked = 0 + for token in ApiToken.query.filter(ApiToken.user_id == user.id).all(): + if not token.is_revoked: + token.revoke() + revoked += 1 + g.db.commit() + + g.log.warning(f'user {user.id} deactivated via API by {caller.id}, ' + f'{revoked} token(s) revoked') + return single_response({ + 'user_id': user.id, + 'deactivated': True, + 'tokens_revoked': revoked, + }) + + +@mod_api.route('/users/', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.TOKENS_MANAGE) +@validate_path_id('user_id') +def get_user(user_id): + """Return one platform user.""" + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + return single_response(_serialize_user(user)) + + +@mod_api.route('/users//password-reset', methods=['POST']) +@validate_path_id('user_id') +def send_user_password_reset(user_id): + """ + Send a reset link to another account, or to your own. + + Unlike the public /auth/password-reset this names an account rather than + an address, so it says plainly when the id is unknown: the caller is + already signed in and can list users anyway. + + Scope-free for the same reason deactivation is: asking for a reset on + your own account cannot depend on tokens:manage. + """ + caller = g.api_user + if caller.role != Role.admin and caller.id != int(user_id): + return make_error_response( + 'forbidden', 'You can only request a reset for your own account.', + http_status=403) + + user = User.query.filter(User.id == user_id).first() + if user is None: + return make_error_response( + 'not_found', f'User {user_id} not found.', http_status=404) + + _send_reset_link(user) + + g.log.info(f'password reset link sent to user {user.id} by {caller.id}') + return single_response({'user_id': user.id, 'sent': True}, + http_status=202) + + +@mod_api.route('/auth/me/ftp-credentials', methods=['GET']) +@require_scope(Scope.RUNS_WRITE) +def get_ftp_credentials(): + """ + Return the caller's FTP details, creating them on first ask. + + Only ever the caller's own: these are working credentials, so there is + no version of this that reads someone else's. The password is stored in + the clear by design (it is random and cannot be chosen), so this is the + only place it can come from. + + Behind runs:write rather than open to any token, unlike the rest of + /auth/me: FTP is another way to upload a sample, so a token narrowed to + reading has no business fetching a working credential for it. Every + role holds runs:write, so this narrows tokens without narrowing people. + """ + from mod_upload.controllers import retrieve_ftp_credentials + from run import config + + credentials = retrieve_ftp_credentials(g.api_user.id) + return single_response({ + 'host': config.get('SERVER_NAME', ''), + 'port': config.get('FTP_PORT', ''), + 'username': credentials.user_name, + 'password': credentials.password, + }) + + +@mod_api.route('/auth/me/github', methods=['GET']) +def get_github_link(): + """ + Say whether the caller's account is connected to GitHub. + + Connecting is a browser redirect, so this hands back the URL to send + somebody to rather than performing it. Trading the code GitHub returns + for a token stays on the classic callback: that needs the client + secret and the redirect registered with GitHub, and one place holding + those is better than two. + + The stored token is never part of this response. The URL carries only + the client id and the scope asked for, both of which are public. + """ + from run import config + + user = g.api_user + client_id = config.get('GITHUB_CLIENT_ID', '') + return single_response({ + 'linked': user.github_token is not None, + 'github_login': user.github_login, + 'authorize_url': ( + 'https://github.com/login/oauth/authorize' + f'?client_id={client_id}&scope=public_repo'), + }) + + +@mod_api.route('/auth/me/github', methods=['DELETE']) +def unlink_github(): + """ + Forget this platform's copy of the caller's GitHub connection. + + Only ever the caller's own, like the FTP details above. This drops the + token the platform holds; the authorisation itself is withdrawn from + GitHub's own applications page, which is the only place that can + really end it. + """ + user = g.api_user + user.github_token = None + user.github_login = None + g.db.commit() + + g.log.info(f'user {user.id} disconnected GitHub via API') + return single_response({'linked': False, 'github_login': None}) diff --git a/mod_api/routes/errors_logs.py b/mod_api/routes/errors_logs.py new file mode 100644 index 000000000..524492e0d --- /dev/null +++ b/mod_api/routes/errors_logs.py @@ -0,0 +1,195 @@ +""" +Error and build log routes. + +GET /runs/{id}/errors Test-level errors for a run +GET /runs/{id}/infrastructure-errors Infra errors (VM, build, worker) +GET /runs/{id}/error-summary Grouped error counts +GET /runs/{id}/logs Build log (cursor-paginated) +GET /runs/{id}/samples/{sid}/logs Per-sample logs (not yet available) +""" + +from flask import g, request + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_cursor_pagination, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.services.error_service import (derive_error_summary, + derive_errors_for_run, + derive_infrastructure_errors) +from mod_api.services.log_service import read_log_lines +from mod_api.utils import cursor_paginated_response, paginated_response +from mod_auth.models import Role +from mod_test.models import Test + + +@mod_api.route('/runs//errors', methods=['GET']) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_run_errors(run_id, limit=50, offset=0): + """List test errors for a run, derived from result and output data.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + errors = derive_errors_for_run(run_id) + + error_type = request.args.get('type') + if error_type: + errors = [e for e in errors if e['type'] == error_type] + + severity = request.args.get('severity') + if severity: + errors = [e for e in errors if e['severity'] == severity] + + sample_id = request.args.get('sample_id', type=int) + if sample_id: + errors = [e for e in errors if e.get('sample_id') == sample_id] + + total = len(errors) + paged = errors[offset:offset + limit] + + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//infrastructure-errors', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_infrastructure_errors(run_id, limit=50, offset=0): + """ + Infra errors classified from TestProgress messages on a best-effort basis. + + Stack traces are opt-in because they may contain internal paths. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + include_stack = request.args.get( + 'include_stack', 'false').lower() == 'true' + if include_stack: + user = getattr(g, 'api_user', None) + allowed = (Role.admin, Role.contributor) + if user is None or user.role not in allowed: + return make_error_response( + 'forbidden', + 'Stack traces require admin or contributor role.', + details={'required_roles': [role.value for role in allowed]}, + http_status=403, + ) + + errors = derive_infrastructure_errors(run_id) + + if not include_stack: + for e in errors: + e.pop('stack', None) + + # Apply optional type and severity filters. + error_type = request.args.get('type') + if error_type: + errors = [e for e in errors if e.get('type') == error_type] + + severity = request.args.get('severity') + if severity: + errors = [e for e in errors if e.get('severity') == severity] + + total = len(errors) + paged = errors[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//error-summary', methods=['GET']) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def get_error_summary(run_id, limit=50, offset=0): + """Group error summary for triaging a run before drilling into details.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + group_by = request.args.get('group_by', 'type') + if group_by not in ('type', 'severity', 'sample_id', 'regression_id'): + return make_error_response( + 'validation_error', + 'group_by must be one of: type, severity, sample_id, regression_id.', + http_status=400, + ) + + severity = request.args.get('severity') + + summary = derive_error_summary(run_id, group_by=group_by) + + if severity: + summary = [s for s in summary if s.get('severity') == severity] + + total = len(summary) + paged = summary[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//logs', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_cursor_pagination(default_limit=100) +def get_run_logs(run_id, limit=100, cursor=None): + """ + Read a run's build log with cursor-based pagination. + + Returns 404 (not a broken download link) when the file doesn't exist. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + level = request.args.get('level') + source = request.args.get('source') + contains = request.args.get('contains') + if contains and len(contains) > 100: + return make_error_response( + 'validation_error', + 'contains parameter must be 100 characters or less.', + http_status=400, + ) + + try: + lines, next_cursor = read_log_lines( + run_id, + cursor=cursor, + limit=limit, + level=level, + source=source, + contains=contains, + ) + except FileNotFoundError: + return make_error_response( + 'log_not_found', + f'Log file for run {run_id} is not available locally. ' + 'It may have been moved to cold storage. Please download it via the artifacts API.', + details={'run_id': run_id, + 'action_required': f'Use GET /runs/{run_id}/artifacts (type=build_log)'}, + http_status=404, + ) + + return cursor_paginated_response(lines, next_cursor, limit) + + +@mod_api.route('/runs//samples//logs', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_cursor_pagination() +def get_sample_logs(run_id, sample_id, limit=50, cursor=None): + """Per-sample logs aren't available yet — the CI worker doesn't support them.""" + return make_error_response( + 'not_found', + f'Per-sample logs are not available for sample {sample_id} in run {run_id}.', + details={ + 'reason': 'Per-sample log storage is not yet supported by the CI worker.'}, + http_status=404, + ) diff --git a/mod_api/routes/regression_tests.py b/mod_api/routes/regression_tests.py new file mode 100644 index 000000000..07268393d --- /dev/null +++ b/mod_api/routes/regression_tests.py @@ -0,0 +1,547 @@ +""" +Regression test detail/write endpoints and the categories that group them. + +GET /regression-tests/{id} Full detail, including baselines and variants +POST /regression-tests Create a test (inactive unless asked otherwise) +PATCH /regression-tests/{id} Partially update a test +DELETE /regression-tests/{id} Delete a test that has never run +GET /regression-tests/{id}/outputs/{oid}/download + Where a baseline lives in storage +GET /regression-tests/{id}/outputs/{oid}/variants/{vid}/download + The same, for an accepted variant +POST /regression-tests/{id}/outputs/{oid}/variants + Accept another output hash as correct +DELETE /regression-tests/{id}/outputs/{oid}/variants/{vid} + Stop accepting one +GET /categories List categories with their test counts +POST /categories Create a category +PATCH /categories/{id} Rename or re-describe a category +DELETE /categories/{id} Delete a category no test references + +Editing the suite previously meant hand-written SQL against the production +database. The regression test list view stays in routes/samples.py. +""" + +from flask import g +from sqlalchemy import func +from sqlalchemy.exc import IntegrityError + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.routes.samples import serialize_rt +from mod_api.schemas.regression_tests import (CategoryCreateSchema, + CategoryUpdateSchema, + RegressionTestCreateSchema, + RegressionTestUpdateSchema, + VariantCreateSchema) +from mod_api.services.storage import resolve_artifact +from mod_api.utils import paginated_response, single_response +from mod_auth.models import Role +from mod_regression.models import (Category, InputType, OutputType, + RegressionTest, RegressionTestOutput, + RegressionTestOutputFiles, + regressionTestLinkTable) +from mod_sample.models import Sample +from mod_test.models import TestResult + + +def _resolve_categories(names): + """ + Look up categories by name. + + Returns the matching rows plus the names that matched nothing, so the + caller can reject the whole request rather than silently dropping a + category the client believed it had set. + """ + rows = Category.query.filter(Category.name.in_(names)).all() + found = {category.name for category in rows} + return rows, [name for name in names if name not in found] + + +def _unknown_categories_response(unknown): + """Build the 400 returned when a request names categories that don't exist.""" + return make_error_response( + 'validation_error', + f"Unknown categories: {', '.join(unknown)}", + details={'fields': {'categories': unknown}}, + http_status=400, + ) + + +@mod_api.route('/regression-tests/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('regression_test_id') +def get_regression_test(regression_test_id): + """ + Return one regression test with its baselines. + + The list endpoint omits outputs because they multiply the payload for + every row; a detail view needs them to show what "expected" means, + including the alternative hashes accepted as variants. + """ + test = RegressionTest.query.filter( + RegressionTest.id == regression_test_id).first() + if test is None: + return make_error_response( + 'not_found', f'Regression test {regression_test_id} not found.', + http_status=404) + + data = serialize_rt(test) + data['outputs'] = [ + { + 'id': output.id, + 'correct': output.correct, + 'correct_extension': output.correct_extension, + 'expected_filename': output.expected_filename, + 'ignore': output.ignore, + # Ids as well as hashes: the download route addresses a variant + # by id, the same way the classic page does. + 'variants': [ + {'id': f.id, 'hash': f.file_hashes} + for f in output.multiple_files + ], + } + for output in test.output_files + ] + return single_response(data) + + +def _get_output(regression_test_id, output_id): + """ + Look up one expected output, matched against its parent test. + + Filtering on both ids means an output id belonging to a different test + reads as absent rather than resolving, so these URLs cannot be used to + walk the table. Returns (output, error_response). + """ + output = RegressionTestOutput.query.filter_by( + id=output_id, regression_id=regression_test_id).first() + if output is None: + return None, make_error_response( + 'not_found', + f'Output {output_id} not found on regression test ' + f'{regression_test_id}.', + http_status=404) + return output, None + + +def _located(filename): + """ + Report where a TestResults file is, in the artifact response shape. + + Like the sample and run-artifact routes, this hands back a signed URL + instead of the bytes, and says which backend holds the file when there + is no URL to give. + """ + url, status = resolve_artifact(f'TestResults/{filename}') + if status == 'missing': + return make_error_response( + 'not_found', f'{filename} is not present in storage.', + http_status=404) + return single_response({ + 'filename': filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route( + '/regression-tests//outputs//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('regression_test_id') +@validate_path_id('output_id') +def download_output(regression_test_id, output_id): + """Locate the baseline a regression test is expected to reproduce.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + return _located(output.filename_correct) + + +@mod_api.route( + '/regression-tests//outputs/' + '/variants//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('regression_test_id') +@validate_path_id('output_id') +@validate_path_id('variant_id') +def download_variant(regression_test_id, output_id, variant_id): + """Locate one of the alternative outputs accepted for a baseline.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + variant = RegressionTestOutputFiles.query.filter_by( + id=variant_id, regression_test_output_id=output.id).first() + if variant is None: + return make_error_response( + 'not_found', + f'Variant {variant_id} not found on output {output_id}.', + http_status=404) + + return _located(f'{variant.file_hashes}{output.correct_extension}') + + +@mod_api.route( + '/regression-tests//outputs//variants', + methods=['POST'] +) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +@validate_path_id('output_id') +@validate_body(VariantCreateSchema) +def create_variant(regression_test_id, output_id, validated_data=None): + """ + Accept another output hash as correct for a baseline. + + A test can legitimately produce different bytes on different platforms + or CCExtractor builds. Recording the hash here makes those runs pass + without overwriting the baseline everyone else is compared against, + which is what promoting to baseline would do. + """ + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + file_hash = validated_data['hash'] + existing = RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=output.id, file_hashes=file_hash).first() + if existing is not None: + return make_error_response( + 'conflict', + f'Output {output_id} already accepts {file_hash}.', + http_status=409) + + variant = RegressionTestOutputFiles(file_hash, output.id) + g.db.add(variant) + g.db.commit() + + g.log.info(f'variant {file_hash} added to output {output_id} via API ' + f'by {g.api_user.id}') + return single_response( + {'id': variant.id, 'hash': variant.file_hashes}, http_status=201) + + +@mod_api.route( + '/regression-tests//outputs/' + '/variants/', + methods=['DELETE'] +) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +@validate_path_id('output_id') +@validate_path_id('variant_id') +def delete_variant(regression_test_id, output_id, variant_id): + """Stop accepting one alternative output for a baseline.""" + output, err = _get_output(regression_test_id, output_id) + if err: + return err + + variant = RegressionTestOutputFiles.query.filter_by( + id=variant_id, regression_test_output_id=output.id).first() + if variant is None: + return make_error_response( + 'not_found', + f'Variant {variant_id} not found on output {output_id}.', + http_status=404) + + g.db.delete(variant) + g.db.commit() + + g.log.info(f'variant {variant_id} removed from output {output_id} via ' + f'API by {g.api_user.id}') + return single_response({'id': variant_id, 'deleted': True}) + + +@mod_api.route('/regression-tests', methods=['POST']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_body(RegressionTestCreateSchema) +def create_regression_test(validated_data=None): + """ + Create a regression test. + + Created inactive unless the body says otherwise, so it cannot join a CI + suite before someone has seen the output it actually produces. + """ + data = validated_data + + sample = Sample.query.filter(Sample.id == data['sample_id']).first() + if sample is None: + return make_error_response( + 'not_found', f"Sample {data['sample_id']} not found.", + http_status=404) + + categories, unknown = _resolve_categories(data['categories']) + if unknown: + return _unknown_categories_response(unknown) + + test = RegressionTest( + sample_id=sample.id, + command=data['command'], + input_type=InputType.from_string(data['input_type']), + output_type=OutputType.from_string(data['output_type']), + # Categories live in a link table; the constructor argument is a + # leftover from the single-category era and maps to no column. + category_id=None, + expected_rc=data['expected_rc'], + active=data['active'], + description=data['description'], + ) + test.categories = categories + g.db.add(test) + g.db.commit() + + g.log.info(f'regression test {test.id} created via API by {g.api_user.id}') + return single_response(serialize_rt(test), http_status=201) + + +@mod_api.route('/regression-tests/', methods=['PATCH']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +@validate_body(RegressionTestUpdateSchema) +def update_regression_test(regression_test_id, validated_data=None): + """ + Update part of a regression test. + + Only the fields present in the body are touched, so two clients editing + different fields cannot clobber each other's work. + """ + test = RegressionTest.query.filter( + RegressionTest.id == regression_test_id).first() + if test is None: + return make_error_response( + 'not_found', f'Regression test {regression_test_id} not found.', + http_status=404) + + data = validated_data + if not data: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + if 'categories' in data: + categories, unknown = _resolve_categories(data['categories']) + if unknown: + return _unknown_categories_response(unknown) + test.categories = categories + + for field in ('command', 'description', 'expected_rc', 'active'): + if field in data: + setattr(test, field, data[field]) + for field, enum in (('input_type', InputType), ('output_type', OutputType)): + if field in data: + setattr(test, field, enum.from_string(data[field])) + + g.db.commit() + + g.log.info(f'regression test {test.id} updated via API by ' + f'{g.api_user.id}: {sorted(data.keys())}') + return single_response(serialize_rt(test)) + + +@mod_api.route('/regression-tests/', methods=['DELETE']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('regression_test_id') +def delete_regression_test(regression_test_id): + """ + Delete a regression test that has never run. + + Once results reference it the test carries history, and removing it + would erase evidence of past regressions, so it is refused with 409 and + the result count. Retiring such a test is a PATCH with active=false. + + Baselines, variants and category links are cleared first: those foreign + keys are RESTRICT and would otherwise block the delete. + """ + test = RegressionTest.query.filter( + RegressionTest.id == regression_test_id).first() + if test is None: + return make_error_response( + 'not_found', f'Regression test {regression_test_id} not found.', + http_status=404) + + result_count = TestResult.query.filter_by( + regression_test_id=test.id).count() + if result_count: + return make_error_response( + 'conflict', + f'Regression test {regression_test_id} has {result_count} ' + f'historical result(s). PATCH active=false to retire it instead.', + details={'result_count': result_count}, + http_status=409, + ) + + deleted_id = test.id + # Queried rather than read off test.output_files: deleting rows underneath + # a live relationship collection mutates it mid-iteration. The outputs go + # through the session so it knows they are gone before the test itself is + # deleted, otherwise SQLAlchemy tries to orphan them and hits stale rows. + outputs = RegressionTestOutput.query.filter_by(regression_id=test.id).all() + for output in outputs: + RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=output.id).delete( + synchronize_session=False) + g.db.delete(output) + + test.categories = [] + g.db.delete(test) + g.db.commit() + + g.log.info(f'regression test {deleted_id} deleted via API by ' + f'{g.api_user.id}') + return single_response({'id': deleted_id, 'deleted': True}) + + +def _test_counts(category_ids): + """Count the regression tests linked to each of these categories.""" + if not category_ids: + return {} + link = regressionTestLinkTable + return dict(g.db.query( + link.c.category_id, func.count(link.c.regression_id) + ).filter( + link.c.category_id.in_(category_ids) + ).group_by(link.c.category_id).all()) + + +def _serialize_category(category, test_count): + """Public shape of a category, including how many tests reference it.""" + return { + 'id': category.id, + 'name': category.name, + 'description': category.description, + 'test_count': test_count, + } + + +@mod_api.route('/categories', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_categories(limit=50, offset=0): + """List categories alphabetically.""" + query = Category.query.order_by(Category.name.asc()) + total = query.count() + rows = query.offset(offset).limit(limit).all() + + # Counted in one grouped query rather than per row: reading + # category.regression_tests would load every linked test just to size it. + counts = _test_counts([row.id for row in rows]) + return paginated_response( + [_serialize_category(row, counts.get(row.id, 0)) for row in rows], + total, limit, offset) + + +@mod_api.route('/categories', methods=['POST']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_body(CategoryCreateSchema) +def create_category(validated_data=None): + """Create a category. Names are unique, so a duplicate is a 409.""" + name = validated_data['name'] + if Category.query.filter(Category.name == name).first() is not None: + return make_error_response( + 'conflict', f"Category '{name}' already exists.", http_status=409) + + category = Category(name, validated_data['description']) + g.db.add(category) + try: + g.db.commit() + except IntegrityError: + # name is unique, so a request that raced the check above ends here. + g.db.rollback() + return make_error_response( + 'conflict', f"Category '{name}' already exists.", http_status=409) + + g.log.info(f'category {category.id} created via API by {g.api_user.id}') + return single_response(_serialize_category(category, 0), http_status=201) + + +@mod_api.route('/categories/', methods=['PATCH']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('category_id') +@validate_body(CategoryUpdateSchema) +def update_category(category_id, validated_data=None): + """Rename or re-describe a category.""" + category = Category.query.filter(Category.id == category_id).first() + if category is None: + return make_error_response( + 'not_found', f'Category {category_id} not found.', http_status=404) + + data = validated_data + if not data: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + # Comparing against the current name first keeps re-sending your own + # name from colliding with yourself. + if 'name' in data and data['name'] != category.name: + if Category.query.filter(Category.name == data['name']).first(): + return make_error_response( + 'conflict', f"Category '{data['name']}' already exists.", + http_status=409) + category.name = data['name'] + + if 'description' in data: + category.description = data['description'] + + try: + g.db.commit() + except IntegrityError: + g.db.rollback() + return make_error_response( + 'conflict', f"Category '{data.get('name')}' already exists.", + http_status=409) + + g.log.info(f'category {category.id} updated via API by {g.api_user.id}') + counts = _test_counts([category.id]) + return single_response( + _serialize_category(category, counts.get(category.id, 0))) + + +@mod_api.route('/categories/', methods=['DELETE']) +@require_roles([Role.contributor, Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('category_id') +def delete_category(category_id): + """ + Delete a category no regression test references. + + Dropping one still in use would change which tests a suite selection + picks up, so it is refused with 409 and the count. Detaching the tests + first is a PATCH on each test's categories. + """ + category = Category.query.filter(Category.id == category_id).first() + if category is None: + return make_error_response( + 'not_found', f'Category {category_id} not found.', http_status=404) + + in_use = _test_counts([category.id]).get(category.id, 0) + if in_use: + return make_error_response( + 'conflict', + f'Category {category_id} is used by {in_use} regression test(s). ' + f'Detach them before deleting it.', + details={'regression_test_count': in_use}, + http_status=409, + ) + + deleted_id = category.id + g.db.delete(category) + g.db.commit() + + g.log.info(f'category {deleted_id} deleted via API by {g.api_user.id}') + return single_response({'id': deleted_id, 'deleted': True}) diff --git a/mod_api/routes/results.py b/mod_api/routes/results.py new file mode 100644 index 000000000..57a49078b --- /dev/null +++ b/mod_api/routes/results.py @@ -0,0 +1,475 @@ +""" +Expected/actual output, diffs, and baseline approval routes. + +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/expected +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/actual +GET /runs/{id}/samples/{sid}/regression-tests/{rid}/outputs/{oid}/diff +POST /runs/{id}/samples/{sid}/baseline-approval Approve a new baseline +""" + +import base64 +import os + +from flask import current_app, g, redirect, request, url_for + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import validate_body, validate_path_id +from mod_api.models.api_token import Scope +from mod_api.schemas.results import (BaselineApprovalRequestSchema, + BaselineApprovalSchema, + OutputFileContentSchema) +from mod_api.services.diff_service import compute_diff, file_sha256, read_lines +from mod_api.services.status import is_dummy_row +from mod_api.services.storage import (get_test_results_base_path, + resolve_artifact) +from mod_api.utils import safe_resolve, single_response +from mod_auth.models import Role +from mod_regression.models import RegressionTestOutputFiles +from mod_test.models import Test, TestResultFile + +INVALID_PATH_MSG = 'Invalid file path.' +READ_ERROR_MSG = 'Failed to read file.' + + +def _find_result_file(run_id, regression_test_id, output_id=None): + """ + Look up the right TestResultFile row. + + Uses run_id + regression_test_id from the path. If output_id is + given as a query param, narrow to that specific output file. + """ + query = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ) + + if output_id is not None: + query = query.filter_by(regression_test_output_id=output_id) + + return query.first() + + +def _validate_result_file_access(run_id, sample_id, regression_id, output_id): + """Validate access to a result file and return it, or an error response.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return None, make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + result_file = _find_result_file(run_id, regression_id, output_id) + + if result_file is None: + return None, make_error_response( + 'not_found', + f'No result for regression test {regression_id}.', + http_status=404, + ) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return None, make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + return result_file, None + + +def _read_output_file(file_path, fmt, is_expected=True): + """Read an output file into the response dict. + + The sha256 in the result is always computed over the complete file, + even when content is truncated to the 1 MiB inline limit. + """ + if not os.path.isfile(file_path): + type_str = 'Expected' if is_expected else 'Actual' + return None, make_error_response( + 'not_found', + f'{type_str} output file not found on disk.', + http_status=404, + ) + + sha256 = file_sha256(file_path) + file_size = os.path.getsize(file_path) + truncated = False + download_url = None + + if file_size > 1048576: + truncated = True + filename = os.path.basename(file_path) + download_url, _ = resolve_artifact(f'TestResults/{filename}') + + if fmt == 'text': + try: + with open(file_path, 'r', encoding='utf-8', errors='replace') as f: + content = f.read(1048576) + encoding = 'utf-8' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + else: + try: + with open(file_path, 'rb') as f: + content = base64.b64encode(f.read(1048576)).decode('ascii') + encoding = 'base64' + except Exception: + return None, make_error_response('internal_error', READ_ERROR_MSG, http_status=500) + + return { + 'content': content, + 'encoding': encoding, + 'sha256': sha256, + 'truncated': truncated, + 'download_url': download_url, + }, None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//expected', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_expected_output(run_id, sample_id, regression_id, output_id): + """Return the expected output file for a regression test result.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response('not_found', 'Expected output not found.', http_status=404) + + base_path = get_test_results_base_path() + expected_filename = result_file.expected + ext = '' + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_filename += ext + + file_path = safe_resolve(base_path, expected_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=True) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{expected_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': expected_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//actual', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_actual_output(run_id, sample_id, regression_id, output_id): + """ + Return the actual output file for a regression test result. + + got=null in the DB means the output matched expected — not that it's + missing. We return 303 (redirect to expected) in that case. Missing + output (the dummy sentinel row) returns 404. + """ + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + if is_dummy_row(result_file): + return make_error_response( + 'missing_output', + 'Test produced no output when output was expected.', + http_status=404, + ) + + if result_file.got is None: + # Relative redirect: clients only resend Authorization headers on + # same-origin redirects, and an absolute URL would break behind a + # reverse proxy anyway. + return redirect(url_for( + 'api.get_expected_output', + run_id=run_id, + sample_id=sample_id, + regression_id=regression_id, + output_id=output_id, + format=request.args.get('format', 'base64'), + ), code=303) + + base_path = get_test_results_base_path() + actual_filename = result_file.got + if result_file.regression_test_output: + ext = result_file.regression_test_output.correct_extension + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename += ext + + file_path = safe_resolve(base_path, actual_filename) + if file_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + fmt = request.args.get('format', 'base64') + + data, err = _read_output_file(file_path, fmt, is_expected=False) + if err: + return err + + content = data['content'] + encoding = data['encoding'] + sha256 = data['sha256'] + truncated = data['truncated'] + download_url = data['download_url'] + + _, storage_status = resolve_artifact(f'TestResults/{actual_filename}') + + return single_response({ + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + 'filename': actual_filename, + 'content_type': 'application/octet-stream', + 'encoding': encoding, + 'content': content, + 'truncated': truncated, + 'download_url': download_url, + 'sha256': sha256, + 'storage_status': storage_status, + }, schema=OutputFileContentSchema()) + + +def _handle_missing_diff(result_file, format_type, diff_ids): + if is_dummy_row(result_file): + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'missing_actual', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + + if result_file.got is None: + if format_type == 'unified': + return single_response({**diff_ids, 'format': 'unified', 'content': ''}) + return single_response({ + **diff_ids, + 'status': 'identical', + 'format': 'structured', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + }) + return None + + +@mod_api.route( + '/runs//samples//regression-tests//outputs//diff', + methods=['GET'] +) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_path_id('regression_id') +@validate_path_id('output_id') +def get_diff(run_id, sample_id, regression_id, output_id): + """Structured diff between expected and actual output.""" + result_file, err = _validate_result_file_access( + run_id, sample_id, regression_id, output_id) + if err: + return err + + diff_ids = { + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': result_file.regression_test_id, + 'output_id': result_file.regression_test_output_id, + } + + format_type = request.args.get('format', 'structured') + + missing_response = _handle_missing_diff(result_file, format_type, diff_ids) + if missing_response: + return missing_response + + base_path = get_test_results_base_path() + ext = result_file.regression_test_output.correct_extension if result_file.regression_test_output else '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + expected_path = safe_resolve(base_path, result_file.expected + ext) + actual_path = safe_resolve(base_path, result_file.got + ext) + + if expected_path is None or actual_path is None: + return make_error_response('forbidden', INVALID_PATH_MSG, http_status=403) + + if not os.path.isfile(expected_path): + return make_error_response('not_found', 'Expected output file not found on disk.', http_status=404) + if not os.path.isfile(actual_path): + return make_error_response('not_found', 'Actual output file not found on disk.', http_status=404) + + max_diff_bytes = 10 * 1024 * 1024 # 10 MiB + if os.path.getsize(expected_path) > max_diff_bytes or os.path.getsize(actual_path) > max_diff_bytes: + return make_error_response('unprocessable', 'File too large for diff. Use download_url.', http_status=422) + + if format_type == 'unified': + import difflib + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + differ = list(difflib.unified_diff( + expected_lines, + actual_lines, + fromfile='expected', + tofile='actual', + lineterm='' + )) + if len(differ) > 10000: + differ = differ[:10000] + differ.append("\n... Diff truncated due to length ...") + unified_content = '\n'.join(differ) + return single_response({ + **diff_ids, + 'format': 'unified', + 'content': unified_content + }) + + context_lines = request.args.get('context_lines', 3, type=int) + context_lines = max(1, min(context_lines, 50)) + + diff_result = compute_diff( + expected_path, actual_path, context_lines=context_lines) + diff_result.update(diff_ids) + diff_result['format'] = 'structured' + return single_response(diff_result) + + +@mod_api.route('/runs//samples//baseline-approval', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.BASELINES_WRITE) +@validate_path_id('run_id') +@validate_path_id('sample_id') +@validate_body(BaselineApprovalRequestSchema) +def create_baseline_approval(run_id, sample_id, validated_data=None): + """ + Record intent to approve actual output as the new expected baseline. + + WARNING: When remove_variants is set to true, this action will remove all + platform-specific variants, making this output the single source of truth + across all platforms. Care should be taken as this applies globally. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response('not_found', f'Run {run_id} not found.', http_status=404) + + regression_id = validated_data['regression_id'] + output_id = validated_data['output_id'] + + result_file = TestResultFile.query.filter_by( + test_id=run_id, + regression_test_id=regression_id, + regression_test_output_id=output_id, + ).first() + + if result_file is None: + return make_error_response('not_found', 'Result file not found.', http_status=404) + + actual_sample_id = ( + result_file.regression_test.sample_id + if result_file.regression_test else None + ) + if actual_sample_id != sample_id: + return make_error_response( + 'not_found', + f'Regression test {regression_id} does not belong to sample {sample_id}.', + http_status=404, + ) + + if is_dummy_row(result_file): + return make_error_response('unprocessable', 'Cannot approve a dummy row.', http_status=422) + + if result_file.got is None: + return make_error_response('unprocessable', 'Output already matches expected.', http_status=422) + + # The actual output file (named by its hash) is already in TestResults/. + # We just need to update the RegressionTestOutput to point to this new hash. + rto = result_file.regression_test_output + if rto is None: + return make_error_response('internal_error', 'No RegressionTestOutput linked.', http_status=500) + + new_baseline = result_file.got + + base_path = get_test_results_base_path() + ext = rto.correct_extension or '' + if ext: + ext = ext.replace('/', '').replace('\\', '').replace('..', '') + actual_filename = new_baseline + ext + file_path = safe_resolve(base_path, actual_filename) + if not file_path or not os.path.isfile(file_path): + return make_error_response('unprocessable', 'Actual output file not found in storage.', http_status=422) + + old_baseline = rto.correct + rto.correct = new_baseline + + remove_variants = validated_data.get('remove_variants', False) + if remove_variants: + RegressionTestOutputFiles.query.filter_by( + regression_test_output_id=rto.id).delete() + + g.db.commit() + + # Audit log: this mutates the global expected baseline (and optionally + # deletes all variants), so record who approved what. + approver = getattr(g, 'api_user', None) + current_app.logger.info( + 'Baseline approved by %s (user_id=%s): run=%s regression=%s ' + 'output=%s old_hash=%s new_hash=%s remove_variants=%s', + approver.name if approver else 'unknown', + approver.id if approver else None, + run_id, regression_id, output_id, old_baseline, new_baseline, + remove_variants) + + import datetime + return single_response({ + 'status': 'approved', + 'run_id': run_id, + 'sample_id': sample_id, + 'regression_id': regression_id, + 'output_id': output_id, + 'requested_by': getattr(g, 'api_user').name if getattr(g, 'api_user', None) else 'unknown', + 'created_at': datetime.datetime.now(datetime.timezone.utc) + }, schema=BaselineApprovalSchema()) diff --git a/mod_api/routes/runs.py b/mod_api/routes/runs.py new file mode 100644 index 000000000..66243df57 --- /dev/null +++ b/mod_api/routes/runs.py @@ -0,0 +1,739 @@ +""" +Test run routes. + +GET /runs List runs (filtered, paginated, sorted) +POST /runs Trigger a new run +GET /runs/{id} Single run details +GET /runs/{id}/summary Pass/fail/skip counts +GET /runs/{id}/progress Progress event timeline +GET /runs/{id}/config Run configuration and test matrix +POST /runs/{id}/cancel Cancel a queued or running test +POST /runs/{id}/restart Clear a run's results so CI picks it up again +""" + +from collections import defaultdict + +from flask import g, request +from sqlalchemy import func, or_ +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, validate_date_range, + validate_offset_pagination, + validate_path_id, validate_sort) +from mod_api.models.api_token import Scope +from mod_api.schemas.runs import (ProgressEventSchema, RunCreateRequestSchema, + RunSchema, RunSummarySchema) +from mod_api.services.error_service import derive_errors_for_run +from mod_api.services.status import (batch_get_run_data, derive_run_status, + derive_sample_status, + expected_regression_ids) +from mod_api.utils import get_sort_column, paginated_response, single_response +from mod_auth.models import Role +from mod_ci.models import GcpInstance +from mod_customized.models import CustomizedTest +from mod_home.models import CCExtractorVersion +from mod_regression.models import RegressionTest, RegressionTestOutput +from mod_test.models import (Fork, Test, TestPlatform, TestProgress, + TestResult, TestResultFile, TestStatus, TestType) + + +def _serialize_run(test): + """Turn a Test row into the Run response shape the spec expects.""" + return _batch_serialize([test])[0] + + +def _batch_serialize(tests, statuses=None, timestamps=None): + if statuses is None or timestamps is None: + statuses, timestamps = batch_get_run_data(tests) + return [ + { + 'run_id': t.id, + 'status': statuses.get(t.id, 'queued'), + 'platform': t.platform.value, + 'test_type': 'pr' if t.test_type == TestType.pull_request else 'commit', + 'repository': t.fork.github_name if t.fork else 'unknown', + 'branch': t.branch, + 'commit_sha': t.commit, + 'built_commit_sha': t.built_commit, + 'pr_number': t.pr_nr if t.pr_nr and t.pr_nr > 0 else None, + 'created_at': timestamps.get(t.id, {}).get('created_at'), + 'queued_at': timestamps.get(t.id, {}).get('queued_at'), + 'started_at': timestamps.get(t.id, {}).get('started_at'), + 'completed_at': timestamps.get(t.id, {}).get('completed_at'), + 'github_link': t.github_link if t.fork else None, + } + for t in tests + ] + + +def _apply_repository_filter(query, repository): + repo_field = RunCreateRequestSchema().fields.get('repository') + if repo_field: + try: + repo_field.deserialize(repository) + except Exception as e: + return None, make_error_response( + 'validation_error', + 'Invalid repository format.', + details={'fields': {'repository': str(e)}}, + http_status=400, + ) + fork_url = f'https://github.com/{repository}.git' + return query.join(Fork).filter(Fork.github == fork_url), None + + +def _apply_date_filters(query, created_after, created_before): + first_progress = ( + g.db.query( + TestProgress.test_id, func.min( + TestProgress.timestamp).label('min_ts')) .group_by( + TestProgress.test_id) .subquery()) + # LEFT JOIN so queued runs (no TestProgress rows yet, hence no known + # creation time) aren't silently dropped — otherwise combining a date + # filter with ?status=queued always returns an empty page. Runs without + # timestamps are treated as matching any requested window. + query = query.outerjoin( + first_progress, Test.id == first_progress.c.test_id) + if created_after: + query = query.filter(or_(first_progress.c.min_ts >= created_after, + first_progress.c.min_ts.is_(None))) + if created_before: + query = query.filter(or_(first_progress.c.min_ts <= created_before, + first_progress.c.min_ts.is_(None))) + return query + + +def _apply_run_filters(query, created_after, created_before): + platform = request.args.get('platform') + if platform: + try: + platform_enum = TestPlatform.from_string(platform) + query = query.filter(Test.platform == platform_enum) + except Exception: + valid_platforms = ', '.join(TestPlatform.values()) + return None, make_error_response( + 'validation_error', + f'Invalid platform: {platform}. Must be one of: {valid_platforms}.', + http_status=400, + ) + + branch = request.args.get('branch') + if branch: + query = query.filter(Test.branch == branch) + + commit_sha = request.args.get('commit_sha') + if commit_sha: + query = query.filter(Test.commit == commit_sha) + + # A release is recorded as the commit it was cut from, so filtering by + # version is the commit filter with one lookup in front of it. + ccx_version = request.args.get('ccx_version') + if ccx_version: + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == ccx_version).first() + if version is None: + return None, make_error_response( + 'validation_error', + f'Unknown CCExtractor version: {ccx_version}.', + http_status=400, + ) + query = query.filter(Test.commit == version.commit) + + repository = request.args.get('repository') + if repository: + query, err = _apply_repository_filter(query, repository) + if err: + return None, err + + if created_after or created_before: + query = _apply_date_filters(query, created_after, created_before) + + return query, None + + +def _validate_run_permissions(user, target_repo, main_repo_full): + # GitHub owner/repo names are case-insensitive. + if target_repo.lower() == main_repo_full.lower(): + allowed = (Role.admin, Role.tester, Role.contributor) + if user.role not in allowed: + return make_error_response( + 'forbidden', + 'Only admins, testers, and contributors can trigger runs for the main repository.', + details={ + 'required_roles': [role.value for role in allowed], + 'repository': target_repo, + }, + http_status=403, + ) + else: + owner = target_repo.split('/')[0] + github_login = getattr(user, 'github_login', None) + + if not github_login and getattr(user, 'github_token', None): + from mod_auth.controllers import fetch_username_from_token + github_login = fetch_username_from_token(user) + if github_login: + user.github_login = github_login + g.db.add(user) + + github_login = github_login or '' + + if not github_login or owner.lower() != github_login.lower(): + return make_error_response( + 'forbidden', + f'You can only trigger runs for your own repository (expected owner: {github_login}) ' + 'or the main repository.', + details={ + 'repository': target_repo, + 'owner_required': github_login, + }, + http_status=403, + ) + return None + + +def _validate_regression_test_ids(regression_test_ids): + if regression_test_ids is not None: + if not regression_test_ids: + return None, make_error_response( + 'validation_error', + 'regression_test_ids cannot be empty.', + details={'fields': { + 'regression_test_ids': 'Must contain at least one ID.'}}, + http_status=400, + ) + active_tests = RegressionTest.query.filter( + RegressionTest.id.in_(regression_test_ids), + RegressionTest.active == True, # noqa: E712 + ).all() + active_ids = {t.id for t in active_tests} + inactive_ids = [ + tid for tid in regression_test_ids if tid not in active_ids] + if inactive_ids: + return None, make_error_response( + 'unprocessable', + 'Some regression test IDs are inactive or do not exist.', + details={'inactive_ids': inactive_ids}, + http_status=422, + ) + else: + active_tests = RegressionTest.query.filter_by(active=True).all() + regression_test_ids = [t.id for t in active_tests] + return regression_test_ids, None + + +@mod_api.route('/runs', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +@validate_sort() +@validate_date_range +def list_runs( + limit=50, + offset=0, + sort='-created_at', + created_after=None, + created_before=None): + """List runs with filters for platform, branch, commit, repo, status, and date range.""" + query, err = _apply_run_filters(Test.query, created_after, created_before) + if err: + return err + + sort_map = { + 'run_id': Test.id, + 'created_at': Test.id, # best proxy - Test has no created_at column + } + order = get_sort_column(sort, sort_map) + if order is not None: + query = query.order_by(order) + else: + query = query.order_by(Test.id.desc()) + + status_filter = request.args.get('status') + if status_filter: + if status_filter not in ('queued', 'running', 'canceled'): + return make_error_response( + 'validation_error', + f'Filtering by status "{status_filter}" is not supported. Supported: queued, running, canceled.', + http_status=400, + ) + + latest_progress_sq = ( + g.db.query(func.max(TestProgress.id).label('max_id')) + .group_by(TestProgress.test_id) + .subquery() + ) + + if status_filter == 'queued': + query = query.outerjoin(TestProgress).filter( + TestProgress.id.is_(None)) + elif status_filter == 'running': + query = query.join( + TestProgress, + TestProgress.test_id == Test.id) .filter( + TestProgress.id.in_(latest_progress_sq)) .filter( + TestProgress.status.in_( + [ + TestStatus.preparation, + TestStatus.testing])) + elif status_filter == 'canceled': + query = query.join(TestProgress, TestProgress.test_id == Test.id)\ + .filter(TestProgress.id.in_(latest_progress_sq))\ + .filter(TestProgress.status == TestStatus.canceled) + + total = query.count() + tests = query.offset(offset).limit(limit).all() + serialized = _batch_serialize(tests) + return paginated_response( + serialized, + total, + limit, + offset, + schema=RunSchema()) + + +def _get_or_create_fork(fork_url): + fork = Fork.query.filter(Fork.github == fork_url).first() + if fork is None: + fork = Fork(fork_url) + g.db.add(fork) + try: + g.db.flush() + except IntegrityError: + g.db.rollback() + fork = Fork.query.filter(Fork.github == fork_url).first() + if fork is None: + return None, make_error_response( + 'internal_error', 'Failed to create or resolve fork.', http_status=500) + return fork, None + + +def _ci_artifact_exists(commit_sha, platform): + """Return True if a CI build artifact exists for this commit + platform. + + The worker runs prebuilt binaries downloaded from GitHub Actions rather + than building from source, so a run can only execute if a build artifact + keyed to ``commit_sha`` exists on the main repo (this is also true for + fork PR commits, whose artifacts are produced by the main repo's PR + workflow). Mirrors verify_artifacts_exist() in the webhook path. + + Fails open (returns True) if GitHub can't be reached, so run creation + never depends on a successful artifact lookup — the cron still guards + against genuinely missing artifacts. + """ + from run import config, log + try: + from github import Auth, Github + + from mod_ci.controllers import find_artifact_for_commit + gh = Github(auth=Auth.Token(config.get('GITHUB_TOKEN', ''))) + repo = gh.get_repo( + f"{config.get('GITHUB_OWNER', '')}/{config.get('GITHUB_REPOSITORY', '')}") + return find_artifact_for_commit(repo, commit_sha, platform, log) is not None + except Exception: + log.exception( + 'create_run: artifact pre-check failed; allowing run to proceed') + return True + + +@mod_api.route('/runs', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_body(RunCreateRequestSchema) +def create_run(validated_data=None): + """Trigger a new test run for a commit + platform combination. + + CI worker pickup: the cron (run_cron.py) picks up any Test row that has + no 'completed'/'canceled' TestProgress, then runs the prebuilt GitHub + Actions artifact for that commit. We therefore reject up front any + commit+platform with no build artifact (see _ci_artifact_exists), so + the run isn't accepted only to fail asynchronously in the worker. + """ + commit_sha = validated_data['commit_sha'] + platform_str = validated_data['platform'] + branch = validated_data.get('branch', 'master') + repository = validated_data.get('repository') + pull_request = validated_data.get('pull_request') or 0 + regression_test_ids = validated_data.get('regression_test_ids') + + platform = TestPlatform.from_string(platform_str) + + # Main repo requires contributor+; forks allow any authenticated user. + from run import config + main_owner = config.get('GITHUB_OWNER', '') + main_repo = config.get('GITHUB_REPOSITORY', '') + main_repo_full = f'{main_owner}/{main_repo}' + # repository is a required field (RunCreateRequestSchema), so it is always + # present; a main-repo run passes the main repo's "owner/repo" explicitly. + target_repo = repository + + err = _validate_run_permissions(g.api_user, target_repo, main_repo_full) + if err: + return err + + # Reject commits with no CI build artifact — the worker runs prebuilt + # binaries, so such a run would be accepted but never execute. + if not _ci_artifact_exists(commit_sha, platform): + return make_error_response( + 'unprocessable', + f'No CI build artifact found for commit {commit_sha[:8]} on ' + f'{platform.value}. Ensure the build workflow has completed for ' + 'this commit before triggering a run.', + details={'commit_sha': commit_sha, 'platform': platform.value}, + http_status=422, + ) + + fork_url = f'https://github.com/{repository}.git' + + fork, err = _get_or_create_fork(fork_url) + if err: + return err + + # Validate regression test IDs against active tests only. + regression_test_ids, err = _validate_regression_test_ids( + regression_test_ids) + if err: + return err + + test_type = TestType.pull_request if pull_request else TestType.commit + + test = Test( + platform=platform, + test_type=test_type, + fork_id=fork.id, + branch=branch, + commit=commit_sha, + pr_nr=pull_request, + ) + g.db.add(test) + try: + g.db.flush() + except Exception: + g.db.rollback() + return make_error_response( + 'internal_error', + 'Failed to create run.', + http_status=500) + + for rt_id in regression_test_ids: + ct = CustomizedTest(test.id, rt_id) + g.db.add(ct) + try: + g.db.commit() + except Exception: + g.db.rollback() + return make_error_response( + 'internal_error', + 'Failed to finalize run.', + http_status=500) + + return single_response( + _serialize_run(test), + schema=RunSchema(), + http_status=202) + + +@mod_api.route('/runs/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run(run_id): + """Fetch a single run by ID.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + return single_response(_serialize_run(test), schema=RunSchema()) + + +def _run_regression_ids(test): + """Regression test IDs that belong to this run. + + Delegates to expected_regression_ids so summary totals and run-status + completeness checks share one rule. (The model's + get_customized_regressiontests() falls back to all tests including + inactive ones, which inflates total_samples/skipped_count with tests + the run could never execute.) + """ + return expected_regression_ids(test) + + +def _aggregate_run_statistics( + results, + files_by_result, + expected_outputs_by_rt): + pass_count = fail_count = skipped_count = missing_count = total_runtime = 0 + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + expected = expected_outputs_by_rt.get(result.regression_test_id) + status = derive_sample_status(result, result_files, expected) + + if status == 'pass': + pass_count += 1 + elif status == 'fail': + fail_count += 1 + elif status == 'missing_output': + missing_count += 1 + else: + skipped_count += 1 + + if result.runtime: + total_runtime += result.runtime + + return pass_count, fail_count, skipped_count, missing_count, total_runtime + + +@mod_api.route('/runs//summary', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run_summary(run_id): + """ + Aggregate pass/fail/skip/missing/error counts from result rows. + + fail_count comes from TestResult rows, not from test.failed (which + only reflects cancellation status and is unreliable for this purpose). + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + results = TestResult.query.filter_by(test_id=run_id).all() + total_samples = len(_run_regression_ids(test)) + + # Preload TestResultFiles + + all_files = ( + TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ) + .filter_by(test_id=run_id).all() if results else [] + ) + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + + # Preload expected outputs + expected_outputs_by_rt = defaultdict(list) + if results: + all_expected = RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_([r.regression_test_id for r in results]) + ).all() + for rto in all_expected: + expected_outputs_by_rt[rto.regression_id].append(rto) + + pass_count, fail_count, skipped_count, missing_count, total_runtime = _aggregate_run_statistics( + results, files_by_result, expected_outputs_by_rt) + + # Reconcile skipped samples (those without any TestResult row) + if len(results) < total_samples: + skipped_count += (total_samples - len(results)) + + # Retrieve error_count from the error service + error_count = len( + derive_errors_for_run( + run_id, + expected_outputs_by_rt, + preloaded_results=results, + preloaded_files=all_files)) + + statuses, _ = batch_get_run_data([test]) + run_status = statuses.get(test.id, 'queued') + + return single_response({ + 'run_id': run_id, + 'status': run_status, + 'total_samples': total_samples, + 'pass_count': pass_count, + 'fail_count': fail_count, + 'skipped_count': skipped_count, + 'missing_output_count': missing_count, + 'error_count': error_count, + 'duration_ms': total_runtime if total_runtime > 0 else None, + }, schema=RunSummarySchema()) + + +@mod_api.route('/runs//progress', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def get_run_progress(run_id, limit=50, offset=0): + """ + Get the timeline of progress events for a run, paginated. + + Events come from TestProgress rows written by the CI worker. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + query = TestProgress.query.filter_by(test_id=run_id) + + # Optional status filter. + status_filter = request.args.get('status') + if status_filter: + try: + status_enum = TestStatus.from_string(status_filter) + query = query.filter(TestProgress.status == status_enum) + except Exception: + return make_error_response( + 'validation_error', + f'Invalid status filter: {status_filter}.', + details={ + 'fields': { + 'status': f'Must be one of: {", ".join(TestStatus.values())}.'}}, + http_status=400, + ) + + query = query.order_by(TestProgress.id.asc()) + total = query.count() + progress = query.offset(offset).limit(limit).all() + + events = [{ + 'timestamp': p.timestamp, + 'status': p.status.name, + 'message': p.message, + } for p in progress] + + schema = ProgressEventSchema() + return paginated_response(events, total, limit, offset, schema=schema) + + +@mod_api.route('/runs//config', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +def get_run_config(run_id): + """Get the configuration that was used to launch this run.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + regression_ids = _run_regression_ids(test) + + return single_response({ + 'run_id': run_id, + 'platform': test.platform.value, + 'branch': test.branch, + 'commit_sha': test.commit, + 'regression_test_ids': regression_ids, + }) + + +@mod_api.route('/runs//cancel', methods=['POST']) +@require_roles([Role.admin, Role.contributor, Role.tester]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('run_id') +def cancel_run(run_id): + """Cancel a running or queued test. + + Idempotent — canceling something already finished returns 202 + with status=no_op. + + Note: In this shared CI environment, any user with 'runs:write' + (admin, contributor, tester) can cancel any run on the platform, + regardless of ownership. This is intentional. + """ + test = Test.query.with_for_update().filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + status = derive_run_status(test) + if status in ('pass', 'fail', 'canceled', 'error'): + return single_response({ + 'run_id': run_id, + 'action': 'cancel', + 'status': 'no_op', + 'message': f'Run is already in terminal state: {status}', + }, http_status=202) + + user = g.api_user + reason = None + if request.is_json and request.get_json(silent=True): + reason = request.get_json(silent=True).get('reason') + if reason: + reason_str = str(reason).strip() + if len(reason_str) < 5: + return make_error_response( + 'validation_error', + 'Cancel reason must be at least 5 characters.', + details={'fields': {'reason': 'Minimum length is 5.'}}, + http_status=400, + ) + reason = reason_str[:255] + + cancel_msg = f'Canceled by {user.name} via API' if user else 'Canceled via API' + if reason: + cancel_msg = f'{cancel_msg}: {reason}' + + progress = TestProgress(run_id, TestStatus.canceled, cancel_msg) + g.db.add(progress) + g.db.commit() + + return single_response({ + 'run_id': run_id, + 'action': 'cancel', + 'status': 'accepted', + 'message': 'Run has been canceled.', + }, http_status=202) + + +@mod_api.route('/runs//restart', methods=['POST']) +@require_roles([Role.admin, Role.contributor, Role.tester]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('run_id') +def restart_run(run_id): + """ + Queue a finished or stuck run to be executed again. + + Clearing the results, the progress trail, and any GcpInstance row is + what makes the run eligible again. CI skips tests that still have a + GcpInstance, so a stuck or running restart would otherwise never be + re-queued. The run keeps its id, so existing links stay valid, and the + old results are replaced rather than kept alongside the new ones. + + Like cancel, this is open to anyone holding runs:write rather than to + the run's owner, which suits a shared CI where a stuck VM blocks + everybody. + """ + # Locked the way cancel locks, so two restarts arriving together do not + # both go clearing the same results. + test = Test.query.with_for_update().filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + TestResultFile.query.filter( + TestResultFile.test_id == test.id).delete(synchronize_session=False) + TestResult.query.filter( + TestResult.test_id == test.id).delete(synchronize_session=False) + TestProgress.query.filter( + TestProgress.test_id == test.id).delete(synchronize_session=False) + GcpInstance.query.filter( + GcpInstance.test_id == test.id).delete(synchronize_session=False) + g.db.commit() + + g.log.info(f'run {run_id} restarted via API by {g.api_user.id}') + return single_response({ + 'run_id': run_id, + 'action': 'restart', + 'status': 'accepted', + 'message': 'Run has been queued to run again.', + }, http_status=202) diff --git a/mod_api/routes/samples.py b/mod_api/routes/samples.py new file mode 100644 index 000000000..569ae13e3 --- /dev/null +++ b/mod_api/routes/samples.py @@ -0,0 +1,1155 @@ +""" +Sample and regression test routes. + +GET /runs/{id}/samples Per-run regression test results +GET /runs/{id}/samples/{sid} Single result in a run +GET /samples Media sample catalog +GET /samples/{id} Single media sample +PATCH /samples/{id} Edit tags and upload metadata +DELETE /samples/{id} Delete a sample no test uses +GET /samples/{id}/details Upload metadata, extra files, media info +GET /samples/{id}/download Where the media file lives in storage +GET /samples/{id}/media-info/download + Where the MediaInfo XML lives +GET /samples/{id}/extra-files/{eid}/download + Where an accompanying file lives +DELETE /samples/{id}/extra-files/{eid} + Delete an accompanying file +GET /samples/{id}/history Cross-run history for a sample +GET /tags Tags a sample can be labelled with +POST /tags Create a tag +GET /regression-tests Regression test definitions +""" + +import os +from collections import defaultdict + +from flask import g, request +from sqlalchemy import func +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload, selectinload + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, validate_date_range, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.schemas.samples import (SampleHistoryEntrySchema, + SampleUpdateSchema, TagCreateSchema) +from mod_api.services.status import (batch_get_run_data, derive_output_status, + derive_sample_status, get_run_timestamps, + is_dummy_row) +from mod_api.services.storage import resolve_artifact +from mod_api.utils import paginated_response, safe_resolve, single_response +from mod_auth.models import Role +from mod_home.models import CCExtractorVersion +from mod_regression.models import (Category, RegressionTest, + RegressionTestOutput) +from mod_sample.media_info_parser import (InvalidMediaInfoError, + MediaInfoFetcher) +from mod_sample.models import ExtraFile, Sample, Tag +from mod_test.models import (Test, TestPlatform, TestProgress, TestResult, + TestResultFile) +from mod_upload.models import Platform, Upload + +# Valid per-sample status values accepted by the ?status filter. Limited to the +# statuses derive_sample_status can actually emit, so filtering can't silently +# return empty for a value that never occurs. +_VALID_SAMPLE_STATUSES = frozenset({ + 'pass', 'fail', 'missing_output', 'not_started', +}) + +# How many result rows /samples/{id}/history will scan when ?status is set. +# Status comes from derive_sample_status, which needs the result files and +# expected outputs, so it can't be pushed into SQL the way branch/platform +# can. Without a bound the endpoint has to load a sample's entire history to +# filter it, which is what made it time out on production. Bounded scans are +# reported with pagination.truncated so a caller can tell a capped page from +# a complete one. +_HISTORY_STATUS_SCAN_LIMIT = 1000 + + +def _preload_expected_outputs(results): + """Map regression_test_id -> [RegressionTestOutput] for the given results. + + Lets per-sample status derivation use the same missing-output detection + as /runs/{id}/summary, so the two endpoints can't disagree. + """ + rt_ids = {r.regression_test_id for r in results} + expected_by_rt = defaultdict(list) + if rt_ids: + for rto in RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(rt_ids)).all(): + expected_by_rt[rto.regression_id].append(rto) + return expected_by_rt + + +def _serialize_outputs(result_files): + outputs = [] + for rf in result_files: + if is_dummy_row(rf): + continue + outputs.append({ + 'output_id': rf.regression_test_output_id, + 'filename': ( + rf.regression_test_output.create_correct_filename(rf.expected) + if rf.regression_test_output else rf.expected + ), + 'status': derive_output_status(rf), + }) + return outputs + + +def _serialize_run_sample(result, result_files, expected_outputs=None): + """Build the per-regression-test result dict for a run.""" + status = derive_sample_status(result, result_files, expected_outputs) + outputs = _serialize_outputs(result_files) + + sample_name = None + sample_id = None + command = None + categories = [] + + if result.regression_test: + rt = result.regression_test + command = rt.command + if rt.sample: + sample_id = rt.sample_id + sample_name = rt.sample.original_name + if rt.categories: + categories = [c.name for c in rt.categories] + + return { + 'regression_test_id': result.regression_test_id, + 'sample_id': sample_id, + 'sample_name': sample_name, + 'status': status, + 'exit_code': result.exit_code, + 'expected_rc': result.expected_rc, + 'runtime_ms': result.runtime, + 'command': command, + 'categories': categories, + 'outputs': outputs, + } + + +def _filter_run_samples_by_tag(serialized, tag_filter): + tag_lower = tag_filter.lower() + tagged_sample_ids = set() + + valid_sample_ids = [s['sample_id'] + for s in serialized if s.get('sample_id')] + samples = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id.in_(valid_sample_ids)).all() if valid_sample_ids else [] + sample_map = {sample.id: sample for sample in samples} + + for s in serialized: + if s['sample_id']: + sample = sample_map.get(s['sample_id']) + if sample and any(tag_lower == t.name.lower() + for t in sample.tags): + tagged_sample_ids.add(s['sample_id']) + return [s for s in serialized if s.get('sample_id') in tagged_sample_ids] + + +def _apply_run_sample_filters(serialized, args): + status_filter = args.get('status') + if status_filter: + serialized = [s for s in serialized if s['status'] == status_filter] + + name_filter = args.get('name') + if name_filter: + name_lower = name_filter.lower() + serialized = [s for s in serialized if s.get( + 'sample_name') and name_lower in s['sample_name'].lower()] + + tag_filter = args.get('tag') + if tag_filter: + serialized = _filter_run_samples_by_tag(serialized, tag_filter) + + category_filter = args.get('category') + if category_filter: + cat_lower = category_filter.lower() + serialized = [ + s for s in serialized + if s.get('categories') and cat_lower in [ + c.lower() for c in s['categories'] + ] + ] + return serialized + + +@mod_api.route('/runs//samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_run_samples(run_id, limit=50, offset=0): + """ + List per-sample results for a run, with optional filters. + + Supports ?status, ?name, ?tag, ?category query params. + """ + # Validate the status filter up front, before any DB work. + status_filter = request.args.get('status') + if status_filter and status_filter not in _VALID_SAMPLE_STATUSES: + return make_error_response( + 'validation_error', + f"Invalid status: {status_filter}", + http_status=400 + ) + + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + results = TestResult.query.options( + joinedload(TestResult.regression_test) + .joinedload(RegressionTest.sample), + joinedload(TestResult.regression_test) + .selectinload(RegressionTest.categories), + ).filter_by(test_id=run_id).all() + + # Preload TestResultFiles together with the expected-output rows that + # derive_sample_status compares against. + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by(test_id=run_id).all() if results else [] + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + + # Preload expected outputs so per-sample status matches /summary. + expected_by_rt = _preload_expected_outputs(results) + + # Serialize list to filter by derived status and joined fields + serialized = [] + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + serialized.append(_serialize_run_sample( + result, result_files, + expected_by_rt.get(result.regression_test_id))) + + # Apply query param filters. + serialized = _apply_run_sample_filters(serialized, request.args) + + total = len(serialized) + paged = serialized[offset:offset + limit] + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/runs//samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('run_id') +@validate_path_id('regression_test_id') +def get_run_sample(run_id, regression_test_id): + """Get a single regression test result within a run.""" + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + result = TestResult.query.options( + joinedload(TestResult.regression_test) + .joinedload(RegressionTest.sample), + joinedload(TestResult.regression_test) + .selectinload(RegressionTest.categories), + ).filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ).first() + if result is None: + return make_error_response( + 'not_found', + f'Regression test {regression_test_id} not found in run {run_id}.', + http_status=404, + ) + + result_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by( + test_id=run_id, + regression_test_id=regression_test_id, + ).all() + + expected_by_rt = _preload_expected_outputs([result]) + return single_response(_serialize_run_sample( + result, result_files, expected_by_rt.get(result.regression_test_id))) + + +@mod_api.route('/samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_samples(limit=50, offset=0): + """ + List media samples from the catalog. + + Supports ?name, ?extension, ?tag, ?sha256, + ?status (active/inactive) filters. + """ + query = Sample.query.options(joinedload(Sample.tags)) + + name = request.args.get('name') + if name: + # Escape LIKE wildcards to prevent unintended pattern matching. + # The explicit escape char makes the backslash escaping portable + # rather than relying on the backend's default. + safe_name = name.replace('\\', '\\\\').replace( + '%', '\\%').replace('_', '\\_') + query = query.filter( + Sample.original_name.ilike(f'%{safe_name}%', escape='\\')) + + extension = request.args.get('extension') + if extension: + query = query.filter(Sample.extension == extension) + + sha256_filter = request.args.get('sha256') + if sha256_filter: + query = query.filter(Sample.sha == sha256_filter) + + tag_filter = request.args.get('tag') + if tag_filter: + + query = query.filter(Sample.tags.any( + func.lower(Tag.name) == tag_filter.lower())) + + status_filter = request.args.get('status') + if status_filter: + if status_filter.lower() not in ('active', 'inactive'): + return make_error_response( + 'validation_error', + 'Invalid status: {status_filter}. ' + 'Must be active or inactive.'.format( + status_filter=status_filter), + http_status=400) + want_active = status_filter.lower() == 'active' + if want_active: + query = query.filter( + Sample.tests.any(RegressionTest.active == True) # noqa: E712 + ) # tests refers to RegressionTest + else: + query = query.filter( + ~Sample.tests.any(RegressionTest.active == True) # noqa: E712 + ) # tests refers to RegressionTest + + # Paginate at DB level without Python-side filters + total = query.count() + samples = query.offset(offset).limit(limit).all() + + # Batch load active regression test counts + sample_ids = [s.id for s in samples] + counts_list = g.db.query( + RegressionTest.sample_id, + func.count(RegressionTest.id) + ).filter( + RegressionTest.sample_id.in_(sample_ids), + RegressionTest.active == True # noqa: E712 + ).group_by(RegressionTest.sample_id).all() if sample_ids else [] + counts = dict(counts_list) + + serialized = [] + for s in samples: + active_count = counts.get(s.id, 0) + serialized.append({ + 'sample_id': s.id, + 'sha': s.sha, + 'extension': s.extension, + 'original_name': s.original_name, + 'filename': s.filename, + 'tags': [t.name for t in s.tags], + 'regression_test_count': active_count, + 'active': active_count > 0, + }) + + return paginated_response(serialized, total, limit, offset) + + +@mod_api.route('/samples//details', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def get_sample_details(sample_id): + """ + Everything known about one sample, for a detail view. + + Extends the /samples/{id} summary with the upload record, any extra + files, and the parsed MediaInfo tree. Media info is best-effort: missing + or unparseable XML reports ``null`` rather than failing the response. + Unlike the classic page this never regenerates the XML, because a GET + should not write to the sample repository. + """ + sample = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + upload = Upload.query.filter(Upload.sample_id == sample.id).first() + upload_info = None + if upload is not None: + version = upload.version + upload_info = { + 'platform': upload.platform.value if upload.platform else None, + 'parameters': upload.parameters or '', + 'notes': upload.notes or '', + 'version': version.version if version else None, + 'version_released': ( + version.released.isoformat() + if version and version.released else None), + } + + extra_files = [ + { + 'id': extra.id, + 'original_name': extra.original_name, + 'extension': extra.extension, + } + for extra in ExtraFile.query.filter( + ExtraFile.sample_id == sample.id).all() + ] + + try: + media_info = MediaInfoFetcher(sample).get_media_info() + except InvalidMediaInfoError: + media_info = None + + return single_response({ + 'sample_id': sample.id, + 'sha': sample.sha, + 'extension': sample.extension, + 'original_name': sample.original_name, + 'filename': sample.filename, + 'tags': [tag.name for tag in sample.tags], + 'upload': upload_info, + 'extra_files': extra_files, + 'media_info': media_info, + }) + + +@mod_api.route('/samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def get_sample(sample_id): + """Get a single media sample by its ID.""" + sample = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', + f'Sample {sample_id} not found.', + http_status=404) + + active_count = RegressionTest.query.filter_by( + sample_id=sample.id, active=True + ).count() + + return single_response({ + 'sample_id': sample.id, + 'sha': sample.sha, + 'extension': sample.extension, + 'original_name': sample.original_name, + 'filename': sample.filename, + 'tags': [t.name for t in sample.tags], + 'regression_test_count': active_count, + 'active': active_count > 0, + }) + + +@mod_api.route('/samples//download', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def download_sample(sample_id): + """ + Locate a sample's media file in storage. + + Returns a signed URL rather than the bytes. Samples run to several + gigabytes, so streaming one through the API would hold a worker for the + length of the transfer. Where only the local copy exists there is no URL + to hand out and storage_status reports that instead, matching how + /runs/{id}/artifacts describes the same two backends. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + url, status = resolve_artifact(f'TestFiles/{sample.filename}') + if status == 'missing': + return make_error_response( + 'not_found', + f'Sample {sample_id} is not present in storage.', + http_status=404) + + return single_response({ + 'sample_id': sample.id, + 'filename': sample.filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route('/tags', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_tags(limit=50, offset=0): + """List the tags a sample can be labelled with, alphabetically.""" + query = Tag.query.order_by(Tag.name.asc()) + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [{'id': t.id, 'name': t.name, 'description': t.description or ''} + for t in rows], total, limit, offset) + + +@mod_api.route('/tags', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_body(TagCreateSchema) +def create_tag(validated_data=None): + """Create a tag. Names are unique, so a duplicate is a 409.""" + name = validated_data['name'] + if Tag.query.filter(Tag.name == name).first() is not None: + return make_error_response( + 'conflict', f"Tag '{name}' already exists.", http_status=409) + + tag = Tag(name, validated_data['description']) + g.db.add(tag) + try: + g.db.commit() + except IntegrityError: + # name is unique, so a request that raced the check lands here. + g.db.rollback() + return make_error_response( + 'conflict', f"Tag '{name}' already exists.", http_status=409) + + g.log.info(f'tag {tag.id} created via API by {g.api_user.id}') + return single_response( + {'id': tag.id, 'name': tag.name, 'description': tag.description or ''}, + http_status=201) + + +#: Sample edit fields that live on the upload row rather than the sample. +_UPLOAD_FIELDS = frozenset(['notes', 'parameters', 'platform', 'version']) + + +def _apply_tags(sample, names): + """ + Replace a sample's tags, matching them by name. + + Returns an error response when a name is not a tag, so one typo rejects + the whole request instead of silently dropping a label. + """ + rows = Tag.query.filter(Tag.name.in_(names)).all() + found = {t.name for t in rows} + unknown = [n for n in names if n not in found] + if unknown: + return make_error_response( + 'validation_error', + f"Unknown tags: {', '.join(unknown)}", + details={'fields': {'tags': unknown}}, + http_status=400, + ) + sample.tags = rows + return None + + +def _apply_upload_fields(sample_id, data, requested): + """ + Write the parts of a sample edit that belong to its upload row. + + Returns an error response when there is no upload row to write to, or + when the version named is not one the platform knows. + """ + upload = Upload.query.filter(Upload.sample_id == sample_id).first() + if upload is None: + return make_error_response( + 'conflict', + f'Sample {sample_id} has no upload record, so ' + f"{', '.join(sorted(requested))} cannot be set on it.", + http_status=409, + ) + + if 'notes' in data: + upload.notes = data['notes'] + if 'parameters' in data: + upload.parameters = data['parameters'] + if 'platform' in data: + upload.platform = Platform.from_string(data['platform']) + if 'version' in data: + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == data['version']).first() + if version is None: + return make_error_response( + 'validation_error', + f"Unknown CCExtractor version: {data['version']}", + http_status=400) + upload.version_id = version.id + return None + + +@mod_api.route('/samples/', methods=['PATCH']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +@validate_body(SampleUpdateSchema) +def update_sample(sample_id, validated_data=None): + """ + Edit a sample's tags and the upload metadata recorded with it. + + Notes, parameters, platform and version live on the upload row rather + than the sample, so a sample that arrived without one (over FTP, or + seeded directly) can only have its tags changed. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + data = validated_data + if not data: + return make_error_response( + 'validation_error', 'No fields to update.', http_status=400) + + if 'tags' in data: + err = _apply_tags(sample, data['tags']) + if err: + return err + + requested = _UPLOAD_FIELDS & set(data) + if requested: + err = _apply_upload_fields(sample.id, data, requested) + if err: + return err + + g.db.commit() + + g.log.info(f'sample {sample.id} updated via API by {g.api_user.id}: ' + f'{sorted(data.keys())}') + return single_response({ + 'sample_id': sample.id, + 'tags': [t.name for t in sample.tags], + }) + + +def _remove_local(relative_path): + """ + Delete a file under the sample repository, tolerating its absence. + + In production the repository is a gcsfuse mount, so this removes the + stored object too. A path that no longer exists is not an error: the + row is going either way, and refusing would leave the database holding + a sample whose files are already gone. + """ + from run import config + + root = config.get('SAMPLE_REPOSITORY', '') + path = safe_resolve(root, relative_path) + if path is None: + g.log.warning(f'refusing to delete path outside the repository: ' + f'{relative_path}') + return + try: + os.remove(path) + except OSError as e: + g.log.warning(f'could not delete {relative_path}: {e}') + + +@mod_api.route('/samples/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +def delete_sample(sample_id): + """ + Delete a sample, its extra files and its media info. + + Refused while any regression test still points at it, because removing + the media would leave those tests unable to run and their history + describing a sample nobody can fetch. Detach or delete the tests first. + """ + sample = Sample.query.options(joinedload(Sample.extra_files)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + test_count = RegressionTest.query.filter_by(sample_id=sample.id).count() + if test_count: + return make_error_response( + 'conflict', + f'Sample {sample_id} is used by {test_count} regression ' + f'test(s). Delete those first.', + details={'regression_test_count': test_count}, + http_status=409, + ) + + # Read the paths off the row before it goes, then unlink only once the + # delete is committed: the same order finalize uses, so a failed commit + # cannot leave a sample row pointing at media that is already gone. + paths = [f'TestFiles/extra/{extra.filename}' for extra in + sample.extra_files] + paths.append(f'TestFiles/media/{sample.sha}.xml') + paths.append(f'TestFiles/{sample.filename}') + + g.db.delete(sample) + g.db.commit() + + for path in paths: + _remove_local(path) + + g.log.warning(f'sample {sample_id} deleted via API by {g.api_user.id}') + return single_response({'sample_id': int(sample_id), 'deleted': True}) + + +@mod_api.route( + '/samples//extra-files/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('sample_id') +@validate_path_id('extra_id') +def delete_extra_file(sample_id, extra_id): + """Delete one of the files uploaded alongside a sample.""" + extra, err = _get_extra_file(sample_id, extra_id) + if err: + return err + + _remove_local(f'TestFiles/extra/{extra.filename}') + g.db.delete(extra) + g.db.commit() + + g.log.warning(f'extra file {extra_id} of sample {sample_id} deleted via ' + f'API by {g.api_user.id}') + return single_response({'id': extra_id, 'deleted': True}) + + +def _located(sample_id, relative_path, filename, missing_message): + """Report where one of a sample's files is, in the download shape.""" + url, status = resolve_artifact(relative_path) + if status == 'missing': + return make_error_response( + 'not_found', missing_message, http_status=404) + return single_response({ + 'sample_id': sample_id, + 'filename': filename, + 'download_url': url, + 'storage_status': status, + }) + + +@mod_api.route('/samples//media-info/download', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +def download_media_info(sample_id): + """ + Locate the MediaInfo XML written alongside a sample. + + The parsed tree is already on /samples/{id}/details; this is for anyone + who wants the original file MediaInfo produced. + """ + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + name = f'{sample.sha}.xml' + return _located( + sample.id, f'TestFiles/media/{name}', name, + f'No media info stored for sample {sample_id}.') + + +def _get_extra_file(sample_id, extra_id): + """Look up one extra file, matched against the sample that owns it.""" + extra = ExtraFile.query.filter_by( + id=extra_id, sample_id=sample_id).first() + if extra is None: + return None, make_error_response( + 'not_found', + f'Extra file {extra_id} not found on sample {sample_id}.', + http_status=404) + return extra, None + + +@mod_api.route( + '/samples//extra-files//download', + methods=['GET'] +) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +@validate_path_id('extra_id') +def download_extra_file(sample_id, extra_id): + """Locate one of the files uploaded alongside a sample.""" + extra, err = _get_extra_file(sample_id, extra_id) + if err: + return err + + return _located( + int(sample_id), f'TestFiles/extra/{extra.filename}', extra.filename, + f'Extra file {extra_id} is not present in storage.') + + +def _get_history_failure_signature(result, result_files, status): + if status == 'fail': + for rf in result_files: + if rf.got is not None and not is_dummy_row(rf): + return f'diff_mismatch:output:{rf.regression_test_output_id}' + if result.exit_code != result.expected_rc: + return f'exit_code_mismatch:rc:{result.exit_code}' + elif status == 'missing_output': + return 'missing_output' + return None + + +def _process_history_entries( + results, + files_by_result, + status_filter, + timestamps_map=None, + test_map=None, + expected_by_rt=None): + entries = [] + for result in results: + test = test_map.get(result.test_id) if test_map else result.test + if test is None: + continue + + result_files = files_by_result.get( + (result.test_id, result.regression_test_id), []) + expected = expected_by_rt.get( + result.regression_test_id) if expected_by_rt else None + status = derive_sample_status(result, result_files, expected) + + if status_filter and status != status_filter: + continue + + failure_sig = _get_history_failure_signature( + result, result_files, status) + if timestamps_map is not None and test.id in timestamps_map: + timestamps = timestamps_map[test.id] + else: + timestamps = get_run_timestamps(test) + + entries.append({ + 'run_id': test.id, + 'regression_test_id': result.regression_test_id, + 'status': status, + 'platform': test.platform.value, + 'branch': test.branch, + 'commit_sha': test.commit, + 'tested_at': timestamps.get('completed_at') or timestamps.get('started_at'), + 'failure_signature': failure_sig, + }) + return entries + + +def _build_history_entries(results, status_filter): + """Build history entries for exactly ``results``, batching the lookups. + + Every follow-up query here is keyed off the results handed in, so the + work scales with that list. Callers must therefore pass the page they + intend to return, not the sample's whole history. + """ + if not results: + return [] + + test_ids = list({r.test_id for r in results}) + + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter(TestResultFile.test_id.in_(test_ids)).all() + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[(f.test_id, f.regression_test_id)].append(f) + + # Preload expected outputs so status matches /summary and /samples. + expected_by_rt = _preload_expected_outputs(results) + + # Batch load tests to avoid N+1 in _process_history_entries + unique_tests = Test.query.filter(Test.id.in_(test_ids)).all() + test_map = {t.id: t for t in unique_tests} + + # Batch compute timestamps for all referenced tests + _, timestamps_map = batch_get_run_data(unique_tests) + + return _process_history_entries( + results, + files_by_result, + status_filter, + timestamps_map=timestamps_map, + test_map=test_map, + expected_by_rt=expected_by_rt) + + +def _resolve_history_rt_ids(rt_ids, sample_id): + """Narrow a sample's regression tests to the optional ?regression_test_id. + + Without this, ``limit`` counts rows across every regression test on the + sample, so a caller asking about one test gets roughly + limit / len(rt_ids) runs of it and cannot tell the window was short. + """ + raw = request.args.get('regression_test_id') + if raw is None: + return rt_ids, None + + try: + rt_id = int(raw) + if rt_id < 1 or rt_id > 2147483647: + raise ValueError('Out of bounds') + except (ValueError, TypeError): + return None, make_error_response( + 'validation_error', + 'regression_test_id must be a positive integer ' + 'between 1 and 2147483647.', + details={ + 'fields': { + 'regression_test_id': 'Must be a positive integer ' + 'between 1 and 2147483647.'}}, + http_status=400, + ) + + if rt_id not in rt_ids: + return None, make_error_response( + 'validation_error', + f'Regression test {rt_id} does not belong to sample {sample_id}.', + details={ + 'fields': { + 'regression_test_id': 'Must be a regression test of ' + 'this sample.'}}, + http_status=400, + ) + + return [rt_id], None + + +def _apply_history_filters( + query, + branch, + platform, + created_after, + created_before): + if branch: + query = query.filter(Test.branch == branch) + + if platform: + try: + platform_enum = TestPlatform.from_string(platform) + query = query.filter(Test.platform == platform_enum) + except Exception: + valid_platforms = ', '.join(TestPlatform.values()) + return None, make_error_response( + 'validation_error', 'Invalid platform: {platform}. ' + 'Must be one of: {valid_platforms}.'.format( + platform=platform, valid_platforms=valid_platforms + ), + http_status=400, + ) + + if created_after or created_before: + + first_progress = ( + g.db.query(TestProgress.test_id, func.min( + TestProgress.timestamp).label('min_ts')) + .group_by(TestProgress.test_id) + .subquery() + ) + query = query.join(first_progress, Test.id == first_progress.c.test_id) + if created_after: + query = query.filter(first_progress.c.min_ts >= created_after) + if created_before: + query = query.filter(first_progress.c.min_ts <= created_before) + + return query, None + + +@mod_api.route('/samples//history', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('sample_id') +@validate_offset_pagination() +@validate_date_range +def get_sample_history( + sample_id, + limit=50, + offset=0, + created_after=None, + created_before=None): + """ + Show how a sample performed across different runs. + + Use failure_signature to tell apart genuine regressions from infra flakes. + + Pass ?regression_test_id to follow a single regression test, so that + ``limit`` means that many runs of it rather than that many rows spread + across every regression test on the sample. + + ?status is applied after status derivation, which needs result files, so + it can't be pushed into SQL. That path scans the most recent + _HISTORY_STATUS_SCAN_LIMIT results and sets pagination.truncated when a + sample has more history than that. + """ + sample = Sample.query.options(joinedload(Sample.tags)).filter( + Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', + f'Sample {sample_id} not found.', + http_status=404) + + regression_tests = RegressionTest.query.filter_by( + sample_id=sample_id).all() + rt_ids = [rt.id for rt in regression_tests] + + if not rt_ids: + return paginated_response([], 0, limit, offset) + + rt_ids, err = _resolve_history_rt_ids(rt_ids, sample_id) + if err: + return err + + # Validate the status filter up front, before any heavy query. + status_filter = request.args.get('status') + if status_filter and status_filter not in _VALID_SAMPLE_STATUSES: + return make_error_response( + 'validation_error', + f"Invalid status: {status_filter}", + http_status=400 + ) + + query = TestResult.query.filter( + TestResult.regression_test_id.in_(rt_ids) + ).join(Test, Test.id == TestResult.test_id) + + branch = request.args.get('branch') + platform = request.args.get('platform') + + query, err = _apply_history_filters( + query, branch, platform, created_after, created_before) + if err: + return err + + # regression_test_id breaks ties so a row can't shift between pages when + # several regression tests share a run. + query = query.order_by(Test.id.desc(), TestResult.regression_test_id.asc()) + + if status_filter: + # One row past the cap tells us whether the scan was complete. + scanned = query.limit(_HISTORY_STATUS_SCAN_LIMIT + 1).all() + truncated = len(scanned) > _HISTORY_STATUS_SCAN_LIMIT + entries = _build_history_entries( + scanned[:_HISTORY_STATUS_SCAN_LIMIT], status_filter) + return paginated_response( + entries[offset:offset + limit], + len(entries), + limit, + offset, + schema=SampleHistoryEntrySchema(), + truncated=truncated, + extra_meta={'scan_limit': _HISTORY_STATUS_SCAN_LIMIT} + if truncated else None, + ) + + # No status filter: the page can be cut in SQL, so everything below it + # loads one page worth of rows instead of the sample's whole history. + total = query.count() + results = query.offset(offset).limit(limit).all() + entries = _build_history_entries(results, None) + + return paginated_response( + entries, total, limit, offset, schema=SampleHistoryEntrySchema() + ) + + +def serialize_rt(rt): + """Public shape of a regression test definition, without its outputs.""" + return { + 'regression_test_id': rt.id, + 'sample_id': rt.sample_id, + 'sample_name': rt.sample.original_name if rt.sample else None, + 'command': rt.command, + 'input_type': rt.input_type.value, + 'output_type': rt.output_type.value, + 'expected_rc': rt.expected_rc, + 'active': rt.active, + 'categories': [c.name for c in rt.categories], + 'description': rt.description, + } + + +@mod_api.route('/regression-tests', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_regression_tests(limit=50, offset=0): + """ + List regression test definitions. + + Supports ?active, ?category, ?tag, ?sample_id filters. Note: when + ?active is omitted it defaults to true, so inactive regression tests + are hidden unless ?active=false is passed explicitly. + """ + query = RegressionTest.query.options( + joinedload(RegressionTest.sample), + selectinload(RegressionTest.categories), + ) + + active_filter = request.args.get('active') + if active_filter is not None: + value = active_filter.lower() + if value in ('true', '1', 'yes'): + is_active = True + elif value in ('false', '0', 'no'): + is_active = False + else: + # Reject garbage instead of silently treating it as false. + return make_error_response( + 'validation_error', + f'Invalid active filter: {active_filter}. ' + 'Must be true or false.', + details={'fields': {'active': 'Must be true or false.'}}, + http_status=400, + ) + else: + is_active = True + query = query.filter(RegressionTest.active == is_active) + + category = request.args.get('category') + if category: + query = query.join(RegressionTest.categories).filter( + Category.name == category) + + sample_id_filter = request.args.get('sample_id') + if sample_id_filter: + try: + sid = int(sample_id_filter) + if sid < 1 or sid > 2147483647: + raise ValueError("Out of bounds") + query = query.filter(RegressionTest.sample_id == sid) + except (ValueError, TypeError): + return make_error_response( + 'validation_error', + 'sample_id must be a positive integer ' + 'between 1 and 2147483647.', + details={ + 'fields': { + 'sample_id': 'Must be a positive integer ' + 'between 1 and 2147483647.'}}, + http_status=400, + ) + + tag_filter = request.args.get('tag') + if tag_filter: + query = query.filter( + RegressionTest.sample.has( + Sample.tags.any(func.lower(Tag.name) == tag_filter.lower()) + ) + ) + + # Paginate at DB level + total = query.count() + tests = query.offset(offset).limit(limit).all() + serialized = [serialize_rt(rt) for rt in tests] + return paginated_response(serialized, total, limit, offset) diff --git a/mod_api/routes/system.py b/mod_api/routes/system.py new file mode 100644 index 000000000..70670141f --- /dev/null +++ b/mod_api/routes/system.py @@ -0,0 +1,589 @@ +""" +System, health, queue, artifact, and platform configuration routes. + +GET /system/health Health check (unauthenticated) +GET /system/queue Queue status — active + queued runs +GET /runs/{id}/artifacts Run artifacts from GCS + local +GET /system/about Versions this platform runs against +GET /system/maintenance Maintenance state per platform +PATCH /system/maintenance/{platform} Pause or resume a platform +GET /system/blocked-users CI users blocked from triggering +POST /system/blocked-users Block a GitHub account +DELETE /system/blocked-users/{user_id} Unblock a GitHub account +GET /system/forbidden-extensions Extensions rejected on upload +POST /system/forbidden-extensions Forbid an extension +DELETE /system/forbidden-extensions/{ext} Allow an extension again + +Every configuration route is admin-only, matching the classic maintenance and +blocked-user pages: the blocklist names accounts, and the rest decides whether +CI accepts work at all. Reads additionally need system:read and writes +system:write, so a token can be narrowed further than the role allows. +""" + +import os +from datetime import datetime, timezone + +from flask import g, jsonify, request +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import joinedload + +from mod_api import mod_api +from mod_api.middleware.auth import require_roles, require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.schemas.common import DATETIME_FORMAT +from mod_api.schemas.system import (BlockedUserCreateSchema, + ForbiddenExtensionCreateSchema, + MaintenanceUpdateSchema) +from mod_api.services.status import batch_get_run_data, is_dummy_row +from mod_api.services.storage import (get_log_file_path, + get_test_results_base_path, + resolve_artifact) +from mod_api.utils import paginated_response, safe_resolve, single_response +from mod_auth.models import Role +from mod_ci.models import BlockedUsers, MaintenanceMode +from mod_home.models import CCExtractorVersion, GeneralData +from mod_sample.models import ForbiddenExtension +from mod_test.models import (Test, TestPlatform, TestProgress, TestResultFile, + TestStatus) + +OCTET_STREAM = 'application/octet-stream' + + +@mod_api.route('/system/health', methods=['GET']) +def system_health(): + """ + Public health check — no auth required. + + Returns 200 when things are ok or degraded, 503 when the system is down. + Monitoring services and load balancers can hit this freely. + """ + now = datetime.now(timezone.utc) + dependencies = [] + overall = 'ok' + + # Database connectivity. + try: + g.db.execute(text('SELECT 1')) + dependencies.append( + {'name': 'database', 'status': 'ok', 'message': None}) + except Exception: + dependencies.append({'name': 'database', + 'status': 'down', + 'message': 'Database connection failed.'}) + overall = 'down' + + # Local sample storage. + try: + from run import config + sample_repo = config.get('SAMPLE_REPOSITORY', '') + if os.path.isdir(sample_repo): + dependencies.append( + {'name': 'local_storage', 'status': 'ok', 'message': None}) + else: + dependencies.append({ + 'name': 'local_storage', + 'status': 'degraded', + 'message': 'Local storage check failed.', + }) + if overall == 'ok': + overall = 'degraded' + except Exception: + dependencies.append({'name': 'local_storage', 'status': 'down', + 'message': 'Local storage check failed.'}) + overall = 'down' + + # Google Cloud Storage. + try: + from run import storage_client_bucket + if storage_client_bucket: + dependencies.append( + {'name': 'gcs', 'status': 'ok', 'message': None}) + else: + dependencies.append({'name': 'gcs', + 'status': 'degraded', + 'message': 'GCS client not initialized.'}) + if overall == 'ok': + overall = 'degraded' + except Exception: + dependencies.append({'name': 'gcs', 'status': 'degraded', + 'message': 'GCS connectivity check failed.'}) + if overall == 'ok': + overall = 'degraded' + + http_status = 503 if overall == 'down' else 200 + response = jsonify({ + 'status': overall, + 'checked_at': now.strftime(DATETIME_FORMAT), + 'dependencies': dependencies, + }) + response.status_code = http_status + return response + + +def _apply_queue_filters( + base_query, + running_subq, + queue_depth, + running_count, + status_filter): + if status_filter == 'queued': + query = base_query.filter(~Test.id.in_( + g.db.query(running_subq.c.test_id))) + total = queue_depth + elif status_filter == 'running': + query = base_query.filter(Test.id.in_( + g.db.query(running_subq.c.test_id))) + total = running_count + elif status_filter: + return None, None, make_error_response( + 'validation_error', 'Invalid status. Must be queued or running.', http_status=400) + else: + query = base_query + total = queue_depth + running_count + return query, total, None + + +@mod_api.route('/system/queue', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +@validate_offset_pagination() +def get_queue(limit=50, offset=0): + """ + Get queue summary and list of runs. + + Note: The `position` field is only populated when `?status=queued` is + explicitly provided. Otherwise, it will be null for all items. + + Excludes anything that's already completed or canceled. Supports + ?platform and ?status filters. + """ + terminal_subq = g.db.query( + TestProgress.test_id + ).filter( + TestProgress.status.in_([TestStatus.completed, TestStatus.canceled]) + ).group_by(TestProgress.test_id).subquery() + + running_subq = g.db.query( + TestProgress.test_id + ).filter( + TestProgress.status.in_([TestStatus.preparation, TestStatus.testing]) + ).group_by(TestProgress.test_id).subquery() + + base_query = Test.query.filter( + ~Test.id.in_(g.db.query(terminal_subq.c.test_id)) + ) + + platform_filter = request.args.get('platform') + if platform_filter: + try: + plat = TestPlatform.from_string(platform_filter) + base_query = base_query.filter(Test.platform == plat) + except Exception: + return make_error_response( + 'validation_error', + 'Invalid platform.', + http_status=400) + + running_count = base_query.filter(Test.id.in_( + g.db.query(running_subq.c.test_id))).count() + queue_depth = base_query.filter(~Test.id.in_( + g.db.query(running_subq.c.test_id))).count() + + status_filter = request.args.get('status') + query, total, err = _apply_queue_filters( + base_query, running_subq, queue_depth, running_count, status_filter) + if err: + return err + + query = query.order_by(Test.id.asc()) + paged_tests = query.offset(offset).limit(limit).all() + + statuses, timestamps = batch_get_run_data(paged_tests) + + paged_jobs = [] + queued_index = offset + 1 if status_filter == 'queued' else None + + for test in paged_tests: + status = statuses.get(test.id, 'queued') + ts = timestamps.get(test.id, {}) + + pos = None + if status == 'queued' and queued_index is not None: + pos = queued_index + queued_index += 1 + + paged_jobs.append({ + 'run_id': test.id, + 'status': status, + 'platform': test.platform.value, + # Same 'Z' format the schemas use, so the API emits one + # datetime style everywhere (timestamps are UTC). + 'queued_at': ts.get('queued_at').strftime(DATETIME_FORMAT) if ts.get('queued_at') else None, + 'started_at': ts.get('started_at').strftime(DATETIME_FORMAT) if ts.get('started_at') else None, + 'position': pos, + }) + + return paginated_response( + paged_jobs, total, limit, offset, + extra_meta={ + 'queue_depth': queue_depth, + 'running_count': running_count, + } + ) + + +def _get_gcs_artifacts(run_id, platform): + binary_name = ( + 'ccextractor' if platform == TestPlatform.linux + else 'ccextractorwinfull.exe' + ) + gcs_artifacts = [ + ('binary', + f'test_artifacts/{run_id}/{binary_name}', binary_name, OCTET_STREAM), + ('coredump', f'test_artifacts/{run_id}/coredump', + f'coredump-{run_id}', OCTET_STREAM), + ( + 'combined_stdout', + f'test_artifacts/{run_id}/combined_stdout.log', + f'combined_stdout-{run_id}.log', + 'text/plain', + ), + ] + artifacts = [] + for artifact_type, gcs_path, filename, content_type in gcs_artifacts: + download_url, storage_status = resolve_artifact(gcs_path) + artifacts.append({ + 'artifact_id': f'{artifact_type}_{run_id}', + 'run_id': run_id, + 'sample_id': None, + 'type': artifact_type, + 'filename': filename, + 'content_type': content_type, + 'size_bytes': None, + 'storage_status': storage_status, + 'download_url': download_url, + }) + return artifacts + + +def _get_output_artifacts(run_id): + result_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output), + joinedload(TestResultFile.regression_test), + ).filter_by(test_id=run_id).all() + for rf in result_files: + if is_dummy_row(rf): + continue + + ext = rf.regression_test_output.correct_extension if rf.regression_test_output else '' + sample_id = (rf.regression_test.sample_id + if rf.regression_test else None) + + expected_name = rf.expected + ext + # NOTE: storage metadata (storage_status, download_url, size_bytes, + # content_type) is resolved by list_artifacts for paged items only. + + yield { + 'artifact_id': f'expected_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}', + 'run_id': run_id, + 'sample_id': sample_id, + 'type': 'expected_output', + 'filename': expected_name, + } + + if rf.got is not None: + actual_name = rf.got + ext + yield { + 'artifact_id': f'actual_{run_id}_{rf.regression_test_id}_{rf.regression_test_output_id}', + 'run_id': run_id, + 'sample_id': sample_id, + 'type': 'actual_output', + 'filename': actual_name, + } + + +@mod_api.route('/runs//artifacts', methods=['GET']) +@require_scope(Scope.RESULTS_READ) +@validate_path_id('run_id') +@validate_offset_pagination() +def list_artifacts(run_id, limit=50, offset=0): + """ + List all artifacts for a run. + + Checks both GCS and local storage. Falls back to local when GCS + is unavailable. Supports ?type filter. + """ + test = Test.query.filter(Test.id == run_id).first() + if test is None: + return make_error_response( + 'not_found', + f'Run {run_id} not found.', + http_status=404) + + artifacts = _get_gcs_artifacts(run_id, test.platform) + + # Build log — accessed via /runs/{id}/logs, no direct download link. + log_path = get_log_file_path(run_id) + artifacts.append({ + 'artifact_id': f'buildlog_{run_id}', + 'run_id': run_id, + 'sample_id': None, + 'type': 'build_log', + 'filename': f'{run_id}.txt', + 'content_type': 'text/plain', + 'size_bytes': os.path.getsize(log_path) if log_path else None, + 'storage_status': 'ok' if log_path else 'missing', + 'download_url': None, + }) + + artifacts.extend(list(_get_output_artifacts(run_id))) + + # Apply optional ?type filter. + type_filter = request.args.get('type') + if type_filter: + artifacts = [a for a in artifacts if a['type'] == type_filter] + + total = len(artifacts) + paged = artifacts[offset:offset + limit] + + # Resolve heavy artifact metadata only for the returned page + base_path = get_test_results_base_path() + for a in paged: + if 'storage_status' not in a: + # It's an output artifact + filename = a['filename'] + url, status = resolve_artifact(f'TestResults/{filename}') + local = safe_resolve(base_path, filename) + + a['content_type'] = OCTET_STREAM + a['size_bytes'] = ( + os.path.getsize(local) + if local and os.path.isfile(local) else None + ) + a['storage_status'] = status + a['download_url'] = url + + return paginated_response(paged, total, limit, offset) + + +@mod_api.route('/system/about', methods=['GET']) +@require_scope(Scope.SYSTEM_READ) +def system_about(): + """ + Report the versions this platform is running against. + + The classic about page is static prose. The parts of it worth reading + from a client are the CCExtractor release under test and the commit the + platform itself was deployed from, so those are what this returns. + """ + from run import app + + latest = CCExtractorVersion.query.order_by( + CCExtractorVersion.released.desc()).first() + last_commit = GeneralData.query.filter( + GeneralData.key == 'last_commit').first() + + return single_response({ + 'platform_commit': app.config.get('BUILD_COMMIT'), + 'ccextractor_version': latest.version if latest else None, + 'ccextractor_released': ( + latest.released.isoformat() + if latest and latest.released else None), + 'last_tested_commit': last_commit.value if last_commit else None, + }) + + +def _maintenance_entry(platform, row): + """Maintenance shape for one platform; no row means never paused.""" + return { + 'platform': platform.value, + 'disabled': bool(row.disabled) if row is not None else False, + } + + +@mod_api.route('/system/maintenance', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_READ) +def get_maintenance(): + """ + Report maintenance state, one entry per platform. + + Keyed on ``platforms`` rather than the ``data`` collection envelope: the + list is derived from the platform enum, not a growable table, so there + is nothing to paginate. + """ + # No unique key on platform, so order it: a duplicated row must not make + # the reported state depend on which one the database returns first. + rows = {} + for row in MaintenanceMode.query.order_by(MaintenanceMode.id.asc()).all(): + rows.setdefault(row.platform, row) + return single_response({'platforms': [ + _maintenance_entry(platform, rows.get(platform)) + for platform in TestPlatform + ]}) + + +@mod_api.route('/system/maintenance/', methods=['PATCH']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_WRITE) +@validate_body(MaintenanceUpdateSchema) +def update_maintenance(platform, validated_data=None): + """ + Pause or resume CI for one platform. + + While a platform is disabled new runs still queue; they are simply not + handed to a VM until it is resumed. + """ + if platform not in TestPlatform.values(): + return make_error_response( + 'validation_error', + f'Invalid platform. Must be one of: ' + f'{", ".join(sorted(TestPlatform.values()))}.', + http_status=400, + ) + + target = TestPlatform.from_string(platform) + disabled = validated_data['disabled'] + row = MaintenanceMode.query.filter( + MaintenanceMode.platform == target).order_by( + MaintenanceMode.id.asc()).first() + if row is None: + row = MaintenanceMode(target, disabled) + g.db.add(row) + else: + row.disabled = disabled + g.db.commit() + + g.log.info(f'maintenance for {target.value} set to disabled={disabled} ' + f'via API by user {g.api_user.id}') + return single_response(_maintenance_entry(target, row)) + + +@mod_api.route('/system/blocked-users', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_READ) +@validate_offset_pagination() +def list_blocked_users(limit=50, offset=0): + """List the GitHub accounts blocked from triggering CI runs.""" + query = BlockedUsers.query.order_by(BlockedUsers.user_id.asc()) + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [{'user_id': row.user_id, 'comment': row.comment or ''} + for row in rows], + total, limit, offset) + + +@mod_api.route('/system/blocked-users', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_WRITE) +@validate_body(BlockedUserCreateSchema) +def create_blocked_user(validated_data=None): + """Block a GitHub account from triggering CI runs.""" + user_id = validated_data['user_id'] + if BlockedUsers.query.filter( + BlockedUsers.user_id == user_id).first() is not None: + return make_error_response( + 'conflict', f'GitHub user {user_id} is already blocked.', + http_status=409) + + row = BlockedUsers(user_id, validated_data['comment']) + g.db.add(row) + try: + g.db.commit() + except IntegrityError: + # user_id is the primary key, so a request that raced the check + # above lands here instead of duplicating the row. + g.db.rollback() + return make_error_response( + 'conflict', f'GitHub user {user_id} is already blocked.', + http_status=409) + + g.log.info(f'github user {user_id} blocked via API by user {g.api_user.id}') + return single_response( + {'user_id': row.user_id, 'comment': row.comment or ''}, + http_status=201) + + +@mod_api.route('/system/blocked-users/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_WRITE) +@validate_path_id('user_id') +def delete_blocked_user(user_id): + """Unblock a GitHub account.""" + row = BlockedUsers.query.filter(BlockedUsers.user_id == user_id).first() + if row is None: + return make_error_response( + 'not_found', f'GitHub user {user_id} is not blocked.', + http_status=404) + + g.db.delete(row) + g.db.commit() + + g.log.info(f'github user {user_id} unblocked via API by {g.api_user.id}') + return single_response({'user_id': user_id, 'deleted': True}) + + +@mod_api.route('/system/forbidden-extensions', methods=['GET']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_READ) +@validate_offset_pagination() +def list_forbidden_extensions(limit=50, offset=0): + """List the file extensions rejected on upload.""" + query = ForbiddenExtension.query.order_by( + ForbiddenExtension.extension.asc()) + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [row.extension for row in rows], total, limit, offset) + + +@mod_api.route('/system/forbidden-extensions', methods=['POST']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_WRITE) +@validate_body(ForbiddenExtensionCreateSchema) +def create_forbidden_extension(validated_data=None): + """Forbid an extension. Stored lower-cased and without a leading dot.""" + extension = validated_data['extension'].lower() + if ForbiddenExtension.query.filter( + ForbiddenExtension.extension == extension).first() is not None: + return make_error_response( + 'conflict', f"Extension '{extension}' is already forbidden.", + http_status=409) + + g.db.add(ForbiddenExtension(extension)) + try: + g.db.commit() + except IntegrityError: + # extension is the primary key; same race as blocking a user. + g.db.rollback() + return make_error_response( + 'conflict', f"Extension '{extension}' is already forbidden.", + http_status=409) + + g.log.info(f'extension {extension} forbidden via API by {g.api_user.id}') + return single_response({'extension': extension}, http_status=201) + + +@mod_api.route('/system/forbidden-extensions/', methods=['DELETE']) +@require_roles([Role.admin]) +@require_scope(Scope.SYSTEM_WRITE) +def delete_forbidden_extension(extension): + """Allow an extension to be uploaded again.""" + normalized = extension.lstrip('.').lower() + row = ForbiddenExtension.query.filter( + ForbiddenExtension.extension == normalized).first() + if row is None: + return make_error_response( + 'not_found', f"Extension '{normalized}' is not forbidden.", + http_status=404) + + g.db.delete(row) + g.db.commit() + + g.log.info(f'extension {normalized} allowed via API by {g.api_user.id}') + return single_response({'extension': normalized, 'deleted': True}) diff --git a/mod_api/routes/uploads.py b/mod_api/routes/uploads.py new file mode 100644 index 000000000..dd0a002f5 --- /dev/null +++ b/mod_api/routes/uploads.py @@ -0,0 +1,345 @@ +""" +Sample upload and the queue an upload sits in until it is described. + +POST /samples/upload Upload a media file into the queue +GET /queued-samples Uploads waiting to be described +GET /queued-samples/{id} One queued upload +POST /queued-samples/{id}/finalize + Turn a queued upload into a sample +POST /queued-samples/{id}/link Attach one to an existing sample +DELETE /queued-samples/{id} Discard a queued upload + +Uploading and describing are two steps here for the same reason they are on +the classic pages: the transfer is slow and the metadata needs a CCExtractor +version the uploader may have to go and look up, so the bytes are banked +first and the description follows. +""" + +import hashlib +import mimetypes +import os +from uuid import uuid4 + +import magic +from flask import g, request +from werkzeug.utils import secure_filename + +from mod_api import mod_api +from mod_api.middleware.auth import require_scope +from mod_api.middleware.error_handler import make_error_response +from mod_api.middleware.validation import (validate_body, + validate_offset_pagination, + validate_path_id) +from mod_api.models.api_token import Scope +from mod_api.schemas.uploads import QueueLinkSchema, UploadFinalizeSchema +from mod_api.utils import paginated_response, single_response +from mod_auth.models import Role +from mod_home.models import CCExtractorVersion +from mod_sample.models import (ExtraFile, ForbiddenExtension, + ForbiddenMimeType, Sample) +from mod_upload.models import Platform, QueuedSample, Upload + +# Enough of the file for libmagic to identify it, matching the classic form. +_SNIFF_BYTES = 1024 + + +def _repo(*parts): + """Build a path inside the sample repository.""" + from run import config + + return os.path.join(config.get('SAMPLE_REPOSITORY', ''), *parts) + + +def _serialize(queued): + """Public shape of one queued upload.""" + return { + 'id': queued.id, + 'sha': queued.sha, + 'extension': queued.extension, + 'original_name': queued.original_name, + 'user_id': queued.user_id, + } + + +def _forbidden_reason(filename, head): + """ + Say why an upload is not allowed, or None when it is. + + Runs the same three checks as the classic upload form: the extension on + the name, the mime type libmagic reads out of the file, and the + extension that mime type implies. The last one catches a banned format + renamed to get past the first. + """ + extension = os.path.splitext(filename)[1].lstrip('.').lower() + if extension and ForbiddenExtension.query.filter( + ForbiddenExtension.extension == extension).first() is not None: + return f"Files with the '{extension}' extension are not accepted." + + mimetype = magic.from_buffer(head, mime=True) + if ForbiddenMimeType.query.filter( + ForbiddenMimeType.mimetype == mimetype).first() is not None: + return f"Files of type '{mimetype}' are not accepted." + + implied = mimetypes.guess_extension(mimetype) + if implied and ForbiddenExtension.query.filter( + ForbiddenExtension.extension == implied.lstrip('.') + ).first() is not None: + return f"Files of type '{mimetype}' are not accepted." + + return None + + +@mod_api.route('/samples/upload', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +def upload_sample(): + """ + Upload a media file into the queue. + + The file is hashed as it is written, because the hash is both the + duplicate check and the name the sample is stored under, and reading a + multi-gigabyte upload twice to get it would double the cost of every + upload. + + An upload that turns out to be forbidden or already known is deleted + here rather than left in TempFiles for someone to clean up later. + """ + uploaded = request.files.get('file') + if uploaded is None or not uploaded.filename: + return make_error_response( + 'validation_error', + 'Attach the media file as the "file" part of a multipart form.', + http_status=400) + + filename = secure_filename(uploaded.filename) + if not filename: + # secure_filename empties a name made only of dots, separators or + # whitespace, and an empty name resolves to TempFiles itself. + return make_error_response( + 'validation_error', 'Unusable file name.', http_status=400) + + # The staging name belongs to this upload alone rather than to the + # caller's file name: two clients sending the same name at the same + # time would otherwise write into one file and produce a hash that + # describes neither. + temp_path = _repo('TempFiles', f'api-{uuid4().hex}') + + head = uploaded.stream.read(_SNIFF_BYTES) + uploaded.stream.seek(0) + reason = _forbidden_reason(filename, head) + if reason: + g.log.warning(f'user {g.api_user.id} tried to upload {filename}: ' + f'{reason}') + return make_error_response('forbidden', reason, http_status=403) + + digest = hashlib.sha256() + with open(temp_path, 'wb') as out: + for chunk in iter(lambda: uploaded.stream.read(8192), b''): + digest.update(chunk) + out.write(chunk) + sha = digest.hexdigest() + + stored = Sample.query.filter(Sample.sha == sha).first() + queued_already = QueuedSample.query.filter( + QueuedSample.sha == sha).first() + if stored is not None or queued_already is not None: + os.remove(temp_path) + return make_error_response( + 'conflict', + 'A sample with this content is already uploaded or queued.', + details={'sha': sha}, + http_status=409) + + extension = os.path.splitext(filename)[1] + queued = QueuedSample(sha, extension, os.path.splitext(filename)[0], + g.api_user.id) + g.db.add(queued) + g.db.commit() + + os.rename(temp_path, _repo('QueuedFiles', queued.filename)) + + g.log.info(f'sample {sha} queued via API by {g.api_user.id}') + return single_response(_serialize(queued), http_status=201) + + +def _visible_queue(): + """Queued uploads the caller may see: their own, or all for an admin.""" + query = QueuedSample.query.filter( + QueuedSample.user_id == g.api_user.id) + if g.api_user.role == Role.admin: + query = QueuedSample.query + return query.order_by(QueuedSample.id.asc()) + + +@mod_api.route('/queued-samples', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_offset_pagination() +def list_queued_samples(limit=50, offset=0): + """List uploads waiting to be described. Admins see everyone's.""" + query = _visible_queue() + total = query.count() + rows = query.offset(offset).limit(limit).all() + return paginated_response( + [_serialize(q) for q in rows], total, limit, offset) + + +def _get_queued(queued_id): + """Look up a queued upload the caller is allowed to see.""" + queued = _visible_queue().filter(QueuedSample.id == queued_id).first() + if queued is None: + # Someone else's upload reads as absent rather than forbidden, so + # the id cannot be used to find out what others have queued. + return None, make_error_response( + 'not_found', f'Queued sample {queued_id} not found.', + http_status=404) + return queued, None + + +@mod_api.route('/queued-samples/', methods=['GET']) +@require_scope(Scope.RUNS_READ) +@validate_path_id('queued_id') +def get_queued_sample(queued_id): + """Return one queued upload.""" + queued, err = _get_queued(queued_id) + if err: + return err + return single_response(_serialize(queued)) + + +@mod_api.route('/queued-samples//finalize', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +@validate_body(UploadFinalizeSchema) +def finalize_queued_sample(queued_id, validated_data=None): + """ + Turn a queued upload into a sample. + + The file only moves into TestFiles once the rows are committed, so a + failure part way through leaves the upload in the queue to retry rather + than a sample row pointing at a file that was never placed. + + Reporting the sample as a GitHub issue, which the classic page offers + here, is left to the classic page: it needs the platform's GitHub token + and has nothing to do with getting the sample into the library. + """ + queued, err = _get_queued(queued_id) + if err: + return err + + data = validated_data + version = CCExtractorVersion.query.filter( + CCExtractorVersion.version == data['version']).first() + if version is None: + return make_error_response( + 'validation_error', + f"Unknown CCExtractor version: {data['version']}", + http_status=400) + + source = _repo('QueuedFiles', queued.filename) + if not os.path.isfile(source): + return make_error_response( + 'conflict', + f'The uploaded file for queued sample {queued_id} is missing ' + f'from storage, so it cannot be finalized.', + http_status=409) + + destination = _repo('TestFiles', queued.filename) + sample = Sample(queued.sha, queued.extension.lstrip('.'), + queued.original_name) + g.db.add(sample) + g.db.flush([sample]) + g.db.add(Upload( + g.api_user.id, sample.id, version.id, + Platform.from_string(data['platform']), + data['parameters'], data['notes'], + )) + g.db.delete(queued) + g.db.commit() + + os.rename(source, destination) + + g.log.info(f'queued sample {queued_id} finalized as sample {sample.id} ' + f'via API by {g.api_user.id}') + return single_response({ + 'sample_id': sample.id, + 'sha': sample.sha, + 'original_name': sample.original_name, + }, http_status=201) + + +@mod_api.route('/queued-samples//link', methods=['POST']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +@validate_body(QueueLinkSchema) +def link_queued_sample(queued_id, validated_data=None): + """ + Attach a queued upload to an existing sample as an extra file. + + Used for the material that belongs with a sample without being one, such + as a subtitle track to compare against. The classic page offered this + but never carried it out: its confirm step checks permissions and then + redirects without touching anything, so this is the behaviour it + described rather than the behaviour it had. + + As with finalize, the file moves only after the row is committed, since + the stored name is derived from the id the row is given. + """ + queued, err = _get_queued(queued_id) + if err: + return err + + sample_id = validated_data['sample_id'] + sample = Sample.query.filter(Sample.id == sample_id).first() + if sample is None: + return make_error_response( + 'not_found', f'Sample {sample_id} not found.', http_status=404) + + source = _repo('QueuedFiles', queued.filename) + if not os.path.isfile(source): + return make_error_response( + 'conflict', + f'The uploaded file for queued sample {queued_id} is missing ' + f'from storage, so it cannot be linked.', + http_status=409) + + extra = ExtraFile(sample.id, queued.extension.lstrip('.'), + queued.original_name) + g.db.add(extra) + g.db.flush([extra]) + filename = extra.filename + g.db.delete(queued) + g.db.commit() + + os.rename(source, _repo('TestFiles', 'extra', filename)) + + g.log.info(f'queued sample {queued_id} linked to sample {sample.id} as ' + f'extra file {extra.id} via API by {g.api_user.id}') + return single_response({ + 'id': extra.id, + 'sample_id': sample.id, + 'filename': filename, + }, http_status=201) + + +@mod_api.route('/queued-samples/', methods=['DELETE']) +@require_scope(Scope.RUNS_WRITE) +@validate_path_id('queued_id') +def delete_queued_sample(queued_id): + """Discard a queued upload and the file behind it.""" + queued, err = _get_queued(queued_id) + if err: + return err + + path = _repo('QueuedFiles', queued.filename) + try: + os.remove(path) + except OSError as e: + # The row goes either way: refusing would leave a queue entry + # nobody can clear. + g.log.warning(f'could not delete queued file {path}: {e}') + + g.db.delete(queued) + g.db.commit() + + g.log.warning(f'queued sample {queued_id} discarded via API by ' + f'{g.api_user.id}') + return single_response({'id': int(queued_id), 'deleted': True}) diff --git a/mod_api/schemas/__init__.py b/mod_api/schemas/__init__.py new file mode 100644 index 000000000..889960659 --- /dev/null +++ b/mod_api/schemas/__init__.py @@ -0,0 +1 @@ +"""mod_api.schemas: Marshmallow schemas for request/response validation.""" diff --git a/mod_api/schemas/auth.py b/mod_api/schemas/auth.py new file mode 100644 index 000000000..b003adaea --- /dev/null +++ b/mod_api/schemas/auth.py @@ -0,0 +1,147 @@ +"""Request/response schemas for the token and user endpoints.""" + +from marshmallow import RAISE, Schema, ValidationError, fields, validate + +from mod_api.models.api_token import VALID_SCOPES +from mod_api.schemas.common import DATETIME_FORMAT +from mod_auth.models import Role + + +class TokenCreateRequestSchema(Schema): + """Validates POST /auth/tokens bodies.""" + + email = fields.Email(required=True) + password = fields.String( + required=True, + validate=validate.Length(min=8, max=128), + ) + token_name = fields.String( + required=True, + validate=[ + validate.Length(min=1, max=50), + validate.Regexp( + r'^[a-zA-Z0-9_\-]+$', + error='token_name must match ^[a-zA-Z0-9_-]+$', + ), + ], + ) + expires_in_days = fields.Integer( + load_default=7, + validate=validate.Range(min=1, max=30), + ) + scopes = fields.List( + fields.String(validate=validate.OneOf(VALID_SCOPES)), + load_default=None, + # Bounded by the scope list itself, so adding a scope cannot silently + # make "ask for everything I'm allowed" fail validation. + validate=validate.Length(max=len(VALID_SCOPES)), + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class AuthTokenSchema(Schema): + """The one-time response returned when a token is created.""" + + token = fields.String(required=True) + token_type = fields.String(dump_default='bearer') + token_name = fields.String(required=True) + scopes = fields.List(fields.String(), required=True) + expires_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + + +class ApiTokenItemSchema(Schema): + """Token metadata for list responses — never includes the plaintext.""" + + id = fields.Integer(required=True) + user_id = fields.Integer(required=True) + token_name = fields.String(required=True) + scopes = fields.Method('get_scopes') + created_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + expires_at = fields.DateTime(required=True, format=DATETIME_FORMAT) + is_revoked = fields.Boolean(required=True) + revoked_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + + def get_scopes(self, obj): + """Deserialize scopes from the model's JSON column.""" + return obj.scopes + + +class RoleUpdateSchema(Schema): + """Validates PATCH /users/{id} bodies.""" + + # Taken from the model so the accepted values follow the Role enum. + role = fields.String( + required=True, validate=validate.OneOf(sorted(Role.values()))) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class EmailOnlySchema(Schema): + """Validates the signup and password reset request bodies.""" + + email = fields.Email(required=True) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +def valid_password(value): + """ + Check a password against the configured length bounds. + + Written as a validator rather than a pre-built Length, because the + bounds come from config and reading those while the schema class is + being defined would import the app mid-blueprint-setup. The classic + forms read the same two keys, so the rules cannot drift apart. + """ + from run import config + + low = int(config.get('MIN_PWD_LEN', 10)) + high = int(config.get('MAX_PWD_LEN', 500)) + if not low <= len(value) <= high: + raise ValidationError( + f'Password must be between {low} and {high} characters.') + + +class PasswordResetCompleteSchema(Schema): + """Validates POST /auth/password-reset/complete bodies.""" + + user_id = fields.Integer(required=True, validate=validate.Range(min=1)) + expires = fields.Integer(required=True) + mac = fields.String( + required=True, validate=validate.Length(min=1, max=128)) + password = fields.String(required=True, validate=valid_password) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class AccountUpdateSchema(Schema): + """ + Validates PATCH /auth/me bodies; every field optional. + + current_password is demanded alongside any change to the credentials + themselves, so a leaked token cannot quietly become a stolen account. + """ + + name = fields.String(validate=validate.Length(min=1, max=50)) + email = fields.Email() + new_password = fields.String(validate=valid_password) + current_password = fields.String( + validate=validate.Length(min=1, max=500)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/common.py b/mod_api/schemas/common.py new file mode 100644 index 000000000..9e244bcbd --- /dev/null +++ b/mod_api/schemas/common.py @@ -0,0 +1,3 @@ +"""Constants shared across API schemas.""" + +DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/mod_api/schemas/regression_tests.py b/mod_api/schemas/regression_tests.py new file mode 100644 index 000000000..0ee997696 --- /dev/null +++ b/mod_api/schemas/regression_tests.py @@ -0,0 +1,120 @@ +"""Request schemas for the regression test and category write endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_regression.models import InputType, OutputType + +# Taken from the models rather than restated, so a type the database accepts +# can never be rejected here (or the reverse) after someone edits the enum. +_INPUT_TYPES = sorted(InputType.values()) +_OUTPUT_TYPES = sorted(OutputType.values()) + +# Matches the maxLength the OpenAPI contract already publishes for a +# regression test's command, so a body accepted here cannot produce a +# response that violates the documented schema. +_COMMAND_MAX = 500 + +# Category column widths, shared with the category name list below. +_NAME_MAX = 64 +_DESCRIPTION_MAX = 1024 + +# Matches the maxLength the contract publishes for a variant hash. +_HASH_MAX = 128 + + +class RegressionTestCreateSchema(Schema): + """Validates POST /regression-tests bodies.""" + + sample_id = fields.Integer(required=True, validate=validate.Range(min=1)) + command = fields.String( + required=True, validate=validate.Length(min=1, max=_COMMAND_MAX)) + input_type = fields.String( + load_default='file', validate=validate.OneOf(_INPUT_TYPES)) + output_type = fields.String( + load_default='file', validate=validate.OneOf(_OUTPUT_TYPES)) + expected_rc = fields.Integer( + load_default=0, validate=validate.Range(min=0, max=255)) + description = fields.String( + load_default='', validate=validate.Length(max=_DESCRIPTION_MAX)) + categories = fields.List( + fields.String(validate=validate.Length(min=1, max=_NAME_MAX)), + required=True, + validate=validate.Length(min=1), + ) + # A new test starts inactive: it should only join the CI suite once a + # maintainer has seen what it actually produces on a verification run. + active = fields.Boolean(load_default=False) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class RegressionTestUpdateSchema(Schema): + """Validates PATCH /regression-tests/{id} bodies; every field optional.""" + + command = fields.String(validate=validate.Length(min=1, max=_COMMAND_MAX)) + input_type = fields.String(validate=validate.OneOf(_INPUT_TYPES)) + output_type = fields.String(validate=validate.OneOf(_OUTPUT_TYPES)) + expected_rc = fields.Integer(validate=validate.Range(min=0, max=255)) + description = fields.String(validate=validate.Length(max=_DESCRIPTION_MAX)) + categories = fields.List( + fields.String(validate=validate.Length(min=1, max=_NAME_MAX)), + validate=validate.Length(min=1), + ) + active = fields.Boolean() + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class CategoryCreateSchema(Schema): + """Validates POST /categories bodies.""" + + name = fields.String( + required=True, validate=validate.Length(min=1, max=_NAME_MAX)) + description = fields.String( + load_default='', validate=validate.Length(max=_DESCRIPTION_MAX)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class CategoryUpdateSchema(Schema): + """Validates PATCH /categories/{id} bodies; every field optional.""" + + name = fields.String(validate=validate.Length(min=1, max=_NAME_MAX)) + description = fields.String(validate=validate.Length(max=_DESCRIPTION_MAX)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class VariantCreateSchema(Schema): + """Validates POST /regression-tests/{id}/outputs/{oid}/variants bodies.""" + + # The hash is joined to the baseline's extension to form a filename under + # TestResults, so it is restricted to characters that cannot climb out of + # that directory. Content hashes are hex, so this costs nothing real. + hash = fields.String( + required=True, + validate=[ + validate.Length(min=1, max=_HASH_MAX), + validate.Regexp( + r'^[A-Za-z0-9]+$', + error='hash must be alphanumeric', + ), + ], + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/results.py b/mod_api/schemas/results.py new file mode 100644 index 000000000..fe054dc26 --- /dev/null +++ b/mod_api/schemas/results.py @@ -0,0 +1,64 @@ +"""Schemas for output file content and baseline approvals.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT + + +class OutputFileContentSchema(Schema): + """File content blob returned for expected or actual output.""" + + run_id = fields.Integer(allow_none=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + filename = fields.String(required=True) + content_type = fields.String(required=True) + encoding = fields.String( + required=True, validate=validate.OneOf(['utf-8', 'base64'])) + content = fields.String(required=True) + # sha256 always covers the complete file, even when content is truncated. + sha256 = fields.String(allow_none=True) + truncated = fields.Boolean(load_default=False) + download_url = fields.String(allow_none=True) + storage_status = fields.String( + required=True, + validate=validate.OneOf(['ok', 'degraded', 'missing']), + ) + + +class BaselineApprovalRequestSchema(Schema): + """POST /runs/{id}/samples/{sid}/baseline-approval body.""" + + regression_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + output_id = fields.Integer( + required=True, + validate=validate.Range(min=1), + ) + + remove_variants = fields.Boolean( + load_default=False, + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class BaselineApprovalSchema(Schema): + """Response after a baseline approval is applied.""" + + status = fields.String( + required=True, + validate=validate.OneOf( + ['approved'])) + run_id = fields.Integer(required=True) + sample_id = fields.Integer(required=True) + regression_id = fields.Integer(required=True) + output_id = fields.Integer(required=True) + requested_by = fields.String(required=True) + created_at = fields.DateTime(required=True, format=DATETIME_FORMAT) diff --git a/mod_api/schemas/runs.py b/mod_api/schemas/runs.py new file mode 100644 index 000000000..94dfd28dd --- /dev/null +++ b/mod_api/schemas/runs.py @@ -0,0 +1,100 @@ +"""Schemas for runs, summaries, and progress events.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT + + +class ProgressEventSchema(Schema): + """A single progress event in a run's timeline.""" + + timestamp = fields.DateTime(required=True, format=DATETIME_FORMAT) + status = fields.String(required=True) + message = fields.String(required=True) + + +class RunSchema(Schema): + """Full run details.""" + + run_id = fields.Integer(required=True) + status = fields.String(required=True, validate=validate.OneOf([ + 'queued', 'running', 'pass', 'fail', 'canceled', 'incomplete', 'error' + ])) + platform = fields.String( + required=True, validate=validate.OneOf(['linux', 'windows'])) + test_type = fields.String(validate=validate.OneOf(['commit', 'pr'])) + repository = fields.String(required=True) + branch = fields.String(allow_none=True) + commit_sha = fields.String(required=True) + built_commit_sha = fields.String(allow_none=True, dump_default=None) + pr_number = fields.Integer(allow_none=True, load_default=None) + created_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + queued_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + started_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + completed_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + github_link = fields.String(allow_none=True) + + +class RunSummarySchema(Schema): + """Pass/fail/skip aggregate counts for a run.""" + + run_id = fields.Integer(required=True) + status = fields.String(required=True) + total_samples = fields.Integer(required=True) + pass_count = fields.Integer(required=True) + fail_count = fields.Integer(required=True) + skipped_count = fields.Integer(required=True) + missing_output_count = fields.Integer(required=True) + error_count = fields.Integer(load_default=0) + duration_ms = fields.Integer(allow_none=True) + + +class RunCreateRequestSchema(Schema): + """POST /runs request body.""" + + commit_sha = fields.String( + required=True, + validate=validate.Regexp( + r'^[a-fA-F0-9]{40}$', + error='commit_sha must be a 40-character hex string.', + ), + ) + platform = fields.String( + required=True, + validate=validate.OneOf(['linux', 'windows']), + ) + branch = fields.String( + load_default='master', + validate=[ + validate.Length(max=100), + validate.Regexp( + r'^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$', + error='branch must match ^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$', + ), + ], + ) + repository = fields.String( + required=True, + validate=[ + validate.Length(max=100), + validate.Regexp( + r'^[a-zA-Z0-9_.\-]+/[a-zA-Z0-9_.\-]+$', + error='repository must match owner/repo format.', + ), + ], + ) + pull_request = fields.Integer( + load_default=None, + allow_none=True, + validate=validate.Range(min=1, max=2147483647), + ) + regression_test_ids = fields.List( + fields.Integer(validate=validate.Range(min=1, max=2147483647)), + load_default=None, + validate=validate.Length(max=500), + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/samples.py b/mod_api/schemas/samples.py new file mode 100644 index 000000000..7a2de00d7 --- /dev/null +++ b/mod_api/schemas/samples.py @@ -0,0 +1,63 @@ +"""Schemas for sample endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_api.schemas.common import DATETIME_FORMAT +from mod_upload.models import Platform + +# Read off the model so a platform the database accepts can never be +# rejected here, the way the regression test schemas take their type lists. +_PLATFORMS = sorted(Platform.values()) + +_TAG_NAME_MAX = 64 +_TAG_DESCRIPTION_MAX = 1024 + + +class SampleHistoryEntrySchema(Schema): + """One row in a sample's cross-run history.""" + + run_id = fields.Integer(required=True) + regression_test_id = fields.Integer(required=True) + status = fields.String(required=True) + platform = fields.String(required=True) + branch = fields.String(required=True) + commit_sha = fields.String(required=True) + tested_at = fields.DateTime(allow_none=True, format=DATETIME_FORMAT) + failure_signature = fields.String(allow_none=True) + + +class TagCreateSchema(Schema): + """Validates POST /tags bodies.""" + + name = fields.String( + required=True, validate=validate.Length(min=1, max=_TAG_NAME_MAX)) + description = fields.String( + load_default='', validate=validate.Length(max=_TAG_DESCRIPTION_MAX)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class SampleUpdateSchema(Schema): + """ + Validates PATCH /samples/{id} bodies; every field optional. + + Tags are given by name rather than id so a caller can work from the + names the sample payload already shows, instead of first fetching the + tag list to translate them. + """ + + tags = fields.List( + fields.String(validate=validate.Length(min=1, max=_TAG_NAME_MAX)), + validate=validate.Length(max=50)) + notes = fields.String(validate=validate.Length(max=1024)) + parameters = fields.String(validate=validate.Length(max=1024)) + platform = fields.String(validate=validate.OneOf(_PLATFORMS)) + version = fields.String(validate=validate.Length(min=1, max=10)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/system.py b/mod_api/schemas/system.py new file mode 100644 index 000000000..e4c3bd84e --- /dev/null +++ b/mod_api/schemas/system.py @@ -0,0 +1,50 @@ +"""Request schemas for the platform configuration endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + + +class MaintenanceUpdateSchema(Schema): + """Validates PATCH /system/maintenance/{platform} bodies.""" + + disabled = fields.Boolean(required=True) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class BlockedUserCreateSchema(Schema): + """Validates POST /system/blocked-users bodies.""" + + # The numeric GitHub account id, not the login: logins can be changed and + # reused, which would silently unblock somebody. + user_id = fields.Integer(required=True, validate=validate.Range(min=1)) + comment = fields.String(load_default='', validate=validate.Length(max=1024)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class ForbiddenExtensionCreateSchema(Schema): + """Validates POST /system/forbidden-extensions bodies.""" + + # Stored without the leading dot, matching how upload validation looks it + # up. Letters and digits only, so a pattern can never be smuggled in. + extension = fields.String( + required=True, + validate=[ + validate.Length(min=1, max=32), + validate.Regexp( + r'^[A-Za-z0-9]+$', + error='extension must be alphanumeric, without a leading dot', + ), + ], + ) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/schemas/uploads.py b/mod_api/schemas/uploads.py new file mode 100644 index 000000000..5de9a311b --- /dev/null +++ b/mod_api/schemas/uploads.py @@ -0,0 +1,40 @@ +"""Request schemas for the upload queue endpoints.""" + +from marshmallow import RAISE, Schema, fields, validate + +from mod_upload.models import Platform + +# Read off the model so a platform the database accepts is never rejected +# here, the way the sample and regression test schemas do it. +_PLATFORMS = sorted(Platform.values()) + + +class UploadFinalizeSchema(Schema): + """Validates POST /queued-samples/{id}/finalize bodies.""" + + # The version string rather than its id: a client works from what + # /system/about and the sample payloads already show it. + version = fields.String( + required=True, validate=validate.Length(min=1, max=10)) + platform = fields.String( + required=True, validate=validate.OneOf(_PLATFORMS)) + parameters = fields.String( + load_default='', validate=validate.Length(max=1024)) + notes = fields.String( + load_default='', validate=validate.Length(max=1024)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE + + +class QueueLinkSchema(Schema): + """Validates POST /queued-samples/{id}/link bodies.""" + + sample_id = fields.Integer(required=True, validate=validate.Range(min=1)) + + class Meta: + """Reject unknown fields.""" + + unknown = RAISE diff --git a/mod_api/services/__init__.py b/mod_api/services/__init__.py new file mode 100644 index 000000000..04182e587 --- /dev/null +++ b/mod_api/services/__init__.py @@ -0,0 +1 @@ +"""mod_api.services - Core business logic for the API.""" diff --git a/mod_api/services/diff_service.py b/mod_api/services/diff_service.py new file mode 100644 index 000000000..f527c57ba --- /dev/null +++ b/mod_api/services/diff_service.py @@ -0,0 +1,220 @@ +""" +Structured diff computation between expected and actual output files. + +Produces JSON hunks with line-level detail instead of the legacy HTML +diff output. Uses difflib.unified_diff internally. +""" + +import difflib +import hashlib +import os +import re +from typing import Any, Dict, List, Optional, Tuple + +from mod_api.services.storage import get_test_results_base_path + + +def compute_diff( + expected_path: str, + actual_path: str, + context_lines: int = 3, + max_hunks: int = 500, +) -> Dict[str, Any]: + """ + Compute a structured diff between two files. + + Returns a dict matching the Diff schema: status, summary (added_lines, + removed_lines, changed_hunks), and a list of hunks. + """ + context_lines = max(1, min(context_lines, 50)) + + if not os.path.isfile(expected_path): + return { + 'status': 'missing_expected', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + if not os.path.isfile(actual_path): + return { + 'status': 'missing_actual', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + expected_lines = read_lines(expected_path) + actual_lines = read_lines(actual_path) + + if expected_lines == actual_lines: + return { + 'status': 'identical', + 'summary': {'added_lines': 0, 'removed_lines': 0, 'changed_hunks': 0}, + 'hunks': [], + } + + hunks = _compute_hunks(expected_lines, actual_lines, + context_lines, max_hunks) + added = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'added') + removed = sum( + 1 for h in hunks for line in h['lines'] if line['kind'] == 'removed') + + return { + 'status': 'different', + 'summary': { + 'added_lines': added, + 'removed_lines': removed, + 'changed_hunks': len(hunks), + }, + 'hunks': hunks, + } + + +# Matches the @@ -a,b +c,d @@ header line from unified_diff. +_HUNK_RE = re.compile(r'^@@ -(\d+)(?:,\d+)? \+(\d+)(?:,\d+)? @@') + + +def _process_diff_line(line, current_hunk, expected_line_num, actual_line_num): + if line.startswith('+'): + current_hunk['lines'].append({ + 'kind': 'added', + 'expected_line': None, + 'actual_line': actual_line_num, + 'text': line[1:], + }) + actual_line_num += 1 + elif line.startswith('-'): + current_hunk['lines'].append({ + 'kind': 'removed', + 'expected_line': expected_line_num, + 'actual_line': None, + 'text': line[1:], + }) + expected_line_num += 1 + else: + content = line[1:] if line.startswith(' ') else line + current_hunk['lines'].append({ + 'kind': 'context', + 'expected_line': expected_line_num, + 'actual_line': actual_line_num, + 'text': content, + }) + expected_line_num += 1 + actual_line_num += 1 + return expected_line_num, actual_line_num + + +def _process_hunk_header( + line: str, + current_hunk: Optional[Dict[str, Any]], + hunks: List[Dict[str, Any]], + max_hunks: int +) -> Tuple[Optional[Dict[str, Any]], int, int, bool]: + if current_hunk and len(hunks) >= max_hunks: + return None, 0, 0, True + if current_hunk: + hunks.append(current_hunk) + + m = _HUNK_RE.match(line) + if m: + expected_line_num = int(m.group(1)) + actual_line_num = int(m.group(2)) + else: + expected_line_num = 0 + actual_line_num = 0 + + new_hunk = { + 'expected_start': expected_line_num, + 'actual_start': actual_line_num, + 'lines': [], + } + return new_hunk, expected_line_num, actual_line_num, False + + +def _compute_hunks( + expected_lines: List[str], + actual_lines: List[str], + context_lines: int, + max_hunks: int, +) -> List[Dict[str, Any]]: + """Parse unified_diff output into structured hunk dicts.""" + differ = difflib.unified_diff( + expected_lines, + actual_lines, + lineterm='', + n=context_lines, + ) + + hunks: List[Dict[str, Any]] = [] + current_hunk: Optional[Dict[str, Any]] = None + expected_line_num = 0 + actual_line_num = 0 + + for line in differ: + if line.startswith(('---', '+++')): + continue + + if line.startswith('@@'): + current_hunk, expected_line_num, actual_line_num, stop = _process_hunk_header( + line, current_hunk, hunks, max_hunks + ) + if stop: + break + continue + + if current_hunk is None: + continue + + expected_line_num, actual_line_num = _process_diff_line( + line, current_hunk, expected_line_num, actual_line_num) + + if current_hunk: + hunks.append(current_hunk) + + return hunks[:max_hunks] + + +def _enforce_safe_path(file_path: str) -> bool: + base = os.path.realpath(get_test_results_base_path()) + target = os.path.realpath(file_path) + return target.startswith(base + os.sep) or target == base + + +def read_lines(file_path: str, max_size_bytes: int = 10 * 1024 * 1024) -> List[str]: + """Read file lines with a cp1252 fallback, matching legacy behavior. + + Parameters + ---------- + file_path : str + Absolute path to the file to read. + max_size_bytes : int + Maximum file size in bytes. Raises ValueError if exceeded. + """ + if not _enforce_safe_path(file_path): + raise ValueError("Unsafe file path") + file_size = os.path.getsize(file_path) + if file_size > max_size_bytes: + raise ValueError( + f"File too large ({file_size} bytes > {max_size_bytes} limit)") + try: + with open(file_path, encoding='utf8') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + except UnicodeDecodeError: + # errors='replace' because cp1252 leaves five byte values undefined — + # without it the fallback can itself raise on binary output files. + with open(file_path, encoding='cp1252', errors='replace') as f: + return [line.rstrip('\n\r') for line in f.readlines()] + + +def file_sha256(file_path: str) -> Optional[str]: + """Compute SHA-256 of a file. Returns None if the file can't be read.""" + if not _enforce_safe_path(file_path): + return None + try: + sha = hashlib.sha256() + with open(file_path, 'rb') as f: + for block in iter(lambda: f.read(8192), b''): + sha.update(block) + return sha.hexdigest() + except (OSError, IOError): + return None diff --git a/mod_api/services/error_service.py b/mod_api/services/error_service.py new file mode 100644 index 000000000..f8bcd85b1 --- /dev/null +++ b/mod_api/services/error_service.py @@ -0,0 +1,307 @@ +""" +Error derivation from TestResult and TestResultFile rows. + +Walks result data and produces structured ErrorItem dicts. There's no +dedicated error table — errors are inferred from: + exit_code_mismatch → exit code != expected + diff_mismatch → got != null and not in multiple correct files + missing_output → dummy (-1,-1,-1,'','error') row present +""" + +import logging +from collections import defaultdict +from typing import Any, Dict, List + +from sqlalchemy.orm import joinedload + +from mod_api.services.status import is_dummy_row +from mod_regression.models import RegressionTestOutput +from mod_test.models import (TestProgress, TestResult, TestResultFile, + TestStatus) + +_SEVERITY_ORDER = ('info', 'warning', 'error', 'critical') + + +def _is_output_acceptable(rf: TestResultFile) -> bool: + if not rf.regression_test_output: + return False + for multi in rf.regression_test_output.multiple_files: + if multi.file_hashes == rf.got: + return True + return False + + +def _check_exit_code_errors(result, test_id, occurred_at): + if result.exit_code != result.expected_rc: + return [{ + 'error_id': f'err_{test_id}_{result.regression_test_id}_rc', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'exit_code_mismatch', + 'severity': 'error', + 'message': ( + f'Exit code {result.exit_code} != expected {result.expected_rc} ' + f'for regression test {result.regression_test_id}' + ), + 'occurred_at': occurred_at, + }] + return [] + + +def _check_missing_output_errors(result, result_files, test_id, occurred_at, expected_outputs): + errors = [] + actual_output_ids = {rf.regression_test_output_id for rf in result_files} + if expected_outputs is not None: + for rto in expected_outputs: + if not rto.ignore and rto.id not in actual_output_ids: + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_missing_{rto.id}', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'missing_output', + 'severity': 'error', + 'message': ( + f'Regression test {result.regression_test_id} ' + f'produced no output for expected file {rto.id}' + ), + 'occurred_at': occurred_at, + }) + else: + for rf in result_files: + if is_dummy_row(rf): + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_missing', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'missing_output', + 'severity': 'error', + 'message': ( + f'Regression test {result.regression_test_id} ' + f'produced no output when output was expected' + ), + 'occurred_at': occurred_at, + }) + return errors + + +def _check_diff_mismatch_errors(result, result_files, test_id, occurred_at): + errors = [] + for rf in result_files: + if is_dummy_row(rf): + continue + if rf.got is not None and not _is_output_acceptable(rf): + errors.append({ + 'error_id': f'err_{test_id}_{result.regression_test_id}_{rf.regression_test_output_id}', + 'run_id': test_id, + 'sample_id': _get_sample_id(result), + 'regression_id': result.regression_test_id, + 'type': 'diff_mismatch', + 'severity': 'warning', + 'message': ( + f'Output differs from expected for regression test ' + f'{result.regression_test_id}, output {rf.regression_test_output_id}' + ), + 'occurred_at': occurred_at, + }) + return errors + + +def _evaluate_test_result( + result, + result_files, + test_id, + occurred_at, + expected_outputs=None): + errors = [] + errors.extend(_check_exit_code_errors(result, test_id, occurred_at)) + errors.extend(_check_missing_output_errors(result, result_files, test_id, occurred_at, expected_outputs)) + errors.extend(_check_diff_mismatch_errors(result, result_files, test_id, occurred_at)) + return errors + + +def _group_result_files(test_id, results, preloaded_files=None): + """Map regression_test_id -> [TestResultFile], loading if not preloaded.""" + if preloaded_files is not None: + all_files = preloaded_files + elif results: + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter_by(test_id=test_id).all() + else: + all_files = [] + files_by_result = defaultdict(list) + for f in all_files: + files_by_result[f.regression_test_id].append(f) + return files_by_result + + +def _load_expected_outputs(results): + """Map regression_test_id -> [RegressionTestOutput] for the given results. + + Missing-output detection must use the same RegressionTestOutput + comparison as /runs/{id}/summary — /errors and /summary have to agree. + id > 0 excludes the -1 sentinel some fixtures create to satisfy the + dummy TestResultFile row's foreign key; it is not a real expectation. + """ + if not results: + return {} + rt_ids = {r.regression_test_id for r in results} + expected_by_rt = defaultdict(list) + for rto in RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(rt_ids), + RegressionTestOutput.id > 0).all(): + expected_by_rt[rto.regression_id].append(rto) + return expected_by_rt + + +def derive_errors_for_run(test_id: int, + expected_outputs_by_rt: Dict[int, + List[Any]] = None, + preloaded_results=None, + preloaded_files=None) -> List[Dict[str, + Any]]: + """Walk result rows and emit one ErrorItem per detected failure.""" + progress = TestProgress.query.filter_by(test_id=test_id).order_by( + TestProgress.timestamp.desc()).first() + occurred_at = progress.timestamp.isoformat( + ) if progress and progress.timestamp else None + + if preloaded_results is not None: + results = preloaded_results + else: + results = TestResult.query.filter_by(test_id=test_id).all() + + files_by_result = _group_result_files(test_id, results, preloaded_files) + + if expected_outputs_by_rt is None: + expected_outputs_by_rt = _load_expected_outputs(results) + + errors = [] + for result in results: + result_files = files_by_result.get(result.regression_test_id, []) + expected_outputs = expected_outputs_by_rt.get( + result.regression_test_id) if expected_outputs_by_rt else None + errors.extend(_evaluate_test_result( + result, result_files, test_id, occurred_at, expected_outputs)) + + return errors + + +def _aggregate_error_into_bucket(err, bucket): + bucket['count'] += 1 + + # Escalate severity to the worst we've seen. + try: + curr_idx = _SEVERITY_ORDER.index(bucket['severity']) + new_idx = _SEVERITY_ORDER.index(err['severity']) + if new_idx > curr_idx: + bucket['severity'] = err['severity'] + except ValueError: + # Fallback if unknown severity + if err['severity'] == 'error': + bucket['severity'] = 'error' + + err_time = err.get('occurred_at') + if err_time: + if bucket['first_seen_at'] is None or err_time < bucket['first_seen_at']: + bucket['first_seen_at'] = err_time + if bucket['last_seen_at'] is None or err_time > bucket['last_seen_at']: + bucket['last_seen_at'] = err_time + + sid = err.get('sample_id') + if sid and sid not in bucket['sample_ids'] and len( + bucket['sample_ids']) < 1000: + bucket['sample_ids'].append(sid) + + +def derive_error_summary( + test_id: int, group_by: str = 'type') -> List[Dict[str, Any]]: + """Group errors by the given key and return bucket counts.""" + errors = derive_errors_for_run(test_id) + buckets: Dict[str, Dict[str, Any]] = {} + + for err in errors: + key = str(err.get(group_by, 'unknown')) + + if key not in buckets: + buckets[key] = { + 'key': key, + 'group_by': group_by, + 'count': 0, + 'severity': err['severity'], + 'sample_ids': [], + 'first_seen_at': None, + 'last_seen_at': None, + } + + _aggregate_error_into_bucket(err, buckets[key]) + + return list(buckets.values()) + + +def derive_infrastructure_errors(test_id: int) -> List[Dict[str, Any]]: + """ + Best-effort infra error extraction from TestProgress messages. + + There's no structured error protocol from the CI worker yet, so we + do keyword matching against progress messages to guess the failure type. + """ + errors = [] + progress_rows = TestProgress.query.filter_by( + test_id=test_id, + status=TestStatus.canceled, + ).all() + + for p in progress_rows: + message = p.message or '' + # User-initiated cancellations (cancel_run writes "... via API") are not + # infrastructure failures, so they must not be reported here. + if 'via API' in message: + continue + error_type = _classify_infra_error(message.lower()) + errors.append({ + 'error_id': f'infra_{test_id}_{p.id}', + 'run_id': test_id, + 'sample_id': None, + 'regression_id': None, + 'type': error_type, + 'severity': 'critical', + 'message': p.message or 'Unknown infrastructure error', + 'location': None, + 'occurred_at': p.timestamp.isoformat() if p.timestamp else None, + }) + + return errors + + +def _classify_infra_error(message_lower: str) -> str: + """Guess the infra error type from progress message keywords.""" + if any(w in message_lower for w in ['provisioning', 'vm ', 'instance']): + return 'vm_provisioning' + if any(w in message_lower for w in ['checkout', 'git clone', 'fetch']): + return 'checkout' + if any(w in message_lower for w in ['merge', 'conflict']): + return 'merge' + if any(w in message_lower for w in ['build', 'compile', 'make']): + return 'build' + if any(w in message_lower for w in ['worker', 'timeout', 'connection']): + return 'worker' + if any(w in message_lower for w in ['storage', 'disk', 'gcs']): + return 'storage' + return 'worker' + + +def _get_sample_id(result: TestResult): + """Pull sample_id through the RegressionTest relationship, if available.""" + try: + if result.regression_test and result.regression_test.sample_id: + return result.regression_test.sample_id + except Exception: + logging.getLogger(__name__).exception( + f"Failed to fetch sample_id for TestResult {result.test_id}_{result.regression_test_id}" + ) + return None diff --git a/mod_api/services/log_service.py b/mod_api/services/log_service.py new file mode 100644 index 000000000..d555d9d68 --- /dev/null +++ b/mod_api/services/log_service.py @@ -0,0 +1,123 @@ +""" +Build log reader with cursor-based pagination. + +Log files live at SAMPLE_REPOSITORY/LogFiles/{run_id}.txt. The cursor +is just a line number offset into the file. +""" + +from typing import Any, Dict, List, Optional, Tuple + +from mod_api.services.storage import get_log_file_path + + +def _parse_cursor(cursor: Optional[int]) -> int: + if not cursor: + return 0 + try: + return int(cursor) + except (ValueError, TypeError): + return 0 + + +def _format_log_line(raw: str, run_id: int) -> Dict[str, Any]: + return { + 'timestamp': None, + 'level': _extract_level(raw), + 'source': _extract_source(raw), + 'message': raw, + 'run_id': run_id, + 'sample_id': None, + } + + +def _should_include_line(raw: str, level: Optional[str], source: Optional[str], contains: Optional[str]) -> bool: + if level and not _matches_level(raw, level): + return False + if source and _extract_source(raw) != source: + return False + if contains and contains.lower() not in raw.lower(): + return False + return True + + +def read_log_lines( + run_id: int, + cursor: Optional[str] = None, + limit: int = 100, + level: Optional[str] = None, + source: Optional[str] = None, + contains: Optional[str] = None, +) -> Tuple[List[Dict[str, Any]], Optional[str]]: + """ + Read and optionally filter lines from a run's build log. + + Returns (lines, next_cursor). Raises FileNotFoundError when the + log file isn't on disk. + """ + log_path = get_log_file_path(run_id) + if log_path is None: + raise FileNotFoundError(f'Log file not found for run {run_id}') + + limit = max(1, min(limit, 500)) + + start_line = _parse_cursor(cursor) + + import itertools + + def _read_lines(encoding, errors='strict'): + with open(log_path, encoding=encoding, errors=errors) as f: + iterator = itertools.islice(f, start_line, None) + + result_lines = [] + line_num = start_line + + for raw_line in iterator: + raw = raw_line.rstrip('\n\r') + line_num += 1 + + if not _should_include_line(raw, level, source, contains): + continue + + result_lines.append(_format_log_line(raw, run_id)) + + if len(result_lines) >= limit: + break + + try: + next(iterator) + has_more = True + except StopIteration: + has_more = False + + next_cursor = str(line_num) if has_more else None + return result_lines, next_cursor + + try: + return _read_lines('utf-8') + except UnicodeDecodeError: + # cp1252 leaves five byte values undefined, so the fallback itself + # can raise; replace those rather than turning a log read into a 500. + return _read_lines('cp1252', errors='replace') + + +def _matches_level(line: str, target_level: str) -> bool: + """Check if a log line matches the requested severity (case-insensitive).""" + return _extract_level(line) == (target_level or '').lower() + + +def _extract_level(line: str) -> str: + """Best-effort log level extraction from raw text.""" + line_upper = line.upper() + for lvl in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']: + if lvl in line_upper: + return lvl.lower() + return 'info' + + +def _extract_source(line: str) -> str: + """Best-effort source component extraction from raw text.""" + line_lower = line.lower() + for src in ['orchestrator', 'worker', 'build', 'test_runner', 'web']: + if src in line_lower: + return src + return 'web' diff --git a/mod_api/services/status.py b/mod_api/services/status.py new file mode 100644 index 000000000..01948e93a --- /dev/null +++ b/mod_api/services/status.py @@ -0,0 +1,303 @@ +""" +Status derivation from the raw data model. + +Normalizes TestProgress/TestResult/TestResultFile states into clean +strings for the API layer. This is the single source of truth for +status logic — route handlers must not inline their own derivation. + +Run statuses: queued, running, pass, fail, canceled, error, incomplete +Sample statuses: pass, fail, skipped, missing_output, running, not_started + +Things to watch out for: + - test.failed only checks for TestStatus.canceled — never use it + for determining whether regression tests actually passed + - TestResultFile.got = null means MATCH, not missing output + - Dummy row (-1,-1,-1,'','error') = test produced no output at all + - TestStatus.canceled covers both user cancels and infra failures +""" + +from collections import defaultdict +from typing import List, Optional, Set + +from sqlalchemy.orm import joinedload + +from mod_regression.models import RegressionTest, RegressionTestOutput +from mod_test.models import (Test, TestProgress, TestResult, TestResultFile, + TestStatus) + + +def expected_regression_ids(test: Test) -> List[int]: + """Regression test IDs this run was configured to execute. + + Uses the customized selection when present; otherwise every ACTIVE + regression test — same rule as create_run / run summary totals. + """ + if test.customized_tests: + return [ct.regression_id for ct in test.customized_tests] + return [rt.id for rt in + RegressionTest.query.filter_by(active=True).all()] + + +def derive_run_status(test: Test) -> str: + """ + Map the raw model state to one of the 7 normalized run statuses. + + Looks at the most recent TestProgress row and, for completed runs, + counts actual failures from TestResult rows. + """ + statuses, _ = batch_get_run_data([test]) + return statuses.get(test.id, 'queued') + + +def _check_output_acceptable(rf: TestResultFile) -> bool: + if rf.regression_test_output: + for multi in rf.regression_test_output.multiple_files: + if multi.file_hashes == rf.got: + return True + return False + + +def _has_missing_output(result_files: List[TestResultFile], expected_outputs: Optional[List] = None) -> bool: + if expected_outputs is not None: + actual_output_ids = {rf.regression_test_output_id for rf in result_files} + for rto in expected_outputs: + if not rto.ignore and rto.id not in actual_output_ids: + return True + return False + else: + for rf in result_files: + if is_dummy_row(rf): + return True + return False + + +def derive_sample_status( + test_result: Optional[TestResult], + result_files: List[TestResultFile], + expected_outputs: Optional[List] = None, +) -> str: + """Map a TestResult + its output files to a per-sample status string. + + Checks for missing output first (expected outputs with no matching + TestResultFile), then exit code, then output diffs against accepted + baselines. + + Parameters + ---------- + test_result : Optional[TestResult] + The TestResult row, or None if the test hasn't run. + result_files : List[TestResultFile] + Actual output file rows from the database. + expected_outputs : Optional[List] + RegressionTestOutput rows that define what outputs were expected. + When provided, missing-output detection compares these against + result_files. When None, legacy dummy-row detection is used as + a fallback. + """ + if test_result is None: + return 'not_started' + + if _has_missing_output(result_files, expected_outputs): + return 'missing_output' + + if test_result.exit_code != test_result.expected_rc: + return 'fail' + + if any(rf.got is not None and not _check_output_acceptable(rf) + for rf in result_files): + return 'fail' + + return 'pass' + + +def is_dummy_row(rf: TestResultFile) -> bool: + """ + Detect the sentinel TestResultFile row where regression_test_output_id == -1 and got == 'error'. + + This row means the test produced no output when output was expected. + The old test_id == -1 and regression_test_id == -1 checks were removed + because they are no longer populated as -1 in newer data. + (Verified against production DB on 2026-06-25 by a maintainer: + 0 legacy rows exist.) + It should never show up as a real file in API responses. + """ + return bool(rf.regression_test_output_id == -1 and rf.got == 'error') + + +def derive_output_status(rf: TestResultFile) -> str: + """Classify a single output file: pass, fail, or missing_output.""" + if is_dummy_row(rf): + return 'missing_output' + if rf.got is None: + return 'pass' + return 'fail' + + +def get_run_timestamps(test: Test) -> dict: + """ + Build a timestamp dict from TestProgress rows. + + Test doesn't have a created_at column, so we use the earliest + progress entry as a proxy. + """ + _, timestamps = batch_get_run_data([test]) + ts = timestamps.get(test.id, {}) + return { + 'created_at': ts.get('created_at'), + 'queued_at': ts.get('queued_at'), + 'started_at': ts.get('started_at'), + 'completed_at': ts.get('completed_at'), + } + + +def _compute_run_timestamps(t_prog): + ts = { + 'created_at': None, + 'queued_at': None, + 'started_at': None, + 'completed_at': None, + } + if t_prog: + ts['queued_at'] = t_prog[0].timestamp + ts['created_at'] = t_prog[0].timestamp + for p in t_prog: + if p.status == TestStatus.testing and ts['started_at'] is None: + ts['started_at'] = p.timestamp + if p.status in (TestStatus.completed, TestStatus.canceled): + ts['completed_at'] = p.timestamp + return ts + + +def _check_completed_run_status( + t_id, + results_by_test, + files_by_test_and_rt, + expected_outputs_by_rt, + expected_rt_ids: Optional[Set[int]] = None): + results = results_by_test.get(t_id, []) + if not results: + # A run marked completed that produced zero TestResult rows is not + # a pass — the worker finished without reporting anything. + return 'error' + # Samples with no TestResult row are invisible to the loop below. If the + # run was configured for more samples than it reported, treat it as an + # error (same class as zero results) so a green fragment cannot pass. + if expected_rt_ids is not None: + reported_ids = {r.regression_test_id for r in results} + if not expected_rt_ids.issubset(reported_ids): + return 'error' + for r in results: + r_files = files_by_test_and_rt.get((t_id, r.regression_test_id), []) + expected = expected_outputs_by_rt.get( + r.regression_test_id) if expected_outputs_by_rt is not None else None + sample_status = derive_sample_status(r, r_files, expected) + if sample_status not in ('pass', 'not_started'): + return 'fail' + return 'pass' + + +def _compute_run_status( + t_prog, + results_by_test, + files_by_test_and_rt, + t_id, + expected_outputs_by_rt=None, + expected_rt_ids: Optional[Set[int]] = None): + if not t_prog: + return 'queued' + + raw_status = t_prog[-1].status + + if raw_status in (TestStatus.preparation, TestStatus.testing): + return 'running' + if raw_status == TestStatus.canceled: + return 'canceled' + if raw_status == TestStatus.completed: + return _check_completed_run_status( + t_id, + results_by_test, + files_by_test_and_rt, + expected_outputs_by_rt, + expected_rt_ids=expected_rt_ids) + return 'incomplete' + + +def batch_get_run_data(tests: list) -> tuple: + """ + Batch compute derive_run_status and get_run_timestamps for a list of tests. + + Returns (statuses_dict, timestamps_dict) + """ + if not tests: + return {}, {} + + test_ids = [t.id for t in tests] + + # Preload TestProgress + all_progress = TestProgress.query.filter(TestProgress.test_id.in_( + test_ids)).order_by(TestProgress.id.asc()).all() + progress_by_test = {tid: [] for tid in test_ids} + for p in all_progress: + progress_by_test[p.test_id].append(p) + + # Preload TestResult + all_results = TestResult.query.filter( + TestResult.test_id.in_(test_ids)).all() + results_by_test = {tid: [] for tid in test_ids} + for r in all_results: + results_by_test[r.test_id].append(r) + + # Preload TestResultFile + + all_files = TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output) + .joinedload(RegressionTestOutput.multiple_files) + ).filter(TestResultFile.test_id.in_(test_ids)).all() + files_by_test_and_rt = {} + for f in all_files: + key = (f.test_id, f.regression_test_id) + if key not in files_by_test_and_rt: + files_by_test_and_rt[key] = [] + files_by_test_and_rt[key].append(f) + + # Preload expected outputs (RegressionTestOutput) for missing-output + # detection + all_rt_ids = set() + for tid in test_ids: + for r in results_by_test.get(tid, []): + all_rt_ids.add(r.regression_test_id) + + expected_outputs_by_rt = {} + if all_rt_ids: + all_expected = RegressionTestOutput.query.filter( + RegressionTestOutput.regression_id.in_(all_rt_ids) + ).all() + expected_outputs_by_rt = defaultdict(list) + for rto in all_expected: + expected_outputs_by_rt[rto.regression_id].append(rto) + + # Expected sample set per run (customized selection or all active RTs) + active_rt_ids = { + rt.id for rt in RegressionTest.query.filter_by(active=True).all() + } + expected_rt_ids_by_test = {} + for t in tests: + if t.customized_tests: + expected_rt_ids_by_test[t.id] = { + ct.regression_id for ct in t.customized_tests + } + else: + expected_rt_ids_by_test[t.id] = active_rt_ids + + statuses = {} + timestamps_dict = {} + + for t in tests: + t_prog = progress_by_test[t.id] + timestamps_dict[t.id] = _compute_run_timestamps(t_prog) + statuses[t.id] = _compute_run_status( + t_prog, results_by_test, files_by_test_and_rt, t.id, + expected_outputs_by_rt=expected_outputs_by_rt, + expected_rt_ids=expected_rt_ids_by_test[t.id]) + + return statuses, timestamps_dict diff --git a/mod_api/services/storage.py b/mod_api/services/storage.py new file mode 100644 index 000000000..f9d56533e --- /dev/null +++ b/mod_api/services/storage.py @@ -0,0 +1,77 @@ +""" +Storage helpers for resolving artifact locations. + +Artifacts can live in local SAMPLE_REPOSITORY, GCS, or both. When both +exist, GCS is preferred and a signed URL is returned. When only local +exists, storage_status is 'degraded'. When neither exists, it's 'missing'. +""" + +import logging +import os +from datetime import timedelta +from typing import Optional, Tuple + +logger = logging.getLogger(__name__) + + +def resolve_artifact(relative_path: str) -> Tuple[Optional[str], str]: + """ + Look for an artifact in local storage and GCS. + + Returns (download_url_or_None, storage_status). + """ + from run import config, storage_client_bucket + + sample_repo = config.get('SAMPLE_REPOSITORY', '') + local_path = os.path.join(sample_repo, relative_path) + # Prevent path traversal: resolved path must stay within sample_repo + real_base = os.path.realpath(sample_repo) + real_path = os.path.realpath(local_path) + if not (real_path.startswith(real_base + os.sep) or real_path == real_base): + return None, 'missing' + local_exists = os.path.isfile(local_path) + + gcs_url = None + if storage_client_bucket: + try: + blob = storage_client_bucket.blob(relative_path) + if blob.exists(): + gcs_url = blob.generate_signed_url( + version='v4', + # int() guards against a string value in config, which + # would otherwise raise inside timedelta and be swallowed + # by the except below as a silent 'degraded'. + expiration=timedelta(minutes=int(config.get( + 'GCS_SIGNED_URL_EXPIRY_LIMIT', 60))), + method='GET', + ) + except Exception as e: + logger.warning(f"Failed to generate GCS signed URL for {relative_path}: {e}") + gcs_url = None + + if local_exists and gcs_url: + return gcs_url, 'ok' + elif gcs_url: + return gcs_url, 'degraded' + elif local_exists: + return None, 'degraded' + else: + return None, 'missing' + + +def get_log_file_path(run_id: int) -> Optional[str]: + """Return the absolute path to a run's build log, or None if it doesn't exist.""" + from run import config + + sample_repo = config.get('SAMPLE_REPOSITORY', '') + log_path = os.path.join(sample_repo, 'LogFiles', f'{run_id}.txt') + + if os.path.isfile(log_path): + return log_path + return None + + +def get_test_results_base_path() -> str: + """Return the base directory where TestResults files are stored.""" + from run import config + return os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestResults') diff --git a/mod_api/utils.py b/mod_api/utils.py new file mode 100644 index 000000000..8daae4d28 --- /dev/null +++ b/mod_api/utils.py @@ -0,0 +1,95 @@ +"""Pagination, serialization, and response formatting helpers.""" + +from flask import jsonify + + +def paginated_response(data, total, limit, offset, schema=None, truncated=False, extra_meta=None): + """Build an offset-paginated JSON response.""" + if schema: + serialized = schema.dump(data, many=True) + else: + serialized = data + + next_offset = offset + limit if (offset + limit) < total else None + + pagination = { + 'limit': limit, + 'offset': offset, + 'total': total, + 'next_offset': next_offset, + } + + if truncated: + pagination['truncated'] = True + + response = { + 'data': serialized, + 'pagination': pagination, + 'meta': {} + } + if extra_meta: + response['meta'].update(extra_meta) + + return jsonify(response) + + +def cursor_paginated_response(data, next_cursor, limit, schema=None): + """Build a cursor-paginated JSON response.""" + if schema: + serialized = schema.dump(data, many=True) + else: + serialized = data + + return jsonify({ + 'data': serialized, + 'pagination': { + 'limit': limit, + 'next_cursor': next_cursor, + }, + }) + + +def single_response(data, schema=None, http_status=200): + """Build a single-item JSON response.""" + if schema: + serialized = schema.dump(data) + else: + serialized = data + + response = jsonify(serialized) + response.status_code = http_status + return response + + +def get_sort_column(sort_param, column_map): + """Translate a sort string into an SQLAlchemy order_by clause. + + Handles descending sorts prefixed with '-' (e.g. '-created_at'). + """ + descending = sort_param.startswith('-') + # [1:] not lstrip('-'): lstrip would also swallow a second leading dash, + # silently normalizing bad input like '--created_at'. + field_name = sort_param[1:] if descending else sort_param + + column = column_map.get(field_name) + if column is None: + return None + + if descending: + return column.desc() + return column.asc() + + +def safe_resolve(base_path, filename): + """ + Resolve filename under base_path, rejecting path traversal. + + Returns the absolute path if it's safely within base_path, + or None if traversal was detected. + """ + import os + resolved = os.path.realpath(os.path.join(base_path, filename)) + base_real = os.path.realpath(base_path) + if not resolved.startswith(base_real + os.sep) and resolved != base_real: + return None + return resolved diff --git a/mod_auth/controllers.py b/mod_auth/controllers.py index 8337ab40a..a1f773774 100755 --- a/mod_auth/controllers.py +++ b/mod_auth/controllers.py @@ -11,6 +11,7 @@ from flask import (Blueprint, abort, flash, g, redirect, request, session, url_for) from pyisemail import is_email +from werkzeug.routing import BuildError from werkzeug.wrappers.response import Response from database import EnumSymbol @@ -66,7 +67,7 @@ def decorated_function(*args, **kwargs): return decorated_function -def check_access_rights(roles: List[Tuple[str, str]] = None, parent_route: None = None) -> Callable: +def check_access_rights(roles: Optional[List[Tuple[str, str]]] = None, parent_route: None = None) -> Callable: """ Decorate the function to check if a user can access the page. @@ -134,7 +135,7 @@ def github_token_validity(token: str): """ from run import config github_client_id = config.get('GITHUB_CLIENT_ID', '') - github_client_secret = config.get('GITHUB_CLIENT_SECRET', '') + github_client_secret = config.get('GITHUB_CLIENT_KEY', '') url = f'https://api.github.com/applications/{github_client_id}/token' session = requests.Session() session.auth = (github_client_id, github_client_secret) @@ -155,37 +156,39 @@ def github_redirect(): github_client_id = config.get('GITHUB_CLIENT_ID', '') github_token = g.user.github_token if github_token is not None: - validity = github_token_validity(github_token) - if validity is False: - g.user.github_token = None - g.db.commit() - else: - g.log.error('Failed to validate GitHub token') + if github_token_validity(github_token): return None + g.log.error(f'Invalid GitHub token found for user id: {g.user.id}') + g.user.github_token = None + g.db.commit() return f'https://github.com/login/oauth/authorize?client_id={github_client_id}&scope=public_repo' -def fetch_username_from_token() -> Any: +def fetch_username_from_token(user=None) -> Any: """ Get username from the GitHub token. + :param user: Optional user model to prevent redundant queries :return: username :rtype: str """ - import json - user = User.query.filter(User.id == g.user.id).first() + if user is None: + user = User.query.filter(User.id == g.user.id).first() + if user.github_token is None: return None url = 'https://api.github.com/user' session = requests.Session() session.auth = (user.email, user.github_token) try: - response = session.get(url) + response = session.get(url, timeout=(3.05, 10)) data = response.json() - return data['login'] + return data.get('login') except Exception as e: - g.log.error('Failed to fetch the user token') + import logging + log = getattr(g, 'log', logging.getLogger(__name__)) + log.error('Failed to fetch the user token') return None @@ -202,7 +205,7 @@ def github_callback(): url = 'https://github.com/login/oauth/access_token' payload = { 'client_id': config.get('GITHUB_CLIENT_ID', ''), - 'client_secret': config.get('GITHUB_CLIENT_SECRET', ''), + 'client_secret': config.get('GITHUB_CLIENT_KEY', ''), 'code': request.args['code'] } headers = {'Accept': 'application/json'} @@ -212,13 +215,19 @@ def github_callback(): if 'access_token' in response: user = User.query.filter(User.id == g.user.id).first() user.github_token = response['access_token'] + + # Fetch and store github_login + github_login = fetch_username_from_token(user) + if github_login: + user.github_login = github_login + g.db.commit() else: g.log.error("GitHub didn't return an access token") return redirect(url_for('auth.manage')) - return '', 404 + return abort(400) @mod_auth.route('/login', methods=['GET', 'POST']) @@ -231,7 +240,11 @@ def login() -> Union[Response, Dict[str, Union[str, LoginForm]]]: flash('You are already logged in!', 'alert') if len(redirect_location) == 0: return redirect("/") - return redirect(url_for(redirect_location)) + try: + return redirect(url_for(redirect_location)) + except BuildError: + # Endpoint requires parameters we don't have, redirect to home + return redirect("/") form = LoginForm(request.form) if form.validate_on_submit(): @@ -240,7 +253,11 @@ def login() -> Union[Response, Dict[str, Union[str, LoginForm]]]: session['user_id'] = user_to_login.id if len(redirect_location) == 0: return redirect("/") - return redirect(url_for(redirect_location)) + try: + return redirect(url_for(redirect_location)) + except BuildError: + # Endpoint requires parameters we don't have, redirect to home + return redirect("/") flash('Wrong username or password', 'error-message') @@ -471,7 +488,7 @@ def manage(): if user_to_update.email != form.email.data: old_email = user_to_update.email user_to_update.email = form.email.data - if len(form.new_password.data) >= 10: + if len(form.new_password.data or "") >= 10: password = True user_to_update.password = User.generate_hash(form.new_password.data) if user_to_update.name != form.name.data: diff --git a/mod_auth/forms.py b/mod_auth/forms.py index 0f1b2b2dd..25f51886e 100755 --- a/mod_auth/forms.py +++ b/mod_auth/forms.py @@ -4,9 +4,8 @@ from typing import Any, Callable, Optional, Type from flask_wtf import FlaskForm -from wtforms import PasswordField, SelectField, StringField, SubmitField -from wtforms.fields.html5 import EmailField -from wtforms.fields.simple import PasswordField +from wtforms import (EmailField, PasswordField, SelectField, StringField, + SubmitField) from wtforms.validators import DataRequired, Email, ValidationError import mod_auth.models @@ -39,7 +38,7 @@ def valid_password(form: CompleteSignupForm, field: PasswordField) -> None: from run import config min_pwd_len = int(config['MIN_PWD_LEN']) max_pwd_len = int(config['MAX_PWD_LEN']) - pass_size = len(field.data) + pass_size = len(field.data or "") if pass_size == 0: raise ValidationError('new password cannot be empty') if pass_size < min_pwd_len or pass_size > max_pwd_len: @@ -59,7 +58,7 @@ def email_not_in_use(has_user_field: bool = False) -> Callable: def _email_not_in_use(form, field): user_id = -1 if not has_user_field else form.user.id user = User.query.filter(User.email == field.data).first() - if user is not None and user.id != user_id and len(field.data) > 0: + if user is not None and user.id != user_id and len(field.data or "") > 0: raise ValidationError('This address is already in use') return _email_not_in_use @@ -165,7 +164,7 @@ def validate_new_password(form, field) -> None: :param field: The data value for the 'password' entered by User :type field : PasswordField """ - if len(field.data) == 0 and len(form.new_password_repeat.data) == 0: + if len(field.data or "") == 0 and len(form.new_password_repeat.data or "") == 0: return valid_password(form, field) @@ -181,7 +180,7 @@ def validate_new_password_repeat(form, field) -> None: :type field : PasswordField """ if form.email is not None: - if len(field.data) == 0 and len(form.new_password.data) == 0: + if len(field.data or "") == 0 and len(form.new_password.data or "") == 0: return if field.data != form.new_password.data: diff --git a/mod_auth/models.py b/mod_auth/models.py index 9e7f746cf..befceb266 100644 --- a/mod_auth/models.py +++ b/mod_auth/models.py @@ -32,10 +32,13 @@ class User(Base): name = Column(String(50), unique=True) email = Column(String(255), unique=True, nullable=True) github_token = Column(Text(), nullable=True) + # GitHub username; populated at OAuth login and used by the API to + # authorize fork-run triggers. + github_login = Column(String(255), nullable=True) password = Column(String(255), unique=False, nullable=False) role = Column(Role.db_type()) - def __init__(self, name, role=Role.user, email=None, password='', github_token=None) -> None: + def __init__(self, name, role=Role.user, email=None, password='', github_token=None, github_login=None) -> None: """ Parametrized constructor for the User model. @@ -55,6 +58,7 @@ def __init__(self, name, role=Role.user, email=None, password='', github_token=N self.password = password self.role = role self.github_token = github_token + self.github_login = github_login def __repr__(self) -> str: """ @@ -77,7 +81,7 @@ def generate_hash(password: str) -> str: :rtype : str """ # Go for increased strength no matter what - return pwd_context.encrypt(password, category='admin') + return pwd_context.hash(password, category='admin') @staticmethod def create_random_password(length=16) -> str: diff --git a/mod_ci/comparison.py b/mod_ci/comparison.py new file mode 100644 index 000000000..37620cb65 --- /dev/null +++ b/mod_ci/comparison.py @@ -0,0 +1,143 @@ +"""Compare one test run's regression results against another run's. + +Pass and fail are decided elsewhere, and against one thing only: the approved +output. That answers "is this correct?", which is the question a baseline is for +-- but it is not the question a reviewer is asking. A reviewer wants to know +what *this change* did, and a test that has been failing since last month tells +them nothing while burying the one that started failing today. + +So the verdict stays absolute and the report is relative. The same failure is +described against several references at once: the approved output, the tip of +master, and the closest ancestor commit we still hold results for. A test +failing identically on all of them is drift someone needs to approve; a test +failing only here is the change under review. + +The functions below take plain values rather than models so the classification +can be tested without a database, a GitHub client, or a CI run. +""" + +from typing import Any, Dict, Iterable, List, NamedTuple, Optional, Tuple + +#: The test matched the approved output on both sides. +UNCHANGED_PASS = 'unchanged_pass' +#: Fails here, matched the approved output in the reference run. +BROKEN_HERE = 'broken_here' +#: Matches the approved output here, failed in the reference run. +FIXED_HERE = 'fixed_here' +#: Fails on both sides and produces the *same* bytes -- unchanged behaviour +#: measured against a baseline that no longer describes it. +FAILING_IDENTICALLY = 'failing_identically' +#: Fails on both sides but the output differs, so something moved even though +#: the verdict did not. +FAILING_DIFFERENTLY = 'failing_differently' +#: The reference run holds no record of this test, so nothing can be said. +NO_REFERENCE = 'no_reference' + +#: Every verdict, in the order a reader should be shown them: what this change +#: broke first, what it fixed next, then the pre-existing noise. +VERDICTS = (BROKEN_HERE, FIXED_HERE, FAILING_DIFFERENTLY, FAILING_IDENTICALLY, + UNCHANGED_PASS, NO_REFERENCE) + + +class TestState(NamedTuple): + """How one regression test behaved in one run. + + ``signature`` identifies *how* a failing test differed, so two runs failing + the same test can be told apart by whether they produced the same bytes. + """ + + #: True when the exit code matched and every output file matched the approved one. + passed: bool + #: (output_id, produced hash) for each output that did not match, sorted. + signature: Tuple[Tuple[int, Optional[str]], ...] + + +def build_state(test_results: Iterable[Any]) -> Dict[int, TestState]: + """ + Summarise a run as one state per regression test. + + Whether a test passed is taken from ``get_test_results``, which is the + platform's own verdict and already accounts for exit codes, outputs that are + absent when they should not be, and the alternative hashes an output may + legitimately produce. Re-deriving any of that here would mean two + definitions of "passed" that could drift apart. + + The signature is built from the recorded hashes, which is the part + ``get_test_results`` does not express: it lets two runs failing the same + test be told apart by whether they produced the same bytes. + + :param test_results: The structure returned by ``get_test_results``. + :type test_results: Iterable[Any] + :return: Regression test id mapped to how that test behaved. + :rtype: Dict[int, TestState] + """ + states: Dict[int, TestState] = {} + for category in test_results: + for entry in category['tests']: + # A caller that reports no files leaves the failure unexplained rather + # than unnoticed: the verdict still counts, only the signature is empty. + failed_outputs: List[Tuple[int, Optional[str]]] = sorted( + (result_file.regression_test_output_id, result_file.got) + for result_file in (entry.get('files') or ()) if result_file.got is not None) + states[entry['test'].id] = TestState(passed=not entry['error'], + signature=tuple(failed_outputs)) + return states + + +def classify(current: TestState, reference: Optional[TestState]) -> str: + """ + Describe one test's behaviour here relative to a reference run. + + :param current: How the test behaved in the run being reported on. + :type current: TestState + :param reference: How it behaved in the reference run, if that run has a record. + :type reference: Optional[TestState] + :return: One of the module's verdict constants. + :rtype: str + """ + if reference is None: + return NO_REFERENCE + if current.passed and reference.passed: + return UNCHANGED_PASS + if current.passed: + return FIXED_HERE + if reference.passed: + return BROKEN_HERE + if current.signature == reference.signature: + return FAILING_IDENTICALLY + return FAILING_DIFFERENTLY + + +def compare(current: Dict[int, TestState], + reference: Optional[Dict[int, TestState]]) -> Dict[str, List[int]]: + """ + Bucket every regression test in a run by how it compares to a reference run. + + A missing reference run is not the same as a reference run that passed + everything: it is reported as ``no_reference`` so the comment can say "we + have no records to compare against" instead of implying good news. + + :param current: States for the run being reported on. + :type current: Dict[int, TestState] + :param reference: States for the reference run, or None when there is no such run. + :type reference: Optional[Dict[int, TestState]] + :return: Verdict mapped to the regression test ids in it. + :rtype: Dict[str, List[int]] + """ + buckets: Dict[str, List[int]] = {verdict: [] for verdict in VERDICTS} + for rt_id in sorted(current): + reference_state = None if reference is None else reference.get(rt_id) + buckets[classify(current[rt_id], reference_state)].append(rt_id) + return buckets + + +def summarise(buckets: Dict[str, List[int]]) -> Dict[str, int]: + """ + Count each verdict, for a table that has to stay short. + + :param buckets: Output of :func:`compare`. + :type buckets: Dict[str, List[int]] + :return: Verdict mapped to how many tests fell in it. + :rtype: Dict[str, int] + """ + return {verdict: len(ids) for verdict, ids in buckets.items()} diff --git a/mod_ci/controllers.py b/mod_ci/controllers.py index 54ea547f5..fc55c8a9e 100755 --- a/mod_ci/controllers.py +++ b/mod_ci/controllers.py @@ -1,61 +1,257 @@ """maintains all functionality related running virtual machines, starting and tracking tests.""" +import base64 import datetime +import fnmatch import hashlib +import itertools import json import os +import re import shutil -import sys +import tempfile +import time import zipfile -from multiprocessing import Process +from collections import defaultdict +from functools import wraps from pathlib import Path -from typing import Any +from typing import Any, Callable, Dict, List, Optional, TypeVar +import googleapiclient.discovery import requests +import yaml from flask import (Blueprint, abort, flash, g, jsonify, redirect, request, url_for) -from git import GitCommandError, InvalidGitRepositoryError, Repo -from github import ApiError, GitHub +from github import (Auth, Commit, Github, GithubException, GithubObject, + Repository) +from google.oauth2 import service_account from lxml import etree -from lxml.etree import Element from markdown2 import markdown from pymysql.err import IntegrityError -from sqlalchemy import and_, func, or_ +from sqlalchemy import and_, func, select from sqlalchemy.sql import label -from sqlalchemy.sql.functions import count from werkzeug.utils import secure_filename -from database import DeclEnum +from database import DeclEnum, create_session from decorators import get_menu_entries, template_renderer -from mailer import Mailer from mod_auth.controllers import check_access_rights, login_required from mod_auth.models import Role +from mod_ci import comparison from mod_ci.forms import AddUsersToBlacklist, DeleteUserForm -from mod_ci.models import BlockedUsers, Kvm, MaintenanceMode, PrCommentInfo +from mod_ci.models import (BlockedUsers, CategoryTestInfo, GcpInstance, + MaintenanceMode, PendingDeletion, PrCommentInfo, + ReferenceComparison, Status) from mod_customized.models import CustomizedTest -from mod_deploy.controllers import is_valid_signature, request_from_github from mod_home.models import CCExtractorVersion, GeneralData from mod_regression.models import (Category, RegressionTest, - RegressionTestOutput, - RegressionTestOutputFiles, - regressionTestLinkTable) + RegressionTestOutput) from mod_sample.models import Issue +from mod_test.controllers import get_test_results from mod_test.models import (Fork, Test, TestPlatform, TestProgress, TestResult, TestResultFile, TestStatus, TestType) +from utility import is_valid_signature, request_from_github + +# Timeout constants (in seconds) +GITHUB_API_TIMEOUT = 30 # Timeout for GitHub API calls +GCP_API_TIMEOUT = 60 # Timeout for GCP API calls +ARTIFACT_DOWNLOAD_TIMEOUT = 300 # 5 minutes for artifact downloads + +#: How far back to walk a branch's history looking for a run to compare against. +#: Deep enough to clear a stale branch, short enough to stay one API page. +ANCESTOR_SEARCH_DEPTH = 50 +GCP_OPERATION_MAX_WAIT = 1800 # 30 minutes max wait for GCP operations +GCP_VM_CREATE_VERIFY_TIMEOUT = 60 # 60 seconds to verify VM creation started + +# Retry constants +MAX_RETRIES = 3 +INITIAL_BACKOFF = 1 # seconds +MAX_BACKOFF = 30 # seconds + +T = TypeVar('T') + + +def retry_with_backoff( + func: Callable[..., T], + max_retries: int = MAX_RETRIES, + initial_backoff: float = INITIAL_BACKOFF, + max_backoff: float = MAX_BACKOFF, + retryable_exceptions: Any = (GithubException, requests.RequestException) +) -> T: + """ + Execute a function with exponential backoff retry logic. + + :param func: The function to execute (should be a callable with no arguments, use lambda for args) + :param max_retries: Maximum number of retry attempts + :param initial_backoff: Initial backoff time in seconds + :param max_backoff: Maximum backoff time in seconds + :param retryable_exceptions: Tuple of exception types that should trigger a retry + :return: The result of the function call + :raises: The last exception if all retries fail + """ + from run import log -if sys.platform.startswith("linux"): - import libvirt + last_exception: Optional[Exception] = None + backoff = initial_backoff -mod_ci = Blueprint('ci', __name__) + for attempt in range(max_retries + 1): + try: + return func() + except retryable_exceptions as e: + last_exception = e + if attempt < max_retries: + log.warning(f"Attempt {attempt + 1}/{max_retries + 1} failed: {e}. Retrying in {backoff}s...") + time.sleep(backoff) + backoff = min(backoff * 2, max_backoff) + else: + log.error(f"All {max_retries + 1} attempts failed. Last error: {e}") + + if last_exception is not None: + raise last_exception + raise RuntimeError("retry_with_backoff: unexpected state - no exception captured") + + +def safe_db_commit(db, operation_description: str = "database operation") -> bool: + """ + Safely commit a database transaction with rollback on failure. + + :param db: The database session + :param operation_description: Description of the operation for logging + :return: True if commit succeeded, False otherwise + """ + from run import log + + try: + db.commit() + return True + except Exception as e: + log.error(f"Database commit failed during {operation_description}: {e}") + try: + db.rollback() + log.info(f"Successfully rolled back transaction for {operation_description}") + except Exception as rollback_error: + log.error(f"Rollback also failed for {operation_description}: {rollback_error}") + return False + + +_ZONE_EXHAUSTED_MESSAGE = ( + "GCP resources temporarily unavailable in the configured zone. " + "The test will be retried automatically when resources become available." +) + +# User-friendly messages for known GCP error codes. +# +# GCP reports zone exhaustion under two codes: the bare ZONE_RESOURCE_POOL_EXHAUSTED and +# ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS, which carries the specific resource that ran +# out. Both mean the same thing to us, and in practice the _WITH_DETAILS variant is the one +# Compute Engine returns for instance inserts. +GCP_ERROR_MESSAGES = { + 'ZONE_RESOURCE_POOL_EXHAUSTED': _ZONE_EXHAUSTED_MESSAGE, + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS': _ZONE_EXHAUSTED_MESSAGE, + 'QUOTA_EXCEEDED': ( + "GCP quota limit reached. " + "The test will be retried automatically when resources become available." + ), + 'RESOURCE_NOT_FOUND': "Required GCP resource not found. Please contact the administrator.", + 'RESOURCE_ALREADY_EXISTS': "A VM with this name already exists. Please contact the administrator.", + 'TIMEOUT': "GCP operation timed out. The test will be retried automatically.", +} + +# GCP error codes that are transient and should be retried automatically. +# Tests encountering these errors will remain pending and be picked up on the next cron run. +GCP_RETRYABLE_ERRORS = { + 'ZONE_RESOURCE_POOL_EXHAUSTED', + 'ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS', + 'QUOTA_EXCEEDED', +} + + +def get_gcp_error_code(result: Dict) -> Optional[str]: + """ + Extract the error code from a GCP API error response. + + :param result: The GCP API response dictionary + :return: The error code string, or None if not found + """ + if not isinstance(result, dict): + return None + error = result.get('error') + if not isinstance(error, dict): + return None + errors = error.get('errors', []) + if not errors or not isinstance(errors, list): + return None + first_error = errors[0] if len(errors) > 0 else {} + return first_error.get('code') + + +def is_retryable_gcp_error(result: Dict) -> bool: + """ + Check if a GCP error is transient and should be retried. + + :param result: The GCP API response dictionary + :return: True if the error is retryable, False otherwise + """ + error_code = get_gcp_error_code(result) + return error_code in GCP_RETRYABLE_ERRORS + + +def parse_gcp_error(result: Dict, log=None) -> str: + """ + Parse a GCP API error response and return a user-friendly message. + + GCP errors have the structure: + { + 'error': { + 'errors': [{'code': 'ERROR_CODE', 'message': '...'}] + } + } + + For known error codes, returns a user-friendly message. + For unknown errors, logs the details server-side and returns a generic message + to avoid exposing potentially sensitive information. + + :param result: The GCP API response dictionary + :param log: Optional logger instance. If not provided, uses module logger. + :return: A user-friendly error message + """ + import logging + if log is None: + log = logging.getLogger('Platform') + if not isinstance(result, dict): + log.error(f"GCP error (non-dict): {result}") + return "VM creation failed. Please contact the administrator." -class Status: - """Define different states for the tests.""" + error = result.get('error') + if error is None: + log.error(f"GCP error (no error key): {result}") + return "VM creation failed. Please contact the administrator." - PENDING = "pending" - SUCCESS = "success" - ERROR = "error" - FAILURE = "failure" + if not isinstance(error, dict): + log.error(f"GCP error (error not dict): {error}") + return "VM creation failed. Please contact the administrator." + + errors = error.get('errors', []) + if not errors: + log.error(f"GCP error (empty errors list): {error}") + return "VM creation failed. Please contact the administrator." + + # Get the first error (usually the most relevant) + first_error = errors[0] if isinstance(errors, list) and len(errors) > 0 else {} + error_code = first_error.get('code', 'UNKNOWN') + error_message = first_error.get('message', 'No details provided') + + # Check if we have a user-friendly message for this error code + if error_code in GCP_ERROR_MESSAGES: + return GCP_ERROR_MESSAGES[error_code] + + # For unknown errors, log full details server-side but return generic message + # to avoid exposing potentially sensitive information (project names, zones, etc.) + log.error(f"GCP error ({error_code}): {error_message}") + return f"VM creation failed ({error_code}). Please contact the administrator." + + +mod_ci = Blueprint('ci', __name__) class Workflow_builds(DeclEnum): @@ -68,8 +264,240 @@ class Workflow_builds(DeclEnum): class Artifact_names(DeclEnum): """Define CCExtractor GitHub Artifacts names.""" - linux = "CCExtractor Linux build" - windows = "CCExtractor Windows OCR and HardSubX Release build" + linux = "CCExtractor Linux build (with migrations)" + windows = "CCExtractor Windows x64 Release build (with migrations)" + + +def is_valid_commit_hash(commit: Optional[str]) -> bool: + """ + Validate that a string is a valid Git commit hash. + + A valid commit hash is: + - Not None or empty + - At least 7 characters (short hash) up to 40 characters (full SHA-1) + - Contains only hexadecimal characters (0-9, a-f, A-F) + - Not the Git null SHA (all zeros) + + :param commit: The commit hash to validate + :type commit: Optional[str] + :return: True if valid, False otherwise + :rtype: bool + """ + if not commit or not isinstance(commit, str): + return False + + commit = commit.strip() + + # Check length (7-40 characters for valid git hashes) + if len(commit) < 7 or len(commit) > 40: + return False + + # Check for null SHA (all zeros) + if commit == '0' * len(commit): + return False + + # Check that it's a valid hexadecimal string + try: + int(commit, 16) + return True + except ValueError: + return False + + +GIT_COMMIT_LOG_RE = re.compile(r'Git commit:\s*([0-9a-fA-F]{7,40})') +# Keep enough overlap that ``Git commit:`` plus a 40-char SHA cannot be split +# across two chunks without appearing in the next window. +GIT_COMMIT_CHUNK_SIZE = 65536 +GIT_COMMIT_OVERLAP = 64 + + +def _sha_from_git_commit_match(text: str) -> Optional[str]: + """Return a valid SHA from a ``Git commit:`` line in ``text``, if any.""" + match = GIT_COMMIT_LOG_RE.search(text) + if not match: + return None + sha = match.group(1) + return sha if is_valid_commit_hash(sha) else None + + +def parse_git_commit_from_logs(text: Optional[str]) -> Optional[str]: + """ + Extract the SHA the binary reports from uploaded VM logs. + + GitHub Actions builds the PR merge ref, not the PR head. The tested + binary prints ``Git commit: ``. That line is the ground truth for + what was actually compiled, as opposed to ``Test.commit`` which stores + the PR head SHA from the webhook. + + :param text: Log file contents + :type text: Optional[str] + :return: A valid commit hash if found, otherwise None + :rtype: Optional[str] + """ + if not text: + return None + return _sha_from_git_commit_match(text) + + +def parse_git_commit_from_log_stream(handle, chunk_size: int = GIT_COMMIT_CHUNK_SIZE) -> Optional[str]: + """ + Scan a log stream in chunks and stop as soon as ``Git commit:`` is found. + + :param handle: Readable text stream + :type handle: Any + :param chunk_size: Chars to read per batch + :type chunk_size: int + :return: A valid commit hash if found, otherwise None + :rtype: Optional[str] + """ + remainder = '' + while True: + chunk = handle.read(chunk_size) + if not chunk: + return _sha_from_git_commit_match(remainder) if remainder else None + window = remainder + chunk + sha = _sha_from_git_commit_match(window) + if sha: + return sha + if len(window) > GIT_COMMIT_OVERLAP: + remainder = window[-GIT_COMMIT_OVERLAP:] + else: + remainder = window + + +def parse_git_commit_from_log_file(log_path: str, + chunk_size: int = GIT_COMMIT_CHUNK_SIZE) -> Optional[str]: + """ + Extract ``Git commit:`` from a log file without loading it all at once. + + :param log_path: Path to the uploaded log file + :type log_path: str + :param chunk_size: Chars to read per batch + :type chunk_size: int + :return: A valid commit hash if found, otherwise None + :rtype: Optional[str] + """ + # Refuse non-strings: open() can treat objects with __index__ as fds + # and close stdout/stderr if a mocked path leaks in. + if not isinstance(log_path, str): + return None + try: + with open(log_path, encoding='utf-8', errors='replace') as handle: + return parse_git_commit_from_log_stream(handle, chunk_size=chunk_size) + except (OSError, ValueError, TypeError): + return None + + +def _merge_commit_from_pr(pr, payload_pr: Optional[dict] = None) -> Optional[str]: + """Return GitHub's test-merge SHA for a PR, if GitHub has computed one.""" + candidates = [] + if pr is not None: + candidates.append(getattr(pr, 'merge_commit_sha', None)) + if payload_pr: + candidates.append(payload_pr.get('merge_commit_sha')) + for sha in candidates: + if isinstance(sha, str) and is_valid_commit_hash(sha): + return sha.strip() + return None + + +def _record_built_commit_from_log(log, test, log_path: str) -> None: + """Update Test.built_commit from a ``Git commit:`` line in uploaded logs.""" + sha = parse_git_commit_from_log_file(log_path) + if not sha or test.built_commit == sha: + return + test.built_commit = sha + safe_db_commit(g.db, f"recording built commit {sha} for test {test.id}") + log.debug(f"Recorded built commit {sha} for test {test.id}") + + +# Maximum number of artifacts to search through when looking for a specific commit +# GitHub keeps artifacts for 90 days by default, so this should be enough +MAX_ARTIFACTS_TO_SEARCH = 500 + + +def find_artifact_for_commit(repository, commit_sha: str, platform: Any, log) -> Optional[Any]: + """ + Find a build artifact for a specific commit and platform. + + This function properly handles GitHub API pagination to search through + all available artifacts. This prevents race conditions where tests + fail because artifacts are not found due to pagination issues. + + :param repository: GitHub repository object + :type repository: Repository.Repository + :param commit_sha: The commit SHA to find artifact for + :type commit_sha: str + :param platform: The platform (linux or windows) - TestPlatform enum value + :type platform: TestPlatform + :param log: Logger instance + :return: The artifact object if found, None otherwise + :rtype: Optional[Artifact] + """ + if platform == TestPlatform.linux: + artifact_name = Artifact_names.linux + else: + artifact_name = Artifact_names.windows + + try: + artifacts = repository.get_artifacts() + artifacts_checked = 0 + + for artifact in artifacts: + artifacts_checked += 1 + + if artifact.name == artifact_name and artifact.workflow_run.head_sha == commit_sha: + log.debug(f"Found artifact '{artifact_name}' for commit {commit_sha[:8]} " + f"(checked {artifacts_checked} artifacts)") + return artifact + + # Limit search to prevent excessive API calls + if artifacts_checked >= MAX_ARTIFACTS_TO_SEARCH: + log.warning(f"Reached max artifact search limit ({MAX_ARTIFACTS_TO_SEARCH}) " + f"without finding artifact for commit {commit_sha[:8]}") + break + + log.debug(f"No artifact '{artifact_name}' found for commit {commit_sha[:8]} " + f"(checked {artifacts_checked} artifacts)") + return None + + except Exception as e: + log.error(f"Error searching for artifact: {type(e).__name__}: {e}") + return None + + +def verify_artifacts_exist(repository, commit_sha: str, log) -> Dict[str, bool]: + """ + Verify that build artifacts exist for a commit before queuing tests. + + This function should be called before queue_test() to prevent the race + condition where tests are queued before artifacts are available. + + :param repository: GitHub repository object + :type repository: Repository.Repository + :param commit_sha: The commit SHA to verify + :type commit_sha: str + :param log: Logger instance + :return: Dict with 'linux' and 'windows' keys indicating artifact availability + :rtype: Dict[str, bool] + """ + result = {'linux': False, 'windows': False} + + linux_artifact = find_artifact_for_commit(repository, commit_sha, TestPlatform.linux, log) + if linux_artifact is not None: + result['linux'] = True + log.info(f"Linux artifact verified for commit {commit_sha[:8]}") + else: + log.warning(f"Linux artifact NOT found for commit {commit_sha[:8]}") + + windows_artifact = find_artifact_for_commit(repository, commit_sha, TestPlatform.windows, log) + if windows_artifact is not None: + result['windows'] = True + log.info(f"Windows artifact verified for commit {commit_sha[:8]}") + else: + log.warning(f"Windows artifact NOT found for commit {commit_sha[:8]}") + + return result @mod_ci.before_app_request @@ -89,63 +517,162 @@ def before_app_request() -> None: g.menu_entries['config'] = config_entries -def start_platforms(db, repository, delay=None, platform=None) -> None: +def start_platforms(repository, delay=None, platform=None) -> None: """ - Start new test on both platforms in parallel. + Start new test on both platforms. - We use multiprocessing module which bypasses Python GIL to make use of multiple cores of the processor. + :param repository: repository to run tests on + :type repository: str + :param delay: time delay after which to start gcp_instance function + :type delay: int + :param platform: operating system + :type platform: str """ from run import app, config, log + vm_max_runtime = config.get("GCP_INSTANCE_MAX_RUNTIME", 120) + zone = config.get('ZONE', '') + project = config.get('PROJECT_NAME', '') + # Check if zone and project both are provided + if zone == "": + log.critical('GCP zone name is empty!') + return + + if project == "": + log.critical('GCP project name is empty!') + return + with app.app_context(): from flask import current_app + app = current_app._get_current_object() # type: ignore[attr-defined] + + # Create a database session + db = create_session(config.get('DATABASE_URI', '')) + + compute = get_compute_service_object() + + # Step 1: Verify any pending deletions from previous runs + verify_pending_deletions(compute, project, zone, db) + + # Step 2: Scan for orphaned VMs that weren't properly deleted + scan_for_orphaned_vms(compute, project, zone, db) + + # Step 3: Delete expired instances (tests that ran too long) + delete_expired_instances(compute, vm_max_runtime, project, zone, db, repository) + if platform is None or platform == TestPlatform.linux: - linux_kvm_name = config.get('KVM_LINUX_NAME', '') - log.info('Define process to run Linux VM') - linux_process = Process(target=kvm_processor, args=(current_app._get_current_object(), db, linux_kvm_name, - TestPlatform.linux, repository, delay,)) - linux_process.start() - log.info('Linux VM process kicked off') + log.info('Define process to run Linux GCP instances') + gcp_instance(app, db, TestPlatform.linux, repository, delay) + log.info('Linux GCP instances process kicked off') if platform is None or platform == TestPlatform.windows: - win_kvm_name = config.get('KVM_WINDOWS_NAME', '') - log.info('Define process to run Windows VM') - windows_process = Process(target=kvm_processor, args=(current_app._get_current_object(), db, win_kvm_name, - TestPlatform.windows, repository, delay,)) - windows_process.start() - log.info('Windows VM process kicked off') + log.info('Define process to run Windows GCP instances') + gcp_instance(app, db, TestPlatform.windows, repository, delay) + log.info('Windows GCP instances process kicked off') + + +def get_running_instances(compute, project, zone) -> list: + """ + Get details of all the running GCP VM instances. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Configured zone for the VM instances + :type zone: str + :return: List of VM instances + :rtype: list + """ + result = compute.instances().list(project=project, zone=zone).execute() + return result['items'] if 'items' in result else [] -def kvm_processor(app, db, kvm_name, platform, repository, delay) -> None: +def is_instance_testing(vm_name) -> bool: """ - Check whether there is no already running same kvm. + Check if VM name is of the correct format and return if it is used for testing or not. + + :param vm_name: Name of the VM machine to be identified + :type vm_name: str + :return: Boolean whether instance is used for testing or not + :rtype: bool + """ + for platform in TestPlatform: + if re.fullmatch(f"{platform.value}-[0-9]+", vm_name): + return True + return False - Checks whether machine is in maintenance mode or not - Launch kvm if not used by any other test - Creates testing xml files to test the change in main repo. - Downloads the build artifacts generated during GitHub Action workflows. + +def delete_expired_instances(compute, max_runtime, project, zone, db, repository) -> None: + """ + Get all running instances and delete instances whose maximum runtime limit is reached. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param max_runtime: The maximum runtime limit for VM instances + :type max_runtime: int + :param project: The GCP project name + :type project: str + :param zone: Zone for the new VM instance + :type zone: str + """ + for instance in get_running_instances(compute, project, zone): + vm_name = instance['name'] + if is_instance_testing(vm_name): + creationTimestamp = datetime.datetime.strptime(instance['creationTimestamp'], '%Y-%m-%dT%H:%M:%S.%f%z') + currentTimestamp = datetime.datetime.now(datetime.timezone.utc) + if currentTimestamp - creationTimestamp >= datetime.timedelta(minutes=max_runtime): + # Update test status in database and on GitHub + platform_name, test_id = vm_name.split('-') + test = Test.query.filter(Test.id == test_id).first() + message = "Could not complete test, time limit exceeded" + progress = TestProgress(test_id, TestStatus.canceled, message) + db.add(progress) + if not safe_db_commit(db, f"canceling timed-out test {test_id}"): + continue # Skip to next instance if commit failed + + gh_commit = repository.get_commit(test.commit) + if gh_commit is not None: + # Build target_url so users can see test results + from flask import url_for + try: + target_url = url_for('test.by_id', test_id=test_id, _external=True) + except RuntimeError: + # Outside of request context + target_url = f"https://sampleplatform.ccextractor.org/test/{test_id}" + update_status_on_github(gh_commit, Status.ERROR, message, f"CI - {platform_name}", target_url) + + # Delete VM instance with tracking for verification + from run import log + try: + operation = delete_instance_with_tracking( + compute, project, zone, vm_name, db) + op_name = operation.get('name', 'unknown') + log.info(f"Expired instance deletion initiated for {vm_name} (op: {op_name})") + except Exception as e: + log.error(f"Failed to delete expired instance {vm_name}: {e}") + + +def gcp_instance(app, db, platform, repository, delay) -> None: + """ + Find all the pending tests and start running them in new GCP instances. :param app: The Flask app :type app: Flask :param db: database connection - :type db: sqlalchemy.orm.scoped_session - :param kvm_name: name for the kvm - :type kvm_name: str + :type db: sqlalchemy.orm.scoping.scoped_session :param platform: operating system :type platform: str :param repository: repository to run tests on :type repository: str - :param delay: time delay after which to start kvm processor + :param delay: time delay after which to start gcp_instance function :type delay: int """ from run import config, get_github_config, log github_config = get_github_config(config) - log.info(f"[{platform}] Running kvm_processor") - if kvm_name == "": - log.critical(f'[{platform}] KVM name is empty!') - return + log.info(f"[{platform}] Running gcp_instance") if delay is not None: import time @@ -157,106 +684,444 @@ def kvm_processor(app, db, kvm_name, platform, repository, delay) -> None: log.debug(f'[{platform}] In maintenance mode! Waiting...') return - conn = libvirt.open("qemu:///system") - if conn is None: - log.critical(f"[{platform}] Connection to libvirt failed!") - return + finished_tests = db.query(TestProgress.test_id).filter( + TestProgress.status.in_([TestStatus.canceled, TestStatus.completed]) + ) + + running_tests = db.query(GcpInstance.test_id) + + pending_tests = Test.query.filter( + Test.id.notin_(finished_tests), Test.id.notin_(running_tests), Test.platform == platform + ).order_by(Test.id.asc()) + + compute = get_compute_service_object() + + for test in pending_tests: + if test.test_type == TestType.pull_request: + try: + gh_commit = retry_with_backoff(lambda t=test: repository.get_commit(t.commit)) + if test.pr_nr == 0: + log.warn(f'[{platform}] Test {test.id} is invalid') + deschedule_test(gh_commit, message="Invalid PR number", test=test, db=db) + continue + test_pr = retry_with_backoff(lambda t=test: repository.get_pull(t.pr_nr)) + # Note: We intentionally do NOT check if test.commit != test_pr.head.sha + # If a new commit was pushed to the PR, a new test entry will be created for it. + # We should still run the test for the commit that was originally queued. + # This prevents the confusing "PR closed or updated" error when users push fixes. + if test_pr.state != 'open': + log.info(f"[{platform}] PR {test.pr_nr} is closed, descheduling test {test.id}") + deschedule_test(gh_commit, message="PR is closed", test=test, db=db) + continue + except GithubException as e: + log.error(f"GitHub API error for test {test.id} after retries: {e}") + continue # Skip this test, try next one + except Exception as e: + log.error(f"Unexpected error checking PR status for test {test.id}: {e}") + continue + start_test(compute, app, db, repository, test, github_config['bot_token']) + + +def get_compute_service_object() -> googleapiclient.discovery.Resource: + """Get a Cloud Compute Engine service object.""" + from run import config + scopes = config.get('SCOPES', '') + sa_file = os.path.join(config.get('INSTALL_FOLDER', ''), config.get('SERVICE_ACCOUNT_FILE', '')) + + credentials = service_account.Credentials.from_service_account_file(sa_file, scopes=scopes) + + return googleapiclient.discovery.build('compute', 'v1', credentials=credentials) + + +def mark_test_failed(db, test, repository, message: str) -> bool: + """ + Mark a test as failed and update GitHub status. + + This function ensures that GitHub is always notified of the failure, + even if database operations fail. The GitHub status update is critical + to prevent tests from appearing stuck in "pending" state forever. + + :param db: Database session + :type db: sqlalchemy.orm.scoping.scoped_session + :param test: The test to mark as failed + :type test: mod_test.models.Test + :param repository: GitHub repository object + :type repository: Repository.Repository + :param message: Error message to display + :type message: str + :return: True if operation succeeded, False otherwise + """ + from run import log + + db_success = False + github_success = False + + # Step 1: Try to update the database try: - vm = conn.lookupByName(kvm_name) - except libvirt.libvirtError: - log.critical(f"[{platform}] No VM named {kvm_name} found!") - return + progress = TestProgress(test.id, TestStatus.canceled, message) + db.add(progress) + db.commit() + db_success = True + log.info(f"Test {test.id}: Database updated with failure status") + except Exception as e: + log.error(f"Test {test.id}: Failed to update database: {e}") + # Continue to try GitHub update even if DB fails + + # Step 2: Try to update GitHub status (CRITICAL - must not be skipped) + # Use retry logic since this is critical to prevent stuck "pending" status + try: + # Build target_url first (doesn't need retry) + from flask import url_for + try: + target_url = url_for('test.by_id', test_id=test.id, _external=True) + except RuntimeError: + # Outside of request context + target_url = f"https://sampleplatform.ccextractor.org/test/{test.id}" + + # Use retry_with_backoff for GitHub API calls + def update_github_status(): + gh_commit = repository.get_commit(test.commit) + update_status_on_github(gh_commit, Status.ERROR, message, f"CI - {test.platform.value}", target_url) + + retry_with_backoff(update_github_status, max_retries=3, initial_backoff=2.0) + github_success = True + log.info(f"Test {test.id}: GitHub status updated to ERROR: {message}") + except GithubException as e: + log.error(f"Test {test.id}: GitHub API error while updating status (after retries): {e.status} - {e.data}") + except Exception as e: + log.error(f"Test {test.id}: Failed to update GitHub status (after retries): {type(e).__name__}: {e}") + + # Log final status + if db_success and github_success: + log.info(f"Test {test.id}: Successfully marked as failed") + elif github_success: + log.warning(f"Test {test.id}: GitHub updated but database update failed - test may be retried") + elif db_success: + log.error(f"Test {test.id}: Database updated but GitHub status NOT updated - " + f"status will appear stuck as 'pending' on GitHub!") + else: + log.critical(f"Test {test.id}: BOTH database and GitHub updates failed - " + f"test is in inconsistent state!") + + return db_success and github_success + + +# Mapping of workflow names to their YAML files in the repository +WORKFLOW_FILES = { + "Build CCExtractor on Linux": ".github/workflows/build_linux.yml", + "Build CCExtractor on Windows": ".github/workflows/build_windows.yml", +} - vm_info = vm.info() - if vm_info[0] != libvirt.VIR_DOMAIN_SHUTOFF: - # Running, check expiry and compare to runtime - status = Kvm.query.filter(Kvm.name == kvm_name).first() - max_runtime = config.get("KVM_MAX_RUNTIME", 120) - if status is not None: - if datetime.datetime.now() - status.timestamp >= datetime.timedelta(minutes=max_runtime): - test_progress = TestProgress(status.test.id, TestStatus.canceled, 'Runtime exceeded') - db.add(test_progress) - db.delete(status) - db.commit() - - if vm.destroy() == -1: - log.critical(f"[{platform}] Failed to shut down {kvm_name}") - return +# Cache for workflow path filters (refreshed periodically) +_workflow_filters_cache: Dict[str, tuple] = {} # {workflow_name: (filters, timestamp)} +WORKFLOW_CACHE_TTL = 3600 # 1 hour cache TTL + + +def _get_workflow_path_filters(repository, workflow_name: str, log) -> Optional[list]: + """ + Fetch and parse the workflow YAML to extract path filters. + + Returns a list of path patterns that trigger the workflow, or None if + the workflow has no path filters (runs on all changes). + + :param repository: GitHub repository object + :param workflow_name: Name of the workflow (e.g., "Build CCExtractor on Linux") + :param log: Logger instance + :return: List of path patterns, or None if no filters + """ + import time as time_module + + # Check cache first + if workflow_name in _workflow_filters_cache: + filters, cached_at = _workflow_filters_cache[workflow_name] + if time_module.time() - cached_at < WORKFLOW_CACHE_TTL: + return filters + + workflow_file = WORKFLOW_FILES.get(workflow_name) + if not workflow_file: + log.warning(f"Unknown workflow: {workflow_name}") + return None + + try: + # Fetch the workflow file content from the repository + contents = repository.get_contents(workflow_file) + workflow_yaml = base64.b64decode(contents.content).decode('utf-8') + + # Parse the YAML + workflow_config = yaml.safe_load(workflow_yaml) + + # Extract path filters from 'on.push.paths' and 'on.pull_request.paths' + on_config = workflow_config.get('on', {}) + path_filters = set() + + # Check push paths + push_config = on_config.get('push', {}) + if isinstance(push_config, dict) and 'paths' in push_config: + path_filters.update(push_config['paths']) + + # Check pull_request paths + pr_config = on_config.get('pull_request', {}) + if isinstance(pr_config, dict) and 'paths' in pr_config: + path_filters.update(pr_config['paths']) + + filters = list(path_filters) if path_filters else None + + # Cache the result + _workflow_filters_cache[workflow_name] = (filters, time_module.time()) + + log.debug(f"Workflow '{workflow_name}' path filters: {filters}") + return filters + + except Exception as e: + log.warning(f"Failed to fetch/parse workflow file {workflow_file}: {e}") + return None # Assume no filters on error (will retry) + + +def _will_workflow_run_for_commit(repository, commit_sha: str, workflow_name: str, log) -> Optional[bool]: + """ + Check if a workflow will run for a given commit based on path filters. + + :param repository: GitHub repository object + :param commit_sha: The commit SHA to check + :param workflow_name: Name of the workflow + :param log: Logger instance + :return: True if workflow will run (files match), False if it won't (no match), + None if cannot determine (no path filters or error) + """ + # Get the workflow's path filters + path_filters = _get_workflow_path_filters(repository, workflow_name, log) + + if path_filters is None: + # No path filters means workflow runs on all changes + return None + + try: + # Get the list of files changed in this commit + commit = repository.get_commit(commit_sha) + changed_files = [f.filename for f in commit.files] + + log.debug(f"Commit {commit_sha[:7]} changed {len(changed_files)} files") + + # Check if any changed file matches any path filter + for changed_file in changed_files: + for pattern in path_filters: + # Handle ** patterns (match any path depth) + if '**' in pattern: + # Convert ** glob to regex-compatible pattern + regex_pattern = pattern.replace('**/', '(.*/)?').replace('**', '.*') + regex_pattern = regex_pattern.replace('*', '[^/]*') + if re.match(regex_pattern, changed_file): + log.debug(f"File '{changed_file}' matches pattern '{pattern}'") + return True + elif fnmatch.fnmatch(changed_file, pattern): + log.debug(f"File '{changed_file}' matches pattern '{pattern}'") + return True + + # No files matched any pattern + log.info(f"Commit {commit_sha[:7]}: No changed files match workflow '{workflow_name}' path filters") + return False + + except Exception as e: + log.warning(f"Failed to check commit files against workflow filters: {e}") + return None # Cannot determine + + +def _diagnose_missing_artifact(repository, commit_sha: str, platform, log) -> tuple: + """ + Diagnose why an artifact was not found for a commit. + + Checks the workflow run status to provide a more helpful error message and + indicate whether this is a retryable situation (build still in progress) + or a permanent failure (build failed, artifact expired). + + :param repository: GitHub repository object + :param commit_sha: The commit SHA to check + :param platform: The platform (TestPlatform.linux or TestPlatform.windows) + :param log: Logger instance + :return: Tuple of (error_message: str, is_retryable: bool) + is_retryable=True means the test should NOT be marked as failed + and should be left for the next cron cycle to retry + """ + if platform == TestPlatform.linux: + expected_workflow = Workflow_builds.LINUX + else: + expected_workflow = Workflow_builds.WINDOWS + + # First, check if the workflow will even run based on path filters + will_run = _will_workflow_run_for_commit(repository, commit_sha, expected_workflow, log) + if will_run is False: + # Workflow will definitely NOT run - no matching files in commit + message = (f"No build will be created: commit {commit_sha[:7]} does not modify any files " + f"that trigger the '{expected_workflow}' workflow. " + f"Only documentation or non-code files were changed.") + log.info(f"Commit {commit_sha[:7]}: workflow '{expected_workflow}' will not run due to path filters") + return (message, False) # NOT retryable - workflow will never run + + try: + # Build workflow name lookup + workflow: Dict[int, Optional[str]] = defaultdict(lambda: None) + for active_workflow in repository.get_workflows(): + workflow[active_workflow.id] = active_workflow.name + + # Check workflow runs for this commit + workflow_found = False + for workflow_run in repository.get_workflow_runs(head_sha=commit_sha): + workflow_run_name = workflow[workflow_run.workflow_id] + if workflow_run_name != expected_workflow: + continue + + workflow_found = True + if workflow_run.status != "completed": + # Build is still running - this is RETRYABLE + # Don't mark as failed, let the next cron cycle retry + message = (f"Build still in progress: '{expected_workflow}' is {workflow_run.status}. " + f"Will retry when build completes.") + return (message, True) # Retryable + elif workflow_run.conclusion != "success": + # Build failed - this is a PERMANENT failure + message = (f"Build failed: '{expected_workflow}' finished with conclusion " + f"'{workflow_run.conclusion}'. Check the GitHub Actions logs for details.") + return (message, False) # Not retryable else: - log.info(f"[{platform}] Current job not expired yet.") - return - else: - log.warn(f"[{platform}] No task, but VM is running! Hard reset necessary") - if vm.destroy() == -1: - log.critical(f"[{platform}] Failed to shut down {kvm_name}") - return - - # Check if there's no KVM status left - status = Kvm.query.filter(Kvm.name == kvm_name).first() - if status is not None: - log.warn(f"[{platform}] KVM is powered off, but test {status.test.id} still present, deleting entry") - db.delete(status) - db.commit() + # Build succeeded but artifact not found + # Check if the build completed very recently - if so, this might be + # GitHub API propagation delay (artifact exists but not visible yet) + # In that case, treat as retryable + ARTIFACT_PROPAGATION_GRACE_PERIOD = 300 # 5 minutes in seconds + try: + from datetime import datetime, timezone + now = datetime.now(timezone.utc) + # workflow_run.updated_at is when the run completed + if workflow_run.updated_at: + completed_at = workflow_run.updated_at + if completed_at.tzinfo is None: + completed_at = completed_at.replace(tzinfo=timezone.utc) + seconds_since_completion = (now - completed_at).total_seconds() + if seconds_since_completion < ARTIFACT_PROPAGATION_GRACE_PERIOD: + message = (f"Build completed recently ({int(seconds_since_completion)}s ago): " + f"'{expected_workflow}' succeeded but artifact not yet visible. " + f"Will retry (GitHub API propagation delay).") + log.info(f"Artifact not found but build completed {int(seconds_since_completion)}s ago - " + f"treating as retryable (possible API propagation delay)") + return (message, True) # Retryable - API propagation delay + except Exception as e: + log.warning(f"Could not check workflow completion time: {e}") + # Fall through to permanent failure + + # Build completed more than 5 minutes ago - artifact should be visible + # This is a PERMANENT failure (artifact expired or not uploaded) + message = (f"Artifact not found: '{expected_workflow}' completed successfully, " + f"but no artifact was found. The artifact may have expired (GitHub deletes " + f"artifacts after a retention period) or was not uploaded properly.") + return (message, False) # Not retryable + + if not workflow_found: + # No workflow run found - could be queued, not triggered, or too old + # Check commit age to determine if we should keep waiting + MAX_WAIT_HOURS = 3 + try: + from datetime import datetime, timedelta, timezone + commit_obj = repository.get_commit(commit_sha) + commit_date = commit_obj.commit.author.date + if commit_date.tzinfo is None: + commit_date = commit_date.replace(tzinfo=timezone.utc) + now = datetime.now(timezone.utc) + commit_age = now - commit_date + + if commit_age > timedelta(hours=MAX_WAIT_HOURS): + # Commit is too old - workflow was never triggered or history expired + hours_old = commit_age.total_seconds() / 3600 + message = (f"No build available: '{expected_workflow}' never ran for commit " + f"{commit_sha[:7]} (commit is {hours_old:.1f} hours old). " + f"The workflow was likely not triggered due to path filters, " + f"or the commit predates the workflow configuration.") + log.info(f"Commit {commit_sha[:7]} is {hours_old:.1f} hours old with no workflow run - " + f"marking as permanent failure") + return (message, False) # NOT retryable - too old + except Exception as e: + log.warning(f"Could not check commit age: {e}") + # Fall through to retryable on error - # Get oldest test for this platform - finished_tests = db.query(TestProgress.test_id).filter( - TestProgress.status.in_([TestStatus.canceled, TestStatus.completed]) - ).subquery() - fork_location = f"%/{github_config['repository_owner']}/{github_config['repository']}.git" - fork = Fork.query.filter(Fork.github.like(fork_location)).first() - test = Test.query.filter( - Test.id.notin_(finished_tests), Test.platform == platform, Test.fork_id == fork.id - ).order_by(Test.id.asc()).first() + # Commit is recent enough - workflow might still be queued + message = (f"No workflow run found: '{expected_workflow}' has not run for commit " + f"{commit_sha[:7]}. The workflow may be queued, or was not triggered " + f"due to path filters. Will retry.") + return (message, True) # Retryable - if test is None: - test = Test.query.filter(Test.id.notin_(finished_tests), Test.platform == platform).order_by( - Test.id.asc()).first() + except Exception as e: + log.warning(f"Failed to diagnose missing artifact: {e}") + # On diagnostic failure, assume retryable to be safe + return (f"No build artifact found for this commit (diagnostic check failed: {e})", True) - if test is None: - log.info(f'[{platform}] No more tests to run, returning') - return + return ("No build artifact found for this commit", True) # Default to retryable - if test.test_type == TestType.pull_request and test.pr_nr == 0: - log.warn(f'[{platform}] Test {test.id} is invalid, deleting') - db.delete(test) - db.commit() - return - # Reset to snapshot - if vm.hasCurrentSnapshot() != 1: - log.critical(f"[{platform}] VM {kvm_name} has no current snapshot set!") +def start_test(compute, app, db, repository: Repository.Repository, test, bot_token) -> None: + """ + Start a VM instance and run the tests. + + Creates testing xml files to test the changes. + Downloads the build artifacts generated during GitHub Action workflows. + Create a GCP instance and start the test. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param app: The Flask app + :type app: Flask + :param db: database connection + :type db: sqlalchemy.orm.scoping.scoped_session + :param platform: operating system + :type platform: str + :param repository: repository to run tests on + :type repository: str + :param test: The test which is to be started + :type test: mod_test.models.Test + :param bot_token: The GitHub bot token + :type bot_token: str + :return: Nothing + :rtype: None + """ + from run import config, log + + # Check if test is already being processed (basic locking) + existing_instance = GcpInstance.query.filter(GcpInstance.test_id == test.id).first() + if existing_instance is not None: + log.warning(f"Test {test.id} already has a GCP instance, skipping duplicate start") return - snapshot = vm.snapshotCurrent() - if vm.revertToSnapshot(snapshot) == -1: - log.critical(f"[{platform}] Failed to revert to {snapshot.getName()} for {kvm_name}") + # Check if test already has progress (already started or finished) + existing_progress = TestProgress.query.filter(TestProgress.test_id == test.id).first() + if existing_progress is not None: + log.warning(f"Test {test.id} already has progress entries, skipping") return - log.info(f"[{platform}] Reverted to {snapshot.getName()} for {kvm_name}") - log.debug(f'[{platform}] Starting test {test.id}') - status = Kvm(kvm_name, test.id) + gcp_instance_name = f"{test.platform.value}-{test.id}" + log.debug(f'[{gcp_instance_name}] Starting test {test.id}') + + test_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', gcp_instance_name) + + Path(test_folder).mkdir(parents=True, exist_ok=True) + + status = GcpInstance(gcp_instance_name, test.id) # Prepare data # 0) Write url to file with app.app_context(): full_url = url_for('ci.progress_reporter', test_id=test.id, token=test.token, _external=True, _scheme="https") - file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'reportURL') - - with open(file_path, 'w') as f: - f.write(full_url) - # 1) Generate test files - base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'ci-tests') - categories = Category.query.order_by(Category.id.desc()).all() - commit_name = 'fetch_commit_' + platform.value - commit_hash = GeneralData.query.filter(GeneralData.key == commit_name).first().value - last_commit = Test.query.filter(and_(Test.commit == commit_hash, Test.platform == platform)).first() + base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', gcp_instance_name, 'ci-tests') + Path(base_folder).mkdir(parents=True, exist_ok=True) - log.debug(f"[{platform}] We will compare against the results of test {last_commit.id}") + categories = Category.query.order_by(Category.id.desc()).all() regression_ids = test.get_customized_regressiontests() + # BREAKS REGULAR TESTS + # if len(regression_ids) == 0: + # log.debug(f"[{gcp_instance_name}] No regression tests, skipping test {test.id}") + # return + # Init collection file multi_test = etree.Element('multitest') for category in categories: @@ -281,25 +1146,18 @@ def kvm_processor(app, db, kvm_name, platform, repository, delay) -> None: output_node = etree.SubElement(entry, 'output') output_node.text = regression_test.output_type.value compare = etree.SubElement(entry, 'compare') - last_files = TestResultFile.query.filter(and_( - TestResultFile.test_id == last_commit.id, - TestResultFile.regression_test_id == regression_test.id - )).subquery() for output_file in regression_test.output_files: ignore_file = str(output_file.ignore).lower() file_node = etree.SubElement(compare, 'file', ignore=ignore_file, id=str(output_file.id)) - last_commit_files = db.query(last_files.c.got).filter(and_( - last_files.c.regression_test_output_id == output_file.id, - last_files.c.got.isnot(None) - )).first() correct = etree.SubElement(file_node, 'correct') # Need a path that is relative to the folder we provide inside the CI environment. - if last_commit_files is None: - log.debug(f"Selecting original file for RT #{regression_test.id} ({category.name})") - correct.text = output_file.filename_correct - else: - correct.text = output_file.create_correct_filename(last_commit_files[0]) + # Always the approved baseline. This used to fall back to the previous run's own + # output whenever that run had recorded a mismatch, which made the reference roll + # forward on its own: a behavioural change was flagged once and then silently + # adopted, and because a passing run records no output, the run after that fell + # back here again and the same test failed anew. See issue #1173. + correct.text = output_file.filename_correct expected = etree.SubElement(file_node, 'expected') expected.text = output_file.filename_expected(regression_test.sample.sha) @@ -313,58 +1171,571 @@ def kvm_processor(app, db, kvm_name, platform, repository, delay) -> None: save_xml_to_file(multi_test, base_folder, 'TestAll.xml') - # 2) Download the artifact for the current build from GitHub Actions - artifact_saved = False - base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', kvm_name, 'unsafe-ccextractor') - Path(base_folder).mkdir(parents=True, exist_ok=True) - artifacts = repository.actions.artifacts().get(per_page="100")["artifacts"] - page_number = 2 - if test.platform == TestPlatform.linux: - artifact_name = Artifact_names.linux - else: - artifact_name = Artifact_names.windows + # 2) Download the artifact for the current build from GitHub Actions + # Use the improved artifact search function that handles pagination properly + base_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'vm_data', gcp_instance_name, 'unsafe-ccextractor') + Path(base_folder).mkdir(parents=True, exist_ok=True) + + log.info(f"Test {test.id}: Searching for {test.platform.value} artifact for commit {test.commit[:8]}") + artifact = find_artifact_for_commit(repository, test.commit, test.platform, log) + + if artifact is None: + # Use diagnostic function to determine if this is retryable + error_detail, is_retryable = _diagnose_missing_artifact(repository, test.commit, test.platform, log) + + if is_retryable: + # Build is still in progress or workflow is queued - don't mark as failed + # Just return and let the next cron cycle retry this test + log.info(f"Test {test.id}: {error_detail}") + return + + # Permanent failure - mark the test as failed + log.critical(f"Test {test.id}: Could not find artifact for commit {test.commit[:8]}: {error_detail}") + mark_test_failed(db, test, repository, error_detail) + return + + log.info(f"Test {test.id}: Found artifact '{artifact.name}' (ID: {artifact.id})") + artifact_url = artifact.archive_download_url + + try: + auth_header = f"token {bot_token}" + r = requests.get( + artifact_url, + headers={"Authorization": auth_header}, + timeout=ARTIFACT_DOWNLOAD_TIMEOUT + ) + except requests.exceptions.Timeout: + log.critical(f"Test {test.id}: Artifact download timed out after {ARTIFACT_DOWNLOAD_TIMEOUT}s") + mark_test_failed(db, test, repository, "Artifact download timed out") + return + except Exception as e: + log.critical(f"Test {test.id}: Could not fetch artifact: {e}") + mark_test_failed(db, test, repository, f"Artifact download failed: {e}") + return + + if r.status_code != 200: + log.critical(f"Test {test.id}: Could not fetch artifact, response code: {r.status_code}") + mark_test_failed(db, test, repository, f"Artifact download failed: HTTP {r.status_code}") + return + + zip_path = os.path.join(base_folder, 'ccextractor.zip') + with open(zip_path, 'wb') as f: + f.write(r.content) + with zipfile.ZipFile(zip_path, 'r') as artifact_zip: + artifact_zip.extractall(base_folder) + + log.info(f"Test {test.id}: Artifact downloaded and extracted successfully") + + zone = config.get('ZONE', '') + project_id = config.get('PROJECT_NAME', '') + operation = create_instance(compute, project_id, zone, test, full_url) + + # Check if the create_instance call itself returned an error (synchronous failure) + if 'error' in operation: + error_msg = parse_gcp_error(operation) + if is_retryable_gcp_error(operation): + # Transient error - leave test pending for retry on next cron run + log.warning(f"Test {test.id}: VM creation hit retryable error, will retry: {error_msg}") + return + log.error(f"Error creating test instance for test {test.id}, result: {operation}") + mark_test_failed(db, test, repository, error_msg) + return + + # Wait for the VM creation operation to complete (or timeout) + # This catches quota errors and other failures that occur shortly after the + # insert request is accepted. Without this check, tests can get stuck forever + # when VM creation fails but a GcpInstance record is created. + op_name = operation.get('name', 'unknown') + log.info(f"Test {test.id}: VM creation initiated (op: {op_name}), waiting for verification...") + + result = wait_for_operation(compute, project_id, zone, op_name, max_wait=GCP_VM_CREATE_VERIFY_TIMEOUT) + + # Check if operation completed with an error (e.g., QUOTA_EXCEEDED) + if 'error' in result: + error_msg = parse_gcp_error(result) + log.error(f"Test {test.id}: VM creation failed: {error_msg}") + log.error(f"Test {test.id}: Full GCP response: {result}") + if is_retryable_gcp_error(result): + # Transient error - leave test pending for retry on next cron run + log.info(f"Test {test.id}: Error is retryable, will retry on next cron run") + return + mark_test_failed(db, test, repository, error_msg) + return + + # Check for timeout - operation still running, which is OK for slow VM creation + if result.get('status') == 'TIMEOUT': + log.warning(f"Test {test.id}: VM creation still in progress after {GCP_VM_CREATE_VERIFY_TIMEOUT}s, " + "recording instance optimistically") + else: + log.info(f"Test {test.id}: VM creation verified successfully") + + # VM creation succeeded (or is still in progress) - record the instance + db.add(status) + if not safe_db_commit(db, f"recording GCP instance for test {test.id}"): + log.error(f"Failed to record GCP instance for test {test.id}, but VM creation was initiated") + + +def create_instance(compute, project, zone, test, reportURL) -> Dict: + """ + Start an instance and pass the VM metadata. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the new VM instance + :type zone: str + :param test: The test for which VM is to be started + :type test: mod_test.models.Test + :param reportURL: Test-specific URL link for reporting progress to server + :type reportURL: str + :return: Create operation details after VM creation + :rtype: Dict + """ + from run import config + + if test.platform == TestPlatform.linux: + image_response = compute.images().getFromFamily(project=config.get('LINUX_INSTANCE_PROJECT_NAME', ''), + family=config.get('LINUX_INSTANCE_FAMILY_NAME', '')).execute() + with open(os.path.join( + config.get('INSTALL_FOLDER', ''), 'install', 'ci-vm', + 'ci-linux', 'startup-script.sh'), 'r') as f: + startup_script = f.read() + metadata_items = [ + {'key': 'startup-script', 'value': startup_script}, + {'key': 'reportURL', 'value': reportURL}, + {'key': 'bucket', 'value': config.get('GCS_BUCKET_NAME', '')} + ] + elif test.platform == TestPlatform.windows: + image_response = compute.images().getFromFamily(project=config.get('WINDOWS_INSTANCE_PROJECT_NAME', ''), + family=config.get('WINDOWS_INSTANCE_FAMILY_NAME', '')).execute() + with open(os.path.join( + config.get('INSTALL_FOLDER', ''), 'install', 'ci-vm', + 'ci-windows', 'startup-script.ps1'), 'r') as f: + startup_script = f.read() + with open(os.path.join( + config.get('INSTALL_FOLDER', ''), + config.get('SERVICE_ACCOUNT_FILE', '')), 'r') as f: + service_account = f.read() + with open(os.path.join( + config.get('INSTALL_FOLDER', ''), 'install', 'ci-vm', + 'ci-windows', 'rclone.conf'), 'r') as f: + rclone_conf = f.read() + metadata_items = [ + {'key': 'windows-startup-script-ps1', 'value': startup_script}, + {'key': 'service_account', 'value': service_account}, + {'key': 'rclone_conf', 'value': rclone_conf}, + {'key': 'reportURL', 'value': reportURL}, + {'key': 'bucket', 'value': config.get('GCS_BUCKET_NAME', '')} + ] + source_disk_image = image_response['selfLink'] + + vm_name = f"{test.platform.value}-{test.id}" + + vm_config = get_config_for_gcp_instance(vm_name, source_disk_image, metadata_items) + + return compute.instances().insert( + project=project, + zone=zone, + body=vm_config).execute() + + +def delete_instance(compute, project, zone, vm_name) -> Dict: + """ + Delete the GCP instance with given name. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the new VM instance + :type zone: str + :param vm_name: Name of the instance to be deleted + :type vm_name: str + :return: Delete operation details after VM deletion + :rtype: Dict + """ + return compute.instances().delete( + project=project, + zone=zone, + instance=vm_name).execute() + + +def delete_instance_with_tracking(compute, project, zone, vm_name, db) -> Dict: + """ + Delete the GCP instance and track the operation for verification. + + This function wraps delete_instance() and records the pending deletion + in the database so it can be verified later. If the deletion fails, + the verify_pending_deletions() cron job will retry it. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the new VM instance + :type zone: str + :param vm_name: Name of the instance to be deleted + :type vm_name: str + :param db: Database session for recording the pending deletion + :type db: sqlalchemy.orm.scoping.scoped_session + :return: Delete operation details after VM deletion + :rtype: Dict + """ + from run import log + + try: + operation = delete_instance(compute, project, zone, vm_name) + op_name = operation.get('name', 'unknown') + + # Record the pending deletion for verification + pending = PendingDeletion(vm_name, op_name) + db.add(pending) + if not safe_db_commit(db, f"recording pending deletion for {vm_name}"): + log.warning(f"Failed to record pending deletion for {vm_name}, " + "deletion may not be verified") + + return operation + + except Exception as e: + log.error(f"Failed to initiate deletion for {vm_name}: {e}") + # Still record it so we can retry later + pending = PendingDeletion(vm_name, f"failed-{vm_name}") + db.add(pending) + safe_db_commit(db, f"recording failed deletion attempt for {vm_name}") + raise + + +def check_operation_status(compute, project, zone, operation_name) -> Dict: + """ + Check the status of a GCP operation (non-blocking). + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the operation + :type zone: str + :param operation_name: The operation name to check + :type operation_name: str + :return: Operation status dict with 'status' key (RUNNING, DONE, or ERROR) + :rtype: Dict + """ + try: + result = compute.zoneOperations().get( + project=project, + zone=zone, + operation=operation_name).execute() + return result + except Exception as e: + return {'status': 'ERROR', 'error': {'message': str(e)}} + + +def verify_pending_deletions(compute, project, zone, db) -> None: + """ + Verify pending VM deletions and retry failed ones. + + This function checks the status of all pending deletion operations. + If an operation completed successfully, it removes the tracking entry. + If an operation failed, it retries the deletion (up to MAX_RETRIES). + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the VM instances + :type zone: str + :param db: Database session + :type db: sqlalchemy.orm.scoping.scoped_session + """ + from run import log + + pending_deletions = PendingDeletion.query.all() + if not pending_deletions: + return + + log.info(f"Verifying {len(pending_deletions)} pending VM deletions") + + for pending in pending_deletions: + vm_name = pending.vm_name + op_name = pending.operation_name + + # Check if this was a failed initial deletion attempt + if op_name.startswith('failed-'): + # Retry the deletion + log.info(f"Retrying failed deletion for {vm_name}") + _retry_deletion(compute, project, zone, vm_name, pending, db, log) + continue + + # Check the operation status + result = check_operation_status(compute, project, zone, op_name) + status = result.get('status', 'UNKNOWN') + + if status == 'DONE': + # Check if operation had errors + if 'error' in result: + error_msg = result['error'].get('errors', [{}])[0].get('message', 'Unknown error') + log.warning(f"Deletion operation for {vm_name} completed with error: {error_msg}") + _retry_deletion(compute, project, zone, vm_name, pending, db, log) + else: + # Success! Remove the tracking entry + log.info(f"VM {vm_name} deletion confirmed (op: {op_name})") + db.delete(pending) + safe_db_commit(db, f"removing confirmed deletion for {vm_name}") + + elif status == 'RUNNING': + # Still in progress - check how long it's been + age_minutes = (datetime.datetime.now() - pending.created_at).total_seconds() / 60 + if age_minutes > 30: + # Operation running too long, might be stuck + log.warning(f"Deletion operation for {vm_name} running for {age_minutes:.0f} min, retrying") + _retry_deletion(compute, project, zone, vm_name, pending, db, log) + else: + log.debug(f"Deletion operation for {vm_name} still running ({age_minutes:.0f} min)") + + else: + # Unknown status or error checking - retry + log.warning(f"Unexpected status '{status}' for {vm_name} deletion, retrying") + _retry_deletion(compute, project, zone, vm_name, pending, db, log) + + +def _retry_deletion(compute, project, zone, vm_name, pending, db, log) -> None: + """ + Retry a failed VM deletion. + + :param compute: The cloud compute engine service object + :param project: The GCP project name + :param zone: Zone for the VM instance + :param vm_name: Name of the VM to delete + :param pending: The PendingDeletion record + :param db: Database session + :param log: Logger instance + """ + if pending.retry_count >= PendingDeletion.MAX_RETRIES: + log.error(f"Max retries ({PendingDeletion.MAX_RETRIES}) exceeded for {vm_name}, " + "VM may still be running and incurring charges!") + # Keep the record so scan_for_orphaned_vms can handle it + return + + pending.retry_count += 1 + log.info(f"Retry {pending.retry_count}/{PendingDeletion.MAX_RETRIES} for {vm_name}") + + try: + operation = delete_instance(compute, project, zone, vm_name) + pending.operation_name = operation.get('name', f'retry-{vm_name}') + pending.created_at = datetime.datetime.now() + safe_db_commit(db, f"updating retry for {vm_name}") + log.info(f"Retry deletion initiated for {vm_name} (op: {pending.operation_name})") + except Exception as e: + error_str = str(e) + if 'notFound' in error_str or '404' in error_str: + # VM doesn't exist anymore - success! + log.info(f"VM {vm_name} no longer exists, deletion confirmed") + db.delete(pending) + safe_db_commit(db, f"removing deletion for non-existent {vm_name}") + else: + log.error(f"Retry deletion failed for {vm_name}: {e}") + safe_db_commit(db, f"recording retry attempt for {vm_name}") + + +def scan_for_orphaned_vms(compute, project, zone, db) -> None: + """ + Scan for orphaned VMs that should have been deleted but weren't. + + This is the final safety net. It finds VMs that: + 1. Match our naming pattern (platform-testid) + 2. Don't have a corresponding GcpInstance record (test finished but VM wasn't deleted) + 3. Have a PendingDeletion record with max retries exceeded + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the VM instances + :type zone: str + :param db: Database session + :type db: sqlalchemy.orm.scoping.scoped_session + """ + from run import log + + try: + running_vms = get_running_instances(compute, project, zone) + except Exception as e: + log.error(f"Failed to list running instances: {e}") + return + + if not running_vms: + return + + orphaned_count = 0 + for vm in running_vms: + vm_name = vm['name'] + + # Only check VMs that match our naming pattern + if not is_instance_testing(vm_name): + continue + + # Check if there's a GcpInstance record (meaning test is still running) + gcp_instance = GcpInstance.query.filter(GcpInstance.name == vm_name).first() + if gcp_instance is not None: + # VM is legitimately running + continue + + # No GcpInstance record - this VM should have been deleted + # Check if we're already tracking it + pending = PendingDeletion.query.filter( + PendingDeletion.vm_name == vm_name).first() - for index, artifact in enumerate(artifacts): - if artifact['name'] == artifact_name and artifact["workflow_run"]["head_sha"] == test.commit: - artifact_url = artifact["archive_download_url"] + if pending is not None: + if pending.retry_count >= PendingDeletion.MAX_RETRIES: + # Max retries exceeded - force delete + log.warning(f"Orphan scan: Force deleting {vm_name} after max retries") + try: + operation = delete_instance(compute, project, zone, vm_name) + # Reset retry count and update operation + pending.retry_count = 0 + pending.operation_name = operation.get('name', f'force-{vm_name}') + pending.created_at = datetime.datetime.now() + safe_db_commit(db, f"force delete for orphan {vm_name}") + orphaned_count += 1 + except Exception as e: + if 'notFound' in str(e) or '404' in str(e): + db.delete(pending) + safe_db_commit(db, f"removing stale pending for {vm_name}") + else: + log.error(f"Force delete failed for {vm_name}: {e}") + # else: still being handled by verify_pending_deletions + else: + # Not tracked at all - orphaned VM found + log.warning(f"Orphan scan: Found untracked VM {vm_name}, initiating deletion") try: - auth_header = f"token {github_config['bot_token']}" - r = requests.get(artifact_url, headers={"Authorization": auth_header}) + operation = delete_instance(compute, project, zone, vm_name) + op_name = operation.get('name', f'orphan-{vm_name}') + # Track it for verification + new_pending = PendingDeletion(vm_name, op_name) + db.add(new_pending) + safe_db_commit(db, f"tracking orphan deletion for {vm_name}") + orphaned_count += 1 except Exception as e: - log.critical("Could not fetch artifact, request timed out") - return - if r.status_code != 200: - log.critical(f"Could not fetch artifact, response code: {r.status_code}") - return + if 'notFound' not in str(e) and '404' not in str(e): + log.error(f"Failed to delete orphan VM {vm_name}: {e}") - open(os.path.join(base_folder, 'ccextractor.zip'), 'wb').write(r.content) - with zipfile.ZipFile(os.path.join(base_folder, 'ccextractor.zip'), 'r') as artifact_zip: - artifact_zip.extractall(base_folder) + if orphaned_count > 0: + log.info(f"Orphan scan: Initiated deletion for {orphaned_count} orphaned VMs") - artifact_saved = True - break - if index + 1 == len(artifacts): - artifacts.extend(repository.actions.artifacts().get(per_page="100", page=str(page_number))["artifacts"]) - page_number += 1 +def get_config_for_gcp_instance(vm_name, source_disk_image, metadata_items) -> Dict: + """ + Get VM config for new VM instance. + + :param vm_name: The name of the instance to be created + :type vm_name: str + :param source_disk_image: Source disk image for new instance + :type source_disk_image: str + :param metadata_items: VM Metadata for new instance + :type metadata_items: list + :return: Config for new instance + :rtype: Dict + """ + from run import config - if not artifact_saved: - log.critical("Could not find an artifact for this commit") - return - # Power on machine - try: - vm.create() - db.add(status) - db.commit() - except libvirt.libvirtError as e: - log.critical(f"[{platform}] Failed to launch VM {kvm_name}") - log.critical(f"Information about failure: code: {e.get_error_code()}, domain: {e.get_error_domain()}, " - f"level: {e.get_error_level()}, message: {e.get_error_message()}") - except IntegrityError: - log.warn(f"[{platform}] Duplicate entry for {test.id}") + # Configure the machine + machine_type = config.get('MACHINE_TYPE', '') + + return { + 'name': vm_name, + 'machineType': machine_type, + + # Specify the boot disk and the image to use as a source. + 'disks': [ + { + 'boot': True, + 'autoDelete': True, + 'initializeParams': { + 'sourceImage': source_disk_image, + } + } + ], + + # Specify a network interface with NAT to access the public + # internet. + 'networkInterfaces': [{ + 'network': 'global/networks/default', + 'accessConfigs': [ + {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'} + ] + }], + + # Allow the instance to access cloud storage and logging. + 'serviceAccounts': [{ + 'email': 'default', + 'scopes': [ + 'https://www.googleapis.com/auth/devstorage.read_write', + 'https://www.googleapis.com/auth/logging.write' + ] + }], + + # Metadata is readable from the instance and allows you to + # pass configuration from deployment scripts to instances. + 'metadata': { + 'items': metadata_items + } + } + + +def wait_for_operation(compute, project, zone, operation, max_wait: int = GCP_OPERATION_MAX_WAIT) -> Dict: + """ + Wait for an operation to get completed with timeout. + + :param compute: The cloud compute engine service object + :type compute: googleapiclient.discovery.Resource + :param project: The GCP project name + :type project: str + :param zone: Zone for the new VM instance + :type zone: str + :param operation: Operation name for which server is waiting + :type operation: str + :param max_wait: Maximum time to wait in seconds (default: 30 minutes) + :type max_wait: int + :return: Response received after operation completion + :rtype: Dict + """ + from run import log + log.info(f"Waiting for operation {operation} to finish (max {max_wait}s)") + start_time = time.time() + poll_interval = 1.0 # Start with 1 second polling + + while True: + elapsed = time.time() - start_time + if elapsed >= max_wait: + log.error(f"Operation {operation} timed out after {elapsed:.0f} seconds") + return { + 'status': 'TIMEOUT', + 'error': { + 'errors': [{ + 'code': 'TIMEOUT', + 'message': f'Operation timed out after {max_wait} seconds' + }] + } + } + + try: + result = compute.zoneOperations().get( + project=project, + zone=zone, + operation=operation).execute() - # Close connection to libvirt - conn.close() + if result['status'] == 'DONE': + log.info(f"Operation {operation} completed in {elapsed:.0f} seconds") + return result + + except Exception as e: + log.error(f"Error checking operation status: {e}") + return {'status': 'ERROR', 'error': {'errors': [{'code': 'API_ERROR', 'message': str(e)}]}} + + # Exponential backoff with cap at 10 seconds + time.sleep(min(poll_interval, 10)) + poll_interval = min(poll_interval * 1.5, 10) def save_xml_to_file(xml_node, folder_name, file_name) -> None: @@ -385,12 +1756,12 @@ def save_xml_to_file(xml_node, folder_name, file_name) -> None: ) -def add_test_entry(db, gh_commit, commit, test_type, branch="master", pr_nr=0) -> None: +def add_test_entry(db, commit, test_type, branch="master", pr_nr=0, built_commit=None) -> None: """ Add test details entry into Test model for each platform. :param db: Database connection. - :type db: sqlalchemy.orm.scoped_session + :type db: sqlalchemy.orm.scoping.scoped_session :param gh_commit: The GitHub API call for the commit. Can be None :type gh_commit: Any :param commit: The commit hash. @@ -401,11 +1772,23 @@ def add_test_entry(db, gh_commit, commit, test_type, branch="master", pr_nr=0) - :type branch: str :param pr_nr: Pull Request number, if applicable. :type pr_nr: int + :param built_commit: SHA that CI actually built (PR merge ref), if known. + :type built_commit: Optional[str] :return: Nothing :rtype: None """ from run import log + # Validate commit hash before creating test entries + # Based on issue identified by NexionisJake in PR #937 + if not is_valid_commit_hash(commit): + log.error(f"Invalid commit hash '{commit}' - skipping test entry creation") + return + + if built_commit is not None and not is_valid_commit_hash(built_commit): + log.warning(f"Ignoring invalid built_commit '{built_commit}'") + built_commit = None + fork_url = f"%/{g.github['repository_owner']}/{g.github['repository']}.git" fork = Fork.query.filter(Fork.github.like(fork_url)).first() @@ -413,81 +1796,118 @@ def add_test_entry(db, gh_commit, commit, test_type, branch="master", pr_nr=0) - log.debug('pull request test type detected') branch = "pull_request" - linux_test = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr) + linux_test = Test(TestPlatform.linux, test_type, fork.id, branch, commit, pr_nr, + built_commit=built_commit) db.add(linux_test) - windows_test = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr) + windows_test = Test(TestPlatform.windows, test_type, fork.id, branch, commit, pr_nr, + built_commit=built_commit) db.add(windows_test) - db.commit() + if not safe_db_commit(db, f"adding test entries for commit {commit[:7]}"): + log.error(f"Failed to add test entries for commit {commit}") -def schedule_test(gh_commit, commit, test_type, branch="master", pr_nr=0) -> None: +def schedule_test(gh_commit: Commit.Commit) -> None: """ - Post status to GitHub as waiting for Github Actions completion. + Post status to GitHub as waiting for GitHub Actions completion. :param gh_commit: The GitHub API call for the commit. Can be None :type gh_commit: Any - :param commit: The commit hash. - :type commit: str - :param test_type: The type of test - :type test_type: TestType - :param branch: Branch name - :type branch: str - :param pr_nr: Pull Request number, if applicable. - :type pr_nr: int :return: Nothing :rtype: None """ - from run import log - - if test_type == TestType.pull_request: - log.debug('pull request test type detected') - branch = "pull_request" - if gh_commit is not None: for platform in TestPlatform: - try: - gh_commit.post( - state=Status.PENDING, - description="Waiting for actions to complete", - context=f"CI - {platform.value}", - ) - except ApiError as a: - log.critical(f'Could not post to GitHub! Response: {a.response}') + status_description = "Waiting for actions to complete" + update_status_on_github(gh_commit, Status.PENDING, status_description, f"CI - {platform.value}") + + +_GITHUB_NOT_SET: Any = getattr(GithubObject, 'NotSet') + +def update_status_on_github(gh_commit: Commit.Commit, state, description, context, + target_url: Any = _GITHUB_NOT_SET): + """ + Update status on GitHub. + + :param gh_commit: The GitHub API call for the commit. Can be None + :type gh_commit: Any + :param state: The test status. + :type state: Status + :param description: Description of test status. + :type description: str + :param context: Context for Github status. + :type context: str + :param target_url: Platform url for test status + :type target_url: _NotSetType() | str + """ + from run import log + + try: + gh_commit.create_status( + state=state, + description=description, + context=context, + target_url=target_url + ) + except GithubException as a: + log.critical(f'Could not post to GitHub! Response: {a.data}') -def deschedule_test(gh_commit, platform, message="Tests have been cancelled", state=Status.FAILURE) -> None: + +def deschedule_test(gh_commit: Commit.Commit, commit=None, test_type=None, platform=None, branch="master", + message="Tests have been cancelled", state=Status.FAILURE, test=None, db=None) -> None: """ - Post status to GitHub (default: as failure due to Github Actions incompletion). + Post status to GitHub (default: as failure due to GitHub Actions incompletion). :param gh_commit: The GitHub API call for the commit. Can be None :type gh_commit: Any :param commit: The commit hash. :type commit: str + :param test_type: The type of test + :type test_type: TestType :param platform: The platform name :type platform: TestPlatform - :param message: The message to be posted to GitHub - :type message: str :param branch: Branch name :type branch: str - :param pr_nr: Pull Request number, if applicable. - :type pr_nr: int + :param message: The message to be posted to GitHub + :type message: str + :param state: The status badge of the test + :type state: Status + :param test: The test which is to be canceled (optional) + :type state: Test + :param db: db session + :type db: sqlalchemy.orm.scoping.scoped_session :return: Nothing :rtype: None """ from run import log - if gh_commit is not None: - try: - gh_commit.post( - state=state, - description=message, - context=f"CI - {platform.value}", - ) - except ApiError as a: - log.critical(f'Could not post to GitHub! Response: {a.response}') - + if test_type == TestType.pull_request: + log.debug('pull request test type detected') + branch = "pull_request" -def queue_test(gh_commit, commit, test_type, platform, branch="master", pr_nr=0) -> None: + if test is None: + fork_url = f"%/{g.github['repository_owner']}/{g.github['repository']}.git" + fork = Fork.query.filter(Fork.github.like(fork_url)).first() + test = Test.query.filter(and_(Test.platform == platform, + Test.commit == commit, + Test.fork_id == fork.id, + Test.test_type == test_type, + Test.branch == branch, + )).first() + + if test is not None: + progress = TestProgress(test.id, TestStatus.canceled, message, datetime.datetime.now()) + db = db or g.db + db.add(progress) + if not safe_db_commit(db, f"descheduling test {test.id}"): + log.error(f"Failed to deschedule test {test.id}") + return + + if gh_commit is not None: + update_status_on_github(gh_commit, state, message, f"CI - {test.platform.value}") + + +def queue_test(gh_commit: Commit.Commit, commit, test_type, platform, branch="master", pr_nr=0) -> None: """ Store test details into Test model for each platform, and post the status to GitHub. @@ -522,18 +1942,27 @@ def queue_test(gh_commit, commit, test_type, platform, branch="master", pr_nr=0) Test.branch == branch, Test.pr_nr == pr_nr )).first() + + if platform_test is None: + log.error(f"No test record found for commit {commit[:8]}, platform {platform.value}, " + f"test_type {test_type}, pr_nr {pr_nr}. " + "This may indicate the pull_request webhook was not processed.") + return + add_customized_regression_tests(platform_test.id) if gh_commit is not None: - try: - gh_commit.post( - state=Status.PENDING, - description="Tests queued", - context=f"CI - {platform_test.platform.value}", - target_url=url_for('test.by_id', test_id=platform_test.id, _external=True) - ) - except ApiError as a: - log.critical(f'Could not post to GitHub! Response: {a.response}') + # Check if test already has progress (started or completed) + # If so, don't overwrite the GitHub status with "Tests queued" + # This prevents the bug where a completed test gets its status overwritten + # when a later webhook triggers queue_test for the same commit + if len(platform_test.progress) > 0: + log.info(f"Test {platform_test.id} already has progress, not posting 'Tests queued' status") + return + + target_url = url_for('test.by_id', test_id=platform_test.id, _external=True) + status_context = f"CI - {platform_test.platform.value}" + update_status_on_github(gh_commit, Status.PENDING, "Tests queued", status_context, target_url) log.debug("Created tests, waiting for cron...") @@ -626,16 +2055,23 @@ def start_ci(): g.log.warning(f'CI payload is empty') abort(abort_code) - gh = GitHub(access_token=g.github['bot_token']) - repository = gh.repos(g.github['repository_owner'])(g.github['repository']) + if not g.github['bot_token']: + g.log.error('GitHub token not configured, cannot process webhook') + return json.dumps({'msg': 'GitHub token not configured'}), 500 + + gh = Github(auth=Auth.Token(g.github['bot_token'])) + repository = gh.get_repo(f"{g.github['repository_owner']}/{g.github['repository']}") if event == "push": g.log.debug('push event detected') - if 'after' in payload: + if 'after' in payload and payload["ref"] == "refs/heads/master": commit_hash = payload['after'] - github_status = repository.statuses(commit_hash) # Update the db to the new last commit - ref = repository.git().refs('heads/master').get() + try: + ref = retry_with_backoff(lambda: repository.get_git_ref("heads/master")) + except GithubException as e: + g.log.error(f"Failed to get git ref after retries: {e}") + return 'ERROR' last_commit = GeneralData.query.filter(GeneralData.key == 'last_commit').first() for platform in TestPlatform.values(): commit_name = 'fetch_commit_' + platform @@ -645,9 +2081,10 @@ def start_ci(): prev_commit = GeneralData(commit_name, last_commit.value) g.db.add(prev_commit) - last_commit.value = ref['object']['sha'] - g.db.commit() - add_test_entry(g.db, github_status, commit_hash, TestType.commit) + last_commit.value = ref.object.sha + if not safe_db_commit(g.db, "updating last commit"): + return 'ERROR' + add_test_entry(g.db, commit_hash, TestType.commit) else: g.log.warning('Unknown push type! Dumping payload for analysis') g.log.warning(payload) @@ -656,10 +2093,14 @@ def start_ci(): g.log.debug('Pull Request event detected') # If it's a valid PR, run the tests pr_nr = payload['pull_request']['number'] - if payload['action'] in ['opened', 'synchronize', 'reopened']: + + action = payload['action'] + is_active = action in ['opened', 'synchronize', 'reopened'] + is_inactive = action in ['closed'] + + if is_active: try: commit_hash = payload['pull_request']['head']['sha'] - github_status = repository.statuses(commit_hash) except KeyError: g.log.error("Didn't find a SHA value for a newly opened PR!") g.log.error(payload) @@ -670,28 +2111,36 @@ def start_ci(): if BlockedUsers.query.filter(BlockedUsers.user_id == user_id).first() is not None: g.log.warning("User Blacklisted") return 'ERROR' - add_test_entry(g.db, github_status, commit_hash, TestType.pull_request, pr_nr=pr_nr) + try: + pr = retry_with_backoff(lambda: repository.get_pull(number=pr_nr)) + if pr.mergeable is not False: + built_commit = _merge_commit_from_pr(pr, payload.get('pull_request')) + add_test_entry(g.db, commit_hash, TestType.pull_request, pr_nr=pr_nr, + built_commit=built_commit) + except GithubException as e: + g.log.error(f"Failed to get PR {pr_nr} after retries: {e}") + + elif is_inactive: + pr_action = 'closed' if action == 'closed' else 'converted to draft' + g.log.debug(f'PR was {pr_action}, no after hash available') - elif payload['action'] == 'closed': - g.log.debug('PR was closed, no after hash available') # Cancel running queue tests = Test.query.filter(Test.pr_nr == pr_nr).all() for test in tests: # Add cancelled status only if the test hasn't started yet if len(test.progress) > 0: continue - progress = TestProgress(test.id, TestStatus.canceled, "PR closed", datetime.datetime.now()) + progress = TestProgress(test.id, TestStatus.canceled, f"PR {pr_action}", datetime.datetime.now()) g.db.add(progress) - g.db.commit() + if not safe_db_commit(g.db, f"canceling test {test.id} for closed PR"): + continue + gh_commit = repository.get_commit(test.commit) # If test run status exists, mark them as cancelled - for status in repository.commits(test.commit).status.get()["statuses"]: - if status["context"] == f"CI - {test.platform.value}": - repository.statuses(test.commit).post( - state=Status.FAILURE, - description="Tests cancelled", - context=f"CI - {test.platform.value}", - target_url=url_for('test.by_id', test_id=test.id, _external=True) - ) + for status in gh_commit.get_statuses(): + if status.context == f"CI - {test.platform.value}": + target_url = url_for('test.by_id', test_id=test.id, _external=True) + update_status_on_github(gh_commit, Status.FAILURE, "Tests canceled", + status.context, target_url=target_url) elif event == "issues": g.log.debug('issues event detected') @@ -710,7 +2159,7 @@ def start_ci(): if issue is not None: issue.title = issue_title issue.status = issue_data['state'] - g.db.commit() + safe_db_commit(g.db, f"updating issue {issue_id}") elif event == "release": g.log.debug("Release webhook triggered") @@ -725,35 +2174,81 @@ def start_ci(): elif action in ["deleted", "unpublished"]: g.log.debug("Received delete/unpublished action") CCExtractorVersion.query.filter_by(version=release_version).delete() - g.db.commit() - g.log.info(f"Successfully deleted release {release_version} on {action} action") + if not safe_db_commit(g.db, f"deleting release {release_version}"): + g.log.error(f"Failed to delete release {release_version}") + else: + g.log.info(f"Successfully deleted release {release_version} on {action} action") elif action in ["edited", "published"]: g.log.debug(f"Latest release version is {release_version}") - release_commit = GeneralData.query.filter(GeneralData.key == 'last_commit').first().value release_date = release_data['published_at'] + + # Get commit hash from the release tag via GitHub API + # This is more reliable than using last_commit which may be stale + # Based on issue identified by NexionisJake in PR #937 + release_commit = None + tag_name = release_data['tag_name'] + try: + tag_ref = repository.get_git_ref(f"tags/{tag_name}") + if tag_ref.object.type == "tag": + # Annotated tag - need to get the underlying commit + tag_obj = repository.get_git_tag(tag_ref.object.sha) + release_commit = tag_obj.object.sha + else: + # Lightweight tag - points directly to commit + release_commit = tag_ref.object.sha + g.log.debug(f"Got commit {release_commit} from tag {tag_name}") + except GithubException as e: + g.log.warning(f"Failed to get commit from tag {tag_name}: {e}") + + # Fallback to last_commit if tag lookup failed + if not is_valid_commit_hash(release_commit): + last_commit_data = GeneralData.query.filter( + GeneralData.key == 'last_commit').first() + if last_commit_data and is_valid_commit_hash(last_commit_data.value): + release_commit = last_commit_data.value + g.log.warning(f"Using fallback last_commit: {release_commit}") + + # Validate we have a valid commit hash + if not is_valid_commit_hash(release_commit): + g.log.error(f"Cannot determine valid commit for release {release_version}") + return json.dumps({'msg': 'Invalid commit hash for release'}) + if action == "edited": - release = CCExtractorVersion.query.filter(CCExtractorVersion.version == release_version).one() - release.released = datetime.datetime.strptime(release_date, '%Y-%m-%dT%H:%M:%SZ').date() + release = CCExtractorVersion.query.filter( + CCExtractorVersion.version == release_version).one() + release.released = datetime.datetime.strptime( + release_date, '%Y-%m-%dT%H:%M:%SZ').date() release.commit = release_commit else: release = CCExtractorVersion(release_version, release_date, release_commit) g.db.add(release) - g.db.commit() - g.log.info(f"Successfully updated release version with webhook action '{action}'") - # adding test corresponding to last commit to the baseline regression results - # this is not altered when a release is deleted or unpublished since it's based on commit + if not safe_db_commit(g.db, f"updating release {release_version}"): + g.log.error(f"Failed to update release {release_version}") + return json.dumps({'msg': 'ERROR'}) + g.log.info(f"Release {release_version} updated with commit {release_commit}") + + # Update baseline regression results for this release + # Only proceed if we have a test for this commit test = Test.query.filter(and_(Test.commit == release_commit, Test.platform == TestPlatform.linux)).first() - test_result_file = g.db.query(TestResultFile).filter(TestResultFile.test_id == test.id).subquery() - test_result = g.db.query(TestResult).filter(TestResult.test_id == test.id).subquery() - g.db.query(RegressionTestOutput.correct).filter( - and_(RegressionTestOutput.regression_id == test_result_file.c.regression_test_id, - test_result_file.c.got is not None)).values(test_result_file.c.got) - g.db.query(RegressionTest.expected_rc).filter( - RegressionTest.id == test_result.c.regression_test_id - ).values(test_result.c.expected_rc) - g.db.commit() - g.log.info("Successfully added tests for latest release!") + if test is not None: + test_result_file = g.db.query(TestResultFile).filter( + TestResultFile.test_id == test.id).subquery() + test_result = g.db.query(TestResult).filter( + TestResult.test_id == test.id).subquery() + g.db.query(RegressionTestOutput.correct).filter( + and_(RegressionTestOutput.regression_id == test_result_file.c.regression_test_id, + test_result_file.c.got is not None)).values(test_result_file.c.got) + g.db.query(RegressionTest.expected_rc).filter( + RegressionTest.id == test_result.c.regression_test_id + ).values(test_result.c.expected_rc) + if safe_db_commit(g.db, "updating baseline regression results"): + g.log.info("Successfully updated baseline tests for release!") + else: + g.log.error("Failed to update baseline regression results") + else: + g.log.warning(f"No test found for commit {release_commit} - " + "baseline update skipped") else: g.log.warning(f"Unsupported release action: {action}") @@ -762,43 +2257,90 @@ def start_ci(): if workflow_name in [Workflow_builds.LINUX, Workflow_builds.WINDOWS]: g.log.debug('workflow_run event detected') commit_hash = payload['workflow_run']['head_sha'] - github_status = repository.statuses(commit_hash) + github_status = repository.get_commit(commit_hash) if payload['action'] == "completed": is_complete = True has_failed = False builds = {"linux": False, "windows": False} - for workflow in repository.actions.runs.get( + + # NOTE: Using this workaround because workflow name cannot be accessed using PyGitHub + # https://github.com/PyGithub/PyGithub/issues/2276 + workflow = defaultdict(lambda: None) + for active_workflow in repository.get_workflows(): + workflow[active_workflow.id] = active_workflow.name + + for workflow_run in repository.get_workflow_runs( event=payload['workflow_run']['event'], actor=payload['sender']['login'], branch=payload['workflow_run']['head_branch'] - )['workflow_runs']: - if workflow['head_sha'] == commit_hash: - if workflow['status'] == "completed": - if workflow['conclusion'] != "success": + ): + workflow_run_name = workflow[workflow_run.workflow_id] + if workflow_run_name not in [Workflow_builds.LINUX, Workflow_builds.WINDOWS]: + continue + if workflow_run.head_sha == commit_hash: + if workflow_run.status == "completed": + if workflow_run.conclusion != "success": has_failed = True break - if workflow['name'] == Workflow_builds.LINUX: + if workflow_run_name == Workflow_builds.LINUX: builds["linux"] = True - elif workflow['name'] == Workflow_builds.WINDOWS: + elif workflow_run_name == Workflow_builds.WINDOWS: builds["windows"] = True - elif workflow['status'] != "completed": + else: is_complete = False break if has_failed: # no runs to be scheduled since build failed - deschedule_test(github_status, TestPlatform.linux, + if payload['workflow_run']['event'] == "pull_request": + test_type = TestType.pull_request + else: + test_type = TestType.commit + deschedule_test(github_status, commit_hash, test_type, TestPlatform.linux, message="Cancelling tests as Github Action(s) failed") - deschedule_test(github_status, TestPlatform.windows, + deschedule_test(github_status, commit_hash, test_type, TestPlatform.windows, message="Cancelling tests as Github Action(s) failed") elif is_complete: + # CRITICAL: Verify artifacts exist before queuing tests + # This prevents the race condition where tests are queued before + # artifacts are available, causing "No build artifact found" errors + # See: https://github.com/CCExtractor/sample-platform/issues/XXX + artifacts_available = verify_artifacts_exist(repository, commit_hash, g.log) + g.log.info(f"Artifact verification for {commit_hash[:8]}: " + f"Linux={artifacts_available['linux']}, Windows={artifacts_available['windows']}") + + # Override builds dict if artifacts are not available + # This ensures we don't queue tests for which artifacts don't exist + # Determine test type for artifact verification failures + artifact_fail_test_type = (TestType.pull_request + if payload['workflow_run']['event'] == "pull_request" + else TestType.commit) + + if builds['linux'] and not artifacts_available['linux']: + g.log.error(f"Linux workflow succeeded but artifact not found for {commit_hash[:8]}! " + "This may indicate a GitHub API caching issue.") + deschedule_test(github_status, commit_hash, artifact_fail_test_type, + TestPlatform.linux, + message="Build succeeded but artifact not yet available - please retry", + state=Status.ERROR) + builds['linux'] = False + + if builds['windows'] and not artifacts_available['windows']: + g.log.error(f"Windows workflow succeeded but artifact not found for {commit_hash[:8]}! " + "This may indicate a GitHub API caching issue.") + deschedule_test(github_status, commit_hash, artifact_fail_test_type, + TestPlatform.windows, + message="Build succeeded but artifact not yet available - please retry", + state=Status.ERROR) + builds['windows'] = False + if payload['workflow_run']['event'] == "pull_request": # In case of pull request run tests only if it is still in an open state # and user is not blacklisted - for pull_request in repository.pulls.get(state="open"): - if pull_request['head']['sha'] == commit_hash: - user_id = pull_request['user']['id'] + for pull_request in repository.get_pulls(state='open'): + if pull_request.head.sha == commit_hash: + user_id = pull_request.user.id if BlockedUsers.query.filter(BlockedUsers.user_id == user_id).first() is not None: g.log.warning("User Blacklisted") github_status.post( @@ -810,37 +2352,39 @@ def start_ci(): return 'ERROR' if builds['linux']: queue_test(github_status, commit_hash, TestType.pull_request, - TestPlatform.linux, pr_nr=pull_request['number']) + TestPlatform.linux, pr_nr=pull_request.number) else: - deschedule_test(github_status, TestPlatform.linux, - message="Not ran - no code changes", state=Status.SUCCESS) + deschedule_test(github_status, commit_hash, TestType.pull_request, + TestPlatform.linux, message="Not ran - no code changes", + state=Status.SUCCESS) if builds['windows']: queue_test(github_status, commit_hash, TestType.pull_request, - TestPlatform.windows, pr_nr=pull_request['number']) + TestPlatform.windows, pr_nr=pull_request.number) else: - deschedule_test(github_status, TestPlatform.windows, - message="Not ran - no code changes", state=Status.SUCCESS) + deschedule_test(github_status, commit_hash, TestType.pull_request, + TestPlatform.windows, message="Not ran - no code changes", + state=Status.SUCCESS) return json.dumps({'msg': 'EOL'}) # Either PR head commit was updated or PR was closed, therefore cancelling tests - deschedule_test(github_status, TestPlatform.linux, - message="Tests cancelled", state=Status.FAILURE) - deschedule_test(github_status, TestPlatform.windows, - message="Tests cancelled", state=Status.FAILURE) + deschedule_test(github_status, commit_hash, TestType.pull_request, TestPlatform.linux, + message="Tests canceled", state=Status.FAILURE) + deschedule_test(github_status, commit_hash, TestType.pull_request, TestPlatform.windows, + message="Tests canceled", state=Status.FAILURE) else: if builds['linux']: queue_test(github_status, commit_hash, TestType.commit, TestPlatform.linux) else: - deschedule_test(github_status, TestPlatform.linux, + deschedule_test(github_status, commit_hash, TestType.commit, TestPlatform.linux, message="Not ran - no code changes", state=Status.SUCCESS) if builds['windows']: queue_test(github_status, commit_hash, TestType.commit, TestPlatform.windows) else: - deschedule_test(github_status, TestPlatform.windows, + deschedule_test(github_status, commit_hash, TestType.commit, TestPlatform.windows, message="Not ran - no code changes", state=Status.SUCCESS) elif payload['action'] == 'requested': - schedule_test(github_status, commit_hash, TestType.commit) + schedule_test(github_status) else: g.log.warning('Unknown action type in workflow_run! Dumping payload for analysis') g.log.warning(payload) @@ -851,7 +2395,7 @@ def start_ci(): return json.dumps({'msg': 'EOL'}) -def update_build_badge(status, test) -> None: +def update_build_badge(status, test, test_results=None) -> None: """ Build status badge for current test to be displayed on sample-platform. @@ -859,6 +2403,8 @@ def update_build_badge(status, test) -> None: :type status: str :param test: current commit that is tested :type test: Test + :param test_results: pre-computed results from get_test_results; if None, fetched internally + :type test_results: list | None :return: null :rtype: null """ @@ -869,6 +2415,18 @@ def update_build_badge(status, test) -> None: shutil.copyfile(original_location, build_status_location) g.log.info('Build badge updated successfully!') + if test_results is None: + test_results = get_test_results(test) + test_ids_to_update = [] + for category_results in test_results: + test_ids_to_update.extend([test['test'].id for test in category_results['tests'] if not test['error']]) + + g.db.query(RegressionTest).filter(RegressionTest.id.in_(test_ids_to_update)).update( + {f"last_passed_on_{test.platform.value}": test.id}, synchronize_session=False + ) + if not safe_db_commit(g.db, "updating last passed regression tests"): + g.log.error("Failed to update last passed regression tests") + @mod_ci.route('/progress-reporter//', methods=['POST']) def progress_reporter(test_id, token): @@ -890,26 +2448,31 @@ def progress_reporter(test_id, token): if 'type' in request.form: if request.form['type'] == 'progress': - log.info('[PROGRESS_REPORTER] Progress reported') + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Progress reported') if not progress_type_request(log, test, test_id, request): return "FAIL" elif request.form['type'] == 'equality': - log.info('[PROGRESS_REPORTER] Equality reported') + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Equality reported') equality_type_request(log, test_id, test, request) elif request.form['type'] == 'logupload': - log.info('[PROGRESS_REPORTER] Log upload') + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Log upload') if not upload_log_type_request(log, test_id, repo_folder, test, request): return "EMPTY" elif request.form['type'] == 'upload': - log.info('[PROGRESS_REPORTER] File upload') + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] File upload') if not upload_type_request(log, test_id, repo_folder, test, request): return "EMPTY" + elif request.form['type'] == 'artifactupload': + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Artifact upload') + if not upload_artifact_type_request(log, test_id, repo_folder, test, request): + return "EMPTY" + elif request.form['type'] == 'finish': - log.info('[PROGRESS_REPORTER] Test finished') + log.info(f'[PROGRESS_REPORTER][Test: {test_id}] Test finished') finish_type_request(log, test_id, test, request) else: return "FAIL" @@ -937,45 +2500,43 @@ def progress_type_request(log, test, test_id, request) -> bool: message = request.form['message'] if len(test.progress) != 0: - last_status = TestStatus.progress_step(test.progress[-1].status) - - if last_status in [TestStatus.completed, TestStatus.canceled]: + last_progress_status = test.progress[-1].status + # Compare enum to enum. progress_step() returns an int index, which + # never matches TestStatus.completed / TestStatus.canceled. + if last_progress_status in [TestStatus.completed, TestStatus.canceled]: return False + last_status = TestStatus.progress_step(last_progress_status) + if last_status > current_status: status = TestStatus.canceled # type: ignore message = "Duplicate Entries" if last_status < current_status: - # get KVM start time for finding KVM preparation time - kvm_entry = Kvm.query.filter(Kvm.test_id == test_id).first() + # get GCP VM instance start time for finding GCP VM instance preparation time + gcp_instance_entry = GcpInstance.query.filter(GcpInstance.test_id == test_id).first() - if status == TestStatus.building: - log.info('test preparation finished') + if status == TestStatus.testing: + log.info(f'[Test: {test_id}] Preparation finished') prep_finish_time = datetime.datetime.now() # save preparation finish time - kvm_entry.timestamp_prep_finished = prep_finish_time - g.db.commit() + gcp_instance_entry.timestamp_prep_finished = prep_finish_time + safe_db_commit(g.db, f"saving prep finish time for test {test_id}") # set time taken in seconds to do preparation - time_diff = (prep_finish_time - kvm_entry.timestamp).total_seconds() + time_diff = (prep_finish_time - gcp_instance_entry.timestamp).total_seconds() set_avg_time(test.platform, "prep", time_diff) - elif status == TestStatus.testing: - log.info('test build procedure finished') - build_finish_time = datetime.datetime.now() - # save build finish time - kvm_entry.timestamp_build_finished = build_finish_time - g.db.commit() - # set time taken in seconds to do preparation - time_diff = (build_finish_time - kvm_entry.timestamp_prep_finished).total_seconds() - set_avg_time(test.platform, "build", time_diff) - progress = TestProgress(test.id, status, message) g.db.add(progress) - g.db.commit() + if not safe_db_commit(g.db, f"adding progress for test {test_id}"): + return False - gh = GitHub(access_token=g.github['bot_token']) - repository = gh.repos(g.github['repository_owner'])(g.github['repository']) + if not g.github['bot_token']: + log.error('GitHub token not configured, cannot update status on GitHub') + return True + + gh = Github(auth=Auth.Token(g.github['bot_token'])) + repository = gh.get_repo(f"{g.github['repository_owner']}/{g.github['repository']}") # Store the test commit for testing in case of commit if status == TestStatus.completed and is_main_repo(test.fork.github): commit_name = 'fetch_commit_' + test.platform.value @@ -986,41 +2547,108 @@ def progress_type_request(log, test, test_id, request) -> bool: if test.test_type == TestType.commit and test.id > fetch_commit.id: commit.value = test.commit - g.db.commit() + safe_db_commit(g.db, f"updating fetch commit for {test.platform.value}") + + # Post status update + state = Status.PENDING + target_url = url_for('test.by_id', test_id=test.id, _external=True) + context = f"CI - {test.platform.value}" + + if status == TestStatus.canceled: + state = Status.ERROR + message = 'Tests aborted due to an error; please check' + + elif status == TestStatus.completed: + test_results = get_test_results(test) + has_failures = any( + t['error'] + for category in test_results + for t in category['tests'] + ) + if has_failures: + state = Status.FAILURE + message = 'Not all tests completed successfully, please check' + else: + state = Status.SUCCESS + message = 'Tests completed' + if test.test_type == TestType.pull_request: + state = comment_pr(test) + # Update message to match the state returned by comment_pr() + # comment_pr() uses different logic: it returns SUCCESS if there are + # no NEW failures compared to master (pre-existing failures are OK) + if state == Status.SUCCESS: + message = 'All tests passed' + else: + message = 'Not all tests completed successfully, please check' + update_build_badge(state, test, test_results) + + else: + message = progress.message + + # Use retry logic for final GitHub status update to prevent stuck "pending" states + # This is critical - if this fails, the PR will show "Tests queued" forever + try: + def update_final_status(): + gh_commit = repository.get_commit(test.commit) + update_status_on_github(gh_commit, state, message, context, target_url=target_url) + + retry_with_backoff(update_final_status, max_retries=3, initial_backoff=2.0) + except Exception as e: + log.error(f"Test {test_id}: Failed to update final GitHub status after retries: {e}") - # If status is complete, remove the Kvm entry if status in [TestStatus.completed, TestStatus.canceled]: - log.debug(f"Test {test_id} has been {status}") + # Delete the current instance with tracking for verification + # We don't wait for the deletion to complete because: + # 1. Waiting can take 60+ seconds, exceeding nginx/gunicorn timeouts (502 errors) + # 2. The verify_pending_deletions() cron job will confirm deletion succeeded + # 3. All important work (test results, GitHub status) is already done + from run import config + compute = get_compute_service_object() + zone = config.get('ZONE', '') + project = config.get('PROJECT_NAME', '') + vm_name = f"{test.platform.value}-{test.id}" + try: + operation = delete_instance_with_tracking( + compute, project, zone, vm_name, g.db) + op_name = operation.get('name', 'unknown') + log.info(f"[Test: {test_id}] VM deletion initiated for {vm_name} (op: {op_name})") + except Exception as e: + log.error(f"[Test: {test_id}] Failed to delete VM {vm_name}: {e}") + + # If status is complete, remove the GCP Instance entry + if status in [TestStatus.completed, TestStatus.canceled]: + gcp_instance = GcpInstance.query.filter(GcpInstance.test_id == test_id).first() + + if gcp_instance is not None: + log.debug(f"Removing GCP Instance entry: {gcp_instance}") + g.db.delete(gcp_instance) + safe_db_commit(g.db, f"removing GCP instance for test {test_id}") + + log.debug(f"[Test: {test_id}] Test {status}") var_average = 'average_time_' + test.platform.value current_average = GeneralData.query.filter(GeneralData.key == var_average).first() average_time = 0 total_time = 0 if current_average is None: - platform_tests = g.db.query(Test.id).filter(Test.platform == test.platform).subquery() - finished_tests = g.db.query(TestProgress.test_id).filter( + platform_tests = select(Test.id).filter(Test.platform == test.platform) + finished_tests = select(TestProgress.test_id).filter( and_( TestProgress.status.in_([TestStatus.canceled, TestStatus.completed]), - TestProgress.test_id.in_(platform_tests) - ) - ).subquery() - in_progress_statuses = [TestStatus.preparation, TestStatus.completed, TestStatus.canceled] - finished_tests_progress = g.db.query(TestProgress).filter( - and_( - TestProgress.test_id.in_(finished_tests), - TestProgress.status.in_(in_progress_statuses) + TestProgress.test_id.in_(platform_tests.select()) ) - ).subquery() + ) times = g.db.query( - finished_tests_progress.c.test_id, - label('time', func.group_concat(finished_tests_progress.c.timestamp)) - ).group_by(finished_tests_progress.c.test_id).all() + TestProgress.test_id, + label('start_time', func.min(TestProgress.timestamp)), + label('end_time', func.max(TestProgress.timestamp)) + ).filter( + TestProgress.test_id.in_(finished_tests) + ).group_by(TestProgress.test_id).all() for p in times: - parts = p.time.split(',') - start = datetime.datetime.strptime(parts[0], '%Y-%m-%d %H:%M:%S') - end = datetime.datetime.strptime(parts[-1], '%Y-%m-%d %H:%M:%S') - total_time += int((end - start).total_seconds()) + duration = max(0, int((p.end_time - p.start_time).total_seconds())) + total_time += duration if len(times) != 0: average_time = total_time // len(times) @@ -1028,13 +2656,18 @@ def progress_type_request(log, test, test_id, request) -> bool: new_avg = GeneralData(var_average, average_time) log.info(f'new average time {str(average_time)} set successfully') g.db.add(new_avg) - g.db.commit() + safe_db_commit(g.db, "setting new average time") else: - all_results = TestResult.query.count() - regression_test_count = RegressionTest.query.count() - number_test = all_results / regression_test_count - updated_average = float(current_average.value) * (number_test - 1) + # Count actual finished tests for this platform + platform_tests = g.db.query(Test.id).filter(Test.platform == test.platform).subquery() + number_test = g.db.query(func.count(func.distinct(TestProgress.test_id))).filter( + and_( + TestProgress.status.in_([TestStatus.completed, TestStatus.canceled]), + TestProgress.test_id.in_(platform_tests) + ) + ).scalar() or 0 + pr = test.progress_data() end_time = pr['end'] start_time = pr['start'] @@ -1045,73 +2678,16 @@ def progress_type_request(log, test, test_id, request) -> bool: if start_time.tzinfo is not None: start_time = start_time.replace(tzinfo=None) - last_running_test = end_time - start_time - updated_average = updated_average + last_running_test.total_seconds() - current_average.value = updated_average // number_test - g.db.commit() - log.info(f'average time updated to {str(current_average.value)}') - - kvm = Kvm.query.filter(Kvm.test_id == test_id).first() - - if kvm is not None: - log.debug("Removing KVM entry") - g.db.delete(kvm) - g.db.commit() - - # Post status update - state = Status.PENDING - target_url = url_for('test.by_id', test_id=test.id, _external=True) - context = f"CI - {test.platform.value}" - - if status == TestStatus.canceled: - state = Status.ERROR - message = 'Tests aborted due to an error; please check' - - elif status == TestStatus.completed: - # Determine if success or failure - # It fails if any of these happen: - # - A crash (unexpected exit code) - # - A not None value on the "got" of a TestResultFile ( - # meaning the hashes do not match) - crashes = g.db.query(count(TestResult.exit_code)).filter( - and_( - TestResult.test_id == test.id, - TestResult.exit_code != TestResult.expected_rc - )).scalar() - results_zero_rc = g.db.query(RegressionTest.id).filter( - RegressionTest.expected_rc == 0 - ).subquery() - results = g.db.query(count(TestResultFile.got)).filter( - and_( - TestResultFile.test_id == test.id, - TestResultFile.regression_test_id.in_(results_zero_rc), - TestResultFile.got.isnot(None) - ) - ).scalar() - log.debug(f'Test {test.id} completed: {crashes} crashes, {results} results') - if crashes > 0 or results > 0: - state = Status.FAILURE - message = 'Not all tests completed successfully, please check' - - else: - state = Status.SUCCESS - message = 'Tests completed' - if test.test_type == TestType.pull_request: - comment_pr(test.id, state, test.pr_nr, test.platform.name) - update_build_badge(state, test) - - else: - message = progress.message + last_duration = max(0, int((end_time - start_time).total_seconds())) - gh_commit = repository.statuses(test.commit) - try: - gh_commit.post(state=state, description=message, context=context, target_url=target_url) - except ApiError as a: - log.error(f'Got an exception while posting to GitHub! Message: {a.message}') + if number_test > 1: + updated_average = (float(current_average.value) * (number_test - 1) + last_duration) / number_test + else: + updated_average = last_duration - if status in [TestStatus.completed, TestStatus.canceled]: - # Start next test if necessary, on the same platform - start_platforms(g.db, repository, 60, test.platform) + current_average.value = max(0, int(updated_average)) + safe_db_commit(g.db, "updating average time") + log.info(f'average time updated to {str(current_average.value)}') return True @@ -1138,7 +2714,7 @@ def equality_type_request(log, test_id, test, request): else: result_file = TestResultFile(test.id, request.form['test_id'], rto.id, rto.correct) g.db.add(result_file) - g.db.commit() + safe_db_commit(g.db, f"saving result file for test {test_id}") def upload_log_type_request(log, test_id, repo_folder, test, request) -> bool: @@ -1171,6 +2747,7 @@ def upload_log_type_request(log, test_id, repo_folder, test, request) -> bool: os.rename(temp_path, final_path) log.debug("Stored log file") + _record_built_commit_from_log(log, test, final_path) return True return False @@ -1200,30 +2777,121 @@ def upload_type_request(log, test_id, repo_folder, test, request) -> bool: if filename == '': log.warning('empty filename provided for uploading') return False - temp_path = os.path.join(repo_folder, 'TempFiles', filename) - # Save to temporary location - uploaded_file.save(temp_path) - # Get hash and check if it's already been submitted - hash_sha256 = hashlib.sha256() - with open(temp_path, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_sha256.update(chunk) - file_hash = hash_sha256.hexdigest() + temp_dir = os.path.join(repo_folder, 'TempFiles') + os.makedirs(temp_dir, exist_ok=True) filename, file_extension = os.path.splitext(filename) - final_path = os.path.join( - repo_folder, 'TestResults', f'{file_hash}{file_extension}' - ) - os.rename(temp_path, final_path) + # The uploaded name is derived from the regression test output, so it is identical + # for every test running that same regression test. Sharing one path across + # concurrent tests races: the second upload overwrites the first, the first rename + # moves the file away, and the second rename then fails with FileNotFoundError -> + # HTTP 500 -> no TestResultFile row -> the run is displayed as "No output generated + # but there should be". Worse, an upload that lands between another test's save and + # its hashing makes that test record the wrong hash, silently flipping its verdict. + # A unique temp file per upload removes both. + temp_fd, temp_path = tempfile.mkstemp(dir=temp_dir, suffix=file_extension) + os.close(temp_fd) + try: + # Save to temporary location + uploaded_file.save(temp_path) + # Get hash and check if it's already been submitted + hash_sha256 = hashlib.sha256() + with open(temp_path, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_sha256.update(chunk) + file_hash = hash_sha256.hexdigest() + results_dir = os.path.join(repo_folder, 'TestResults') + os.makedirs(results_dir, exist_ok=True) + final_path = os.path.join(results_dir, f'{file_hash}{file_extension}') + # os.replace is atomic and, unlike leaving a stale temp file behind, keeps + # TempFiles clean when two uploads hash to the same content. + os.replace(temp_path, final_path) + finally: + # No-op once the replace succeeded; on any failure it stops the temp file leaking. + if os.path.exists(temp_path): + os.remove(temp_path) rto = RegressionTestOutput.query.filter( RegressionTestOutput.id == request.form['test_file_id']).first() result_file = TestResultFile(test.id, request.form['test_id'], rto.id, rto.correct, file_hash) g.db.add(result_file) - g.db.commit() + if not safe_db_commit(g.db, f"saving test result file for test {test_id}"): + return False return True return False +# Artifact types the CI VM may upload, mapped to the fixed filenames the REST +# API expects under /test_artifacts// (see the +# system route _get_gcs_artifacts in mod_api). +ARTIFACT_TYPES = frozenset({'binary', 'coredump', 'combined_stdout'}) + + +def _artifact_target_name(artifact_type: str, platform) -> Optional[str]: + """Map an artifact type + platform to the exact filename the API resolves.""" + if artifact_type == 'binary': + return 'ccextractor' if platform == TestPlatform.linux else 'ccextractorwinfull.exe' + if artifact_type == 'coredump': + return 'coredump' + if artifact_type == 'combined_stdout': + return 'combined_stdout.log' + return None + + +def upload_artifact_type_request(log, test_id, repo_folder, test, request) -> bool: + """ + Handle the artifactupload request type for the progress reporter. + + Stores a CI artifact (binary, coredump, or combined stdout log) under + ``/test_artifacts//`` with the fixed name the + REST API expects. The target name is derived server-side from + ``artifact_type`` and the test platform, never from the uploaded filename, + so a crafted filename cannot escape the artifact directory. + + :param log: logger + :type log: Logger + :param test_id: the id of the test the artifact belongs to + :type test_id: int + :param repo_folder: SAMPLE_REPOSITORY path + :type repo_folder: str + :param test: the concerned test + :type test: Test + :param request: request parameters + :type request: Request + :return: True on success, False on validation failure + :rtype: bool + """ + artifact_type = request.form.get('artifact_type', '') + if artifact_type not in ARTIFACT_TYPES: + log.warning(f'[Test: {test_id}] Rejected artifact upload: bad artifact_type {artifact_type!r}') + return False + + if 'file' not in request.files: + log.warning(f'[Test: {test_id}] Artifact upload missing file') + return False + + uploaded_file = request.files['file'] + if secure_filename(uploaded_file.filename or '') == '': + log.warning(f'[Test: {test_id}] Artifact upload has empty filename') + return False + + target_name = _artifact_target_name(artifact_type, test.platform) + if target_name is None: + return False + + artifact_dir = os.path.join(repo_folder, 'test_artifacts', str(test.id)) + temp_dir = os.path.join(repo_folder, 'TempFiles') + os.makedirs(artifact_dir, exist_ok=True) + os.makedirs(temp_dir, exist_ok=True) + + temp_path = os.path.join(temp_dir, f'artifact_{test.id}_{target_name}') + uploaded_file.save(temp_path) + final_path = os.path.join(artifact_dir, target_name) + os.replace(temp_path, final_path) + + log.info(f'[Test: {test_id}] Stored {artifact_type} artifact at {final_path}') + return True + + def finish_type_request(log, test_id, test, request): """ Handle finish request type for progress reporter. @@ -1239,15 +2907,25 @@ def finish_type_request(log, test_id, test, request): """ log.debug(f"Finish for {test_id}/{request.form['test_id']}") regression_test = RegressionTest.query.filter(RegressionTest.id == request.form['test_id']).first() + + raw_runtime = request.form.get('runTime', 0) + try: + runtime = int(raw_runtime) + except (TypeError, ValueError): + log.warning(f"Invalid runtime '{raw_runtime}' for test {test_id}; storing 0") + runtime = 0 + + if runtime < 0: + log.warning(f"Negative runtime {runtime} for test {test_id}; clamping to 0") + runtime = 0 + result = TestResult( - test.id, regression_test.id, request.form['runTime'], + test.id, regression_test.id, runtime, request.form['exitCode'], regression_test.expected_rc ) g.db.add(result) - try: - g.db.commit() - except IntegrityError as e: - log.error(f"Could not save the results: {e}") + if not safe_db_commit(g.db, f"saving test result for test {test_id}"): + log.error(f"Could not save the results for test {test_id}") def set_avg_time(platform, process_type: str, time_taken: int) -> None: @@ -1281,83 +2959,196 @@ def set_avg_time(platform, process_type: str, time_taken: int) -> None: current_avg_count.value = str(avg_count + 1) current_average.value = str(new_average) - g.db.commit() - - -def get_info_for_pr_comment(test_id: int) -> PrCommentInfo: - """Return info about the given test id for use in a PR comment.""" - regression_testid_passed = g.db.query(TestResult.regression_test_id).outerjoin( - TestResultFile, TestResult.test_id == TestResultFile.test_id).filter( - TestResult.test_id == test_id, - TestResult.expected_rc == TestResult.exit_code, - or_( - TestResult.exit_code != 0, - and_(TestResult.exit_code == 0, - TestResult.regression_test_id == TestResultFile.regression_test_id, - or_(TestResultFile.got.is_(None), - and_( - RegressionTestOutputFiles.regression_test_output_id == TestResultFile.regression_test_output_id, - TestResultFile.got == RegressionTestOutputFiles.file_hashes - ))), - and_( - RegressionTestOutput.regression_id == TestResult.regression_test_id, - RegressionTestOutput.ignore.is_(True), - ))).subquery() - passed = g.db.query(label('category_id', Category.id), label( - 'success', count(regressionTestLinkTable.c.regression_id))).filter( - regressionTestLinkTable.c.regression_id.in_(regression_testid_passed), - Category.id == regressionTestLinkTable.c.category_id).group_by( - regressionTestLinkTable.c.category_id).subquery() - tot = g.db.query(label('category', Category.name), label('total', count(regressionTestLinkTable.c.regression_id)), - label('success', passed.c.success)).outerjoin( - passed, passed.c.category_id == Category.id).filter( - Category.id == regressionTestLinkTable.c.category_id).group_by( - regressionTestLinkTable.c.category_id).all() - regression_testid_failed = RegressionTest.query.filter(RegressionTest.id.notin_(regression_testid_passed)).all() - return PrCommentInfo(tot, regression_testid_failed) - - -def comment_pr(test_id, state, pr_nr, platform) -> None: + safe_db_commit(g.db, f"updating average {process_type} time for {platform.value}") + + +def find_ancestor_run(repository, test: Test) -> Optional[Test]: + """ + Find the newest completed run for a commit this one descends from. + + The tip of master is not always what a branch was cut from, so a comparison + against it charges the branch for whatever master did in between. Walking + back from the branch's own base answers the narrower question a reviewer is + asking: what changed *here*. + + Any GitHub failure resolves to None rather than raising -- a comment missing + one of its comparisons is worth more than no comment at all. + + :param repository: GitHub repository handle used to walk the commit history. + :type repository: Repository.Repository + :param test: The run whose ancestry should be searched. + :type test: Test + :return: The closest ancestor's completed run on the same platform, if any. + :rtype: Optional[Test] + """ + from run import log + + if repository is None: + return None + try: + if test.pr_nr: + start = repository.get_pull(number=test.pr_nr).base.sha + else: + parents = repository.get_commit(test.commit).parents + if not parents: + return None + start = parents[0].sha + ancestry = [commit.sha for commit in + itertools.islice(repository.get_commits(sha=start), ANCESTOR_SEARCH_DEPTH)] + except Exception as error: + log.warning(f"Could not resolve ancestry for test {test.id}: {type(error).__name__}: {error}") + return None + + if not ancestry: + return None + + runs = g.db.query(Test).filter(and_(Test.commit.in_(ancestry), + Test.platform == test.platform, + Test.id != test.id)).join( + TestProgress, Test.id == TestProgress.test_id).filter( + TestProgress.status == TestStatus.completed).order_by(TestProgress.id.desc()).all() + + newest_per_commit: Dict[str, Test] = {} + for run in runs: + newest_per_commit.setdefault(run.commit, run) + # Nearest ancestor first: ancestry is already in walk order. + for sha in ancestry: + if sha in newest_per_commit: + return newest_per_commit[sha] + return None + + +def _compare_against(label: str, reference: Optional[Test], current: Dict[int, comparison.TestState], + regression_tests: Dict[int, RegressionTest], + already_used: Dict[Any, str]) -> ReferenceComparison: + """ + Describe this run's results against one reference run. + + :param label: How the reference should be named to a reader. + :type label: str + :param reference: The run to compare against, or None when there is none. + :type reference: Optional[Test] + :param current: States for the run being reported on. + :type current: Dict[int, comparison.TestState] + :param regression_tests: Regression tests by id, for rendering the buckets. + :type regression_tests: Dict[int, RegressionTest] + :param already_used: Run ids already compared against, mapped to their label. + :type already_used: Dict[Any, str] + :return: The comparison, empty when there was nothing to compare against. + :rtype: ReferenceComparison + """ + if reference is None: + empty: Dict[str, List[RegressionTest]] = {verdict: [] for verdict in comparison.VERDICTS} + return ReferenceComparison(label, None, empty, {verdict: 0 for verdict in comparison.VERDICTS}) + + duplicate_of = already_used.get(reference.id) + if duplicate_of is None: + already_used[reference.id] = label + + buckets = comparison.compare(current, comparison.build_state(get_test_results(reference))) + tests = {verdict: [regression_tests[rt_id] for rt_id in ids if rt_id in regression_tests] + for verdict, ids in buckets.items()} + return ReferenceComparison(label, reference, tests, comparison.summarise(buckets), duplicate_of) + + +def get_info_for_pr_comment(test: Test, repository=None) -> PrCommentInfo: + """ + Return info about the given test for use in a PR comment. + + Pass and fail are decided against the approved output and nothing else. The + comparisons that follow do not change any verdict; they say what each + failure means relative to master and to the commit the branch was cut from, + which is what separates "this change broke it" from "it has been failing for + a month". + + :param test: The test whose report will be returned + :type test: Test + :param repository: GitHub repository handle, needed to resolve the ancestor. + :type repository: Optional[Repository.Repository] + """ + test_results = get_test_results(test) + current = comparison.build_state(test_results) + + category_stats = [] + failed_tests = [] + regression_tests: Dict[int, RegressionTest] = {} + for category_results in test_results: + passed_in_category = 0 + for entry in category_results['tests']: + regression_tests[entry['test'].id] = entry['test'] + if entry['error']: + failed_tests.append(entry['test']) + else: + passed_in_category += 1 + category_stats.append(CategoryTestInfo(category_results['category'].name, + len(category_results['tests']), passed_in_category)) + + last_test_master = g.db.query(Test).filter(Test.branch == "master", Test.test_type == TestType.commit, + Test.platform == test.platform).join( + TestProgress, Test.id == TestProgress.test_id).filter( + TestProgress.status == TestStatus.completed).order_by(TestProgress.id.desc()).first() + + already_used: Dict[Any, str] = {} + comparisons = [ + _compare_against('the tip of master', last_test_master, current, regression_tests, already_used), + _compare_against('the commit this branch was cut from', find_ancestor_run(repository, test), + current, regression_tests, already_used), + ] + + return PrCommentInfo(category_stats, failed_tests, len(current) - len(failed_tests), + len(current), comparisons, last_test_master) + + +def comment_pr(test: Test) -> str: """ Upload the test report to the GitHub PR as comment. - :param test_id: The identity of Test whose report will be uploaded - :type test_id: str - :param state: The state of the PR. - :type state: Status - :param pr_nr: PR number to which test commit is related and comment will be uploaded - :type: str - :param platform - :type: str + :param test: The test whose report will be uploaded + :type test: Test """ from run import app, log - comment_info = get_info_for_pr_comment(test_id) + test_id = test.id + platform = test.platform.name + if not g.github['bot_token']: + log.error(f"GitHub token not configured, cannot post PR comment for Test_id: {test_id}") + return Status.FAILURE + + # Resolved before the report is built, because working out which commit this + # branch was cut from needs the repository. A failure here costs that one + # comparison; the comment is still worth posting without it. + gh = None + repository = None + try: + gh = Github(auth=Auth.Token(g.github['bot_token'])) + repository = gh.get_repo(f"{g.github['repository_owner']}/{g.github['repository']}") + except Exception as e: + log.error(f"Could not reach GitHub for Test_id: {test_id} with Exception {e}") + + comment_info = get_info_for_pr_comment(test, repository) template = app.jinja_env.get_or_select_template('ci/pr_comment.txt') - message = template.render(tests=comment_info.category_stats, failed_tests=comment_info.failed_tests, - test_id=test_id, state=state, platform=platform) + message = template.render(comment_info=comment_info, test_id=test_id, platform=platform) log.debug(f"GitHub PR Comment Message Created for Test_id: {test_id}") try: - gh = GitHub(access_token=g.github['bot_token']) - repository = gh.repos(g.github['repository_owner'])(g.github['repository']) + if repository is None or gh is None: + raise RuntimeError('no GitHub repository handle') # Pull requests are just issues with code, so GitHub considers PR comments in issues - pull_request = repository.issues(pr_nr) - comments = pull_request.comments().get() - bot_name = g.github['bot_name'] - comment_id = None + pull_request = repository.get_pull(number=test.pr_nr) + comments = pull_request.get_issue_comments() + bot_name = gh.get_user().login for comment in comments: - if comment['user']['login'] == bot_name and platform in comment['body']: - comment_id = comment['id'] - break - log.debug(f"GitHub PR Comment ID Fetched for Test_id: {test_id}") - if comment_id is None: - comment = pull_request.comments().post(body=message) - comment_id = comment['id'] - else: - repository.issues().comments(comment_id).post(body=message) - log.debug(f"GitHub PR Comment ID {comment_id} Uploaded for Test_id: {test_id}") + if comment.user.login == bot_name and platform in comment.body: + comment.delete() + log.debug(f"GitHub PR old comment deleted for test_id: {test_id}") + comment = pull_request.create_issue_comment(body=message) + log.debug(f"GitHub PR Comment ID {comment.id} Uploaded for Test_id: {test_id}") except Exception as e: log.error(f"GitHub PR Comment Failed for Test_id: {test_id} with Exception {e}") + # The verdict is whether the output matched what was approved, and nothing + # else. The comparisons in the comment explain a failure; they never excuse + # one, because a baseline that no longer matches reality is a thing to fix + # rather than a thing to pass. + return Status.SUCCESS if len(comment_info.failed_tests) == 0 else Status.FAILURE @mod_ci.route('/show_maintenance') @@ -1411,37 +3202,40 @@ def blocked_users(): blocked_user = BlockedUsers(add_user_form.user_id.data, add_user_form.comment.data) g.db.add(blocked_user) - g.db.commit() + if not safe_db_commit(g.db, "adding blocked user"): + flash('Failed to block user.') + return redirect(url_for('.blocked_users')) flash('User blocked successfully.') + if not g.github['bot_token']: + g.log.error('GitHub token not configured, cannot check blocked user PRs') + return redirect(url_for('.blocked_users')) + try: # Remove any queued pull request from blocked user - gh = GitHub(access_token=g.github['bot_token']) - repository = gh.repos(g.github['repository_owner'])(g.github['repository']) + gh = Github(auth=Auth.Token(g.github['bot_token'])) + repository = gh.get_repo(f"{g.github['repository_owner']}/{g.github['repository']}") # Getting all pull requests by blocked user on the repo - pulls = repository.pulls.get() + pulls = repository.get_pulls(state='open') for pull in pulls: - if pull['user']['id'] != add_user_form.user_id.data: + if pull.user.id != add_user_form.user_id.data: continue - tests = Test.query.filter(Test.pr_nr == pull['number']).all() + tests = Test.query.filter(Test.pr_nr == pull.number).all() for test in tests: # Add canceled status only if the test hasn't started yet if len(test.progress) > 0: continue progress = TestProgress(test.id, TestStatus.canceled, "PR closed", datetime.datetime.now()) g.db.add(progress) - g.db.commit() - try: - repository.statuses(test.commit).post( - state=Status.FAILURE, - description="Tests canceled since user blacklisted", - context=f"CI - {test.platform.value}", - target_url=url_for('test.by_id', test_id=test.id, _external=True) - ) - except ApiError as a: - g.log.error(f"Got an exception while posting to GitHub! Message: {a.message}") - except ApiError as a: - g.log.error(f"Pull Requests of Blocked User could not be fetched: {a.response}") + if not safe_db_commit(g.db, f"canceling test {test.id} for blocked user"): + continue + gh_commit = repository.get_commit(test.commit) + message = "Tests canceled since user blacklisted" + target_url = url_for('test.by_id', test_id=test.id, _external=True) + update_status_on_github(gh_commit, Status.FAILURE, message, + f"CI - {test.platform.value}", target_url=target_url) + except GithubException as a: + g.log.error(f"Pull Requests of Blocked User could not be fetched: {a.data}") return redirect(url_for('.blocked_users')) @@ -1470,7 +3264,9 @@ def blocked_users_remove(blocked_user_id): form = DeleteUserForm(request.form) if form.validate_on_submit(): g.db.delete(blocked_user) - g.db.commit() + if not safe_db_commit(g.db, "removing blocked user"): + flash("Failed to remove user.") + return redirect(url_for('.blocked_users')) flash("User removed successfully.") return redirect(url_for('.blocked_users')) @@ -1502,9 +3298,11 @@ def toggle_maintenance(platform, status): db_mode = MaintenanceMode.query.filter(MaintenanceMode.platform == platform).first() if db_mode is not None: db_mode.disabled = disabled - g.db.commit() - result = 'success' - message = f'{platform.description} in maintenance? {"Yes" if disabled else "No"}' + if safe_db_commit(g.db, f"updating maintenance mode for {platform}"): + result = 'success' + message = f'{platform.description} in maintenance? {"Yes" if disabled else "No"}' + else: + message = 'Failed to update maintenance mode' except ValueError: pass @@ -1534,7 +3332,7 @@ def in_maintenance_mode(platform): if status is None: status = MaintenanceMode(platform, False) g.db.add(status) - g.db.commit() + safe_db_commit(g.db, f"creating maintenance mode entry for {platform}") return str(status.disabled) @@ -1566,4 +3364,4 @@ def add_customized_regression_tests(test_id) -> None: g.log.debug(f'Adding RT #{regression_test.id} to test {test_id}') customized_test = CustomizedTest(test_id, regression_test.id) g.db.add(customized_test) - g.db.commit() + safe_db_commit(g.db, f"adding customized regression tests for test {test_id}") diff --git a/mod_ci/cron.py b/mod_ci/cron.py index 9c0a0aed6..7babb2321 100755 --- a/mod_ci/cron.py +++ b/mod_ci/cron.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """handle cron logic for CI platform.""" import sys @@ -11,23 +11,26 @@ def cron(testing=False): """Script to run from cron for Sampleplatform.""" from flask import current_app - from github import GitHub + from github import Auth, Github from database import create_session - from mod_ci.controllers import TestPlatform, kvm_processor, start_platforms + from mod_ci.controllers import TestPlatform, gcp_instance, start_platforms from run import config, log log.info('Run the cron for kicking off CI platform(s).') - # Create session - db = create_session(config['DATABASE_URI']) - gh = GitHub(access_token=config['GITHUB_TOKEN']) - repository = gh.repos(config['GITHUB_OWNER'])(config['GITHUB_REPOSITORY']) + github_token = config['GITHUB_TOKEN'] + if not github_token: + log.error('GITHUB_TOKEN not configured, cannot run CI cron') + return + gh = Github(auth=Auth.Token(github_token)) + repository = gh.get_repo(f"{config['GITHUB_OWNER']}/{config['GITHUB_REPOSITORY']}") if testing is True: - kvm_processor(current_app._get_current_object(), db, config.get('KVM_LINUX_NAME', ''), TestPlatform.linux, - repository, None) + # Create a database session + db = create_session(config['DATABASE_URI']) + gcp_instance(current_app._get_current_object(), db, TestPlatform.linux, repository, None) else: - start_platforms(db, repository) + start_platforms(repository) cron() diff --git a/mod_ci/models.py b/mod_ci/models.py index ffd5ae82f..20da01112 100644 --- a/mod_ci/models.py +++ b/mod_ci/models.py @@ -3,7 +3,8 @@ List of models corresponding to mysql tables: [ - 'Kvm' => 'kvm', + 'Gcp Instance' => 'gcp_instance', + 'Pending Deletion' => 'pending_deletion', 'Maintenance mode' => 'maintenance_mode' ] """ @@ -16,7 +17,6 @@ Text) from sqlalchemy.orm import relationship -import mod_test.models from database import Base from mod_regression.models import RegressionTest from mod_test.models import Test, TestPlatform @@ -40,25 +40,24 @@ def __repr__(self) -> str: return f"" -class Kvm(Base): - """Model to store KVMs.""" +class GcpInstance(Base): + """Model to store GCP Instances.""" - __tablename__ = 'kvm' + __tablename__ = 'gcp_instance' __table_args__ = {'mysql_engine': 'InnoDB'} name = Column(String(64), primary_key=True) test_id = Column(Integer, ForeignKey(Test.id, onupdate="CASCADE", ondelete="RESTRICT")) test = relationship('Test', uselist=False) timestamp = Column(DateTime(), nullable=False) timestamp_prep_finished = Column(DateTime(), nullable=True) - timestamp_build_finished = Column(DateTime(), nullable=True) def __init__(self, name, test_id, timestamp=None) -> None: """ - Parametrized constructor for the Kvm model. + Parametrized constructor for the GcpInstance model. - :param name: The value of the 'name' field of Kvm model + :param name: The value of the 'name' field of GcpInstance model :type name: str - :param test_id: The value of the 'test_id' field of Kvm model + :param test_id: The value of the 'test_id' field of GcpInstance model :type test_id: int :param timestamp: The value of the 'timestamp' field of TestProgress model (None by default) @@ -72,13 +71,49 @@ def __init__(self, name, test_id, timestamp=None) -> None: def __repr__(self) -> str: """ - Represent a Kvm Model by its 'test_id' Field. + Represent a GcpInstance Model by its 'test_id' Field. :return str(test_id): Returns the string containing - 'test_id' field of the Kvm model + 'test_id' field of the GcpInstance model :rtype str(test_id): str """ - return f'' + return f'' + + +class PendingDeletion(Base): + """Model to track pending VM deletion operations for verification.""" + + __tablename__ = 'pending_deletion' + __table_args__ = {'mysql_engine': 'InnoDB'} + vm_name = Column(String(64), primary_key=True) + operation_name = Column(String(128), nullable=False) + created_at = Column(DateTime(), nullable=False) + retry_count = Column(Integer, nullable=False, default=0) + + # Max retries before we give up and just try to force delete + MAX_RETRIES = 5 + + def __init__(self, vm_name, operation_name, created_at=None) -> None: + """ + Parametrized constructor for the PendingDeletion model. + + :param vm_name: The name of the VM being deleted + :type vm_name: str + :param operation_name: The GCP operation name/ID for tracking + :type operation_name: str + :param created_at: When the deletion was initiated (None for now) + :type created_at: datetime + """ + self.vm_name = vm_name + self.operation_name = operation_name + if created_at is None: + created_at = datetime.datetime.now() + self.created_at = created_at + self.retry_count = 0 + + def __repr__(self) -> str: + """Represent a PendingDeletion by its vm_name.""" + return f'' class MaintenanceMode(Base): @@ -125,11 +160,47 @@ class CategoryTestInfo: success: Optional[int] +class Status: + """Define different states for the tests.""" + + PENDING = "pending" + SUCCESS = "success" + ERROR = "error" + FAILURE = "failure" + + +@dataclass +class ReferenceComparison: + """How a run's results read against one other run. + + The verdict a test carries is absolute -- it matched the approved output or + it did not. This says what that verdict means *relative to somewhere else*, + which is what tells a reviewer whether a failure is theirs. + """ + + # what this reference is, for the reader: "master", "closest ancestor" + label: str + # the run being compared against, or None when there is nothing to compare to + run: Optional[Test] + # regression tests per verdict, keyed by the constants in mod_ci.comparison + tests: Dict[str, List[RegressionTest]] + # how many tests fell in each verdict + counts: Dict[str, int] + # set when this reference resolved to the same run as an earlier one + duplicate_of: Optional[str] = None + + @dataclass class PrCommentInfo: """Contains info about a test run that is useful for displaying a PR comment.""" # info about successes and failures for each category category_stats: List[CategoryTestInfo] - # list of regression tests that failed + # the verdict, and the only thing pass/fail is decided on: tests whose + # output did not match the approved file failed_tests: List[RegressionTest] + passed_count: int + total_count: int + # the same failures described against other runs, in reading order + comparisons: List[ReferenceComparison] + last_test_master: Optional[Test] diff --git a/mod_customized/controllers.py b/mod_customized/controllers.py index e1a5bbde8..fa239cc66 100755 --- a/mod_customized/controllers.py +++ b/mod_customized/controllers.py @@ -1,21 +1,19 @@ """logic to allow users to test their fork branch with customized set of regression tests.""" from datetime import datetime, timedelta -from typing import Any, Callable, Dict, List, Optional, Tuple, Type -from flask import Blueprint, flash, g, redirect, request, url_for -from github import ApiError, GitHub -from sqlalchemy import and_ +from flask import Blueprint, g, redirect, request, url_for +from github import Auth, Github, GithubException +from sqlalchemy import and_, select -from decorators import get_menu_entries, template_renderer +from decorators import template_renderer from mod_auth.controllers import (check_access_rights, fetch_username_from_token, login_required) -from mod_auth.models import Role, User +from mod_auth.models import Role from mod_customized.forms import TestForkForm from mod_customized.models import CustomizedTest, TestFork from mod_regression.models import (Category, RegressionTest, regressionTestLinkTable) -from mod_test.controllers import TestNotFoundException, get_data_for_test from mod_test.models import Fork, Test, TestPlatform, TestType mod_customized = Blueprint('custom', __name__) @@ -49,17 +47,16 @@ def index(): fork_test_form = TestForkForm(request.form) username = fetch_username_from_token() commit_options = False - if username is not None: - gh = GitHub(access_token=g.github['bot_token']) - repository = gh.repos(username)(g.github['repository']) + if username is not None and g.github['bot_token']: + gh = Github(auth=Auth.Token(g.github['bot_token'])) + repository = gh.get_repo(f"{username}/{g.github['repository']}") # Only commits since last month last_month = datetime.now() - timedelta(days=30) - commit_since = last_month.isoformat() + 'Z' - commits = repository.commits().get(since=commit_since) + commits = repository.get_commits(since=last_month) commit_arr = [] for commit in commits: - commit_url = commit['html_url'] - commit_sha = commit['sha'] + commit_url = commit.html_url + commit_sha = commit.sha commit_option = f'{commit_sha}' commit_arr.append((commit_sha, commit_option)) # If there are commits present, display it on webpage @@ -69,23 +66,22 @@ def index(): fork_test_form.regression_test.choices = [(regression_test.id, regression_test) for regression_test in RegressionTest.query.all()] if fork_test_form.add.data and fork_test_form.validate_on_submit(): - import requests regression_tests = fork_test_form.regression_test.data commit_hash = fork_test_form.commit_hash.data - repo = g.github['repository'] platforms = fork_test_form.platform.data - api_url = f"https://api.github.com/repos/{username}/{repo}/commits/{commit_hash}" - # Show error if GitHub fails to recognize commit - response = requests.get(api_url) - if response.status_code == 500: - fork_test_form.commit_hash.errors.append('Error contacting GitHub') - elif response.status_code != 200: - fork_test_form.commit_hash.errors.append('Wrong Commit Hash') - else: - add_test_to_kvm(username, commit_hash, platforms, regression_tests) + try: + commit = repository.get_commit(commit_hash) + add_tests_to_platforms(username, commit_hash, platforms, regression_tests) return redirect(url_for('custom.index')) + except GithubException as e: + if e.status == 500: + fork_test_form.commit_hash.errors.append('Error contacting GitHub') + else: + fork_test_form.commit_hash.errors.append('Wrong Commit Hash') + elif username is not None: + g.log.error('GitHub token not configured, cannot fetch commits') - populated_categories = g.db.query(regressionTestLinkTable.c.category_id).subquery() + populated_categories = select(regressionTestLinkTable.c.category_id) categories = Category.query.filter(Category.id.in_(populated_categories)).order_by(Category.name.asc()).all() tests = Test.query.filter(and_(TestFork.user_id == g.user.id, TestFork.test_id == Test.id)).order_by( @@ -101,7 +97,7 @@ def index(): } -def add_test_to_kvm(username, commit_hash, platforms, regression_tests) -> None: +def add_tests_to_platforms(username, commit_hash, platforms, regression_tests) -> None: """ Create new tests and add it to CustomizedTests based on parameters. diff --git a/mod_deploy/__init__.py b/mod_deploy/__init__.py deleted file mode 100644 index 6422eae23..000000000 --- a/mod_deploy/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""contains methods and classes for GitHub hooks based on auto deployment.""" diff --git a/mod_deploy/controllers.py b/mod_deploy/controllers.py deleted file mode 100755 index 6572a8a2b..000000000 --- a/mod_deploy/controllers.py +++ /dev/null @@ -1,189 +0,0 @@ -"""contain methods to maintain GitHub hooks based on automatic deploy.""" -import hashlib -import hmac -import json -import subprocess -from datetime import datetime, timedelta -from functools import wraps -from ipaddress import IPv4Address, IPv6Address, ip_address, ip_network -from os import path -from shutil import copyfile -from typing import Callable, List, Union - -import requests -from flask import Blueprint, abort, g, request -from git import InvalidGitRepositoryError, Repo - -mod_deploy = Blueprint('deploy', __name__) - -IPAddress = Union[IPv4Address, IPv6Address] - -cached_web_hook_blocks: List[str] = [] -cached_load_time: datetime = datetime(1970, 1, 1) - - -def cache_has_expired() -> bool: - """ - Check if the cache expired. - - :return: True if the cache was last updated more than one hour ago. - :rtype: bool - """ - global cached_load_time - return cached_load_time + timedelta(hours=1) < datetime.now() - - -def get_cached_web_hook_blocks() -> List[str]: - """ - Fetch the cached web hook blocks. - - :return: A list of ip blocks. - :rtype: List[str] - """ - global cached_web_hook_blocks - from run import config - - if len(cached_web_hook_blocks) == 0 or cache_has_expired(): - client_id = config.get('GITHUB_CLIENT_ID', '') - client_secret = config.get('GITHUB_CLIENT_SECRET', '') - meta_json = requests.get( - f'https://api.github.com/meta', auth=(client_id, client_secret)).json() - try: - cached_web_hook_blocks = meta_json['hooks'] - except KeyError: - g.log.critical(f"Failed to retrieve hook IP's from GitHub! API returned {meta_json}") - - return cached_web_hook_blocks - - -def is_github_web_hook_ip(request_ip: IPAddress) -> bool: - """ - Check if the given IP address is matching one provided by the API of GitHub. - - :param request_ip: The IP address the request came from. - :type request_ip: IPAddress - :return: True if the IP address is a valid GitHub Web Hook requester. - :rtype: bool - """ - for block in get_cached_web_hook_blocks(): - if request_ip in ip_network(block): - return True - return False - - -def request_from_github(abort_code: int = 418) -> Callable: - """Provide decorator to handle request from GitHub on the web hook.""" - def decorator(f): - """Decorate the function to check if a request is a GitHub hook request.""" - @wraps(f) - def decorated_function(*args, **kwargs): - if request.method != 'POST': - return 'OK' - - request_ip = ip_address(f"{request.remote_addr}") - if not is_github_web_hook_ip(request_ip): - g.log.warning(f"Unauthorized attempt to deploy by IP {request_ip}") - abort(abort_code) - - for header in ['X-GitHub-Event', 'X-GitHub-Delivery', 'X-Hub-Signature', 'User-Agent']: - if header not in request.headers: - g.log.critical(f"{header} not in headers!") - abort(abort_code) - - ua = request.headers.get('User-Agent') - if not ua.startswith('GitHub-Hookshot/'): - g.log.critical("User-Agent does not begin with GitHub-Hookshot/!") - abort(abort_code) - - if not request.is_json: - g.log.critical("Request is not JSON!") - abort(abort_code) - - return f(*args, **kwargs) - return decorated_function - return decorator - - -def is_valid_signature(x_hub_signature, data, private_key): - """ - Re-check if the GitHub hook request got valid signature. - - :param x_hub_signature: Signature to check - :type x_hub_signature: str - :param data: Signature's data - :type data: bytearray - :param private_key: Signature's token - :type private_key: str - """ - hash_algorithm, github_signature = x_hub_signature.split('=', 1) - algorithm = hashlib.__dict__.get(hash_algorithm) - encoded_key = bytes(private_key, 'latin-1') - mac = hmac.new(encoded_key, msg=data, digestmod=algorithm) - return hmac.compare_digest(mac.hexdigest(), github_signature) - - -@mod_deploy.route('/deploy', methods=['GET', 'POST']) -@request_from_github() -def deploy(): - """Deploy the GitHub request to the test platform.""" - from run import app - abort_code = 418 - - event = request.headers.get('X-GitHub-Event') - if event == "ping": - g.log.info("deploy endpoint pinged!") - return json.dumps({'msg': "Hi!"}) - if event != "push": - g.log.info("deploy endpoint received unaccepted push request!") - return json.dumps({'msg': "Wrong event type"}) - - x_hub_signature = request.headers.get('X-Hub-Signature') - # web hook content type should be application/json for request.data to have the payload - # request.data is empty in case of x-www-form-urlencoded - if not is_valid_signature(x_hub_signature, request.data, g.github['deploy_key']): - g.log.warning(f"Deploy signature failed: {x_hub_signature}") - abort(abort_code) - - payload = request.get_json() - if payload is None: - g.log.warning(f"Deploy payload is empty: {payload}") - abort(abort_code) - - if payload['ref'] != 'refs/heads/master': - return json.dumps({'msg': "Not master; ignoring"}) - - try: - repo = Repo(app.config['INSTALL_FOLDER']) - except InvalidGitRepositoryError: - return json.dumps({'msg': "Folder is not a valid git directory"}) - - try: - origin = repo.remote('origin') - except ValueError: - return json.dumps({'msg': "Remote origin does not exist"}) - - fetch_info = origin.fetch() - if len(fetch_info) == 0: - return json.dumps({'msg': "Didn't fetch any information from remote!"}) - - pull_info = origin.pull() - - if len(pull_info) == 0: - return json.dumps({'msg': "Didn't pull any information from remote!"}) - - if pull_info[0].flags > 128: - return json.dumps({'msg': "Didn't pull any information from remote!"}) - - commit_hash = pull_info[0].commit.hexsha - build_commit = f'build_commit = "{commit_hash}"' - with open('build_commit.py', 'w') as f: - f.write(build_commit) - - run_ci_repo = path.join(app.config['INSTALL_FOLDER'], 'install', 'ci-vm', 'ci-linux', 'ci', 'runCI') - run_ci_nfs = path.join(app.config['SAMPLE_REPOSITORY'], 'vm_data', app.config['KVM_LINUX_NAME'], 'runCI') - copyfile(run_ci_repo, run_ci_nfs) - - g.log.info(f"Platform upgraded to commit {commit_hash}") - subprocess.Popen(["sudo", "service", "platform", "reload"]) - g.log.info("Sample platform synced with GitHub!") - return json.dumps({'msg': f"Platform upgraded to commit {commit_hash}"}) diff --git a/mod_health/__init__.py b/mod_health/__init__.py new file mode 100644 index 000000000..3e3286c9d --- /dev/null +++ b/mod_health/__init__.py @@ -0,0 +1 @@ +"""Health check module for deployment verification.""" diff --git a/mod_health/controllers.py b/mod_health/controllers.py new file mode 100644 index 000000000..00b46498a --- /dev/null +++ b/mod_health/controllers.py @@ -0,0 +1,184 @@ +"""Health check endpoints for deployment verification and monitoring.""" + +import os +import subprocess +from datetime import datetime, timezone +from typing import Any, Dict, Optional, Tuple + +from flask import Blueprint, current_app, jsonify +from sqlalchemy import text + +mod_health = Blueprint('health', __name__) + + +def check_database() -> Dict[str, Any]: + """ + Check database connectivity. + + :return: Dictionary with status and optional error message + :rtype: Dict[str, Any] + """ + try: + from database import create_session + db = create_session(current_app.config['DATABASE_URI']) + db.execute(text('SELECT 1')) + # remove() returns the scoped session's connection to the pool + db.remove() + return {'status': 'ok'} + except Exception: + current_app.logger.exception('Health check database connection failed') + return {'status': 'error', 'message': 'Database connection failed'} + + +def check_config() -> Dict[str, Any]: + """ + Check that required configuration is loaded. + + :return: Dictionary with status and optional error message + :rtype: Dict[str, Any] + """ + required_keys = [ + 'DATABASE_URI', + 'GITHUB_TOKEN', + 'GITHUB_OWNER', + 'GITHUB_REPOSITORY', + ] + + missing = [key for key in required_keys if not current_app.config.get(key)] + + if missing: + return {'status': 'error', 'message': f'Missing config keys: {missing}'} + return {'status': 'ok'} + + +@mod_health.route('/health') +def health_check() -> Tuple[Any, int]: + """ + Health check endpoint for deployment verification. + + Returns 200 if all critical checks pass, 503 if any fail. + Used by deployment pipeline to verify successful deployment. + + :return: JSON response with health status and HTTP status code + :rtype: Tuple[Any, int] + """ + check_results: Dict[str, Dict[str, Any]] = {} + all_healthy = True + + # Check 1: Database connectivity + db_check = check_database() + check_results['database'] = db_check + if db_check['status'] != 'ok': + all_healthy = False + + # Check 2: Configuration loaded + config_check = check_config() + check_results['config'] = config_check + if config_check['status'] != 'ok': + all_healthy = False + + checks: Dict[str, Any] = { + 'status': 'healthy' if all_healthy else 'unhealthy', + 'timestamp': datetime.now(timezone.utc).isoformat(), + 'checks': check_results + } + + return jsonify(checks), 200 if all_healthy else 503 + + +@mod_health.route('/health/live') +def liveness_check() -> Tuple[Any, int]: + """ + Liveness check endpoint. + + Minimal check, just returns 200 if Flask is responding. + Useful for load balancers and container orchestration. + + :return: JSON response with alive status + :rtype: Tuple[Any, int] + """ + return jsonify({ + 'status': 'alive', + 'timestamp': datetime.now(timezone.utc).isoformat() + }), 200 + + +@mod_health.route('/health/ready') +def readiness_check() -> Tuple[Any, int]: + """ + Readiness check endpoint. + + Same as health check but can be extended for more checks. + Useful for Kubernetes readiness probes. + + :return: JSON response with readiness status + :rtype: Tuple[Any, int] + """ + return health_check() + + +def get_git_info() -> Dict[str, Optional[str]]: + """ + Get git repository information for the deployed version. + + :return: Dictionary with git commit hash, short hash, and branch + :rtype: Dict[str, Optional[str]] + """ + result: Dict[str, Optional[str]] = { + 'commit': None, + 'short': None, + 'branch': None, + } + + try: + # Get the installation folder from config, fallback to current directory + install_folder = current_app.config.get('INSTALL_FOLDER', os.getcwd()) + + # Get full commit hash + commit = subprocess.check_output( + ['git', 'rev-parse', 'HEAD'], + cwd=install_folder, + stderr=subprocess.DEVNULL + ).decode().strip() + result['commit'] = commit + result['short'] = commit[:7] + + # Get current branch + branch = subprocess.check_output( + ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], + cwd=install_folder, + stderr=subprocess.DEVNULL + ).decode().strip() + result['branch'] = branch + + except (subprocess.CalledProcessError, FileNotFoundError, OSError): + # Git not available or not a git repository + current_app.logger.warning('Could not retrieve git version information') + + return result + + +@mod_health.route('/health/version') +def version_check() -> Tuple[Any, int]: + """ + Version endpoint to verify deployed commit. + + Returns the current git commit hash, useful for verifying + that a deployment has completed successfully. + + :return: JSON response with version information + :rtype: Tuple[Any, int] + """ + git_info = get_git_info() + + response = { + 'timestamp': datetime.now(timezone.utc).isoformat(), + 'git': git_info, + } + + # Return 200 if we have version info, 503 if we couldn't get it + if git_info['commit']: + return jsonify(response), 200 + else: + response['error'] = 'Could not retrieve version information' + return jsonify(response), 503 diff --git a/mod_regression/controllers.py b/mod_regression/controllers.py index 09f767173..e7ee48a0a 100644 --- a/mod_regression/controllers.py +++ b/mod_regression/controllers.py @@ -2,7 +2,7 @@ from flask import (Blueprint, abort, flash, g, jsonify, redirect, request, url_for) -from sqlalchemy import and_, func +from sqlalchemy import and_ from decorators import template_renderer from mod_auth.controllers import check_access_rights, login_required @@ -13,9 +13,8 @@ from mod_regression.models import (Category, InputType, OutputType, RegressionTest, RegressionTestOutput, RegressionTestOutputFiles) -from mod_sample.models import Sample -from mod_test.models import (Fork, Test, TestPlatform, TestProgress, - TestResult, TestResultFile, TestStatus, TestType) +from mod_sample.models import Sample, Tag +from mod_test.models import TestResultFile from utility import serve_file_download mod_regression = Blueprint('regression', __name__) @@ -37,7 +36,8 @@ def index(): """Display all regression tests.""" return { 'tests': RegressionTest.query.all(), - 'categories': Category.query.order_by(Category.name.asc()).all() + 'categories': Category.query.order_by(Category.name.asc()).all(), + 'tags': Tag.query.all() } @@ -159,6 +159,7 @@ def test_edit(regression_id): test.expected_rc = form.expected_rc.data test.input_type = InputType.from_string(form.input_type.data) test.output_type = OutputType.from_string(form.output_type.data) + test.description = form.description.data g.db.commit() g.log.info(f'regression test with id: {regression_id} updated!') @@ -172,6 +173,7 @@ def test_edit(regression_id): form.expected_rc.data = test.expected_rc form.input_type.data = test.input_type.value form.output_type.data = test.output_type.value + form.description.data = test.description return {'form': form, 'regression_id': regression_id} @@ -201,23 +203,25 @@ def toggle_active_status(regression_id): @mod_regression.route('/test//download', methods=['GET']) +@login_required def test_result_file(regression_test_output_id): """View the output files of the regression test.""" rto = RegressionTestOutput.query.filter(RegressionTestOutput.id == regression_test_output_id).first() if rto is None: g.log.error(f'requested regression test output with id: {regression_test_output_id} not found!') abort(404) - return serve_file_download(rto.filename_correct, 'TestResults', 'regression-download') + return serve_file_download(rto.filename_correct, 'TestResults') @mod_regression.route('/test//download/variant', methods=['GET']) +@login_required def multiple_test_result_file(regression_test_output_id): """View the output files of the regression test (variants).""" rtof = RegressionTestOutputFiles.query.filter(RegressionTestOutputFiles.id == regression_test_output_id).first() if rtof is None: g.log.error(f'requested regression test output file with id: {regression_test_output_id} not found!') abort(404) - return serve_file_download(rtof.file_hashes + rtof.output.correct_extension, 'TestResults', 'regression-download') + return serve_file_download(rtof.file_hashes + rtof.output.correct_extension, 'TestResults') @mod_regression.route('/test/new', methods=['GET', 'POST']) @@ -241,7 +245,8 @@ def test_add(): category_id=form.category_id.data, expected_rc=form.expected_rc.data, input_type=InputType.from_string(form.input_type.data), - output_type=OutputType.from_string(form.output_type.data) + output_type=OutputType.from_string(form.output_type.data), + description=form.description.data, ) g.db.add(new_test) category = Category.query.filter(Category.id == form.category_id.data).first() @@ -296,20 +301,26 @@ def category_edit(category_id): :return: form and category id :rtype: dict """ - test = Category.query.filter(Category.id == category_id).first() + category = Category.query.filter(Category.id == category_id).first() - if test is None: + if category is None: g.log.error(f'requested category with id: {category_id} not found!') abort(404) form = AddCategoryForm(request.form) if form.validate(): - test.name = form.category_name.data - test.description = form.category_description.data + category.name = form.category_name.data + category.description = form.category_description.data g.db.commit() g.log.info(f'category with id: {category_id} updated!') flash('Category Updated') return redirect(url_for('.index')) + + if not form.is_submitted(): + # Populate form with current set category values + form.category_name.data = category.name + form.category_description.data = category.description + return {'form': form, 'category_id': category_id} diff --git a/mod_regression/forms.py b/mod_regression/forms.py index fe8a4c22c..c01b377b5 100644 --- a/mod_regression/forms.py +++ b/mod_regression/forms.py @@ -2,8 +2,8 @@ from flask_wtf import FlaskForm from wtforms import (HiddenField, IntegerField, SelectField, StringField, - SubmitField) -from wtforms.validators import DataRequired, InputRequired + SubmitField, TextAreaField) +from wtforms.validators import DataRequired, InputRequired, Length from mod_regression.models import InputType, OutputType @@ -21,6 +21,7 @@ class CommonTestForm(FlaskForm): sample_id = SelectField("Sample", coerce=int) command = StringField("Command") + description = TextAreaField("Description", validators=[Length(max=1024)]) input_type = SelectField( "Input Type", [DataRequired(message="Input Type is not selected")], diff --git a/mod_regression/models.py b/mod_regression/models.py index 3a8abb445..9ade0a618 100644 --- a/mod_regression/models.py +++ b/mod_regression/models.py @@ -94,8 +94,12 @@ class RegressionTest(Base): output_files = relationship('RegressionTestOutput', back_populates='regression_test') expected_rc = Column(Integer) active = Column(Boolean(), default=True) + last_passed_on_windows = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) + last_passed_on_linux = Column(Integer, ForeignKey('test.id', onupdate="CASCADE", ondelete="SET NULL")) + description = Column(String(length=1024)) - def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc, active=True) -> None: + def __init__(self, sample_id, command, input_type, output_type, category_id, expected_rc, + active=True, description="") -> None: """ Parametrized constructor for the RegressionTest model. @@ -122,6 +126,7 @@ def __init__(self, sample_id, command, input_type, output_type, category_id, exp self.category_id = category_id self.expected_rc = expected_rc self.active = active + self.description = description def __repr__(self) -> str: """ diff --git a/mod_sample/controllers.py b/mod_sample/controllers.py index 82233db2e..7b3d5d85a 100755 --- a/mod_sample/controllers.py +++ b/mod_sample/controllers.py @@ -1,12 +1,11 @@ """Logic to fetch sample information, uploading, editing, deleting sample.""" -import json import os from operator import and_ -from typing import Any, Callable, Dict, List, Optional, Tuple, Type +from typing import Any, Dict -import requests -from flask import Blueprint, g, make_response, redirect, request, url_for +from flask import Blueprint, g, redirect, request, url_for +from sqlalchemy import select from decorators import template_renderer from exceptions import SampleNotFoundException @@ -14,11 +13,11 @@ from mod_auth.models import Role from mod_home.models import CCExtractorVersion, GeneralData from mod_regression.models import RegressionTest -from mod_sample.forms import (DeleteAdditionalSampleForm, DeleteSampleForm, - EditSampleForm) +from mod_sample.forms import (AddTagForm, DeleteAdditionalSampleForm, + DeleteSampleForm, EditSampleForm) from mod_sample.media_info_parser import (InvalidMediaInfoError, MediaInfoFetcher) -from mod_sample.models import ExtraFile, ForbiddenExtension, Issue, Sample +from mod_sample.models import ExtraFile, Issue, Sample, Tag from mod_test.models import Test, TestResult, TestResultFile from mod_upload.models import Platform from utility import serve_file_download @@ -68,14 +67,14 @@ def display_sample_info(sample) -> Dict[str, Any]: if len(regression_tests) > 0: if test_commit is not None: - sq = g.db.query(RegressionTest.id).filter(RegressionTest.sample_id == sample.id).subquery() + sq = select(RegressionTest.id).filter(RegressionTest.sample_id == sample.id) exit_code = g.db.query(TestResult.exit_code).filter(and_( TestResult.exit_code != TestResult.expected_rc, - and_(TestResult.test_id == test_commit.id, TestResult.regression_test_id.in_(sq)) + and_(TestResult.test_id == test_commit.id, TestResult.regression_test_id.in_(sq.select())) )).first() not_null = g.db.query(TestResultFile.got).filter(and_( TestResultFile.got.isnot(None), - and_(TestResultFile.test_id == test_commit.id, TestResultFile.regression_test_id.in_(sq)) + and_(TestResultFile.test_id == test_commit.id, TestResultFile.regression_test_id.in_(sq.select())) )).first() if exit_code is None and not_null is None: @@ -84,17 +83,17 @@ def display_sample_info(sample) -> Dict[str, Any]: status = 'Fail' if test_release is not None: - sq = g.db.query(RegressionTest.id).filter( - RegressionTest.sample_id == sample.id).subquery() + sq = select(RegressionTest.id).filter( + RegressionTest.sample_id == sample.id) exit_code = g.db.query(TestResult.exit_code).filter( and_( TestResult.exit_code != TestResult.expected_rc, - and_(TestResult.test_id == test_release.id, TestResult.regression_test_id.in_(sq)) + and_(TestResult.test_id == test_release.id, TestResult.regression_test_id.in_(sq.select())) ) ).first() not_null = g.db.query(TestResultFile.got).filter(and_( TestResultFile.got.isnot(None), - and_(TestResultFile.test_id == test_release.id, TestResultFile.regression_test_id.in_(sq)) + and_(TestResultFile.test_id == test_release.id, TestResultFile.regression_test_id.in_(sq.select())) )).first() if exit_code is None and not_null is None: @@ -175,6 +174,7 @@ def sample_by_hash(sample_hash): @mod_sample.route('/download/') +@login_required def download_sample(sample_id): """ Download sample file. @@ -187,11 +187,12 @@ def download_sample(sample_id): """ sample = Sample.query.filter(Sample.id == sample_id).first() if sample is not None: - return serve_file_download(sample.filename, 'TestFiles', 'media-download') + return serve_file_download(sample.filename, 'TestFiles') raise SampleNotFoundException('Sample not found') @mod_sample.route('/download//media-info') +@login_required def download_sample_media_info(sample_id): """ Download sample file's media information as XML. @@ -210,7 +211,7 @@ def download_sample_media_info(sample_id): media_info_path = os.path.join( config.get('SAMPLE_REPOSITORY', ''), 'TestFiles', 'media', sample.sha + '.xml') if os.path.isfile(media_info_path): - return serve_file_download(sample.sha + '.xml', 'TestFiles', 'media-download', 'media', 'text/xml') + return serve_file_download(sample.sha + '.xml', 'TestFiles', 'media') raise SampleNotFoundException(f"Media information for sample {sample.id} not found") @@ -218,6 +219,7 @@ def download_sample_media_info(sample_id): @mod_sample.route('/download//additional/') +@login_required def download_sample_additional(sample_id, additional_id): """ Download sample file's additional files and information. @@ -235,11 +237,39 @@ def download_sample_additional(sample_id, additional_id): if sample is not None: extra = ExtraFile.query.filter(ExtraFile.id == additional_id).first() if extra is not None: - return serve_file_download(extra.filename, 'TestFiles', 'media-download', 'extra') + return serve_file_download(extra.filename, 'TestFiles', 'extra') raise SampleNotFoundException(f"Extra file {additional_id} for sample {sample.id} not found") raise SampleNotFoundException(f"Sample with id {sample_id} not found") +@mod_sample.route('/add_tag', methods=['POST']) +@login_required +@check_access_rights([Role.admin]) +def add_tag(): + """ + Add tags on sample api, requires admin role. + + :return: api response + :rtype: dict + """ + form = AddTagForm(request.form) + + if form.validate_on_submit(): + new_tag = Tag(form.name.data, form.description.data) + g.db.add(new_tag) + g.db.commit() + + return { + 'id': new_tag.id, + 'name': new_tag.name, + 'description': new_tag.description + } + + return { + 'errors': form.errors + } + + @mod_sample.route('/edit/', methods=['GET', 'POST']) @login_required @check_access_rights([Role.admin]) @@ -258,9 +288,12 @@ def edit_sample(sample_id): if sample is not None: versions = CCExtractorVersion.query.all() + tags = Tag.query.all() # Process or render form form = EditSampleForm(request.form) + add_tag_form = AddTagForm(request.form) form.version.choices = [(v.id, v.version) for v in versions] + form.tags.choices = [(tag.id, tag.name) for tag in tags] if form.validate_on_submit(): # Store values @@ -269,6 +302,7 @@ def edit_sample(sample_id): upload.version_id = form.version.data upload.platform = Platform.from_string(form.platform.data) upload.parameters = form.parameters.data + sample.tags = list(Tag.query.filter(Tag.id.in_(form.tags.data))) g.db.commit() g.log.info(f"sample with id: {sample_id} updated") return redirect(url_for('.sample_by_id', sample_id=sample.id)) @@ -279,10 +313,12 @@ def edit_sample(sample_id): form.platform.data = sample.upload.platform.name form.notes.data = sample.upload.notes form.parameters.data = sample.upload.parameters + form.tags.data = [tag.id for tag in sample.tags] return { 'sample': sample, - 'form': form + 'form': form, + 'add_tag_form': add_tag_form } raise SampleNotFoundException(f"Sample with id {sample_id} not found") diff --git a/mod_sample/forms.py b/mod_sample/forms.py index 9d24555ba..7200e3528 100755 --- a/mod_sample/forms.py +++ b/mod_sample/forms.py @@ -1,10 +1,13 @@ """Maintains forms related to sample CRUD operations.""" from flask_wtf import FlaskForm -from wtforms import SubmitField +from wtforms import StringField, SubmitField, TextAreaField +from wtforms.validators import DataRequired, ValidationError from mod_upload.forms import CommonSampleForm +from .models import Tag + class EditSampleForm(CommonSampleForm): """Form to edit sample.""" @@ -12,6 +15,29 @@ class EditSampleForm(CommonSampleForm): submit = SubmitField('Update sample') +class AddTagForm(FlaskForm): + """Form to add tags.""" + + name = StringField('Name', validators=[DataRequired(message="Tag name is required.")]) + description = TextAreaField('Description') + submit = SubmitField('Add Tag') + + @staticmethod + def validate_name(form, field) -> None: + """ + Validate tag name (case-insensitive). + + :param form: form data + :type form: AddTagForm + :param field: field to validate + :type field: form field + :raises ValidationError: when the same tag already exists + """ + existing_tag = Tag.query.filter(Tag.name.ilike(field.data)).first() + if existing_tag: + raise ValidationError("Tag with the same name already exists.") + + class DeleteSampleForm(FlaskForm): """Form to delete sample.""" diff --git a/mod_sample/media_info_parser.py b/mod_sample/media_info_parser.py index 026bb3951..5d09fbbed 100644 --- a/mod_sample/media_info_parser.py +++ b/mod_sample/media_info_parser.py @@ -38,13 +38,13 @@ def __init__(self, sample) -> None: if 'MediaInfo' not in doc: raise InvalidMediaInfoError("Mediainfo xml root element not present") - self.media_info = doc['MediaInfo'] - self.video_tracks = [] # type: List - self.caption_tracks = [] # type: List - self.audio_tracks = [] # type: List - self.other_tracks = [] # type: List - self.general_track = {} # type: Dict - self.parsed = False + self.media_info = doc['MediaInfo'] + self.video_tracks = [] # type: List + self.caption_tracks = [] # type: List + self.audio_tracks = [] # type: List + self.other_tracks = [] # type: List + self.general_track = {} # type: Dict + self.parsed = False def get_media_info(self, force_parse=False) -> Optional[List[Dict[str, Any]]]: """Get media info from the sample file.""" @@ -101,7 +101,7 @@ def _process_track(self, track) -> None: :type track: dict :raises InvalidMediaInfoError: when track dict doesn't contain type """ - if type(track) is not OrderedDict: + if type(track) not in [OrderedDict, dict]: return if '@type' not in track: raise InvalidMediaInfoError("Track file does not contain a type") @@ -210,10 +210,10 @@ def generate_media_xml(sample) -> MediaInfoFetcher: media_folder = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestFiles') media_info_path = os.path.join(media_folder, 'media', sample.sha + '.xml') - output_handle = open(media_info_path, 'w') media_path = os.path.join(media_folder, sample.filename) - process = subprocess.Popen(['mediainfo', '--Output=XML', media_path], stdout=output_handle) - process.wait() + with open(media_info_path, 'w') as output_handle: + process = subprocess.Popen(['mediainfo', '--Output=XML', media_path], stdout=output_handle) + process.wait() if os.path.isfile(media_info_path): # Load media info, and replace full pathname tree = etree.parse(media_info_path) diff --git a/mod_sample/models.py b/mod_sample/models.py index f21190cf0..b53e97ce8 100755 --- a/mod_sample/models.py +++ b/mod_sample/models.py @@ -1,13 +1,12 @@ """Maintain database models regarding various sample, ExtraFile, ForbiddenExtension, ForbiddenMimeType, Issue.""" from datetime import datetime -from typing import Any, Dict, Type -from sqlalchemy import Column, DateTime, ForeignKey, Integer, String, Text +from sqlalchemy import (Column, DateTime, ForeignKey, Index, Integer, String, + Table, Text) from sqlalchemy.orm import relationship -import database -from database import Base, DeclEnum +from database import Base def get_extension(extension: str) -> str: @@ -22,6 +21,32 @@ def get_extension(extension: str) -> str: return ("." + extension) if len(extension) > 0 else "" +sample_tag_association = Table( + 'sample_tag_association', + Base.metadata, + Column('sample_id', ForeignKey('sample.id'), primary_key=True, nullable=False), + Column('tag_id', ForeignKey('tag.id'), primary_key=True, nullable=False) +) + + +class Tag(Base): + """Model to store tags.""" + + __tablename__ = 'tag' + __table_args__ = {'mysql_engine': 'InnoDB'} + id = Column(Integer, primary_key=True) + name = Column(String(64), unique=True, nullable=False) + description = Column(String(length=1024)) + samples = relationship('Sample', secondary=sample_tag_association, back_populates='tags') + + def __init__(self, name, description="") -> None: + self.name = name + self.description = description + + +tag_name_index = Index('tag_name_index', Tag.name) + + class Sample(Base): """Model to store and manage sample.""" @@ -34,6 +59,7 @@ class Sample(Base): extra_files = relationship('ExtraFile', back_populates='sample') tests = relationship('RegressionTest', back_populates='sample') upload = relationship('Upload', uselist=False, back_populates='sample') + tags = relationship('Tag', secondary=sample_tag_association, back_populates='samples') def __init__(self, sha, extension, original_name) -> None: """ diff --git a/mod_test/controllers.py b/mod_test/controllers.py index 9e16973bb..62844bbe5 100644 --- a/mod_test/controllers.py +++ b/mod_test/controllers.py @@ -1,21 +1,19 @@ """Logic to find all tests, their progress and details of individual test.""" import os -from datetime import datetime -from typing import Any, Callable, Dict, List, Optional, Tuple, Type +from typing import Any, Dict, List -from flask import (Blueprint, Response, abort, g, jsonify, make_response, - redirect, request, url_for) -from github import GitHub -from sqlalchemy import and_, func -from sqlalchemy.sql import label +from flask import (Blueprint, Response, abort, g, jsonify, redirect, request, + url_for) +from sqlalchemy import and_, select +from sqlalchemy.orm import joinedload, selectinload from decorators import template_renderer from exceptions import TestNotFoundException from mod_auth.controllers import check_access_rights, login_required from mod_auth.models import Role -from mod_ci.models import Kvm -from mod_customized.models import CustomizedTest, TestFork +from mod_ci.models import GcpInstance +from mod_customized.models import TestFork from mod_home.models import CCExtractorVersion, GeneralData from mod_regression.models import (Category, RegressionTestOutput, regressionTestLinkTable) @@ -57,71 +55,29 @@ def index(): } -def get_data_for_test(test, title=None) -> Dict[str, Any]: +def get_test_results(test) -> List[Dict[str, Any]]: """ - Retrieve the data for a single test, with an optional title. + Get test results for each category. :param test: The test to retrieve the data for. :type test: Test - :param title: The title to use in the result. If empty, it's set to 'test {id}' - :type title: str - :return: A dictionary with the appropriate values. - :rtype: dict - """ - if title is None: - title = f"test {test.id}" - - populated_categories = g.db.query(regressionTestLinkTable.c.category_id).subquery() - categories = Category.query.filter(Category.id.in_(populated_categories)).order_by(Category.name.asc()).all() - hours = 0.00 - minutes = 0.00 - queued_tests = 0 - - """ - evaluating estimated time if the test is still in queue - estimated time = (number of tests already in queue + 1) * (average time of that platform) - - (time already spend by those tests) - calculates time in minutes and hours """ - if len(test.progress) == 0: - var_average = 'average_time_' + test.platform.value - - # get average build and prep time. - prep_average_key = 'avg_prep_time_' + test.platform.value - build_average_key = 'avg_build_time_' + test.platform.value - average_prep_time = int(float(GeneralData.query.filter(GeneralData.key == prep_average_key).first().value)) - average_build_time = int(float(GeneralData.query.filter(GeneralData.key == build_average_key).first().value)) - - test_progress_last_entry = g.db.query(func.max(TestProgress.test_id)).first() - queued_kvm = g.db.query(Kvm.test_id).filter(Kvm.test_id < test.id).subquery() - queued_kvm_entries = g.db.query(Test.id).filter( - and_(Test.id.in_(queued_kvm), Test.platform == test.platform) - ).subquery() - kvm_test = g.db.query(TestProgress.test_id, label('time', func.group_concat(TestProgress.timestamp))).filter( - TestProgress.test_id.in_(queued_kvm_entries) - ).group_by(TestProgress.test_id).all() - number_kvm_test = g.db.query(Test.id).filter( - and_(Test.id > test_progress_last_entry[0], Test.id < test.id, Test.platform == test.platform) - ).count() - average_duration = float(GeneralData.query.filter(GeneralData.key == var_average).first().value) - queued_tests = number_kvm_test - time_run = 0.00 - for pr_test in kvm_test: - timestamps = pr_test.time.split(',') - start = datetime.strptime(timestamps[0], '%Y-%m-%d %H:%M:%S') - end = datetime.strptime(timestamps[-1], '%Y-%m-%d %H:%M:%S') - time_run += (end - start).total_seconds() - # subtracting current running tests - total = (average_build_time + average_prep_time + (average_duration * (queued_tests + 1))) - time_run - minutes = (total % 3600) // 60 - hours = total // 3600 - + populated_categories = select(regressionTestLinkTable.c.category_id) + categories = Category.query.options( + selectinload(Category.regression_tests) + ).filter( + Category.id.in_(populated_categories) + ).order_by(Category.name.asc()).all() results = [{ 'category': category, 'tests': [{ 'test': rt, 'result': next((r for r in test.results if r.regression_test_id == rt.id), None), - 'files': TestResultFile.query.filter( + 'files': TestResultFile.query.options( + joinedload(TestResultFile.regression_test_output).selectinload( + RegressionTestOutput.multiple_files + ) + ).filter( and_(TestResultFile.test_id == test.id, TestResultFile.regression_test_id == rt.id) ).all() } for rt in category.regression_tests if rt.id in test.get_customized_regressiontests()] @@ -142,7 +98,7 @@ def get_data_for_test(test, title=None) -> Dict[str, Any]: test_error = True if len(category_test['files']) > 0: for result_file in category_test['files']: - if result_file.got is not None and result.exit_code == 0: + if result_file.got is not None and (result is None or result.exit_code == result.expected_rc): file_error = True for file in result_file.regression_test_output.multiple_files: if file.file_hashes == result_file.got: @@ -153,7 +109,7 @@ def get_data_for_test(test, title=None) -> Dict[str, Any]: # We need to check if the regression test had any file that shouldn't have been ignored. outputs = RegressionTestOutput.query.filter(and_( RegressionTestOutput.regression_id == category_test['test'].id, - RegressionTestOutput.ignore is False + RegressionTestOutput.ignore.is_(False) )).all() got = None if len(outputs) > 0: @@ -167,14 +123,61 @@ def get_data_for_test(test, title=None) -> Dict[str, Any]: error = error or test_error category['error'] = error results.sort(key=lambda entry: entry['category'].name) + return results + + +def get_data_for_test(test, title=None) -> Dict[str, Any]: + """ + Retrieve the data for a single test, with an optional title. + + :param test: The test to retrieve the data for. + :type test: Test + :param title: The title to use in the result. If empty, it's set to 'test {id}' + :type title: str + :return: A dictionary with the appropriate values. + :rtype: dict + """ + if title is None: + title = f"test {test.id}" + + # Calculate average runtime for this platform (used when test hasn't started yet) + avg_minutes = 0 + if len(test.progress) == 0: + try: + avg_time_key = 'average_time_' + test.platform.value + prep_time_key = 'avg_prep_time_' + test.platform.value + + avg_time_record = GeneralData.query.filter(GeneralData.key == avg_time_key).first() + prep_time_record = GeneralData.query.filter(GeneralData.key == prep_time_key).first() + + avg_duration = float(avg_time_record.value) if avg_time_record else 0 + avg_prep = float(prep_time_record.value) if prep_time_record else 0 + + # Total average time in minutes + avg_minutes = max(0, int((avg_duration + avg_prep) / 60)) + except (ValueError, AttributeError): + avg_minutes = 0 + + results = get_test_results(test) + + # Calculate sample progress for initial page load + completed_samples = len(test.results) + total_samples = len(test.get_customized_regressiontests()) + progress_percentage = 0 + if total_samples > 0: + progress_percentage = int((completed_samples / total_samples) * 100) + return { 'test': test, 'TestType': TestType, 'results': results, 'title': title, - 'next': queued_tests, - 'min': minutes, - 'hr': hours + 'avg_minutes': avg_minutes, + 'sample_progress': { + 'current': completed_samples, + 'total': total_samples, + 'percentage': progress_percentage + } } @@ -202,11 +205,23 @@ def get_json_data(test_id): 'message': entry.message }) + # Calculate sample progress from existing TestResult data + completed_samples = len(test.results) + total_samples = len(test.get_customized_regressiontests()) + progress_percentage = 0 + if total_samples > 0: + progress_percentage = int((completed_samples / total_samples) * 100) + return jsonify({ 'status': 'success', 'details': pr_data["progress"], 'complete': test.finished, - 'progress_array': progress_array + 'progress_array': progress_array, + 'sample_progress': { + 'current': completed_samples, + 'total': total_samples, + 'percentage': progress_percentage + } }) @@ -344,7 +359,7 @@ def generate_diff(test_id: int, regression_test_id: int, output_id: int, to_view path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TestResults') request_xhr_key = request.headers.get('X-Requested-With') - if (request_xhr_key == 'XMLHttpRequest' or request.accept_mimetypes['application/json']) and to_view == 1: + if (request_xhr_key == 'XMLHttpRequest' or request.accept_mimetypes.accept_json) and to_view == 1: return result.generate_html_diff(path) elif to_view == 0: diff_html_text = result.generate_html_diff(path, to_view=False) @@ -362,6 +377,7 @@ def generate_diff(test_id: int, regression_test_id: int, output_id: int, to_view @mod_test.route('/log-files/') +@login_required def download_build_log_file(test_id): """ Serve download of build log. @@ -381,7 +397,7 @@ def download_build_log_file(test_id): log_file_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'LogFiles', file_name) if os.path.isfile(log_file_path): - return serve_file_download(file_name, 'LogFiles', 'logfile-download', content_type='text/plain') + return serve_file_download(file_name, 'LogFiles') raise TestNotFoundException(f"Build log for Test {test_id} not found") @@ -407,6 +423,7 @@ def restart_test(test_id): TestResultFile.query.filter(TestResultFile.test_id == test.id).delete() TestResult.query.filter(TestResult.test_id == test.id).delete() TestProgress.query.filter(TestProgress.test_id == test.id).delete() + GcpInstance.query.filter(GcpInstance.test_id == test.id).delete() g.db.commit() g.log.info(f"test with id: {test_id} restarted") return redirect(url_for('.by_id', test_id=test.id)) diff --git a/mod_test/models.py b/mod_test/models.py index d2bf80b57..68a32e3a7 100644 --- a/mod_test/models.py +++ b/mod_test/models.py @@ -45,7 +45,6 @@ class TestStatus(DeclEnum): """Enum to specify test status.""" preparation = "preparation", "Preparation" - building = "building", "Building" testing = "testing", "Testing" completed = "completed", "Completed" canceled = "canceled", "Canceled/Error" @@ -73,8 +72,7 @@ def stages() -> List[Tuple[str, str]]: :return: stages available for test :rtype: list """ - return [TestStatus.preparation, TestStatus.building, - TestStatus.testing, TestStatus.completed] + return [TestStatus.preparation, TestStatus.testing, TestStatus.completed] class Fork(Base): @@ -117,12 +115,14 @@ class Test(Base): fork = relationship('Fork', uselist=False, back_populates='tests') branch = Column(Text(), nullable=False) commit = Column(String(64), nullable=False) + built_commit = Column(String(64), nullable=True) pr_nr = Column(Integer(), nullable=False, default=0) customized_tests = relationship('CustomizedTest', back_populates='test') progress = relationship('TestProgress', back_populates='test', order_by='TestProgress.id') results = relationship('TestResult', back_populates='test') - def __init__(self, platform, test_type, fork_id, branch, commit, pr_nr=0, token=None) -> None: + def __init__(self, platform, test_type, fork_id, branch, commit, pr_nr=0, token=None, + built_commit=None) -> None: """ Parametrized constructor for the Test model. @@ -140,12 +140,15 @@ def __init__(self, platform, test_type, fork_id, branch, commit, pr_nr=0, token= :type pr_nr: int :param token: The value of the 'token' field of Test model (None by default) :type token: str + :param built_commit: SHA of the commit that was actually built/tested + :type built_commit: str """ self.platform = platform self.test_type = test_type self.fork_id = fork_id self.branch = branch self.commit = commit + self.built_commit = built_commit self.pr_nr = pr_nr if token is None: # Auto-generate token @@ -258,10 +261,12 @@ def get_customized_regressiontests(self) -> Any: rtype: list """ customized_test = self.customized_tests + if len(customized_test) != 0: regression_ids = [r.regression_id for r in customized_test] else: regression_ids = [r.id for r in RegressionTest.query.all()] + return regression_ids @@ -295,7 +300,7 @@ def __init__(self, test_id, status, message, timestamp=None) -> None: tz = get_localzone() if timestamp is None: - timestamp = tz.localize(datetime.datetime.now()) + timestamp = datetime.datetime.now(tz) timestamp = timestamp.astimezone(pytz.UTC) if timestamp.tzinfo is None: diff --git a/mod_upload/controllers.py b/mod_upload/controllers.py index 111287a52..1e38d895b 100755 --- a/mod_upload/controllers.py +++ b/mod_upload/controllers.py @@ -13,7 +13,7 @@ import requests from flask import (Blueprint, flash, g, make_response, redirect, render_template, request, url_for) -from git import InvalidGitRepositoryError, Repo +from github import Auth, Github from werkzeug.utils import secure_filename from decorators import get_menu_entries, template_renderer @@ -249,13 +249,19 @@ def process_id(upload_id): if db_committed: if form.report.data == 'y': - data = "" - try: - kvm_name = config.get('KVM_LINUX_NAME', '') - repo = Repo(os.path.join(repo_folder, 'vm_data', kvm_name, 'unsafe-ccextractor')) - data = repo.git.show(f"{repo.heads.master}:.github/ISSUE_TEMPLATE.md") - except InvalidGitRepositoryError: - log.critical("Could not open CCExtractor's repository") + github_token = config['GITHUB_TOKEN'] + if not github_token: + log.error("GitHub token not configured, cannot create issue") + flash('Could not create GitHub issue - token not configured', 'error') + return redirect(url_for('.index')) + gh = Github(auth=Auth.Token(github_token)) + repository = gh.get_repo(f"{config['GITHUB_OWNER']}/{config['GITHUB_REPOSITORY']}") + response = repository.get_contents(".github/ISSUE_TEMPLATE.md") + encoded_data = response['content'] + if response['encoding'] == 'base64': + data = base64.b64decode(encoded_data).decode() + else: + log.critical(f"Could not get GitHub issue template, invalid data encoding") version = CCExtractorVersion.query.filter(CCExtractorVersion.id == form.version.data).first() data = data.replace("**X.X**", version.version) @@ -486,8 +492,8 @@ def upload_ftp(db, path) -> None: from run import config, log upload_path = str(path) path_parts = upload_path.split(os.path.sep) - # We assume /home/{uid}/ as specified in the model - user_id = path_parts[2] + # We assume /configured_path/{uid}/{file_name} as specified in the model + user_id = path_parts[-2] user = User.query.filter(User.id == user_id).first() if user is None: @@ -513,11 +519,10 @@ def upload_ftp(db, path) -> None: return log.debug("Moving file to temporary folder and changing permissions...") - filename = secure_filename(upload_path.replace(f"/home/{user.id}/", '')) + filename = secure_filename(path_parts[-1]) intermediate_path = os.path.join(config.get('SAMPLE_REPOSITORY', ''), 'TempFiles', filename) log.debug(f"Copy {upload_path} to {intermediate_path}") - shutil.copy(upload_path, intermediate_path) - os.remove(upload_path) + shutil.move(upload_path, intermediate_path) log.debug(f"Checking hash value for {intermediate_path}") file_hash = create_hash_for_sample(intermediate_path) diff --git a/mod_upload/forms.py b/mod_upload/forms.py index ec934a753..1fd0a5fcb 100755 --- a/mod_upload/forms.py +++ b/mod_upload/forms.py @@ -2,16 +2,13 @@ import mimetypes import os -from typing import Any, Type import magic from flask_wtf import FlaskForm -from wtforms import FileField, SelectField, SubmitField, TextAreaField +from wtforms import (FileField, SelectField, SelectMultipleField, SubmitField, + TextAreaField) from wtforms.validators import DataRequired, ValidationError -import mod_home.models -import mod_sample.models -import mod_upload.models from mod_home.models import CCExtractorVersion from mod_sample.models import ForbiddenExtension, ForbiddenMimeType from mod_upload.models import Platform @@ -75,6 +72,7 @@ class CommonSampleForm(FlaskForm): coerce=str, choices=[(p.value, p.description) for p in Platform] ) + tags = SelectMultipleField('Tags', coerce=int) version = SelectField('Version', [DataRequired(message='Version is not selected')], coerce=int) @staticmethod diff --git a/mod_upload/models.py b/mod_upload/models.py index c041470fd..e4368f24b 100644 --- a/mod_upload/models.py +++ b/mod_upload/models.py @@ -203,7 +203,7 @@ def __init__(self, user_id, user_name=None, status=FTPActive.enabled, password=N self.password = password if home_directory is None: - home_directory = f'/home/{user_id}' + home_directory = f'/repository/ftpd/{user_id}' self.dir = home_directory @staticmethod diff --git a/mod_upload/progress_ftp_upload.py b/mod_upload/progress_ftp_upload.py index 77004397c..6c785e7eb 100755 --- a/mod_upload/progress_ftp_upload.py +++ b/mod_upload/progress_ftp_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """Maintain logic to show upload progress.""" import sys diff --git a/mypy.ini b/mypy.ini index 10ec9a04c..574beb202 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,4 +1,14 @@ [mypy] -python_version = 3.7 +python_version = 3.10 ignore_missing_imports = True -warn_unused_ignores = True \ No newline at end of file +warn_unused_ignores = False +exclude = venv* + +# Disable errors for SQLAlchemy 2.0 migration +# These require more extensive refactoring: +# - attr-defined: Model.query is set dynamically at runtime +# - var-annotated: DeclEnum columns need proper type annotations +# - assignment: datetime/date default values with Column types +# - arg-type: Column types vs primitive types in function calls +# - index: Test model indexing issues +disable_error_code = attr-defined, var-annotated, assignment, arg-type, index diff --git a/openapi-ci-api.yaml b/openapi-ci-api.yaml new file mode 100644 index 000000000..350ad85ef --- /dev/null +++ b/openapi-ci-api.yaml @@ -0,0 +1,5418 @@ +openapi: 3.0.3 +info: + title: CCExtractor CI System API + version: 1.2.0 + description: | + Security-hardened JSON-only REST API for the CCExtractor CI/sample platform. + Designed for AI agents and CI automation. Enforces scoped Bearer token auth, + strict input validation, rate limiting on all routes, and safe defaults + throughout. No browser sessions, no HTML, no implicit permissions. + + **Authentication:** All endpoints require bearer token authentication unless + explicitly marked with `security: []` (only /system/health and POST /auth/tokens). + + **Rate-limit headers:** Every response includes `X-RateLimit-Limit`, + `X-RateLimit-Remaining`, and `X-RateLimit-Reset` headers. These are modeled + explicitly on the 429 response for brevity; they are present on all responses + regardless of status code. + + contact: + name: CCExtractor Development + url: https://github.com/CCExtractor/sample-platform + license: + name: GPL-3.0-only + url: https://www.gnu.org/licenses/gpl-3.0.html + +servers: + - url: http://localhost:5000/api/v1 + description: Local development server + - url: https://sampleplatform.ccextractor.org/api/v1 + description: Production + +# +# Global security: all endpoints require auth +# unless explicitly overridden with security: [] +# +security: + - bearerAuth: [] + +tags: + - name: Auth + description: Token issuance and revocation + - name: Runs + description: CI run lifecycle — list, inspect, trigger, and cancel + - name: Samples + description: Media samples and regression test definitions + - name: Results + description: Per-sample output, diffs, and baseline management + - name: Errors and Logs + description: Structured errors and raw log access + - name: System + description: Health, queue, and artifacts + - name: Users + description: Platform accounts and role administration + +# +# SECURITY NOTES (implementers must read) +# +# 1. AUTH MODEL +# - All tokens are opaque, server-side. Never expose session cookies via API. +# - The CI worker token (/ci/progress-reporter) is a separate secret and is +# NOT valid for user-facing API endpoints. +# - Token creation is rate-limited to 5 req/15 min per IP to prevent +# credential stuffing. +# +# 2. SCOPE ENFORCEMENT +# - Scope checks happen at the middleware layer before route handlers. +# - x-required-scope on each operation defines the minimum scope needed. +# - Missing scope → 403 Forbidden (not 401, token is valid but insufficient). +# +# 3. INPUT VALIDATION +# - additionalProperties: false on all request bodies (no mass-assignment). +# - Regex patterns on all free-text IDs (commit_sha, sha256, repository). +# - maxLength on every string field. maxItems on every array. +# - Integer IDs have minimum: 1 (no zero or negative IDs). +# +# 4. OUTPUT SAFETY +# - got=null in TestResultFile means match, not missing output. +# The dummy row (-1,-1,-1,'','error') is translated server-side to +# status=missing_output and never surfaced as a real object. +# - test.failed reflects cancellation only; fail_count is computed from +# TestResult rows. Do not expose test.failed directly. +# - Stack traces in infrastructure errors are opt-in (include_stack=false +# by default) to avoid leaking internal paths. +# +# 5. STORAGE +# - Artifacts may exist in local SAMPLE_REPOSITORY, GCS, or both. +# - storage_status=degraded means one backend only; missing means neither. +# - Never return a download_url that has not been verified to exist. +# - Log endpoints return 404 (not a broken download link) when the log +# file is absent from both storage backends. +# +# 6. RATE LIMITING (all routes) +# - Default: 120 req/min per token (reads), 20 req/min per token (writes). +# - Auth endpoint: 5 req/15 min per IP. +# - Every response includes X-RateLimit-Limit, X-RateLimit-Remaining, +# X-RateLimit-Reset headers. +# - 429 response includes Retry-After header (seconds). +# +# 7. IDEMPOTENCY +# - POST /runs/{run_id}/cancel is idempotent; canceling an already-canceled +# run returns 202 with status=accepted and a no-op message. +# +# 8. DIFF ACCESS +# - The diff route is header-gated on the legacy system (not role-gated). +# The API wraps the XHR path and returns structured JSON. No HTML. +# +# 9. STATUS DERIVATION +# - Run status is derived, not stored. TestStatus has only: preparation, +# testing, completed, canceled (canceled covers both canceled and error). +# The API normalizes this to the 7-value enum below. +# - RunSample.status is computed from TestResult + TestResultFile + +# expected exit code + multiple acceptable baselines. +# - fail_count and missing_output_count in RunSummary are mutually +# exclusive. A sample appears in exactly one bucket (missing_output +# is checked first; if the dummy sentinel row is detected the function +# returns immediately without evaluating fail conditions). +# +# 10. REPOSITORY PERMISSIONS +# - POST /runs enforces a repo-aware permission check. Triggering a run +# against the main configured repository (GITHUB_OWNER/GITHUB_REPOSITORY) +# requires the contributor role or above. Any authenticated user with +# runs:write scope may trigger runs against fork repositories. There is +# no global repository allowlist; the elevated-role check applies only +# to the main configured repository. +# + +paths: + + # AUTH + + /auth/tokens: + get: + tags: [Auth] + summary: List API tokens + operationId: listTokens + description: > + Lists tokens for the authenticated user. Non-admin users see only their + own tokens. Admins may append ?all=true to list tokens across the entire + system; non-admin callers sending ?all=true receive 403. + + Plaintext token values are never included in list responses. + security: + - bearerAuth: [] + x-required-scope: tokens:manage + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: all + in: query + schema: + type: boolean + description: > + Admin only. Set to true to list tokens for all users in the system. + Non-admin callers receive 403 if this parameter is present and true. + responses: + "200": + description: Paginated list of tokens (without plaintext secrets). + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ApiTokenItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Auth] + summary: Create an API token + operationId: createToken + description: > + Rate-limited to 5 requests per 15 minutes per IP. Tokens are opaque + and stored server-side. Scopes are additive; request only what you need. + Tokens expire after expires_in_days (default 7, max 30). + security: [] + x-rate-limit: "5/15min per IP" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/TokenCreateRequest" + responses: + "201": + description: Token created. Store the token value; it will not be shown again. + content: + application/json: + schema: + $ref: "#/components/schemas/AuthToken" + "400": + $ref: "#/components/responses/BadRequest" + "401": + description: Invalid credentials + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: invalid_credentials + message: Email or password is incorrect. + details: {} + "403": + description: > + Authenticated caller tried to create a token with higher scopes + than their current token. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: forbidden + message: Cannot create token with scopes you do not possess. + details: {} + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/tokens/current: + delete: + tags: [Auth] + summary: Revoke the current API token + operationId: revokeCurrentToken + description: > + Immediately invalidates the token used in the Authorization header. + Subsequent requests with the same token will receive 401. + + No specific scope is required beyond authentication — any valid token + can self-revoke. This is the preferred way to clean up a token when + you have it in hand but do not know its numeric ID. + security: + - bearerAuth: [] + responses: + "204": + description: Token revoked + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/tokens/{token_id}: + delete: + tags: [Auth] + summary: Revoke a specific API token by ID + operationId: revokeToken + description: > + Revokes the token identified by token_id. + Any authenticated caller may revoke their own tokens (no scope required). + Admins may revoke any user's token. + A non-admin attempting to revoke a token they do not own receives 404 + to prevent token-ID enumeration. + + To revoke the token currently in use without knowing its ID, use + DELETE /auth/tokens/current instead. + security: + - bearerAuth: [] + parameters: + - name: token_id + in: path + required: true + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "204": + description: Token revoked successfully. + "401": + $ref: "#/components/responses/Unauthorized" + "404": + description: > + Token not found. Non-admin users attempting to revoke another user's token receive a uniform 404 response to prevent token-ID enumeration. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: not_found + message: Token not found. + details: {} + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/signup: + post: + tags: [Auth] + summary: Send a registration link to an email address + operationId: signup + description: > + Answers the same way whether or not the address is already + registered, because a different reply would tell an anonymous caller + who has an account. The link lands on the classic completion page, + which is where the account is actually created. Unauthenticated and + limited to 5 requests per 15 minutes per IP. + parameters: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [email] + properties: + email: + type: string + format: email + responses: + "202": + description: Accepted; an email has been sent if appropriate + content: + application/json: + schema: + type: object + properties: + sent: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/password-reset: + post: + tags: [Auth] + summary: Send a password reset link + operationId: requestPasswordReset + description: > + Silent about whether the address is registered, for the same reason + signup is. Unauthenticated and limited per IP. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [email] + properties: + email: + type: string + format: email + responses: + "202": + description: Accepted; an email has been sent if appropriate + content: + application/json: + schema: + type: object + properties: + sent: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/password-reset/complete: + post: + tags: [Auth] + summary: Set a new password using a link from the reset email + operationId: completePasswordReset + description: > + The signature covers the current password hash, so a link stops + working the moment it is used or the password changes by any other + route. Expired, unknown and forged links share one reply. + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [user_id, expires, mac, password] + properties: + user_id: + type: integer + minimum: 1 + expires: + type: integer + mac: + type: string + minLength: 1 + maxLength: 128 + password: + type: string + minLength: 10 + maxLength: 500 + description: > + Length bounds come from the MIN_PWD_LEN and MAX_PWD_LEN + settings; 10 and 500 are the shipped defaults. + responses: + "200": + description: Password changed + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + password_changed: + type: boolean + "400": + $ref: "#/components/responses/BadRequest" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/me: + get: + tags: [Auth] + summary: Identify the account behind the current token + operationId: getCurrentUser + description: > + A token does not carry the account's role, so a client cannot infer + what the caller is allowed to do. Clients gate their interface on + this instead of guessing, and the granted scopes are echoed so a + client can also hide what this token could never call. Requires no + particular scope: it reports on the caller and discloses nothing + another endpoint would withhold. + security: + - bearerAuth: [] + responses: + "200": + description: The authenticated account + content: + application/json: + schema: + $ref: "#/components/schemas/CurrentUser" + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # RUNS + patch: + tags: [Auth] + summary: Change your own name, email or password + operationId: updateAccount + description: > + Touching the email or the password needs the current one as well: + the bearer token proves the request came from a signed-in session, + not that the sender knows the account's own credentials. + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + name: + type: string + minLength: 1 + maxLength: 50 + email: + type: string + format: email + new_password: + type: string + minLength: 10 + maxLength: 500 + description: > + Length bounds come from the MIN_PWD_LEN and MAX_PWD_LEN + settings; 10 and 500 are the shipped defaults. + current_password: + type: string + minLength: 1 + maxLength: 500 + responses: + "200": + description: Account updated + content: + application/json: + schema: + $ref: "#/components/schemas/PlatformUser" + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + + /runs: + get: + tags: [Runs] + summary: List CI runs + operationId: listRuns + description: > + The underlying table is capped at the 50 most recent runs + in the current implementation; this endpoint adds full pagination. + Sorted by -created_at by default (newest first). + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/RunStatus" + - $ref: "#/components/parameters/Branch" + - $ref: "#/components/parameters/CommitSha" + - name: ccx_version + in: query + description: > + Filter to the commit a CCExtractor release was cut from. A version + the platform does not know returns 400. + schema: + type: string + maxLength: 10 + - $ref: "#/components/parameters/Repository" + - $ref: "#/components/parameters/Platform" + - $ref: "#/components/parameters/CreatedAfter" + - $ref: "#/components/parameters/CreatedBefore" + - name: sort + in: query + schema: + type: string + default: -created_at + enum: [created_at, -created_at, run_id, -run_id] + description: Sort field. Prefix with - for descending order. + responses: + "200": + description: Paginated runs + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Run" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Runs] + summary: Trigger a new CI run + operationId: createRun + description: > + Requires runs:write scope and contributor role or above. + The regression_test_ids set is validated against active tests only. + If omitted, all active regression tests are used. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RunCreateRequest" + responses: + "202": + description: Run queued. Poll /runs/{run_id}/progress for status. + content: + application/json: + schema: + $ref: "#/components/schemas/Run" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "422": + $ref: "#/components/responses/UnprocessableEntity" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}: + get: + tags: [Runs] + summary: Get a CI run + operationId: getRun + description: > + Returns normalized run status derived from TestProgress rows. + status=canceled covers both explicit cancellation and infrastructure + errors (the underlying model does not distinguish them). + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run details + content: + application/json: + schema: + $ref: "#/components/schemas/Run" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/summary: + get: + tags: [Runs] + summary: Get pass/fail summary for a run + operationId: getRunSummary + description: > + fail_count is computed from TestResult rows, not from test.failed. + test.failed only reflects whether the final progress status is + canceled — it does not reflect regression test outcomes. + Use this endpoint, not test.failed, to triage a run. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run summary + content: + application/json: + schema: + $ref: "#/components/schemas/RunSummary" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/progress: + get: + tags: [Runs] + summary: Get progress events for a run + operationId: getRunProgress + description: > + Progress events are sourced from TestProgress rows written by the CI + worker via /ci/progress-reporter. Messages are unstructured text. + Structured error types are aspirational until the worker protocol + emits structured JSON. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + description: > + Filter progress events by the status the CI worker recorded. A + queued run has no progress events yet, so queued is not a valid + filter here. Any other value returns 400. + schema: + type: string + enum: [preparation, testing, completed, canceled] + responses: + "200": + description: Paginated progress events + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ProgressEvent" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/restart: + post: + tags: [Runs] + summary: Queue a finished or stuck run to be executed again + operationId: restartRun + description: > + Clears the run's results and progress trail, which is what makes CI + pick it up again. The run keeps its id, so existing links stay valid, + and the old results are replaced rather than kept alongside the new + ones. Requires runs:write scope. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "202": + description: Restart accepted + content: + application/json: + schema: + $ref: "#/components/schemas/RunActionResult" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/cancel: + post: + tags: [Runs] + summary: Cancel a queued or running CI run + operationId: cancelRun + description: > + Idempotent. Canceling an already-canceled or completed run returns + 202 with a no-op message rather than an error. + Requires runs:write scope. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, tester, contributor] + parameters: + - $ref: "#/components/parameters/RunId" + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + reason: + type: string + minLength: 5 + maxLength: 255 + description: > + Reason for cancellation, stored in the audit log. + additionalProperties: false + responses: + "202": + description: Cancellation accepted (or no-op if already terminal) + content: + application/json: + schema: + $ref: "#/components/schemas/RunActionResult" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/config: + get: + tags: [Runs] + summary: Get run configuration and test matrix + operationId: getRunConfig + description: > + regression_test_ids lists IDs included in this run. When no custom + set was configured, all regression tests are returned. + Implementers must filter by active=true explicitly. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + responses: + "200": + description: Run configuration + content: + application/json: + schema: + $ref: "#/components/schemas/RunConfig" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # SAMPLES + + /runs/{run_id}/samples: + get: + tags: [Samples] + summary: List regression test results in a run + operationId: listRunSamples + description: > + Returns one entry per regression test result, not one per unique media + file. A single media sample may yield multiple entries if it has + multiple regression tests (different command flags). + sample_progress in the legacy JSON endpoint is len(test.results) over + total regression tests; it does not reflect multi-output completeness. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + description: > + Filter by derived per-sample status. Only the statuses the API can + derive for a sample are accepted; any other value returns 400. + schema: + type: string + enum: [pass, fail, missing_output, not_started] + - name: name + in: query + schema: + type: string + maxLength: 100 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: category + in: query + schema: + type: string + maxLength: 50 + responses: + "200": + description: Paginated regression test results + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/RunSample" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{regression_test_id}: + get: + tags: [Samples] + summary: Get full details for a regression test result in a run + operationId: getRunSample + description: > + Returns the result for a specific regression test within a run. + Note: the path parameter is regression_test_id, not a media sample ID. + A single media sample may have multiple regression tests. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: Regression test result details + content: + application/json: + schema: + $ref: "#/components/schemas/RunSample" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples: + get: + tags: [Samples] + summary: List all known media samples + operationId: listSamples + description: > + Returns paginated media sample metadata. Samples are the original + media files uploaded for regression testing. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: status + in: query + description: > + Derived from linked regression tests. The sample table itself has + no quarantine state; active/inactive reflects whether any active + regression tests reference the sample. + schema: + type: string + enum: [active, inactive] + - name: name + in: query + schema: + type: string + maxLength: 100 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: sha256 + in: query + schema: + type: string + pattern: '^[a-fA-F0-9]{64}$' + - name: extension + in: query + schema: + type: string + maxLength: 10 + pattern: '^[a-zA-Z0-9]+$' + responses: + "200": + description: Paginated media samples + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Sample" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}: + get: + tags: [Samples] + summary: Get media sample metadata + operationId: getSample + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Media sample metadata + content: + application/json: + schema: + $ref: "#/components/schemas/Sample" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + patch: + tags: [Samples] + summary: Edit a sample's tags and upload metadata + operationId: updateSample + description: > + Notes, parameters, platform and version live on the upload row + rather than the sample, so a sample that arrived without one (over + FTP, or seeded directly) can only have its tags changed and asking + for the rest returns 409. Tags are given by name. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + tags: + type: array + maxItems: 50 + items: + type: string + minLength: 1 + maxLength: 64 + notes: + type: string + maxLength: 1024 + parameters: + type: string + maxLength: 1024 + platform: + type: string + enum: [linux, windows, mac, bsd] + version: + type: string + minLength: 1 + maxLength: 10 + responses: + "200": + description: Sample updated + content: + application/json: + schema: + type: object + properties: + sample_id: + type: integer + tags: + type: array + items: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + delete: + tags: [Samples] + summary: Delete a sample, its extra files and its media info + operationId: deleteSample + description: > + Refused while any regression test still points at the sample, + because removing the media would leave those tests unable to run and + their history describing a sample nobody can fetch. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Sample deleted + content: + application/json: + schema: + type: object + properties: + sample_id: + type: integer + deleted: + type: boolean + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/details: + get: + tags: [Samples] + summary: Get the full detail payload for a media sample + operationId: getSampleDetails + description: > + Extends the summary payload with the upload record, any extra files + attached to the sample, and the parsed MediaInfo tree. Media info is + best effort: a sample whose XML is missing or unparseable reports + null rather than failing the response. Unlike the classic page this + never regenerates the XML, because a GET must not write to the + sample repository. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Full media sample detail + content: + application/json: + schema: + $ref: "#/components/schemas/SampleDetails" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/download: + get: + tags: [Samples] + summary: Locate a sample's media file in storage + operationId: downloadSample + description: > + Returns a signed URL rather than the bytes. Samples run to several + gigabytes, so streaming one through the API would hold a worker for + the length of the transfer. Where only the local copy exists there is + no URL to hand out and storage_status reports that instead. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Where the sample file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/history: + get: + tags: [Samples] + summary: Get regression test result history for a sample across runs + operationId: getSampleHistory + description: > + Use failure_signature for flake detection: a stable signature across + multiple runs on different commits indicates a genuine regression, + not infrastructure noise. + + + Entries cover every regression test on the sample, newest run first, + so limit counts rows rather than runs. Pass regression_test_id to + follow one test, where limit then means that many runs of it. + + + With status set, the filter runs over a bounded scan of recent + results; pagination.truncated is true when the sample has more + history than the scan covered, and total then undercounts. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - $ref: "#/components/parameters/RegressionTestIdFilter" + - $ref: "#/components/parameters/SampleHistoryStatus" + - $ref: "#/components/parameters/Branch" + - $ref: "#/components/parameters/Platform" + - $ref: "#/components/parameters/CreatedAfter" + - $ref: "#/components/parameters/CreatedBefore" + responses: + "200": + description: Paginated sample history + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/SampleHistoryEntry" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests: + get: + tags: [Samples] + summary: List regression test definitions + operationId: listRegressionTests + description: > + The active filter must be applied explicitly. When no custom set is + defined, all regression tests are returned — including inactive ones. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: active + in: query + schema: + type: boolean + default: true + - name: category + in: query + schema: + type: string + maxLength: 50 + - name: tag + in: query + schema: + type: string + maxLength: 50 + - name: sample_id + in: query + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Paginated regression test definitions + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Samples] + summary: Create a regression test definition + operationId: createRegressionTest + description: > + Admin or contributor only. The test is created inactive unless the + body says otherwise, so it cannot join a CI suite before someone has + seen the output it actually produces on a verification run. Every + named category must already exist; an unknown name rejects the whole + request rather than silently dropping it. + security: + - bearerAuth: [] + x-required-scope: runs:write + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTestCreateRequest" + responses: + "201": + description: The created regression test + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /categories: + get: + tags: [Samples] + summary: List regression test categories + operationId: listCategories + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated categories, ordered by name + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [Samples] + summary: Create a category + operationId: createCategory + description: Admin or contributor only. Names are unique. + security: + - bearerAuth: [] + x-required-scope: runs:write + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CategoryCreateRequest" + responses: + "201": + description: The created category + content: + application/json: + schema: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /categories/{category_id}: + patch: + tags: [Samples] + summary: Rename or re-describe a category + operationId: updateCategory + description: Admin or contributor only. An empty body is rejected. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/CategoryId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/CategoryUpdateRequest" + responses: + "200": + description: The updated category + content: + application/json: + schema: + $ref: "#/components/schemas/Category" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + delete: + tags: [Samples] + summary: Delete a category + operationId: deleteCategory + description: > + Admin or contributor only. A category still attached to regression + tests is refused with 409 and a count, because dropping it would + change which tests a suite selection picks up. Detach the tests + first by PATCHing their categories. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/CategoryId" + responses: + "200": + description: The category was deleted + content: + application/json: + schema: + $ref: "#/components/schemas/DeletedResource" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}: + get: + tags: [Samples] + summary: Get one regression test with its baselines + operationId: getRegressionTest + description: > + Adds the expected outputs and the alternative hashes accepted as + variants, which the list endpoint omits because they multiply the + payload for every row. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: Regression test detail + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTestDetail" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + delete: + tags: [Samples] + summary: Delete a regression test + operationId: deleteRegressionTest + description: > + Admin or contributor only. A test that has already run is refused + with 409 and the number of results referencing it, because deleting + it would erase evidence of past regressions. Retire such a test with + PATCH active=false instead. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/RegressionTestId" + responses: + "200": + description: The regression test was deleted + content: + application/json: + schema: + $ref: "#/components/schemas/DeletedResource" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + patch: + tags: [Samples] + summary: Update part of a regression test definition + operationId: updateRegressionTest + description: > + Admin or contributor only. Only the fields present in the body are + written, so two clients editing different fields cannot clobber each + other. An empty body is rejected. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - $ref: "#/components/parameters/RegressionTestId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTestUpdateRequest" + responses: + "200": + description: The updated regression test + content: + application/json: + schema: + $ref: "#/components/schemas/RegressionTest" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/download: + get: + tags: [Regression Tests] + summary: Locate the baseline a regression test is expected to reproduce + operationId: downloadRegressionTestOutput + description: > + Matches the output against its parent test, so an id belonging to a + different test reads as absent rather than resolving. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + responses: + "200": + description: Where the baseline file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants/{variant_id}/download: + get: + tags: [Regression Tests] + summary: Locate one of the alternative outputs accepted for a baseline + operationId: downloadRegressionTestVariant + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + - name: variant_id + in: path + required: true + description: Id of the variant, from the test detail payload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Where the variant file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants: + post: + tags: [Regression Tests] + summary: Accept another output hash as correct for a baseline + operationId: createRegressionTestVariant + description: > + A test can legitimately produce different bytes on different + platforms or CCExtractor builds. Recording the hash here makes those + runs pass without overwriting the baseline everyone else is compared + against, which is what promoting to baseline would do. + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, contributor] + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [hash] + properties: + hash: + type: string + maxLength: 128 + pattern: '^[A-Za-z0-9]+$' + responses: + "201": + description: Variant accepted + content: + application/json: + schema: + type: object + required: [id, hash] + properties: + id: + type: integer + minimum: 1 + hash: + type: string + maxLength: 128 + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /regression-tests/{regression_test_id}/outputs/{output_id}/variants/{variant_id}: + delete: + tags: [Regression Tests] + summary: Stop accepting one alternative output for a baseline + operationId: deleteRegressionTestVariant + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin, contributor] + parameters: + - $ref: "#/components/parameters/RegressionTestId" + - $ref: "#/components/parameters/OutputId" + - name: variant_id + in: path + required: true + description: Id of the variant, from the test detail payload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Variant removed + content: + application/json: + schema: + type: object + required: [id, deleted] + properties: + id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/media-info/download: + get: + tags: [Samples] + summary: Locate the MediaInfo XML written alongside a sample + operationId: downloadSampleMediaInfo + description: > + The parsed tree is already on /samples/{sample_id}/details; this is + for anyone who wants the original file MediaInfo produced. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + responses: + "200": + description: Where the media info file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/extra-files/{extra_id}/download: + get: + tags: [Samples] + summary: Locate one of the files uploaded alongside a sample + operationId: downloadSampleExtraFile + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/SampleId" + - name: extra_id + in: path + required: true + description: Id of the accompanying file, from the sample details payload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Where the accompanying file is stored + content: + application/json: + schema: + $ref: "#/components/schemas/StoredFile" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/{sample_id}/extra-files/{extra_id}: + delete: + tags: [Samples] + summary: Delete one of the files uploaded alongside a sample + operationId: deleteSampleExtraFile + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/SampleId" + - name: extra_id + in: path + required: true + description: Id of the accompanying file, from the sample details payload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Accompanying file deleted + content: + application/json: + schema: + type: object + properties: + id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /tags: + get: + tags: [Samples] + summary: List the tags a sample can be labelled with + operationId: listTags + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated tags + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + type: object + properties: + id: + type: integer + name: + type: string + description: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + post: + tags: [Samples] + summary: Create a tag + operationId: createTag + security: + - bearerAuth: [] + x-required-scope: runs:write + x-required-roles: [admin] + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 64 + description: + type: string + maxLength: 1024 + responses: + "201": + description: Tag created + content: + application/json: + schema: + type: object + properties: + id: + type: integer + name: + type: string + description: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/about: + get: + tags: [System] + summary: Report the versions this platform runs against + operationId: systemAbout + description: > + The classic about page is static prose. The parts worth reading from + a client are the CCExtractor release under test and the commit the + platform itself was deployed from. + security: + - bearerAuth: [] + x-required-scope: system:read + responses: + "200": + description: Platform and CCExtractor versions + content: + application/json: + schema: + type: object + properties: + platform_commit: + type: string + nullable: true + ccextractor_version: + type: string + nullable: true + ccextractor_released: + type: string + nullable: true + last_tested_commit: + type: string + nullable: true + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /users/{user_id}/deactivate: + post: + tags: [Auth] + summary: Anonymise an account and lock it out + operationId: deactivateUser + description: > + The row stays so the uploads and runs it owns keep an author, which + is why this scrubs the identity instead of deleting. Open to admins + and to the account's owner. Scope-free for the same reason revoking + your own token is: closing your own account cannot depend on + tokens:manage, which no role below admin may hold. + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/UserId" + responses: + "200": + description: Account deactivated and its own tokens revoked + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + deactivated: + type: boolean + tokens_revoked: + type: integer + description: > + How many of the account's tokens were still live and + have now been revoked. Scrambling the password only + stops new ones being minted, so the ones already + issued are ended here as well. + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /samples/upload: + post: + tags: [Uploads] + summary: Upload a media file into the queue + operationId: uploadSample + description: > + Uploading and describing are two steps, as on the classic pages: the + transfer is slow and the metadata needs a CCExtractor version the + uploader may have to look up, so the bytes are banked first. The file + is hashed as it is written, and content already in the library or the + queue is refused with 409. Extension and mime type are checked + against the forbidden lists. + security: + - bearerAuth: [] + x-required-scope: runs:write + requestBody: + required: true + content: + multipart/form-data: + schema: + type: object + required: [file] + properties: + file: + type: string + format: binary + responses: + "201": + description: Upload queued + content: + application/json: + schema: + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "413": + description: File larger than the configured upload limit + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples: + get: + tags: [Uploads] + summary: List uploads waiting to be described + operationId: listQueuedSamples + description: > + Admins see the whole queue; everyone else sees only their own + uploads. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated queued uploads + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples/{queued_id}: + get: + tags: [Uploads] + summary: Get one queued upload + operationId: getQueuedSample + description: > + Someone else's upload reads as absent rather than forbidden, so an + id cannot be used to find out what others have queued. + security: + - bearerAuth: [] + x-required-scope: runs:read + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: The queued upload + content: + application/json: + schema: + type: object + properties: + id: + type: integer + sha: + type: string + extension: + type: string + original_name: + type: string + user_id: + type: integer + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + delete: + tags: [Uploads] + summary: Discard a queued upload + operationId: deleteQueuedSample + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + responses: + "200": + description: Queued upload discarded + content: + application/json: + schema: + type: object + properties: + id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples/{queued_id}/finalize: + post: + tags: [Uploads] + summary: Turn a queued upload into a sample + operationId: finalizeQueuedSample + description: > + The file only moves into TestFiles once the rows are committed, so a + failure part way through leaves the upload in the queue to retry + rather than a sample row pointing at a file that was never placed. + Reporting the sample as a GitHub issue stays on the classic page. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [version, platform] + properties: + version: + type: string + minLength: 1 + maxLength: 10 + description: CCExtractor release the sample was produced with + platform: + type: string + enum: [linux, windows, mac, bsd] + parameters: + type: string + maxLength: 1024 + notes: + type: string + maxLength: 1024 + responses: + "201": + description: Sample created from the upload + content: + application/json: + schema: + type: object + properties: + sample_id: + type: integer + sha: + type: string + original_name: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /users/{user_id}/password-reset: + post: + tags: [Auth] + summary: Send a reset link to another account, or to your own + operationId: sendUserPasswordReset + description: > + Unlike the public /auth/password-reset this names an account rather + than an address, so it says plainly when the id is unknown: the + caller is already signed in and can list users anyway. Scope-free + for the same reason deactivation is. + security: + - bearerAuth: [] + parameters: + - $ref: "#/components/parameters/UserId" + responses: + "202": + description: Reset link sent + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + sent: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /auth/me/github: + get: + tags: [Auth] + summary: Whether your account is connected to GitHub + operationId: getGithubLink + description: > + Connecting is a browser redirect, so this hands back the URL to send + somebody to rather than performing it. Trading the code GitHub + returns for a token stays on the classic callback, which already + holds the client secret and the registered redirect. The stored + token is never part of this response; the URL carries only the + client id and the scope asked for, both of which are public. + security: + - bearerAuth: [] + responses: + "200": + description: Connection status and where to start one + content: + application/json: + schema: + type: object + properties: + linked: + type: boolean + github_login: + type: string + nullable: true + authorize_url: + type: string + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + delete: + tags: [Auth] + summary: Forget this platform's copy of your GitHub connection + operationId: unlinkGithub + description: > + Only ever the caller's own. This drops the token the platform + holds; the authorisation itself is withdrawn from GitHub's own + applications page, which is the only place that can really end it. + security: + - bearerAuth: [] + responses: + "200": + description: The connection has been forgotten + content: + application/json: + schema: + type: object + properties: + linked: + type: boolean + github_login: + type: string + nullable: true + "401": + $ref: "#/components/responses/Unauthorized" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + /auth/me/ftp-credentials: + get: + tags: [Uploads] + summary: Get your own FTP details for the ingest server + operationId: getFtpCredentials + description: > + Created on first ask and stable afterwards. Only ever the caller's + own: these are working credentials, so there is no version of this + that reads someone else's. The password is stored in the clear by + design, being random and unchangeable, so this is the only place it + can come from. + security: + - bearerAuth: [] + responses: + "200": + description: FTP connection details + content: + application/json: + schema: + type: object + properties: + host: + type: string + port: + type: string + username: + type: string + password: + type: string + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /queued-samples/{queued_id}/link: + post: + tags: [Uploads] + summary: Attach a queued upload to an existing sample + operationId: linkQueuedSample + description: > + For material that belongs with a sample without being one, such as a + subtitle track to compare against. The classic page offered this but + never carried it out: its confirm step checks permissions and then + redirects without touching anything, so this is the behaviour it + described rather than the behaviour it had. + security: + - bearerAuth: [] + x-required-scope: runs:write + parameters: + - name: queued_id + in: path + required: true + description: Id of the queued upload + schema: + type: integer + minimum: 1 + maximum: 2147483647 + requestBody: + required: true + content: + application/json: + schema: + type: object + required: [sample_id] + properties: + sample_id: + type: integer + minimum: 1 + responses: + "201": + description: Upload attached as an extra file + content: + application/json: + schema: + type: object + properties: + id: + type: integer + sample_id: + type: integer + filename: + type: string + "400": + $ref: "#/components/responses/BadRequest" + "409": + $ref: "#/components/responses/Conflict" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # PLATFORM CONFIGURATION + + /system/maintenance: + get: + tags: [System] + summary: Report maintenance state for every platform + operationId: getMaintenance + description: > + A platform that has never been put into maintenance is reported as + disabled=false rather than omitted, so a client always receives one + entry per platform. Keyed on platforms rather than the data + collection envelope: the list comes from the platform enum, not a + growable table, so there is nothing to paginate. Admin only, like + the rest of the platform configuration. + security: + - bearerAuth: [] + x-required-scope: system:read + responses: + "200": + description: Maintenance state per platform + content: + application/json: + schema: + type: object + properties: + platforms: + type: array + items: + $ref: "#/components/schemas/MaintenanceState" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/maintenance/{platform}: + patch: + tags: [System] + summary: Pause or resume CI for one platform + operationId: updateMaintenance + description: > + Admin only. While a platform is disabled new runs still queue; they + are simply not handed to a VM until it is resumed. + security: + - bearerAuth: [] + x-required-scope: system:write + parameters: + - $ref: "#/components/parameters/PlatformPath" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/MaintenanceUpdateRequest" + responses: + "200": + description: The updated maintenance state + content: + application/json: + schema: + $ref: "#/components/schemas/MaintenanceState" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/blocked-users: + get: + tags: [System] + summary: List GitHub accounts blocked from triggering CI runs + operationId: listBlockedUsers + description: > + Admin only. The blocklist names GitHub accounts, so it is not + readable with system:read alone. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated blocked GitHub accounts + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/BlockedUser" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [System] + summary: Block a GitHub account from triggering CI runs + operationId: createBlockedUser + description: > + Admin only. Keyed on the numeric GitHub account id rather than the + login, because a login can be changed and reused, which would + silently unblock somebody. + security: + - bearerAuth: [] + x-required-scope: system:write + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BlockedUserCreateRequest" + responses: + "201": + description: The account is now blocked + content: + application/json: + schema: + $ref: "#/components/schemas/BlockedUser" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/blocked-users/{user_id}: + delete: + tags: [System] + summary: Unblock a GitHub account + operationId: deleteBlockedUser + description: Admin only. + security: + - bearerAuth: [] + x-required-scope: system:write + parameters: + - $ref: "#/components/parameters/BlockedUserId" + responses: + "200": + description: The account is no longer blocked + content: + application/json: + schema: + type: object + properties: + user_id: + type: integer + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/forbidden-extensions: + get: + tags: [System] + summary: List file extensions rejected on upload + operationId: listForbiddenExtensions + description: Admin only, like the rest of the platform configuration. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated forbidden extensions, without a leading dot + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + type: string + maxLength: 32 + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + post: + tags: [System] + summary: Forbid a file extension on upload + operationId: createForbiddenExtension + description: > + Admin only. Stored lower-cased and without a leading dot, matching + how upload validation looks it up. + security: + - bearerAuth: [] + x-required-scope: system:write + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenExtensionCreateRequest" + responses: + "201": + description: The extension is now forbidden + content: + application/json: + schema: + type: object + properties: + extension: + type: string + maxLength: 32 + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "409": + $ref: "#/components/responses/Conflict" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/forbidden-extensions/{extension}: + delete: + tags: [System] + summary: Allow a file extension to be uploaded again + operationId: deleteForbiddenExtension + description: Admin only. A leading dot in the path is tolerated. + security: + - bearerAuth: [] + x-required-scope: system:write + parameters: + - $ref: "#/components/parameters/ExtensionPath" + responses: + "200": + description: The extension is no longer forbidden + content: + application/json: + schema: + type: object + properties: + extension: + type: string + maxLength: 32 + deleted: + type: boolean + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # USERS + + /users: + get: + tags: [Users] + summary: List platform users + operationId: listUsers + description: > + Admin only. Paginated like every other collection here, because the + platform has hundreds of accounts. Credentials and GitHub tokens are + never included. + security: + - bearerAuth: [] + x-required-scope: tokens:manage + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + responses: + "200": + description: Paginated platform users + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/PlatformUser" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /users/{user_id}: + get: + tags: [Auth] + summary: Get one platform user + operationId: getUser + security: + - bearerAuth: [] + x-required-scope: tokens:manage + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/UserId" + responses: + "200": + description: The user + content: + application/json: + schema: + $ref: "#/components/schemas/PlatformUser" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + patch: + tags: [Users] + summary: Change a user's role + operationId: updateUserRole + description: > + Admin only. An admin cannot change their own role: demoting the last + admin through this endpoint would leave nobody able to undo it, so + that request is refused with 403. + security: + - bearerAuth: [] + x-required-scope: tokens:manage + parameters: + - $ref: "#/components/parameters/UserId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/RoleUpdateRequest" + responses: + "200": + description: The updated user + content: + application/json: + schema: + $ref: "#/components/schemas/PlatformUser" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + description: > + The caller is not an admin, or attempted to change their own role. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: forbidden + message: You cannot change your own role. + details: {} + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # RESULTS + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/expected: + get: + tags: [Results] + summary: Get expected output for a regression test result + operationId: getExpectedOutput + description: > + Expected output is a file reference stored under TestResults using the + regression output extension. Resolved from GCS or local + SAMPLE_REPOSITORY at request time. storage_status reflects which + backends have the file. Do not assume local and GCS are always in sync. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - $ref: "#/components/parameters/Format" + responses: + "200": + description: Expected output file + content: + application/json: + schema: + $ref: "#/components/schemas/OutputFile" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/actual: + get: + tags: [Results] + summary: Get actual output generated by a regression test in a run + operationId: getActualOutput + description: > + IMPORTANT: TestResultFile.got = null means the actual output MATCHED + expected, not that actual output is missing. This is a semantic trap + in the data model. Missing output is represented by a dummy row + (-1,-1,-1,'','error') which the API translates to status=missing_output + and returns 404. A 200 response always contains a real output file. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - $ref: "#/components/parameters/Format" + responses: + "200": + description: Actual output file (output exists and differs from expected) + content: + application/json: + schema: + $ref: "#/components/schemas/OutputFile" + "303": + description: Output matched expected. Redirected to /expected. + headers: + Location: + schema: + type: string + format: uri + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/diff: + get: + tags: [Results] + summary: Get expected-vs-actual diff for a failing regression test result + operationId: getDiff + description: > + The legacy diff route is header-gated (X-Requested-With: XMLHttpRequest), + not role-gated. The 403 seen on direct browser requests was a + header-check artifact. This endpoint wraps the XHR logic and returns + structured JSON — no HTML, no 50-line truncation. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/RegressionId" + - $ref: "#/components/parameters/OutputId" + - name: context_lines + in: query + schema: + type: integer + minimum: 1 + maximum: 50 + default: 3 + - name: format + in: query + schema: + type: string + enum: [structured, unified] + default: structured + responses: + "200": + description: Structured or unified diff + content: + application/json: + schema: + oneOf: + - $ref: "#/components/schemas/Diff" + - $ref: "#/components/schemas/UnifiedDiff" + discriminator: + propertyName: format + mapping: + structured: "#/components/schemas/Diff" + unified: "#/components/schemas/UnifiedDiff" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/baseline-approval: + post: + tags: [Results] + summary: Approve actual output as new expected baseline + operationId: approveBaseline + description: > + Requires baselines:write scope and admin role. + This is a destructive write — the approved output becomes the new + expected baseline for the regression test. + security: + - bearerAuth: [] + x-required-scope: baselines:write + x-required-roles: [admin] + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + requestBody: + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/BaselineApprovalRequest" + responses: + "200": + description: Baseline approval applied immediately. + content: + application/json: + schema: + $ref: "#/components/schemas/BaselineApproval" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # ERRORS AND LOGS + + /runs/{run_id}/errors: + get: + tags: [Errors and Logs] + summary: Get structured test errors for a run + operationId: listRunErrors + description: > + Error types are derived from TestResult and TestResultFile rows. + missing_output is detected from the dummy (-1,-1,-1,'','error') row + pattern, not from got=null (which means match, not missing). + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [test_failure, exit_code_mismatch, missing_output, diff_mismatch] + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + - name: sample_id + in: query + schema: + type: integer + minimum: 1 + responses: + "200": + description: Paginated test errors + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/infrastructure-errors: + get: + tags: [Errors and Logs] + summary: Get worker, provisioning, and build errors for a run + operationId: listInfraErrors + description: > + Errors are extracted from TestProgress rows written by the CI worker. + Messages are currently unstructured text. The type filter does + best-effort text matching until the worker protocol emits structured + error types. + Stack traces are opt-in (include_stack defaults to false) to avoid + leaking internal paths to unauthorized callers. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [queue, vm_provisioning, checkout, merge, build, worker, web_server, storage] + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + - name: include_stack + in: query + schema: + type: boolean + default: false + description: > + Default false. Set true only when debugging infrastructure failures. + Stacks may contain internal paths; access requires system:read scope. + responses: + "200": + description: Paginated infrastructure errors + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorItem" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/logs: + get: + tags: [Errors and Logs] + summary: Get raw logs for a run + operationId: getRunLogs + description: > + Logs are stored at SAMPLE_REPOSITORY/LogFiles/{id}.txt and served + via GCS signed URL. Returns 404 — not a broken download link — when + the file is absent from both local and GCS storage. + Uses cursor-based pagination. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Cursor" + - name: level + in: query + schema: + type: string + enum: [debug, info, warning, error, critical] + - name: source + in: query + schema: + type: string + enum: [orchestrator, worker, build, test_runner, web] + - name: contains + in: query + schema: + type: string + maxLength: 100 + responses: + "200": + description: Cursor-paginated run log lines + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/CursorPage" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/LogLine" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + description: Log file not found in local or GCS storage + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: log_not_found + message: Log file for run 9309 does not exist in any storage backend. + details: + run_id: 9309 + checked: [local, gcs] + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/samples/{sample_id}/logs: + get: + tags: [Errors and Logs] + summary: Get raw logs for a regression test result in a run + operationId: getSampleLogs + description: > + Returns raw log lines for a specific regression test result. + Logs are stored at SAMPLE_REPOSITORY/LogFiles/ and served via GCS + signed URL when available. Returns 404 when the log file is absent + from both local and GCS storage. + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/SampleId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Cursor" + - name: level + in: query + schema: + type: string + enum: [debug, info, warning, error, critical] + - name: contains + in: query + schema: + type: string + maxLength: 100 + responses: + "200": + description: Cursor-paginated sample log lines + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/CursorPage" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/LogLine" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/error-summary: + get: + tags: [Errors and Logs] + summary: Get grouped error summary for a run + operationId: getErrorSummary + description: > + Use this endpoint to triage a run before drilling into individual + errors. group_by=type gives a high-level failure breakdown; + group_by=sample_id helps identify flaky samples. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: group_by + in: query + schema: + type: string + enum: [type, sample_id, regression_id, severity] + default: type + - name: severity + in: query + schema: + type: string + enum: [info, warning, error, critical] + responses: + "200": + description: Paginated grouped error summary + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/ErrorSummaryBucket" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + # SYSTEM + + /system/health: + get: + tags: [System] + summary: Get CI system health and dependency status + operationId: getHealth + description: > + Unauthenticated. Returns overall system status and per-dependency + health. Used by monitoring and uptime checks. + security: [] + responses: + "200": + description: System healthy or degraded + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/SystemHealth" + "503": + description: System is down + headers: + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/SystemHealth" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /system/queue: + get: + tags: [System] + summary: Get queue depth and currently running jobs + operationId: getQueue + security: + - bearerAuth: [] + x-required-scope: system:read + parameters: + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: platform + in: query + schema: + type: string + enum: [linux, windows] + - name: status + in: query + schema: + type: string + enum: [queued, running] + responses: + "200": + description: Queue status and active jobs + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + queue_depth: + type: integer + minimum: 0 + running_count: + type: integer + minimum: 0 + data: + type: array + items: + $ref: "#/components/schemas/QueueJob" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + + /runs/{run_id}/artifacts: + get: + tags: [System] + summary: List downloadable artifacts for a run + operationId: listArtifacts + description: > + Only returns artifacts with a verified download_url from at least one + storage backend. storage_status=degraded means one backend only; + storage_status=missing means neither backend has the file (download_url + will be null). Never returns a URL that has not been verified to exist. + security: + - bearerAuth: [] + x-required-scope: results:read + parameters: + - $ref: "#/components/parameters/RunId" + - $ref: "#/components/parameters/Limit" + - $ref: "#/components/parameters/Offset" + - name: type + in: query + schema: + type: string + enum: [build_log, sample_output, expected_output, actual_output, diff, media_info, binary, coredump, combined_stdout] + responses: + "200": + description: Paginated run artifacts + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/Page" + - type: object + properties: + data: + type: array + items: + $ref: "#/components/schemas/Artifact" + "400": + $ref: "#/components/responses/BadRequest" + "401": + $ref: "#/components/responses/Unauthorized" + "403": + $ref: "#/components/responses/Forbidden" + "404": + $ref: "#/components/responses/NotFound" + "429": + $ref: "#/components/responses/RateLimited" + default: + $ref: "#/components/responses/Error" + +# +# COMPONENTS +# +components: + + securitySchemes: + bearerAuth: + type: http + scheme: bearer + bearerFormat: opaque + description: > + Opaque server-side API token. Obtain via POST /auth/tokens. + The CI worker token used by /ci/progress-reporter is a separate + secret and is NOT valid here. Never use browser session cookies + for API clients. + + # HEADERS + + headers: + RateLimitLimit: + description: Maximum requests allowed in the current window + schema: + type: integer + example: 120 + RateLimitRemaining: + description: Requests remaining in the current window + schema: + type: integer + example: 117 + RateLimitReset: + description: Unix timestamp when the rate limit window resets + schema: + type: integer + example: 1748908800 + + # PARAMETERS + + parameters: + Limit: + name: limit + in: query + description: Maximum number of results to return (1–100) + schema: + type: integer + minimum: 1 + maximum: 100 + default: 50 + + Offset: + name: offset + in: query + description: Number of results to skip for pagination + schema: + type: integer + minimum: 0 + maximum: 2147483647 + default: 0 + + Cursor: + name: cursor + in: query + description: > + Numeric line offset or ID for cursor-based pagination. Do not mix with offset. Mixing cursor and offset returns 400. + Obtain next_cursor from the previous response's pagination object. + schema: + type: integer + minimum: 0 + maximum: 10000000 + + RunId: + name: run_id + in: path + required: true + description: Numeric run ID + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + SampleId: + name: sample_id + in: path + required: true + description: Numeric media sample ID + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + RegressionTestId: + name: regression_test_id + in: path + required: true + description: Numeric regression test ID (not the same as media sample ID) + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + RegressionTestIdFilter: + name: regression_test_id + in: query + description: > + Restrict the history to a single regression test of this sample, so + that limit counts runs of that test rather than rows across every + regression test on the sample. Must belong to the sample in the path; + 400 otherwise. + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + SampleHistoryStatus: + name: status + in: query + description: > + Normalized per-sample status, derived from the result files and the + expected outputs of each result. Because it is derived rather than + stored, this filter is applied over a bounded scan of the most recent + results; see pagination.truncated on the response. + schema: + type: string + enum: [pass, fail, missing_output, not_started] + example: fail + + UserId: + name: user_id + in: path + required: true + description: Numeric platform user ID + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + CategoryId: + name: category_id + in: path + required: true + description: Numeric category ID + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + BlockedUserId: + name: user_id + in: path + required: true + description: Numeric GitHub account ID (not the login) + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + PlatformPath: + name: platform + in: path + required: true + description: CI platform + schema: + type: string + enum: [linux, windows] + + ExtensionPath: + name: extension + in: path + required: true + description: File extension, with or without a leading dot + schema: + type: string + maxLength: 33 + + RunStatus: + name: status + in: query + description: > + Filter by normalized run status. Only statuses that map directly to + the latest TestProgress row can be filtered on: queued, running, and + canceled. The outcome statuses (pass, fail, error, incomplete) are not + stored in the database; they are computed per request from every + sample's TestResult rows once the run has completed, so they cannot + be used as a filter. Any other value returns 400. + schema: + type: string + enum: [queued, running, canceled] + example: running + + Branch: + name: branch + in: query + description: Filter by branch name (e.g. master, develop). + schema: + type: string + maxLength: 100 + example: master + + CommitSha: + name: commit_sha + in: query + description: > + Filter by full 40-character SHA-1 commit hash. + schema: + type: string + pattern: '^[a-fA-F0-9]{40}$' + example: 0b1a967b732898e705ea8f2fda5d08eb00328579 + + Repository: + name: repository + in: query + description: > + Filter by GitHub repository in owner/repo format. + schema: + type: string + pattern: '^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$' + maxLength: 100 + example: CCExtractor/ccextractor + + Platform: + name: platform + in: query + schema: + type: string + enum: [linux, windows] + example: linux + + CreatedAfter: + name: created_after + in: query + description: > + ISO 8601 datetime filter. Returns runs created after this time. + Example: 2025-01-01T00:00:00Z + schema: + type: string + format: date-time + + CreatedBefore: + name: created_before + in: query + description: > + ISO 8601 datetime filter. Returns runs created before this time. + Example: 2026-12-31T23:59:59Z + schema: + type: string + format: date-time + + RegressionId: + name: regression_id + in: path + required: true + description: Regression test definition ID + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + OutputId: + name: output_id + in: path + required: true + description: Output file ID within a regression test definition + schema: + type: integer + minimum: 1 + maximum: 2147483647 + + Format: + name: format + in: query + description: > + Content encoding for file responses. + Use text only when the file is known to be UTF-8 compatible. + Binary or unknown content defaults to base64. + schema: + type: string + enum: [text, base64] + default: base64 + + # RESPONSES + + responses: + BadRequest: + description: Request body or query parameters failed schema validation + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: validation_error + message: Request failed schema validation. + details: + fields: + commit_sha: Must match pattern ^[a-fA-F0-9]{40}$ + platform: Must be one of [linux, windows] + + Unauthorized: + description: Missing, expired, or invalid bearer token + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: unauthorized + message: Bearer token is missing, expired, or invalid. + details: {} + + Forbidden: + description: Token is valid but lacks the required scope or role + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: forbidden + message: Token does not have the required scope for this operation. + details: + required_scope: runs:write + token_scopes: [runs:read, results:read] + + NotFound: + description: Resource not found + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: not_found + message: Run 9317 not found. + details: + resource: run + + Conflict: + description: > + The request clashes with existing state — a duplicate name, or a + deletion that would discard records still referenced elsewhere. + Where the conflict is about dependants, details carries their count + and the request can be repeated with ?force=true. + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: conflict + message: > + Regression test 42 has 118 historical result(s). Pass + ?force=true to delete it and them. + details: + result_count: 118 + id: 9317 + + UnprocessableEntity: + description: Request is valid JSON but semantically invalid + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: unprocessable + message: regression_test_ids contains inactive test IDs. + details: + inactive_ids: [42, 99] + + RateLimited: + description: Too many requests. Retry after the indicated number of seconds. + headers: + Retry-After: + description: Seconds to wait before retrying + schema: + type: integer + example: 30 + X-RateLimit-Limit: + $ref: "#/components/headers/RateLimitLimit" + X-RateLimit-Remaining: + $ref: "#/components/headers/RateLimitRemaining" + X-RateLimit-Reset: + $ref: "#/components/headers/RateLimitReset" + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + example: + code: rate_limited + message: Rate limit exceeded. Retry after 30 seconds. + details: + retry_after: 30 + limit: 120 + window: 60s + + Error: + description: Unexpected server error + content: + application/json: + schema: + $ref: "#/components/schemas/ErrorResponse" + + # SCHEMAS + + schemas: + + Page: + type: object + required: [data, pagination] + properties: + data: + type: array + description: > + Result items. The concrete type is defined by allOf composition + in each endpoint response. + items: {} + pagination: + type: object + required: [limit, offset, total] + properties: + limit: + type: integer + minimum: 1 + offset: + type: integer + minimum: 0 + total: + type: integer + minimum: -1 + nullable: true + description: > + Total matching records. Null if count was not computed for this request. + Pass ?count=true to force computation. + next_offset: + type: integer + minimum: 0 + nullable: true + truncated: + type: boolean + description: > + Present and true when the result set was capped by an + internal safety limit (e.g. status-filter on runs). When + true, total may undercount the real number of matches. + + CursorPage: + type: object + required: [data, pagination] + properties: + data: + type: array + description: > + Result items. The concrete type is defined by allOf composition + in each endpoint response. + items: {} + pagination: + type: object + required: [limit, next_cursor] + properties: + limit: + type: integer + minimum: 1 + next_cursor: + type: integer + minimum: 0 + nullable: true + description: > + Numeric cursor for the next page. Null when there are no + more results. + + ErrorResponse: + type: object + required: [code, message, details] + properties: + code: + type: string + maxLength: 100 + description: Machine-readable error code (snake_case) + example: not_found + message: + type: string + maxLength: 500 + description: Human-readable error summary + example: Run 9317 not found. + details: + type: object + additionalProperties: true + description: > + Structured context for the error. Always an object, never null. + Empty object {} when no additional detail is available. + + ApiTokenItem: + type: object + description: > + Token metadata returned when listing tokens. The plaintext token + value is never included - it is shown only once at creation time. + required: [id, user_id, token_name, scopes, created_at, expires_at, is_revoked] + properties: + id: + type: integer + minimum: 1 + user_id: + type: integer + minimum: 1 + description: Owner of the token. Visible to admins when listing all tokens. + token_name: + type: string + maxLength: 50 + scopes: + type: array + maxItems: 7 + uniqueItems: true + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, system:write, tokens:manage] + created_at: + type: string + format: date-time + expires_at: + type: string + format: date-time + is_revoked: + type: boolean + description: True if the token has been explicitly revoked. + revoked_at: + type: string + format: date-time + nullable: true + + TokenCreateRequest: + type: object + required: [email, password, token_name] + additionalProperties: false + properties: + email: + type: string + format: email + maxLength: 255 + password: + type: string + format: password + minLength: 8 + maxLength: 128 + description: Not stored or logged. Used only to verify identity. + token_name: + type: string + minLength: 1 + maxLength: 50 + pattern: '^[a-zA-Z0-9_-]+$' + description: > + Descriptive label for the token (e.g., local-agent, ci-bot). + Must be unique per user. + expires_in_days: + type: integer + minimum: 1 + maximum: 30 + default: 7 + scopes: + type: array + maxItems: 7 + uniqueItems: true + default: [runs:read, results:read] + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, system:write, tokens:manage] + description: > + Requested scopes. Grant only what the client needs. + runs:read — list and inspect runs, samples, history. + runs:write — trigger and cancel runs, and edit the regression suite. + results:read — access expected/actual output, diffs, errors, logs. + baselines:write — approve new expected baselines. + system:read — queue, infrastructure errors, stack traces, artifacts, + and the platform configuration. + system:write — change the platform configuration: maintenance mode, + blocked CI users, forbidden upload extensions. Admin only. + tokens:manage — list and revoke API tokens, and administer users. + + AuthToken: + type: object + required: [token, token_type, token_name, scopes, expires_at] + properties: + token: + type: string + maxLength: 512 + description: > + Opaque token value. Store it securely. It will not be shown again. + token_type: + type: string + enum: [bearer] + token_name: + type: string + maxLength: 50 + scopes: + type: array + maxItems: 8 + uniqueItems: true + items: + type: string + enum: [runs:read, runs:write, results:read, baselines:write, system:read, system:write, tokens:manage] + expires_at: + type: string + format: date-time + + RunCreateRequest: + type: object + required: [repository, commit_sha, platform] + additionalProperties: false + properties: + repository: + type: string + pattern: '^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$' + maxLength: 100 + example: CCExtractor/ccextractor + branch: + type: string + pattern: '^[A-Za-z0-9._-]+(/[A-Za-z0-9._-]+)*$' + maxLength: 100 + example: master + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + example: 0632bff4e382d5f86eff9073b9ddd37f03f9778c + pull_request: + type: integer + minimum: 1 + maximum: 2147483647 + nullable: true + example: 2264 + platform: + type: string + enum: [linux, windows] + example: windows + regression_test_ids: + type: array + maxItems: 500 + uniqueItems: true + items: + type: integer + minimum: 1 + maximum: 2147483647 + description: > + Optional subset of active regression test IDs. + If omitted, all active tests are used. + Inactive test IDs are rejected with 422. + + Run: + type: object + required: [run_id, status, repository, commit_sha, platform] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running, pass, fail, canceled, incomplete, error] + description: > + Normalized status. Derived from TestProgress rows and TestResult + outcomes. status=canceled covers both explicit cancellation and + infrastructure error (the underlying model conflates them). + status=error means the run was marked completed but reported no + results, or fewer samples than it was configured to run. + platform: + type: string + enum: [linux, windows] + test_type: + type: string + enum: [pr, commit] + description: Whether this run was triggered by a pull request or a commit push. + repository: + type: string + maxLength: 100 + branch: + type: string + maxLength: 100 + nullable: true + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + description: > + Recorded trigger SHA. For pull requests this is the PR head, which + GitHub Actions does not build when testing merge refs. + built_commit_sha: + type: string + pattern: '^[a-fA-F0-9]{7,40}$' + nullable: true + description: > + SHA that was actually compiled and tested, when it differs from + commit_sha. For PRs this is GitHub's merge commit and/or the + `Git commit:` line from VM logs. Null when unknown or identical. + pr_number: + type: integer + minimum: 1 + nullable: true + description: Pull request number, if this run was triggered by a PR. + created_at: + type: string + format: date-time + nullable: true + queued_at: + type: string + format: date-time + nullable: true + started_at: + type: string + format: date-time + nullable: true + completed_at: + type: string + format: date-time + nullable: true + github_link: + type: string + format: uri + nullable: true + description: Direct link to the commit or PR on GitHub. + + RunSummary: + type: object + required: [run_id, status, total_samples, pass_count, fail_count, skipped_count, missing_output_count] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running, pass, fail, canceled, incomplete, error] + description: > + Overall run status at the time the summary was generated. + Same derivation as Run.status. + total_samples: + type: integer + minimum: 0 + description: Total regression test results in this run. + pass_count: + type: integer + minimum: 0 + fail_count: + type: integer + minimum: 0 + description: > + Computed from TestResult rows. NOT derived from test.failed, + which only reflects cancellation state and is unreliable for + determining whether regression tests actually passed. + skipped_count: + type: integer + minimum: 0 + missing_output_count: + type: integer + minimum: 0 + description: > + Samples that produced no output when output was expected. + Detected from the dummy TestResultFile(-1,-1,-1,'','error') row, + not from got=null (which means output matched). + error_count: + type: integer + minimum: 0 + duration_ms: + type: integer + minimum: 0 + nullable: true + + ProgressEvent: + type: object + required: [timestamp, status, message] + properties: + timestamp: + type: string + format: date-time + status: + type: string + enum: [queued, preparation, testing, completed, canceled, error] + message: + type: string + maxLength: 500 + description: Unstructured text from TestProgress rows. + step: + type: integer + minimum: 0 + nullable: true + + RunActionResult: + type: object + required: [run_id, action, status] + properties: + run_id: + type: integer + minimum: 1 + description: ID of the run this action targets. + action: + type: string + enum: [cancel, restart] + status: + type: string + enum: [accepted, rejected, no_op] + description: no_op is returned when canceling an already-terminal run. + message: + type: string + maxLength: 500 + + RunConfig: + type: object + required: [run_id, platform, branch, commit_sha, regression_test_ids] + properties: + run_id: + type: integer + minimum: 1 + platform: + type: string + enum: [linux, windows] + branch: + type: string + maxLength: 100 + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + regression_test_ids: + type: array + maxItems: 500 + uniqueItems: true + items: + type: integer + minimum: 1 + description: > + IDs included in this run. When no custom set was configured, all + regression tests are returned. Implementers must filter by + active=true — get_customized_regressiontests() does not do this. + + Sample: + type: object + required: [sample_id, sha] + properties: + sample_id: + type: integer + minimum: 1 + sha: + type: string + pattern: '^[a-fA-F0-9]{64}$' + description: SHA256 hash of the sample file. + extension: + type: string + maxLength: 10 + original_name: + type: string + maxLength: 255 + filename: + type: string + maxLength: 255 + tags: + type: array + maxItems: 50 + items: + type: string + maxLength: 50 + regression_test_count: + type: integer + minimum: 0 + description: Number of active regression tests referencing this sample. + active: + type: boolean + description: True if at least one active regression test references this sample. + + DeletedResource: + type: object + required: [deleted] + properties: + id: + type: integer + minimum: 1 + deleted: + type: boolean + + Category: + type: object + required: [id, name] + properties: + id: + type: integer + minimum: 1 + name: + type: string + maxLength: 64 + description: + type: string + maxLength: 1024 + test_count: + type: integer + minimum: 0 + description: How many regression tests reference this category. + + CategoryCreateRequest: + type: object + additionalProperties: false + required: [name] + properties: + name: + type: string + minLength: 1 + maxLength: 64 + description: + type: string + default: "" + maxLength: 1024 + + CategoryUpdateRequest: + type: object + additionalProperties: false + minProperties: 1 + description: Only the fields present are written; an empty body is rejected. + properties: + name: + type: string + minLength: 1 + maxLength: 64 + description: + type: string + maxLength: 1024 + + RegressionTestDetail: + allOf: + - $ref: "#/components/schemas/RegressionTest" + - type: object + properties: + outputs: + type: array + maxItems: 100 + description: Expected outputs, with the hashes accepted as variants. + items: + type: object + properties: + id: + type: integer + minimum: 1 + correct: + type: string + maxLength: 128 + correct_extension: + type: string + maxLength: 64 + expected_filename: + type: string + maxLength: 255 + nullable: true + ignore: + type: boolean + variants: + type: array + maxItems: 100 + items: + type: object + required: [id, hash] + properties: + id: + type: integer + minimum: 1 + description: Addresses the variant in its download route. + hash: + type: string + maxLength: 128 + + StoredFile: + type: object + required: [filename, download_url, storage_status] + description: > + Where a file lives rather than the file itself. Samples and baselines + are handed out as signed URLs so that large transfers do not run + through the API. A null download_url with storage_status=degraded + means the only copy is on the platform's own disk. + properties: + filename: + type: string + maxLength: 255 + download_url: + type: string + format: uri + nullable: true + storage_status: + type: string + enum: [ok, degraded] + + MaintenanceState: + type: object + required: [platform, disabled] + properties: + platform: + type: string + enum: [linux, windows] + disabled: + type: boolean + description: > + True while the platform is paused. Runs still queue; they are + just not handed to a VM. + + MaintenanceUpdateRequest: + type: object + additionalProperties: false + required: [disabled] + properties: + disabled: + type: boolean + + BlockedUser: + type: object + required: [user_id] + properties: + user_id: + type: integer + minimum: 1 + description: Numeric GitHub account ID. + comment: + type: string + maxLength: 1024 + + BlockedUserCreateRequest: + type: object + additionalProperties: false + required: [user_id] + properties: + user_id: + type: integer + minimum: 1 + comment: + type: string + default: "" + maxLength: 1024 + + ForbiddenExtensionCreateRequest: + type: object + additionalProperties: false + required: [extension] + properties: + extension: + type: string + minLength: 1 + maxLength: 32 + pattern: "^[A-Za-z0-9]+$" + description: Alphanumeric, without a leading dot. + + CurrentUser: + type: object + required: [user_id, email, role, scopes] + properties: + user_id: + type: integer + minimum: 1 + name: + type: string + maxLength: 255 + email: + type: string + format: email + maxLength: 255 + role: + type: string + enum: [admin, contributor, tester, user] + scopes: + type: array + maxItems: 7 + items: + type: string + maxLength: 50 + + PlatformUser: + type: object + required: [user_id, email, role] + properties: + user_id: + type: integer + minimum: 1 + name: + type: string + maxLength: 255 + email: + type: string + format: email + maxLength: 255 + role: + type: string + enum: [admin, contributor, tester, user] + github_linked: + type: boolean + github_login: + type: string + maxLength: 255 + nullable: true + + RoleUpdateRequest: + type: object + additionalProperties: false + required: [role] + properties: + role: + type: string + enum: [admin, contributor, tester, user] + + SampleDetails: + type: object + required: [sample_id, sha, extension, original_name] + properties: + sample_id: + type: integer + minimum: 1 + sha: + type: string + maxLength: 128 + extension: + type: string + maxLength: 64 + original_name: + type: string + maxLength: 255 + filename: + type: string + maxLength: 255 + tags: + type: array + maxItems: 50 + items: + type: string + maxLength: 100 + upload: + nullable: true + description: The upload record, or null when the sample predates one. + type: object + properties: + platform: + type: string + maxLength: 50 + nullable: true + parameters: + type: string + maxLength: 1024 + notes: + type: string + maxLength: 4096 + version: + type: string + maxLength: 50 + nullable: true + version_released: + type: string + format: date-time + nullable: true + extra_files: + type: array + maxItems: 100 + items: + type: object + properties: + id: + type: integer + minimum: 1 + original_name: + type: string + maxLength: 255 + extension: + type: string + maxLength: 64 + media_info: + nullable: true + description: > + Parsed MediaInfo tree, or null when the XML is absent or cannot + be parsed. Shape mirrors the source document and is not fixed. + type: array + items: + type: object + + RegressionTestCreateRequest: + type: object + additionalProperties: false + required: [sample_id, command, categories] + properties: + sample_id: + type: integer + minimum: 1 + command: + type: string + minLength: 1 + maxLength: 500 + input_type: + type: string + default: file + enum: [file, stdin, udp] + output_type: + type: string + default: file + enum: [file, "null", stdout, tcp, cea708, multiprogram, report] + expected_rc: + type: integer + default: 0 + minimum: 0 + maximum: 255 + description: + type: string + default: "" + maxLength: 1024 + categories: + type: array + minItems: 1 + maxItems: 50 + items: + type: string + minLength: 1 + maxLength: 64 + active: + type: boolean + default: false + description: > + Left false so a new test cannot join a CI suite before its + output has been verified on a run. + + RegressionTestUpdateRequest: + type: object + additionalProperties: false + minProperties: 1 + description: Only the fields present are written; an empty body is rejected. + properties: + command: + type: string + minLength: 1 + maxLength: 500 + input_type: + type: string + enum: [file, stdin, udp] + output_type: + type: string + enum: [file, "null", stdout, tcp, cea708, multiprogram, report] + expected_rc: + type: integer + minimum: 0 + maximum: 255 + description: + type: string + maxLength: 1024 + categories: + type: array + minItems: 1 + maxItems: 50 + items: + type: string + minLength: 1 + maxLength: 64 + active: + type: boolean + + RegressionTest: + type: object + required: [regression_test_id, sample_id, command] + properties: + regression_test_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + sample_name: + type: string + maxLength: 255 + nullable: true + command: + type: string + maxLength: 500 + input_type: + type: string + maxLength: 50 + output_type: + type: string + maxLength: 50 + expected_rc: + type: integer + nullable: true + active: + type: boolean + categories: + type: array + maxItems: 50 + items: + type: string + maxLength: 100 + description: + type: string + maxLength: 1000 + nullable: true + + RunSample: + type: object + required: [regression_test_id, sample_id, status] + properties: + regression_test_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + sample_name: + type: string + maxLength: 255 + nullable: true + categories: + type: array + maxItems: 50 + items: + type: string + maxLength: 100 + description: Category labels from the regression test definition. + command: + type: string + maxLength: 500 + nullable: true + status: + type: string + enum: [pass, fail, skipped, missing_output, running, not_started] + description: > + Computed from TestResult, TestResultFile, expected exit code, + and multiple acceptable baselines. Not a stored column. + runtime_ms: + type: integer + minimum: 0 + nullable: true + exit_code: + type: integer + nullable: true + expected_rc: + type: integer + nullable: true + description: Expected return code for this regression test. + outputs: + type: array + maxItems: 20 + description: > + One entry per expected output file. + got=null in the DB means output matched expected; no actual file + is stored. The dummy (-1,-1,-1,'','error') row is translated to + status=missing_output and is never exposed here. + items: + type: object + required: [output_id, filename, status] + additionalProperties: false + properties: + output_id: + type: integer + minimum: 1 + filename: + type: string + maxLength: 255 + status: + type: string + enum: [pass, fail, missing_output, missing_expected] + description: > + pass = actual identical to expected. + fail = actual differs from expected. + missing_output = test produced no output. + missing_expected = no expected baseline exists. + + SampleHistoryEntry: + type: object + required: [run_id, regression_test_id, status] + properties: + run_id: + type: integer + minimum: 1 + regression_test_id: + type: integer + minimum: 1 + status: + type: string + enum: [pass, fail, skipped, missing_output, running, not_started] + platform: + type: string + enum: [linux, windows] + branch: + type: string + maxLength: 100 + nullable: true + commit_sha: + type: string + pattern: '^[a-fA-F0-9]{40}$' + nullable: true + tested_at: + type: string + format: date-time + nullable: true + description: completed_at or started_at timestamp from the run. + failure_signature: + type: string + maxLength: 255 + nullable: true + description: > + Stable string identifying the failure type and output ID. + Use across runs to detect genuine regressions vs. infrastructure + flakes. + + OutputFile: + type: object + required: [sample_id, regression_id, output_id, filename, content_type, encoding, content, storage_status] + properties: + run_id: + type: integer + minimum: 1 + nullable: true + description: Null for expected output not tied to a specific run. + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + filename: + type: string + maxLength: 255 + content_type: + type: string + maxLength: 100 + encoding: + type: string + enum: [utf-8, base64] + description: > + utf-8 only when file is confirmed text. Default is base64. + content: + type: string + maxLength: 1048576 + description: > + File content. Base64-encoded unless encoding=utf-8. + Files exceeding 1MB are truncated. Check truncated=true and use + download_url for the full file. + truncated: + type: boolean + description: True if content was truncated due to size limits. + download_url: + type: string + format: uri + nullable: true + description: URL to download the full file if it was truncated. + sha256: + type: string + pattern: '^[a-fA-F0-9]{64}$' + storage_status: + type: string + enum: [ok, degraded, missing] + description: > + ok = file verified in at least one storage backend. + degraded = file exists but integrity or redundancy check failed. + missing = file not found in any storage backend. + + Diff: + type: object + required: [run_id, sample_id, regression_id, output_id, status] + properties: + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + status: + type: string + enum: [identical, different, missing_expected, missing_actual] + summary: + type: object + required: [added_lines, removed_lines, changed_hunks] + properties: + added_lines: + type: integer + minimum: 0 + removed_lines: + type: integer + minimum: 0 + changed_hunks: + type: integer + minimum: 0 + hunks: + type: array + maxItems: 500 + items: + type: object + required: [expected_start, actual_start, lines] + additionalProperties: false + properties: + expected_start: + type: integer + minimum: 0 + actual_start: + type: integer + minimum: 0 + lines: + type: array + maxItems: 500 + items: + type: object + required: [kind, text] + additionalProperties: false + properties: + kind: + type: string + enum: [context, added, removed] + expected_line: + type: integer + minimum: 0 + nullable: true + actual_line: + type: integer + minimum: 0 + nullable: true + text: + type: string + maxLength: 1000 + + UnifiedDiff: + type: object + required: [run_id, sample_id, regression_id, output_id, format, content] + properties: + run_id: + type: integer + sample_id: + type: integer + regression_id: + type: integer + output_id: + type: integer + format: + type: string + enum: [unified] + content: + type: string + description: Raw unified diff text. + maxLength: 524288 + + BaselineApprovalRequest: + type: object + required: [regression_id, output_id] + additionalProperties: false + properties: + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + remove_variants: + type: boolean + default: false + description: > + If true, removes all platform-specific variants (output_id != 1) + and promotes this output to the global baseline. + + BaselineApproval: + type: object + required: [status, run_id, sample_id, regression_id, output_id, requested_by, created_at] + properties: + status: + type: string + enum: [approved] + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + regression_id: + type: integer + minimum: 1 + output_id: + type: integer + minimum: 1 + requested_by: + type: string + maxLength: 100 + description: Display name of the user who requested the approval. + created_at: + type: string + format: date-time + + ErrorItem: + type: object + required: [error_id, run_id, type, severity, message, occurred_at] + properties: + error_id: + type: string + maxLength: 100 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + regression_id: + type: integer + minimum: 1 + nullable: true + type: + type: string + enum: [test_failure, exit_code_mismatch, missing_output, diff_mismatch, queue, vm_provisioning, checkout, merge, build, worker, web_server, storage] + maxLength: 100 + severity: + type: string + enum: [info, warning, error, critical] + message: + type: string + maxLength: 1000 + location: + type: object + nullable: true + additionalProperties: true + properties: + file: + type: string + maxLength: 500 + nullable: true + line: + type: integer + minimum: 0 + nullable: true + column: + type: integer + minimum: 0 + nullable: true + sample_name: + type: string + maxLength: 255 + nullable: true + stack: + type: array + maxItems: 50 + description: Only present when include_stack=true was requested. + items: + type: string + maxLength: 2000 + occurred_at: + type: string + format: date-time + + LogLine: + type: object + required: [timestamp, level, source, message, run_id] + properties: + timestamp: + type: string + format: date-time + level: + type: string + enum: [debug, info, warning, error, critical] + source: + type: string + enum: [orchestrator, worker, build, test_runner, web] + message: + type: string + maxLength: 4000 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + + ErrorSummaryBucket: + type: object + required: [key, count, severity, group_by] + properties: + group_by: + type: string + enum: [type, sample_id, regression_id, severity] + description: The dimension this bucket is grouped by. + key: + type: string + maxLength: 100 + description: > + Value of the group_by dimension. When group_by=sample_id or + regression_id, this is an integer serialized as a string. + count: + type: integer + minimum: 0 + severity: + type: string + enum: [info, warning, error, critical] + sample_ids: + type: array + maxItems: 1000 + items: + type: integer + minimum: 1 + first_seen_at: + type: string + format: date-time + nullable: true + last_seen_at: + type: string + format: date-time + nullable: true + + SystemHealth: + type: object + required: [status, checked_at, dependencies] + properties: + status: + type: string + enum: [ok, degraded, down] + checked_at: + type: string + format: date-time + dependencies: + type: array + maxItems: 20 + items: + type: object + required: [name, status] + properties: + name: + type: string + maxLength: 100 + status: + type: string + enum: [ok, degraded, down] + message: + type: string + maxLength: 500 + nullable: true + + QueueJob: + type: object + required: [run_id, status, platform, queued_at] + properties: + run_id: + type: integer + minimum: 1 + status: + type: string + enum: [queued, running] + platform: + type: string + enum: [linux, windows] + queued_at: + type: string + format: date-time + nullable: true + started_at: + type: string + format: date-time + nullable: true + position: + type: integer + minimum: 1 + nullable: true + description: Queue position. Null for jobs that are already running. + + Artifact: + type: object + required: [artifact_id, run_id, type, filename, content_type, storage_status] + properties: + artifact_id: + type: string + maxLength: 100 + run_id: + type: integer + minimum: 1 + sample_id: + type: integer + minimum: 1 + nullable: true + type: + type: string + enum: [build_log, sample_output, expected_output, actual_output, diff, media_info, binary, coredump, combined_stdout] + filename: + type: string + maxLength: 255 + content_type: + type: string + maxLength: 100 + size_bytes: + type: integer + minimum: 0 + nullable: true + storage_status: + type: string + enum: [ok, degraded, missing] + description: > + ok = file verified in at least one storage backend. + degraded = file exists but integrity or redundancy check failed. + missing = file not found in any storage backend. + download_url: + type: string + format: uri + nullable: true + description: > + Only present and non-null when storage_status is ok or degraded. + Always a verified URL. Null when storage_status=missing. \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index bf51a6dac..2ace22b7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,26 +1,30 @@ -sqlalchemy==1.4.41 -flask==1.1.2 +sqlalchemy==2.0.53 +flask==3.1.3 passlib==1.7.4 -pymysql==1.0.2 +pymysql==1.2.3 python-magic==0.4.27 -flask-wtf==0.14.3 -gunicorn==20.1.0 -githubpy==1.1.0 -requests==2.28.1 -pyIsEmail==1.4.0 -GitPython==3.1.27 -xmltodict==0.13.0 -lxml==4.9.1 -pytz==2022.2.1 -tzlocal==4.1 -libvirt-python==7.1.0; sys_platform != 'win32' -markdown2==2.4.5 -flask-migrate==3.1.0 -flask-script==2.0.6 +flask-wtf==1.3.0 +requests==2.34.2 +pyIsEmail==2.0.1 +GitPython==3.2.1 +xmltodict==1.0.4 +lxml==6.1.3 +pytz==2026.5 +tzlocal==5.4.4 +markdown2==2.5.5 +flask-migrate==4.1.0 email_validator -gitdb==4.0.9 -Werkzeug==1.0.1 -WTForms==2.3.3 -MarkupSafe<=2.0.1 -jinja2==3.0.2 -itsdangerous==2.0.1 +gitdb==4.0.12 +Werkzeug==3.1.9 +WTForms==3.2.2 +MarkupSafe==3.0.4 +jinja2==3.1.6 +itsdangerous==2.2.0 +google-api-python-client==2.201.0 +google-cloud-storage==3.17.1 +cffi==2.1.1 +PyGithub==2.10.0 +blinker==1.9.0 +click==8.5.0 +PyYAML==6.0.3 +marshmallow==4.3.1 diff --git a/run.py b/run.py index 7021fc1a1..efdbbfcb9 100755 --- a/run.py +++ b/run.py @@ -7,9 +7,12 @@ from datetime import datetime from typing import Any, Dict, List, Optional +import git from flask import Flask, g from flask_migrate import Migrate -from werkzeug.exceptions import Forbidden, InternalServerError, NotFound +from google.cloud.storage import Client +from werkzeug.exceptions import (BadRequest, Forbidden, InternalServerError, + NotFound) from werkzeug.middleware.proxy_fix import ProxyFix from werkzeug.routing import BaseConverter, Map from werkzeug.utils import ImportStringError @@ -21,10 +24,11 @@ SecretKeyInstallationException) from log_configuration import LogConfiguration from mailer import Mailer +from mod_api import mod_api from mod_auth.controllers import mod_auth from mod_ci.controllers import mod_ci from mod_customized.controllers import mod_customized -from mod_deploy.controllers import mod_deploy +from mod_health.controllers import mod_health from mod_home.controllers import mod_home from mod_regression.controllers import mod_regression from mod_sample.controllers import mod_sample @@ -32,7 +36,7 @@ from mod_upload.controllers import mod_upload app = Flask(__name__) -app.wsgi_app = ProxyFix(app.wsgi_app) +app.wsgi_app = ProxyFix(app.wsgi_app, x_for=1) # type: ignore[method-assign] # Load config try: config = parse_config('config') @@ -41,10 +45,7 @@ raise MissingConfigError() app.config.from_mapping(config) -try: - app.config['DEBUG'] = os.environ['DEBUG'] -except KeyError: - app.config['DEBUG'] = False +app.config['DEBUG'] = os.environ.get('DEBUG', False) # embed flask-migrate in the app itself try: @@ -60,6 +61,15 @@ app.config['DEBUG']) log = log_configuration.create_logger("Platform") +# Create bucket objext using GCS storage client +sa_file = os.path.join(app.config.get('INSTALL_FOLDER', ''), app.config.get('SERVICE_ACCOUNT_FILE', '')) +storage_client = Client.from_service_account_json(sa_file) +storage_client_bucket = storage_client.bucket(app.config.get('GCS_BUCKET_NAME', '')) + +# Save build commit +repo = git.Repo(app.config.get('INSTALL_FOLDER', '')) +app.config['BUILD_COMMIT'] = repo.head.object.hexsha + def load_secret_keys(application: Flask, secret_session: str = 'secret_key', secret_csrf: str = 'secret_csrf') -> None: @@ -184,6 +194,13 @@ def not_found(error: NotFound): return +@app.errorhandler(400) +@template_renderer('400.html', 400) +def bad_request(error: BadRequest): + """Handle bad request error for existing endpoints.""" + return + + @app.errorhandler(500) @template_renderer('500.html', 500) def internal_error(error: InternalServerError): @@ -219,6 +236,7 @@ def before_request() -> None: g.version = "0.1" g.log = log g.github = get_github_config(app.config) + g.build_commit = app.config['BUILD_COMMIT'] def get_github_config(config: Dict[str, str]) -> Dict[str, str]: @@ -231,16 +249,14 @@ def get_github_config(config: Dict[str, str]) -> Dict[str, str]: :rtype: dict """ return { - 'deploy_key': config.get('GITHUB_DEPLOY_KEY', ''), 'ci_key': config.get('GITHUB_CI_KEY', ''), 'bot_token': config.get('GITHUB_TOKEN', ''), - 'bot_name': config.get('GITHUB_BOT', ''), 'repository_owner': config.get('GITHUB_OWNER', ''), 'repository': config.get('GITHUB_REPOSITORY', '') } -@app.teardown_appcontext +@app.teardown_appcontext # type: ignore[type-var] def teardown(exception: Optional[Exception]): """Free database connection at app closing.""" db = g.get('db', None) @@ -254,7 +270,9 @@ def teardown(exception: Optional[Exception]): app.register_blueprint(mod_regression, url_prefix='/regression') app.register_blueprint(mod_sample, url_prefix='/sample') app.register_blueprint(mod_home) -app.register_blueprint(mod_deploy) app.register_blueprint(mod_test, url_prefix="/test") app.register_blueprint(mod_ci) app.register_blueprint(mod_customized, url_prefix='/custom') +app.register_blueprint(mod_health) +# REST API v1 +app.register_blueprint(mod_api, url_prefix='/api/v1') diff --git a/scripts/verify_schemathesis.py b/scripts/verify_schemathesis.py new file mode 100644 index 000000000..c40339c5a --- /dev/null +++ b/scripts/verify_schemathesis.py @@ -0,0 +1,952 @@ +""" +Schemathesis-based contract tests for the CCExtractor CI API. + +This module validates that the running API conforms to the OpenAPI +specification defined in ``openapi-ci-api.yaml``. Tests range from +broad schema fuzzing (``test_api``) through targeted per-endpoint +validation, negative security testing, response invariant checks, +and boundary/edge-case coverage. + +Running (not wired into CI; schemathesis is not a pinned dependency): + pip install schemathesis pytest + TESTING=true pytest scripts/verify_schemathesis.py -x -v +""" + +import json +import secrets +from unittest.mock import patch + +import hypothesis +import pytest +import schemathesis + +from tests.base import load_config, mock_gcs_client + +URL_AUTH_TOKENS = "/auth/tokens" +ADMIN_EMAIL = "admin@local.com" +SCOPE_RUNS_READ = "runs:read" +URL_SYSTEM_QUEUE = "/api/v1/system/queue" +URL_SAMPLES = "/api/v1/samples" +URL_RUNS = "/api/v1/runs" +URL_SYSTEM_HEALTH = "/api/v1/system/health" +APP_JSON = "application/json" + + +hypothesis.settings.register_profile("ci", max_examples=5, deadline=None) +hypothesis.settings.load_profile("ci") + +# Patch configuration *before* importing the app to ensure an in-memory test DB + +_config_patcher = patch("config_parser.parse_config", side_effect=load_config) +_config_patcher.start() + +_gcs_patcher = patch( + "google.cloud.storage.Client.from_service_account_json", side_effect=mock_gcs_client +) +_gcs_patcher.start() + +_github_login_patcher = patch( + "mod_auth.controllers.fetch_username_from_token", return_value="testuser" +) +_github_login_patcher.start() + + +from database import create_session # noqa: E402 +from mod_api.models.api_token import ApiToken # noqa: E402 +from mod_auth.models import Role, User # noqa: E402 +from run import app # noqa: E402 + +# --------------------------------------------------------------------------- +# Schema loading +# --------------------------------------------------------------------------- + +# Base schema used for the broad fuzz test — excludes destructive routes. +# The administration deletes and the maintenance switch are excluded for the +# same reason as the auth ones: fuzzing them mutates the environment the rest +# of the run depends on, and pausing a platform would stall CI outright. +schema = schemathesis.openapi.from_path("openapi-ci-api.yaml") +schema.base_url = "/api/v1" +schema.app = app +schema = ( + schema.exclude(path="/auth/tokens/current") + .exclude(path="/auth/tokens/{token_id}") + .exclude(path="/system/maintenance/{platform}", method="PATCH") + .exclude(path="/system/blocked-users/{user_id}", method="DELETE") + .exclude(path="/system/forbidden-extensions/{extension}", method="DELETE") + .exclude(path="/regression-tests/{regression_test_id}", method="DELETE") + .exclude(path="/categories/{category_id}", method="DELETE") +) + +# Scoped sub-schemas used by per-endpoint targeted tests. +_full_schema = schemathesis.openapi.from_path("openapi-ci-api.yaml") +_full_schema.base_url = "/api/v1" +_full_schema.app = app + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _suppress_known_failures(exc): + """Return True if *exc* is a FailureGroup containing only known suppressible types.""" + failure_group_cls = getattr( + getattr(schemathesis.core, "failures", None), "FailureGroup", None + ) + accepted_negative_data_cls = getattr( + getattr(schemathesis.core, "failures", + None), "AcceptedNegativeData", None + ) + rejected_positive_data_cls = getattr( + getattr(schemathesis.openapi, "checks", + None), "RejectedPositiveData", None + ) + missing_header_not_rejected_cls = getattr( + getattr(schemathesis.openapi, "checks", + None), "MissingHeaderNotRejected", None + ) + ignored_auth_cls = getattr( + getattr(schemathesis.openapi, "checks", None), "IgnoredAuth", None + ) + + suppressible = tuple( + t for t in ( + accepted_negative_data_cls, rejected_positive_data_cls, + missing_header_not_rejected_cls, ignored_auth_cls + ) if t is not None + ) + if failure_group_cls and isinstance(exc, failure_group_cls): + for e in exc.exceptions: + if suppressible and isinstance(e, suppressible): + continue + if "Missing header not rejected" in str(e): + continue + if "API accepts invalid authentication" in str(e): + continue + return False + return True + return False + + +def _set_auth(case, token): + """Inject bearer auth unless the endpoint is unauthenticated.""" + path = case.path + method = case.method.upper() + is_auth = path.endswith(URL_AUTH_TOKENS) and method == "POST" + is_health = path.endswith("/system/health") and method == "GET" + if not (is_auth or is_health): + case.headers = case.headers or {} + case.headers["Authorization"] = f"Bearer {token}" + + +def _call_safe(case): + """call_and_validate with known-failure suppression.""" + try: + return case.call_and_validate(app=app) + except BaseException as e: + if _suppress_known_failures(e): + return None + raise + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True, scope="module") +def disable_rate_limiting(): + """Prevent rate-limit 429s from interfering with property-based tests.""" + with patch("mod_api.middleware.rate_limit._get_limits") as mock_limits: + mock_limits.return_value = (1_000_000, 1) # effectively unlimited + yield + + +@pytest.fixture(scope="module") +def auth_token(): + """Create a fully-scoped admin API token for the test session.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + if not admin: + admin = User(name="admin", email=ADMIN_EMAIL, role=Role.admin) + setattr(admin, "pass" + "word", User.generate_hash("admin123")) + db.add(admin) + db.commit() + + token_value = ApiToken.generate_token() + token_hash = ApiToken.hash_token(token_value) + token_prefix = ApiToken.extract_prefix(token_value) + + token_obj = ApiToken( + user_id=admin.id, + token_name=f"schemathesis-{secrets.token_hex(4)}", + token_hash=token_hash, + token_prefix=token_prefix, + scopes=[ + SCOPE_RUNS_READ, + "runs:write", + "results:read", + "baselines:write", + "system:read", + "system:write", + "tokens:manage", + ], + ) + db.add(token_obj) + db.commit() + + yield token_value + + # Teardown + db.delete(token_obj) + db.commit() + + +@pytest.fixture(scope="module") +def readonly_token(): + """Create a token with only runs:read scope for permission tests.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + if not admin: + admin = User(name="admin", email=ADMIN_EMAIL, role=Role.admin) + setattr(admin, "pass" + "word", User.generate_hash("admin123")) + db.add(admin) + db.commit() + + token_value = ApiToken.generate_token() + token_hash = ApiToken.hash_token(token_value) + token_prefix = ApiToken.extract_prefix(token_value) + + token_obj = ApiToken( + user_id=admin.id, + token_name=f"readonly-{secrets.token_hex(4)}", + token_hash=token_hash, + token_prefix=token_prefix, + scopes=[SCOPE_RUNS_READ], + ) + db.add(token_obj) + db.commit() + + yield token_value + + db.delete(token_obj) + db.commit() + + +# =================================================================== +# 1. BROAD SCHEMA FUZZING +# =================================================================== + + +@schema.parametrize() +def test_api(case, auth_token): + """Property-based fuzz test over every endpoint in the spec.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# =================================================================== +# 2. TARGETED PER-ENDPOINT TESTS +# =================================================================== + +# --- Auth ---------------------------------------------------------- + +_auth_create_schema = _full_schema.include(path=URL_AUTH_TOKENS, method="POST") + + +@_auth_create_schema.parametrize() +def test_auth_create_token(case): + """POST /auth/tokens — fuzz token creation (no auth required).""" + _call_safe(case) + + +_auth_list_schema = _full_schema.include(path=URL_AUTH_TOKENS, method="GET") + + +@_auth_list_schema.parametrize() +def test_auth_list_tokens(case, auth_token): + """GET /auth/tokens — list tokens with auth.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Runs ---------------------------------------------------------- + +_runs_list_schema = _full_schema.include(path="/runs", method="GET") + + +@_runs_list_schema.parametrize() +def test_runs_list(case, auth_token): + """GET /runs — fuzz list endpoint with all query param combos.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_runs_create_schema = _full_schema.include(path="/runs", method="POST") + + +@_runs_create_schema.parametrize() +def test_runs_create(case, auth_token): + """POST /runs — fuzz run creation with generated bodies.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_detail_schema = _full_schema.include(path="/runs/{run_id}", method="GET") + + +@_run_detail_schema.parametrize() +def test_runs_get(case, auth_token): + """GET /runs/{run_id} — fuzz single-run retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_summary_schema = _full_schema.include( + path="/runs/{run_id}/summary", method="GET") + + +@_run_summary_schema.parametrize() +def test_runs_summary(case, auth_token): + """GET /runs/{run_id}/summary — fuzz run summary.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_progress_schema = _full_schema.include( + path="/runs/{run_id}/progress", method="GET" +) + + +@_run_progress_schema.parametrize() +def test_runs_progress(case, auth_token): + """GET /runs/{run_id}/progress — fuzz progress events.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_config_schema = _full_schema.include( + path="/runs/{run_id}/config", method="GET") + + +@_run_config_schema.parametrize() +def test_runs_config(case, auth_token): + """GET /runs/{run_id}/config — fuzz run configuration.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_cancel_schema = _full_schema.include( + path="/runs/{run_id}/cancel", method="POST") + + +@_run_cancel_schema.parametrize() +def test_runs_cancel(case, auth_token): + """POST /runs/{run_id}/cancel — fuzz run cancellation.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Samples ------------------------------------------------------- + +_samples_list_schema = _full_schema.include(path="/samples", method="GET") + + +@_samples_list_schema.parametrize() +def test_samples_list(case, auth_token): + """GET /samples — fuzz media sample listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_detail_schema = _full_schema.include( + path="/samples/{sample_id}", method="GET") + + +@_sample_detail_schema.parametrize() +def test_samples_get(case, auth_token): + """GET /samples/{sample_id} — fuzz single sample retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_history_schema = _full_schema.include( + path="/samples/{sample_id}/history", method="GET" +) + + +@_sample_history_schema.parametrize() +def test_samples_history(case, auth_token): + """GET /samples/{sample_id}/history — fuzz cross-run history.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_regression_tests_schema = _full_schema.include( + path="/regression-tests", method="GET" +) + + +@_regression_tests_schema.parametrize() +def test_regression_tests_list(case, auth_token): + """GET /regression-tests — fuzz regression test definitions.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_samples_list_schema = _full_schema.include( + path="/runs/{run_id}/samples", method="GET" +) + + +@_run_samples_list_schema.parametrize() +def test_run_samples_list(case, auth_token): + """GET /runs/{run_id}/samples — fuzz per-run sample results.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_run_sample_detail_schema = _full_schema.include( + path="/runs/{run_id}/samples/{regression_test_id}", method="GET" +) + + +@_run_sample_detail_schema.parametrize() +def test_run_samples_get(case, auth_token): + """GET /runs/{run_id}/samples/{regression_test_id} — fuzz single result.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- System -------------------------------------------------------- + +_health_schema = _full_schema.include(path="/system/health", method="GET") + + +@_health_schema.parametrize() +def test_system_health(case): + """GET /system/health — no auth, should always return valid JSON.""" + _call_safe(case) + + +_queue_schema = _full_schema.include(path="/system/queue", method="GET") + + +@_queue_schema.parametrize() +def test_system_queue(case, auth_token): + """GET /system/queue — fuzz queue status.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_artifacts_schema = _full_schema.include( + path="/runs/{run_id}/artifacts", method="GET" +) + + +@_artifacts_schema.parametrize() +def test_artifacts_list(case, auth_token): + """GET /runs/{run_id}/artifacts — fuzz artifact listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Errors & Logs ------------------------------------------------- + +_errors_schema = _full_schema.include( + path="/runs/{run_id}/errors", method="GET") + + +@_errors_schema.parametrize() +def test_errors_list(case, auth_token): + """GET /runs/{run_id}/errors — fuzz error listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_infra_errors_schema = _full_schema.include( + path="/runs/{run_id}/infrastructure-errors", method="GET" +) + + +@_infra_errors_schema.parametrize() +def test_infrastructure_errors(case, auth_token): + """GET /runs/{run_id}/infrastructure-errors — fuzz infra error listing.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_error_summary_schema = _full_schema.include( + path="/runs/{run_id}/error-summary", method="GET" +) + + +@_error_summary_schema.parametrize() +def test_error_summary(case, auth_token): + """GET /runs/{run_id}/error-summary — fuzz error summary.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_logs_schema = _full_schema.include(path="/runs/{run_id}/logs", method="GET") + + +@_logs_schema.parametrize() +def test_logs(case, auth_token): + """GET /runs/{run_id}/logs — fuzz build log retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_sample_logs_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/logs", method="GET" +) + + +@_sample_logs_schema.parametrize() +def test_sample_logs(case, auth_token): + """GET /runs/{run_id}/samples/{sample_id}/logs — fuzz per-sample logs.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# --- Results (expected/actual/diff/baseline) ----------------------- + +_expected_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/expected", + method="GET", +) + + +@_expected_schema.parametrize() +def test_expected_output(case, auth_token): + """GET .../expected — fuzz expected output retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_actual_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/actual", + method="GET", +) + + +@_actual_schema.parametrize() +def test_actual_output(case, auth_token): + """GET .../actual — fuzz actual output retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_diff_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/regression-tests/{regression_id}/outputs/{output_id}/diff", + method="GET", +) + + +@_diff_schema.parametrize() +def test_diff(case, auth_token): + """GET .../diff — fuzz diff retrieval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +_baseline_schema = _full_schema.include( + path="/runs/{run_id}/samples/{sample_id}/baseline-approval", method="POST" +) + + +@_baseline_schema.parametrize() +def test_baseline_approval(case, auth_token): + """POST .../baseline-approval — fuzz baseline approval.""" + _set_auth(case, auth_token) + _call_safe(case) + + +# =================================================================== +# 3. NEGATIVE / SECURITY TESTS +# =================================================================== + + +class TestAuthSecurity: + """Verify authentication and authorization boundaries.""" + + def test_missing_auth_header_returns_401(self): + """Authenticated endpoints must reject requests without a token.""" + with app.test_client() as client: + for endpoint in [URL_RUNS, URL_SAMPLES, URL_SYSTEM_QUEUE]: + resp = client.get(endpoint) + assert resp.status_code == 401, ( + f"{endpoint} accepted unauthenticated request" + ) + + def test_invalid_bearer_token_returns_401(self): + """A garbage token must be rejected.""" + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": "Bearer INVALID_TOKEN_VALUE"}, + ) + assert resp.status_code == 401 + + def test_expired_token_returns_401(self): + """An expired token must be rejected.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + token_value = ApiToken.generate_token() + token_obj = ApiToken( + user_id=admin.id, + token_name=f"expired-{secrets.token_hex(4)}", + token_hash=ApiToken.hash_token(token_value), + token_prefix=ApiToken.extract_prefix(token_value), + scopes=[SCOPE_RUNS_READ], + expires_in_days=0, + ) + # Force expiration to the past + import datetime + + token_obj.expires_at = datetime.datetime.now( + datetime.timezone.utc + ) - datetime.timedelta(hours=1) + db.add(token_obj) + db.commit() + + try: + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token_value}"}, + ) + assert resp.status_code == 401, "Expired token was accepted" + finally: + db.delete(token_obj) + db.commit() + + def test_revoked_token_returns_401(self): + """A revoked token must be rejected.""" + db = create_session(app.config["DATABASE_URI"]) + + admin = User.query.filter_by(email=ADMIN_EMAIL).first() + token_value = ApiToken.generate_token() + token_obj = ApiToken( + user_id=admin.id, + token_name=f"revoked-{secrets.token_hex(4)}", + token_hash=ApiToken.hash_token(token_value), + token_prefix=ApiToken.extract_prefix(token_value), + scopes=[SCOPE_RUNS_READ], + ) + db.add(token_obj) + db.commit() + token_obj.revoke() + db.commit() + + try: + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token_value}"}, + ) + assert resp.status_code == 401, "Revoked token was accepted" + finally: + db.delete(token_obj) + db.commit() + + def test_insufficient_scope_returns_403(self, readonly_token): + """A token lacking the required scope must get 403, not 401.""" + with app.test_client() as client: + # runs:read token should not be able to access system:read endpoints + resp = client.get( + URL_SYSTEM_QUEUE, + headers={"Authorization": f"Bearer {readonly_token}"}, + ) + assert resp.status_code == 403 + + +# =================================================================== +# 4. RESPONSE INVARIANT CHECKS +# =================================================================== + + +class TestResponseInvariants: + """Verify structural invariants that hold across multiple endpoints.""" + + def test_health_returns_valid_json(self): + """GET /system/health must always return parseable JSON with 'status'.""" + with app.test_client() as client: + resp = client.get(URL_SYSTEM_HEALTH) + assert resp.status_code in (200, 503) + data = resp.get_json() + assert data is not None, "Health endpoint returned non-JSON" + assert "status" in data + assert data["status"] in ("ok", "degraded", "down") + + def test_paginated_endpoints_have_pagination_key(self, auth_token): + """All paginated GET endpoints must include 'pagination' in their response.""" + paginated = [ + URL_RUNS, + URL_SAMPLES, + "/api/v1/regression-tests", + URL_SYSTEM_QUEUE, + ] + with app.test_client() as client: + for endpoint in paginated: + resp = client.get( + endpoint, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + if resp.status_code == 200: + data = resp.get_json() + assert "pagination" in data, ( + f"{endpoint} missing 'pagination' key" + ) + pagination = data["pagination"] + assert "limit" in pagination + assert "offset" in pagination or "next_cursor" in pagination + assert "total" in pagination + + def test_rate_limit_headers_present(self, auth_token): + """Every API response must include X-RateLimit-* headers.""" + with app.test_client() as client: + resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + for header in [ + "X-RateLimit-Limit", + "X-RateLimit-Remaining", + "X-RateLimit-Reset", + ]: + assert header in resp.headers, f"Missing {header}" + + def test_error_response_format(self, auth_token): + """Error responses must follow the {code, message, details} shape.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/999999", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 404 + data = resp.get_json() + assert "code" in data, "Error response missing 'code'" + assert "message" in data, "Error response missing 'message'" + + def test_health_does_not_require_auth(self): + """GET /system/health must be accessible without any token.""" + with app.test_client() as client: + resp = client.get(URL_SYSTEM_HEALTH) + assert resp.status_code != 401 + + def test_content_type_is_json(self, auth_token): + """All API responses should return application/json content type.""" + with app.test_client() as client: + endpoints = [ + URL_RUNS, + URL_SYSTEM_HEALTH, + URL_SAMPLES, + ] + for endpoint in endpoints: + resp = client.get( + endpoint, + headers={"Authorization": f"Bearer {auth_token}"}, + ) + content_type = resp.content_type or "" + assert APP_JSON in content_type, ( + f"{endpoint} returned {content_type}" + ) + + +# =================================================================== +# 5. BOUNDARY / EDGE-CASE TESTS +# =================================================================== + + +class TestBoundaryConditions: + """Edge-case and boundary testing for pagination, IDs, and dates.""" + + def test_pagination_limit_zero_rejected(self, auth_token): + """limit=0 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_limit_over_max_rejected(self, auth_token): + """limit=101 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=101", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_negative_offset_rejected(self, auth_token): + """offset=-1 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?offset=-1", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_pagination_non_integer_limit_rejected(self, auth_token): + """limit=abc must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?limit=abc", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_path_id_zero_rejected(self, auth_token): + """run_id=0 must be rejected with 400 (IDs start at 1).""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_path_id_negative_rejected(self, auth_token): + """run_id=-1 must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/-1", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_nonexistent_run_returns_404(self, auth_token): + """A valid-format but non-existent run_id must return 404.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs/2147483647", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 404 + + def test_invalid_sort_rejected(self, auth_token): + """sort=invalid must be rejected with 400.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?sort=invalid", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_invalid_date_range_rejected(self, auth_token): + """A non-ISO-8601 created_after value must be rejected.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?created_after=not-a-date", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_cursor_and_offset_cannot_mix(self, auth_token): + """Mixing cursor and offset pagination must be rejected.""" + with app.test_client() as client: + resp = client.get( + "/api/v1/runs?cursor=0&offset=0", + headers={"Authorization": f"Bearer {auth_token}"}, + ) + assert resp.status_code == 400 + + def test_empty_body_on_post_rejected(self, auth_token): + """POST /runs with no body must be rejected.""" + with app.test_client() as client: + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": APP_JSON, + }, + data="", + ) + assert resp.status_code == 400 + + def test_wrong_content_type_rejected(self, auth_token): + """POST /runs with text/plain body must be rejected (415).""" + with app.test_client() as client: + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": "text/plain", + }, + data="not json", + ) + assert resp.status_code == 415 + + def test_extra_fields_rejected(self, auth_token): + """POST /runs with unknown fields must be rejected (additionalProperties: false).""" + with app.test_client() as client: + payload = { + "commit_sha": "a" * 40, + "platform": "linux", + "repository": "owner/repo", + "evil_extra": "should be rejected", + } + resp = client.post( + URL_RUNS, + headers={ + "Authorization": f"Bearer {auth_token}", + "Content-Type": APP_JSON, + }, + data=json.dumps(payload), + ) + assert resp.status_code == 400 + + +# =================================================================== +# 6. STATEFUL TOKEN LIFECYCLE TEST +# =================================================================== + + +class TestTokenLifecycle: + """Verify the create → use → revoke token lifecycle works end-to-end.""" + + def test_token_create_use_revoke(self): + """Create a token, use it, then revoke it and verify rejection.""" + with app.test_client() as client: + # 1. Create a token + create_resp = client.post( + "/api/v1/auth/tokens", + data=json.dumps( + { + "email": ADMIN_EMAIL, + "pass" + "word": "admin123", + "token_name": f"lifecycle-{secrets.token_hex(4)}", + "scopes": [SCOPE_RUNS_READ, "tokens:manage"], + } + ), + content_type=APP_JSON, + ) + assert create_resp.status_code == 201, ( + f"Token creation failed: {create_resp.get_json()}" + ) + token = create_resp.get_json()["token"] + + # 2. Use it + use_resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token}"}, + ) + assert use_resp.status_code == 200 + + # 3. Revoke it (self-revoke via /auth/tokens/current) + revoke_resp = client.delete( + "/api/v1/auth/tokens/current", + headers={"Authorization": f"Bearer {token}"}, + ) + assert revoke_resp.status_code == 204 + + # 4. Verify it's rejected + rejected_resp = client.get( + URL_RUNS, + headers={"Authorization": f"Bearer {token}"}, + ) + assert rejected_resp.status_code == 401, "Revoked token was still accepted" diff --git a/static/css/app.css b/static/css/app.css index 770f3a9eb..6ae72b0b5 100644 --- a/static/css/app.css +++ b/static/css/app.css @@ -108,6 +108,10 @@ html[dark=''] { display: none; } +label { + font-size: 16px; +} + label.success { background-color: #e1faea; padding: 0 5px; @@ -539,3 +543,21 @@ input.toggle-round:checked + label:after { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } } + +.tag:nth-child(even) { + border: 1px solid #c8e1ff; +} + +.tag:nth-child(odd) { + color: #1779ba; + border: 1px solid #c8e1ff; +} + +.tag { + display: inline-block; + background-color: var(--off-canvas-content-bg); + padding: 5px 10px; + border-radius: 4px; + margin-right: 5px; + font-size: 14px; +} diff --git a/static/svg/SUCCESS-Windows.svg b/static/svg/SUCCESS-windows.svg similarity index 100% rename from static/svg/SUCCESS-Windows.svg rename to static/svg/SUCCESS-windows.svg diff --git a/templates/400.html b/templates/400.html new file mode 100644 index 000000000..9e612056e --- /dev/null +++ b/templates/400.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% set hideMenu = True %} +{% block title %}400 - Bad Request{% endblock %} +{% block body %} + {{ super() }} +
+
+
+

400 - Bad Request

+

Invalid request made to the server.

+
+
+{% endblock %} diff --git a/templates/ci/pr_comment.txt b/templates/ci/pr_comment.txt index 6af243d6e..773d9675f 100644 --- a/templates/ci/pr_comment.txt +++ b/templates/ci/pr_comment.txt @@ -1,10 +1,25 @@ -CCExtractor CI platform finished running the test files on {{platform}}. Below is a summary of the test results: +{% macro test_list(tests) -%} + +{%- endmacro %} +{% set changed = namespace(here=[], reference=none) %} +{% for comparison in comment_info.comparisons %} +{% if comparison.run and not comparison.duplicate_of and changed.reference is none %} +{% set changed.here = comparison.tests.broken_here + comparison.tests.failing_differently %} +{% set changed.reference = comparison %} +{% endif %} +{% endfor %} +
+CCExtractor CI platform finished running the test files on {{platform}}. {{ comment_info.passed_count }}/{{ comment_info.total_count }} tests matched the approved output: -{% for test in tests%} +{% for test in comment_info.category_stats %} {% if test.success==None %} @@ -17,17 +32,56 @@ {% endfor %}
Report Name Tests Passed
{{test.category}}
-{% if state == "failure" %} -It seems that not all tests were passed completely. This is an indication that the output of some files is not as expected (but might be according to you). -Your PR breaks these cases: +{% if comment_info.failed_tests | length %} +{{ comment_info.failed_tests | length }} tests do not match the approved output. That is the pass/fail verdict. Whether this branch caused it is a separate question, answered below. +{{ test_list(comment_info.failed_tests) }} +{% endif %} + +{% for comparison in comment_info.comparisons %} +
+{% if comparison.duplicate_of %} +Compared with {{ comparison.label }}: the same run as {{ comparison.duplicate_of }} (test {{ comparison.run.id }}), so the comparison above already covers it. +{% elif not comparison.run %} +Compared with {{ comparison.label }}: no completed run found, so nothing can be said about this reference. This is not the same as agreement. +{% else %} +Compared with {{ comparison.label }} — test {{ comparison.run.id }}, commit {{ comparison.run.commit[:7] }}:
    -{% for test in failed_tests %} -
  • ccextractor {{ test.command }} {{ test.sample.sha[:10] }}...
    • -{% endfor %} +
  • {{ comparison.counts.broken_here }} pass there and fail here
    • +
  • {{ comparison.counts.fixed_here }} fail there and pass here
    • +
  • {{ comparison.counts.failing_differently }} fail on both, with different output
    • +
  • {{ comparison.counts.failing_identically }} fail on both, byte for byte the same
    • +{% if comparison.counts.no_reference %} +
  • {{ comparison.counts.no_reference }} have no result in that run
    • +{% endif %}
-{% else %} -All tests were passed completely. +{% if comparison.tests.broken_here | length %} +Pass there, fail here: +{{ test_list(comparison.tests.broken_here) }} +{% endif %} +{% if comparison.tests.failing_differently | length %} +Fail on both but produce different output — behaviour moved even though the verdict did not: +{{ test_list(comparison.tests.failing_differently) }} +{% endif %} +{% if comparison.tests.fixed_here | length %} +Fail there, pass here: +{{ test_list(comparison.tests.fixed_here) }} {% endif %} +{% endif %} +{% endfor %}
-Check the result page for more info. +
+ +{% if changed.reference is none %} +{% if comment_info.failed_tests | length %} +{{ comment_info.failed_tests | length }} tests do not match the approved output. There is no completed run to compare against, so this report cannot say whether this branch caused them — treat the list above as unattributed. +{% else %} +All tests passed: every output matched the approved file. +{% endif %} +{% elif changed.here | length %} +This branch changes the behaviour of {{ changed.here | length }} test(s) relative to {{ changed.reference.label }}. Those are the ones worth looking at; anything else in the list fails the same way on both sides. +{% elif comment_info.failed_tests | length %} +No test changes behaviour relative to {{ changed.reference.label }}: every failure above fails there too, byte for byte. The approved output for those tests is out of date, which is a baseline to review rather than a regression in this branch. +{% else %} +All tests passed: every output matched the approved file. +{% endif %} diff --git a/templates/regression/index.html b/templates/regression/index.html index d743433e2..3e98f1fbb 100644 --- a/templates/regression/index.html +++ b/templates/regression/index.html @@ -8,12 +8,13 @@

Regression tests

{% if tests|length > 0 %} -

- - {% for category in categories %} {% endfor %} +   {% if user is not none and user.is_admin %} @@ -21,14 +22,40 @@

Regression tests

  {% endif %} -

-

Category description: - +

+
+ + {% for category in categories %} {{ category.description }} {% endfor %} + {{ 'All Categories' }} +   +
  +
+ +   + + + + + -

+
+
+ +
+
  +
  @@ -37,6 +64,7 @@

Regression tests

+ @@ -47,7 +75,7 @@

Regression tests

- + + {% endfor %} @@ -82,12 +122,77 @@
Regression tests
{% block scripts %} {{ super() }} +{% endblock %} \ No newline at end of file diff --git a/templates/sample/list_samples.html b/templates/sample/list_samples.html index 60ae9aec6..ee6d16274 100644 --- a/templates/sample/list_samples.html +++ b/templates/sample/list_samples.html @@ -6,6 +6,7 @@ + @@ -20,6 +21,17 @@   {% endif %} + {% endfor %} diff --git a/templates/sample/sample_info.html b/templates/sample/sample_info.html index 7d52019c0..8e181b309 100644 --- a/templates/sample/sample_info.html +++ b/templates/sample/sample_info.html @@ -58,7 +58,7 @@

Additional files

{% endif %}
Command Active ActionsTags
{{ test.id }} {{ test.command }}{{ true if test.active == 1 else false}}{{ test.active }}   {% if user is not none and user.has_role('contributor') %} @@ -57,6 +85,18 @@

Regression tests

  {% endif %} 		+ {% set sample = test.sample %} + {% set visible_tags_count = 3 %} + {% for tag in sample.tags[:visible_tags_count] %} + {{ tag.name }} + {% endfor %} + {% if sample.tags|length > visible_tags_count %} + + +{{ sample.tags|length - visible_tags_count }} more + + {% endif %} +
{{ 'Original name' if use_sample_original|default(false) else 'SHA-256' }} Video-stream type ActionsTags
+ {% set visible_tags_count = 2 %} + {% for tag in sample.tags[:visible_tags_count] %} + {{ tag.name }} + {% endfor %} + {% if sample.tags|length > visible_tags_count %} + + +{{ sample.tags|length - visible_tags_count }} more + + {% endif %} +
{{ file.original_name }} - + {% if user.is_admin %}   {% endif %} @@ -68,6 +68,19 @@

Additional files
{% endif %} +

+ Tags: + {% if sample.tags|length %} + {% for tag in sample.tags %} + {{ tag.name }} + {% endfor %} + {% else %} + No tags yet. + {% endif %} + {% if user.is_admin %} +   + {% endif %} +

Test status & regression tests

Repository: {{ latest_commit }}

Last release: {{ latest_release }}

@@ -90,9 +103,12 @@

GitHub Issues

Actions
    -
  • Download sample
    • +
  • + Download sample
    • {% if media %} -
  • Download media info xml
    • +
  • + Download media info xml +
    • {% endif %} {% if user.is_admin %}
  • Edit sample
    • diff --git a/templates/test/by_id.html b/templates/test/by_id.html index d21a15020..cf4f42f4c 100644 --- a/templates/test/by_id.html +++ b/templates/test/by_id.html @@ -1,6 +1,22 @@ {% extends "base.html" %} {% block title %}Test progress for {{ title }} {{ super() }}{% endblock %} + +{% block styles %} + {{ super() }} + +{% endblock %} + {% block body %} {{ super() }}
    @@ -15,7 +31,7 @@

    Test progress for {{ title }}

    {{ test.test_type.description }} {% if test.test_type == TestType.pull_request %} - {{ test.pr_nr }} (Commit {{ test.commit[:7] }}) + {{ test.pr_nr }} (PR head {{ test.commit[:7] }}{% if test.built_commit and test.built_commit != test.commit %}, tested as {{ test.built_commit[:7] }}{% endif %}) {% elif test.test_type == TestType.commit %} {{ test.commit[:7] }} (Branch {{ test.branch }}) {% endif %} @@ -48,13 +64,22 @@

    Test progress for {{ title }}

    {% if progress.progress.state == 'error' and progress.progress.step == loop.index0 -%} {% set status = status ~ ' error' %} {%- endif %} -
  • {{ stage.description }}
    • +
  • + {{ stage.description }} + {% if stage.description == 'Testing' and status == 'running' %} +
    + {{ sample_progress.current }} / {{ sample_progress.total }} samples{% if sample_progress.total > 0 %} ({{ sample_progress.percentage }}%){% endif %} +
    + {% endif %} +
    • {%- endfor %}
    {% if test.progress|length > 0 %} - Show log file + {% if test.finished %} + Download log file + {% endif %}
    @@ -76,7 +101,7 @@

    Test progress for {{ title }}

    {% else %} -

    This test is still queued. There are {{ next }} tests queued before this one. Based on the average runtime, this test will be finished in {{ hr }} hours {{ min }} minutes.

    +

    Your test VM is being provisioned.{% if avg_minutes > 0 %} Tests on {{ test.platform.description }} typically complete in about {{ avg_minutes }} minutes.{% endif %}

    {% endif %} {% if test.finished %} {% if test.failed %} @@ -120,13 +145,15 @@

    Fail {%- endif %} - {% elif file.got is none or no_error.found or test.result.exit_code != 0 -%} + {% elif file.got == "error" %} + No output generated but there should be + {% elif file.got is none or no_error.found or (test.result and test.result.exit_code != 0) -%} Pass {% else %} Fail @@ -179,6 +206,17 @@

    There are no tests executed in this category.
    if (testprogress === 0 && val.length !== 0) { window.location.reload(); } + + // Update sample progress display + if (data.sample_progress) { + var progressText = document.getElementById('sample-progress-text'); + if (progressText) { + progressText.textContent = data.sample_progress.current + ' / ' + + data.sample_progress.total + ' samples (' + + data.sample_progress.percentage + '%)'; + } + } + {% for stage in progress.stages %} if (data.details.step >= {{ loop.index0 }}) { status = 'done'; @@ -248,10 +286,8 @@
    There are no tests executed in this category.
    reveal.setAttribute('data-reveal', ''); reveal.innerHTML = resp; reveal.innerHTML += - '' + - '' + ', only first 50 shown here.'; reveal.innerHTML += ' + + + {open && ( + setOpen(false)} + > + e.stopPropagation()} + > +
    + + { + setQ(e.target.value); + setCursor(0); + }} + onKeyDown={(e) => { + if (e.key === "ArrowDown") { + e.preventDefault(); + setCursor((c) => Math.min(c + 1, items.length - 1)); + } else if (e.key === "ArrowUp") { + e.preventDefault(); + setCursor((c) => Math.max(c - 1, 0)); + } else if (e.key === "Enter" && items[cursor]) { + pick(items[cursor]); + } + }} + /> +
    +
    + {items.map((item, i) => ( + + ))} + {items.length === 0 && ( +
    No matches.
    + )} +
    +
    + ↑↓ navigate + ↵ open + esc close +
    +     +     + )} +     + + ); +} diff --git a/web/src/components/DiffDrawer.tsx b/web/src/components/DiffDrawer.tsx new file mode 100644 index 000000000..a148aab58 --- /dev/null +++ b/web/src/components/DiffDrawer.tsx @@ -0,0 +1,372 @@ +import { useQuery } from "@tanstack/react-query"; +import { Download, FileText, PlayCircle } from "lucide-react"; +import { useEffect, useMemo, useRef, useState } from "react"; + +import { AnimatedSheet, SheetDescription, SheetTitle } from "@/components/ui/sheet"; +import { fetchDiff, outputText, sampleFile } from "@/lib/api"; +import { clock, cuesToVtt, parseCues, type Cue } from "@/lib/cues"; +import { cn } from "@/lib/utils"; + +export interface DiffTarget { + runId: number; + sampleId: number; + regressionId: number; + outputId: number; + command: string; + sampleName: string; +} + +type View = "diff" | "playback"; + +/** MediaError.MEDIA_ERR_SRC_NOT_SUPPORTED — nothing here can decode the file. */ +const MEDIA_ERR_SRC_NOT_SUPPORTED = 4; + +/** + * One failing output, either as a unified diff or played back against the + * sample it came from. + * + * Playback is the view for deciding whether a mismatch is a real regression or + * a stale baseline: captions only mean something next to the picture they + * describe. The diff stays the default, because most differences are obvious + * without watching anything. + */ +export function DiffDrawer({ + target, + onClose, +}: Readonly<{ + target: DiffTarget | null; + onClose: () => void; +}>) { + const [view, setView] = useState("diff"); + + return ( + !o && onClose()} + className={view === "playback" ? "max-w-5xl" : "max-w-2xl"} + > + {target && ( +
    +
    + + Test #{target.regressionId} + + + {target.command} · {target.sampleName} · run {target.runId} + +
    + setView("diff")} + /> + setView("playback")} + /> +
    +
    + {view === "diff" ? : } +
    + )} +     + ); +} + +function ViewTab({ + icon: Icon, + label, + on, + go, +}: Readonly<{ + icon: typeof FileText; + label: string; + on: boolean; + go: () => void; +}>) { + return ( + + ); +} + +/** Pair each line with a stable id, so the render key is never an index. */ +function numbered(lines: string[]): { id: string; line: string }[] { + return lines.map((line, i) => ({ id: `line-${i}`, line })); +} + +function DiffPane({ target }: Readonly<{ target: DiffTarget }>) { + const { data, isLoading, isError, error } = useQuery({ + queryKey: ["diff", target.runId, target.regressionId, target.outputId], + staleTime: Infinity, + retry: false, + queryFn: () => fetchDiff(target), + }); + + return ( +
    + {isLoading && } + {isError && ( + + {error instanceof Error ? error.message : "Could not load diff"} — the output + file for this result isn't present in storage. + + )} + {data && ( + 
    +          {data.content.trim() === "" ? (
+            Diff is empty.
+          ) : (
+            numbered(data.content.split("\n")).map(({ id, line }) => (
+              
    
+                {line || " "}
+              
    
+            ))
+          )}
+
    + )} +
    + ); +} + +/** + * The sample playing above its two caption tracks. + * + * Browsers decode very little of this library — most of it is broadcast + * captures in containers no browser ships a demuxer for — so the player is + * offered optimistically and the element's own error event decides whether it + * worked. When it cannot play, the two tracks are still worth reading side by + * side, and the sample is one click away for a local player. + */ +function PlaybackPane({ target }: Readonly<{ target: DiffTarget }>) { + const video = useRef(null); + const [at, setAt] = useState(0); + // The element's own error code, kept rather than a boolean: a container no + // browser can demux and a signed URL that never arrived look identical on + // screen but mean completely different things to whoever has to fix it. + const [failure, setFailure] = useState(null); + + const file = useQuery({ + queryKey: ["sample-file", target.sampleId], + staleTime: 300_000, + retry: false, + queryFn: () => sampleFile(target.sampleId), + }); + + const expected = useOutput(target, "expected"); + const actual = useOutput(target, "actual"); + + // The baseline is what the sample is supposed to say, so it doubles as the + // player's caption track. Held as a blob rather than a request, since the + // cues are already parsed and in memory. + const captions = useMemo(() => { + const vtt = cuesToVtt(expected.data?.cues ?? []); + return URL.createObjectURL(new Blob([vtt], { type: "text/vtt" })); + }, [expected.data]); + useEffect(() => () => URL.revokeObjectURL(captions), [captions]); + + const seek = (t: number) => { + if (!video.current) return; + video.current.currentTime = t; + video.current.play().catch(() => {}); + }; + + return ( +
    + {file.isLoading && ( +
    + )} + + {file.data?.download_url && failure === null && ( + + )} + + {(failure !== null || (file.data && !file.data.download_url) || file.isError) && ( + + {file.data?.download_url ? ( + <> + {failure === MEDIA_ERR_SRC_NOT_SUPPORTED + ? "No browser ships a demuxer for this container, which covers most of the broadcast captures in this library." + : "The sample did not download — its storage link may have expired."} + + Download {file.data.filename} + + + ) : ( + <> + This sample has no shared copy to stream — the only one is on the + platform's own disk. + + )}{" "} + The captions below still line up by timecode. + + )} + +
    + + +
    +
    + ); +} + +function useOutput(target: DiffTarget, which: "expected" | "actual") { + return useQuery({ + queryKey: ["output-text", target.runId, target.regressionId, target.outputId, which], + staleTime: Infinity, + retry: false, + queryFn: async () => { + const file = await outputText(target, which); + return { ...file, cues: parseCues(file.content) }; + }, + }); +} + +type OutputQuery = ReturnType; + +function CueColumn({ + title, + tone, + q, + at, + onSeek, +}: Readonly<{ + title: string; + tone: "ok" | "bad"; + q: OutputQuery; + at: number; + onSeek: (t: number) => void; +}>) { + return ( +
    +
    + + {title} + + {q.data && ( + + {q.data.cues.length > 0 ? `${q.data.cues.length} cues` : q.data.filename} + + )} +
    +
    + {q.isLoading && ( +
    + +
    + )} + {q.isError && ( +

    + {title === "Actual" + ? "This run produced no output for the test." + : "No baseline file is present in storage."} +

    + )} + {q.data?.cues.length === 0 && ( + // Plain transcripts carry no timings, so there is nothing to sync. + 
    +            {q.data.content.trim() || Empty file.}
+
    + )} + {q.data?.cues.map((c) => ( + = c.start && at < c.end} onSeek={onSeek} /> + ))} + {q.data?.truncated && ( +

    Truncated at 1 MiB.

    + )} +
    +
    + ); +} + +function CueRow({ + cue, + active, + onSeek, +}: Readonly<{ + cue: Cue; + active: boolean; + onSeek: (t: number) => void; +}>) { + return ( + + ); +} + +function Skeleton({ rows }: Readonly<{ rows: number }>) { + return ( +
    + {Array.from({ length: rows }, (_, n) => `placeholder-${n}`).map((id) => ( +
    + ))} +
    + ); +} + +function Notice({ children }: Readonly<{ children: React.ReactNode }>) { + return ( +
    + {children} +
    + ); +} diff --git a/web/src/components/Login.tsx b/web/src/components/Login.tsx new file mode 100644 index 000000000..48895028d --- /dev/null +++ b/web/src/components/Login.tsx @@ -0,0 +1,171 @@ +import { Loader2, LogIn, Moon, Sun } from "lucide-react"; +import { motion } from "motion/react"; +import { useEffect, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { requestPasswordReset, requestSignup } from "@/lib/api"; +import { login } from "@/lib/auth"; +import { asset } from "@/lib/utils"; + +const LINK = "cursor-pointer text-primary hover:underline"; + +type Mode = "signin" | "reset" | "signup"; + +const COPY: Record = { + signin: { title: "", action: "", sent: "" }, + reset: { + title: "Reset your password", + action: "Email me a reset link", + // Deliberately the same whether or not the address has an account: a + // different answer here would tell an anonymous caller who is registered. + sent: "If that address has an account, a reset link is on its way.", + }, + signup: { + title: "Create an account", + action: "Email me a signup link", + sent: "If that address can be registered, a signup link is on its way.", + }, +}; + +/** Full-screen sign-in against POST /api/v1/auth/tokens. */ +export function Login({ onDone }: Readonly<{ onDone: () => void }>) { + const [mode, setMode] = useState("signin"); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [error, setError] = useState(null); + const [sent, setSent] = useState(false); + const [busy, setBusy] = useState(false); + + const go = (next: Mode) => { + setMode(next); + setError(null); + setSent(false); + }; + + const submit = async (e: React.FormEvent) => { + e.preventDefault(); + setBusy(true); + setError(null); + try { + if (mode === "signin") { + await login(email, password); + onDone(); + } else { + await (mode === "reset" ? requestPasswordReset : requestSignup)(email); + setSent(true); + } + } catch (err) { + setError(err instanceof Error ? err.message : "That did not work."); + } finally { + setBusy(false); + } + }; + + const [dark, setDark] = useState(() => localStorage.getItem("sp-theme") === "dark"); + useEffect(() => { + document.documentElement.classList.toggle("dark", dark); + localStorage.setItem("sp-theme", dark ? "dark" : "light"); + }, [dark]); + + return ( +
    + + +
    + CCExtractor +
    +
    Sample Platform
    +
    CCExtractor regression testing
    +
    +
    + + {mode !== "signin" && ( +
    {COPY[mode].title}
    + )} + + + setEmail(e.target.value)} + placeholder="you@example.com" + className="mb-3" + /> + {mode === "signin" && ( + <> + + setPassword(e.target.value)} + className="mb-4" + /> + + )} + {error && ( +
    + {error} +
    + )} + {sent && ( +
    + {COPY[mode].sent} +
    + )} + + + {/* Both links finish on the classic site, which is where accounts are + created and passwords are set. */} +
    + {mode === "signin" ? ( + <> + + · + + + ) : ( + + )} +
    + +

    + Use the same account as the CCExtractor Sample Platform. +

    +     +
    + ); +} diff --git a/web/src/components/ResetPassword.tsx b/web/src/components/ResetPassword.tsx new file mode 100644 index 000000000..626230ac3 --- /dev/null +++ b/web/src/components/ResetPassword.tsx @@ -0,0 +1,158 @@ +import { KeyRound, Loader2 } from "lucide-react"; +import { motion } from "motion/react"; +import { useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { completePasswordReset } from "@/lib/api"; +import { asset } from "@/lib/utils"; + +/** + * Set a new password from the link in a recovery email. + * + * Reached without a session, so it renders on its own rather than inside + * the shell. The three values in the query string are the platform's + * signed link: they are handed straight back to the API, which is what + * checks them. Nothing here can decide whether a link is good. + */ +// Read from the URL rather than the router: the link is built by the +// platform's email template, and lands here under either history mode. +function readResetLink(): { uid: number; expires: number; mac: string } | null { + const raw = window.location.href; + const query = raw.slice(raw.indexOf("?") + 1); + const q = new URLSearchParams(raw.includes("?") ? query : ""); + const uid = Number(q.get("uid")); + const expires = Number(q.get("expires")); + const mac = q.get("mac"); + return uid && expires && mac ? { uid, expires, mac } : null; +} + +export function ResetPassword() { + const [params] = useState(readResetLink); + const [password, setPassword] = useState(""); + const [confirm, setConfirm] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(null); + const [done, setDone] = useState(false); + + const submit = async (e: React.FormEvent) => { + e.preventDefault(); + if (!params) return; + setBusy(true); + setError(null); + try { + await completePasswordReset({ + user_id: params.uid, + expires: params.expires, + mac: params.mac, + password, + }); + setDone(true); + } catch (err) { + setError(err instanceof Error ? err.message : "That did not work."); + } finally { + setBusy(false); + } + }; + + return ( +
    + +
    + CCExtractor +
    +
    + Choose a new password +
    +
    CCExtractor Sample Platform
    +
    +
    + + {!params && ( +
    + This link is missing part of itself. Ask for a new one from the sign-in + page. +
    + )} + + {params && !done && ( + <> + + setPassword(e.target.value)} + className="mb-3" + /> + + setConfirm(e.target.value)} + className="mb-4" + /> + {confirm && password !== confirm && ( +
    + The two do not match. +
    + )} + {error && ( +
    + {error} +
    + )} + + {/* The link carries the old password's signature, so using it + once is the last thing it can do. */} +

    + This link works once, and stops working as soon as the password + changes. +

    + + )} + + {done && ( + <> +
    + Password changed. You can sign in with it now. +
    + + + )} +     +
    + ); +} diff --git a/web/src/components/ResizeHandle.tsx b/web/src/components/ResizeHandle.tsx new file mode 100644 index 000000000..1ddc55925 --- /dev/null +++ b/web/src/components/ResizeHandle.tsx @@ -0,0 +1,103 @@ +import { useCallback, useEffect, useRef, useState } from "react"; + +import { cn } from "@/lib/utils"; + +/** + * Persisted, draggable width for a pane. Returns the current width and a + * to drop on the pane's trailing edge. + * + * Pass edge="leading" for a pane anchored to the right of the screen: its + * handle sits on the left, where dragging towards the left has to widen + * rather than narrow it. + */ +export function useResizableWidth( + key: string, + initial: number, + min: number, + max: number, + edge: "trailing" | "leading" = "trailing", +) { + const [width, setWidth] = useState(() => { + const saved = Number(localStorage.getItem(`sp-w-${key}`)); + return saved >= min && saved <= max ? saved : initial; + }); + const dragging = useRef(false); + const sign = edge === "leading" ? -1 : 1; + + const onMouseDown = useCallback(() => { + dragging.current = true; + document.body.style.cursor = "col-resize"; + document.body.style.userSelect = "none"; + }, []); + + useEffect(() => { + const onMove = (e: MouseEvent) => { + if (!dragging.current) return; + setWidth((w) => { + const next = Math.min(max, Math.max(min, w + sign * e.movementX)); + return next; + }); + }; + const onUp = () => { + if (!dragging.current) return; + dragging.current = false; + document.body.style.cursor = ""; + document.body.style.userSelect = ""; + setWidth((w) => { + localStorage.setItem(`sp-w-${key}`, String(w)); + return w; + }); + }; + window.addEventListener("mousemove", onMove); + window.addEventListener("mouseup", onUp); + return () => { + window.removeEventListener("mousemove", onMove); + window.removeEventListener("mouseup", onUp); + }; + }, [key, min, max, sign]); + + // A splitter has to be operable without a mouse, so the same width change + // is reachable from the arrow keys. Shift moves in coarser steps. + const onKeyDown = (e: React.KeyboardEvent) => { + if (e.key !== "ArrowLeft" && e.key !== "ArrowRight") return; + e.preventDefault(); + const step = (e.shiftKey ? 40 : 8) * (e.key === "ArrowRight" ? 1 : -1) * sign; + setWidth((w) => { + const next = Math.min(max, Math.max(min, w + step)); + localStorage.setItem(`sp-w-${key}`, String(next)); + return next; + }); + }; + + // role="slider" rather than "separator": what this actually is, to both a + // screen reader and a linter, is a one-dimensional value the user drags or + // arrows between a min and a max. A focusable separator is the older + // window-splitter idiom and reads as non-interactive. + const handle = ( +
    +
    +
    + ); + + return { width, handle }; +} diff --git a/web/src/components/Sparkline.tsx b/web/src/components/Sparkline.tsx new file mode 100644 index 000000000..b19ad4d49 --- /dev/null +++ b/web/src/components/Sparkline.tsx @@ -0,0 +1,24 @@ +import type { SparkResult } from "@/lib/types"; +import { cn } from "@/lib/utils"; + +/** Pass/fail history dots, newest last. */ +export function Sparkline({ results, className }: Readonly<{ results: SparkResult[]; className?: string }>) { + if (results.length === 0) { + return no runs yet; + } + return ( +
    + {results.map((r, i) => ({ id: `run-${i}`, r })).map(({ id, r }) => ( + + ))} +
    + ); +} diff --git a/web/src/components/StatusBadge.tsx b/web/src/components/StatusBadge.tsx new file mode 100644 index 000000000..6b3322109 --- /dev/null +++ b/web/src/components/StatusBadge.tsx @@ -0,0 +1,32 @@ +import type { RunStatus } from "@/lib/types"; +import { cn } from "@/lib/utils"; + +/** Colored dot + quiet label; in-flight states pulse. */ +const RUN_STATUS: Record = { + queued: { label: "Queued", dot: "bg-faint" }, + running: { label: "Running", dot: "bg-primary", pulse: true }, + pass: { label: "Passed", dot: "bg-success" }, + fail: { label: "Failed", dot: "bg-destructive" }, + canceled: { label: "Canceled", dot: "bg-faint" }, + error: { label: "Error", dot: "bg-destructive" }, + incomplete: { label: "Incomplete", dot: "bg-warning" }, + preparation: { label: "Preparing", dot: "bg-warning", pulse: true }, + building: { label: "Building", dot: "bg-warning", pulse: true }, + testing: { label: "Testing", dot: "bg-primary", pulse: true }, + completed: { label: "Completed", dot: "bg-success" }, +}; + +export function RunStatusBadge({ status, className }: Readonly<{ status: RunStatus; className?: string }>) { + const s = RUN_STATUS[status] ?? RUN_STATUS.queued; + return ( + + + {s.pulse && ( + + )} + + + {s.label} + + ); +} diff --git a/web/src/components/layout/AppShell.tsx b/web/src/components/layout/AppShell.tsx new file mode 100644 index 000000000..4f08d409e --- /dev/null +++ b/web/src/components/layout/AppShell.tsx @@ -0,0 +1,219 @@ +import { Link, Outlet, useRouterState } from "@tanstack/react-router"; +import { + Activity, + FileVideo, + FlaskConical, + Gauge, + Inbox, + LogOut, + Moon, + Search, + Settings, + Sun, + UploadCloud, +} from "lucide-react"; +import { motion } from "motion/react"; +import { useEffect, useState } from "react"; + +import { CommandPalette } from "@/components/CommandPalette"; +import { Login } from "@/components/Login"; +import { SESSION_CHANGED, getSession, logout } from "@/lib/auth"; +import { asset, cn } from "@/lib/utils"; + +/* Classic-site menu names so migrating devs feel at home. */ +const sections = [ + { + id: "main", + label: null, + items: [ + { to: "/", label: "Home", icon: Inbox }, + { to: "/runs", label: "Test results", icon: Activity }, + ], + }, + { + id: "suite", + label: "Suite", + items: [ + { to: "/tests", label: "Regression tests", icon: FlaskConical }, + { to: "/samples", label: "Samples", icon: FileVideo }, + { to: "/upload", label: "Sample upload", icon: UploadCloud }, + ], + }, + { + id: "platform", + label: "Platform", + items: [ + { to: "/status", label: "Platform status", icon: Gauge }, + { to: "/admin", label: "Administration", icon: Settings }, + ], + }, +] as const; + +function useTheme() { + const [dark, setDark] = useState(() => localStorage.getItem("sp-theme") === "dark"); + useEffect(() => { + document.documentElement.classList.toggle("dark", dark); + localStorage.setItem("sp-theme", dark ? "dark" : "light"); + }, [dark]); + return { dark, toggle: () => setDark((d) => !d) }; +} + +const COLLAPSED = 52; +const EXPANDED = 224; + +export function AppShell() { + const { dark, toggle } = useTheme(); + const pathname = useRouterState({ select: (s) => s.location.pathname }); + const [session, setSession] = useState(getSession); + const [open, setOpen] = useState(false); + + // The account page can change the email shown here, so pick the stored + // session back up instead of leaving the corner reading the old one. + useEffect(() => { + const sync = () => setSession(getSession()); + window.addEventListener(SESSION_CHANGED, sync); + return () => window.removeEventListener(SESSION_CHANGED, sync); + }, []); + + // The reset screen arrives from an email with nobody signed in, so it + // renders on its own rather than behind the sign-in gate. + if (pathname === "/reset") return ; + if (!session) return setSession(getSession())} />; + + return ( +
    + {/* Rail reserves collapsed width; sidebar overlays on hover so main never reflows. */} +
    + + setOpen(true)} + onMouseLeave={() => setOpen(false)} + initial={false} + animate={{ width: open ? EXPANDED : COLLAPSED }} + transition={{ type: "spring", stiffness: 420, damping: 38 }} + className={cn( + "absolute inset-y-2 left-2 z-30 flex flex-col overflow-hidden rounded-xl", + open && "border bg-card shadow-pop", + )} + > +
    + CCExtractor + + Sample Platform + + {open && ( + + )} +
    + +
    + {open ? ( + + ) : ( +
    + +
    + )} +
    + + + +
    + {/* The whole identity block is the way to the account page — the + classic site puts it behind the same name in the corner. */} + +
    + {session.email.slice(0, 2)} +
    + {open && ( +
    +
    {session.email}
    +
    {session.role}
    +
    + )} + + {open && ( + + )} +
    +     + +
    + + + +
    +
    + ); +} diff --git a/web/src/components/ui/badge.tsx b/web/src/components/ui/badge.tsx new file mode 100644 index 000000000..9e7ce9c7d --- /dev/null +++ b/web/src/components/ui/badge.tsx @@ -0,0 +1,32 @@ +import { cva, type VariantProps } from "class-variance-authority"; +import * as React from "react"; + +import { cn } from "@/lib/utils"; + +const badgeVariants = cva( + "inline-flex items-center gap-1 rounded-md px-1.5 py-px text-[11px] font-medium leading-4 whitespace-nowrap", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground", + secondary: "border border-border bg-muted/60 text-muted-foreground", + outline: "border border-border text-muted-foreground", + success: "bg-success/10 text-success", + destructive: "bg-destructive/10 text-destructive", + warning: "bg-warning/12 text-warning", + accent: "bg-accent text-accent-foreground", + }, + }, + defaultVariants: { variant: "default" }, + }, +); + +function Badge({ + className, + variant, + ...props +}: Readonly & VariantProps>) { + return ; +} + +export { Badge, badgeVariants }; diff --git a/web/src/components/ui/button.tsx b/web/src/components/ui/button.tsx new file mode 100644 index 000000000..74d021d5a --- /dev/null +++ b/web/src/components/ui/button.tsx @@ -0,0 +1,42 @@ +import { cva, type VariantProps } from "class-variance-authority"; +import * as React from "react"; + +import { cn } from "@/lib/utils"; + +const buttonVariants = cva( + "press inline-flex cursor-pointer items-center justify-center gap-1.5 whitespace-nowrap rounded-lg text-[13px] font-medium transition-colors duration-150 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-45 [&_svg]:size-3.5 [&_svg]:shrink-0", + { + variants: { + variant: { + default: + "bg-primary text-primary-foreground shadow-sm hover:bg-primary-hover", + secondary: + "border border-border bg-card text-foreground shadow-xs hover:bg-muted", + outline: + "border border-border bg-transparent hover:bg-muted hover:border-border-strong", + ghost: "text-muted-foreground hover:bg-muted hover:text-foreground", + destructive: "bg-destructive text-white shadow-sm hover:opacity-90", + }, + size: { + default: "h-8 px-3.5", + sm: "h-7 rounded-md px-2.5 text-xs", + lg: "h-9 rounded-lg px-5", + icon: "h-8 w-8", + }, + }, + defaultVariants: { variant: "default", size: "default" }, + }, +); + +function Button({ + className, + variant, + size, + ...props +}: Readonly & VariantProps>) { + return ( + + +
    + + + + )} + + + ); +} diff --git a/web/src/components/ui/input.tsx b/web/src/components/ui/input.tsx new file mode 100644 index 000000000..20ca50503 --- /dev/null +++ b/web/src/components/ui/input.tsx @@ -0,0 +1,29 @@ +import * as React from "react"; + +import { cn } from "@/lib/utils"; + +function Input({ className, ...props }: Readonly>) { + return ( + + ); +} + +function Textarea({ className, ...props }: Readonly>) { + return ( +