diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..3ec5303 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,63 @@ +.git +.gitignore +.gitattributes +.github +.env +.env.* + +**/__pycache__ +**/*.py[cod] +**/.pytest_cache +**/.mypy_cache +**/.ruff_cache +**/.cache +.coverage +.coverage.* +coverage.xml +htmlcov + +.venv +venv +env +ENV +**/*.egg-info +build +dist + +frontend/node_modules +frontend/dist +frontend/.test-dist +node_modules + +*.db +*.db-* +*.sqlite +*.sqlite3 +*.sqlite-* +src/data/** +!src/data/*.py +**/raw/** +**/processed/** +**/preprocessed/** +**/browser-profile/** +**/browser_profiles/** +**/playwright-report/** +**/test-results/** + +docs +exports +**/exports/** +**/audit-staging/** +**/*.tfstate +**/*.tfstate.* + +*.log +*.tmp +*.temp +*.bak +*.swp +*.swo +.DS_Store +._* + +*.ipynb diff --git a/.env.example b/.env.example index dae4d22..b65cb95 100644 --- a/.env.example +++ b/.env.example @@ -1,10 +1,66 @@ -# Copy to .env. The real .env is ignored by Git; never commit secrets. -JWT_SECRET_KEY=replace-with-a-long-random-secret -JWT_ALGORITHM=HS256 +# Copy to .env for local work only. The real .env is ignored by Git; never commit secrets. +APP_ENV=development +# Operational deployments default to PostgreSQL. The SQLite mode is local/test +# migration compatibility only; shadow_compare requires a separate snapshot. +# DATA_RUNTIME_MODE=postgresql +# SHADOW_SQLITE_PATH=/absolute/path/to/coherent-shadow-snapshot.db +AUTH_MODE=disabled ACCESS_TOKEN_EXPIRE_MINUTES=30 -BFF_ADMIN_USER=admin -BFF_ADMIN_PASSWORD=password -CORE_API_URL=http://localhost:8080 +SESSION_COOKIE_SECURE=false +CORS_ORIGINS=http://localhost:5173,http://127.0.0.1:5173 + +# Production/staging must use OIDC and inject these values through the runtime. +# AUTH_MODE=oidc +# OIDC_ISSUER=https://identity.example.invalid/ +# OIDC_AUDIENCE=foundation-intelligence-api +# OIDC_JWKS_URL=https://identity.example.invalid/.well-known/jwks.json +# OIDC_ROLE_CLAIM=roles + +# Explicit local-only authentication. Use unique values; no defaults are supplied. +# AUTH_MODE=development +# DEV_AUTH_ENABLED=true +# DEV_AUTH_USERNAME= +# DEV_AUTH_PASSWORD= +# DEV_AUTH_SECRET= +# DEV_AUTH_ALLOWED_HOSTS=127.0.0.1,::1,localhost + +# Public, non-production demo only. Both values are required together. This +# mode has no login/session and exposes only the reviewed aggregate route +# allowlist in bff.security; all other API routes remain protected. +# APP_ENV=demo +# DATA_RUNTIME_MODE=postgresql +# AUTH_MODE=public_readonly +# DEV_AUTH_ENABLED=false +# CORE_PROXY_ENABLED=false +# CORS_ORIGINS= + +RATE_LIMIT_REQUESTS=120 +RATE_LIMIT_WINDOW_SECONDS=60 +MAX_REQUEST_BODY_BYTES=1048576 +REQUEST_TIMEOUT_SECONDS=30 + +# PostgreSQL readiness/runtime configuration. Prefer a mounted secret file. +# DATABASE_URL=postgresql+asyncpg://user:password@localhost:5432/foundation_intelligence +DATABASE_HOST=127.0.0.1 +DATABASE_PORT=5432 +DATABASE_NAME=foundation_intelligence +DATABASE_USER=foundation_app +# DATABASE_PASSWORD_FILE=/absolute/path/to/postgres_password +# AWS/RDS tasks use encrypted PostgreSQL transport. +DATABASE_SSL_MODE=disable +DATABASE_POOL_SIZE=5 +DATABASE_MAX_OVERFLOW=5 +DATABASE_POOL_TIMEOUT_SECONDS=5 +DATABASE_CONNECT_TIMEOUT_SECONDS=5 +DATABASE_STATEMENT_TIMEOUT_MS=30000 + +CORE_PROXY_ENABLED=false +CORE_API_URL=http://127.0.0.1:8080 +# CORE_API_ALLOWED_HOSTS=127.0.0.1 +# CORE_PROXY_ALLOWED_PATHS=v1/data +# CORE_PROXY_ALLOWED_METHODS=GET +# CORE_PROXY_FORWARD_HEADERS=accept,content-type,x-request-id +# CORE_API_BEARER_TOKEN= DB_PATH=src/data/charities.db DATA_PATH=src/data/raw/register_of_charities_results.json SCORE_CONFIG_PATH=config/scoring.example.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e458e5..5946b53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,65 +1,284 @@ -name: Philea CI Pipeline +name: Pull Request Quality Gate on: - push: - branches: [ main, dev ] pull_request: - branches: [ main, dev ] - -jobs: - backend: - runs-on: ubuntu-latest + branches: [main, dev] + push: + branches: [main, dev, 91-clean-up-code-for-aws-integration] - steps: - - name: Checkout Code - uses: actions/checkout@v4 +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.12' - cache: 'pip' +permissions: + contents: read + actions: read + security-events: write - - name: Install Dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt +env: + PYTHON_VERSION: "3.12" + NODE_VERSION: "22" + PYTHONPATH: src - - name: Run Linter (Flake8) - run: | - # Stop the build if there are Python syntax errors or undefined names - flake8 src/ --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 src/ --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics +jobs: + backend-quality: + name: Backend compile, lint, type, test and coverage + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: requirements.txt + - name: Install hash-locked Python dependencies + run: | + python -m pip install --require-hashes -r requirements-locking.txt + python -m pip install --require-hashes -r requirements.txt + python -m pip check + - name: Compile + run: python -m compileall -q src scripts + - name: Blocking Flake8 + run: flake8 src scripts --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Typed production contracts + run: mypy --config-file mypy.ini + - name: Backend tests and coverage + env: + GEMINI_API_KEY: "disabled-ci-placeholder" + run: python -m pytest -q --cov=src --cov-report=xml --cov-report=term --cov-fail-under=70 + - name: Generate and verify deterministic SBOMs + run: | + python scripts/generate_sbom.py + git diff --exit-code -- docs/remediation/evidence/sbom + - name: Licence policy + run: python scripts/check_licenses.py --node-modules frontend/node_modules + - uses: actions/upload-artifact@v4 + with: + name: backend-quality-evidence + path: | + coverage.xml + docs/remediation/evidence/sbom/backend.cdx.json + docs/remediation/evidence/sbom/frontend.cdx.json + if-no-files-found: error + retention-days: 14 - - name: Syntax Compile Check (Build) - run: | - python -m compileall src/ + frontend-quality: + name: Frontend lint, test, build, Playwright and accessibility + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Deterministic install + working-directory: frontend + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Frontend lint + working-directory: frontend + run: npm run lint + - name: Frontend unit tests + working-directory: frontend + run: npm test + - name: Frontend production build and bundle budget + working-directory: frontend + run: npm run build + - name: Playwright journeys and axe accessibility + working-directory: frontend + env: + PHASE7_CHROME_PATH: /usr/bin/google-chrome + run: npm run test:e2e + - uses: actions/upload-artifact@v4 + with: + name: frontend-build + path: frontend/dist + if-no-files-found: error + retention-days: 14 - - name: Run Unit Tests - env: - GEMINI_API_KEY: "dummy_key" - run: | - PYTHONPATH=src pytest src/tests --cov=bff --cov-fail-under=70 + dependency-security: + name: Dependency, secret and licence scanning + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Python dependency vulnerability scan + uses: pypa/gh-action-pip-audit@v1.1.0 + with: + inputs: requirements.txt + - uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: npm dependency vulnerability scan + working-directory: frontend + run: npm audit --audit-level=high --ignore-scripts + - name: Secret scan + uses: gitleaks/gitleaks-action@v2 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - frontend: + codeql: + name: CodeQL SAST (${{ matrix.language }}) runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + language: [python, javascript-typescript] + steps: + - uses: actions/checkout@v4 + - uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + - uses: github/codeql-action/autobuild@v3 + - uses: github/codeql-action/analyze@v3 + container-security: + name: Container build, scan and runtime contract + runs-on: ubuntu-latest + timeout-minutes: 35 steps: - - name: Checkout Code - uses: actions/checkout@v4 + - uses: actions/checkout@v4 + - uses: docker/setup-buildx-action@v3 + - name: Build digest-pinned data-free backend + run: docker build --target backend-runtime --tag foundation-intelligence-backend:ci . + - name: Non-root, health, size and no-data assertions + env: + MAXIMUM_IMAGE_BYTES: "524288000" + run: scripts/verify_container_image.sh foundation-intelligence-backend:ci + - name: Generate container SBOM + uses: anchore/sbom-action@v0 + with: + image: foundation-intelligence-backend:ci + format: cyclonedx-json + output-file: container.cdx.json + - name: Container vulnerability scan + uses: aquasecurity/trivy-action@0.30.0 + with: + image-ref: foundation-intelligence-backend:ci + format: sarif + output: trivy-container.sarif + severity: HIGH,CRITICAL + ignore-unfixed: true + exit-code: "1" + - uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: trivy-container.sarif + - uses: actions/upload-artifact@v4 + if: always() + with: + name: container-security-evidence + path: | + container.cdx.json + trivy-container.sarif + if-no-files-found: error + retention-days: 14 - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: '22' - cache: 'npm' - cache-dependency-path: frontend/package-lock.json + terraform-security: + name: Terraform formatting, validation and security + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Offline structure and security contract + run: PYTHONPATH=src python scripts/validate_terraform_static.py + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.8 + terraform_wrapper: false + - name: Terraform formatting + run: terraform fmt -check -recursive infra/terraform + - name: Require reviewed provider locks + run: | + test -f infra/terraform/environments/dev/.terraform.lock.hcl + test -f infra/terraform/environments/staging/.terraform.lock.hcl + - name: Terraform init and validate without backends + run: | + terraform -chdir=infra/terraform/environments/dev init -backend=false -lockfile=readonly -input=false + terraform -chdir=infra/terraform/environments/dev validate + terraform -chdir=infra/terraform/environments/staging init -backend=false -lockfile=readonly -input=false + terraform -chdir=infra/terraform/environments/staging validate + - name: Terraform security scan + uses: aquasecurity/trivy-action@0.30.0 + with: + scan-type: config + scan-ref: infra/terraform + severity: HIGH,CRITICAL + exit-code: "1" - - name: Install Frontend Dependencies - working-directory: frontend - run: npm ci + postgresql-migration: + name: Empty PostgreSQL, Alembic and reconciliation + runs-on: ubuntu-latest + timeout-minutes: 35 + services: + postgres: + image: postgres:16.14-alpine3.24@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 + env: + POSTGRES_DB: foundation_intelligence + POSTGRES_USER: foundation_app + POSTGRES_PASSWORD: local-ci-password + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U foundation_app -d foundation_intelligence" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + env: + RUN_POSTGRES_INTEGRATION: "1" + DATABASE_URL: postgresql+asyncpg://foundation_app:local-ci-password@127.0.0.1:5432/foundation_intelligence + DATABASE_STATEMENT_TIMEOUT_MS: "120000" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: pip + cache-dependency-path: requirements.txt + - name: Install hash-locked Python dependencies + run: | + python -m pip install --require-hashes -r requirements-locking.txt + python -m pip install --require-hashes -r requirements.txt + - name: Alembic empty-database migration + run: alembic upgrade head + - name: PostgreSQL schema, pipeline and governance integration + run: >- + python -m pytest -q + src/tests/test_sqlite_to_postgres_migration.py + src/tests/test_postgres_schema.py + src/tests/test_durable_pipeline.py + src/tests/test_governance_retention.py + - name: API golden contract tests + run: python -m pytest -q src/tests/test_api_golden.py + - name: Performance smoke and pool isolation + run: python -m pytest -q src/tests/test_ci_performance_smoke.py - - name: Build Frontend - working-directory: frontend - run: npm run build + required-gate: + name: Required PR gate + if: always() + needs: + - backend-quality + - frontend-quality + - dependency-security + - codeql + - container-security + - terraform-security + - postgresql-migration + runs-on: ubuntu-latest + steps: + - name: Require every gate + env: + RESULTS: ${{ join(needs.*.result, ',') }} + run: test "$RESULTS" = "success,success,success,success,success,success,success" diff --git a/.github/workflows/deploy-production.yml b/.github/workflows/deploy-production.yml new file mode 100644 index 0000000..71c65d5 --- /dev/null +++ b/.github/workflows/deploy-production.yml @@ -0,0 +1,16 @@ +name: Production Deployment Disabled + +on: + workflow_dispatch: + +permissions: + contents: read + id-token: write + +jobs: + production-disabled: + if: ${{ false }} + runs-on: ubuntu-latest + environment: production + steps: + - run: echo "Production deployment remains disabled until all approvals and gates pass." diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml new file mode 100644 index 0000000..e13faf0 --- /dev/null +++ b/.github/workflows/deploy-staging.yml @@ -0,0 +1,245 @@ +name: Protected Staging Deployment + +on: + workflow_dispatch: + inputs: + confirmation: + description: "Type I_APPROVE_STAGING to start the protected workflow" + required: true + type: string + +concurrency: + group: staging-deployment + cancel-in-progress: false + +permissions: + contents: read + id-token: write + actions: read + +env: + AWS_REGION: ${{ vars.AWS_STAGING_REGION }} + TF_ROOT: infra/terraform/environments/staging + PYTHONPATH: src + +jobs: + build: + name: Build immutable candidate artefacts + if: github.event.inputs.confirmation == 'I_APPROVE_STAGING' + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: frontend/package-lock.json + - name: Build backend image + run: | + docker build --target backend-runtime --tag foundation-intelligence-backend:${GITHUB_SHA} . + scripts/verify_container_image.sh foundation-intelligence-backend:${GITHUB_SHA} + docker save foundation-intelligence-backend:${GITHUB_SHA} -o backend-image.tar + - name: Build deterministic frontend + working-directory: frontend + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + run: | + npm ci --ignore-scripts --no-audit --no-fund + npm run lint + npm test + npm run build + - uses: actions/upload-artifact@v4 + with: + name: staging-candidate-${{ github.sha }} + path: | + backend-image.tar + frontend/dist + if-no-files-found: error + retention-days: 3 + + publish-and-plan: + name: Publish immutable images and produce reviewed plan + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: staging-publish + outputs: + api_image: ${{ steps.images.outputs.api_image }} + worker_image: ${{ steps.images.outputs.worker_image }} + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: staging-candidate-${{ github.sha }} + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_STAGING_PUBLISH_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: staging-publish-${{ github.run_id }} + - uses: aws-actions/amazon-ecr-login@v2 + - name: Push candidate and resolve immutable ECR digests + id: images + env: + AWS_ACCOUNT_ID: ${{ vars.AWS_STAGING_ACCOUNT_ID }} + run: | + docker load -i backend-image.tar + api_repository="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/foundation-intelligence-staging/api" + worker_repository="${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/foundation-intelligence-staging/worker" + docker tag foundation-intelligence-backend:${GITHUB_SHA} "${api_repository}:${GITHUB_SHA}" + docker tag foundation-intelligence-backend:${GITHUB_SHA} "${worker_repository}:${GITHUB_SHA}" + docker push "${api_repository}:${GITHUB_SHA}" + docker push "${worker_repository}:${GITHUB_SHA}" + api_digest=$(aws ecr describe-images --repository-name foundation-intelligence-staging/api --image-ids imageTag=${GITHUB_SHA} --query 'imageDetails[0].imageDigest' --output text) + worker_digest=$(aws ecr describe-images --repository-name foundation-intelligence-staging/worker --image-ids imageTag=${GITHUB_SHA} --query 'imageDetails[0].imageDigest' --output text) + test -n "$api_digest" + test -n "$worker_digest" + echo "api_image=${api_repository}@${api_digest}" >> "$GITHUB_OUTPUT" + echo "worker_image=${worker_repository}@${worker_digest}" >> "$GITHUB_OUTPUT" + echo "frontend_digest=${api_digest}" >> "$GITHUB_OUTPUT" + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.8 + terraform_wrapper: false + - name: Initialize approved encrypted state backend + run: >- + terraform -chdir=${TF_ROOT} init -reconfigure -lockfile=readonly -input=false + -backend-config=bucket=${{ vars.AWS_STAGING_STATE_BUCKET }} + -backend-config=key=foundation-intelligence/staging/terraform.tfstate + -backend-config=region=${AWS_REGION} + -backend-config=dynamodb_table=${{ vars.AWS_STAGING_STATE_LOCK_TABLE }} + - name: Create non-destructive review plan + env: + TF_VAR_aws_region: ${{ env.AWS_REGION }} + TF_VAR_api_image: ${{ steps.images.outputs.api_image }} + TF_VAR_worker_image: ${{ steps.images.outputs.worker_image }} + TF_VAR_frontend_origin_image_digest: ${{ steps.images.outputs.frontend_digest }} + TF_VAR_github_repository: ${{ github.repository }} + TF_VAR_github_oidc_provider_arn: ${{ vars.AWS_GITHUB_OIDC_PROVIDER_ARN }} + TF_VAR_domain_name: ${{ vars.STAGING_API_DOMAIN }} + TF_VAR_hosted_zone_id: ${{ vars.AWS_STAGING_HOSTED_ZONE_ID }} + TF_VAR_manage_dns: "true" + run: | + terraform -chdir=${TF_ROOT} fmt -check + terraform -chdir=${TF_ROOT} validate + terraform -chdir=${TF_ROOT} plan -input=false -out=reviewed.tfplan + terraform -chdir=${TF_ROOT} show -no-color reviewed.tfplan > staging-plan.txt + terraform -chdir=${TF_ROOT} show -json reviewed.tfplan > staging-plan.json + - uses: actions/upload-artifact@v4 + with: + name: reviewed-staging-plan-${{ github.run_id }} + path: | + infra/terraform/environments/staging/reviewed.tfplan + infra/terraform/environments/staging/staging-plan.txt + infra/terraform/environments/staging/staging-plan.json + if-no-files-found: error + retention-days: 3 + + deploy: + name: Apply reviewed plan and run deployment gates + needs: [build, publish-and-plan] + runs-on: ubuntu-latest + timeout-minutes: 45 + environment: staging + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 + with: + name: staging-candidate-${{ github.sha }} + - uses: actions/download-artifact@v4 + with: + name: reviewed-staging-plan-${{ github.run_id }} + path: infra/terraform/environments/staging + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + - uses: actions/setup-node@v4 + with: + node-version: "22" + cache: npm + cache-dependency-path: frontend/package-lock.json + - uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_STAGING_DEPLOYMENT_ROLE_ARN }} + aws-region: ${{ env.AWS_REGION }} + role-session-name: staging-deploy-${{ github.run_id }} + - uses: hashicorp/setup-terraform@v3 + with: + terraform_version: 1.9.8 + terraform_wrapper: false + - name: Initialize the same approved backend + run: >- + terraform -chdir=${TF_ROOT} init -reconfigure -lockfile=readonly -input=false + -backend-config=bucket=${{ vars.AWS_STAGING_STATE_BUCKET }} + -backend-config=key=foundation-intelligence/staging/terraform.tfstate + -backend-config=region=${AWS_REGION} + -backend-config=dynamodb_table=${{ vars.AWS_STAGING_STATE_LOCK_TABLE }} + - name: Capture rollback task definitions + run: | + aws ecs describe-services --cluster foundation-intelligence-staging --services api worker --query 'services[].{name:serviceName,taskDefinition:taskDefinition}' > previous-task-definitions.json + test "$(jq length previous-task-definitions.json)" -eq 2 + - name: Apply exactly the reviewed plan + run: terraform -chdir=${TF_ROOT} apply -input=false reviewed.tfplan + - name: Run Alembic as a private one-off task + run: | + release=$(terraform -chdir=${TF_ROOT} output -json release_gate) + network=$(jq -cn --argjson release "$release" '{awsvpcConfiguration:{subnets:$release.application_subnets,securityGroups:[$release.security_group_id],assignPublicIp:"DISABLED"}}') + overrides='{"containerOverrides":[{"name":"release-gate","command":["alembic","upgrade","head"]}]}' + task=$(aws ecs run-task --cluster "$(jq -r .cluster_name <<<"$release")" --task-definition "$(jq -r .task_definition_arn <<<"$release")" --launch-type FARGATE --network-configuration "$network" --overrides "$overrides" --query 'tasks[0].taskArn' --output text) + aws ecs wait tasks-stopped --cluster "$(jq -r .cluster_name <<<"$release")" --tasks "$task" + test "$(aws ecs describe-tasks --cluster "$(jq -r .cluster_name <<<"$release")" --tasks "$task" --query 'tasks[0].containers[0].exitCode' --output text)" = "0" + - name: Deploy API and workers after migration + run: | + release=$(terraform -chdir=${TF_ROOT} output -json release_gate) + aws ecs update-service --cluster "$(jq -r .cluster_name <<<"$release")" --service api --task-definition "$(jq -r .api_task_definition <<<"$release")" >/dev/null + aws ecs update-service --cluster "$(jq -r .cluster_name <<<"$release")" --service worker --task-definition "$(jq -r .worker_task_definition <<<"$release")" >/dev/null + aws ecs wait services-stable --cluster "$(jq -r .cluster_name <<<"$release")" --services api worker + - name: Deploy frontend and invalidate exact distribution + run: | + release=$(terraform -chdir=${TF_ROOT} output -json release_gate) + aws s3 sync frontend/dist "s3://$(jq -r .frontend_bucket <<<"$release")/" --delete --only-show-errors + aws cloudfront create-invalidation --distribution-id "$(jq -r .cloudfront_id <<<"$release")" --paths '/*' >/dev/null + - name: Smoke and authenticated load gate + env: + STAGING_TEST_BEARER_TOKEN: ${{ secrets.STAGING_TEST_BEARER_TOKEN }} + run: | + python -m pip install --require-hashes -r requirements-locking.txt + python -m pip install --require-hashes -r requirements.txt + curl --fail --silent --show-error "${{ vars.STAGING_API_URL }}/health/ready" + python scripts/http_load_smoke.py --base-url "${{ vars.STAGING_API_URL }}" --token "$STAGING_TEST_BEARER_TOKEN" --samples 10 --concurrency 5 + - name: PostgreSQL reconciliation release gate + run: | + release=$(terraform -chdir=${TF_ROOT} output -json release_gate) + network=$(jq -cn --argjson release "$release" '{awsvpcConfiguration:{subnets:$release.application_subnets,securityGroups:[$release.security_group_id],assignPublicIp:"DISABLED"}}') + task=$(aws ecs run-task --cluster "$(jq -r .cluster_name <<<"$release")" --task-definition "$(jq -r .task_definition_arn <<<"$release")" --launch-type FARGATE --network-configuration "$network" --query 'tasks[0].taskArn' --output text) + aws ecs wait tasks-stopped --cluster "$(jq -r .cluster_name <<<"$release")" --tasks "$task" + test "$(aws ecs describe-tasks --cluster "$(jq -r .cluster_name <<<"$release")" --tasks "$task" --query 'tasks[0].containers[0].exitCode' --output text)" = "0" + - name: Browser E2E and accessibility + working-directory: frontend + env: + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" + PHASE7_CHROME_PATH: /usr/bin/google-chrome + PHASE7_BASE_URL: ${{ vars.STAGING_FRONTEND_URL }} + run: | + npm ci --ignore-scripts --no-audit --no-fund + npm run test:e2e + - name: Security and rollback-readiness assertions + run: | + release=$(terraform -chdir=${TF_ROOT} output -json release_gate) + aws ecs describe-task-definition --task-definition "$(jq -r .api_task_definition <<<"$release")" > deployed-api-task.json + jq -e '.taskDefinition.containerDefinitions[0].image | contains("@sha256:")' deployed-api-task.json + jq -e '.taskDefinition.containerDefinitions[0].readonlyRootFilesystem == true' deployed-api-task.json + jq -e 'all(.[]; .taskDefinition | length > 0)' previous-task-definitions.json + - uses: actions/upload-artifact@v4 + if: always() + with: + name: staging-deployment-evidence-${{ github.run_id }} + path: | + previous-task-definitions.json + deployed-api-task.json + infra/terraform/environments/staging/staging-plan.txt + infra/terraform/environments/staging/staging-plan.json + if-no-files-found: warn + retention-days: 30 diff --git a/.gitignore b/.gitignore index af9ecff..74b694e 100644 --- a/.gitignore +++ b/.gitignore @@ -176,3 +176,11 @@ src/data/processed/ src/data/raw/charity_commission_bulk/ src/data/pipeline_status.json src/data/pipeline_run.log + +# Terraform local state and generated plans (provider locks stay tracked) +**/.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +terraform.tfvars +*.auto.tfvars diff --git a/Dockerfile b/Dockerfile index f7611d1..d918850 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,25 +1,111 @@ -FROM python:3.12-slim +# syntax=docker/dockerfile:1.8@sha256:e87caa74dcb7d46cd820352bfea12591f3dba3ddc4285e19c7dcd13359f7cefd + +ARG PYTHON_IMAGE=python:3.12.13-slim-bookworm@sha256:d50fb7611f86d04a3b0471b46d7557818d88983fc3136726336b2a4c657aa30b +ARG NODE_IMAGE=node:22.22.2-alpine@sha256:8ea2348b068a9544dae7317b4f3aafcdc032df1647bb7d768a05a5cad1a7683f +ARG NGINX_IMAGE=nginxinc/nginx-unprivileged:1.30.4-alpine3.24@sha256:44e36330f74d4f3a1d4e222acca9e23b401fb87811a7597024502bb759c4dd49 + +FROM ${PYTHON_IMAGE} AS backend-builder + +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 +WORKDIR /build +COPY requirements-locking.txt requirements-runtime.txt ./ +RUN python -m pip install \ + --index-url=https://pypi.org/simple \ + --only-binary=:all: \ + --require-hashes \ + -r requirements-locking.txt \ + && python -m pip wheel \ + --index-url=https://pypi.org/simple \ + --no-build-isolation \ + --no-deps \ + --require-hashes \ + --wheel-dir=/wheels \ + -r requirements-runtime.txt -WORKDIR /app -# Install system dependencies (build tools, etc. if required) -RUN apt-get update && apt-get install -y --no-install-recommends \ - build-essential \ - && rm -rf /var/lib/apt/lists/* +FROM ${PYTHON_IMAGE} AS backend-runtime -# Copy and install requirements -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +ARG TARGETARCH +ARG TARGETOS +LABEL org.opencontainers.image.title="Foundation Intelligence API" \ + org.opencontainers.image.description="Data-free FastAPI runtime" \ + org.opencontainers.image.source="https://github.com/netlight/FoundationIntelligencePlatform" \ + io.foundation-intelligence.target-platform="${TARGETOS}/${TARGETARCH}" -# Copy source code -COPY src/ /app/src/ +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app/src \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + PIP_NO_CACHE_DIR=1 \ + TMPDIR=/tmp/fip \ + DATA_PATH=/app/runtime-data/unavailable.json \ + DB_PATH=/app/runtime-data/unavailable.db \ + SCORE_CONFIG_PATH=/app/config/scoring.example.json -# Set Python search path -ENV PYTHONPATH=/app/src -ENV DATA_PATH=/app/src/data/raw/register_of_charities_results.json +RUN groupadd --system --gid 10001 app \ + && useradd --system --uid 10001 --gid 10001 --home-dir /nonexistent --shell /usr/sbin/nologin app \ + && install -d -o 10001 -g 10001 -m 0700 /tmp/fip -# Expose port +COPY --from=backend-builder /wheels /wheels +RUN python -m pip install --no-index --find-links=/wheels /wheels/* \ + && rm -rf /wheels + +WORKDIR /app +COPY --chown=10001:10001 src/bff /app/src/bff +COPY --chown=10001:10001 src/data/*.py /app/src/data/ +COPY --chown=10001:10001 src/governance /app/src/governance +COPY --chown=10001:10001 src/migration /app/src/migration +COPY --chown=10001:10001 src/observability /app/src/observability +COPY --chown=10001:10001 src/transition /app/src/transition +COPY --chown=10001:10001 src/pipelines /app/src/pipelines +COPY --chown=10001:10001 src/preprocessing /app/src/preprocessing +COPY --chown=10001:10001 src/scoring /app/src/scoring +COPY --chown=10001:10001 src/scrapers /app/src/scrapers +COPY --chown=10001:10001 config/scoring.example.json /app/config/scoring.example.json +COPY --chown=10001:10001 config/data-governance.json /app/config/data-governance.json +COPY --chown=10001:10001 config/observability.json /app/config/observability.json +COPY --chown=10001:10001 config/source-pipelines.json /app/config/source-pipelines.json +COPY --chown=10001:10001 config/runtime-transition.json /app/config/runtime-transition.json +COPY --chown=10001:10001 alembic.ini /app/alembic.ini +COPY --chown=10001:10001 alembic /app/alembic + +USER 10001:10001 EXPOSE 8000 +STOPSIGNAL SIGTERM +HEALTHCHECK --interval=10s --timeout=3s --start-period=20s --retries=5 \ + CMD ["python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=2).read()"] +ENTRYPOINT ["python", "-m", "uvicorn"] +CMD ["bff.main:app", "--host", "0.0.0.0", "--port", "8000", "--no-server-header", "--timeout-graceful-shutdown", "30"] + + +FROM ${NODE_IMAGE} AS frontend-build + +ENV NODE_ENV=production +WORKDIR /frontend +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm ci --include=dev --ignore-scripts --registry=https://registry.npmjs.org +COPY frontend/ ./ +RUN npm run build + + +FROM ${NGINX_IMAGE} AS frontend-runtime + +LABEL org.opencontainers.image.title="Foundation Intelligence Frontend" \ + org.opencontainers.image.description="Static Vite production build" +COPY docker/frontend-nginx.conf /etc/nginx/conf.d/default.conf +COPY --from=frontend-build /frontend/dist /usr/share/nginx/html +USER 101:101 +EXPOSE 8080 +STOPSIGNAL SIGQUIT +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=5 \ + CMD ["wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/"] + + +FROM frontend-runtime AS frontend-runtime-ecs -# Start server -CMD ["uvicorn", "bff.main:app", "--host", "0.0.0.0", "--port", "8000"] +LABEL org.opencontainers.image.description="Static Vite production build for a shared ECS task network namespace" +COPY docker/frontend-nginx.ecs.conf /etc/nginx/conf.d/default.conf +EXPOSE 8080 +HEALTHCHECK --interval=10s --timeout=3s --start-period=5s --retries=5 \ + CMD ["wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/"] diff --git a/README.md b/README.md index 41d47d6..71c92e4 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,11 @@ The platform deliberately separates source facts, normalized source values, dete | Capability | Status | Current evidence / limitation | |---|---|---| | Cached UK organization ingestion | Complete | 65 normalized UK-side organizations in the current rebuilt database | -| Cached 360Giving grant ingestion | Sampled | 200,000 observed 360Giving transactions with source provenance; the active database is a reproducible capped sample, not complete 360Giving coverage | +| Cached 360Giving grant ingestion | Sampled | 302,546 observed transactions with source provenance in the current migration source; coverage is not complete 360Giving | | Cached Philea organization ingestion | Complete | 299 records; organization-level only, with no grants assigned | | DACH foundation intelligence | Partial | Philea and deterministic geography normalization provide some European/DACH discoverability; this is not a complete DACH registry or grant dataset | -| Organization directory and detail | Complete | SQLite-backed API and UI, with source/type/coverage labels | +| Organization directory and detail | Complete | PostgreSQL-backed operational API/UI; SQLite is retained only as a local migration source | +| PostgreSQL migration/cutover | Complete locally | Exact reconciliation, 18 zero-difference shadow projections, full restore and dataset rollback pass; AWS/production cutover is unexecuted | | Programme-area enrichment | Complete | Versioned deterministic taxonomy/rules and evidence; accuracy has not been externally validated | | Geographic-focus enrichment | Complete | Versioned deterministic rules and evidence; distinct from headquarters and beneficiary geography | | Grant list and network summary | Complete | Observed 360Giving transactions only | @@ -37,15 +38,18 @@ scrapers ──► consolidation ──► deterministic enrichment ──► JS Philea normalization/dedup ───────────┘ │ ▼ - atomic SQLite rebuild + coherent SQLite migration source + │ + deterministic versioned migration/reconcile │ ▼ React/Vite UI ◄── cookie-authenticated JSON ── FastAPI BFF │ - ├── organization/grant repository + ├── async PostgreSQL repositories + ├── versioned analytics/materialisations + ├── durable job/outbox workers ├── experimental score engine/config - ├── pipeline monitor/trigger - └── optional news service + └── optional approval-gated news service ``` The major components are: @@ -54,7 +58,8 @@ The major components are: - `src/preprocessing/consolidate.py`: maps Charity Commission and 360Giving records into common organization and grant records. - `src/preprocessing/enrichment.py`: the one active, versioned source of programme and geography taxonomy/rules. It keeps source values and inferred values separate. - `src/preprocessing/philea_adapter.py`: normalizes all cached Philea records, assigns stable negative local IDs, maps organization types, records provenance, and performs conservative cross-source deduplication. -- `src/data/db_loader.py`: owns schema version 6, validates tables/version, loads JSONL strictly into a staging database, and publishes it atomically only after validation. +- `src/data/db_loader.py`: maintains the coherent SQLite migration source for local ingestion compatibility; it is not an operational production datastore. +- `alembic/`, `src/migration/`, and `src/bff/postgres/`: own the PostgreSQL schema, lossless versioned migration/reconciliation, rollback controls and operational repositories. - `src/pipelines/backfill_ecb_exchange_rates.py`: fetches/caches official ECB daily EXR rates and atomically backfills reproducible EUR values without altering source amounts. - `src/pipelines/run_pipeline.py`: orchestrates collection, consolidation, enrichment, reports, and database publication. - `src/bff/`: FastAPI entry point, demo cookie authentication, repositories, organization/grant endpoints, pipeline controls, proxy, and optional news summary. @@ -66,10 +71,11 @@ The major components are: The checked-in source caches currently contain 62 Charity Commission records, a baseline 360Giving cache, an observed 360Giving publisher-pilot import, and 299 Philea member records. Consolidation creates 65 UK-side organization rows alongside the Philea records. -The current regenerated database contains: +The current preserved SQLite migration source and reconciled PostgreSQL active dataset contain: -- 364 organizations: 65 primarily from the Charity Commission/360Giving and 299 from Philea. -- 200,000 observed 360Giving grants in GBP, USD, EUR, ILS, and CHF. Original amount and currency remain source facts; this is the current reproducible ingestion cap, not the complete 360Giving dataset. +- 373 organizations across Charity Commission/360Giving and Philea identities. +- 302,546 observed grants in GBP, USD, EUR, ILS, and CHF. Original amount and currency remain source facts; this is a reproducible sampled source, not complete 360Giving coverage. +- 397,469 current registry rows and 345 accepted registry/profile links. - Auto EUR display uses the official daily ECB EXR reference rate for the award date; weekends and ECB holidays use the previous published business-day rate. Each converted grant stores the conversion status, rate, rate date, and source. 34 pre-1999 GBP grants remain explicitly unconverted because no ECB EUR reference rate exists for their award dates. - 299 Philea organizations marked `organization_level_only`; no grant is attached to a synthetic Philea ID. @@ -124,7 +130,7 @@ Geographic concepts are intentionally distinct: - Geographic focus: where an organization says it works or funds; source and inferred values remain separate. - Beneficiary/project geography: a transaction's destination; used by recipient-region filters and the map. -The `Global Grant Distribution` map currently resolves beneficiary-country geography for 71,286 of the 200,000 ingested grants (35.64%). These produce 71,403 country associations; 62 grants have multiple associated countries. It uses `beneficiary_geography_normalized` first and falls back only to explicit ISO country codes or explicit country names in the original `beneficiary_geography` source field. It never consults funder headquarters, recipient registered offices, or inferred operating regions. England, Scotland, Wales, and Northern Ireland roll up to the United Kingdom shape while their original labels are retained in country detail. The remaining 128,714 records are reported as unmapped rather than assigned a fabricated country. +The `Global Grant Distribution` map currently resolves beneficiary-country geography for 104,191 of the 302,546 ingested grants (34.44%). These produce 104,309 country associations because explicitly multi-country grants retain every supported association. It uses `beneficiary_geography_normalized` first and falls back only to explicit ISO country codes or explicit country names in the original `beneficiary_geography` source field. It never consults funder headquarters, recipient registered offices, or inferred operating regions. England, Scotland, Wales, and Northern Ireland roll up to the United Kingdom shape while their original labels are retained in country detail. The remaining 198,355 records are reported as unmapped rather than assigned a fabricated country. The Overview `Filters` button opens a non-layout-shifting, scrollable global grant-filter drawer for award date, currency, beneficiary geography, programme area, donor, recipient, and time aggregation. `Auto · EUR converted` includes every eligible source currency using stored historical ECB rates; selecting `GBP`, `USD`, `EUR`, `CHF`, or `ILS` instead shows only grants originally recorded in that currency. One applied grant scope drives the Overview KPIs, map, trends, and programme allocation; coverage counters explicitly change from ingested to filtered grants. The map header retains only display controls and the illustrative-connections toggle. Organization-directory search, income/expenditure, and headquarters filters remain independent because they describe organization records rather than the filtered grant population. @@ -138,7 +144,7 @@ The endpoint requires `beneficiary_country` as a canonical ISO alpha-2 code and For a selected country, multi-country grants count once for activity and recency, but their full amount is excluded from the country-attributable funding total. In `currency=auto`, only stored EUR values with `native_eur`, `ecb_award_date`, or `ecb_previous_business_day` conversion status are monetary eligible. In an explicit currency mode, only original grants in that selected currency are considered. A verified Directory profile is linked only where it already exists; source-only funders remain source-only and must never be converted into invented profiles. -The current active 200,000-grant database is a reproducible **sampled ingestion cap**, not a complete 360Giving dataset. The import record is `src/data/processed/360giving_registry_publisher_pilot_import_200000.json`. The optimized source-funder list uses normalized beneficiary/programme indexes and a versioned narrow fact table; supported loaders invalidate it after data changes. See `docs/donor_directory_performance_report.md` for measured GB/US repository timings and the explicit HTTP/browser validation limits. +The current active 302,546-grant dataset is reproducible and checksum-bound, but remains sampled rather than complete 360Giving coverage. Its exact source/target counts and controls are recorded in `docs/remediation/evidence/phase4-migration-manifest.json`. The optimized source-funder list uses normalized beneficiary/programme indexes and a versioned narrow fact table; supported loaders invalidate it after data changes. See the Phase-6 performance evidence for current PostgreSQL timings; older donor-directory reports remain historical evidence. The map CTA writes `funder_country` plus canonical `grant_*` filters to the URL. Search, status, sorting, page, sources, and selected donor also survive refresh/back/forward and copied URLs. Desktop uses a compact list plus right-side detail panel; tablet/mobile use a full-width/full-screen sheet. Real-browser acceptance is still outstanding and the legacy donor view remains available; see `docs/donor_directory_responsive_validation.md`. @@ -168,24 +174,24 @@ To use a reviewed configuration, copy the example, preserve the validated schema ## Local setup -Prerequisites: Python 3.12, Node.js 22, and npm. +Prerequisites: Python 3.12, Node.js 22/npm, Docker Desktop and Compose. ```bash -git checkout 86-implement-ui-for-final-presentation +git checkout 91-clean-up-code-for-aws-integration python3.12 -m venv venv -./venv/bin/python -m pip install --upgrade pip -./venv/bin/pip install -r requirements.txt +./venv/bin/python -m pip install --require-hashes -r requirements-locking.txt +./venv/bin/python -m pip install --require-hashes -r requirements.txt cp .env.example .env cd frontend -npm ci +npm ci --ignore-scripts --no-audit --no-fund cp .env.example .env cd .. ``` The root `.env` is ignored by Git and loaded by the BFF configuration. `frontend/.env` is also ignored, but every `VITE_*` value is embedded in browser code and must not contain a real secret. The demo username/password are visible client configuration and are not production authentication. -Relevant backend variables are `JWT_SECRET_KEY`, `JWT_ALGORITHM`, `ACCESS_TOKEN_EXPIRE_MINUTES`, `BFF_ADMIN_USER`, `BFF_ADMIN_PASSWORD`, `CORE_API_URL`, `DB_PATH`, `DATA_PATH`, `SCORE_CONFIG_PATH`, and the optional `ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_BASE_URL`, and `CLAUDE_MODEL`. Frontend variables are `VITE_API_BASE_URL`, `VITE_BFF_USERNAME`, and `VITE_BFF_PASSWORD`; the two demo credentials must match the BFF values. +Operationally relevant variables include `DATA_RUNTIME_MODE` (PostgreSQL by default), `DATABASE_*`, `AUTH_MODE`, `OIDC_*`, `CORS_ORIGINS`, request/rate limits, proxy allowlists and `SCORE_CONFIG_PATH`. See `docs/remediation/environment-variable-reference.md`. Optional news credentials remain separately approval-gated. Every frontend `VITE_*` value is public build-time configuration and must not contain a secret. ### Build the presentation database diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..44db501 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,43 @@ +[alembic] +script_location = %(here)s/alembic +prepend_sys_path = %(here)s/src +path_separator = os +timezone = UTC + +# The runtime URL is assembled from DATABASE_URL or DATABASE_* plus the mounted +# password file by alembic/env.py. No credential belongs in this file. +sqlalchemy.url = postgresql+asyncpg:// + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..19efb3d --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,74 @@ +"""Alembic environment using the same asyncpg configuration as the API.""" + +from __future__ import annotations + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import pool +from sqlalchemy.ext.asyncio import async_engine_from_config + +from bff.database import DatabaseSettings + + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = None + + +def _configured_url() -> str: + # Both consumers below receive this value directly. No ConfigParser boundary + # exists here, so percent-encoded credentials must remain untouched. + return DatabaseSettings.from_env().raw_sqlalchemy_url() + + +def run_migrations_offline() -> None: + context.configure( + url=_configured_url(), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + transactional_ddl=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def _run_sync_migrations(connection) -> None: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + transactional_ddl=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +async def _run_async_migrations() -> None: + section = config.get_section(config.config_ini_section) or {} + section["sqlalchemy.url"] = _configured_url() + connectable = async_engine_from_config( + section, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + try: + async with connectable.connect() as connection: + await connection.run_sync(_run_sync_migrations) + finally: + await connectable.dispose() + + +def run_migrations_online() -> None: + asyncio.run(_run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..520e8d9 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from typing import Optional, Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Optional[str] = ${repr(down_revision)} +branch_labels: Optional[Union[str, Sequence[str]]] = ${repr(branch_labels)} +depends_on: Optional[Union[str, Sequence[str]]] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/0001_postgresql_foundation.py b/alembic/versions/0001_postgresql_foundation.py new file mode 100644 index 0000000..69f9860 --- /dev/null +++ b/alembic/versions/0001_postgresql_foundation.py @@ -0,0 +1,1186 @@ +"""Create the versioned PostgreSQL serving and operational schema. + +Revision ID: 0001_postgresql_foundation +Revises: None +Create Date: 2026-07-29 +""" + +from __future__ import annotations + +from typing import Optional, Sequence, Union + +from alembic import op + + +revision: str = "0001_postgresql_foundation" +down_revision: Optional[str] = None +branch_labels: Optional[Union[str, Sequence[str]]] = None +depends_on: Optional[Union[str, Sequence[str]]] = None + + +def _execute_all(statements: Sequence[str]) -> None: + for statement in statements: + op.execute(statement) + + +def upgrade() -> None: + _execute_all( + [ + "CREATE EXTENSION IF NOT EXISTS pg_trgm", + """ + CREATE TABLE dataset_versions ( + dataset_version TEXT PRIMARY KEY, + revision BIGINT GENERATED ALWAYS AS IDENTITY, + status TEXT NOT NULL DEFAULT 'created', + is_active BOOLEAN NOT NULL DEFAULT FALSE, + source_checksum CHAR(64), + source_schema_version TEXT, + code_revision CHAR(40), + previous_dataset_version TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + approved_at TIMESTAMPTZ, + activated_at TIMESTAMPTZ, + rejected_at TIMESTAMPTZ, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT uq_dataset_versions_revision UNIQUE (revision), + CONSTRAINT fk_dataset_versions_previous + FOREIGN KEY (previous_dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_dataset_versions_status CHECK ( + status IN ('created', 'loading', 'validating', 'rejected', + 'approved', 'active', 'rolled_back', 'failed') + ), + CONSTRAINT ck_dataset_versions_activation CHECK ( + (is_active AND status = 'active' AND activated_at IS NOT NULL) + OR (NOT is_active AND status <> 'active') + ), + CONSTRAINT ck_dataset_versions_checksum CHECK ( + source_checksum IS NULL OR source_checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_dataset_versions_code_revision CHECK ( + code_revision IS NULL OR code_revision ~ '^[a-f0-9]{40}$' + ) + ) + """, + """ + CREATE UNIQUE INDEX uq_dataset_versions_single_active + ON dataset_versions ((is_active)) WHERE is_active + """, + """ + CREATE TABLE migration_runs ( + migration_run_id UUID PRIMARY KEY, + target_dataset_version TEXT NOT NULL, + source_database_checksum CHAR(64) NOT NULL, + source_schema_version TEXT NOT NULL, + source_fact_version TEXT NOT NULL, + target_schema_version TEXT NOT NULL, + code_revision CHAR(40) NOT NULL, + status TEXT NOT NULL DEFAULT 'created', + started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + source_counts JSONB NOT NULL DEFAULT '{}'::jsonb, + target_counts JSONB NOT NULL DEFAULT '{}'::jsonb, + reconciliation_results JSONB NOT NULL DEFAULT '{}'::jsonb, + errors JSONB NOT NULL DEFAULT '[]'::jsonb, + actor_id TEXT NOT NULL, + actor_type TEXT NOT NULL, + CONSTRAINT uq_migration_runs_target UNIQUE (target_dataset_version), + CONSTRAINT fk_migration_runs_dataset FOREIGN KEY (target_dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_migration_runs_checksum CHECK ( + source_database_checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_migration_runs_revision CHECK ( + code_revision ~ '^[a-f0-9]{40}$' + ), + CONSTRAINT ck_migration_runs_status CHECK ( + status IN ('created', 'loading', 'validating', 'rejected', + 'approved', 'active', 'rolled_back', 'failed') + ), + CONSTRAINT ck_migration_runs_actor_type CHECK ( + actor_type IN ('human', 'service', 'ci') + ), + CONSTRAINT ck_migration_runs_completion CHECK ( + completed_at IS NULL OR completed_at >= started_at + ) + ) + """, + """ + CREATE TABLE charities ( + dataset_version TEXT NOT NULL, + charity_id BIGINT NOT NULL, + name TEXT NOT NULL, + type TEXT, + website TEXT, + email TEXT, + address TEXT, + city TEXT, + state TEXT, + country TEXT, + latitude NUMERIC(9, 6), + longitude NUMERIC(9, 6), + annual_income NUMERIC(24, 4), + annual_expenditure NUMERIC(24, 4), + thematic_focus TEXT, + geographic_focus TEXT, + raw_source_data JSONB, + programme_areas_source JSONB, + programme_areas_inferred JSONB, + programme_area_scores JSONB, + programme_area_method TEXT, + programme_area_evidence JSONB, + programme_area_review_required BOOLEAN NOT NULL DEFAULT FALSE, + geographic_focus_source JSONB, + geographic_focus_inferred JSONB, + headquarters_country TEXT, + headquarters_region TEXT, + geography_method TEXT, + geography_confidence NUMERIC(6, 5), + geography_evidence JSONB, + geography_review_required BOOLEAN NOT NULL DEFAULT FALSE, + enrichment_rule_version TEXT, + enrichment_review_reasons JSONB, + insufficient_source_text BOOLEAN NOT NULL DEFAULT FALSE, + normalized_name TEXT, + normalized_domain TEXT, + organization_type TEXT, + primary_source TEXT, + source_names JSONB, + source_record_id TEXT, + source_url TEXT, + source_records JSONB, + ingestion_timestamp TIMESTAMPTZ, + transaction_coverage TEXT, + deduplication_status TEXT, + deduplication_candidates JSONB, + PRIMARY KEY (dataset_version, charity_id), + CONSTRAINT uq_charities_source_identity + UNIQUE (dataset_version, primary_source, source_record_id), + CONSTRAINT fk_charities_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_charities_name CHECK (btrim(name) <> ''), + CONSTRAINT ck_charities_latitude CHECK ( + latitude IS NULL OR latitude BETWEEN -90 AND 90 + ), + CONSTRAINT ck_charities_longitude CHECK ( + longitude IS NULL OR longitude BETWEEN -180 AND 180 + ), + CONSTRAINT ck_charities_geography_confidence CHECK ( + geography_confidence IS NULL + OR geography_confidence BETWEEN 0 AND 1 + ), + CONSTRAINT ck_charities_programme_method CHECK ( + programme_area_method IS NULL OR programme_area_method IN ( + 'deterministic_regex', 'source_normalization', + 'source_normalization+deterministic_regex', 'unavailable' + ) + ), + CONSTRAINT ck_charities_geography_method CHECK ( + geography_method IS NULL OR geography_method IN ( + 'deterministic_regex', 'source_normalization', + 'source_normalization+deterministic_regex', 'unavailable' + ) + ) + ) + """, + """ + CREATE INDEX ix_charities_normalized_name + ON charities (dataset_version, normalized_name, charity_id) + """, + """ + CREATE TABLE charity_programme_categories ( + dataset_version TEXT NOT NULL, + charity_id BIGINT NOT NULL, + programme_area TEXT NOT NULL, + provenance TEXT NOT NULL, + confidence NUMERIC(6, 5), + PRIMARY KEY (dataset_version, charity_id, programme_area, provenance), + CONSTRAINT fk_charity_programmes_charity + FOREIGN KEY (dataset_version, charity_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_charity_programmes_area CHECK (btrim(programme_area) <> ''), + CONSTRAINT ck_charity_programmes_provenance CHECK ( + provenance IN ('source', 'inferred', 'manual', 'unclassified') + ), + CONSTRAINT ck_charity_programmes_confidence CHECK ( + confidence IS NULL OR confidence BETWEEN 0 AND 1 + ) + ) + """, + """ + CREATE TABLE charity_geographic_areas ( + dataset_version TEXT NOT NULL, + charity_id BIGINT NOT NULL, + country_code CHAR(2) NOT NULL, + administrative_region TEXT NOT NULL DEFAULT '', + provenance TEXT NOT NULL, + confidence NUMERIC(6, 5), + PRIMARY KEY ( + dataset_version, charity_id, country_code, + administrative_region, provenance + ), + CONSTRAINT fk_charity_geographies_charity + FOREIGN KEY (dataset_version, charity_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_charity_geographies_country CHECK ( + country_code ~ '^[A-Z]{2}$' + ), + CONSTRAINT ck_charity_geographies_provenance CHECK ( + provenance IN ('source', 'inferred', 'manual', 'unclassified') + ), + CONSTRAINT ck_charity_geographies_confidence CHECK ( + confidence IS NULL OR confidence BETWEEN 0 AND 1 + ) + ) + """, + """ + CREATE TABLE charity_registry_organizations ( + dataset_version TEXT NOT NULL, + registry_id TEXT NOT NULL, + charity_number TEXT NOT NULL, + linked_charity_number TEXT, + registered_name TEXT NOT NULL, + normalized_name TEXT NOT NULL, + registration_status TEXT, + registration_date DATE, + removal_date DATE, + income NUMERIC(24, 4), + expenditure NUMERIC(24, 4), + financial_period_end_date DATE, + address_line_one TEXT, + address_line_two TEXT, + address_line_three TEXT, + address_line_four TEXT, + address_line_five TEXT, + postcode TEXT, + city TEXT, + administrative_region TEXT, + country_code CHAR(2), + registered_latitude NUMERIC(9, 6), + registered_longitude NUMERIC(9, 6), + activity_text TEXT, + source_name TEXT NOT NULL, + source_record_updated_at TIMESTAMPTZ, + imported_at TIMESTAMPTZ NOT NULL, + is_current_source_record BOOLEAN NOT NULL DEFAULT TRUE, + search_vector TSVECTOR GENERATED ALWAYS AS ( + to_tsvector( + 'simple'::regconfig, + coalesce(registered_name, '') || ' ' || + coalesce(normalized_name, '') || ' ' || + coalesce(charity_number, '') || ' ' || + coalesce(postcode, '') || ' ' || + coalesce(activity_text, '') + ) + ) STORED, + PRIMARY KEY (dataset_version, registry_id), + CONSTRAINT fk_registry_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_registry_id CHECK (btrim(registry_id) <> ''), + CONSTRAINT ck_registry_charity_number CHECK (btrim(charity_number) <> ''), + CONSTRAINT ck_registry_registered_name CHECK (btrim(registered_name) <> ''), + CONSTRAINT ck_registry_normalized_name CHECK (btrim(normalized_name) <> ''), + CONSTRAINT ck_registry_country CHECK ( + country_code IS NULL OR country_code ~ '^[A-Z]{2}$' + ), + CONSTRAINT ck_registry_latitude CHECK ( + registered_latitude IS NULL OR registered_latitude BETWEEN -90 AND 90 + ), + CONSTRAINT ck_registry_longitude CHECK ( + registered_longitude IS NULL OR registered_longitude BETWEEN -180 AND 180 + ), + CONSTRAINT ck_registry_dates CHECK ( + removal_date IS NULL OR registration_date IS NULL + OR removal_date >= registration_date + ) + ) + """, + """ + CREATE INDEX ix_registry_charity_number + ON charity_registry_organizations + (dataset_version, charity_number, registry_id) + """, + """ + CREATE INDEX ix_registry_linked_charity_number + ON charity_registry_organizations + (dataset_version, linked_charity_number, registry_id) + """, + """ + CREATE INDEX ix_registry_status_income + ON charity_registry_organizations + (dataset_version, registration_status, income DESC, registry_id) + """, + """ + CREATE INDEX ix_registry_status_expenditure + ON charity_registry_organizations + (dataset_version, registration_status, expenditure DESC, registry_id) + """, + """ + CREATE INDEX ix_registry_country_region + ON charity_registry_organizations + (dataset_version, country_code, administrative_region, registry_id) + """, + """ + CREATE INDEX ix_registry_search_vector + ON charity_registry_organizations USING GIN (search_vector) + """, + """ + CREATE INDEX ix_registry_registered_name_trgm + ON charity_registry_organizations USING GIN (registered_name gin_trgm_ops) + """, + """ + CREATE INDEX ix_registry_normalized_name_trgm + ON charity_registry_organizations USING GIN (normalized_name gin_trgm_ops) + """, + """ + CREATE TABLE grants ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + funding_charity_id BIGINT, + funding_name TEXT, + funding_org_source_id TEXT, + recipient_name TEXT NOT NULL, + recipient_charity_id BIGINT, + recipient_org_source_id TEXT, + amount NUMERIC(24, 4), + amount_eur NUMERIC(24, 4), + currency CHAR(3), + description TEXT, + award_date DATE, + recipient_latitude NUMERIC(9, 6), + recipient_longitude NUMERIC(9, 6), + recipient_region TEXT, + beneficiary_geography TEXT, + project_geography TEXT, + programme_area_source TEXT, + tags JSONB, + geographic_focus TEXT, + source TEXT, + source_record_id TEXT, + source_url TEXT, + ingestion_timestamp TIMESTAMPTZ, + raw_grant_data JSONB, + programme_area_inferred TEXT, + programme_area_scores JSONB, + programme_area_method TEXT, + programme_area_evidence JSONB, + programme_area_review_required BOOLEAN NOT NULL DEFAULT FALSE, + beneficiary_geography_normalized TEXT, + geographic_focus_inferred TEXT, + geography_method TEXT, + geography_confidence NUMERIC(6, 5), + geography_evidence JSONB, + geography_review_required BOOLEAN NOT NULL DEFAULT FALSE, + enrichment_rule_version TEXT, + enrichment_review_reasons JSONB, + insufficient_source_text BOOLEAN NOT NULL DEFAULT FALSE, + exchange_rate NUMERIC(24, 12), + exchange_rate_date DATE, + exchange_rate_source TEXT, + conversion_status TEXT, + PRIMARY KEY (dataset_version, grant_id), + CONSTRAINT uq_grants_source_identity + UNIQUE (dataset_version, source, source_record_id), + CONSTRAINT fk_grants_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_grants_funder FOREIGN KEY (dataset_version, funding_charity_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_grants_recipient FOREIGN KEY (dataset_version, recipient_charity_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_grants_id CHECK (btrim(grant_id) <> ''), + CONSTRAINT ck_grants_recipient CHECK (btrim(recipient_name) <> ''), + CONSTRAINT ck_grants_currency CHECK ( + currency IS NULL OR currency ~ '^[A-Z]{3}$' + ), + CONSTRAINT ck_grants_latitude CHECK ( + recipient_latitude IS NULL OR recipient_latitude BETWEEN -90 AND 90 + ), + CONSTRAINT ck_grants_longitude CHECK ( + recipient_longitude IS NULL OR recipient_longitude BETWEEN -180 AND 180 + ), + CONSTRAINT ck_grants_geography_confidence CHECK ( + geography_confidence IS NULL OR geography_confidence BETWEEN 0 AND 1 + ), + CONSTRAINT ck_grants_exchange_rate CHECK ( + exchange_rate IS NULL OR exchange_rate > 0 + ), + CONSTRAINT ck_grants_programme_method CHECK ( + programme_area_method IS NULL OR programme_area_method IN ( + 'deterministic_regex', 'source_normalization', + 'source_normalization+deterministic_regex', 'unavailable' + ) + ), + CONSTRAINT ck_grants_geography_method CHECK ( + geography_method IS NULL OR geography_method IN ( + 'deterministic_regex', 'source_normalization', + 'source_normalization+deterministic_regex', 'unavailable' + ) + ), + CONSTRAINT ck_grants_conversion_status CHECK ( + conversion_status IS NULL OR conversion_status IN ( + 'native_eur', 'ecb_monthly_average', + 'unavailable_missing_rate', 'not_applicable', 'invalid_currency' + ) + ) + ) + """, + """ + CREATE INDEX ix_grants_funder + ON grants (dataset_version, funding_charity_id, grant_id) + """, + """ + CREATE INDEX ix_grants_recipient + ON grants (dataset_version, recipient_charity_id, grant_id) + """, + """ + CREATE INDEX ix_grants_source_date + ON grants (dataset_version, source, award_date, grant_id) + """, + """ + CREATE INDEX ix_grants_currency_date + ON grants (dataset_version, currency, award_date, grant_id) + """, + """ + CREATE INDEX ix_grants_conversion_status + ON grants (dataset_version, conversion_status, grant_id) + """, + """ + CREATE TABLE grant_beneficiary_countries ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + country_code CHAR(2) NOT NULL, + country_name TEXT NOT NULL, + PRIMARY KEY (dataset_version, grant_id, country_code), + CONSTRAINT fk_grant_countries_grant + FOREIGN KEY (dataset_version, grant_id) + REFERENCES grants(dataset_version, grant_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_grant_countries_code CHECK ( + country_code ~ '^[A-Z]{2}$' + ), + CONSTRAINT ck_grant_countries_name CHECK (btrim(country_name) <> '') + ) + """, + """ + CREATE INDEX ix_grant_countries_lookup + ON grant_beneficiary_countries + (dataset_version, country_code, grant_id) + """, + """ + CREATE TABLE grant_beneficiary_terms ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + term TEXT NOT NULL, + PRIMARY KEY (dataset_version, grant_id, term), + CONSTRAINT fk_grant_terms_grant + FOREIGN KEY (dataset_version, grant_id) + REFERENCES grants(dataset_version, grant_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_grant_terms_term CHECK (btrim(term) <> '') + ) + """, + """ + CREATE INDEX ix_grant_terms_lookup + ON grant_beneficiary_terms (dataset_version, term, grant_id) + """, + """ + CREATE TABLE grant_programme_categories ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + programme_area TEXT NOT NULL, + PRIMARY KEY (dataset_version, grant_id, programme_area), + CONSTRAINT fk_grant_programmes_grant + FOREIGN KEY (dataset_version, grant_id) + REFERENCES grants(dataset_version, grant_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_grant_programmes_area CHECK (btrim(programme_area) <> '') + ) + """, + """ + CREATE INDEX ix_grant_programmes_lookup + ON grant_programme_categories + (dataset_version, programme_area, grant_id) + """, + """ + CREATE TABLE grant_overview_facts ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + source_namespace TEXT NOT NULL, + award_date DATE, + award_date_status TEXT NOT NULL, + currency CHAR(3), + original_amount_minor BIGINT, + original_amount_status TEXT NOT NULL, + eur_amount_minor BIGINT, + eur_amount_status TEXT NOT NULL, + conversion_status TEXT, + funding_name TEXT NOT NULL, + funding_name_normalized TEXT NOT NULL, + recipient_name TEXT NOT NULL, + recipient_name_normalized TEXT NOT NULL, + country_count INTEGER NOT NULL, + programme_category_count INTEGER NOT NULL, + programme_provenance TEXT NOT NULL, + invalid_source_label BOOLEAN NOT NULL DEFAULT FALSE, + low_confidence_inference BOOLEAN NOT NULL DEFAULT FALSE, + origin_country_code CHAR(2), + origin_country_name TEXT, + origin_source TEXT, + data_revision TEXT NOT NULL, + PRIMARY KEY (dataset_version, grant_id), + CONSTRAINT fk_overview_facts_grant + FOREIGN KEY (dataset_version, grant_id) + REFERENCES grants(dataset_version, grant_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_overview_source CHECK (btrim(source_namespace) <> ''), + CONSTRAINT ck_overview_currency CHECK ( + currency IS NULL OR currency ~ '^[A-Z]{3}$' + ), + CONSTRAINT ck_overview_country CHECK ( + origin_country_code IS NULL OR origin_country_code ~ '^[A-Z]{2}$' + ), + CONSTRAINT ck_overview_counts CHECK ( + country_count >= 0 AND programme_category_count >= 0 + ), + CONSTRAINT ck_overview_award_status CHECK ( + award_date_status IN ('valid', 'missing', 'invalid') + ), + CONSTRAINT ck_overview_original_status CHECK ( + original_amount_status IN ('valid', 'missing', 'negative', 'zero', 'invalid') + ), + CONSTRAINT ck_overview_eur_status CHECK ( + eur_amount_status IN ('valid', 'missing', 'negative', 'zero', 'invalid') + ), + CONSTRAINT ck_overview_provenance CHECK ( + programme_provenance IN ('source', 'inferred', 'unclassified', 'manual') + ) + ) + """, + """ + CREATE INDEX ix_overview_funder + ON grant_overview_facts + (dataset_version, source_namespace, funding_name_normalized, grant_id) + """, + """ + CREATE INDEX ix_overview_recipient + ON grant_overview_facts + (dataset_version, source_namespace, recipient_name_normalized, grant_id) + """, + """ + CREATE INDEX ix_overview_source_date + ON grant_overview_facts + (dataset_version, source_namespace, award_date, grant_id) + """, + """ + CREATE TABLE grant_source_funder_facts ( + dataset_version TEXT NOT NULL, + grant_id TEXT NOT NULL, + country_code CHAR(2) NOT NULL, + country_name TEXT NOT NULL, + source_namespace TEXT NOT NULL, + source_funder_key TEXT NOT NULL, + identity_method TEXT NOT NULL, + source_organization_id TEXT, + normalized_name_fallback TEXT, + display_name TEXT NOT NULL, + recipient_key TEXT NOT NULL, + recipient_name TEXT NOT NULL, + award_date DATE, + currency CHAR(3), + original_amount_minor BIGINT, + original_amount_status TEXT NOT NULL, + eur_amount_minor BIGINT, + eur_amount_status TEXT NOT NULL, + conversion_status TEXT, + country_count INTEGER NOT NULL, + linked_profile_id BIGINT, + publisher_source_url TEXT, + source_record_id TEXT, + data_revision TEXT NOT NULL, + PRIMARY KEY (dataset_version, grant_id, country_code), + CONSTRAINT fk_funder_facts_grant + FOREIGN KEY (dataset_version, grant_id) + REFERENCES grants(dataset_version, grant_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_funder_facts_profile + FOREIGN KEY (dataset_version, linked_profile_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_funder_facts_country CHECK ( + country_code ~ '^[A-Z]{2}$' + ), + CONSTRAINT ck_funder_facts_currency CHECK ( + currency IS NULL OR currency ~ '^[A-Z]{3}$' + ), + CONSTRAINT ck_funder_facts_identity CHECK ( + identity_method IN ('source_id', 'normalized_name', 'manual', 'unknown') + ), + CONSTRAINT ck_funder_facts_count CHECK (country_count >= 1), + CONSTRAINT ck_funder_facts_original_status CHECK ( + original_amount_status IN ('valid', 'missing', 'negative', 'zero', 'invalid') + ), + CONSTRAINT ck_funder_facts_eur_status CHECK ( + eur_amount_status IN ('valid', 'missing', 'negative', 'zero', 'invalid') + ) + ) + """, + """ + CREATE INDEX ix_funder_facts_country_key + ON grant_source_funder_facts + (dataset_version, country_code, source_funder_key, grant_id) + """, + """ + CREATE INDEX ix_funder_facts_profile + ON grant_source_funder_facts + (dataset_version, linked_profile_id, source_funder_key, grant_id) + """, + """ + CREATE INDEX ix_funder_facts_source_name + ON grant_source_funder_facts + (dataset_version, source_namespace, display_name, grant_id) + """, + """ + CREATE TABLE organization_registry_links ( + dataset_version TEXT NOT NULL, + registry_id TEXT NOT NULL, + enriched_organization_id BIGINT NOT NULL, + match_status TEXT NOT NULL, + match_method TEXT NOT NULL, + match_confidence NUMERIC(6, 5), + match_reason TEXT, + reviewed_at TIMESTAMPTZ, + created_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (dataset_version, registry_id, enriched_organization_id), + CONSTRAINT fk_registry_links_registry + FOREIGN KEY (dataset_version, registry_id) + REFERENCES charity_registry_organizations(dataset_version, registry_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_registry_links_charity + FOREIGN KEY (dataset_version, enriched_organization_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_registry_links_status CHECK ( + match_status IN ('candidate', 'accepted', 'rejected', 'needs_review') + ), + CONSTRAINT ck_registry_links_method CHECK ( + match_method IN ( + 'exact_identifier', 'normalized_name', 'manual', 'unmatched' + ) + ), + CONSTRAINT ck_registry_links_confidence CHECK ( + match_confidence IS NULL OR match_confidence BETWEEN 0 AND 1 + ), + CONSTRAINT ck_registry_links_dates CHECK (updated_at >= created_at) + ) + """, + """ + CREATE INDEX ix_registry_links_profile + ON organization_registry_links + (dataset_version, enriched_organization_id, match_status, registry_id) + """, + """ + CREATE INDEX ix_registry_links_status + ON organization_registry_links + (dataset_version, registry_id, match_status, enriched_organization_id) + """, + """ + CREATE TABLE source_funder_link_overrides ( + source_namespace TEXT NOT NULL, + source_organization_id TEXT NOT NULL, + link_mode TEXT NOT NULL, + target_profile_id BIGINT, + target_dataset_version TEXT, + reason TEXT, + updated_by TEXT, + updated_at TIMESTAMPTZ NOT NULL, + revision BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (source_namespace, source_organization_id), + CONSTRAINT fk_funder_overrides_target + FOREIGN KEY (target_dataset_version, target_profile_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_funder_overrides_source CHECK ( + btrim(source_namespace) <> '' AND btrim(source_organization_id) <> '' + ), + CONSTRAINT ck_funder_overrides_mode CHECK ( + link_mode IN ('observed_only', 'link_profile', 'unlink', 'blocked') + ), + CONSTRAINT ck_funder_overrides_target CHECK ( + (link_mode = 'link_profile' AND target_profile_id IS NOT NULL + AND target_dataset_version IS NOT NULL) + OR (link_mode <> 'link_profile' AND target_profile_id IS NULL + AND target_dataset_version IS NULL) + ), + CONSTRAINT ck_funder_overrides_revision CHECK (revision >= 0) + ) + """, + """ + CREATE FUNCTION enforce_override_revision() RETURNS TRIGGER + LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP = 'UPDATE' AND NEW.revision <> OLD.revision + 1 THEN + RAISE EXCEPTION 'override revision must increase by exactly one'; + END IF; + RETURN NEW; + END; + $$ + """, + """ + CREATE TRIGGER trg_source_funder_override_revision + BEFORE UPDATE ON source_funder_link_overrides + FOR EACH ROW EXECUTE FUNCTION enforce_override_revision() + """, + """ + CREATE TABLE source_funder_profile_cache ( + dataset_version TEXT NOT NULL, + source_funder_key TEXT NOT NULL, + profile_id BIGINT NOT NULL, + status TEXT NOT NULL, + payload JSONB, + error TEXT, + updated_at TIMESTAMPTZ NOT NULL, + job_token UUID, + link_revision BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (dataset_version, source_funder_key), + CONSTRAINT fk_funder_cache_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT fk_funder_cache_profile + FOREIGN KEY (dataset_version, profile_id) + REFERENCES charities(dataset_version, charity_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_funder_cache_key CHECK (btrim(source_funder_key) <> ''), + CONSTRAINT ck_funder_cache_status CHECK ( + status IN ('pending', 'ready', 'failed', 'stale') + ), + CONSTRAINT ck_funder_cache_payload CHECK ( + (status = 'ready' AND payload IS NOT NULL AND error IS NULL) + OR status <> 'ready' + ), + CONSTRAINT ck_funder_cache_revision CHECK (link_revision >= 0) + ) + """, + """ + CREATE INDEX ix_funder_cache_profile + ON source_funder_profile_cache + (dataset_version, profile_id, status, source_funder_key) + """, + """ + CREATE TABLE exchange_rates ( + currency CHAR(3) NOT NULL, + rate_date DATE NOT NULL, + eur_reference_rate NUMERIC(24, 12) NOT NULL, + source_series TEXT NOT NULL, + source_url TEXT NOT NULL, + retrieved_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (currency, rate_date), + CONSTRAINT ck_exchange_rates_currency CHECK ( + currency ~ '^[A-Z]{3}$' AND currency <> 'EUR' + ), + CONSTRAINT ck_exchange_rates_value CHECK (eur_reference_rate > 0), + CONSTRAINT ck_exchange_rates_source CHECK ( + btrim(source_series) <> '' AND btrim(source_url) <> '' + ) + ) + """, + """ + CREATE TABLE job_runs ( + job_run_id UUID PRIMARY KEY, + job_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'created', + dataset_version TEXT, + idempotency_key TEXT NOT NULL, + requested_by TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + attempt INTEGER NOT NULL DEFAULT 1, + max_attempts INTEGER NOT NULL DEFAULT 1, + input JSONB NOT NULL DEFAULT '{}'::jsonb, + result JSONB, + error_class TEXT, + error_message TEXT, + CONSTRAINT uq_job_runs_idempotency UNIQUE (job_type, idempotency_key), + CONSTRAINT fk_job_runs_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT ck_job_runs_type CHECK (btrim(job_type) <> ''), + CONSTRAINT ck_job_runs_status CHECK ( + status IN ('created', 'queued', 'running', 'succeeded', 'failed', + 'cancelled', 'timed_out', 'dead_lettered') + ), + CONSTRAINT ck_job_runs_attempts CHECK ( + attempt >= 1 AND max_attempts >= 1 AND attempt <= max_attempts + ), + CONSTRAINT ck_job_runs_timestamps CHECK ( + (started_at IS NULL OR started_at >= requested_at) + AND (completed_at IS NULL OR started_at IS NOT NULL) + AND (completed_at IS NULL OR completed_at >= started_at) + ) + ) + """, + """ + CREATE INDEX ix_job_runs_status_requested + ON job_runs (status, requested_at DESC, job_run_id) + """, + """ + CREATE TABLE job_events ( + job_event_id UUID PRIMARY KEY, + job_run_id UUID NOT NULL, + sequence BIGINT NOT NULL, + event_type TEXT NOT NULL, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + actor_id TEXT, + details JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT uq_job_events_sequence UNIQUE (job_run_id, sequence), + CONSTRAINT fk_job_events_run FOREIGN KEY (job_run_id) + REFERENCES job_runs(job_run_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_job_events_sequence CHECK (sequence >= 1), + CONSTRAINT ck_job_events_type CHECK (btrim(event_type) <> '') + ) + """, + """ + CREATE INDEX ix_job_events_occurred + ON job_events (job_run_id, occurred_at, sequence) + """, + """ + CREATE TABLE audit_events ( + audit_event_id UUID PRIMARY KEY, + request_id TEXT NOT NULL, + actor_id TEXT NOT NULL, + actor_role TEXT, + action TEXT NOT NULL, + target TEXT, + reason TEXT, + outcome TEXT NOT NULL, + http_status INTEGER, + dataset_version TEXT, + error_class TEXT, + occurred_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + details JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT fk_audit_events_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT ck_audit_events_request CHECK (btrim(request_id) <> ''), + CONSTRAINT ck_audit_events_actor CHECK (btrim(actor_id) <> ''), + CONSTRAINT ck_audit_events_action CHECK (btrim(action) <> ''), + CONSTRAINT ck_audit_events_role CHECK ( + actor_role IS NULL OR actor_role IN ( + 'anonymous', 'viewer', 'analyst', 'operator', 'administrator', + 'service' + ) + ), + CONSTRAINT ck_audit_events_outcome CHECK ( + outcome IN ('allowed', 'denied', 'succeeded', 'failed') + ), + CONSTRAINT ck_audit_events_http_status CHECK ( + http_status IS NULL OR http_status BETWEEN 100 AND 599 + ) + ) + """, + """ + CREATE INDEX ix_audit_events_actor_time + ON audit_events (actor_id, occurred_at DESC, audit_event_id) + """, + """ + CREATE INDEX ix_audit_events_action_time + ON audit_events (action, occurred_at DESC, audit_event_id) + """, + """ + CREATE FUNCTION forbid_audit_event_mutation() RETURNS TRIGGER + LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'audit_events is append-only'; + END; + $$ + """, + """ + CREATE TRIGGER trg_audit_events_no_update + BEFORE UPDATE ON audit_events + FOR EACH ROW EXECUTE FUNCTION forbid_audit_event_mutation() + """, + """ + CREATE TRIGGER trg_audit_events_no_delete + BEFORE DELETE ON audit_events + FOR EACH ROW EXECUTE FUNCTION forbid_audit_event_mutation() + """, + """ + CREATE TABLE source_ingestion_runs ( + source_ingestion_run_id UUID PRIMARY KEY, + source_namespace TEXT NOT NULL, + dataset_version TEXT NOT NULL, + job_run_id UUID, + status TEXT NOT NULL DEFAULT 'created', + source_version TEXT, + source_uri TEXT, + object_checksum CHAR(64), + record_count BIGINT, + started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + metrics JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT uq_ingestion_source_version + UNIQUE (source_namespace, source_version, dataset_version), + CONSTRAINT fk_ingestion_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_ingestion_job FOREIGN KEY (job_run_id) + REFERENCES job_runs(job_run_id) + ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT ck_ingestion_source CHECK (btrim(source_namespace) <> ''), + CONSTRAINT ck_ingestion_status CHECK ( + status IN ('created', 'fetching', 'validating', 'loaded', + 'rejected', 'failed') + ), + CONSTRAINT ck_ingestion_checksum CHECK ( + object_checksum IS NULL OR object_checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_ingestion_count CHECK ( + record_count IS NULL OR record_count >= 0 + ), + CONSTRAINT ck_ingestion_completion CHECK ( + completed_at IS NULL OR completed_at >= started_at + ) + ) + """, + """ + CREATE TABLE data_quality_issues ( + data_quality_issue_id UUID PRIMARY KEY, + dataset_version TEXT NOT NULL, + migration_run_id UUID, + source_table TEXT NOT NULL, + source_record_id TEXT NOT NULL, + field_name TEXT, + original_value JSONB, + reason TEXT NOT NULL, + rule TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'open', + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + resolved_at TIMESTAMPTZ, + resolution TEXT, + CONSTRAINT fk_quality_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_quality_migration FOREIGN KEY (migration_run_id) + REFERENCES migration_runs(migration_run_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_quality_source CHECK ( + btrim(source_table) <> '' AND btrim(source_record_id) <> '' + ), + CONSTRAINT ck_quality_reason CHECK ( + btrim(reason) <> '' AND btrim(rule) <> '' + ), + CONSTRAINT ck_quality_status CHECK ( + status IN ('open', 'quarantined', 'accepted', 'corrected', 'dismissed') + ), + CONSTRAINT ck_quality_resolution CHECK ( + (status IN ('open', 'quarantined') AND resolved_at IS NULL) + OR (status IN ('accepted', 'corrected', 'dismissed') + AND resolved_at IS NOT NULL) + ) + ) + """, + """ + CREATE INDEX ix_quality_dataset_status + ON data_quality_issues + (dataset_version, status, source_table, source_record_id) + """, + """ + CREATE TABLE materialization_versions ( + materialization_version_id UUID PRIMARY KEY, + dataset_version TEXT NOT NULL, + materialization_name TEXT NOT NULL, + revision BIGINT NOT NULL, + status TEXT NOT NULL DEFAULT 'created', + is_active BOOLEAN NOT NULL DEFAULT FALSE, + row_count BIGINT, + checksum CHAR(64), + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at TIMESTAMPTZ, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT uq_materialization_revision + UNIQUE (dataset_version, materialization_name, revision), + CONSTRAINT fk_materialization_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_materialization_name CHECK ( + btrim(materialization_name) <> '' + ), + CONSTRAINT ck_materialization_revision CHECK (revision >= 1), + CONSTRAINT ck_materialization_status CHECK ( + status IN ('created', 'building', 'validated', 'active', + 'superseded', 'failed') + ), + CONSTRAINT ck_materialization_activation CHECK ( + (is_active AND status = 'active' AND activated_at IS NOT NULL) + OR (NOT is_active AND status <> 'active') + ), + CONSTRAINT ck_materialization_count CHECK ( + row_count IS NULL OR row_count >= 0 + ), + CONSTRAINT ck_materialization_checksum CHECK ( + checksum IS NULL OR checksum ~ '^[a-f0-9]{64}$' + ) + ) + """, + """ + CREATE UNIQUE INDEX uq_materialization_single_active + ON materialization_versions (dataset_version, materialization_name) + WHERE is_active + """, + """ + CREATE TABLE retention_actions ( + retention_action_id UUID PRIMARY KEY, + target_type TEXT NOT NULL, + target_id TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'reported', + dry_run BOOLEAN NOT NULL DEFAULT TRUE, + legal_hold BOOLEAN NOT NULL DEFAULT FALSE, + reason TEXT NOT NULL, + requested_by TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + approved_by TEXT, + approved_at TIMESTAMPTZ, + executed_at TIMESTAMPTZ, + manifest JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT ck_retention_target CHECK ( + btrim(target_type) <> '' AND btrim(target_id) <> '' + ), + CONSTRAINT ck_retention_status CHECK ( + status IN ('reported', 'held', 'approved', 'executed', + 'cancelled', 'failed') + ), + CONSTRAINT ck_retention_approval CHECK ( + dry_run OR (approved_by IS NOT NULL AND approved_at IS NOT NULL) + ), + CONSTRAINT ck_retention_hold CHECK ( + NOT legal_hold OR status IN ('reported', 'held', 'cancelled') + ), + CONSTRAINT ck_retention_execution CHECK ( + executed_at IS NULL OR (status = 'executed' AND NOT dry_run) + ) + ) + """, + """ + CREATE TABLE export_jobs ( + export_job_id UUID PRIMARY KEY, + job_run_id UUID, + dataset_version TEXT NOT NULL, + export_type TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'created', + requested_by TEXT NOT NULL, + requested_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + object_key TEXT, + checksum CHAR(64), + row_count BIGINT, + expires_at TIMESTAMPTZ, + CONSTRAINT fk_export_job_run FOREIGN KEY (job_run_id) + REFERENCES job_runs(job_run_id) + ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT fk_export_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_export_type CHECK (btrim(export_type) <> ''), + CONSTRAINT ck_export_status CHECK ( + status IN ('created', 'queued', 'running', 'succeeded', + 'failed', 'expired') + ), + CONSTRAINT ck_export_checksum CHECK ( + checksum IS NULL OR checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_export_count CHECK (row_count IS NULL OR row_count >= 0), + CONSTRAINT ck_export_completion CHECK ( + completed_at IS NULL OR completed_at >= requested_at + ) + ) + """, + """ + CREATE INDEX ix_export_jobs_status_requested + ON export_jobs (status, requested_at DESC, export_job_id) + """, + """ + CREATE TABLE idempotency_records ( + actor_id TEXT NOT NULL, + action TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + request_hash CHAR(64) NOT NULL, + status TEXT NOT NULL DEFAULT 'reserved', + response_status INTEGER, + response_body JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + completed_at TIMESTAMPTZ, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (actor_id, action, idempotency_key), + CONSTRAINT ck_idempotency_actor CHECK (btrim(actor_id) <> ''), + CONSTRAINT ck_idempotency_action CHECK (btrim(action) <> ''), + CONSTRAINT ck_idempotency_key CHECK (btrim(idempotency_key) <> ''), + CONSTRAINT ck_idempotency_hash CHECK ( + request_hash ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_idempotency_status CHECK ( + status IN ('reserved', 'completed', 'failed') + ), + CONSTRAINT ck_idempotency_response CHECK ( + response_status IS NULL OR response_status BETWEEN 100 AND 599 + ), + CONSTRAINT ck_idempotency_expiry CHECK (expires_at > created_at), + CONSTRAINT ck_idempotency_completion CHECK ( + (status = 'reserved' AND completed_at IS NULL) + OR (status <> 'reserved' AND completed_at IS NOT NULL) + ) + ) + """, + """ + CREATE INDEX ix_idempotency_expiry + ON idempotency_records (expires_at, status) + """, + ] + ) + + +def downgrade() -> None: + _execute_all( + [ + "DROP TABLE IF EXISTS idempotency_records", + "DROP TABLE IF EXISTS export_jobs", + "DROP TABLE IF EXISTS retention_actions", + "DROP TABLE IF EXISTS materialization_versions", + "DROP TABLE IF EXISTS data_quality_issues", + "DROP TABLE IF EXISTS source_ingestion_runs", + "DROP TRIGGER IF EXISTS trg_audit_events_no_delete ON audit_events", + "DROP TRIGGER IF EXISTS trg_audit_events_no_update ON audit_events", + "DROP TABLE IF EXISTS audit_events", + "DROP FUNCTION IF EXISTS forbid_audit_event_mutation()", + "DROP TABLE IF EXISTS job_events", + "DROP TABLE IF EXISTS job_runs", + "DROP TABLE IF EXISTS exchange_rates", + "DROP TABLE IF EXISTS source_funder_profile_cache", + "DROP TRIGGER IF EXISTS trg_source_funder_override_revision ON source_funder_link_overrides", + "DROP TABLE IF EXISTS source_funder_link_overrides", + "DROP FUNCTION IF EXISTS enforce_override_revision()", + "DROP TABLE IF EXISTS organization_registry_links", + "DROP TABLE IF EXISTS grant_source_funder_facts", + "DROP TABLE IF EXISTS grant_overview_facts", + "DROP TABLE IF EXISTS grant_programme_categories", + "DROP TABLE IF EXISTS grant_beneficiary_terms", + "DROP TABLE IF EXISTS grant_beneficiary_countries", + "DROP TABLE IF EXISTS grants", + "DROP TABLE IF EXISTS charity_registry_organizations", + "DROP TABLE IF EXISTS charity_geographic_areas", + "DROP TABLE IF EXISTS charity_programme_categories", + "DROP TABLE IF EXISTS charities", + "DROP TABLE IF EXISTS migration_runs", + "DROP TABLE IF EXISTS dataset_versions", + ] + ) diff --git a/alembic/versions/0002_exchange_rate_period.py b/alembic/versions/0002_exchange_rate_period.py new file mode 100644 index 0000000..52fc9bf --- /dev/null +++ b/alembic/versions/0002_exchange_rate_period.py @@ -0,0 +1,46 @@ +"""Preserve the monthly precision of grant exchange-rate periods. + +Revision ID: 0002_exchange_rate_period +Revises: 0001_postgresql_foundation +""" + +from alembic import op + + +revision = "0002_exchange_rate_period" +down_revision = "0001_postgresql_foundation" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE grants + ALTER COLUMN exchange_rate_date TYPE VARCHAR(7) + USING to_char(exchange_rate_date, 'YYYY-MM') + """ + ) + op.execute( + """ + ALTER TABLE grants + ADD CONSTRAINT ck_grants_exchange_rate_period + CHECK ( + exchange_rate_date IS NULL + OR exchange_rate_date ~ '^[0-9]{4}-(0[1-9]|1[0-2])$' + ) + """ + ) + + +def downgrade() -> None: + op.execute( + "ALTER TABLE grants DROP CONSTRAINT ck_grants_exchange_rate_period" + ) + op.execute( + """ + ALTER TABLE grants + ALTER COLUMN exchange_rate_date TYPE DATE + USING (exchange_rate_date || '-01')::date + """ + ) diff --git a/alembic/versions/0003_grant_award_timestamp.py b/alembic/versions/0003_grant_award_timestamp.py new file mode 100644 index 0000000..b1ee96a --- /dev/null +++ b/alembic/versions/0003_grant_award_timestamp.py @@ -0,0 +1,44 @@ +"""Preserve original grant award-date timestamp precision. + +Revision ID: 0003_grant_award_timestamp +Revises: 0002_exchange_rate_period +""" + +from alembic import op + + +revision = "0003_grant_award_timestamp" +down_revision = "0002_exchange_rate_period" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.execute( + """ + ALTER TABLE grants + ALTER COLUMN award_date TYPE TEXT + USING award_date::text + """ + ) + op.execute( + """ + ALTER TABLE grants + ADD CONSTRAINT ck_grants_award_date_iso + CHECK ( + award_date IS NULL + OR award_date ~ '^[0-9]{4}-[0-9]{2}-[0-9]{2}' + ) + """ + ) + + +def downgrade() -> None: + op.execute("ALTER TABLE grants DROP CONSTRAINT ck_grants_award_date_iso") + op.execute( + """ + ALTER TABLE grants + ALTER COLUMN award_date TYPE DATE + USING substring(award_date FROM 1 FOR 10)::date + """ + ) diff --git a/alembic/versions/0004_versioned_analytics_materializations.py b/alembic/versions/0004_versioned_analytics_materializations.py new file mode 100644 index 0000000..bb78435 --- /dev/null +++ b/alembic/versions/0004_versioned_analytics_materializations.py @@ -0,0 +1,696 @@ +"""Add versioned dashboard materializations and refresh function. + +Revision ID: 0004_versioned_analytics +Revises: 0003_grant_award_timestamp +""" + +from alembic import op + + +revision = "0004_versioned_analytics" +down_revision = "0003_grant_award_timestamp" +branch_labels = None +depends_on = None + + +TABLES = ( + "analytics_scope_totals", + "analytics_country_aggregates", + "analytics_country_connections", + "analytics_period_aggregates", + "analytics_programme_aggregates", + "analytics_entity_rankings", + "analytics_country_funder_rankings", + "analytics_funder_relationships", + "analytics_filter_values", +) + + +def upgrade() -> None: + statements = ( + """ + CREATE TABLE analytics_scope_totals ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + total_grants BIGINT NOT NULL, + known_geography_grants BIGINT NOT NULL, + multi_country_grants BIGINT NOT NULL, + invalid_amount_grants BIGINT NOT NULL, + missing_date_grants BIGINT NOT NULL, + negative_amount_grants BIGINT NOT NULL, + zero_amount_grants BIGINT NOT NULL, + classified_grants BIGINT NOT NULL, + unclassified_grants BIGINT NOT NULL, + source_classified_grants BIGINT NOT NULL, + inferred_classified_grants BIGINT NOT NULL, + multiple_programme_grants BIGINT NOT NULL, + invalid_source_label_grants BIGINT NOT NULL, + low_confidence_grants BIGINT NOT NULL, + total_amount_minor NUMERIC(30, 4) NOT NULL, + maximum_amount_minor NUMERIC(30, 4), + first_award_date DATE, + latest_award_date DATE, + refreshed_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (dataset_version, amount_basis, currency), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (currency ~ '^[A-Z]{3}$') + ) + """, + """ + CREATE TABLE analytics_country_aggregates ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + country_code CHAR(2) NOT NULL, + country_name TEXT NOT NULL, + grant_count BIGINT NOT NULL, + total_amount_minor NUMERIC(30, 4) NOT NULL, + distinct_funders BIGINT NOT NULL, + distinct_recipients BIGINT NOT NULL, + association_count BIGINT NOT NULL, + multi_country_count BIGINT NOT NULL, + invalid_amount_count BIGINT NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, country_code + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (currency ~ '^[A-Z]{3}$'), + CHECK (country_code ~ '^[A-Z]{2}$') + ) + """, + """ + CREATE INDEX ix_analytics_country_amount + ON analytics_country_aggregates + (dataset_version, amount_basis, currency, total_amount_minor DESC, country_code) + """, + """ + CREATE TABLE analytics_country_connections ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + origin_country_code CHAR(2) NOT NULL, + origin_country_name TEXT NOT NULL, + destination_country_code CHAR(2) NOT NULL, + destination_country_name TEXT NOT NULL, + grant_count BIGINT NOT NULL, + origin_sources TEXT[] NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, + origin_country_code, destination_country_code + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (origin_country_code ~ '^[A-Z]{2}$'), + CHECK (destination_country_code ~ '^[A-Z]{2}$') + ) + """, + """ + CREATE INDEX ix_analytics_country_connection_rank + ON analytics_country_connections + (dataset_version, amount_basis, currency, grant_count DESC, + origin_country_code, destination_country_code) + """, + """ + CREATE TABLE analytics_period_aggregates ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + granularity TEXT NOT NULL, + period_start DATE NOT NULL, + source_record_count BIGINT NOT NULL, + grant_count BIGINT NOT NULL, + total_amount_minor NUMERIC(30, 4) NOT NULL, + mapped_grant_count BIGINT NOT NULL, + unmapped_grant_count BIGINT NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, granularity, period_start + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (granularity IN ('monthly', 'yearly')) + ) + """, + """ + CREATE TABLE analytics_programme_aggregates ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + programme_area TEXT NOT NULL, + distinct_grant_count BIGINT NOT NULL, + weighted_grant_count NUMERIC(30, 8) NOT NULL, + allocated_amount_minor NUMERIC(30, 4) NOT NULL, + source_classified_count BIGINT NOT NULL, + inferred_classified_count BIGINT NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, programme_area + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')) + ) + """, + """ + CREATE TABLE analytics_entity_rankings ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + entity_role TEXT NOT NULL, + entity_key TEXT NOT NULL, + organization_id BIGINT, + organization_name TEXT NOT NULL, + total_amount_minor NUMERIC(30, 4) NOT NULL, + grant_count BIGINT NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, entity_role, entity_key + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (entity_role IN ('funder', 'recipient')) + ) + """, + """ + CREATE INDEX ix_analytics_entity_rank + ON analytics_entity_rankings + (dataset_version, amount_basis, currency, entity_role, + total_amount_minor DESC, entity_key) + """, + """ + CREATE TABLE analytics_country_funder_rankings ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + country_code CHAR(2) NOT NULL, + country_name TEXT NOT NULL, + source_funder_key TEXT NOT NULL, + source_namespace TEXT NOT NULL, + source_organization_id TEXT, + display_name TEXT NOT NULL, + identity_method TEXT NOT NULL, + source_ids TEXT[] NOT NULL, + sources TEXT[] NOT NULL, + observed_profile_id BIGINT, + grant_count BIGINT NOT NULL, + recipient_count BIGINT NOT NULL, + first_award_date DATE, + latest_award_date DATE, + selected_amount_minor NUMERIC(30, 4) NOT NULL, + included_count BIGINT NOT NULL, + multi_country_count BIGINT NOT NULL, + conversion_excluded BIGINT NOT NULL, + missing_count BIGINT NOT NULL, + invalid_count BIGINT NOT NULL, + negative_count BIGINT NOT NULL, + fallback_amount_minor NUMERIC(30, 4), + fallback_currency CHAR(3), + fallback_count BIGINT NOT NULL, + publisher_source_url TEXT, + override_identifier TEXT NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, country_code, + source_funder_key + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')) + ) + """, + """ + CREATE INDEX ix_analytics_country_funder_amount + ON analytics_country_funder_rankings + (dataset_version, amount_basis, currency, country_code, + selected_amount_minor DESC, source_funder_key) + """, + """ + CREATE INDEX ix_analytics_country_funder_activity + ON analytics_country_funder_rankings + (dataset_version, amount_basis, currency, country_code, + grant_count DESC, latest_award_date DESC, source_funder_key) + """, + """ + CREATE TABLE analytics_funder_relationships ( + dataset_version TEXT NOT NULL, + amount_basis TEXT NOT NULL, + currency CHAR(3) NOT NULL, + country_code CHAR(2) NOT NULL, + source_funder_key TEXT NOT NULL, + recipient_key TEXT NOT NULL, + recipient_name TEXT NOT NULL, + total_amount_minor NUMERIC(30, 4) NOT NULL, + grant_count BIGINT NOT NULL, + rank_within_funder INTEGER NOT NULL, + PRIMARY KEY ( + dataset_version, amount_basis, currency, country_code, + source_funder_key, recipient_key + ), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (amount_basis IN ('eur_converted', 'original')), + CHECK (rank_within_funder BETWEEN 1 AND 50) + ) + """, + """ + CREATE INDEX ix_analytics_funder_relationship_rank + ON analytics_funder_relationships + (dataset_version, amount_basis, currency, country_code, + source_funder_key, rank_within_funder) + """, + """ + CREATE TABLE analytics_filter_values ( + dataset_version TEXT NOT NULL, + dimension TEXT NOT NULL, + value TEXT NOT NULL, + usage_count BIGINT NOT NULL, + PRIMARY KEY (dataset_version, dimension, value), + FOREIGN KEY (dataset_version) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE CASCADE, + CHECK (dimension IN ( + 'beneficiary_country', 'currency', 'programme_area', 'source' + )) + ) + """, + """ + CREATE INDEX ix_registry_current_normalized_name + ON charity_registry_organizations + (dataset_version, normalized_name, registry_id) + WHERE is_current_source_record + """, + """ + CREATE INDEX ix_grants_funding_source_id + ON grants (dataset_version, funding_org_source_id, grant_id) + WHERE funding_org_source_id IS NOT NULL + """, + """ + CREATE INDEX ix_grants_recipient_source_id + ON grants (dataset_version, recipient_org_source_id, grant_id) + WHERE recipient_org_source_id IS NOT NULL + """, + ) + for statement in statements: + op.execute(statement) + op.execute(_refresh_function_sql()) + + +def _refresh_function_sql() -> str: + return r""" + CREATE FUNCTION refresh_analytics_materializations(p_dataset_version TEXT) + RETURNS BIGINT + LANGUAGE plpgsql + AS $$ + DECLARE + inserted_rows BIGINT := 0; + current_rows BIGINT := 0; + version_uuid UUID; + BEGIN + IF NOT EXISTS ( + SELECT 1 FROM dataset_versions WHERE dataset_version=p_dataset_version + ) THEN + RAISE EXCEPTION 'unknown dataset version %', p_dataset_version; + END IF; + + DELETE FROM analytics_filter_values WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_funder_relationships WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_country_funder_rankings WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_entity_rankings WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_programme_aggregates WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_period_aggregates WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_country_aggregates WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_country_connections WHERE dataset_version=p_dataset_version; + DELETE FROM analytics_scope_totals WHERE dataset_version=p_dataset_version; + + INSERT INTO analytics_scope_totals + WITH scoped AS ( + SELECT dataset_version, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, award_date, + country_count, programme_category_count, programme_provenance, + invalid_source_label, low_confidence_inference, + eur_amount_minor AS amount_minor, + eur_amount_status AS amount_status + FROM grant_overview_facts WHERE dataset_version=p_dataset_version + UNION ALL + SELECT dataset_version, 'original', currency, award_date, + country_count, programme_category_count, programme_provenance, + invalid_source_label, low_confidence_inference, + original_amount_minor, original_amount_status + FROM grant_overview_facts + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ) + SELECT dataset_version, amount_basis, selected_currency, + COUNT(*), COUNT(*) FILTER (WHERE country_count>0), + COUNT(*) FILTER (WHERE country_count>1), + COUNT(*) FILTER (WHERE amount_status IN ('missing','invalid')), + COUNT(*) FILTER (WHERE award_date IS NULL), + COUNT(*) FILTER (WHERE amount_status='negative'), + COUNT(*) FILTER (WHERE amount_status='zero'), + COUNT(*) FILTER (WHERE programme_category_count>0), + COUNT(*) FILTER (WHERE programme_category_count=0), + COUNT(*) FILTER (WHERE programme_provenance='source'), + COUNT(*) FILTER (WHERE programme_provenance='inferred'), + COUNT(*) FILTER (WHERE programme_category_count>1), + COUNT(*) FILTER (WHERE invalid_source_label), + COUNT(*) FILTER (WHERE low_confidence_inference), + COALESCE(SUM(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), 0), + MAX(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), + MIN(award_date), MAX(award_date), CURRENT_TIMESTAMP + FROM scoped + GROUP BY dataset_version, amount_basis, selected_currency; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_country_connections + WITH scoped AS ( + SELECT dataset_version, grant_id, source_namespace, + origin_country_code, origin_country_name, + 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency + FROM grant_overview_facts + WHERE dataset_version=p_dataset_version + AND origin_country_code IS NOT NULL + UNION ALL + SELECT dataset_version, grant_id, source_namespace, + origin_country_code, origin_country_name, + 'original', currency + FROM grant_overview_facts + WHERE dataset_version=p_dataset_version + AND origin_country_code IS NOT NULL AND currency IS NOT NULL + ) + SELECT fact.dataset_version, fact.amount_basis, fact.selected_currency, + fact.origin_country_code, MIN(fact.origin_country_name), + country.country_code, MIN(country.country_name), + COUNT(DISTINCT fact.grant_id), + array_agg(DISTINCT fact.source_namespace) + FROM scoped AS fact + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=fact.dataset_version + AND country.grant_id=fact.grant_id + WHERE country.country_code<>fact.origin_country_code + GROUP BY fact.dataset_version, fact.amount_basis, fact.selected_currency, + fact.origin_country_code, country.country_code; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_country_aggregates + WITH scoped AS ( + SELECT fact.*, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, + eur_amount_minor AS amount_minor, + eur_amount_status AS amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version + UNION ALL + SELECT fact.*, 'original', currency, original_amount_minor, + original_amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ) + SELECT fact.dataset_version, fact.amount_basis, fact.selected_currency, + country.country_code, MIN(country.country_name), + COUNT(DISTINCT fact.grant_id), + COALESCE(SUM( + CASE WHEN fact.amount_status NOT IN ('missing','invalid','negative') + THEN fact.amount_minor::numeric / NULLIF(fact.country_count, 0) + ELSE 0 END + ), 0), + COUNT(DISTINCT fact.funding_name), + COUNT(DISTINCT fact.recipient_name), COUNT(*), + COUNT(*) FILTER (WHERE fact.country_count>1), + COUNT(*) FILTER (WHERE fact.amount_status IN ('missing','invalid','negative')) + FROM scoped AS fact + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=fact.dataset_version + AND country.grant_id=fact.grant_id + GROUP BY fact.dataset_version, fact.amount_basis, fact.selected_currency, + country.country_code; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_period_aggregates + WITH scoped AS ( + SELECT dataset_version, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, award_date, country_count, + eur_amount_minor AS amount_minor, eur_amount_status AS amount_status + FROM grant_overview_facts WHERE dataset_version=p_dataset_version + UNION ALL + SELECT dataset_version, 'original', currency, award_date, country_count, + original_amount_minor, original_amount_status + FROM grant_overview_facts + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ), granular AS ( + SELECT scoped.*, granularity, + date_trunc(granularity, award_date)::date AS period_start + FROM scoped CROSS JOIN (VALUES ('month'), ('year')) AS levels(granularity) + WHERE award_date IS NOT NULL + ) + SELECT dataset_version, amount_basis, selected_currency, + CASE granularity WHEN 'month' THEN 'monthly' ELSE 'yearly' END, + period_start, COUNT(*), + COUNT(*) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), + COALESCE(SUM(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), 0), + COUNT(*) FILTER (WHERE country_count>0), + COUNT(*) FILTER (WHERE country_count=0) + FROM granular + GROUP BY dataset_version, amount_basis, selected_currency, + granularity, period_start; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_programme_aggregates + WITH scoped AS ( + SELECT fact.*, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, + eur_amount_minor AS amount_minor, + eur_amount_status AS amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version + UNION ALL + SELECT fact.*, 'original', currency, original_amount_minor, + original_amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ) + SELECT fact.dataset_version, fact.amount_basis, fact.selected_currency, + category.programme_area, COUNT(DISTINCT fact.grant_id), + SUM(1.0 / NULLIF(fact.programme_category_count, 0)), + COALESCE(SUM( + CASE WHEN fact.amount_status NOT IN ('missing','invalid','negative') + THEN fact.amount_minor::numeric + / NULLIF(fact.programme_category_count, 0) + ELSE 0 END + ), 0), + COUNT(DISTINCT fact.grant_id) FILTER ( + WHERE fact.programme_provenance='source' + ), + COUNT(DISTINCT fact.grant_id) FILTER ( + WHERE fact.programme_provenance='inferred' + ) + FROM scoped AS fact + JOIN grant_programme_categories AS category + ON category.dataset_version=fact.dataset_version + AND category.grant_id=fact.grant_id + GROUP BY fact.dataset_version, fact.amount_basis, fact.selected_currency, + category.programme_area; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_entity_rankings + WITH scoped AS ( + SELECT fact.*, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, + eur_amount_minor AS amount_minor, eur_amount_status AS amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version + UNION ALL + SELECT fact.*, 'original', currency, original_amount_minor, + original_amount_status + FROM grant_overview_facts AS fact + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ), entities AS ( + SELECT scoped.dataset_version, scoped.amount_basis, + scoped.selected_currency, 'funder' AS role, + COALESCE(grant_row.funding_charity_id::text, + 'name:' || scoped.funding_name_normalized) AS entity_key, + grant_row.funding_charity_id AS organization_id, + scoped.funding_name AS organization_name, + scoped.amount_minor, scoped.amount_status, scoped.grant_id + FROM scoped + JOIN grants AS grant_row USING (dataset_version, grant_id) + WHERE scoped.funding_name IS NOT NULL + UNION ALL + SELECT scoped.dataset_version, scoped.amount_basis, + scoped.selected_currency, 'recipient', + COALESCE(grant_row.recipient_charity_id::text, + 'name:' || scoped.recipient_name_normalized), + grant_row.recipient_charity_id, scoped.recipient_name, + scoped.amount_minor, scoped.amount_status, scoped.grant_id + FROM scoped + JOIN grants AS grant_row USING (dataset_version, grant_id) + WHERE scoped.recipient_name IS NOT NULL + ) + SELECT dataset_version, amount_basis, selected_currency, role, entity_key, + MAX(organization_id), MIN(organization_name), + COALESCE(SUM(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), 0), COUNT(DISTINCT grant_id) + FROM entities + GROUP BY dataset_version, amount_basis, selected_currency, role, entity_key; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_country_funder_rankings + WITH scoped AS ( + SELECT fact.*, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, + eur_amount_minor AS amount_minor, eur_amount_status AS amount_status + FROM grant_source_funder_facts AS fact + WHERE dataset_version=p_dataset_version + UNION ALL + SELECT fact.*, 'original', currency, original_amount_minor, + original_amount_status + FROM grant_source_funder_facts AS fact + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ) + SELECT dataset_version, amount_basis, selected_currency, country_code, + MIN(country_name), source_funder_key, MIN(source_namespace), + MIN(source_organization_id), MIN(display_name), MIN(identity_method), + COALESCE(array_agg(DISTINCT source_organization_id) + FILTER (WHERE source_organization_id IS NOT NULL), ARRAY[]::text[]), + array_agg(DISTINCT source_namespace), MAX(linked_profile_id), + COUNT(DISTINCT grant_id), COUNT(DISTINCT recipient_key), + MIN(award_date), MAX(award_date), + COALESCE(SUM(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), 0), + COUNT(*) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), + COUNT(*) FILTER (WHERE country_count>1), + COUNT(*) FILTER ( + WHERE conversion_status IS NULL OR conversion_status='unavailable_missing_rate' + ), + COUNT(*) FILTER (WHERE original_amount_status='missing'), + COUNT(*) FILTER (WHERE original_amount_status='invalid'), + COUNT(*) FILTER (WHERE original_amount_status='negative'), + SUM(original_amount_minor) FILTER ( + WHERE original_amount_status NOT IN ('negative','invalid','missing') + ), + MIN(currency), + COUNT(*) FILTER ( + WHERE original_amount_status NOT IN ('negative','invalid','missing') + ), + MIN(publisher_source_url), + MIN(COALESCE(NULLIF(source_organization_id, ''), + 'source-funder-key:' || source_funder_key)) + FROM scoped + GROUP BY dataset_version, amount_basis, selected_currency, + country_code, source_funder_key; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_funder_relationships + WITH scoped AS ( + SELECT fact.*, 'eur_converted'::text AS amount_basis, + 'EUR'::char(3) AS selected_currency, + eur_amount_minor AS amount_minor, eur_amount_status AS amount_status + FROM grant_source_funder_facts AS fact + WHERE dataset_version=p_dataset_version + UNION ALL + SELECT fact.*, 'original', currency, original_amount_minor, + original_amount_status + FROM grant_source_funder_facts AS fact + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + ), aggregated AS ( + SELECT dataset_version, amount_basis, selected_currency, country_code, + source_funder_key, recipient_key, MIN(recipient_name) AS recipient_name, + COALESCE(SUM(amount_minor) FILTER ( + WHERE amount_status NOT IN ('missing','invalid','negative') + ), 0) AS total_amount_minor, + COUNT(DISTINCT grant_id) AS grant_count + FROM scoped + GROUP BY dataset_version, amount_basis, selected_currency, country_code, + source_funder_key, recipient_key + ), ranked AS ( + SELECT aggregated.*, + row_number() OVER ( + PARTITION BY dataset_version, amount_basis, selected_currency, + country_code, source_funder_key + ORDER BY total_amount_minor DESC, recipient_key + ) AS relationship_rank + FROM aggregated + ) + SELECT dataset_version, amount_basis, selected_currency, country_code, + source_funder_key, recipient_key, recipient_name, + total_amount_minor, grant_count, relationship_rank + FROM ranked WHERE relationship_rank<=50; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + INSERT INTO analytics_filter_values + SELECT p_dataset_version, 'beneficiary_country', country_name, COUNT(*) + FROM grant_beneficiary_countries WHERE dataset_version=p_dataset_version + GROUP BY country_name + UNION ALL + SELECT p_dataset_version, 'currency', currency, COUNT(*) + FROM grant_overview_facts + WHERE dataset_version=p_dataset_version AND currency IS NOT NULL + GROUP BY currency + UNION ALL + SELECT p_dataset_version, 'programme_area', programme_area, COUNT(*) + FROM grant_programme_categories WHERE dataset_version=p_dataset_version + GROUP BY programme_area + UNION ALL + SELECT p_dataset_version, 'source', source_namespace, COUNT(*) + FROM grant_overview_facts WHERE dataset_version=p_dataset_version + GROUP BY source_namespace; + GET DIAGNOSTICS current_rows = ROW_COUNT; + inserted_rows := inserted_rows + current_rows; + + version_uuid := ( + substring(md5(p_dataset_version || '-dashboard-analytics-v1'), 1, 8) + || '-' || substring(md5(p_dataset_version || '-dashboard-analytics-v1'), 9, 4) + || '-' || substring(md5(p_dataset_version || '-dashboard-analytics-v1'), 13, 4) + || '-' || substring(md5(p_dataset_version || '-dashboard-analytics-v1'), 17, 4) + || '-' || substring(md5(p_dataset_version || '-dashboard-analytics-v1'), 21, 12) + )::uuid; + INSERT INTO materialization_versions ( + materialization_version_id, dataset_version, materialization_name, + revision, status, is_active, row_count, activated_at, metadata + ) VALUES ( + version_uuid, p_dataset_version, 'dashboard_analytics', 1, + 'active', TRUE, inserted_rows, CURRENT_TIMESTAMP, + jsonb_build_object('function', 'refresh_analytics_materializations', + 'relationship_limit', 50) + ) + ON CONFLICT (dataset_version, materialization_name, revision) + DO UPDATE SET status='active', is_active=TRUE, row_count=inserted_rows, + activated_at=CURRENT_TIMESTAMP, + metadata=EXCLUDED.metadata; + RETURN inserted_rows; + END; + $$ + """ + + +def downgrade() -> None: + op.execute("DROP FUNCTION IF EXISTS refresh_analytics_materializations(TEXT)") + op.execute("DROP INDEX IF EXISTS ix_grants_recipient_source_id") + op.execute("DROP INDEX IF EXISTS ix_grants_funding_source_id") + op.execute( + "DROP INDEX IF EXISTS ix_registry_current_normalized_name" + ) + for table in reversed(TABLES): + op.execute(f"DROP TABLE IF EXISTS {table}") diff --git a/alembic/versions/0005_durable_pipeline.py b/alembic/versions/0005_durable_pipeline.py new file mode 100644 index 0000000..f3c028c --- /dev/null +++ b/alembic/versions/0005_durable_pipeline.py @@ -0,0 +1,343 @@ +"""Add durable pipeline coordination and immutable object manifests. + +Revision ID: 0005_durable_pipeline +Revises: 0004_versioned_analytics +""" + +from alembic import op + + +revision = "0005_durable_pipeline" +down_revision = "0004_versioned_analytics" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + statements = ( + """ + CREATE TABLE source_configurations ( + source_name TEXT PRIMARY KEY, + source_owner TEXT NOT NULL, + technical_owner TEXT NOT NULL, + legal_status TEXT NOT NULL, + licence_status TEXT NOT NULL, + terms_url TEXT, + rate_limit_per_minute INTEGER NOT NULL, + user_agent TEXT NOT NULL, + freshness_sla_hours INTEGER NOT NULL, + schedule_expression TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT FALSE, + governance_blocked BOOLEAN NOT NULL DEFAULT TRUE, + last_success_at TIMESTAMPTZ, + watermark TEXT, + classification TEXT NOT NULL, + retention_class TEXT NOT NULL, + schema_version TEXT NOT NULL, + credentials_reference TEXT, + retry_limit INTEGER NOT NULL, + timeout_seconds INTEGER NOT NULL, + maximum_pages INTEGER NOT NULL, + maximum_records BIGINT NOT NULL, + configuration_checksum CHAR(64) NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_source_configuration_name CHECK (btrim(source_name) <> ''), + CONSTRAINT ck_source_configuration_owners CHECK ( + btrim(source_owner) <> '' AND btrim(technical_owner) <> '' + ), + CONSTRAINT ck_source_configuration_legal CHECK ( + legal_status IN ('approved', 'unresolved', 'restricted', 'prohibited') + AND licence_status IN ('approved', 'unresolved', 'restricted', 'prohibited') + ), + CONSTRAINT ck_source_configuration_limits CHECK ( + rate_limit_per_minute >= 1 + AND freshness_sla_hours >= 1 + AND retry_limit BETWEEN 0 AND 20 + AND timeout_seconds BETWEEN 1 AND 86400 + AND maximum_pages >= 1 + AND maximum_records >= 1 + ), + CONSTRAINT ck_source_configuration_schedule CHECK ( + btrim(schedule_expression) <> '' + AND btrim(user_agent) <> '' + AND btrim(classification) <> '' + AND btrim(retention_class) <> '' + AND btrim(schema_version) <> '' + ), + CONSTRAINT ck_source_configuration_governance CHECK ( + NOT enabled OR ( + NOT governance_blocked + AND legal_status = 'approved' + AND licence_status = 'approved' + ) + ), + CONSTRAINT ck_source_configuration_checksum CHECK ( + configuration_checksum ~ '^[a-f0-9]{64}$' + ) + ) + """, + """ + CREATE TABLE storage_objects ( + storage_object_id UUID PRIMARY KEY, + zone TEXT NOT NULL, + bucket_alias TEXT NOT NULL, + object_key TEXT NOT NULL, + object_version TEXT NOT NULL, + checksum CHAR(64) NOT NULL, + content_length BIGINT NOT NULL, + content_type TEXT NOT NULL, + source_name TEXT, + source_ingestion_run_id UUID, + immutable BOOLEAN NOT NULL DEFAULT TRUE, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT uq_storage_object_version UNIQUE ( + zone, bucket_alias, object_key, object_version + ), + CONSTRAINT fk_storage_object_ingestion FOREIGN KEY (source_ingestion_run_id) + REFERENCES source_ingestion_runs(source_ingestion_run_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_storage_object_zone CHECK ( + zone IN ('raw', 'validated', 'curated', 'export') + ), + CONSTRAINT ck_storage_object_location CHECK ( + btrim(bucket_alias) <> '' + AND btrim(object_key) <> '' + AND btrim(object_version) <> '' + AND btrim(content_type) <> '' + ), + CONSTRAINT ck_storage_object_checksum CHECK ( + checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_storage_object_length CHECK (content_length >= 0), + CONSTRAINT ck_storage_object_raw_immutable CHECK ( + zone <> 'raw' OR immutable + ) + ) + """, + """ + CREATE INDEX ix_storage_objects_source_created + ON storage_objects (source_name, created_at DESC, storage_object_id) + """, + """ + CREATE TABLE ingestion_run_manifests ( + ingestion_run_manifest_id UUID PRIMARY KEY, + source_ingestion_run_id UUID NOT NULL UNIQUE, + schema_version TEXT NOT NULL, + source_name TEXT NOT NULL, + source_version TEXT, + dataset_version TEXT NOT NULL, + raw_object_id UUID, + validated_object_id UUID, + curated_object_id UUID, + watermark_before TEXT, + watermark_after TEXT, + record_count BIGINT NOT NULL, + accepted_count BIGINT NOT NULL, + rejected_count BIGINT NOT NULL, + quarantined_count BIGINT NOT NULL, + retry_count INTEGER NOT NULL, + manifest_checksum CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + manifest JSONB NOT NULL, + CONSTRAINT fk_ingestion_manifest_run FOREIGN KEY (source_ingestion_run_id) + REFERENCES source_ingestion_runs(source_ingestion_run_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_ingestion_manifest_dataset FOREIGN KEY (dataset_version) + REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_ingestion_manifest_raw FOREIGN KEY (raw_object_id) + REFERENCES storage_objects(storage_object_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_ingestion_manifest_validated FOREIGN KEY (validated_object_id) + REFERENCES storage_objects(storage_object_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_ingestion_manifest_curated FOREIGN KEY (curated_object_id) + REFERENCES storage_objects(storage_object_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_ingestion_manifest_identity CHECK ( + btrim(schema_version) <> '' AND btrim(source_name) <> '' + ), + CONSTRAINT ck_ingestion_manifest_counts CHECK ( + record_count >= 0 + AND accepted_count >= 0 + AND rejected_count >= 0 + AND quarantined_count >= 0 + AND retry_count >= 0 + AND accepted_count + rejected_count + quarantined_count <= record_count + ), + CONSTRAINT ck_ingestion_manifest_checksum CHECK ( + manifest_checksum ~ '^[a-f0-9]{64}$' + ) + ) + """, + """ + CREATE FUNCTION forbid_immutable_pipeline_record_mutation() RETURNS TRIGGER + LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'immutable pipeline records cannot be changed or deleted'; + END; + $$ + """, + """ + CREATE TRIGGER trg_ingestion_manifests_immutable + BEFORE UPDATE OR DELETE ON ingestion_run_manifests + FOR EACH ROW EXECUTE FUNCTION forbid_immutable_pipeline_record_mutation() + """, + """ + CREATE FUNCTION protect_immutable_storage_object() RETURNS TRIGGER + LANGUAGE plpgsql AS $$ + BEGIN + IF OLD.immutable OR OLD.zone = 'raw' THEN + RAISE EXCEPTION 'immutable storage objects cannot be changed or deleted'; + END IF; + RETURN OLD; + END; + $$ + """, + """ + CREATE TRIGGER trg_storage_objects_immutable + BEFORE UPDATE OR DELETE ON storage_objects + FOR EACH ROW EXECUTE FUNCTION protect_immutable_storage_object() + """, + """ + CREATE TABLE job_dispatch_outbox ( + job_dispatch_outbox_id UUID PRIMARY KEY, + job_run_id UUID NOT NULL UNIQUE, + queue_name TEXT NOT NULL, + message_body JSONB NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + publish_attempts INTEGER NOT NULL DEFAULT 0, + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + queue_message_id TEXT, + published_at TIMESTAMPTZ, + last_error_class TEXT, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT fk_job_dispatch_run FOREIGN KEY (job_run_id) + REFERENCES job_runs(job_run_id) + ON UPDATE CASCADE ON DELETE CASCADE, + CONSTRAINT ck_job_dispatch_queue CHECK (btrim(queue_name) <> ''), + CONSTRAINT ck_job_dispatch_status CHECK ( + status IN ('pending', 'publishing', 'published', 'failed', 'dead_lettered') + ), + CONSTRAINT ck_job_dispatch_attempts CHECK (publish_attempts >= 0), + CONSTRAINT ck_job_dispatch_published CHECK ( + (status = 'published' AND published_at IS NOT NULL AND queue_message_id IS NOT NULL) + OR status <> 'published' + ) + ) + """, + """ + CREATE INDEX ix_job_dispatch_due + ON job_dispatch_outbox (status, next_attempt_at, created_at) + """, + """ + CREATE TABLE worker_heartbeats ( + worker_id TEXT PRIMARY KEY, + queue_name TEXT NOT NULL, + job_run_id UUID, + status TEXT NOT NULL, + started_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + heartbeat_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + metadata JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT fk_worker_heartbeat_job FOREIGN KEY (job_run_id) + REFERENCES job_runs(job_run_id) + ON UPDATE CASCADE ON DELETE SET NULL, + CONSTRAINT ck_worker_heartbeat_identity CHECK ( + btrim(worker_id) <> '' AND btrim(queue_name) <> '' + ), + CONSTRAINT ck_worker_heartbeat_status CHECK ( + status IN ('idle', 'running', 'draining', 'stopped') + ) + ) + """, + """ + CREATE INDEX ix_worker_heartbeats_queue_time + ON worker_heartbeats (queue_name, heartbeat_at DESC, worker_id) + """, + """ + ALTER TABLE job_runs + ADD COLUMN queue_name TEXT NOT NULL DEFAULT 'pipeline', + ADD COLUMN heartbeat_at TIMESTAMPTZ, + ADD COLUMN lease_expires_at TIMESTAMPTZ, + ADD COLUMN timeout_seconds INTEGER NOT NULL DEFAULT 3600, + ADD COLUMN failure_reason TEXT, + ADD COLUMN last_good_dataset_version TEXT, + ADD CONSTRAINT fk_job_runs_last_good_dataset FOREIGN KEY ( + last_good_dataset_version + ) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE SET NULL, + ADD CONSTRAINT ck_job_runs_queue CHECK (btrim(queue_name) <> ''), + ADD CONSTRAINT ck_job_runs_timeout CHECK ( + timeout_seconds BETWEEN 1 AND 86400 + ), + ADD CONSTRAINT ck_job_runs_lease CHECK ( + lease_expires_at IS NULL OR heartbeat_at IS NOT NULL + ) + """, + """ + ALTER TABLE source_ingestion_runs + ADD COLUMN accepted_count BIGINT NOT NULL DEFAULT 0, + ADD COLUMN rejected_count BIGINT NOT NULL DEFAULT 0, + ADD COLUMN quarantined_count BIGINT NOT NULL DEFAULT 0, + ADD COLUMN retry_count INTEGER NOT NULL DEFAULT 0, + ADD COLUMN failure_reason TEXT, + ADD COLUMN watermark_before TEXT, + ADD COLUMN watermark_after TEXT, + ADD COLUMN last_good_dataset_version TEXT, + ADD CONSTRAINT fk_ingestion_last_good_dataset FOREIGN KEY ( + last_good_dataset_version + ) REFERENCES dataset_versions(dataset_version) + ON UPDATE CASCADE ON DELETE SET NULL, + ADD CONSTRAINT ck_ingestion_result_counts CHECK ( + accepted_count >= 0 + AND rejected_count >= 0 + AND quarantined_count >= 0 + AND retry_count >= 0 + AND ( + record_count IS NULL + OR accepted_count + rejected_count + quarantined_count <= record_count + ) + ) + """, + ) + for statement in statements: + op.execute(statement) + + +def downgrade() -> None: + op.execute("ALTER TABLE source_ingestion_runs DROP CONSTRAINT ck_ingestion_result_counts") + op.execute("ALTER TABLE source_ingestion_runs DROP CONSTRAINT fk_ingestion_last_good_dataset") + for column in ( + "last_good_dataset_version", + "watermark_after", + "watermark_before", + "failure_reason", + "retry_count", + "quarantined_count", + "rejected_count", + "accepted_count", + ): + op.execute(f"ALTER TABLE source_ingestion_runs DROP COLUMN {column}") + op.execute("ALTER TABLE job_runs DROP CONSTRAINT ck_job_runs_lease") + op.execute("ALTER TABLE job_runs DROP CONSTRAINT ck_job_runs_timeout") + op.execute("ALTER TABLE job_runs DROP CONSTRAINT ck_job_runs_queue") + op.execute("ALTER TABLE job_runs DROP CONSTRAINT fk_job_runs_last_good_dataset") + for column in ( + "last_good_dataset_version", + "failure_reason", + "timeout_seconds", + "lease_expires_at", + "heartbeat_at", + "queue_name", + ): + op.execute(f"ALTER TABLE job_runs DROP COLUMN {column}") + op.execute("DROP TABLE worker_heartbeats") + op.execute("DROP TABLE job_dispatch_outbox") + op.execute("DROP TRIGGER trg_ingestion_manifests_immutable ON ingestion_run_manifests") + op.execute("DROP TRIGGER trg_storage_objects_immutable ON storage_objects") + op.execute("DROP FUNCTION protect_immutable_storage_object()") + op.execute("DROP FUNCTION forbid_immutable_pipeline_record_mutation()") + op.execute("DROP TABLE ingestion_run_manifests") + op.execute("DROP TABLE storage_objects") + op.execute("DROP TABLE source_configurations") diff --git a/alembic/versions/0006_governance_retention.py b/alembic/versions/0006_governance_retention.py new file mode 100644 index 0000000..c4847a9 --- /dev/null +++ b/alembic/versions/0006_governance_retention.py @@ -0,0 +1,296 @@ +"""Add governance, holds and non-destructive retention evidence. + +Revision ID: 0006_governance_retention +Revises: 0005_durable_pipeline +""" + +from alembic import op + + +revision = "0006_governance_retention" +down_revision = "0005_durable_pipeline" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + statements = ( + """ + CREATE TABLE retention_policies ( + retention_class TEXT PRIMARY KEY, + classification TEXT NOT NULL, + policy_status TEXT NOT NULL DEFAULT 'proposed', + archive_after_days INTEGER, + delete_after_days INTEGER, + export_expire_after_days INTEGER, + destructive_deletion_enabled BOOLEAN NOT NULL DEFAULT FALSE, + production_approved BOOLEAN NOT NULL DEFAULT FALSE, + approval_reference TEXT, + configuration_checksum CHAR(64) NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT ck_retention_policy_identity CHECK ( + btrim(retention_class) <> '' AND btrim(classification) <> '' + ), + CONSTRAINT ck_retention_policy_status CHECK ( + policy_status IN ('proposed', 'approved', 'retired') + ), + CONSTRAINT ck_retention_policy_windows CHECK ( + (archive_after_days IS NULL OR archive_after_days >= 1) + AND (delete_after_days IS NULL OR delete_after_days >= 1) + AND (export_expire_after_days IS NULL OR export_expire_after_days >= 1) + ), + CONSTRAINT ck_retention_policy_deletion_approval CHECK ( + NOT destructive_deletion_enabled OR ( + production_approved + AND policy_status='approved' + AND approval_reference IS NOT NULL + AND delete_after_days IS NOT NULL + ) + ), + CONSTRAINT ck_retention_policy_checksum CHECK ( + configuration_checksum ~ '^[a-f0-9]{64}$' + ) + ) + """, + """ + INSERT INTO retention_policies ( + retention_class, classification, policy_status, + destructive_deletion_enabled, production_approved, + configuration_checksum + ) VALUES ( + 'exports', 'exports', 'proposed', FALSE, FALSE, + repeat('0', 64) + ) + """, + """ + CREATE TABLE data_holds ( + data_hold_id UUID PRIMARY KEY, + hold_type TEXT NOT NULL, + scope_type TEXT NOT NULL, + scope_id TEXT NOT NULL, + reason TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'active', + created_by TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMPTZ, + released_by TEXT, + released_at TIMESTAMPTZ, + release_reason TEXT, + CONSTRAINT ck_data_hold_type CHECK ( + hold_type IN ('legal', 'incident') + ), + CONSTRAINT ck_data_hold_scope CHECK ( + btrim(scope_type) <> '' AND btrim(scope_id) <> '' + ), + CONSTRAINT ck_data_hold_reason CHECK ( + btrim(reason) <> '' AND btrim(created_by) <> '' + ), + CONSTRAINT ck_data_hold_status CHECK ( + status IN ('active', 'released', 'expired') + ), + CONSTRAINT ck_data_hold_release CHECK ( + (status='active' AND released_at IS NULL AND released_by IS NULL) + OR (status IN ('released', 'expired') + AND released_at IS NOT NULL AND released_by IS NOT NULL) + ), + CONSTRAINT ck_data_hold_expiry CHECK ( + expires_at IS NULL OR expires_at > created_at + ) + ) + """, + """ + CREATE INDEX ix_data_holds_active_scope + ON data_holds (scope_type, scope_id, hold_type, created_at DESC) + WHERE status='active' + """, + """ + CREATE TABLE restore_verifications ( + restore_verification_id UUID PRIMARY KEY, + target_type TEXT NOT NULL, + target_id TEXT NOT NULL, + backup_reference TEXT NOT NULL, + backup_checksum CHAR(64) NOT NULL, + verification_status TEXT NOT NULL, + verified_by TEXT NOT NULL, + verified_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + evidence JSONB NOT NULL DEFAULT '{}'::jsonb, + CONSTRAINT ck_restore_verification_target CHECK ( + btrim(target_type) <> '' AND btrim(target_id) <> '' + ), + CONSTRAINT ck_restore_verification_backup CHECK ( + btrim(backup_reference) <> '' + AND backup_checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_restore_verification_status CHECK ( + verification_status IN ('passed', 'failed') + ), + CONSTRAINT ck_restore_verification_actor CHECK (btrim(verified_by) <> '') + ) + """, + """ + CREATE TABLE deletion_manifests ( + deletion_manifest_id UUID PRIMARY KEY, + retention_action_id UUID NOT NULL UNIQUE, + target_type TEXT NOT NULL, + target_id TEXT NOT NULL, + retention_class TEXT NOT NULL, + action_type TEXT NOT NULL, + dry_run BOOLEAN NOT NULL DEFAULT TRUE, + object_count BIGINT NOT NULL DEFAULT 0, + record_count BIGINT NOT NULL DEFAULT 0, + total_bytes BIGINT NOT NULL DEFAULT 0, + target_checksums JSONB NOT NULL DEFAULT '[]'::jsonb, + hold_ids UUID[] NOT NULL DEFAULT '{}'::uuid[], + restore_verification_id UUID, + authorised_by TEXT, + manifest_checksum CHAR(64) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + manifest JSONB NOT NULL, + CONSTRAINT fk_deletion_manifest_action FOREIGN KEY (retention_action_id) + REFERENCES retention_actions(retention_action_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_deletion_manifest_policy FOREIGN KEY (retention_class) + REFERENCES retention_policies(retention_class) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT fk_deletion_manifest_restore FOREIGN KEY (restore_verification_id) + REFERENCES restore_verifications(restore_verification_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + CONSTRAINT ck_deletion_manifest_target CHECK ( + btrim(target_type) <> '' AND btrim(target_id) <> '' + ), + CONSTRAINT ck_deletion_manifest_action CHECK ( + action_type IN ('report', 'archive', 'delete') + ), + CONSTRAINT ck_deletion_manifest_counts CHECK ( + object_count >= 0 AND record_count >= 0 AND total_bytes >= 0 + ), + CONSTRAINT ck_deletion_manifest_checksum CHECK ( + manifest_checksum ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_deletion_manifest_initial_safety CHECK ( + dry_run AND action_type <> 'delete' + ) + ) + """, + """ + CREATE TABLE data_subject_requests ( + data_subject_request_id UUID PRIMARY KEY, + request_type TEXT NOT NULL, + subject_reference_hash CHAR(64) NOT NULL, + status TEXT NOT NULL DEFAULT 'received', + identity_verified BOOLEAN NOT NULL DEFAULT FALSE, + received_at TIMESTAMPTZ NOT NULL DEFAULT CURRENT_TIMESTAMP, + due_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + decision TEXT, + handled_by TEXT, + audit_reference TEXT, + CONSTRAINT ck_data_subject_request_type CHECK ( + request_type IN ('access', 'correction', 'deletion', 'restriction', 'objection') + ), + CONSTRAINT ck_data_subject_reference CHECK ( + subject_reference_hash ~ '^[a-f0-9]{64}$' + ), + CONSTRAINT ck_data_subject_status CHECK ( + status IN ('received', 'identity_pending', 'in_review', 'held', + 'completed', 'rejected', 'cancelled') + ), + CONSTRAINT ck_data_subject_completion CHECK ( + (status IN ('completed', 'rejected', 'cancelled') + AND completed_at IS NOT NULL AND decision IS NOT NULL) + OR (status NOT IN ('completed', 'rejected', 'cancelled') + AND completed_at IS NULL) + ) + ) + """, + """ + ALTER TABLE retention_actions + ADD COLUMN retention_class TEXT, + ADD COLUMN action_type TEXT NOT NULL DEFAULT 'report', + ADD COLUMN hold_ids UUID[] NOT NULL DEFAULT '{}'::uuid[], + ADD COLUMN restore_verification_id UUID, + ADD COLUMN manifest_checksum CHAR(64), + ADD COLUMN production_approved BOOLEAN NOT NULL DEFAULT FALSE, + ADD CONSTRAINT fk_retention_action_policy FOREIGN KEY (retention_class) + REFERENCES retention_policies(retention_class) + ON UPDATE CASCADE ON DELETE RESTRICT, + ADD CONSTRAINT fk_retention_action_restore FOREIGN KEY (restore_verification_id) + REFERENCES restore_verifications(restore_verification_id) + ON UPDATE CASCADE ON DELETE RESTRICT, + ADD CONSTRAINT ck_retention_action_type CHECK ( + action_type IN ('report', 'archive', 'delete') + ), + ADD CONSTRAINT ck_retention_action_manifest_checksum CHECK ( + manifest_checksum IS NULL OR manifest_checksum ~ '^[a-f0-9]{64}$' + ), + ADD CONSTRAINT ck_retention_action_destructive_safety CHECK ( + action_type <> 'delete' OR ( + NOT dry_run AND production_approved + AND approved_by IS NOT NULL AND approved_at IS NOT NULL + AND restore_verification_id IS NOT NULL + AND cardinality(hold_ids)=0 + ) + ) + """, + """ + ALTER TABLE export_jobs + ADD COLUMN retention_class TEXT NOT NULL DEFAULT 'exports', + ADD COLUMN hold_until TIMESTAMPTZ, + ADD COLUMN expiration_reported_at TIMESTAMPTZ, + ADD CONSTRAINT fk_export_retention_policy FOREIGN KEY (retention_class) + REFERENCES retention_policies(retention_class) + ON UPDATE CASCADE ON DELETE RESTRICT, + ADD CONSTRAINT ck_export_hold_until CHECK ( + hold_until IS NULL OR hold_until > requested_at + ) + """, + """ + CREATE FUNCTION forbid_governance_evidence_mutation() RETURNS TRIGGER + LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'governance evidence is append-only'; + END; + $$ + """, + """ + CREATE TRIGGER trg_restore_verifications_immutable + BEFORE UPDATE OR DELETE ON restore_verifications + FOR EACH ROW EXECUTE FUNCTION forbid_governance_evidence_mutation() + """, + """ + CREATE TRIGGER trg_deletion_manifests_immutable + BEFORE UPDATE OR DELETE ON deletion_manifests + FOR EACH ROW EXECUTE FUNCTION forbid_governance_evidence_mutation() + """, + ) + for statement in statements: + op.execute(statement) + + +def downgrade() -> None: + op.execute("DROP TRIGGER trg_deletion_manifests_immutable ON deletion_manifests") + op.execute("DROP TRIGGER trg_restore_verifications_immutable ON restore_verifications") + op.execute("DROP FUNCTION forbid_governance_evidence_mutation()") + op.execute("ALTER TABLE export_jobs DROP CONSTRAINT ck_export_hold_until") + op.execute("ALTER TABLE export_jobs DROP CONSTRAINT fk_export_retention_policy") + for column in ("expiration_reported_at", "hold_until", "retention_class"): + op.execute(f"ALTER TABLE export_jobs DROP COLUMN {column}") + op.execute("ALTER TABLE retention_actions DROP CONSTRAINT ck_retention_action_destructive_safety") + op.execute("ALTER TABLE retention_actions DROP CONSTRAINT ck_retention_action_manifest_checksum") + op.execute("ALTER TABLE retention_actions DROP CONSTRAINT ck_retention_action_type") + op.execute("ALTER TABLE retention_actions DROP CONSTRAINT fk_retention_action_restore") + op.execute("ALTER TABLE retention_actions DROP CONSTRAINT fk_retention_action_policy") + for column in ( + "production_approved", + "manifest_checksum", + "restore_verification_id", + "hold_ids", + "action_type", + "retention_class", + ): + op.execute(f"ALTER TABLE retention_actions DROP COLUMN {column}") + op.execute("DROP TABLE data_subject_requests") + op.execute("DROP TABLE deletion_manifests") + op.execute("DROP TABLE restore_verifications") + op.execute("DROP TABLE data_holds") + op.execute("DROP TABLE retention_policies") diff --git a/alembic/versions/__init__.py b/alembic/versions/__init__.py new file mode 100644 index 0000000..78a71cf --- /dev/null +++ b/alembic/versions/__init__.py @@ -0,0 +1 @@ +"""Foundation Intelligence PostgreSQL schema revisions.""" diff --git a/config/data-governance.json b/config/data-governance.json new file mode 100644 index 0000000..18f8b82 --- /dev/null +++ b/config/data-governance.json @@ -0,0 +1,252 @@ +{ + "configuration_version": "1", + "policy_status": "proposed", + "destructive_deletion_enabled": false, + "production_activation_approved": false, + "restore_before_delete_required": true, + "data_owners": { + "business_owner": "unassigned", + "data_owner": "unassigned", + "privacy_owner": "unassigned", + "legal_owner": "unassigned", + "security_owner": "unassigned", + "technical_owner": "data-platform" + }, + "classifications": [ + { + "classification": "public_organisation_data", + "retention_class": "public_organisation_data", + "exposure": "authenticated", + "archive_after_days": 365, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "contact_data", + "retention_class": "contact_data", + "exposure": "restricted", + "archive_after_days": 90, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "personal_email_addresses", + "retention_class": "personal_email_addresses", + "exposure": "restricted", + "archive_after_days": 30, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "postal_addresses", + "retention_class": "postal_addresses", + "exposure": "restricted", + "archive_after_days": 90, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "raw_source_evidence", + "retention_class": "raw_source_evidence", + "exposure": "internal", + "archive_after_days": 30, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "article_metadata", + "retention_class": "article_metadata", + "exposure": "authenticated", + "archive_after_days": 30, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "article_content", + "retention_class": "ephemeral_article_content", + "exposure": "internal", + "archive_after_days": 1, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "pipeline_logs", + "retention_class": "pipeline_logs", + "exposure": "restricted", + "archive_after_days": 30, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "exports", + "retention_class": "exports", + "exposure": "restricted", + "archive_after_days": 7, + "delete_after_days": null, + "export_expire_after_days": 7 + }, + { + "classification": "audit_events", + "retention_class": "audit_events", + "exposure": "restricted", + "archive_after_days": 365, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "derived_classifications", + "retention_class": "derived_classifications", + "exposure": "authenticated", + "archive_after_days": 365, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "enriched_profiles", + "retention_class": "derived_profile", + "exposure": "authenticated", + "archive_after_days": 365, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "credentials", + "retention_class": "credentials", + "exposure": "never", + "archive_after_days": null, + "delete_after_days": null, + "export_expire_after_days": null + }, + { + "classification": "user_identities", + "retention_class": "user_identities", + "exposure": "restricted", + "archive_after_days": 90, + "delete_after_days": null, + "export_expire_after_days": null + } + ], + "field_exposure_policies": { + "health": ["status", "service", "checks"], + "organization_summary": [ + "id", + "name", + "reg_charity_number", + "reg_status", + "organization_type", + "foundation_regions", + "funding_regions", + "tags", + "source" + ], + "pipeline_event": [ + "job_event_id", + "job_run_id", + "sequence", + "event_type", + "occurred_at", + "job_type", + "details" + ], + "job_history": [ + "job_run_id", + "job_type", + "status", + "dataset_version", + "requested_at", + "started_at", + "completed_at", + "attempt", + "max_attempts", + "error_class", + "error_message", + "queue_name", + "heartbeat_at", + "lease_expires_at", + "timeout_seconds", + "failure_reason", + "last_good_dataset_version" + ], + "source_configuration": [ + "source_name", + "source_owner", + "technical_owner", + "legal_status", + "licence_status", + "terms_url", + "rate_limit_per_minute", + "freshness_sla_hours", + "schedule_expression", + "enabled", + "governance_blocked", + "last_success_at", + "watermark", + "classification", + "retention_class", + "schema_version", + "retry_limit", + "timeout_seconds", + "maximum_pages", + "maximum_records" + ] + }, + "log_redaction": { + "sensitive_keys": [ + "authorization", + "cookie", + "password", + "passwd", + "token", + "secret", + "api_key", + "api-key", + "email", + "email_address", + "personal_email", + "address", + "postal_address", + "connection_string", + "database_url", + "article_body", + "raw_payload" + ], + "replacement": "[REDACTED]" + }, + "privacy_checklist": { + "lawful_basis_review": "unresolved", + "data_protection_impact_assessment": "not_started", + "data_subject_contact": "unassigned", + "source_terms_review": "unresolved", + "international_transfer_review": "unresolved", + "processor_register": "not_started", + "breach_response_owner": "unassigned" + }, + "backup_policy": { + "status": "proposed", + "encryption_required": true, + "restore_test_required": true, + "cross_account_copy": "unresolved", + "retention_days": null + }, + "pitr_policy": { + "status": "proposed", + "required": true, + "retention_days": null + }, + "service_recovery": { + "status": "unresolved", + "rto_hours": null, + "rpo_hours": null + }, + "data_subject_workflow": [ + "receive_request", + "verify_identity", + "locate_scoped_records", + "check_legal_and_incident_holds", + "review_source_and_legal_constraints", + "approve_or_reject", + "execute_only_after_separate_authorisation", + "record_audit_evidence", + "notify_requester" + ] +} diff --git a/config/golden/api-contract.json b/config/golden/api-contract.json new file mode 100644 index 0000000..881276b --- /dev/null +++ b/config/golden/api-contract.json @@ -0,0 +1,25 @@ +{ + "health": { + "legacy": {"status": "healthy", "service": "bff"}, + "live": {"status": "healthy", "service": "bff"}, + "ready": { + "status": "ready", + "checks": { + "postgresql": "healthy", + "schema_version": "healthy", + "active_dataset": "healthy", + "critical_configuration": "healthy", + "queue": "healthy" + } + } + }, + "required_paths": [ + "/health/live", + "/health/ready", + "/api/charities", + "/api/charities/grants/overview", + "/api/charities/grants/map", + "/api/charities/directory/organizations", + "/api/admin/pipeline/status" + ] +} diff --git a/config/golden/transition-domain.json b/config/golden/transition-domain.json new file mode 100644 index 0000000..d8b90db --- /dev/null +++ b/config/golden/transition-domain.json @@ -0,0 +1,4301 @@ +{ + "fixture_version": "1", + "non_database_journeys": { + "manual_refresh_permissions": "operator_role_and_idempotency_required", + "news": "live_network_calls_disabled_in_local_acceptance", + "pipeline_status": "durable_postgresql_job_status_contract" + }, + "semantic_contract": { + "country_filter": [ + { + "grant_count": 101127, + "total_eur_minor": 460419494571 + } + ], + "country_totals": [ + { + "country_code": "GB", + "country_name": "United Kingdom", + "grant_count": 101127, + "single_country_eur_minor": 460419494571 + }, + { + "country_code": "US", + "country_name": "United States", + "grant_count": 600, + "single_country_eur_minor": 4883920784 + }, + { + "country_code": "NG", + "country_name": "Nigeria", + "grant_count": 318, + "single_country_eur_minor": 319453244 + }, + { + "country_code": "KE", + "country_name": "Kenya", + "grant_count": 275, + "single_country_eur_minor": 3414700436 + }, + { + "country_code": "UG", + "country_name": "Uganda", + "grant_count": 263, + "single_country_eur_minor": 3013649323 + }, + { + "country_code": "ZA", + "country_name": "South Africa", + "grant_count": 237, + "single_country_eur_minor": 2473174492 + }, + { + "country_code": "IN", + "country_name": "India", + "grant_count": 106, + "single_country_eur_minor": 878513677 + }, + { + "country_code": "ZW", + "country_name": "Zimbabwe", + "grant_count": 105, + "single_country_eur_minor": 993195007 + }, + { + "country_code": "TZ", + "country_name": "Tanzania", + "grant_count": 86, + "single_country_eur_minor": 4801094023 + }, + { + "country_code": "LK", + "country_name": "Sri Lanka", + "grant_count": 84, + "single_country_eur_minor": 26842936 + }, + { + "country_code": "MW", + "country_name": "Malawi", + "grant_count": 84, + "single_country_eur_minor": 1381182712 + }, + { + "country_code": "ZM", + "country_name": "Zambia", + "grant_count": 80, + "single_country_eur_minor": 1813222781 + }, + { + "country_code": "GH", + "country_name": "Ghana", + "grant_count": 74, + "single_country_eur_minor": 2563914554 + }, + { + "country_code": "IL", + "country_name": "Israel", + "grant_count": 69, + "single_country_eur_minor": 1536757316 + }, + { + "country_code": "SL", + "country_name": "Sierra Leone", + "grant_count": 54, + "single_country_eur_minor": 1672575227 + }, + { + "country_code": "RW", + "country_name": "Rwanda", + "grant_count": 40, + "single_country_eur_minor": 1071797520 + }, + { + "country_code": "CA", + "country_name": "Canada", + "grant_count": 39, + "single_country_eur_minor": 78053300 + }, + { + "country_code": "ET", + "country_name": "Ethiopia", + "grant_count": 35, + "single_country_eur_minor": 1135350127 + }, + { + "country_code": "NP", + "country_name": "Nepal", + "grant_count": 35, + "single_country_eur_minor": 259675618 + }, + { + "country_code": "IS", + "country_name": "Iceland", + "grant_count": 34, + "single_country_eur_minor": 473356008 + }, + { + "country_code": "NL", + "country_name": "Netherlands", + "grant_count": 25, + "single_country_eur_minor": 1063677086 + }, + { + "country_code": "PK", + "country_name": "Pakistan", + "grant_count": 25, + "single_country_eur_minor": 18611076 + }, + { + "country_code": "IE", + "country_name": "Ireland", + "grant_count": 24, + "single_country_eur_minor": 85129164 + }, + { + "country_code": "GR", + "country_name": "Greece", + "grant_count": 21, + "single_country_eur_minor": 111216421 + }, + { + "country_code": "AF", + "country_name": "Afghanistan", + "grant_count": 20, + "single_country_eur_minor": 82861105 + }, + { + "country_code": "CM", + "country_name": "Cameroon", + "grant_count": 20, + "single_country_eur_minor": 62759674 + }, + { + "country_code": "BD", + "country_name": "Bangladesh", + "grant_count": 19, + "single_country_eur_minor": 508772266 + }, + { + "country_code": "RO", + "country_name": "Romania", + "grant_count": 18, + "single_country_eur_minor": 12643101 + }, + { + "country_code": "BR", + "country_name": "Brazil", + "grant_count": 17, + "single_country_eur_minor": 327752371 + }, + { + "country_code": "FR", + "country_name": "France", + "grant_count": 17, + "single_country_eur_minor": 37433803 + }, + { + "country_code": "ES", + "country_name": "Spain", + "grant_count": 16, + "single_country_eur_minor": 11780085 + }, + { + "country_code": "DE", + "country_name": "Germany", + "grant_count": 14, + "single_country_eur_minor": 98797447 + }, + { + "country_code": "IT", + "country_name": "Italy", + "grant_count": 14, + "single_country_eur_minor": 37860522 + }, + { + "country_code": "CD", + "country_name": "Democratic Republic of the Congo", + "grant_count": 13, + "single_country_eur_minor": 352370008 + }, + { + "country_code": "ID", + "country_name": "Indonesia", + "grant_count": 12, + "single_country_eur_minor": 37156714 + }, + { + "country_code": "LR", + "country_name": "Liberia", + "grant_count": 11, + "single_country_eur_minor": 562909109 + }, + { + "country_code": "LS", + "country_name": "Lesotho", + "grant_count": 10, + "single_country_eur_minor": 7435269 + }, + { + "country_code": "MG", + "country_name": "Madagascar", + "grant_count": 10, + "single_country_eur_minor": 62235687 + }, + { + "country_code": "MY", + "country_name": "Malaysia", + "grant_count": 10, + "single_country_eur_minor": 7088124 + }, + { + "country_code": "LB", + "country_name": "Lebanon", + "grant_count": 9, + "single_country_eur_minor": 26512804 + }, + { + "country_code": "AE", + "country_name": "United Arab Emirates", + "grant_count": 8, + "single_country_eur_minor": 7479414 + }, + { + "country_code": "JM", + "country_name": "Jamaica", + "grant_count": 8, + "single_country_eur_minor": 340400679 + }, + { + "country_code": "SS", + "country_name": "South Sudan", + "grant_count": 8, + "single_country_eur_minor": 193169388 + }, + { + "country_code": "MM", + "country_name": "Myanmar", + "grant_count": 7, + "single_country_eur_minor": 256842365 + }, + { + "country_code": "MZ", + "country_name": "Mozambique", + "grant_count": 7, + "single_country_eur_minor": 507564963 + }, + { + "country_code": "PH", + "country_name": "Philippines", + "grant_count": 7, + "single_country_eur_minor": 36060779 + }, + { + "country_code": "SN", + "country_name": "Senegal", + "grant_count": 7, + "single_country_eur_minor": 217481812 + }, + { + "country_code": "SY", + "country_name": "Syria", + "grant_count": 7, + "single_country_eur_minor": 55060016 + }, + { + "country_code": "CO", + "country_name": "Colombia", + "grant_count": 6, + "single_country_eur_minor": 135776664 + }, + { + "country_code": "HR", + "country_name": "Croatia", + "grant_count": 6, + "single_country_eur_minor": 4024234 + }, + { + "country_code": "MO", + "country_name": "Macao", + "grant_count": 6, + "single_country_eur_minor": 1391174 + }, + { + "country_code": "NZ", + "country_name": "New Zealand", + "grant_count": 6, + "single_country_eur_minor": 8044335 + }, + { + "country_code": "PE", + "country_name": "Peru", + "grant_count": 6, + "single_country_eur_minor": 7982772 + }, + { + "country_code": "SO", + "country_name": "Somalia", + "grant_count": 6, + "single_country_eur_minor": 138191070 + }, + { + "country_code": "TG", + "country_name": "Togo", + "grant_count": 6, + "single_country_eur_minor": 111572094 + }, + { + "country_code": "AU", + "country_name": "Australia", + "grant_count": 5, + "single_country_eur_minor": 4505456 + }, + { + "country_code": "BW", + "country_name": "Botswana", + "grant_count": 5, + "single_country_eur_minor": 2419182 + }, + { + "country_code": "CH", + "country_name": "Switzerland", + "grant_count": 5, + "single_country_eur_minor": 64923399 + }, + { + "country_code": "IM", + "country_name": "Isle of Man", + "grant_count": 5, + "single_country_eur_minor": 856834 + }, + { + "country_code": "IQ", + "country_name": "Iraq", + "grant_count": 5, + "single_country_eur_minor": 26327131 + }, + { + "country_code": "JO", + "country_name": "Jordan", + "grant_count": 5, + "single_country_eur_minor": 88062244 + }, + { + "country_code": "PS", + "country_name": "Palestine", + "grant_count": 5, + "single_country_eur_minor": 12875756 + }, + { + "country_code": "SD", + "country_name": "Sudan", + "grant_count": 5, + "single_country_eur_minor": 113679799 + }, + { + "country_code": "TH", + "country_name": "Thailand", + "grant_count": 5, + "single_country_eur_minor": 22018116 + }, + { + "country_code": "BE", + "country_name": "Belgium", + "grant_count": 4, + "single_country_eur_minor": 140913612 + }, + { + "country_code": "GM", + "country_name": "Gambia", + "grant_count": 4, + "single_country_eur_minor": 4340986 + }, + { + "country_code": "GT", + "country_name": "Guatemala", + "grant_count": 4, + "single_country_eur_minor": 11559630 + }, + { + "country_code": "MA", + "country_name": "Morocco", + "grant_count": 4, + "single_country_eur_minor": 6510941 + }, + { + "country_code": "QA", + "country_name": "Qatar", + "grant_count": 4, + "single_country_eur_minor": 8514469 + }, + { + "country_code": "BJ", + "country_name": "Benin", + "grant_count": 3, + "single_country_eur_minor": 4782273 + }, + { + "country_code": "CN", + "country_name": "China", + "grant_count": 3, + "single_country_eur_minor": 673248 + }, + { + "country_code": "HT", + "country_name": "Haiti", + "grant_count": 3, + "single_country_eur_minor": 11144700 + }, + { + "country_code": "ML", + "country_name": "Mali", + "grant_count": 3, + "single_country_eur_minor": 194292255 + }, + { + "country_code": "SE", + "country_name": "Sweden", + "grant_count": 3, + "single_country_eur_minor": 613964 + }, + { + "country_code": "AR", + "country_name": "Argentina", + "grant_count": 2, + "single_country_eur_minor": 4675392 + }, + { + "country_code": "BB", + "country_name": "Barbados", + "grant_count": 2, + "single_country_eur_minor": 2013953 + }, + { + "country_code": "BF", + "country_name": "Burkina Faso", + "grant_count": 2, + "single_country_eur_minor": 66270931 + }, + { + "country_code": "BO", + "country_name": "Bolivia", + "grant_count": 2, + "single_country_eur_minor": 233743 + }, + { + "country_code": "DK", + "country_name": "Denmark", + "grant_count": 2, + "single_country_eur_minor": 5993977 + }, + { + "country_code": "GN", + "country_name": "Guinea", + "grant_count": 2, + "single_country_eur_minor": 4899580 + }, + { + "country_code": "HK", + "country_name": "Hong Kong", + "grant_count": 2, + "single_country_eur_minor": 457047 + }, + { + "country_code": "HN", + "country_name": "Honduras", + "grant_count": 2, + "single_country_eur_minor": 13541713 + }, + { + "country_code": "KH", + "country_name": "Cambodia", + "grant_count": 2, + "single_country_eur_minor": 6931158 + }, + { + "country_code": "LY", + "country_name": "Libya", + "grant_count": 2, + "single_country_eur_minor": 3133762 + }, + { + "country_code": "NA", + "country_name": "Namibia", + "grant_count": 2, + "single_country_eur_minor": 105531990 + }, + { + "country_code": "PG", + "country_name": "Papua New Guinea", + "grant_count": 2, + "single_country_eur_minor": 3967858 + }, + { + "country_code": "PL", + "country_name": "Poland", + "grant_count": 2, + "single_country_eur_minor": 1476226 + }, + { + "country_code": "RU", + "country_name": "Russia", + "grant_count": 2, + "single_country_eur_minor": 1504183 + }, + { + "country_code": "SB", + "country_name": "Solomon Islands", + "grant_count": 2, + "single_country_eur_minor": 224391117 + }, + { + "country_code": "SK", + "country_name": "Slovakia", + "grant_count": 2, + "single_country_eur_minor": 1638074 + }, + { + "country_code": "TR", + "country_name": "Türkiye", + "grant_count": 2, + "single_country_eur_minor": 2258611 + }, + { + "country_code": "UA", + "country_name": "Ukraine", + "grant_count": 2, + "single_country_eur_minor": 6024800 + }, + { + "country_code": "YE", + "country_name": "Yemen", + "grant_count": 2, + "single_country_eur_minor": 146126496 + }, + { + "country_code": "BA", + "country_name": "Bosnia and Herzegovina", + "grant_count": 1, + "single_country_eur_minor": 31023250 + }, + { + "country_code": "BI", + "country_name": "Burundi", + "grant_count": 1, + "single_country_eur_minor": 240758 + }, + { + "country_code": "CG", + "country_name": "Republic of the Congo", + "grant_count": 1, + "single_country_eur_minor": 0 + }, + { + "country_code": "CL", + "country_name": "Chile", + "grant_count": 1, + "single_country_eur_minor": 0 + }, + { + "country_code": "CU", + "country_name": "Cuba", + "grant_count": 1, + "single_country_eur_minor": 787777 + }, + { + "country_code": "DM", + "country_name": "Dominica", + "grant_count": 1, + "single_country_eur_minor": 0 + }, + { + "country_code": "DO", + "country_name": "Dominican Republic", + "grant_count": 1, + "single_country_eur_minor": 32185891 + }, + { + "country_code": "EC", + "country_name": "Ecuador", + "grant_count": 1, + "single_country_eur_minor": 307337 + }, + { + "country_code": "EG", + "country_name": "Egypt", + "grant_count": 1, + "single_country_eur_minor": 44712613 + }, + { + "country_code": "FJ", + "country_name": "Fiji", + "grant_count": 1, + "single_country_eur_minor": 1174393 + }, + { + "country_code": "HU", + "country_name": "Hungary", + "grant_count": 1, + "single_country_eur_minor": 953477 + }, + { + "country_code": "LA", + "country_name": "Laos", + "grant_count": 1, + "single_country_eur_minor": 0 + }, + { + "country_code": "SG", + "country_name": "Singapore", + "grant_count": 1, + "single_country_eur_minor": 232901 + }, + { + "country_code": "SV", + "country_name": "El Salvador", + "grant_count": 1, + "single_country_eur_minor": 4228123 + }, + { + "country_code": "TD", + "country_name": "Chad", + "grant_count": 1, + "single_country_eur_minor": 5754101 + }, + { + "country_code": "TJ", + "country_name": "Tajikistan", + "grant_count": 1, + "single_country_eur_minor": 1168046 + }, + { + "country_code": "TT", + "country_name": "Trinidad and Tobago", + "grant_count": 1, + "single_country_eur_minor": 184474 + }, + { + "country_code": "VC", + "country_name": "Saint Vincent and the Grenadines", + "grant_count": 1, + "single_country_eur_minor": 160084 + }, + { + "country_code": "VE", + "country_name": "Venezuela", + "grant_count": 1, + "single_country_eur_minor": 2232858 + }, + { + "country_code": "VN", + "country_name": "Vietnam", + "grant_count": 1, + "single_country_eur_minor": 4355589 + } + ], + "currency_statuses": [ + { + "conversion_status": "ecb_monthly_average", + "currency": "CAD", + "grant_count": 7 + }, + { + "conversion_status": "ecb_monthly_average", + "currency": "CHF", + "grant_count": 10 + }, + { + "conversion_status": "native_eur", + "currency": "EUR", + "grant_count": 65 + }, + { + "conversion_status": "ecb_monthly_average", + "currency": "GBP", + "grant_count": 300388 + }, + { + "conversion_status": "unavailable_missing_rate", + "currency": "GBP", + "grant_count": 419 + }, + { + "conversion_status": "ecb_monthly_average", + "currency": "ILS", + "grant_count": 23 + }, + { + "conversion_status": "unavailable_missing_rate", + "currency": "KES", + "grant_count": 10 + }, + { + "conversion_status": "unavailable_missing_rate", + "currency": "TZS", + "grant_count": 1 + }, + { + "conversion_status": "ecb_monthly_average", + "currency": "USD", + "grant_count": 1615 + }, + { + "conversion_status": "unavailable_missing_rate", + "currency": "USD", + "grant_count": 2 + }, + { + "conversion_status": "ecb_monthly_average", + "currency": "ZAR", + "grant_count": 6 + } + ], + "date_filter": [ + { + "grant_count": 174384, + "total_eur_minor": 1048824524416 + } + ], + "donor_filter": [ + { + "grant_count": 52395, + "total_eur_minor": 809092991717 + } + ], + "donor_rankings": [ + { + "grant_count": 4895, + "identity": "name:the wellcome trust", + "name": "The Wellcome Trust", + "total_eur_minor": 597753032612 + }, + { + "grant_count": 3291, + "identity": "name:greater london authority", + "name": "Greater London Authority", + "total_eur_minor": 164583113436 + }, + { + "grant_count": 490, + "identity": "name:arcadia", + "name": "ARCADIA", + "total_eur_minor": 111424747780 + }, + { + "grant_count": 7085, + "identity": "name:esmée fairbairn foundation", + "name": "Esmée Fairbairn Foundation", + "total_eur_minor": 89011002522 + }, + { + "grant_count": 1477, + "identity": "326568", + "name": "Comic Relief", + "total_eur_minor": 59488868401 + }, + { + "grant_count": 4613, + "identity": "name:city bridge foundation", + "name": "City Bridge Foundation", + "total_eur_minor": 57545867986 + }, + { + "grant_count": 4198, + "identity": "1102927", + "name": "Paul Hamlyn Foundation", + "total_eur_minor": 53312080222 + }, + { + "grant_count": 230, + "identity": "251988", + "name": "Gatsby Charitable Foundation", + "total_eur_minor": 48321345644 + }, + { + "grant_count": 6551, + "identity": "name:henry smith foundation", + "name": "Henry Smith Foundation", + "total_eur_minor": 46381309659 + }, + { + "grant_count": 1761, + "identity": "name:guy s and st thomas charity", + "name": "Guy's and St Thomas' Charity", + "total_eur_minor": 44338695552 + }, + { + "grant_count": 7802, + "identity": "name:garfield weston foundation", + "name": "Garfield Weston Foundation", + "total_eur_minor": 42035293598 + }, + { + "grant_count": 3303, + "identity": "name:wolfson foundation", + "name": "Wolfson Foundation", + "total_eur_minor": 39671218857 + }, + { + "grant_count": 2019, + "identity": "name:sport england", + "name": "Sport England", + "total_eur_minor": 38669280434 + }, + { + "grant_count": 6375, + "identity": "name:bbc children in need", + "name": "BBC Children in Need", + "total_eur_minor": 31579784784 + }, + { + "grant_count": 4260, + "identity": "name:northern rock foundation", + "name": "Northern Rock Foundation", + "total_eur_minor": 29439969295 + }, + { + "grant_count": 520, + "identity": "name:lloyd s register foundation", + "name": "Lloyd's Register Foundation", + "total_eur_minor": 27997282471 + }, + { + "grant_count": 1991, + "identity": "name:cabinet office", + "name": "Cabinet Office", + "total_eur_minor": 23524270619 + }, + { + "grant_count": 1213, + "identity": "name:pears foundation", + "name": "Pears Foundation", + "total_eur_minor": 21331224047 + }, + { + "grant_count": 150, + "identity": "name:local trust", + "name": "Local Trust", + "total_eur_minor": 20875238600 + }, + { + "grant_count": 2370, + "identity": "name:wales council for voluntary action", + "name": "Wales Council for Voluntary Action", + "total_eur_minor": 19758669944 + }, + { + "grant_count": 3475, + "identity": "name:the tudor trust", + "name": "The Tudor Trust", + "total_eur_minor": 19010924782 + }, + { + "grant_count": 3765, + "identity": "name:the national lottery community fund", + "name": "The National Lottery Community Fund", + "total_eur_minor": 18216909889 + }, + { + "grant_count": 3944, + "identity": "name:lloyds bank foundation for england and wales", + "name": "Lloyds Bank Foundation for England and Wales", + "total_eur_minor": 16493606106 + }, + { + "grant_count": 411, + "identity": "name:the julia rausing trust", + "name": "The Julia Rausing Trust", + "total_eur_minor": 15212282424 + }, + { + "grant_count": 222, + "identity": "name:the national lottery heritage fund", + "name": "The National Lottery Heritage Fund", + "total_eur_minor": 15090635787 + } + ], + "grant_list": [ + { + "amount_minor": 5600891, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "0", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 4877300, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1110", + "recipient_name": "Alcohol Change UK" + }, + { + "amount_minor": 4932000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1118", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 1762100, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1122", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 4558900, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1128", + "recipient_name": "Hear Us" + }, + { + "amount_minor": 4000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1131", + "recipient_name": "Time & Talents" + }, + { + "amount_minor": 4944400, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1137", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 3000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1141", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 5174400, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1144", + "recipient_name": "King's College London" + }, + { + "amount_minor": 400000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1157", + "recipient_name": "King's College London" + }, + { + "amount_minor": 1000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1500", + "recipient_name": "King's College London" + }, + { + "amount_minor": 2200000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1501", + "recipient_name": "Re-Instate Ltd" + }, + { + "amount_minor": 4788500, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1502", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 4200000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1503", + "recipient_name": "Good Vibrations" + }, + { + "amount_minor": 1000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1504", + "recipient_name": "Mental Fight Club" + }, + { + "amount_minor": 4510500, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1505", + "recipient_name": "Beat" + }, + { + "amount_minor": 2000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1506", + "recipient_name": "Hear Us" + }, + { + "amount_minor": 3600000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1507", + "recipient_name": "Greenwich & Lewisham Young People’s Theatre" + }, + { + "amount_minor": 4950000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1508", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + }, + { + "amount_minor": 5000000, + "currency": "GBP", + "funding_name": "Maudsley Charity", + "grant_id": "1509", + "recipient_name": "South London and Maudsley NHS Foundation Trust" + } + ], + "map_relationships": [ + { + "destination_country_code": "NG", + "destination_country_name": "Nigeria", + "grant_count": 309, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ZA", + "destination_country_name": "South Africa", + "grant_count": 154, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "US", + "destination_country_name": "United States", + "grant_count": 134, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "UG", + "destination_country_name": "Uganda", + "grant_count": 101, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "KE", + "destination_country_name": "Kenya", + "grant_count": 100, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "LK", + "destination_country_name": "Sri Lanka", + "grant_count": 78, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "IN", + "destination_country_name": "India", + "grant_count": 68, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "TZ", + "destination_country_name": "Tanzania", + "grant_count": 54, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MW", + "destination_country_name": "Malawi", + "grant_count": 52, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "GH", + "destination_country_name": "Ghana", + "grant_count": 50, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ZM", + "destination_country_name": "Zambia", + "grant_count": 49, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SL", + "destination_country_name": "Sierra Leone", + "grant_count": 40, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CA", + "destination_country_name": "Canada", + "grant_count": 27, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "RW", + "destination_country_name": "Rwanda", + "grant_count": 26, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ZW", + "destination_country_name": "Zimbabwe", + "grant_count": 22, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "PK", + "destination_country_name": "Pakistan", + "grant_count": 21, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "GR", + "destination_country_name": "Greece", + "grant_count": 19, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ET", + "destination_country_name": "Ethiopia", + "grant_count": 18, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "NL", + "destination_country_name": "Netherlands", + "grant_count": 18, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ES", + "destination_country_name": "Spain", + "grant_count": 15, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "FR", + "destination_country_name": "France", + "grant_count": 15, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "IE", + "destination_country_name": "Ireland", + "grant_count": 15, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CM", + "destination_country_name": "Cameroon", + "grant_count": 14, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "NP", + "destination_country_name": "Nepal", + "grant_count": 14, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "RO", + "destination_country_name": "Romania", + "grant_count": 14, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BR", + "destination_country_name": "Brazil", + "grant_count": 10, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "LR", + "destination_country_name": "Liberia", + "grant_count": 10, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MY", + "destination_country_name": "Malaysia", + "grant_count": 10, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BD", + "destination_country_name": "Bangladesh", + "grant_count": 9, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "AE", + "destination_country_name": "United Arab Emirates", + "grant_count": 8, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "JM", + "destination_country_name": "Jamaica", + "grant_count": 8, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ID", + "destination_country_name": "Indonesia", + "grant_count": 7, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MZ", + "destination_country_name": "Mozambique", + "grant_count": 7, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "IT", + "destination_country_name": "Italy", + "grant_count": 6, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MO", + "destination_country_name": "Macao", + "grant_count": 6, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SN", + "destination_country_name": "Senegal", + "grant_count": 6, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "AU", + "destination_country_name": "Australia", + "grant_count": 5, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CD", + "destination_country_name": "Democratic Republic of the Congo", + "grant_count": 5, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "DE", + "destination_country_name": "Germany", + "grant_count": 5, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "LB", + "destination_country_name": "Lebanon", + "grant_count": 5, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "NZ", + "destination_country_name": "New Zealand", + "grant_count": 5, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CO", + "destination_country_name": "Colombia", + "grant_count": 4, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "JO", + "destination_country_name": "Jordan", + "grant_count": 4, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "QA", + "destination_country_name": "Qatar", + "grant_count": 4, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "TG", + "destination_country_name": "Togo", + "grant_count": 4, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BJ", + "destination_country_name": "Benin", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BW", + "destination_country_name": "Botswana", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CH", + "destination_country_name": "Switzerland", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CN", + "destination_country_name": "China", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "IL", + "destination_country_name": "Israel", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MA", + "destination_country_name": "Morocco", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "ML", + "destination_country_name": "Mali", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "PE", + "destination_country_name": "Peru", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "PH", + "destination_country_name": "Philippines", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SO", + "destination_country_name": "Somalia", + "grant_count": 3, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "AR", + "destination_country_name": "Argentina", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BE", + "destination_country_name": "Belgium", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "GN", + "destination_country_name": "Guinea", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "HK", + "destination_country_name": "Hong Kong", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "LS", + "destination_country_name": "Lesotho", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MM", + "destination_country_name": "Myanmar", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "NA", + "destination_country_name": "Namibia", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "PL", + "destination_country_name": "Poland", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SB", + "destination_country_name": "Solomon Islands", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SD", + "destination_country_name": "Sudan", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SE", + "destination_country_name": "Sweden", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SK", + "destination_country_name": "Slovakia", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SS", + "destination_country_name": "South Sudan", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "YE", + "destination_country_name": "Yemen", + "grant_count": 2, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BF", + "destination_country_name": "Burkina Faso", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "BO", + "destination_country_name": "Bolivia", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CG", + "destination_country_name": "Republic of the Congo", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CL", + "destination_country_name": "Chile", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "CU", + "destination_country_name": "Cuba", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "DM", + "destination_country_name": "Dominica", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "DO", + "destination_country_name": "Dominican Republic", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "EG", + "destination_country_name": "Egypt", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "HR", + "destination_country_name": "Croatia", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "HU", + "destination_country_name": "Hungary", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "IQ", + "destination_country_name": "Iraq", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "KH", + "destination_country_name": "Cambodia", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "LA", + "destination_country_name": "Laos", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "MG", + "destination_country_name": "Madagascar", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "RU", + "destination_country_name": "Russia", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "SG", + "destination_country_name": "Singapore", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "TR", + "destination_country_name": "Türkiye", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "TT", + "destination_country_name": "Trinidad and Tobago", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + }, + { + "destination_country_code": "VC", + "destination_country_name": "Saint Vincent and the Grenadines", + "grant_count": 1, + "origin_country_code": "GB", + "origin_country_name": "United Kingdom" + } + ], + "monthly_trends": [ + { + "grant_count": 1, + "period": "1991-04", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1995-05", + "total_eur_minor": 0 + }, + { + "grant_count": 2, + "period": "1995-07", + "total_eur_minor": 0 + }, + { + "grant_count": 4, + "period": "1995-10", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1995-11", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1995-12", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1996-01", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1996-06", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1996-07", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1996-11", + "total_eur_minor": 0 + }, + { + "grant_count": 3, + "period": "1996-12", + "total_eur_minor": 0 + }, + { + "grant_count": 3, + "period": "1997-02", + "total_eur_minor": 0 + }, + { + "grant_count": 23, + "period": "1997-03", + "total_eur_minor": 0 + }, + { + "grant_count": 3, + "period": "1997-04", + "total_eur_minor": 0 + }, + { + "grant_count": 2, + "period": "1997-05", + "total_eur_minor": 0 + }, + { + "grant_count": 7, + "period": "1997-06", + "total_eur_minor": 0 + }, + { + "grant_count": 12, + "period": "1997-07", + "total_eur_minor": 0 + }, + { + "grant_count": 36, + "period": "1997-09", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1997-10", + "total_eur_minor": 0 + }, + { + "grant_count": 4, + "period": "1997-11", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1997-12", + "total_eur_minor": 0 + }, + { + "grant_count": 1, + "period": "1998-01", + "total_eur_minor": 0 + }, + { + "grant_count": 8, + "period": "1998-02", + "total_eur_minor": 0 + }, + { + "grant_count": 17, + "period": "1998-03", + "total_eur_minor": 0 + }, + { + "grant_count": 33, + "period": "1998-04", + "total_eur_minor": 0 + }, + { + "grant_count": 21, + "period": "1998-05", + "total_eur_minor": 0 + }, + { + "grant_count": 36, + "period": "1998-06", + "total_eur_minor": 0 + }, + { + "grant_count": 40, + "period": "1998-07", + "total_eur_minor": 0 + }, + { + "grant_count": 46, + "period": "1998-08", + "total_eur_minor": 0 + }, + { + "grant_count": 13, + "period": "1998-09", + "total_eur_minor": 0 + }, + { + "grant_count": 45, + "period": "1998-10", + "total_eur_minor": 0 + }, + { + "grant_count": 23, + "period": "1998-11", + "total_eur_minor": 0 + }, + { + "grant_count": 28, + "period": "1998-12", + "total_eur_minor": 0 + }, + { + "grant_count": 17, + "period": "1999-01", + "total_eur_minor": 30752235 + }, + { + "grant_count": 46, + "period": "1999-02", + "total_eur_minor": 226415782 + }, + { + "grant_count": 24, + "period": "1999-03", + "total_eur_minor": 383446852 + }, + { + "grant_count": 12, + "period": "1999-04", + "total_eur_minor": 11189167 + }, + { + "grant_count": 24, + "period": "1999-05", + "total_eur_minor": 107432197 + }, + { + "grant_count": 24, + "period": "1999-06", + "total_eur_minor": 15253884 + }, + { + "grant_count": 75, + "period": "1999-07", + "total_eur_minor": 132887823 + }, + { + "grant_count": 9, + "period": "1999-08", + "total_eur_minor": 161955932 + }, + { + "grant_count": 57, + "period": "1999-09", + "total_eur_minor": 267352315 + }, + { + "grant_count": 56, + "period": "1999-10", + "total_eur_minor": 538529781 + }, + { + "grant_count": 4, + "period": "1999-11", + "total_eur_minor": 251978679 + }, + { + "grant_count": 72, + "period": "1999-12", + "total_eur_minor": 345952817 + }, + { + "grant_count": 27, + "period": "2000-01", + "total_eur_minor": 197305113 + }, + { + "grant_count": 39, + "period": "2000-02", + "total_eur_minor": 210364104 + }, + { + "grant_count": 42, + "period": "2000-03", + "total_eur_minor": 255823660 + }, + { + "grant_count": 18, + "period": "2000-04", + "total_eur_minor": 17534964 + }, + { + "grant_count": 49, + "period": "2000-05", + "total_eur_minor": 192703172 + }, + { + "grant_count": 13, + "period": "2000-06", + "total_eur_minor": 12634244 + }, + { + "grant_count": 52, + "period": "2000-07", + "total_eur_minor": 215230941 + }, + { + "grant_count": 11, + "period": "2000-08", + "total_eur_minor": 33059371 + }, + { + "grant_count": 49, + "period": "2000-09", + "total_eur_minor": 462714461 + }, + { + "grant_count": 19, + "period": "2000-10", + "total_eur_minor": 1835594815 + }, + { + "grant_count": 27, + "period": "2000-11", + "total_eur_minor": 18006569 + }, + { + "grant_count": 56, + "period": "2000-12", + "total_eur_minor": 507594749 + }, + { + "grant_count": 22, + "period": "2001-01", + "total_eur_minor": 24667308 + }, + { + "grant_count": 43, + "period": "2001-02", + "total_eur_minor": 588725464 + }, + { + "grant_count": 36, + "period": "2001-03", + "total_eur_minor": 63819957 + }, + { + "grant_count": 51, + "period": "2001-04", + "total_eur_minor": 270913197 + }, + { + "grant_count": 28, + "period": "2001-05", + "total_eur_minor": 47387178 + }, + { + "grant_count": 45, + "period": "2001-06", + "total_eur_minor": 291244232 + }, + { + "grant_count": 30, + "period": "2001-07", + "total_eur_minor": 299037978 + }, + { + "grant_count": 57, + "period": "2001-08", + "total_eur_minor": 430756839 + }, + { + "grant_count": 23, + "period": "2001-09", + "total_eur_minor": 301252892 + }, + { + "grant_count": 73, + "period": "2001-10", + "total_eur_minor": 466213950 + }, + { + "grant_count": 35, + "period": "2001-11", + "total_eur_minor": 59139684 + }, + { + "grant_count": 63, + "period": "2001-12", + "total_eur_minor": 538917468 + }, + { + "grant_count": 23, + "period": "2002-01", + "total_eur_minor": 734821741 + }, + { + "grant_count": 83, + "period": "2002-02", + "total_eur_minor": 704053855 + }, + { + "grant_count": 48, + "period": "2002-03", + "total_eur_minor": 296419302 + }, + { + "grant_count": 35, + "period": "2002-04", + "total_eur_minor": 525775971 + }, + { + "grant_count": 91, + "period": "2002-05", + "total_eur_minor": 405123363 + }, + { + "grant_count": 49, + "period": "2002-06", + "total_eur_minor": 432576195 + }, + { + "grant_count": 68, + "period": "2002-07", + "total_eur_minor": 4965982392 + }, + { + "grant_count": 48, + "period": "2002-08", + "total_eur_minor": 87419273 + }, + { + "grant_count": 61, + "period": "2002-09", + "total_eur_minor": 551174056 + }, + { + "grant_count": 24, + "period": "2002-10", + "total_eur_minor": 35428064 + }, + { + "grant_count": 66, + "period": "2002-11", + "total_eur_minor": 118284485 + }, + { + "grant_count": 65, + "period": "2002-12", + "total_eur_minor": 445000693 + }, + { + "grant_count": 53, + "period": "2003-01", + "total_eur_minor": 585323629 + }, + { + "grant_count": 55, + "period": "2003-02", + "total_eur_minor": 39499597 + }, + { + "grant_count": 137, + "period": "2003-03", + "total_eur_minor": 614136155 + }, + { + "grant_count": 19, + "period": "2003-04", + "total_eur_minor": 35772256 + }, + { + "grant_count": 120, + "period": "2003-05", + "total_eur_minor": 417060232 + }, + { + "grant_count": 39, + "period": "2003-06", + "total_eur_minor": 81419033 + }, + { + "grant_count": 110, + "period": "2003-07", + "total_eur_minor": 793080963 + }, + { + "grant_count": 33, + "period": "2003-08", + "total_eur_minor": 29764749 + }, + { + "grant_count": 60, + "period": "2003-09", + "total_eur_minor": 600343059 + }, + { + "grant_count": 74, + "period": "2003-10", + "total_eur_minor": 2895820265 + }, + { + "grant_count": 29, + "period": "2003-11", + "total_eur_minor": 37237016 + }, + { + "grant_count": 138, + "period": "2003-12", + "total_eur_minor": 1394124759 + }, + { + "grant_count": 70, + "period": "2004-01", + "total_eur_minor": 1404382321 + }, + { + "grant_count": 60, + "period": "2004-02", + "total_eur_minor": 349663897 + }, + { + "grant_count": 80, + "period": "2004-03", + "total_eur_minor": 555117093 + }, + { + "grant_count": 53, + "period": "2004-04", + "total_eur_minor": 194825574 + }, + { + "grant_count": 92, + "period": "2004-05", + "total_eur_minor": 1119360150 + }, + { + "grant_count": 41, + "period": "2004-06", + "total_eur_minor": 538862268 + }, + { + "grant_count": 141, + "period": "2004-07", + "total_eur_minor": 1620326787 + }, + { + "grant_count": 34, + "period": "2004-08", + "total_eur_minor": 41936358 + }, + { + "grant_count": 52, + "period": "2004-09", + "total_eur_minor": 228698696 + }, + { + "grant_count": 61, + "period": "2004-10", + "total_eur_minor": 584957843 + }, + { + "grant_count": 60, + "period": "2004-11", + "total_eur_minor": 159033133 + }, + { + "grant_count": 44, + "period": "2004-12", + "total_eur_minor": 1214642461 + }, + { + "grant_count": 56, + "period": "2005-01", + "total_eur_minor": 146264554 + }, + { + "grant_count": 59, + "period": "2005-02", + "total_eur_minor": 475893682 + }, + { + "grant_count": 94, + "period": "2005-03", + "total_eur_minor": 4509680693 + }, + { + "grant_count": 23, + "period": "2005-04", + "total_eur_minor": 28598076 + }, + { + "grant_count": 78, + "period": "2005-05", + "total_eur_minor": 2505566926 + }, + { + "grant_count": 32, + "period": "2005-06", + "total_eur_minor": 62543078 + }, + { + "grant_count": 83, + "period": "2005-07", + "total_eur_minor": 1705356281 + }, + { + "grant_count": 31, + "period": "2005-08", + "total_eur_minor": 287824463 + }, + { + "grant_count": 41, + "period": "2005-09", + "total_eur_minor": 79411726 + }, + { + "grant_count": 70, + "period": "2005-10", + "total_eur_minor": 918865473 + }, + { + "grant_count": 32, + "period": "2005-11", + "total_eur_minor": 106891888 + }, + { + "grant_count": 69, + "period": "2005-12", + "total_eur_minor": 1462891570 + }, + { + "grant_count": 31, + "period": "2006-01", + "total_eur_minor": 88936701 + }, + { + "grant_count": 118, + "period": "2006-02", + "total_eur_minor": 363644661 + }, + { + "grant_count": 84, + "period": "2006-03", + "total_eur_minor": 2630184559 + }, + { + "grant_count": 90, + "period": "2006-04", + "total_eur_minor": 759828597 + }, + { + "grant_count": 107, + "period": "2006-05", + "total_eur_minor": 659962720 + }, + { + "grant_count": 137, + "period": "2006-06", + "total_eur_minor": 974623877 + }, + { + "grant_count": 86, + "period": "2006-07", + "total_eur_minor": 922400353 + }, + { + "grant_count": 38, + "period": "2006-08", + "total_eur_minor": 45327902 + }, + { + "grant_count": 45, + "period": "2006-09", + "total_eur_minor": 280912739 + }, + { + "grant_count": 148, + "period": "2006-10", + "total_eur_minor": 2228936468 + }, + { + "grant_count": 58, + "period": "2006-11", + "total_eur_minor": 541255631 + }, + { + "grant_count": 142, + "period": "2006-12", + "total_eur_minor": 2859874063 + }, + { + "grant_count": 51, + "period": "2007-01", + "total_eur_minor": 132885080 + }, + { + "grant_count": 82, + "period": "2007-02", + "total_eur_minor": 1293085896 + }, + { + "grant_count": 99, + "period": "2007-03", + "total_eur_minor": 2160154511 + }, + { + "grant_count": 990, + "period": "2007-04", + "total_eur_minor": 4426941179 + }, + { + "grant_count": 162, + "period": "2007-05", + "total_eur_minor": 1161106490 + }, + { + "grant_count": 59, + "period": "2007-06", + "total_eur_minor": 625934608 + }, + { + "grant_count": 162, + "period": "2007-07", + "total_eur_minor": 1320979705 + }, + { + "grant_count": 199, + "period": "2007-08", + "total_eur_minor": 205499205 + }, + { + "grant_count": 140, + "period": "2007-09", + "total_eur_minor": 845243044 + }, + { + "grant_count": 52, + "period": "2007-10", + "total_eur_minor": 720920806 + }, + { + "grant_count": 83, + "period": "2007-11", + "total_eur_minor": 754378043 + }, + { + "grant_count": 73, + "period": "2007-12", + "total_eur_minor": 563580145 + }, + { + "grant_count": 73, + "period": "2008-01", + "total_eur_minor": 683267656 + }, + { + "grant_count": 70, + "period": "2008-02", + "total_eur_minor": 764410369 + }, + { + "grant_count": 47, + "period": "2008-03", + "total_eur_minor": 735953093 + }, + { + "grant_count": 100, + "period": "2008-04", + "total_eur_minor": 386264879 + }, + { + "grant_count": 46, + "period": "2008-05", + "total_eur_minor": 617769578 + }, + { + "grant_count": 155, + "period": "2008-06", + "total_eur_minor": 1886604220 + }, + { + "grant_count": 110, + "period": "2008-07", + "total_eur_minor": 1061680944 + }, + { + "grant_count": 33, + "period": "2008-08", + "total_eur_minor": 258045921 + }, + { + "grant_count": 134, + "period": "2008-09", + "total_eur_minor": 955200981 + }, + { + "grant_count": 120, + "period": "2008-10", + "total_eur_minor": 1274927123 + }, + { + "grant_count": 161, + "period": "2008-11", + "total_eur_minor": 4239640425 + }, + { + "grant_count": 108, + "period": "2008-12", + "total_eur_minor": 1549817522 + }, + { + "grant_count": 84, + "period": "2009-01", + "total_eur_minor": 1671506907 + }, + { + "grant_count": 182, + "period": "2009-02", + "total_eur_minor": 675363880 + }, + { + "grant_count": 94, + "period": "2009-03", + "total_eur_minor": 2140986489 + }, + { + "grant_count": 157, + "period": "2009-04", + "total_eur_minor": 1937656708 + }, + { + "grant_count": 188, + "period": "2009-05", + "total_eur_minor": 6701278543 + }, + { + "grant_count": 284, + "period": "2009-06", + "total_eur_minor": 19536137581 + }, + { + "grant_count": 297, + "period": "2009-07", + "total_eur_minor": 3751989819 + }, + { + "grant_count": 110, + "period": "2009-08", + "total_eur_minor": 664403237 + }, + { + "grant_count": 206, + "period": "2009-09", + "total_eur_minor": 1813524707 + }, + { + "grant_count": 276, + "period": "2009-10", + "total_eur_minor": 1953805636 + }, + { + "grant_count": 199, + "period": "2009-11", + "total_eur_minor": 3663263260 + }, + { + "grant_count": 240, + "period": "2009-12", + "total_eur_minor": 1536801866 + }, + { + "grant_count": 194, + "period": "2010-01", + "total_eur_minor": 1125456090 + }, + { + "grant_count": 239, + "period": "2010-02", + "total_eur_minor": 1376299925 + }, + { + "grant_count": 210, + "period": "2010-03", + "total_eur_minor": 2278301102 + }, + { + "grant_count": 180, + "period": "2010-04", + "total_eur_minor": 2330521631 + }, + { + "grant_count": 236, + "period": "2010-05", + "total_eur_minor": 1356248779 + }, + { + "grant_count": 242, + "period": "2010-06", + "total_eur_minor": 1786392796 + }, + { + "grant_count": 362, + "period": "2010-07", + "total_eur_minor": 8834519233 + }, + { + "grant_count": 142, + "period": "2010-08", + "total_eur_minor": 1019980096 + }, + { + "grant_count": 291, + "period": "2010-09", + "total_eur_minor": 2021848441 + }, + { + "grant_count": 730, + "period": "2010-10", + "total_eur_minor": 15858227302 + }, + { + "grant_count": 286, + "period": "2010-11", + "total_eur_minor": 4015973722 + }, + { + "grant_count": 244, + "period": "2010-12", + "total_eur_minor": 4494065826 + }, + { + "grant_count": 184, + "period": "2011-01", + "total_eur_minor": 3644699154 + }, + { + "grant_count": 203, + "period": "2011-02", + "total_eur_minor": 6887313773 + }, + { + "grant_count": 227, + "period": "2011-03", + "total_eur_minor": 2238521192 + }, + { + "grant_count": 278, + "period": "2011-04", + "total_eur_minor": 5781299928 + }, + { + "grant_count": 294, + "period": "2011-05", + "total_eur_minor": 11114961270 + }, + { + "grant_count": 274, + "period": "2011-06", + "total_eur_minor": 10139393504 + }, + { + "grant_count": 234, + "period": "2011-07", + "total_eur_minor": 1526916661 + }, + { + "grant_count": 125, + "period": "2011-08", + "total_eur_minor": 965957680 + }, + { + "grant_count": 217, + "period": "2011-09", + "total_eur_minor": 1887109628 + }, + { + "grant_count": 439, + "period": "2011-10", + "total_eur_minor": 25666943833 + }, + { + "grant_count": 253, + "period": "2011-11", + "total_eur_minor": 3148204622 + }, + { + "grant_count": 307, + "period": "2011-12", + "total_eur_minor": 9984726477 + }, + { + "grant_count": 166, + "period": "2012-01", + "total_eur_minor": 3844401518 + }, + { + "grant_count": 325, + "period": "2012-02", + "total_eur_minor": 8141977369 + }, + { + "grant_count": 389, + "period": "2012-03", + "total_eur_minor": 11067050166 + }, + { + "grant_count": 211, + "period": "2012-04", + "total_eur_minor": 8151653600 + }, + { + "grant_count": 396, + "period": "2012-05", + "total_eur_minor": 10995124401 + }, + { + "grant_count": 298, + "period": "2012-06", + "total_eur_minor": 6490567944 + }, + { + "grant_count": 322, + "period": "2012-07", + "total_eur_minor": 4768697702 + }, + { + "grant_count": 116, + "period": "2012-08", + "total_eur_minor": 580754197 + }, + { + "grant_count": 301, + "period": "2012-09", + "total_eur_minor": 7333256685 + }, + { + "grant_count": 404, + "period": "2012-10", + "total_eur_minor": 20149799491 + }, + { + "grant_count": 378, + "period": "2012-11", + "total_eur_minor": 13000198673 + }, + { + "grant_count": 424, + "period": "2012-12", + "total_eur_minor": 12631647717 + }, + { + "grant_count": 267, + "period": "2013-01", + "total_eur_minor": 3259014594 + }, + { + "grant_count": 284, + "period": "2013-02", + "total_eur_minor": 1882831225 + }, + { + "grant_count": 281, + "period": "2013-03", + "total_eur_minor": 2306028929 + }, + { + "grant_count": 377, + "period": "2013-04", + "total_eur_minor": 8825779952 + }, + { + "grant_count": 410, + "period": "2013-05", + "total_eur_minor": 5783971097 + }, + { + "grant_count": 419, + "period": "2013-06", + "total_eur_minor": 5551936899 + }, + { + "grant_count": 357, + "period": "2013-07", + "total_eur_minor": 11275177616 + }, + { + "grant_count": 197, + "period": "2013-08", + "total_eur_minor": 1076440302 + }, + { + "grant_count": 553, + "period": "2013-09", + "total_eur_minor": 2773175206 + }, + { + "grant_count": 368, + "period": "2013-10", + "total_eur_minor": 8027178385 + }, + { + "grant_count": 321, + "period": "2013-11", + "total_eur_minor": 5009644067 + }, + { + "grant_count": 408, + "period": "2013-12", + "total_eur_minor": 8205775697 + }, + { + "grant_count": 279, + "period": "2014-01", + "total_eur_minor": 3075059735 + }, + { + "grant_count": 399, + "period": "2014-02", + "total_eur_minor": 4361114812 + }, + { + "grant_count": 572, + "period": "2014-03", + "total_eur_minor": 5446216097 + }, + { + "grant_count": 398, + "period": "2014-04", + "total_eur_minor": 3165635080 + }, + { + "grant_count": 388, + "period": "2014-05", + "total_eur_minor": 2848858552 + }, + { + "grant_count": 565, + "period": "2014-06", + "total_eur_minor": 5746400187 + }, + { + "grant_count": 445, + "period": "2014-07", + "total_eur_minor": 2338180049 + }, + { + "grant_count": 201, + "period": "2014-08", + "total_eur_minor": 1145229508 + }, + { + "grant_count": 510, + "period": "2014-09", + "total_eur_minor": 4747744701 + }, + { + "grant_count": 408, + "period": "2014-10", + "total_eur_minor": 4034527533 + }, + { + "grant_count": 448, + "period": "2014-11", + "total_eur_minor": 5154013667 + }, + { + "grant_count": 583, + "period": "2014-12", + "total_eur_minor": 9101369183 + }, + { + "grant_count": 488, + "period": "2015-01", + "total_eur_minor": 4241142452 + }, + { + "grant_count": 372, + "period": "2015-02", + "total_eur_minor": 1770427411 + }, + { + "grant_count": 820, + "period": "2015-03", + "total_eur_minor": 7686254661 + }, + { + "grant_count": 427, + "period": "2015-04", + "total_eur_minor": 4361748800 + }, + { + "grant_count": 479, + "period": "2015-05", + "total_eur_minor": 6519340710 + }, + { + "grant_count": 732, + "period": "2015-06", + "total_eur_minor": 6869682528 + }, + { + "grant_count": 608, + "period": "2015-07", + "total_eur_minor": 8574730616 + }, + { + "grant_count": 283, + "period": "2015-08", + "total_eur_minor": 853509299 + }, + { + "grant_count": 682, + "period": "2015-09", + "total_eur_minor": 6014358330 + }, + { + "grant_count": 564, + "period": "2015-10", + "total_eur_minor": 5161326646 + }, + { + "grant_count": 739, + "period": "2015-11", + "total_eur_minor": 10419055755 + }, + { + "grant_count": 611, + "period": "2015-12", + "total_eur_minor": 5218610380 + }, + { + "grant_count": 498, + "period": "2016-01", + "total_eur_minor": 10068638482 + }, + { + "grant_count": 490, + "period": "2016-02", + "total_eur_minor": 8638421439 + }, + { + "grant_count": 971, + "period": "2016-03", + "total_eur_minor": 27034826640 + }, + { + "grant_count": 1250, + "period": "2016-04", + "total_eur_minor": 7676701843 + }, + { + "grant_count": 751, + "period": "2016-05", + "total_eur_minor": 5711374533 + }, + { + "grant_count": 1117, + "period": "2016-06", + "total_eur_minor": 11077124210 + }, + { + "grant_count": 751, + "period": "2016-07", + "total_eur_minor": 11840736809 + }, + { + "grant_count": 399, + "period": "2016-08", + "total_eur_minor": 4624730267 + }, + { + "grant_count": 828, + "period": "2016-09", + "total_eur_minor": 6472351912 + }, + { + "grant_count": 837, + "period": "2016-10", + "total_eur_minor": 5841152618 + }, + { + "grant_count": 1208, + "period": "2016-11", + "total_eur_minor": 5523161916 + }, + { + "grant_count": 1151, + "period": "2016-12", + "total_eur_minor": 6830146422 + }, + { + "grant_count": 649, + "period": "2017-01", + "total_eur_minor": 4206906877 + }, + { + "grant_count": 805, + "period": "2017-02", + "total_eur_minor": 6604647621 + }, + { + "grant_count": 1507, + "period": "2017-03", + "total_eur_minor": 7963497721 + }, + { + "grant_count": 1456, + "period": "2017-04", + "total_eur_minor": 15043161971 + }, + { + "grant_count": 1045, + "period": "2017-05", + "total_eur_minor": 5137855626 + }, + { + "grant_count": 1671, + "period": "2017-06", + "total_eur_minor": 7108533824 + }, + { + "grant_count": 1558, + "period": "2017-07", + "total_eur_minor": 9991346395 + }, + { + "grant_count": 1358, + "period": "2017-08", + "total_eur_minor": 2256198489 + }, + { + "grant_count": 1108, + "period": "2017-09", + "total_eur_minor": 9909556708 + }, + { + "grant_count": 1239, + "period": "2017-10", + "total_eur_minor": 8214203254 + }, + { + "grant_count": 2432, + "period": "2017-11", + "total_eur_minor": 10219457700 + }, + { + "grant_count": 1329, + "period": "2017-12", + "total_eur_minor": 5610401770 + }, + { + "grant_count": 1104, + "period": "2018-01", + "total_eur_minor": 8328096039 + }, + { + "grant_count": 956, + "period": "2018-02", + "total_eur_minor": 3561319862 + }, + { + "grant_count": 2138, + "period": "2018-03", + "total_eur_minor": 7911523013 + }, + { + "grant_count": 1755, + "period": "2018-04", + "total_eur_minor": 13726127215 + }, + { + "grant_count": 1276, + "period": "2018-05", + "total_eur_minor": 5734796412 + }, + { + "grant_count": 1939, + "period": "2018-06", + "total_eur_minor": 11434338073 + }, + { + "grant_count": 1248, + "period": "2018-07", + "total_eur_minor": 6823534327 + }, + { + "grant_count": 691, + "period": "2018-08", + "total_eur_minor": 3837326824 + }, + { + "grant_count": 1007, + "period": "2018-09", + "total_eur_minor": 6944276389 + }, + { + "grant_count": 1500, + "period": "2018-10", + "total_eur_minor": 8658230064 + }, + { + "grant_count": 1784, + "period": "2018-11", + "total_eur_minor": 6981457234 + }, + { + "grant_count": 1578, + "period": "2018-12", + "total_eur_minor": 7027781588 + }, + { + "grant_count": 1066, + "period": "2019-01", + "total_eur_minor": 7758511228 + }, + { + "grant_count": 1550, + "period": "2019-02", + "total_eur_minor": 5357469696 + }, + { + "grant_count": 2345, + "period": "2019-03", + "total_eur_minor": 11871691688 + }, + { + "grant_count": 1559, + "period": "2019-04", + "total_eur_minor": 11047156545 + }, + { + "grant_count": 1239, + "period": "2019-05", + "total_eur_minor": 6606900054 + }, + { + "grant_count": 2021, + "period": "2019-06", + "total_eur_minor": 10840368560 + }, + { + "grant_count": 1563, + "period": "2019-07", + "total_eur_minor": 9354782564 + }, + { + "grant_count": 782, + "period": "2019-08", + "total_eur_minor": 1969941717 + }, + { + "grant_count": 1262, + "period": "2019-09", + "total_eur_minor": 6178557369 + }, + { + "grant_count": 1514, + "period": "2019-10", + "total_eur_minor": 4951891710 + }, + { + "grant_count": 1637, + "period": "2019-11", + "total_eur_minor": 7534740275 + }, + { + "grant_count": 1701, + "period": "2019-12", + "total_eur_minor": 10849888199 + }, + { + "grant_count": 1327, + "period": "2020-01", + "total_eur_minor": 7941885945 + }, + { + "grant_count": 1179, + "period": "2020-02", + "total_eur_minor": 6056954617 + }, + { + "grant_count": 2740, + "period": "2020-03", + "total_eur_minor": 18624421859 + }, + { + "grant_count": 6250, + "period": "2020-04", + "total_eur_minor": 23320511945 + }, + { + "grant_count": 4465, + "period": "2020-05", + "total_eur_minor": 11849190690 + }, + { + "grant_count": 4311, + "period": "2020-06", + "total_eur_minor": 16523569210 + }, + { + "grant_count": 3595, + "period": "2020-07", + "total_eur_minor": 9877716534 + }, + { + "grant_count": 1621, + "period": "2020-08", + "total_eur_minor": 7107206148 + }, + { + "grant_count": 2722, + "period": "2020-09", + "total_eur_minor": 8499387753 + }, + { + "grant_count": 2254, + "period": "2020-10", + "total_eur_minor": 12733630264 + }, + { + "grant_count": 2179, + "period": "2020-11", + "total_eur_minor": 7358642148 + }, + { + "grant_count": 2720, + "period": "2020-12", + "total_eur_minor": 10157291500 + }, + { + "grant_count": 1901, + "period": "2021-01", + "total_eur_minor": 6922744527 + }, + { + "grant_count": 1768, + "period": "2021-02", + "total_eur_minor": 36725288387 + }, + { + "grant_count": 3206, + "period": "2021-03", + "total_eur_minor": 13874329124 + }, + { + "grant_count": 1760, + "period": "2021-04", + "total_eur_minor": 12003634783 + }, + { + "grant_count": 1870, + "period": "2021-05", + "total_eur_minor": 7274218477 + }, + { + "grant_count": 2634, + "period": "2021-06", + "total_eur_minor": 13003044540 + }, + { + "grant_count": 2539, + "period": "2021-07", + "total_eur_minor": 12369170076 + }, + { + "grant_count": 1340, + "period": "2021-08", + "total_eur_minor": 8731958094 + }, + { + "grant_count": 1776, + "period": "2021-09", + "total_eur_minor": 17161124841 + }, + { + "grant_count": 2073, + "period": "2021-10", + "total_eur_minor": 7939791024 + }, + { + "grant_count": 2888, + "period": "2021-11", + "total_eur_minor": 28303455708 + }, + { + "grant_count": 2221, + "period": "2021-12", + "total_eur_minor": 11272256677 + }, + { + "grant_count": 1476, + "period": "2022-01", + "total_eur_minor": 7386346434 + }, + { + "grant_count": 1831, + "period": "2022-02", + "total_eur_minor": 9820263239 + }, + { + "grant_count": 3065, + "period": "2022-03", + "total_eur_minor": 21998129799 + }, + { + "grant_count": 3162, + "period": "2022-04", + "total_eur_minor": 16757618835 + }, + { + "grant_count": 2908, + "period": "2022-05", + "total_eur_minor": 19370132026 + }, + { + "grant_count": 2436, + "period": "2022-06", + "total_eur_minor": 10272898994 + }, + { + "grant_count": 3363, + "period": "2022-07", + "total_eur_minor": 10617399305 + }, + { + "grant_count": 1714, + "period": "2022-08", + "total_eur_minor": 8272519902 + }, + { + "grant_count": 2831, + "period": "2022-09", + "total_eur_minor": 25577796879 + }, + { + "grant_count": 3835, + "period": "2022-10", + "total_eur_minor": 7811574091 + }, + { + "grant_count": 3240, + "period": "2022-11", + "total_eur_minor": 9722882601 + }, + { + "grant_count": 3257, + "period": "2022-12", + "total_eur_minor": 14549589380 + }, + { + "grant_count": 3034, + "period": "2023-01", + "total_eur_minor": 10206493027 + }, + { + "grant_count": 2799, + "period": "2023-02", + "total_eur_minor": 11537635908 + }, + { + "grant_count": 3293, + "period": "2023-03", + "total_eur_minor": 16578894376 + }, + { + "grant_count": 2613, + "period": "2023-04", + "total_eur_minor": 12860496642 + }, + { + "grant_count": 2795, + "period": "2023-05", + "total_eur_minor": 43246313788 + }, + { + "grant_count": 3056, + "period": "2023-06", + "total_eur_minor": 19795970388 + }, + { + "grant_count": 3104, + "period": "2023-07", + "total_eur_minor": 23731567940 + }, + { + "grant_count": 2519, + "period": "2023-08", + "total_eur_minor": 10252987561 + }, + { + "grant_count": 3357, + "period": "2023-09", + "total_eur_minor": 54311597998 + }, + { + "grant_count": 3388, + "period": "2023-10", + "total_eur_minor": 18266505367 + }, + { + "grant_count": 3817, + "period": "2023-11", + "total_eur_minor": 16881978792 + }, + { + "grant_count": 2747, + "period": "2023-12", + "total_eur_minor": 12362173690 + }, + { + "grant_count": 2928, + "period": "2024-01", + "total_eur_minor": 25258434326 + }, + { + "grant_count": 3066, + "period": "2024-02", + "total_eur_minor": 10516541857 + }, + { + "grant_count": 4538, + "period": "2024-03", + "total_eur_minor": 19140494538 + }, + { + "grant_count": 3452, + "period": "2024-04", + "total_eur_minor": 18609085782 + }, + { + "grant_count": 4277, + "period": "2024-05", + "total_eur_minor": 28413774759 + }, + { + "grant_count": 3691, + "period": "2024-06", + "total_eur_minor": 69626524123 + }, + { + "grant_count": 4886, + "period": "2024-07", + "total_eur_minor": 32244854242 + }, + { + "grant_count": 2583, + "period": "2024-08", + "total_eur_minor": 5294870640 + }, + { + "grant_count": 3935, + "period": "2024-09", + "total_eur_minor": 59588203330 + }, + { + "grant_count": 3254, + "period": "2024-10", + "total_eur_minor": 15927773158 + }, + { + "grant_count": 3997, + "period": "2024-11", + "total_eur_minor": 17921053046 + }, + { + "grant_count": 2798, + "period": "2024-12", + "total_eur_minor": 18461722782 + }, + { + "grant_count": 2442, + "period": "2025-01", + "total_eur_minor": 22500877428 + }, + { + "grant_count": 1998, + "period": "2025-02", + "total_eur_minor": 10994005272 + }, + { + "grant_count": 4109, + "period": "2025-03", + "total_eur_minor": 17024153003 + }, + { + "grant_count": 2466, + "period": "2025-04", + "total_eur_minor": 14737330610 + }, + { + "grant_count": 2217, + "period": "2025-05", + "total_eur_minor": 25099479388 + }, + { + "grant_count": 2268, + "period": "2025-06", + "total_eur_minor": 39187909588 + }, + { + "grant_count": 1895, + "period": "2025-07", + "total_eur_minor": 13263198948 + }, + { + "grant_count": 1155, + "period": "2025-08", + "total_eur_minor": 10241938193 + }, + { + "grant_count": 1911, + "period": "2025-09", + "total_eur_minor": 44393871160 + }, + { + "grant_count": 1769, + "period": "2025-10", + "total_eur_minor": 9236035164 + }, + { + "grant_count": 1395, + "period": "2025-11", + "total_eur_minor": 6658775052 + }, + { + "grant_count": 1884, + "period": "2025-12", + "total_eur_minor": 13389208853 + }, + { + "grant_count": 667, + "period": "2026-01", + "total_eur_minor": 12473930413 + }, + { + "grant_count": 785, + "period": "2026-02", + "total_eur_minor": 5229722701 + }, + { + "grant_count": 856, + "period": "2026-03", + "total_eur_minor": 4332557864 + }, + { + "grant_count": 360, + "period": "2026-04", + "total_eur_minor": 1071530341 + }, + { + "grant_count": 230, + "period": "2026-05", + "total_eur_minor": 1124864560 + }, + { + "grant_count": 234, + "period": "2026-06", + "total_eur_minor": 1278379080 + }, + { + "grant_count": 91, + "period": "2026-07", + "total_eur_minor": 202110060 + }, + { + "grant_count": 1, + "period": "2026-10", + "total_eur_minor": 0 + } + ], + "organisation_details": [ + { + "charity_id": -24788, + "headquarters_country": "Netherlands", + "name": "Women Win", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24787, + "headquarters_country": "Switzerland", + "name": "Toni Piëch Foundation", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24786, + "headquarters_country": "United Kingdom", + "name": "Thomson Reuters Foundation", + "organization_type": "membership organization", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24785, + "headquarters_country": "United Kingdom", + "name": "The Social Change Nest", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24784, + "headquarters_country": "United States", + "name": "The Global Fund for Children", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24783, + "headquarters_country": null, + "name": "Suna and İnan Kıraç Foundation", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24781, + "headquarters_country": "Sweden", + "name": "Hidden Universe: Biodiversity Foundation (HUB)", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24780, + "headquarters_country": "Germany", + "name": "Andrea von Braun Stiftung", + "organization_type": "foundation", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24510, + "headquarters_country": "Ukraine", + "name": "Philanthropy in Ukraine", + "organization_type": "membership organization", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + }, + { + "charity_id": -24509, + "headquarters_country": null, + "name": "Confederación Española de Cajas de Ahorros", + "organization_type": "philanthropy infrastructure organization", + "primary_source": "Philea", + "transaction_coverage": "organization_level_only" + } + ], + "overview_totals": [ + { + "classified_grants": 302546, + "grant_count": 302546, + "mapped_grants": 104191, + "total_eur_minor": 2243598670770 + } + ], + "programme_filter": [ + { + "grant_count": 7368, + "total_eur_minor": 248473543343 + } + ], + "programme_totals": [ + { + "grant_count": 167843, + "programme_area": "Unclassified", + "represented_eur_minor": 786337497895 + }, + { + "grant_count": 41109, + "programme_area": "Education", + "represented_eur_minor": 400235015153 + }, + { + "grant_count": 40958, + "programme_area": "Health", + "represented_eur_minor": 647419873578 + }, + { + "grant_count": 39811, + "programme_area": "Youth/Children Development", + "represented_eur_minor": 270070988130 + }, + { + "grant_count": 17752, + "programme_area": "Arts & Culture", + "represented_eur_minor": 184431544131 + }, + { + "grant_count": 17234, + "programme_area": "Socio-economic Development, Poverty", + "represented_eur_minor": 119682815730 + }, + { + "grant_count": 7368, + "programme_area": "tech-enablement", + "represented_eur_minor": 248473543343 + }, + { + "grant_count": 6356, + "programme_area": "Sciences & Research", + "represented_eur_minor": 403415755556 + }, + { + "grant_count": 4451, + "programme_area": "Diversity & Inclusion", + "represented_eur_minor": 87084214363 + }, + { + "grant_count": 4379, + "programme_area": "Humanitarian & Disaster Relief", + "represented_eur_minor": 31594041182 + }, + { + "grant_count": 3357, + "programme_area": "Civil society, Voluntarism & Non-Profit Sector", + "represented_eur_minor": 27404067929 + }, + { + "grant_count": 3001, + "programme_area": "Citizenship, Social Justice & Public Affairs", + "represented_eur_minor": 46749881112 + }, + { + "grant_count": 2884, + "programme_area": "Environment/Climate", + "represented_eur_minor": 159586257722 + }, + { + "grant_count": 1740, + "programme_area": "Food, Agriculture & Nutrition", + "represented_eur_minor": 20923682687 + }, + { + "grant_count": 578, + "programme_area": "Human/Civil Rights", + "represented_eur_minor": 8409676106 + }, + { + "grant_count": 62, + "programme_area": "Peace & Conflict Resolution", + "represented_eur_minor": 645036856 + } + ], + "recipient_filter": [ + { + "grant_count": 6089, + "total_eur_minor": 434560511133 + } + ], + "recipient_rankings": [ + { + "grant_count": 576, + "identity": "name:university of oxford", + "name": "University Of Oxford", + "total_eur_minor": 79787774359 + }, + { + "grant_count": 1, + "identity": "name:wellcome leap inc", + "name": "Wellcome Leap Inc.", + "total_eur_minor": 55025784898 + }, + { + "grant_count": 423, + "identity": "name:university of cambridge", + "name": "University Of Cambridge", + "total_eur_minor": 53329684843 + }, + { + "grant_count": 427, + "identity": "name:university college london", + "name": "University College London", + "total_eur_minor": 44052184795 + }, + { + "grant_count": 521, + "identity": "name:king s college london", + "name": "King's College London", + "total_eur_minor": 31270966015 + }, + { + "grant_count": 937, + "identity": "name:guy s and st thomas nhs foundation trust", + "name": "Guy's and St Thomas' NHS Foundation Trust", + "total_eur_minor": 21964774747 + }, + { + "grant_count": 529, + "identity": "205846", + "name": "A la Ronde", + "total_eur_minor": 21957928322 + }, + { + "grant_count": 64, + "identity": "name:liverpool school of tropical medicine", + "name": "Liverpool School of Tropical Medicine", + "total_eur_minor": 21423882549 + }, + { + "grant_count": 292, + "identity": "name:university of edinburgh", + "name": "University of Edinburgh", + "total_eur_minor": 21363226773 + }, + { + "grant_count": 234, + "identity": "name:imperial college london", + "name": "Imperial College London", + "total_eur_minor": 20434525907 + }, + { + "grant_count": 107, + "identity": "name:university of dundee", + "name": "University of Dundee", + "total_eur_minor": 14960691907 + }, + { + "grant_count": 222, + "identity": "220949", + "name": "BRITISH RED CROSS SOCIETY", + "total_eur_minor": 14254727026 + }, + { + "grant_count": 114, + "identity": "name:london school of hygiene and tropical medicine", + "name": "LONDON SCHOOL OF HYGIENE AND TROPICAL MEDICINE", + "total_eur_minor": 12256849404 + }, + { + "grant_count": 82, + "identity": "name:uk community foundations", + "name": "UK Community Foundations", + "total_eur_minor": 10771526317 + }, + { + "grant_count": 1, + "identity": "name:london and quadrant housing trust", + "name": "London & Quadrant Housing Trust", + "total_eur_minor": 9612908030 + }, + { + "grant_count": 143, + "identity": "name:university of manchester", + "name": "University Of Manchester", + "total_eur_minor": 9074678403 + }, + { + "grant_count": 8, + "identity": "name:the sainsbury laboratory", + "name": "The Sainsbury Laboratory", + "total_eur_minor": 8643726866 + }, + { + "grant_count": 151, + "identity": "name:university of glasgow", + "name": "University of Glasgow", + "total_eur_minor": 8381509901 + }, + { + "grant_count": 95, + "identity": "name:university of exeter", + "name": "UNIVERSITY OF EXETER", + "total_eur_minor": 8222140269 + }, + { + "grant_count": 198, + "identity": "name:university of bristol", + "name": "UNIVERSITY OF BRISTOL", + "total_eur_minor": 7734488683 + }, + { + "grant_count": 2, + "identity": "name:boston university", + "name": "Boston University", + "total_eur_minor": 7123851236 + }, + { + "grant_count": 242, + "identity": "name:south london and maudsley nhs foundation trust", + "name": "South London and Maudsley NHS Foundation Trust", + "total_eur_minor": 6864800530 + }, + { + "grant_count": 111, + "identity": "name:university of birmingham", + "name": "University of Birmingham", + "total_eur_minor": 6723125608 + }, + { + "grant_count": 16, + "identity": "name:yale university", + "name": "Yale University", + "total_eur_minor": 6533875930 + }, + { + "grant_count": 15, + "identity": "name:embl - european bioinformatics institute", + "name": "EMBL - European Bioinformatics Institute", + "total_eur_minor": 6145008032 + } + ], + "registry_search": [ + { + "charity_number": "1010442", + "country_code": "GB", + "registered_name": "1 TO 1 SCRIPTURES TRUST", + "registration_status": "Removed", + "registry_id": "cc:1010442" + }, + { + "charity_number": "1137030", + "country_code": "GB", + "registered_name": "10:10 TRUST", + "registration_status": "Removed", + "registry_id": "cc:4050549" + }, + { + "charity_number": "1218470", + "country_code": "GB", + "registered_name": "1008 TIRTHANKAR SHRI SHANTINATH JINALAYA DIGAMBAR JAIN HARROW TRUST LONDON UK", + "registration_status": "Registered", + "registry_id": "cc:5286533" + }, + { + "charity_number": "1113553", + "country_code": "GB", + "registered_name": "101 (CITY OF LONDON) ENGINEER REGIMENT (RICHMOND HILL) CHARITABLE TRUST", + "registration_status": "Registered", + "registry_id": "cc:3011102" + }, + { + "charity_number": "294975", + "country_code": "GB", + "registered_name": "'1067 TRUST FUND'", + "registration_status": "Removed", + "registry_id": "cc:294975" + }, + { + "charity_number": "254425", + "country_code": "GB", + "registered_name": "10TH PRINCESS MARY'S OWN GURKHA RIFLE REGIMENTAL TRUST", + "registration_status": "Removed", + "registry_id": "cc:254425" + }, + { + "charity_number": "1033082", + "country_code": "GB", + "registered_name": "129 INNER WHEEL DISTRICT COMMITTEE TRUST FUND", + "registration_status": "Removed", + "registry_id": "cc:1033082" + }, + { + "charity_number": "248816", + "country_code": "GB", + "registered_name": "12TH (ROYAL) LANCERS ASSOCIATION TRUST FUND", + "registration_status": "Removed", + "registry_id": "cc:21101" + }, + { + "charity_number": "1158832", + "country_code": "GB", + "registered_name": "13:33 TRUST", + "registration_status": "Registered", + "registry_id": "cc:5046410" + }, + { + "charity_number": "1161219", + "country_code": "GB", + "registered_name": "13 RIVERS TRUST", + "registration_status": "Registered", + "registry_id": "cc:5055214" + } + ], + "sankey": [ + { + "amount_minor": 2239937979, + "currency": "GBP", + "funding_name": "Masonic Charitable Foundation", + "grant_count": 6837, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 129988282, + "currency": "GBP", + "funding_name": "R L Glasspool Charity Trust", + "grant_count": 4866, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 181605700, + "currency": "GBP", + "funding_name": "St. Martin-in-the-Fields Charity", + "grant_count": 4576, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 188509309, + "currency": "GBP", + "funding_name": "Hampton Fund", + "grant_count": 3825, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 331537079, + "currency": "GBP", + "funding_name": "Foundation Scotland", + "grant_count": 3004, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 361360997, + "currency": "GBP", + "funding_name": "Barnwood Trust", + "grant_count": 2560, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 31856759, + "currency": "GBP", + "funding_name": "Cloudesley", + "grant_count": 2535, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 206092500, + "currency": "GBP", + "funding_name": "The Teaching Staff Trust", + "grant_count": 2373, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 39854711, + "currency": "GBP", + "funding_name": "The Greggs Foundation", + "grant_count": 2023, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 368654494, + "currency": "GBP", + "funding_name": "Foothold", + "grant_count": 1484, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 114209611, + "currency": "GBP", + "funding_name": "Longleigh Foundation", + "grant_count": 1300, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 18140808016, + "currency": "GBP", + "funding_name": "Guy's and St Thomas' Charity", + "grant_count": 935, + "recipient_name": "Guy's and St Thomas' NHS Foundation Trust" + }, + { + "amount_minor": 114997148, + "currency": "GBP", + "funding_name": "Wiltshire and Swindon Community Foundation", + "grant_count": 711, + "recipient_name": "Individual Recipient" + }, + { + "amount_minor": 41061225, + "currency": "GBP", + "funding_name": "CareTech foundation", + "grant_count": 652, + "recipient_name": "Individual" + }, + { + "amount_minor": 1423354035, + "currency": "GBP", + "funding_name": "Imperial Health Charity", + "grant_count": 575, + "recipient_name": "Imperial College Healthcare NHS Trust" + }, + { + "amount_minor": 65531682563, + "currency": "GBP", + "funding_name": "The Wellcome Trust", + "grant_count": 498, + "recipient_name": "University of Oxford" + }, + { + "amount_minor": 218391707, + "currency": "GBP", + "funding_name": "Imperial Health Charity", + "grant_count": 471, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 16316503, + "currency": "GBP", + "funding_name": "Somerset Community Foundation", + "grant_count": 387, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 300703631, + "currency": "GBP", + "funding_name": "The Paul Mellon Centre for Studies in British Art", + "grant_count": 380, + "recipient_name": "Unknown Recipient" + }, + { + "amount_minor": 34563208, + "currency": "GBP", + "funding_name": "Two Ridings Community Foundation", + "grant_count": 343, + "recipient_name": "Individual Recipient" + }, + { + "amount_minor": 31770815957, + "currency": "GBP", + "funding_name": "The Wellcome Trust", + "grant_count": 342, + "recipient_name": "University of Cambridge" + }, + { + "amount_minor": 34502947967, + "currency": "GBP", + "funding_name": "The Wellcome Trust", + "grant_count": 333, + "recipient_name": "University College London" + }, + { + "amount_minor": 90521469, + "currency": "GBP", + "funding_name": "Wiltshire and Swindon Community Foundation", + "grant_count": 313, + "recipient_name": "Grant to Individual" + }, + { + "amount_minor": 67821800, + "currency": "GBP", + "funding_name": "Haberdashers' Company", + "grant_count": 273, + "recipient_name": "Unknown recipient" + }, + { + "amount_minor": 17541810377, + "currency": "GBP", + "funding_name": "The Wellcome Trust", + "grant_count": 262, + "recipient_name": "University of Edinburgh" + } + ], + "score_components": { + "components": { + "funding_capacity_fit": { + "available": true, + "confidence": 0.9, + "score": 100.0, + "weight": 0.15, + "weighted_score": 15.0 + }, + "geographic_fit": { + "available": true, + "confidence": 0.95, + "score": 100.0, + "weight": 0.25, + "weighted_score": 25.0 + }, + "historical_grant_size_fit": { + "available": true, + "confidence": 0.9, + "score": 100.0, + "weight": 0.15, + "weighted_score": 15.0 + }, + "organization_type_fit": { + "available": true, + "confidence": 0.95, + "score": 100.0, + "weight": 0.1, + "weighted_score": 10.0 + }, + "thematic_fit": { + "available": true, + "confidence": 0.95, + "score": 100.0, + "weight": 0.35, + "weighted_score": 35.0 + } + }, + "confidence": 0.935, + "configuration_status": "experimental", + "data_completeness": 1.0, + "score": 100.0, + "score_version": "example-relevance-v2" + }, + "yearly_trends": [ + { + "grant_count": 1, + "period": "1991", + "total_eur_minor": 0 + }, + { + "grant_count": 9, + "period": "1995", + "total_eur_minor": 0 + }, + { + "grant_count": 7, + "period": "1996", + "total_eur_minor": 0 + }, + { + "grant_count": 92, + "period": "1997", + "total_eur_minor": 0 + }, + { + "grant_count": 311, + "period": "1998", + "total_eur_minor": 0 + }, + { + "grant_count": 420, + "period": "1999", + "total_eur_minor": 2473147464 + }, + { + "grant_count": 402, + "period": "2000", + "total_eur_minor": 3958566163 + }, + { + "grant_count": 506, + "period": "2001", + "total_eur_minor": 3382076147 + }, + { + "grant_count": 661, + "period": "2002", + "total_eur_minor": 9302059390 + }, + { + "grant_count": 867, + "period": "2003", + "total_eur_minor": 7523581713 + }, + { + "grant_count": 788, + "period": "2004", + "total_eur_minor": 8011806581 + }, + { + "grant_count": 668, + "period": "2005", + "total_eur_minor": 12289788410 + }, + { + "grant_count": 1084, + "period": "2006", + "total_eur_minor": 12355888271 + }, + { + "grant_count": 2152, + "period": "2007", + "total_eur_minor": 14210708712 + }, + { + "grant_count": 1157, + "period": "2008", + "total_eur_minor": 14413582711 + }, + { + "grant_count": 2317, + "period": "2009", + "total_eur_minor": 46046718633 + }, + { + "grant_count": 3356, + "period": "2010", + "total_eur_minor": 46497834943 + }, + { + "grant_count": 3035, + "period": "2011", + "total_eur_minor": 82986047722 + }, + { + "grant_count": 3730, + "period": "2012", + "total_eur_minor": 107155129463 + }, + { + "grant_count": 4242, + "period": "2013", + "total_eur_minor": 63976953969 + }, + { + "grant_count": 5196, + "period": "2014", + "total_eur_minor": 51164349104 + }, + { + "grant_count": 6805, + "period": "2015", + "total_eur_minor": 67690187588 + }, + { + "grant_count": 10251, + "period": "2016", + "total_eur_minor": 111339367091 + }, + { + "grant_count": 16157, + "period": "2017", + "total_eur_minor": 92265767956 + }, + { + "grant_count": 16976, + "period": "2018", + "total_eur_minor": 90968807040 + }, + { + "grant_count": 18239, + "period": "2019", + "total_eur_minor": 94321899605 + }, + { + "grant_count": 35363, + "period": "2020", + "total_eur_minor": 140050408613 + }, + { + "grant_count": 25976, + "period": "2021", + "total_eur_minor": 175581016258 + }, + { + "grant_count": 33118, + "period": "2022", + "total_eur_minor": 162157151485 + }, + { + "grant_count": 36522, + "period": "2023", + "total_eur_minor": 250032615477 + }, + { + "grant_count": 43405, + "period": "2024", + "total_eur_minor": 321003332583 + }, + { + "grant_count": 25509, + "period": "2025", + "total_eur_minor": 226726782659 + }, + { + "grant_count": 3224, + "period": "2026", + "total_eur_minor": 25713095019 + } + ] + }, + "source_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651" +} diff --git a/config/observability.json b/config/observability.json new file mode 100644 index 0000000..85118ae --- /dev/null +++ b/config/observability.json @@ -0,0 +1,45 @@ +{ + "configuration_version": "1", + "service": "foundation-intelligence-api", + "expected_schema_version": "0006_governance_retention", + "metrics": [ + {"name": "readiness_success", "type": "gauge", "unit": "None", "dimensions": ["service", "environment"]}, + {"name": "api_request_duration_ms", "type": "histogram", "unit": "Milliseconds", "dimensions": ["service", "environment", "operation", "status"]}, + {"name": "api_errors_total", "type": "counter", "unit": "Count", "dimensions": ["service", "environment", "operation", "error_class"]}, + {"name": "db_pool_checked_out", "type": "gauge", "unit": "Count", "dimensions": ["service", "environment"]}, + {"name": "db_pool_utilization_ratio", "type": "gauge", "unit": "None", "dimensions": ["service", "environment"]}, + {"name": "query_duration_ms", "type": "histogram", "unit": "Milliseconds", "dimensions": ["service", "environment", "operation"]}, + {"name": "cache_hit_ratio", "type": "gauge", "unit": "None", "dimensions": ["service", "environment", "cache"]}, + {"name": "dataset_age_seconds", "type": "gauge", "unit": "Seconds", "dimensions": ["service", "environment", "dataset_version"]}, + {"name": "source_freshness_seconds", "type": "gauge", "unit": "Seconds", "dimensions": ["service", "environment", "source"]}, + {"name": "pipeline_duration_ms", "type": "histogram", "unit": "Milliseconds", "dimensions": ["service", "environment", "job_type", "status"]}, + {"name": "pipeline_failures_total", "type": "counter", "unit": "Count", "dimensions": ["service", "environment", "job_type", "error_class"]}, + {"name": "pipeline_record_count", "type": "gauge", "unit": "Count", "dimensions": ["service", "environment", "source", "result"]}, + {"name": "reconciliation_mismatch_count", "type": "gauge", "unit": "Count", "dimensions": ["service", "environment", "dataset_version"]}, + {"name": "conversion_gap_count", "type": "gauge", "unit": "Count", "dimensions": ["service", "environment", "dataset_version"]}, + {"name": "programme_coverage_ratio", "type": "gauge", "unit": "None", "dimensions": ["service", "environment", "dataset_version"]}, + {"name": "geography_coverage_ratio", "type": "gauge", "unit": "None", "dimensions": ["service", "environment", "dataset_version"]}, + {"name": "queue_oldest_message_age_seconds", "type": "gauge", "unit": "Seconds", "dimensions": ["service", "environment", "queue"]}, + {"name": "dlq_depth", "type": "gauge", "unit": "Count", "dimensions": ["service", "environment", "queue"]}, + {"name": "worker_failures_total", "type": "counter", "unit": "Count", "dimensions": ["service", "environment", "worker", "error_class"]}, + {"name": "task_restarts_total", "type": "counter", "unit": "Count", "dimensions": ["service", "environment", "task_family"]}, + {"name": "estimated_cost_usd", "type": "gauge", "unit": "None", "dimensions": ["environment", "service_category"]} + ], + "alarms": [ + {"name": "readiness-failure", "metric": "readiness_success", "comparison": "LessThanThreshold", "threshold": 1, "evaluation_periods": 2, "period_seconds": 60, "runbook": "readiness-failure"}, + {"name": "api-5xx-budget", "metric": "api_errors_total", "comparison": "GreaterThanThreshold", "threshold": 5, "evaluation_periods": 5, "period_seconds": 60, "runbook": "bad-deployment"}, + {"name": "api-latency-budget", "metric": "api_request_duration_ms", "comparison": "GreaterThanThreshold", "threshold": 1000, "evaluation_periods": 5, "period_seconds": 60, "runbook": "bad-deployment"}, + {"name": "stale-data", "metric": "dataset_age_seconds", "comparison": "GreaterThanThreshold", "threshold": 691200, "evaluation_periods": 2, "period_seconds": 3600, "runbook": "stale-dataset"}, + {"name": "pipeline-failure", "metric": "pipeline_failures_total", "comparison": "GreaterThanThreshold", "threshold": 0, "evaluation_periods": 1, "period_seconds": 300, "runbook": "ingestion-failure"}, + {"name": "reconciliation-mismatch", "metric": "reconciliation_mismatch_count", "comparison": "GreaterThanThreshold", "threshold": 0, "evaluation_periods": 1, "period_seconds": 300, "runbook": "migration-failure"}, + {"name": "dlq-messages", "metric": "dlq_depth", "comparison": "GreaterThanThreshold", "threshold": 0, "evaluation_periods": 1, "period_seconds": 60, "runbook": "dlq-replay"}, + {"name": "queue-backlog", "metric": "queue_oldest_message_age_seconds", "comparison": "GreaterThanThreshold", "threshold": 900, "evaluation_periods": 3, "period_seconds": 60, "runbook": "queue-backlog"}, + {"name": "conversion-gap-increase", "metric": "conversion_gap_count", "comparison": "GreaterThanThreshold", "threshold": 432, "evaluation_periods": 1, "period_seconds": 3600, "runbook": "stale-dataset"}, + {"name": "programme-coverage-decrease", "metric": "programme_coverage_ratio", "comparison": "LessThanThreshold", "threshold": 0.44, "evaluation_periods": 1, "period_seconds": 3600, "runbook": "stale-dataset"}, + {"name": "geography-coverage-decrease", "metric": "geography_coverage_ratio", "comparison": "LessThanThreshold", "threshold": 0.34, "evaluation_periods": 1, "period_seconds": 3600, "runbook": "stale-dataset"}, + {"name": "cost-threshold", "metric": "estimated_cost_usd", "comparison": "GreaterThanThreshold", "threshold": 500, "evaluation_periods": 1, "period_seconds": 86400, "runbook": "cost-spike"}, + {"name": "rds-cpu", "metric": "aws_rds_cpu_utilization", "comparison": "GreaterThanThreshold", "threshold": 80, "evaluation_periods": 5, "period_seconds": 60, "runbook": "rds-outage"}, + {"name": "rds-connections", "metric": "aws_rds_database_connections", "comparison": "GreaterThanThreshold", "threshold": 80, "evaluation_periods": 5, "period_seconds": 60, "runbook": "rds-outage"}, + {"name": "rds-storage", "metric": "aws_rds_free_storage_bytes", "comparison": "LessThanThreshold", "threshold": 10737418240, "evaluation_periods": 3, "period_seconds": 300, "runbook": "rds-outage"} + ] +} diff --git a/config/runtime-transition.json b/config/runtime-transition.json new file mode 100644 index 0000000..fa8e276 --- /dev/null +++ b/config/runtime-transition.json @@ -0,0 +1,48 @@ +{ + "configuration_version": "1", + "default_operational_mode": "postgresql", + "temporary_modes": [ + "sqlite_migration_source", + "shadow_compare" + ], + "production_fallback": false, + "shadow": { + "maximum_pending_comparisons": 8, + "maximum_response_bytes": 2097152, + "timeout_seconds": 20, + "maximum_recorded_differences": 100, + "approved_unordered_paths": [ + "$.available_currencies", + "$.available_sources", + "$.metadata.sources" + ], + "ignored_operational_paths": [ + "$.metadata.data_mode", + "$.metadata.dataset_version", + "$.search_strategy" + ] + }, + "journeys": [ + "dashboard", + "map", + "map_relationships", + "date_filters", + "country_filters", + "programme_filters", + "donor_filters", + "recipient_filters", + "monthly_trends", + "yearly_trends", + "donor_ranking", + "recipient_ranking", + "registry_search", + "profile_detail", + "grant_list", + "drill_down", + "sankey", + "score", + "news", + "pipeline_status", + "manual_refresh_permissions" + ] +} diff --git a/config/source-pipelines.json b/config/source-pipelines.json new file mode 100644 index 0000000..e5d4253 --- /dev/null +++ b/config/source-pipelines.json @@ -0,0 +1,197 @@ +{ + "configuration_version": "1", + "sources": [ + { + "source_name": "360giving", + "source_owner": "unassigned", + "technical_owner": "data-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 30, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 168, + "schedule_expression": "cron(0 2 ? * MON *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "public_organisation_and_grant_data", + "retention_class": "raw_source_evidence", + "schema_version": "360giving-api-v1", + "credentials_reference": null, + "retry_limit": 3, + "timeout_seconds": 30, + "maximum_pages": 10000, + "maximum_records": 2000000 + }, + { + "source_name": "charity_commission", + "source_owner": "unassigned", + "technical_owner": "data-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 30, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 168, + "schedule_expression": "cron(0 3 ? * MON *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "public_organisation_contact_and_financial_data", + "retention_class": "raw_source_evidence", + "schema_version": "charity-commission-api-v2", + "credentials_reference": "CHARITY_COMMISSION_API_KEY", + "retry_limit": 3, + "timeout_seconds": 30, + "maximum_pages": 10000, + "maximum_records": 1000000 + }, + { + "source_name": "philea", + "source_owner": "unassigned", + "technical_owner": "data-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 10, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 720, + "schedule_expression": "cron(0 4 1 * ? *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "public_organisation_data", + "retention_class": "raw_source_evidence", + "schema_version": "philea-web-v1", + "credentials_reference": null, + "retry_limit": 2, + "timeout_seconds": 30, + "maximum_pages": 1000, + "maximum_records": 100000 + }, + { + "source_name": "hinchilla", + "source_owner": "unassigned", + "technical_owner": "data-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 10, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 720, + "schedule_expression": "cron(0 5 1 * ? *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "public_organisation_data", + "retention_class": "raw_source_evidence", + "schema_version": "hinchilla-web-v1", + "credentials_reference": null, + "retry_limit": 2, + "timeout_seconds": 30, + "maximum_pages": 1000, + "maximum_records": 100000 + }, + { + "source_name": "ecb", + "source_owner": "unassigned", + "technical_owner": "data-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 12, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 48, + "schedule_expression": "cron(0 6 ? * MON-FRI *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "public_reference_data", + "retention_class": "raw_source_evidence", + "schema_version": "ecb-exr-csv-v1", + "credentials_reference": null, + "retry_limit": 3, + "timeout_seconds": 90, + "maximum_pages": 100, + "maximum_records": 1000000 + }, + { + "source_name": "google_news_rss", + "source_owner": "unassigned", + "technical_owner": "application-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 10, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 24, + "schedule_expression": "cron(0 7 * * ? *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "article_metadata", + "retention_class": "article_metadata", + "schema_version": "google-news-rss-v1", + "credentials_reference": null, + "retry_limit": 2, + "timeout_seconds": 10, + "maximum_pages": 5, + "maximum_records": 500 + }, + { + "source_name": "article_content", + "source_owner": "unassigned", + "technical_owner": "application-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 10, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 24, + "schedule_expression": "cron(0 7 * * ? *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "article_content", + "retention_class": "ephemeral_article_content", + "schema_version": "bounded-html-v1", + "credentials_reference": null, + "retry_limit": 1, + "timeout_seconds": 10, + "maximum_pages": 100, + "maximum_records": 100 + }, + { + "source_name": "anthropic_news_summary", + "source_owner": "unassigned", + "technical_owner": "application-platform", + "legal_status": "unresolved", + "licence_status": "unresolved", + "terms_url": null, + "rate_limit_per_minute": 1, + "user_agent": "FoundationIntelligencePlatform/1.0 (+source-governance-required)", + "freshness_sla_hours": 24, + "schedule_expression": "cron(0 8 * * ? *)", + "enabled": false, + "governance_blocked": true, + "last_success": null, + "watermark": null, + "classification": "derived_classification", + "retention_class": "derived_profile", + "schema_version": "news-summary-v1", + "credentials_reference": "ANTHROPIC_API_KEY", + "retry_limit": 0, + "timeout_seconds": 45, + "maximum_pages": 1, + "maximum_records": 8 + } + ] +} diff --git a/docker-bake.hcl b/docker-bake.hcl new file mode 100644 index 0000000..a007a3c --- /dev/null +++ b/docker-bake.hcl @@ -0,0 +1,19 @@ +group "default" { + targets = ["backend", "frontend"] +} + +target "backend" { + context = "." + dockerfile = "Dockerfile" + target = "backend-runtime" + tags = ["foundation-intelligence-backend:local"] + platforms = ["linux/amd64", "linux/arm64"] +} + +target "frontend" { + context = "." + dockerfile = "Dockerfile" + target = "frontend-runtime" + tags = ["foundation-intelligence-frontend:local"] + platforms = ["linux/amd64", "linux/arm64"] +} diff --git a/docker-compose.ecs-local.yml b/docker-compose.ecs-local.yml new file mode 100644 index 0000000..8fc9481 --- /dev/null +++ b/docker-compose.ecs-local.yml @@ -0,0 +1,43 @@ +services: + backend: + image: foundation-intelligence-backend:aws-readiness-local + environment: + APP_ENV: "${ECS_APP_ENV:-development}" + AUTH_MODE: "${ECS_AUTH_MODE:-development}" + DEV_AUTH_ENABLED: "${ECS_DEV_AUTH_ENABLED:-true}" + DATABASE_USER: "${ECS_DATABASE_USER:-foundation_app}" + DEV_AUTH_USERNAME: ${DEV_AUTH_USERNAME:?Set DEV_AUTH_USERNAME for the isolated local test} + DEV_AUTH_PASSWORD: ${DEV_AUTH_PASSWORD:?Set DEV_AUTH_PASSWORD for the isolated local test} + DEV_AUTH_SECRET: ${DEV_AUTH_SECRET:?Set DEV_AUTH_SECRET for the isolated local test} + DEV_AUTH_ALLOWED_HOSTS: ${DEV_AUTH_ALLOWED_HOSTS:-127.0.0.1,::1,localhost} + ports: + - "127.0.0.1:${ECS_FRONTEND_HOST_PORT:-8081}:8080" + + frontend: + image: foundation-intelligence-frontend:aws-readiness-local + + frontend-ecs: + image: foundation-intelligence-frontend:ecs-local + build: + context: . + dockerfile: Dockerfile + target: frontend-runtime-ecs + network_mode: service:backend + depends_on: + backend: + condition: service_healthy + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=32m,uid=101,gid=101 + - /var/cache/nginx:rw,noexec,nosuid,size=32m,uid=101,gid=101 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + restart: unless-stopped diff --git a/docker-compose.yml b/docker-compose.yml index 3cb2c05..f1e9445 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,15 +1,132 @@ -version: '3.8' +name: foundation-intelligence + +x-backend-common: &backend-common + image: foundation-intelligence-backend:local + build: + context: . + dockerfile: Dockerfile + target: backend-runtime + environment: &backend-environment + APP_ENV: development + DATA_RUNTIME_MODE: postgresql + AUTH_MODE: disabled + CORS_ORIGINS: http://127.0.0.1:8080,http://localhost:8080 + DATABASE_HOST: postgres + DATABASE_PORT: "5432" + DATABASE_NAME: foundation_intelligence + DATABASE_USER: foundation_app + DATABASE_PASSWORD_FILE: /run/secrets/postgres_password + DATABASE_CONNECT_TIMEOUT_SECONDS: "5" + DATABASE_STATEMENT_TIMEOUT_MS: "30000" + secrets: + - postgres_password + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=128m,uid=10001,gid=10001,mode=0700 + init: true + stop_grace_period: 35s + cap_drop: + - ALL + security_opt: + - no-new-privileges:true services: - bff: - build: . - ports: - - "8000:8000" - volumes: - - ./src:/app/src + postgres: + image: postgres:16.14-alpine3.24@sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777 environment: - - JWT_SECRET_KEY=bff_super_secret_signing_key_for_session_tokens_2026 - - CORE_API_URL=http://localhost:8080 - - DATA_PATH=/app/src/data/raw/register_of_charities_results.json - # Command overridden with --reload for development hot-reloading - command: uvicorn bff.main:app --host 0.0.0.0 --port 8000 --reload + POSTGRES_DB: foundation_intelligence + POSTGRES_USER: foundation_app + POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 + secrets: + - postgres_password + volumes: + - postgres_data:/var/lib/postgresql/data + tmpfs: + - /tmp:rw,noexec,nosuid,size=64m + - /run/postgresql:rw,nosuid,size=16m + ports: + - "127.0.0.1:${POSTGRES_HOST_PORT:-5432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U foundation_app -d foundation_intelligence"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 10s + restart: unless-stopped + stop_grace_period: 30s + + backend: + <<: *backend-common + ports: + - "127.0.0.1:${BACKEND_HOST_PORT:-8000}:8000" + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: + - CMD + - python + - -c + - import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=2).read() + interval: 5s + timeout: 3s + retries: 12 + start_period: 15s + restart: unless-stopped + + frontend: + image: foundation-intelligence-frontend:local + build: + context: . + dockerfile: Dockerfile + target: frontend-runtime + ports: + - "127.0.0.1:${FRONTEND_HOST_PORT:-8080}:8080" + depends_on: + backend: + condition: service_healthy + read_only: true + tmpfs: + - /tmp:rw,noexec,nosuid,size=32m,uid=101,gid=101 + - /var/cache/nginx:rw,noexec,nosuid,size=32m,uid=101,gid=101 + cap_drop: + - ALL + security_opt: + - no-new-privileges:true + healthcheck: + test: ["CMD", "wget", "-q", "-O", "/dev/null", "http://127.0.0.1:8080/"] + interval: 5s + timeout: 3s + retries: 12 + start_period: 5s + restart: unless-stopped + + migration: + <<: *backend-common + profiles: ["operations"] + depends_on: + postgres: + condition: service_healthy + entrypoint: ["alembic"] + command: ["upgrade", "head"] + restart: "no" + + worker: + <<: *backend-common + profiles: ["operations"] + depends_on: + postgres: + condition: service_healthy + migration: + condition: service_completed_successfully + entrypoint: ["python", "-m"] + command: ["pipelines.run_pipeline", "--source", "consolidate"] + restart: "no" + +secrets: + postgres_password: + file: ${POSTGRES_PASSWORD_FILE:?Set POSTGRES_PASSWORD_FILE to a local secret file} + +volumes: + postgres_data: diff --git a/docker/frontend-nginx.conf b/docker/frontend-nginx.conf new file mode 100644 index 0000000..d53e2bd --- /dev/null +++ b/docker/frontend-nginx.conf @@ -0,0 +1,44 @@ +server { + listen 8080; + server_name _; + server_tokens off; + + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://backend:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Request-ID $request_id; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 3s; + proxy_read_timeout 35s; + client_max_body_size 1m; + } + + location = /health { + proxy_pass http://backend:8000/health; + proxy_set_header X-Request-ID $request_id; + proxy_connect_timeout 3s; + proxy_read_timeout 5s; + } + + location ^~ /health/ { + proxy_pass http://backend:8000; + proxy_set_header Host $host; + proxy_set_header X-Request-ID $request_id; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 3s; + proxy_read_timeout 5s; + } + + location / { + try_files $uri $uri/ /index.html; + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; + } +} diff --git a/docker/frontend-nginx.ecs.conf b/docker/frontend-nginx.ecs.conf new file mode 100644 index 0000000..5d2067d --- /dev/null +++ b/docker/frontend-nginx.ecs.conf @@ -0,0 +1,44 @@ +server { + listen 8080; + server_name _; + server_tokens off; + + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://127.0.0.1:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Request-ID $request_id; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 3s; + proxy_read_timeout 35s; + client_max_body_size 1m; + } + + location = /health { + proxy_pass http://127.0.0.1:8000/health; + proxy_set_header X-Request-ID $request_id; + proxy_connect_timeout 3s; + proxy_read_timeout 5s; + } + + location ^~ /health/ { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Request-ID $request_id; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 3s; + proxy_read_timeout 5s; + } + + location / { + try_files $uri $uri/ /index.html; + add_header X-Content-Type-Options nosniff always; + add_header Referrer-Policy strict-origin-when-cross-origin always; + add_header Content-Security-Policy "default-src 'self'; img-src 'self' data: https:; style-src 'self' 'unsafe-inline'; script-src 'self'; connect-src 'self'; font-src 'self' data:; object-src 'none'; base-uri 'self'; frame-ancestors 'none'" always; + } +} diff --git a/docs/audits/audit-command-log.md b/docs/audits/audit-command-log.md new file mode 100644 index 0000000..54eb477 --- /dev/null +++ b/docs/audits/audit-command-log.md @@ -0,0 +1,188 @@ +# Audit Command Log + +Audit date: 2026-07-28 +Working directory for repository commands: `/Users/manuelgrabmayer/netlight - github/FoundationIntelligencePlatform` + +Commands below were actually executed. Read-only discovery commands that differed only by a search expression are grouped, with their patterns and results stated. Secrets and environment values are intentionally absent. Exit `0` means the command itself succeeded; HTTP status is recorded separately because `curl` normally exits 0 for 4xx/5xx responses. + +## Initial state and inventory + +| Command | Exit | Result | +|---|---:|---| +| `pwd` | 0 | Expected repository path. | +| `date -u '+%Y-%m-%dT%H:%M:%SZ'` and `date '+%Y-%m-%dT%H:%M:%S%z %Z'` | 0 | Audit start recorded as 2026-07-28T19:16:52Z / 21:16:52 CEST. | +| `git status --short --branch` | 0 | Branch clean at start, ahead 1. | +| `git branch --show-current` | 0 | `12-fr-12-add-dashboard-filtering-and-drill-down`. | +| `git rev-parse HEAD` | 0 | `97d9b02491866edc9b5d1aec1183dc73e2914626`. | +| `git remote -v` | 0 | GitHub remote identified; no network write executed. | +| `uname -a`, `sw_vers`, `uname -m` | 0 | macOS 26.4.1 / Darwin 25.4.0 / arm64. | +| `python3 --version`, `venv/bin/python --version`, `node --version`, `npm --version`, `docker --version`, `docker-compose --version` | 0 | Python 3.12.3, Node 22.20.0, npm 10.9.3, Docker 28.1.1 CLI, standalone Compose 2.36.0. | +| `docker compose version` | 1 | Docker CLI reported `unknown command`; standalone `docker-compose` used. | +| `rg --files`, `find . -maxdepth 3 -type f`, `find . -name AGENTS.md -print` | 0 | Repository, config, test, workflow and data inventories created; only dependency-local Recharts AGENTS file found. | +| `du -ah src/data frontend .git | sort -h` and `find ... -exec stat ...` variants | 0 | Active DB 2.10 GB; registry DB 1.70 GB; large JSON/JSONL/raw artifacts identified. | +| `git ls-files`, `git check-ignore -v .env src/data/charities.db frontend/dist .pytest_cache` | 0 | Secrets/DB/build/test artifacts ignored; large raw JSON and notebooks tracked. | +| `git ls-files -z | xargs -0 ...` secret/size scans and `rg` secret-pattern scans | 0 | No high-confidence API token/private key in tracked files; no developer absolute-path coupling found. | +| `awk -F= '/^[A-Za-z_][A-Za-z0-9_]*=/{print $1}' .env .env.example frontend/.env.example` | 0 | Variable names inventoried without values. | +| `find . -maxdepth 4 \( -name '*.tf' -o -name 'template.yaml' -o -name 'cdk.json' -o -name 'serverless.yml' \)` | 0 | No AWS IaC found. | +| `rg -n 'sqlite3|PRAGMA|INSERT OR REPLACE|ON CONFLICT|strftime|rowid|json_extract|json_each|fts5|COLLATE NOCASE' src -g '*.py'` | 0 | SQLite portability surface quantified. | +| `rg -n 'ArgumentParser\(|@click\.command|typer\.Typer' src -g '*.py'` | 0 | Sixteen CLI entry files identified. | + +## Installation, tests, lint and build + +| Command | Exit | Result | +|---|---:|---| +| `cd frontend && npm ci` | 0 | 74 packages installed from lockfile. | +| `cd frontend && npm run test` | 0 | 8 passed, 0 failed. | +| `cd frontend && npm run lint` | 0 | Five React hook dependency warnings, no error exit. | +| `cd frontend && npm run build` | 0 | Vite production build passed; main JS 1,963.84 KB / 612.12 KB gzip; chunk warning. | +| `PYTHONPATH=src venv/bin/python -m pytest src/tests --cov=bff --cov-report=term-missing --cov-fail-under=70` | 0 | 280 passed, 3 warnings, 76.07% coverage. | +| `PYTHONPATH=src venv/bin/python -m pytest --collect-only -q src/tests` | 0 | Complete backend test/function inventory collected. | +| `venv/bin/python -m compileall -q src` | 0 | Python source compiled. | +| `venv/bin/python -m flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics` | 0 | No blocking syntax/undefined-name class issue. | +| `venv/bin/python -m flake8 src --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics` | 0 | 1,071 advisory findings; command intentionally never fails. | +| `cd frontend && npm audit --audit-level=high` | 1 | Registry/DNS unavailable under restricted network; escalation not approved. Vulnerability result unknown. | +| `python3 -m venv /private/tmp/fip-audit-venv` | 0 | Clean isolated environment created. | +| `/private/tmp/fip-audit-venv/bin/pip install -r requirements.txt` | 1 | Restricted DNS/network prevented native clean install; escalation not approved. | +| `command -v gitleaks trivy osv-scanner pip-audit` | non-zero | None installed. | + +## Development startup and local processes + +| Command | Exit / state | Result | +|---|---:|---| +| `./start_backend.sh` | running | FastAPI development process started on 127.0.0.1:8000. | +| `cd frontend && npm run dev -- --host 127.0.0.1` | running | Vite development process started on 127.0.0.1:5173. | +| `curl -I http://127.0.0.1:5173` | 0 / HTTP 200 | Frontend reachable. | +| `curl -sS http://127.0.0.1:8000/health` | 0 / HTTP 200 | Backend healthy. | +| Interactive `Ctrl-C` against prior backend/frontend sessions, then the same starts again | clean stop | Stop/restart behavior verified before the audit; current restarted processes were intentionally left running. | +| A second sandboxed `./start_backend.sh` while port 8000 was occupied | 3 | Sandbox bind attempt reported operation not permitted; inconclusive and therefore rerun with local-process permission. | +| A second permitted `./start_backend.sh` while port 8000 was occupied | 3 | Clear `[Errno 48] ... address already in use`; original backend remained healthy. | +| `cd frontend && npm run dev -- --host 127.0.0.1 --strictPort --port 5173` while 5173 was occupied | 1 | Clear `Port 5173 is already in use`; original frontend remained healthy. | + +## SQLite staging, validation and restore + +| Command | Exit | Result | +|---|---:|---| +| `sqlite3 -readonly src/data/charities.db ".backup '/private/tmp/fip-audit-staging-20260728.db'"` | 0 | Coherent 2,100,543,488-byte staging copy created. | +| `shasum -a 256 /private/tmp/fip-audit-staging-20260728.db` | 0 | `609208...895d` before staging migration exercise. | +| `sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA quick_check;'` | 0 | `ok`, 15.15 s. | +| `sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA integrity_check;'` | 0 | `ok`, 39.08 s. | +| `sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA foreign_key_check;'` | 0 | Zero rows. | +| `sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db '.tables'` and `.schema`/`PRAGMA table_info`/`foreign_key_list`/`index_list` queries | 0 | Schema, tables, indexes, FKs and FTS triggers inventoried. | +| Read-only SQL `COUNT`, `COUNT(DISTINCT ...)`, duplicate, orphan, amount/date, currency, geo, programme, organization and `dbstat` queries against staging | 0 | Counts and anomaly cohorts recorded in database report. | +| `PYTHONPATH=src venv/bin/python` staging script calling `create_tables(reset=False)` twice and `validate_database(require_foreign_keys=True)` | 0 | Idempotent; schema 7, unchanged counts, valid. | +| SQLite `.backup` from staging to `/private/tmp/fip-audit-restore-20260728.db` | 0 | Restore copy created. | +| `shasum -a 256 /private/tmp/fip-audit-restore-20260728.db` | 0 | `6f665f...0755`; logical row/FK/quick checks passed. | +| `PYTHONPATH=src venv/bin/python -m data.benchmark_registry --db /private/tmp/fip-audit-staging-20260728.db --query foundation --charity-number 200027` | 0 | FTS 651.171 ms, exact 1.997 ms, filtered 8.89 ms; expected indexes used. | + +## CLI entry-point checks + +The following exact pattern was run for each listed path: + +```zsh +PYTHONPATH=src venv/bin/python "$cli" --help >/dev/null 2>&1 +``` + +All returned exit 0: + +```text +src/data/benchmark_registry.py +src/data/registry.py +src/pipelines/backfill_ecb_exchange_rates.py +src/pipelines/curate_europe_tech_grants.py +src/pipelines/extend_observed_360giving_pilot.py +src/pipelines/import_observed_360giving_grants.py +src/pipelines/prewarm_grant_overview_cache.py +src/pipelines/reclassify_grant_enrichment.py +src/pipelines/run_pipeline.py +src/pipelines/sample_360giving_publishers.py +src/preprocessing/extract_geo_topic.py +src/preprocessing/extract_impressum.py +src/scrapers/360giving.py +src/scrapers/hinchilla.py +src/scrapers/philea.py +src/scrapers/register_of_charities.py +``` + +These checks prove command startup/help only. Live scraper/import commands were not executed. + +## API inventory, success, timing and error cases + +| Command | Exit / HTTP | Result | +|---|---:|---| +| `curl -sS http://127.0.0.1:8000/openapi.json \| jq -r '.paths \| keys[]'` | 0 | 31 paths inventoried. | +| OpenAPI `jq` query enumerating each GET/POST/PUT/PATCH/DELETE and `operationId` | 0 | 36 operations; core proxy has a duplicate operation-ID warning across methods. | +| Initial timing loop using `/api/health`, `/api/grants/overview`, `/api/registry/...`, `/api/grants/source-funders...` | 0 / HTTP 404 except charities | Incorrect guessed paths were identified, discarded and logged honestly. | +| Corrected 10-sample loop for `/health`, `/api/charities/grants/overview`, `/api/charities/directory/organizations?limit=10`; 5 samples for `/api/charities?limit=20` | 0 / HTTP 200 | Warm timings recorded in performance report. | +| Five calls to `/api/charities/grants/funders?beneficiary_country=DE&limit=20` | 0 / HTTP 200 | 0.605–2.009 s. | +| One call to funders without `beneficiary_country` | 0 / HTTP 422 | Required-query validation confirmed. | +| `curl` GETs for charity 1075920 detail, grants and Sankey | 0 / HTTP 200 | 812 grants; 31 nodes/30 links. Temporary responses stored in `/private/tmp`. | +| `curl -H 'Content-Type: application/json' -d '{"target_profile":{"programme_areas":["Education"],"geographies":["Germany"],"minimum_annual_expenditure":100000,"target_average_grant_amount":50000,"currency":"EUR","organization_types":["Foundation"]}}' http://127.0.0.1:8000/api/charities/1075920/score` | 0 / HTTP 200 | Score 90.0, confidence 0.875, not a prediction. | +| Directory `limit=101`, `income_min=-1`; invalid score type; invalid news lookback; missing funder country | 0 / HTTP 422 | Input constraints and sanitized details confirmed. | +| `curl -sS http://127.0.0.1:8000/api/charities/999999999` | 0 / HTTP 404 | Clear missing-resource message. | +| `curl -sS http://127.0.0.1:8000/api/core/health` | 0 / HTTP 503 | Unavailable fixed downstream returns sanitized 503. | +| Concurrent read-only request set: heavy map plus trends/themes/charities/registry/funders | 0 / HTTP 200 | Heavy map 67.5668 s; other requests each waited about 55.435 s. | + +All active-DB API calls above were read-only except the score POST, which is deterministic/read-only. Valid admin/enrich/relink/reset/cache mutations were not called during the audit. + +## Browser commands + +Headless Chrome was run with a fresh profile and `--no-sandbox --disable-gpu`. Each command exited 0 and wrote the named file: + +```zsh +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-1440 --window-size=1440,900 --screenshot='docs/audits/screenshots/overview-1440x900.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-warm --window-size=1440,900 --virtual-time-budget=10000 --screenshot='docs/audits/screenshots/overview-warm-1440x900.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-mobile --window-size=390,844 --virtual-time-budget=12000 --screenshot='docs/audits/screenshots/overview-mobile-390x844.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-ipadl --window-size=1024,768 --virtual-time-budget=12000 --screenshot='docs/audits/screenshots/overview-ipad-landscape-1024x768.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-ipadp --window-size=768,1024 --virtual-time-budget=12000 --screenshot='docs/audits/screenshots/overview-ipad-portrait-768x1024.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-large --window-size=1920,1080 --virtual-time-budget=8000 --screenshot='docs/audits/screenshots/overview-large-1920x1080.png' 'http://127.0.0.1:5173/' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-donors --window-size=1440,900 --virtual-time-budget=12000 --screenshot='docs/audits/screenshots/donor-directory-1440x900.png' 'http://127.0.0.1:5173/?view=donors' +'/Applications/Google Chrome.app/Contents/MacOS/Google Chrome' --headless=new --disable-gpu --no-sandbox --user-data-dir=/private/tmp/fip-chrome-audit-loaded --window-size=1440,900 --hide-scrollbars --run-all-compositor-stages-before-draw --virtual-time-budget=45000 --screenshot='docs/audits/screenshots/overview-loaded-1440x900.png' 'http://127.0.0.1:5173/' +``` + +Browser console/runtime logs also showed repeated React duplicate-key errors for `Awarded to-5Rights Foundation`. + +## Docker commands + +| Command | Exit | Result | +|---|---:|---| +| `docker info` | 1 initially | Docker daemon unavailable. | +| `open -a 'Docker 3'` | 0 | Installed Docker Desktop variant started; daemon later responsive. | +| `docker-compose build bff` | 1 initially | `docker-credential-desktop` missing. No credential file changed. | +| `mkdir -p /private/tmp/fip-audit-docker-config` and an empty `config.json` (`{}`) created for audit-only Docker config | 0 | Avoided dependency on missing credential helper; contains no credentials. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose build bff` | 0 | Clean dependency install and image build passed; 8.81 GB context, 9.37 GB image. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker image inspect foundationintelligenceplatform-bff` | 0 | ARM64, root/default user, no healthcheck. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker history foundationintelligenceplatform-bff --no-trunc` | 0 | 8.81 GB `COPY src` layer and large build-essential layer identified. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker run --rm ...` inspection commands | 0 | Embedded 2.10 GB active DB and 1.70 GB registry DB confirmed; UID 0. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose up -d bff` | 0 | Development Compose service started. | +| `curl -sS http://127.0.0.1:8000/health` against Compose service | 0 / HTTP 200 | Start health verified. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose restart bff` | 0 | Restart passed; health returned 200. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose down` | 0 | Container/network stopped and removed cleanly. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker run -d --name fip-audit-prodlike -p 127.0.0.1:8001:8000 foundationintelligenceplatform-bff` | 0 | Dockerfile default Uvicorn command started. | +| Immediate `curl` to `http://127.0.0.1:8001/health` | 52 / HTTP 000 | Empty reply during application startup. | +| Repeated `curl -sS http://127.0.0.1:8001/health` | 0 / HTTP 200 | Default-image startup verified after initialization. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker stop fip-audit-prodlike` | 0 | Audit container stopped. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker rm fip-audit-prodlike` | 0 | Audit-only container removed. Image retained for evidence. | + +## Final artifact and service validation + +| Command | Exit | Result | +|---|---:|---| +| `test -s` plus `wc -l -c` for every required Markdown report | 0 | All eight required/detail reports exist and are non-empty. | +| `awk` status aggregation over `feature-test-matrix.md` | 0 | 74 rows: 32 PASS, 37 PARTIAL, 4 FAIL, 1 NOT TESTABLE. | +| `awk` status aggregation over `requirements-traceability.md` | 0 | 32 rows: 4 PASS, 16 PARTIAL, 9 FAIL, 3 NOT IMPLEMENTED. | +| `find docs/audits/screenshots -maxdepth 1 -type f -name '*.png' \| wc -l` | 0 | Eight screenshots. | +| `sips -g pixelWidth -g pixelHeight docs/audits/screenshots/*.png` (iterated) | 0 | All eight PNG dimensions match their filenames. | +| High-confidence private-key/AWS/Google/OpenAI token-pattern scan over `docs/audits` | 0 | No match. | +| `stat` on staging and restore DB paths | 0 | Both retained at 2,100,543,488 bytes. | +| `git status --short --branch` | 0 | Only `docs/audits/` untracked; branch still ahead 1. | +| OpenAPI `jq` count | 0 | 31 paths and 36 HTTP operations. | +| Final `curl` health calls to ports 8000 and 5173 | 0 / HTTP 200 | Original backend/frontend still running after port-conflict tests. | +| `DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker ps --format ...` | 0 | No running audit container. | + +## Commands deliberately not executed + +- No `git commit`, `git push`, AWS CLI, Terraform, CDK, CloudFormation or deployment command. +- No valid active admin pipeline trigger, enrichment, relink, reset or cache mutation. +- No `DELETE`, `DROP`, reset, vacuum, repair or mass-update against the active database. +- No live full scraper/import refresh and no paid AI call during the audit. +- No deletion of staging/restore databases, raw/processed data or the large Docker image. diff --git a/docs/audits/aws-migration-plan.md b/docs/audits/aws-migration-plan.md new file mode 100644 index 0000000..278da4d --- /dev/null +++ b/docs/audits/aws-migration-plan.md @@ -0,0 +1,251 @@ +# AWS Migration Plan + +Audit date: 2026-07-28 +Recommendation: **hybrid managed architecture, with ECS/Fargate for the API and batch workers, PostgreSQL for operational data, and S3/Parquet/Athena for raw history and analytical workloads.** + +This is a plan only. No AWS resource was created and no deployment was attempted. + +## Architecture options + +| Option | Shape | Advantages | Disadvantages / fit | Verdict | +|---|---|---|---|---| +| A — Lambda-centric serverless | API Gateway + Lambda + Aurora Serverless + Step Functions/Lambda + S3 | Fine-grained scaling, low idle compute, managed integration. | Current 2.1 GB SQLite, 28–68 s synchronous queries, long scrapers, subprocess jobs, local locks and large dependencies are poor Lambda fits. Rework would be high and scraping duration/NAT cost risky. | Do not use as a lift-and-shift. Lambda may serve small control-plane functions later. | +| B — Container-first managed | CloudFront/S3 frontend + ALB/API Gateway + ECS/Fargate API and workers + RDS PostgreSQL + S3 | Closest safe evolution; supports long jobs, controlled resources and normal PostgreSQL connections. | More baseline cost and scaling configuration than Lambda; still requires data/query redesign. | Best near-term operational foundation. | +| C — Data-lake-first analytics | S3 raw/curated Parquet + Glue Catalog + Athena, with a thin API | Cheap durable history, reproducible raw lineage and strong ad-hoc analytics. | Athena is not suitable for every interactive profile write/read, admin mutation or low-latency UI call. | Required analytical complement, not the only serving store. | +| D — Hybrid (recommended) | B plus C: Fargate API/workers, RDS/Aurora PostgreSQL, S3/Parquet/Athena and precomputed caches | Separates operational, batch and analytical workloads; migration can be phased and rolled back. | More components and governance; demands explicit ownership/observability. | Recommended. | + +## Recommended target architecture + +| Current component | AWS target | Reason | +|---|---|---| +| React/Vite dev server | S3 private origin + CloudFront + WAF | Immutable static deploy, CDN caching and TLS; no frontend runtime container needed. | +| FastAPI BFF | ECS Fargate service behind internal/public ALB or API Gateway, autoscaled | Current API is container-suitable after removing embedded data and blocking I/O. | +| Public authentication | Cognito or enterprise OIDC at ALB/API layer; application RBAC | Admin and write APIs cannot remain anonymous. | +| Admin API | Separate private route/service, OIDC admin role, VPN/internal access if possible | Reduces public attack surface and isolates mutations. | +| Scrapers/imports | One-shot ECS Fargate tasks | Long, dependency-heavy, network-bound jobs do not fit short functions reliably. | +| Scheduling/orchestration | EventBridge Scheduler + Step Functions | Explicit schedules, retries, dependencies, timeouts and run history. | +| Job queue | SQS with DLQ | Durable backpressure and retry isolation for enrichment/profile jobs. | +| Operational DB | RDS PostgreSQL initially; evaluate Aurora only against measured concurrency/availability/cost | Relational constraints, concurrent reads/writes and normal migration tooling. RDS is simpler and predictable at current scale. | +| Raw source/cache files | Versioned S3 buckets/prefixes with KMS and lifecycle | Durable, immutable lineage; removes local filesystem and image coupling. | +| Historical/curated analytics | Partitioned Parquet on S3 + Glue Catalog + Athena | Lower cost for source history, reconciliation and broad scans. | +| Interactive aggregates | PostgreSQL materialized tables/views; optional ElastiCache only after measurement | Overview/map must not scan/rebuild large facts per request. | +| Secrets | Secrets Manager or SSM Parameter Store, KMS, task roles | Removes `.env` from deployed hosts and supports rotation. | +| Logs/metrics/traces | CloudWatch Logs/Metrics/Alarms, structured JSON, OpenTelemetry/X-Ray optional | Request IDs, job counts, quality metrics, errors and latency need durable observability. | +| Images | ECR with multi-architecture manifests, immutable digests and scanning | Current tested image is ARM64-only and 9.37 GB. | +| Delivery | GitHub Actions OIDC to AWS, Terraform or CDK, environment promotion | No static AWS credentials; reviewable and repeatable delivery. | + +## Data placement and S3 strategy + +Suggested logical prefixes/buckets, split further by account/environment: + +```text +s3://fip--raw//ingest_date=YYYY-MM-DD/run_id=/... +s3://fip--validated//schema_version=/ingest_date=.../... +s3://fip--curated//fact_version=/award_year=YYYY/... +s3://fip--exports///... +s3://fip--audit//manifests-and-reconciliation-only +``` + +- Enable versioning, bucket-owner enforcement, block public access, SSE-KMS and least-privilege task roles. +- Store manifests with source URL, retrieval time, HTTP metadata/checksum, schema version, record counts and job ID. +- Raw data is immutable. Corrections produce a new version rather than overwriting evidence. +- Lifecycle transitions raw historical versions to colder storage after an approved period; delete only under an approved retention policy. +- Do not place SQLite files or raw datasets in application images or Git. +- Encrypt exports separately and expire them quickly; avoid caching personal/contact data at CloudFront. + +## Athena strategy + +Athena is useful for: + +- source-to-curated reconciliation and historical audits; +- broad grant trend queries across versions; +- data-quality dashboards and anomaly investigation; +- cost-efficient, infrequent scans of raw/historical datasets. + +It is not the primary store for: + +- organization profile mutations/link overrides; +- sub-second directory/detail endpoints; +- transactionally consistent admin workflow state; +- request-time map aggregation for every user. + +Partition Parquet primarily by dataset, source and award/ingest year/month. Avoid high-cardinality partitions such as individual organization IDs. Compact small files per batch. + +## PostgreSQL mapping + +| SQLite construct | PostgreSQL design | +|---|---| +| Integer/text primary IDs | Preserve source IDs as text where semantically external; introduce generated surrogate keys only when needed. | +| JSON stored as TEXT | `jsonb` with validation/check constraints, or normalized child tables for queried fields. | +| FTS5 table/triggers | Generated/maintained `tsvector` plus GIN index; benchmark prefix/fuzzy behavior and ranking parity. | +| `INSERT OR REPLACE` | Explicit `ON CONFLICT (...) DO UPDATE`; never rely on delete/reinsert side effects. | +| `strftime` | `date_trunc`, `extract`, typed `date`/`timestamptz`. | +| `COLLATE NOCASE` | `citext`, normalized search columns or ICU collation chosen explicitly. | +| `rowid` | Declared primary key/identity and deterministic ordering. | +| `json_each` / `json_extract` | `jsonb_array_elements`, `->`, `->>` or normalized relationships. | +| SQLite file atomic publish | Load versioned staging schema/tables, validate, then transactional view/table-version switch. | +| `PRAGMA foreign_keys` | Always-enforced constraints; use controlled `NOT VALID`/deferred validation only during bulk load. | +| Local cache table | Versioned materialized aggregate tables, refreshed by worker; optional Redis after measurement. | + +Use a migration tool such as Alembic. Every migration must be forward-tested on an empty DB and an anonymized/current-shape snapshot, with a documented rollback or restore strategy. + +## Delivery phases and gates + +### Phase 0 — Blocker remediation and contracts + +Work: + +- protect or disable all mutation/admin/proxy routes by default; +- add `.dockerignore`, separate data from image, non-root multi-stage build and pinned Python dependencies; +- define source, retention, privacy, schema and SLO contracts; +- add request IDs, structured job metadata and DB-aware readiness; +- isolate synchronous DB work and precompute overview/map facts. + +Gate 0: + +- no P0 and no unresolved public-deployment P1 security finding; +- cold primary view p95 target agreed and met on representative data; +- image target agreed (for example under 500 MB without data) and multi-arch build passes; +- retention/privacy/source approvals recorded. + +Rollback: code-only; keep current local workflow and active DB unchanged. + +### Phase 1 — AWS foundation + +Work: + +- create separate development/staging AWS account/environment via Terraform/CDK; +- VPC, private subnets, endpoints, KMS, S3, ECR, CloudWatch and budgets; +- GitHub Actions OIDC role with environment protection; +- static frontend deployment and a data-free API container; +- Cognito/enterprise OIDC and private admin plane. + +Gate 1: + +- IaC plan reviewed; security checks pass; destroy/recreate in non-production is demonstrated; +- no public S3/RDS/ECS admin access; secret rotation path tested; +- image SBOM, dependency and container scans pass policy. + +Rollback: revert CloudFront/API origin to the existing non-AWS environment; destroy only tagged non-production stacks through IaC. + +### Phase 2 — S3 ingestion and worker orchestration + +Work: + +- write raw/validated outputs to versioned S3; +- run one source at a time in ECS tasks through Step Functions; +- persist run manifests, watermarks, counts, checksums and validation results; +- SQS/DLQ for enrichment work; alarms for stale/failed runs. + +Gate 2: + +- repeated run is idempotent; +- failure preserves last approved dataset; +- source count/checksum and retry simulations pass; +- lifecycle is approved but destructive expiration remains disabled until sign-off. + +Rollback: stop schedules, retain S3 versions, return pipeline reads to local/cached inputs. + +### Phase 3 — PostgreSQL schema and bulk migration + +Work: + +- provision RDS PostgreSQL in private subnets with backups/PITR; +- implement Alembic schema and loader; +- migrate base organizations/grants, then child facts/evidence, registry/search, overrides/cache metadata; +- create serving indexes/materializations and query limits. + +Gate 3 — mandatory reconciliation: + +1. Table row counts equal expected counts. +2. Distinct grant/source IDs and registry IDs match. +3. FK/orphan checks return zero unexpected rows. +4. Duplicate/anomaly cohorts equal signed baselines. +5. Control totals match: 302,546 grants; EUR 22,435,986,707.70 under implemented non-negative semantics; 104,191 mapped; 134,554 classified. +6. API golden fixtures match or have approved documented differences. +7. Search quality and query plans meet budgets. +8. Backup restore to a fresh instance succeeds. + +Rollback: keep source DB read-only snapshot and last approved PostgreSQL snapshot; point application feature flag back to SQLite/local environment. Do not discard source artifacts. + +### Phase 4 — Shadow read and performance validation + +Work: + +- send representative read queries to both backends and compare asynchronously; +- exercise dashboard, directory, map, filters, drill-down, profiles, Sankey, score and news; +- load test concurrency and failure modes; +- tune indexes/materializations and autoscaling. + +Gate 4: + +- zero unexplained control-total or entity-identity differences; +- p95/p99 and error-rate SLOs pass sustained representative load; +- no request blocks unrelated health/readiness requests; +- security/DAST and accessibility/E2E suites pass. + +Rollback: disable PostgreSQL read flag and continue shadow diagnostics. + +### Phase 5 — Staging cutover + +Work: + +- stop staging writers, take final coherent snapshot, load/validate delta; +- switch staging API to PostgreSQL and S3; +- exercise operator runbooks, alerts, DLQ replay and restore. + +Gate 5: + +- 72-hour stability window or agreed equivalent; +- on-call ownership, dashboards, alarms and runbooks accepted; +- recovery objectives demonstrated. + +Rollback: switch DB/origin feature flags back, preserve failed target for forensic comparison, replay no writes until reconciliation. + +### Phase 6 — Production preparation and cutover + +Work: + +- repeat snapshot/delta/reconciliation with change freeze; +- progressive traffic shift/canary; +- monitor totals, p95/p99, 4xx/5xx, queue age, job failures and cost. + +Gate 6: + +- production security/privacy approval; +- rollback practiced; PITR and object recovery demonstrated; +- no open P0/P1; operational acceptance signed. + +Rollback: weighted traffic back to previous origin, stop new writers, restore/reconcile from the last known-good point. Never attempt bidirectional replay without a documented conflict policy. + +## CI/CD target gates + +Pull requests should run: + +1. backend compile, blocking lint/type checks, unit/integration tests and coverage; +2. frontend lint with warnings treated by policy, unit tests and production build; +3. Playwright E2E on representative seed data; +4. secret scan, SCA, licence/SBOM, SAST and container/IaC scan; +5. data migration on empty and snapshot-shaped PostgreSQL; +6. Docker multi-arch build with image-size and non-root assertions; +7. Terraform/CDK format, validation, policy and plan. + +Staging promotion adds reconciliation, smoke/E2E, load, DAST, restore and rollback tests. Production promotion requires protected approval, immutable image digest and successful staging evidence. + +## Observability and alarms + +- Structured fields: `request_id`, `job_id`, `source`, `dataset_version`, `schema_version`, `record_count`, `retry`, `duration_ms`, `status`, `error_class`. +- Metrics: API p50/p95/p99, errors, DB pool/query duration, cache hit, overview refresh age, source watermark age, records accepted/rejected/quarantined, queue age, DLQ depth, ECS task failures and spend. +- Alarms: readiness failures, 5xx/error budget, latency, stale source, reconciliation mismatch, missing conversion spike, quality threshold breach, DLQ > 0 and budget anomaly. +- Redact query strings, authorization headers, article bodies, emails and addresses from general logs. + +## Work required before AWS begins + +The migration should not begin as a deployment exercise until Gate 0 is met. Architecture/IaC prototyping with synthetic data is acceptable, but no public or production dataset should be exposed. Required first actions: + +1. OIDC/RBAC and private admin/control plane; disable anonymous writes/proxy. +2. Data-free, non-root, slim multi-arch image and reproducible dependency lock. +3. Fix event-loop blocking and meet a representative dashboard/map p95 budget. +4. Approve retention, privacy, source/licence and data ownership contracts. +5. Define PostgreSQL schema, reconciliation controls and reversible loader. +6. Add CI security/migration/container gates and DB-aware readiness/observability. diff --git a/docs/audits/aws-readiness-audit-2026.md b/docs/audits/aws-readiness-audit-2026.md new file mode 100644 index 0000000..c40ca0e --- /dev/null +++ b/docs/audits/aws-readiness-audit-2026.md @@ -0,0 +1,648 @@ +# Foundation Intelligence Platform — AWS Readiness Audit 2026 + +Audit date: 2026-07-28 +Repository revision: `97d9b02491866edc9b5d1aec1183dc73e2914626` +Branch: `12-fr-12-add-dashboard-filtering-and-drill-down` + +## 1. Audit result + +# NO-GO + +The application is a credible functional prototype with broad backend test coverage and an internally sound SQLite dataset. It is **not ready to begin an AWS deployment/migration execution** against production data. Architecture and synthetic-data IaC prototyping may proceed, but the public/control plane, data packaging, request concurrency, governance and delivery blockers below must be resolved first. + +The strongest positive evidence is: + +- 280 backend tests pass with 76.07% measured BFF coverage; +- 8 frontend tests pass; lint and production build complete; +- SQLite `quick_check`/`integrity_check` are `ok`, with zero foreign-key violations; +- dashboard control totals reconcile exactly with SQL; +- staging migration/init is idempotent and a restore copy validates; +- Docker build and both Compose/default-image starts function locally; +- the fully loaded desktop dashboard, map, profiles, Sankey, score, filters and directories are implemented. + +The release-blocking evidence is: + +- anonymous admin, mutation and generic proxy endpoints would expose data/resource-changing functions if deployed; +- a map call took 67.57 s and caused unrelated calls to wait about 55.44 s because synchronous SQLite work blocks the async server; +- the Docker build sends 8.81 GB and produces a 9.37 GB image containing the 2.10 GB active DB and 1.70 GB registry DB; +- no PostgreSQL implementation, AWS IaC, deployment/rollback pipeline, production auth/RBAC, retention policy or operational SLO/observability exists; +- cold dashboard requests consistently took roughly 28–35 s; the mobile primary view overflows horizontally. + +## 2. Critical blockers + +| Finding | Severity | Blocker | +|---|---:|---| +| FIP-001 | P0 CRITICAL | Anonymous admin/mutation/proxy surface is unsafe for any public AWS deployment. | +| FIP-002 | P1 HIGH | Synchronous SQLite work blocks the event loop; primary/heavy queries take 28–68 s and serialize users. | +| FIP-003 | P1 HIGH | 9.37 GB root-run image embeds databases and source/processed data. | +| FIP-004 | P1 HIGH | SQLite/filesystem batch architecture is not horizontally safe or PostgreSQL-portable without substantial redesign. | +| FIP-005 | P1 HIGH | No AWS IaC, secure delivery, migration/reconciliation, rollback or production operations gates. | +| FIP-006 | P1 HIGH | Retention/privacy/data-source governance is absent while public/contact data and logs are exposed. | + +## 3. System state and audit safety + +| Item | Observed state | +|---|---| +| Host | macOS 26.4.1, Darwin 25.4.0, Apple Silicon/arm64, Europe/Berlin. | +| Runtime | Python 3.12.3, Node 22.20.0, npm 10.9.3. | +| Git | Branch ahead of origin by one pre-existing local commit; clean at audit start. No commit/push during audit. | +| Backend | FastAPI development process running on 127.0.0.1:8000 at handoff. | +| Frontend | Vite development process running on 127.0.0.1:5173 at handoff. | +| Active database | `src/data/charities.db`, 2,100,543,488 bytes; audit access was read-only. | +| Staging | Coherent SQLite `.backup` at `/private/tmp/fip-audit-staging-20260728.db`; migration tests only here. | +| Restore evidence | `/private/tmp/fip-audit-restore-20260728.db`, logically validated. | +| Docker | Daemon running; audit containers stopped/removed; 9.37 GB image retained pending approval. | +| AWS | No resource, account action, deployment, push or external write performed. | + +No active DB migration, valid admin job, scraper import, enrichment, relink, reset, domain-data deletion or source overwrite was run. Temporary copies were left in `/private/tmp` so evidence remains recoverable until the user approves cleanup. + +### Startup and failure-mode matrix + +| Scenario | Evidence | Status | +|---|---|---:| +| Frontend dependency install | `npm ci` from lockfile succeeded. | PASS | +| Native Python clean install | Isolated venv created, but package-registry DNS/network was blocked. | NOT TESTABLE | +| Start without Docker | `./start_backend.sh` and Vite dev command start; both endpoints return 200. | PASS | +| Start with Docker | Compose lifecycle and Dockerfile default command return health 200. | PASS | +| Stop/restart processes | Prior dev processes stopped via `Ctrl-C` and restarted; Compose restart/down succeeded. | PASS | +| Full host/OS reboot | Not performed; it would be outside a non-destructive repository audit. | NOT TESTABLE | +| Database init/migration | Idempotent twice on staging only. | PASS | +| Seed/demo data | No separate seed command found; application uses cached/full source data. | NOT IMPLEMENTED | +| Empty cache | Cache-miss/fallback behavior covered in backend tests; fresh browser still showed severe cold latency. | PARTIAL | +| Missing optional environment variables | Default Docker image started without passing source/AI variables; health/read DB worked. Source/AI features require their configured credentials. | PARTIAL | +| Invalid/unavailable downstream configuration | Current fixed core downstream unavailable; proxy returned sanitized 503. Validation tests cover malformed inputs. | PASS | +| Missing/corrupt/locked DB and failed publish | Existing tests use temporary copies and pass; active DB was not damaged to reproduce them. | PASS on staging/tests | +| Missing directories | Pipeline/temp-directory creation and failure paths covered by tests, not by deleting active directories. | PARTIAL | +| Backend port 8000 occupied | Second `./start_backend.sh` exited 3 with `address already in use`; original service stayed healthy. | PASS | +| Frontend port 5173 occupied | `npm run dev -- --strictPort --port 5173` exited 1 with clear error; original service stayed healthy. | PASS | + +## 4. Current architecture + +### Components + +- Frontend: React, Vite, TypeScript/JavaScript, Recharts/map rendering, localStorage favorites/news-run state. +- Backend: FastAPI BFF with 31 OpenAPI paths and 36 operations. +- Persistence: direct synchronous `sqlite3` repository, 2.10 GB single file, JSON-in-TEXT, FTS5 and materialized/cache tables. +- Ingestion: Python CLIs for 360Giving, Charity Commission, Philea, Hinchilla, optional Impressum crawling, curation, enrichment, ECB conversion and cache prewarming. +- Jobs: FastAPI background task launches subprocess commands; status, logs and `O_EXCL` locks are local files; final DB publication is file replacement. +- External services: public source endpoints/RSS, optional Gemini, Anthropic-compatible news summarization and a fixed downstream core API proxy. +- Delivery: backend Dockerfile and development Compose; GitHub Actions unit/coverage/frontend build only. + +### Data flow + +```mermaid +flowchart LR + EXT["External sources
360Giving · Charity Commission · Philea · Hinchilla · ECB · News/AI"]:::external + SCR["Scraper / import
async batch candidate"]:::batch + VAL["Validation"]:::batch + RAW["Raw/cache files
LOCAL FS · SPOF"]:::local + NORM["Normalization"]:::batch + DEDUP["Deduplication / identity"]:::batch + ENR["Geo · programme · currency · optional AI enrichment"]:::batch + STAGE["Staging SQLite
LOCAL FS"]:::localdb + DB[("Active SQLite 2.10 GB
DB dependency · SPOF")]:::localdb + API["FastAPI BFF
sync SQLite inside async handlers"]:::sync + UI["React dashboard"]:::sync + FILTER["Filters / map / trends"]:::sync + DRILL["Drill-down"]:::sync + DETAIL["Profile · grants · Sankey · score · news"]:::sync + + EXT -->|"external network"| SCR + SCR --> VAL --> RAW --> NORM --> DEDUP --> ENR --> STAGE + STAGE -->|"validate + atomic file replace"| DB + DB -->|"synchronous SQL"| API --> UI --> FILTER --> DRILL --> DETAIL + + S3[("AWS candidate: S3 raw/Parquet")]:::aws + ORCH["AWS candidate: EventBridge + Step Functions"]:::aws + WORK["AWS candidate: ECS/Fargate workers + SQS/DLQ"]:::aws + PG[("AWS candidate: RDS PostgreSQL")]:::aws + SERVE["AWS candidate: ECS API + ALB/API Gateway + OIDC/WAF"]:::aws + CDN["AWS candidate: S3 frontend + CloudFront"]:::aws + + ORCH -.-> WORK + EXT -.-> WORK -.-> S3 + S3 -.-> PG + PG -.-> SERVE -.-> CDN + + classDef external fill:#fff2cc,stroke:#a66d00; + classDef batch fill:#d9eaf7,stroke:#246; + classDef local fill:#f8d7da,stroke:#a00; + classDef localdb fill:#f5c2c7,stroke:#800,stroke-width:2px; + classDef sync fill:#e2e3e5,stroke:#555; + classDef aws fill:#d1e7dd,stroke:#175c36,stroke-dasharray:5 3; +``` + +Synchronous steps are the UI/API/request path. Scraping and enrichment are logically asynchronous batch work but currently run as local processes. Red nodes are local filesystem/database dependencies and single points of failure; green dashed nodes are managed-service candidates. + +## 5. Features tested + +The complete inventory is in `feature-test-matrix.md`. + +| Metric | Count | +|---|---:| +| Identified feature/operation entries | 74 | +| Inspected or exercised | 74 | +| PASS | 32 | +| PARTIAL | 37 | +| FAIL | 4 | +| NOT TESTABLE | 1 | + +The one `NOT TESTABLE` feature is the dependency-vulnerability/reproducibility audit: registry/package metadata access was blocked, and the requested escalation was not approved. Docker nevertheless demonstrated that current ranged Python requirements can install in a clean container; it also proved they resolve to different versions than the host, so installation success is not reproducibility. + +## 6. Requirements status + +Full per-requirement evidence, files, tests, risks, AWS effects and actions are in `requirements-traceability.md`. + +### Epics + +| EP-00 | EP-01 | EP-02 | EP-03 | EP-04 | EP-05 | EP-06 | +|---|---|---|---|---|---|---| +| PARTIAL | PARTIAL | PARTIAL | PARTIAL | FAIL | FAIL | FAIL | + +### Functional requirements + +| ID | Status | ID | Status | ID | Status | +|---|---|---|---|---|---| +| FR-01 | PARTIAL | FR-07 | PARTIAL | FR-13 | PASS | +| FR-02 | PARTIAL | FR-08 | PASS | FR-14 | NOT IMPLEMENTED | +| FR-03 | PARTIAL | FR-09 | PASS | FR-15 | NOT IMPLEMENTED | +| FR-04 | PARTIAL | FR-10 | PARTIAL | FR-16 | NOT IMPLEMENTED | +| FR-05 | PARTIAL | FR-11 | PASS | FR-17 | PARTIAL | +| FR-06 | PARTIAL | FR-12 | PARTIAL | | | + +### Non-functional requirements + +| NFR-01 | NFR-02 | NFR-03 | NFR-04 | NFR-05 | NFR-06 | NFR-07 | NFR-08 | +|---|---|---|---|---|---|---|---| +| FAIL | FAIL | PARTIAL | FAIL | FAIL | FAIL | FAIL | PARTIAL | + +Totals: 4 PASS, 16 PARTIAL, 9 FAIL, 3 NOT IMPLEMENTED across 32 requirements. + +## 7. Findings + +### FIP-001 — Anonymous control plane and mutation surface + +- **Severity:** P0 CRITICAL. +- **Affected component:** FastAPI admin, funder enrichment/relink/reset/cache, registry enrichment and generic core proxy routes. +- **Description:** The application has no authentication/authorization layer. If reachable publicly, callers can start resource-intensive pipelines, alter matching/enrichment state, populate/reset caches and bridge requests to the configured downstream core. +- **Reproduction:** Inspect OpenAPI; call status/log/proxy without credentials. The unavailable proxy returned a sanitized 503, confirming anonymous reachability. Valid mutation was intentionally not performed on active data. +- **Expected:** Public reads follow policy; every control/write operation requires OIDC identity, least-privilege role and auditable intent. Proxy paths/methods/headers are allowlisted. +- **Actual:** No identity or permission dependency is present. The UI labels a local “Netlight Guest / Administrator” without real authentication. +- **Evidence:** 36 OpenAPI operations; security dependency/middleware inventory; endpoint tests; route code; anonymous local calls. +- **Cause:** Prototype/demo trust model was retained while BFF auth was removed. +- **Impact:** Unauthorized resource use, data-state changes, downstream access and potential security incident. +- **AWS impact:** Absolute public-deployment blocker; WAF alone is insufficient. +- **Recommended solution:** Default-deny writes, separate private admin service/routes, OIDC/Cognito/enterprise SSO, RBAC, path/header allowlist, CSRF/idempotency protection where relevant, audit actor/reason/result. +- **Effort:** Large (2–4 weeks plus identity integration/security review). +- **Dependencies:** Identity provider, role model, downstream core trust policy, threat model. +- **Regression test:** Unauthenticated writes/proxy return 401; wrong role 403; permitted role succeeds with immutable audit event; rate limits and path allowlist enforced. + +### FIP-002 — Event-loop blocking and extreme cold/heavy latency + +- **Severity:** P1 HIGH. +- **Affected component:** FastAPI handlers, `SQLiteCharityRepository`, overview/map/ranking/search queries. +- **Description:** Async routes execute synchronous SQLite aggregation. Heavy work blocks unrelated requests on the sole event loop. +- **Reproduction:** Call the GBP map with connections while issuing trends/themes/charities/registry/funder requests concurrently. +- **Expected:** Primary views meet an agreed p95 (recommended <=3 s) and one heavy query does not stall health or unrelated reads. +- **Actual:** Map 67.5668 s; peer calls ~55.435 s; fresh dashboard 28–35 s. Warm overview is ~3.24 ms, masking cold behavior. +- **Evidence:** `performance/runtime-measurements.md`, request logs, cold/loading and fully loaded screenshots. +- **Cause:** Request-time full aggregations and blocking `sqlite3` inside async handlers, with incomplete cache coverage. +- **Impact:** Timeouts, unusable UX and no safe concurrency headroom. +- **AWS impact:** Scaling more tasks hides but does not fix the query/event-loop design and raises cost. +- **Recommended solution:** PostgreSQL async pool or explicit worker threads during transition; versioned materialized overview/map/ranking facts; cache; query limits; remove redundant startup fetches. +- **Effort:** Large (3–6 weeks including load tests). +- **Dependencies:** Target data model, SLOs, representative load/dataset. +- **Regression test:** Concurrent load where a worst-case map call cannot push health/read p95 over budget; cold/warm p50/p95/p99 gates. + +### FIP-003 — Application image embeds 8+ GB of data and runs as root + +- **Severity:** P1 HIGH. +- **Affected component:** `Dockerfile`, build context, Compose. +- **Description:** No `.dockerignore`; `COPY src/` copies active/processed databases and data. Build tools remain in the runtime image; user is root; no image healthcheck/multi-stage/frontend image strategy. +- **Reproduction:** Build with standalone Compose, inspect image/history and list `/app/src/data`. +- **Expected:** Small reproducible data-free image, non-root runtime, healthcheck, pinned dependencies and multi-arch artifacts. +- **Actual:** 8.81 GB context, 9,368,380,422-byte ARM64 image, 2.10 GB active DB and 1.70 GB registry DB embedded. +- **Evidence:** Docker build/history/inspect/run outputs. +- **Cause:** Broad copy and absence of image/data separation. +- **Impact:** Slow builds/pulls, stale/confidential data copies, disk/ECR cost and unsafe runtime permissions. +- **AWS impact:** Blocks acceptable ECR/ECS deployment. +- **Recommended solution:** `.dockerignore`, explicit package/module copy, multi-stage wheels, non-root UID, healthcheck, pinned lock, S3/RDS data, buildx AMD64+ARM64. +- **Effort:** Medium (2–5 days, plus dependency-lock work). +- **Dependencies:** Runtime data-access redesign and artifact policy. +- **Regression test:** CI asserts no `src/data` payload, non-root UID, healthcheck, supported architectures and image-size budget. + +### FIP-004 — SQLite/filesystem architecture is not cloud-portable + +- **Severity:** P1 HIGH. +- **Affected component:** repositories, schema, pipeline publication, locks/status/logs, FTS/search. +- **Description:** Extensive SQLite dialect/API coupling and single-host filesystem state require a real persistence/job redesign, not a connection-string change. +- **Reproduction:** Static SQL inventory and staging schema/migration analysis. +- **Expected:** Managed transactional store, horizontally safe jobs, durable raw/run state and reversible migrations. +- **Actual:** Direct `sqlite3`, FTS5, pragmas, `INSERT OR REPLACE`, `strftime`, `rowid`, `json_each/extract`, local locks/logs and file replacement. +- **Evidence:** `database-integrity-report.md`; about 111 direct sqlite references and dozens of dialect-specific constructs. +- **Cause:** Optimized local prototype architecture. +- **Impact:** High migration defect risk, inconsistent semantics and single-host operation. +- **AWS impact:** Fundamental data-plane migration blocker. +- **Recommended solution:** RDS PostgreSQL schema/Alembic, versioned loader/reconciliation, S3 raw/Parquet, ECS workers, Step Functions and SQS/DLQ. +- **Effort:** Extra large (6–12+ weeks depending on scope). +- **Dependencies:** Data contracts, retention, performance target and dual-read strategy. +- **Regression test:** Empty/snapshot migration, table/key/FK/control-total reconciliation, golden API diff, search parity, restore and rollback rehearsal. + +### FIP-005 — Delivery and operations gates are incomplete + +- **Severity:** P1 HIGH. +- **Affected component:** `.github/workflows/ci.yml`, repository infrastructure/operations. +- **Description:** CI runs backend coverage and frontend build only for main/dev. It lacks frontend tests/lint, container, SCA/SAST/secret/IaC scans, DB migration/reconciliation, staging deploy, smoke/load/DAST and rollback. +- **Reproduction:** Inspect the only workflow and search for IaC/deployment configuration. +- **Expected:** Reviewable IaC and protected staged delivery with immutable artifacts, security/data gates and rollback proof. +- **Actual:** No AWS IaC/deploy/rollback workflow or operational runbook. +- **Evidence:** Workflow/config inventory; local verified commands show what can be promoted into CI. +- **Cause:** Delivery has not progressed beyond local/validation CI. +- **Impact:** Unrepeatable deployment, undetected supply-chain/data migration regressions and poor recoverability. +- **AWS impact:** Blocks controlled staging/production delivery. +- **Recommended solution:** Terraform/CDK, GitHub OIDC, environment promotion, SBOM/scans, migration/reconciliation, smoke/E2E/load and rollback jobs. +- **Effort:** Large (3–6 weeks alongside platform foundation). +- **Dependencies:** AWS accounts, IAM/security standards, target architecture and SLOs. +- **Regression test:** Recreate non-prod from IaC; deploy immutable digest; fail a gate; demonstrate automatic/manual rollback and restore. + +### FIP-006 — Data governance, privacy and retention are absent + +- **Severity:** P1 HIGH. +- **Affected component:** registry/profile data, news/contact extraction, pipeline logs, raw/processed artifacts and exports. +- **Description:** No approved retention schedule, legal/source register, data classification, deletion workflow, privacy review or data-subject process was found. Logs and endpoints can expose addresses/emails/contact evidence. +- **Reproduction:** Inspect schema/artifacts/log samples, admin log endpoint and repository documentation. +- **Expected:** Classified data, lawful/source terms, minimal/redacted logs, lifecycle policy and auditable deletion/hold/restore procedure. +- **Actual:** Indefinite local accumulation and anonymous log access. +- **Evidence:** `data-retention-and-deletion-candidates.md`; source/log/security inventory. +- **Cause:** Governance was not implemented for prototype data flows. +- **Impact:** Compliance/privacy exposure and uncontrolled storage cost. +- **AWS impact:** Blocks production S3/RDS lifecycle and public deployment. +- **Recommended solution:** Source/licence register, DPIA/privacy review, dataset owners, retention/holds, field-level exposure rules, redaction and approved deletion workflow. +- **Effort:** Large and cross-functional (2–6 weeks for baseline, ongoing governance). +- **Dependencies:** Legal/privacy/security and product decisions. +- **Regression test:** Policy-as-code lifecycle review, access tests, log redaction fixtures, deletion dry run plus restore/hold tests. + +### FIP-007 — Currency documentation and anomaly cohorts need resolution + +- **Severity:** P2 MEDIUM. +- **Affected component:** ECB pipeline, README, grant facts/overview. +- **Description:** Stored/tested conversion uses ECB monthly averages, while README describes daily award-date/previous-business-day behavior. 432 grants lack conversion, two grants are negative and one is future-dated. +- **Reproduction:** Group conversion status/currency and compare API total/README/tests. +- **Expected:** One approved, versioned conversion contract with disclosed exclusions and quality gates. +- **Actual:** Central total is internally correct but documentation semantics differ. +- **Evidence:** Exact SQL/API reconciliation in database report. +- **Cause:** Documentation drift and incomplete rates/source anomalies. +- **Impact:** Analysts can misunderstand totals; future migrations may “correct” behavior inconsistently. +- **AWS impact:** Baseline/reconciliation must freeze the intended metric before cutover. +- **Recommended solution:** Approve conversion policy, fix docs/status naming, quarantine anomalies and add control-total fixtures. +- **Effort:** Small–medium (2–5 days plus business approval). +- **Dependencies:** Product/data owner decision. +- **Regression test:** Currency/date fixture matrix and fixed overview control totals across SQLite/PostgreSQL. + +### FIP-008 — Mobile primary view overflows + +- **Severity:** P2 MEDIUM. +- **Affected component:** overview header, KPI grid, map mode controls/responsive CSS. +- **Description:** At 390×844, controls/cards extend beyond the viewport and content is cropped. +- **Reproduction:** Fresh headless Chrome screenshot at 390×844. +- **Expected:** No horizontal page overflow; all primary controls accessible. +- **Actual:** Right-hand KPI/control content is off-screen; prolonged loader remains. +- **Evidence:** `screenshots/overview-mobile-390x844.png`; tablet/desktop comparison screenshots. +- **Cause:** Fixed/minimum widths and multi-column layout without mobile collapse/wrap. +- **Impact:** Mobile users cannot reliably use primary features. +- **AWS impact:** CDN deployment does not solve client layout; staging UX gate needed. +- **Recommended solution:** Responsive single-column/cards, scroll-contained map controls and breakpoint tests. +- **Effort:** Medium (2–5 days). +- **Dependencies:** UX acceptance criteria. +- **Regression test:** Playwright screenshots/overflow assertions at 320/390/768/1024/1440 widths. + +### FIP-009 — Frontend bundle/fetch/render quality debt + +- **Severity:** P2 MEDIUM. +- **Affected component:** React `App`, loading/fetch effects and rendered grant lists. +- **Description:** Main JS is 1.96 MB, lint emits five hook-dependency warnings, runtime logs show repeated duplicate React keys, and fresh views appear to repeat/shared-block startup requests. +- **Reproduction:** `npm run lint/build`, browser runtime logs and fresh-route timings. +- **Expected:** Stable dependency effects, unique keys, bounded chunks and independent loading states. +- **Actual:** Warnings, duplicate-key errors and large main bundle. +- **Evidence:** Build/lint logs and repeated `Awarded to-5Rights Foundation` console errors. +- **Cause:** Large central component/state graph and non-unique display-derived keys. +- **Impact:** Stale/repeated fetches, reconciliation bugs and slower download/parse. +- **AWS impact:** Higher CDN transfer and noisy production behavior. +- **Recommended solution:** Fix dependencies/abort stale requests, use stable IDs, split routes/heavy map/chart dependencies and add console-error E2E gate. +- **Effort:** Medium (1–2 weeks). +- **Dependencies:** UI refactor boundaries and caching strategy. +- **Regression test:** Zero console errors/warnings in E2E, request-count assertions and bundle-size budget. + +### FIP-010 — Dependency security/reproducibility is unknown + +- **Severity:** P2 MEDIUM. +- **Affected component:** `requirements.txt`, npm dependencies, CI and images. +- **Description:** Python requirements use ranges; host and clean image resolved different FastAPI/Anthropic versions. Vulnerability scanners are absent and registry audit was blocked by the controlled environment. +- **Reproduction:** Compare installed host/container versions; attempt `npm audit` and isolated pip install. +- **Expected:** Locked hashes, SBOM, licence/SCA gates and repeatable build. +- **Actual:** Clean Docker install succeeds but is not version-reproducible; vulnerability status unknown. +- **Evidence:** FastAPI 0.139.2 host vs 0.140.13 image; Anthropic 0.118.0 vs 0.120.2; blocked audit logs. +- **Cause:** Ranged requirements and missing supply-chain pipeline. +- **Impact:** Future builds can change behavior or include unknown vulnerabilities. +- **AWS impact:** ECR artifact cannot be confidently promoted. +- **Recommended solution:** Generate reviewed lock with hashes, Renovate/Dependabot policy, SBOM, SCA/container scan and signed digest. +- **Effort:** Medium (2–5 days initial, ongoing updates). +- **Dependencies:** Network-enabled CI and security policy. +- **Regression test:** Rebuild twice to identical dependency manifest; policy fails on prohibited vulnerability/licence. + +### FIP-011 — Health, logs and job observability are not production-grade + +- **Severity:** P2 MEDIUM. +- **Affected component:** middleware, `/health`, admin status/log files and jobs. +- **Description:** Health reports process only; logs lack durable correlation/run metrics; status is a local file; no alerts, traces, quality metrics or DLQ. +- **Reproduction:** Call health while inspecting code/status/log formats and concurrency behavior. +- **Expected:** Separate liveness/readiness, request/job IDs, structured metrics, alerts and durable run history. +- **Actual:** Healthy process can be blocked by a query or have unavailable dependencies; logs may expose data. +- **Evidence:** Health 200 normally, queued behavior during heavy request, local status/log files. +- **Cause:** Development-oriented observability. +- **Impact:** Slow detection, incomplete audit trail and weak recovery. +- **AWS impact:** Autoscaling/alarms cannot act on meaningful state. +- **Recommended solution:** DB-aware readiness, structured redacted logs, CloudWatch metrics/alarms, job manifest and SQS DLQ. +- **Effort:** Medium (1–2 weeks). +- **Dependencies:** Target services/SLOs and privacy rules. +- **Regression test:** Inject DB/source/job failures and assert readiness, metrics, alarm and redacted log behavior. + +### FIP-012 — Live source completeness, rate policy and legal evidence not verified + +- **Severity:** P2 MEDIUM. +- **Affected component:** all scraper/news source integrations. +- **Description:** Mocked retry/pagination/SSRF tests are strong, but no audit-time live full scrape was performed and no approved source/robots/licence register exists. Google News RSS does not prove complete historical pagination. +- **Reproduction:** Review tests/source clients and cached-source distributions; note restricted/no-cost audit boundary. +- **Expected:** Contracted source behavior, legal/politeness policy, freshness SLA and live canary evidence. +- **Actual:** Code/cached data pass; external completeness and current terms remain assumptions. +- **Evidence:** All CLI help exits 0, scraper tests pass, current grants all 360Giving; no live command in command log. +- **Cause:** External/cost/legal constraints and missing formal source governance. +- **Impact:** Silent data gaps or source blocking. +- **AWS impact:** Schedules/quotas/egress cannot be safely finalized. +- **Recommended solution:** Approved source register, low-volume canaries, rate budgets, robots/terms checks and watermark alarms. +- **Effort:** Medium, source-dependent. +- **Dependencies:** Credentials, network, legal approval and source owners. +- **Regression test:** Scheduled canary with schema/checksum/count/freshness assertions and safe retry cap. + +### FIP-013 — Schema versioning and FK enforcement are connection-dependent + +- **Severity:** P3 LOW. +- **Affected component:** SQLite connections/migrations/metadata. +- **Description:** `PRAGMA user_version=0`, application schema is metadata version 7, and fresh connections default to `foreign_keys=0`. +- **Reproduction:** Read pragmas/metadata on staging. +- **Expected:** One authoritative migration ledger and invariant FK enforcement. +- **Actual:** Application validation succeeds, but enforcement depends on connection setup. +- **Evidence:** Integrity/FK/migration report. +- **Cause:** Application-managed schema history and SQLite defaults. +- **Impact:** A new connection path could bypass FK enforcement. +- **AWS impact:** PostgreSQL will reveal load-order/constraint assumptions. +- **Recommended solution:** Central connection factory assertion now; Alembic ledger and always-on constraints in target. +- **Effort:** Small now, medium as part of migration. +- **Dependencies:** Migration design. +- **Regression test:** Every repository connection asserts FK on; mutation tests fail on orphan insert. + +### FIP-014 — Complexity and advisory lint debt raise change risk + +- **Severity:** P3 LOW. +- **Affected component:** backend repositories/routes/pipeline and frontend hooks. +- **Description:** Advisory flake8 reports 1,071 findings, including complexity up to 74 and five F811 redefinitions; deprecation warnings are present. +- **Reproduction:** Run full advisory flake8 and tests. +- **Expected:** Maintainable functions with clear lint policy and no ambiguous redefinitions. +- **Actual:** Blocking syntax lint passes and tests pass, but high-complexity hot paths are risky to port. +- **Evidence:** `get_grant_overview` complexity 74; map/repository methods also high; FastAPI/Starlette deprecation warnings. +- **Cause:** Feature growth in central repository/UI modules and advisory-only style policy. +- **Impact:** PostgreSQL/security/performance changes are harder to review and regress. +- **AWS impact:** Increases migration lead time and defect probability. +- **Recommended solution:** Targeted decomposition around data-access/query contracts, remove redefinitions/deprecations, adopt incremental lint baseline. +- **Effort:** Medium–large, phased. +- **Dependencies:** Do after behavior/golden tests, not as broad unaudited rewrite. +- **Regression test:** Complexity/duplicate-definition gate for changed files plus unchanged golden API/data results. + +## 8. Entity and provenance separation + +- **Donors/source funders:** ranked from observed `fundingOrganization` facts and keyed separately from curated organization profiles. A source funder may remain `observed_only`, be linked through a reviewed override, or have a cached enriched profile. The Germany ranking and observed-only detail behavior were tested. +- **Recipients:** award recipients remain grant transaction parties. Recipient IDs/names drive grant lists, summaries and Sankey relationships; they are not silently treated as donors. Charity 1075920 returned 812 recipient-side grants in the tested role. +- **Organizations/registry records:** the 373 curated organizations and 397,469 Charity Commission directory records are distinct populations joined through 345 explicit accepted links. The 299 Philea profiles are valid organization-only records with no implied grant transactions. +- **Source evidence:** source name/record ID/URL, ingestion timestamp, original currency/amount, classification method/evidence/confidence/review flags and override revisions were inspected. Eight curated organizations have blank/unknown source provenance and are review candidates. + +These distinctions must remain explicit in PostgreSQL keys and API contracts; a broad “organization” merge would corrupt donor/recipient roles and legitimate registry constituent records. + +## 9. Database status + +| Area | Status | +|---|---| +| Integrity | PASS: `quick_check=ok`, `integrity_check=ok`. | +| Foreign-key violations | 0 on staging and restore. | +| Orphans | 0 for checked grant/organization relationships. | +| Identity duplicates | 0 duplicate grant source IDs; registry IDs unique. | +| Business duplicates | 4,271 grant business-key groups; retain pending source-aware review. | +| Registry shared charity numbers | 9,073 groups; mostly legitimate constituent records, not deletion candidates. | +| Migration readiness | FAIL for direct cutover; strong baseline for a controlled loader/reconciliation project. | +| Control totals | 302,546 grants; EUR 22,435,986,707.70 implemented overview total; 104,191 mapped; 134,554 classified. | + +Detailed schema, storage, anomalies and PostgreSQL mapping are in `database-integrity-report.md`. + +## 10. Data retention status + +Must preserve: active DB until rollback expiry, immutable raw source evidence, source IDs/URLs/timestamps, enrichment evidence/rule versions, negative/zero correction records, registry constituents, overrides/revisions and accepted audit baselines. + +Archive candidates: superseded coherent DB snapshots, processed/JSONL batches, raw versions and redacted job histories, preferably encrypted/versioned in S3 with an approved lifecycle. + +Potential technical deletion after explicit approval: 4.2 GB of `/private/tmp` staging/restore copies, incomplete temp virtualenv/browser profiles/API samples, ignored build/test caches and the 9.37 GB local Docker image. + +Nothing was deleted because there is no approved retention policy and the audit had an explicit non-destructive mandate. Full classification is in `data-retention-and-deletion-candidates.md`. + +## 11. Performance status + +| Component | Status and evidence | +|---|---| +| Dashboard | FAIL cold: roughly 28–35 s; loaded by 45 s. Warm overview API is fast. | +| API | Mixed: health/list/detail are milliseconds warm; funder ranking 0.6–2.0 s; heavy calls serialize peers. | +| Database | Structurally sound; exact/indexed registry queries fast, FTS sample 651 ms; dynamic aggregates dominate. | +| Map | FAIL: 67.57 s with connections. | +| Directory | Registry page ~2 ms warm; FTS 651 ms; donor view still loading at 12 s in a fresh browser. | +| Scrapers | Functional/retry tests pass; no live throughput/rate-limit benchmark. | +| Frontend artifact | Main JS 1.96 MB / 612 KB gzip; chunk warning. | +| Docker | 8.81 GB context, 9.37 GB image. | + +Raw samples and limitations are in `performance/runtime-measurements.md`. + +## 12. Security status + +- Critical: anonymous writes/admin/proxy; public deployment readiness is **FAIL**. +- Secrets: `.env` is ignored; no high-confidence secret/private key was found in tracked files or notebooks. This is not a substitute for CI secret scanning. +- Dependencies: vulnerability status **NOT TESTABLE** in this environment; no gitleaks/Trivy/OSV/pip-audit installed; Python dependencies are not locked reproducibly. +- API: no inbound rate limiting, OIDC/RBAC, private admin plane or formal CSRF/idempotency/audit actor controls. +- News: SSRF defenses include public-IP checks, redirect revalidation, bounds and timeouts and have tests; DNS-rebinding/TOCTOU and provider egress still need review. +- Logs/privacy: admin logs are anonymous and may include names/emails/addresses; public registry/profile fields require classification and legal review. +- Proxy: fixed downstream URL reduces arbitrary-host SSRF, but the bridge remains anonymous and forwards methods/authorization to a trusted service. +- Container: root/default user, embedded data, build tools and no scan/healthcheck. + +## 13. Test coverage and honesty + +Executed: + +- all existing backend tests: 280 passed, 76.07% BFF coverage; +- all existing frontend tests: 8 passed; +- frontend lint/build and backend compile/blocking/advisory lint; +- API success/error/timing calls, OpenAPI inventory and concurrency test; +- desktop/tablet/mobile browser rendering including a fully loaded view; +- Docker clean build, Compose start/restart/stop and Dockerfile default start/stop; +- coherent SQLite backup, integrity/FK/schema/count/anomaly/migration/restore/index benchmarks; +- all 16 Python CLI help/startup surfaces. + +Not executed or not fully testable: + +- live full scrapers/import/pipeline refresh (would use external systems and write large/active artifacts); +- valid active admin/enrich/relink/reset/cache mutations (non-destructive mandate); these passed with temp DB/mocks; +- live audit-time Gemini/Anthropic/news cost-bearing calls; news behavior used mocks and existing earlier runtime evidence; +- npm/Python vulnerability audit and clean native dependency install (restricted registry/DNS, escalation rejected); +- AMD64/multi-arch container, real AWS, PostgreSQL, production load/soak, DAST, screen reader/keyboard/axe and complete cross-browser testing; +- downstream core happy-path integration (service unavailable; sanitized 503 verified). + +Remaining assumptions: + +- cached third-party data is representative of source schema/current terms; +- fixture-based scraper tests reflect current external behavior; +- local Apple Silicon performance is diagnostic, not production sizing; +- source licences/privacy bases and target SLO/RTO/RPO still require owner approval. + +Staging-only tests: schema initialization/migration, validation, restore, integrity/FK checks, repository benchmark and mutation/failure tests. Active DB was read-only. + +## 14. Docker and CI/CD readiness + +Docker starts, but is not deployable as-is. Required changes: data-free `.dockerignore`/copy set, non-root multi-stage runtime, locked dependencies, healthcheck, frontend static artifact, ARM64+AMD64 build, SBOM and container scan. Compose is development-only because it uses a source mount and `--reload`; `version: 3.8` is obsolete. + +CI currently validates backend tests/coverage and frontend build. Before AWS it must add frontend tests/lint, E2E/accessibility, dependency/secret/SAST/container/IaC scans, image-size/user assertions, PostgreSQL migrations/reconciliation, staging smoke/load/DAST, immutable promotion and rollback/restore evidence. + +## 15. Recommended AWS architecture + +Use the hybrid option: + +1. React build in private S3 behind CloudFront/WAF. +2. FastAPI read service in ECS Fargate behind ALB/API Gateway, with OIDC and autoscaling. +3. Separate private/admin service or route plane with strict RBAC. +4. RDS PostgreSQL initially; evaluate Aurora only from measured availability/concurrency/cost. +5. ECS one-shot scraper/import/enrichment workers, EventBridge schedules, Step Functions orchestration and SQS/DLQ. +6. Versioned/KMS-encrypted S3 raw and validated zones; curated Parquet + Glue/Athena for history/reconciliation. +7. Versioned/materialized PostgreSQL serving facts; add ElastiCache only after profiling. +8. Secrets Manager/Parameter Store, task roles/private networking and explicit egress. +9. CloudWatch structured logs/metrics/alarms, request/job IDs, DB-aware readiness and quality/freshness dashboards. +10. GitHub Actions OIDC + Terraform/CDK, protected staged promotion and reversible cutover. + +The three evaluated base options and phased gates are documented in `aws-migration-plan.md`. + +## 16. Required work before AWS + +### Before beginning migration execution + +1. Resolve FIP-001: OIDC/RBAC, private admin plane, default-deny mutations and constrained proxy. +2. Resolve FIP-003: data-free slim non-root multi-arch image and dependency lock/SBOM. +3. Resolve FIP-002: eliminate request blocking and meet agreed primary-view/concurrency p95. +4. Approve data/source/privacy/retention contracts and redaction/exposure rules. +5. Freeze PostgreSQL schema/control totals and build reversible loader/reconciliation plan. +6. Define IaC, accounts, IAM, observability, SLO/RTO/RPO and CI security/migration gates. + +### During migration + +- establish S3 raw/validated/curated zones and run manifests; +- move source jobs to ECS/Step Functions/SQS/DLQ; +- migrate PostgreSQL in dependency order and reconcile counts, identities, FKs, duplicates and central metrics; +- implement FTS/search and materialized aggregate parity; +- shadow reads and golden API comparisons before traffic switch. + +### After first staging deployment + +- run sustained load/soak, DAST, multi-browser E2E/a11y, failure injection and cost tests; +- exercise alarms, DLQ replay, backup restore and origin/DB rollback; +- close all unexplained data/query differences and observe a stability window. + +### Before production + +- no open P0/P1; +- production security/privacy/legal and operational acceptance; +- tested canary/progressive traffic, PITR/object restore and rollback; +- signed retention lifecycle, incident/on-call runbooks, budgets and SLOs. + +## 17. Verified commands + +Only commands actually executed are presented as working. + +### Installation + +```zsh +cd frontend +npm ci +``` + +Native clean `pip install -r requirements.txt` was **not verified** because registry/DNS access was blocked. The Docker build did install the same requirements in a clean image. + +### Development without Docker + +```zsh +./start_backend.sh +cd frontend +npm run dev -- --host 127.0.0.1 +``` + +### Tests and build + +```zsh +PYTHONPATH=src venv/bin/python -m pytest src/tests --cov=bff --cov-report=term-missing --cov-fail-under=70 +venv/bin/python -m compileall -q src +venv/bin/python -m flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics +cd frontend +npm run test +npm run lint +npm run build +``` + +### Database check and benchmark + +```zsh +sqlite3 -readonly src/data/charities.db ".backup '/private/tmp/fip-audit-staging-20260728.db'" +sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA quick_check;' +sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA integrity_check;' +sqlite3 -readonly /private/tmp/fip-audit-staging-20260728.db 'PRAGMA foreign_key_check;' +PYTHONPATH=src venv/bin/python -m data.benchmark_registry --db /private/tmp/fip-audit-staging-20260728.db --query foundation --charity-number 200027 +``` + +### Docker build/start/stop + +```zsh +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose build bff +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose up -d bff +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose restart bff +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose down +``` + +### Production-like image command + +```zsh +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker run -d --name fip-audit-prodlike -p 127.0.0.1:8001:8000 foundationintelligenceplatform-bff +curl -sS http://127.0.0.1:8001/health +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker stop fip-audit-prodlike +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker rm fip-audit-prodlike +``` + +### Scraper/import commands + +Only `--help` startup was verified for all scraper/import CLIs. No live command is claimed as functioning in this environment. See `audit-command-log.md` for the exact list. + +Stopping non-Docker dev servers was verified interactively with `Ctrl-C`; the restarted backend/frontend were left running intentionally. + +## 18. Files created + +- `docs/audits/aws-readiness-audit-2026.md` +- `docs/audits/feature-test-matrix.md` +- `docs/audits/requirements-traceability.md` +- `docs/audits/database-integrity-report.md` +- `docs/audits/aws-migration-plan.md` +- `docs/audits/audit-command-log.md` +- `docs/audits/data-retention-and-deletion-candidates.md` +- `docs/audits/performance/runtime-measurements.md` +- eight PNG screenshots under `docs/audits/screenshots/` + +## 19. Final test-honesty statement + +This conclusion distinguishes inspected code, mocked/temp-DB tests, read-only full-data calls and live external execution. It does not infer a passing external integration from a mocked test, does not call a help screen a successful scrape, does not treat a warm cache as cold performance, and does not treat a passing local ARM64 image as multi-architecture readiness. No AWS action, commit, push, active domain mutation or data deletion occurred. diff --git a/docs/audits/data-retention-and-deletion-candidates.md b/docs/audits/data-retention-and-deletion-candidates.md new file mode 100644 index 0000000..065cfaa --- /dev/null +++ b/docs/audits/data-retention-and-deletion-candidates.md @@ -0,0 +1,84 @@ +# Data Retention and Deletion Candidates + +Audit date: 2026-07-28 + +## Safety statement + +No domain data, database row, raw source artifact or user-owned change was deleted during this audit. Candidates are classifications for review, not deletion instructions. Every deletion requires an approved retention policy, an identified owner, a recoverability check and explicit authorization. + +## Data that must be preserved + +| Data | Why preserve it | Risk if removed | Recommended target | +|---|---|---|---| +| `src/data/charities.db` active database | Current integrated application state and audit baseline. | Irrecoverable service/data loss. | Keep read-only baseline until PostgreSQL reconciliation and rollback window close; then archive encrypted under policy. | +| Raw 360Giving, Charity Commission, Philea and other source artifacts | Source evidence, reproducibility and reprocessing. | Cannot explain or rebuild derived results. | Versioned immutable S3 raw zone with checksum/manifests and lifecycle. | +| Source IDs, URLs, timestamps and evidence fields | Identity, provenance and auditability. | False deduplication/linking and unverifiable claims. | Preserve in operational schema and curated history. | +| Enrichment method, confidence, review flags and rule versions | Explains geo/programme classifications. | Silent model/rule drift. | Preserve with each derived record/version. | +| Link overrides and profile-cache revision history | Operator decisions and correction trail. | Repeated bad matching or unexplained profile changes. | Transactional table with actor/time/reason audit record. | +| Negative and zero grants | Can represent corrections, reversals or valid source semantics. | Incorrect totals and loss of source fidelity. | Retain; explicitly classify and exclude/include according to documented metric rules. | +| Duplicate business-key groups | Unique source IDs indicate possible legitimate tranches/awards. | Accidental loss of distinct grants. | Keep until source-aware adjudication. | +| Registry rows sharing a charity number | Often constituent funds or linked registrations. | Directory records and legal identities lost. | Keep; model charity-number relationship rather than deduplicate destructively. | +| Removed registry records | Historical/legal status evidence. | Incorrect longitudinal and due-diligence results. | Archive/retain according to source and legal policy; do not treat `Removed` as deletable. | + +## Archive candidates + +| Candidate | Current observation | Archive approach | Risk / prerequisite | Approval | +|---|---|---|---|---| +| Superseded coherent SQLite snapshots | Each is about 2.10 GB. | Compressed encrypted S3 archive with checksum, schema/fact versions and expiry. | Must retain at least the accepted rollback/PITR horizon. | Data owner + platform owner. | +| `src/data/processed/charity_commission_register.sqlite3` | About 1.70 GB, reproducible registry cache. | S3 versioned processing artifact; rebuild manifest required. | Removing local copy breaks current workflows until S3 access is implemented. | Data pipeline owner. | +| Large processed/pilot JSON and JSONL batches | Hundreds of MB; reproducible but useful for provenance/rebuild. | Convert curated analytical history to Parquet; retain raw separately. | Validate record counts/checksums and loader before local cleanup. | Data owner. | +| Pipeline logs/status history | Local files provide limited operational evidence and may contain contact data. | Short-lived redacted CloudWatch logs; immutable run summary separately. | Privacy/incident requirements determine retention. | Security/privacy + operations. | +| Generated exports | Potentially contain filtered organization/contact data. | Encrypted per-job S3 prefix with short TTL and access log. | Must not be indefinitely public/cacheable. | Product/data owner. | +| Audit screenshots and Markdown reports | Small, useful evidence, no secret observed. | Keep in Git if approved; screenshots are loading/performance evidence. | Recheck screenshots before publishing externally. | Repository owner. | + +## Quarantine and review candidates + +These records should remain stored but be excluded or flagged in affected metrics until reviewed. + +| Candidate | Count / evidence | Reason | Suggested action | +|---|---:|---|---| +| Future-dated grant | 1: `360G-TheSeafarersCharity-1756`, 2026-10-30, GBP 50,000 | Could be a planned award or source-date error. | Compare with source; expose a future-date flag. | +| Negative grant amounts | 2; minimum −10,000 | Likely corrections/reversals; overview already excludes them from the non-negative total. | Add transaction type/correction semantics and regression totals. | +| Zero grant amounts | 2,101 | May be missing/cancelled/non-monetary. | Classify by source fields; do not delete by amount alone. | +| Missing EUR conversions | 432 | Cannot participate in EUR totals. | Backfill only from approved rate policy; preserve original currency/amount. | +| Blank/unknown curated source provenance | 8 organizations | Weak auditability. | Repair from source records or mark explicitly unknown. | +| Negative registry income | 2 rows | May be corrections/source quality. | Confirm against Charity Commission source; flag rather than clamp. | +| Exact business-key duplicate grant groups | 4,271 groups, 14,529 rows beyond one/group | May be repeated source exports or legitimate distinct awards. | Compare source award identifiers and transaction semantics. | +| Low-confidence/review programme or geo records | 72 low-confidence programme; hundreds of review flags | Analysis may overstate certainty. | Human review queue and coverage thresholds. | +| Invalid programme source labels | 234,774 | Indicates normalization mismatch or broadly applied flag. | Investigate rule/label vocabulary before PostgreSQL baseline is signed. | + +## Potential technical deletion candidates + +These are not domain-data deletion candidates. They can be regenerated, but none was automatically removed except the ephemeral test container `fip-audit-prodlike`, which was created solely for this audit and then stopped/removed after verification. + +| Candidate | Location / size | Recoverability | Risk | Required approval / safe precondition | +|---|---|---|---|---| +| Audit staging DB | `/private/tmp/fip-audit-staging-20260728.db`, 2.10 GB | Recreate using SQLite `.backup` from the active DB. | Removes the tested migration copy and its evidence. | Repository/data owner after reports are accepted. | +| Audit restore DB | `/private/tmp/fip-audit-restore-20260728.db`, 2.10 GB | Recreate from staging/source snapshot. | Removes restore evidence. | Same as above. | +| Incomplete isolated virtualenv | `/private/tmp/fip-audit-venv` | Fully reproducible; install failed before dependencies downloaded. | Minimal. | Audit owner after acceptance. | +| Headless Chrome profiles | `/private/tmp/fip-chrome-audit-*` | Reproducible. | Could contain only local session/cache state; inspect ownership first. | Audit owner. | +| Temporary API JSON responses | `/private/tmp/fip-api-*.json`, `/private/tmp/fip-error-*.json` | Reproducible from local read-only calls. | May contain public organization detail/contact fields. | Audit owner; remove after report acceptance. | +| Temporary Docker config | `/private/tmp/fip-audit-docker-config/config.json` containing `{}` | Reproducible. | None; no credentials stored. | Audit owner. | +| Local Docker image | `foundationintelligenceplatform-bff`, 9,368,380,422 B | Rebuildable, though build is slow and unpinned. | Frees substantial disk; deletion removes reproducibility evidence. | Repository owner after Docker finding accepted. | +| Frontend `dist/` | About 2 MB, ignored | Rebuild with `npm run build`. | None if dependencies remain available. | Developer/operator. | +| `.coverage`, `.pytest_cache`, `__pycache__` | Local test artifacts, ignored | Recreate with tests. | None. | Developer/operator. | +| `node_modules` | Installed dependency tree, ignored | Recreate with `npm ci`; network required. | Offline rebuild unavailable. | Developer/operator when network/lockfile available. | + +## Not deletion candidates without a separate investigation + +- The 4,271 business-key duplicate groups and 9,073 duplicate charity-number groups. +- Removed registry organizations. +- Organizations without grant data, including the 299 Philea records. +- Negative/zero financial rows and grants. +- Raw source snapshots or source evidence. +- Active link overrides, cached profiles or audit run metadata. + +## Required governance before automated lifecycle + +1. Assign data owner and system owner per dataset. +2. Record legal basis, source licence/terms and permitted redistribution. +3. Classify contact/address/article data and define data-subject handling. +4. Define active, archive, quarantine and deletion periods by dataset. +5. Define litigation/incident holds and backup/PITR interaction. +6. Test restore before enabling expiration. +7. Require two-person approval for production bulk deletion and retain an immutable deletion manifest. diff --git a/docs/audits/database-integrity-report.md b/docs/audits/database-integrity-report.md new file mode 100644 index 0000000..4175224 --- /dev/null +++ b/docs/audits/database-integrity-report.md @@ -0,0 +1,207 @@ +# Database Integrity Report + +Audit date: 2026-07-28 +Source database: `src/data/charities.db` (read-only during audit) +Staging copy: `/private/tmp/fip-audit-staging-20260728.db` +Restore copy: `/private/tmp/fip-audit-restore-20260728.db` + +## Result + +The active SQLite database is structurally intact and internally consistent: `quick_check` and `integrity_check` returned `ok`, and `PRAGMA foreign_key_check` returned zero rows. The database is not ready for direct PostgreSQL cutover. The repository relies heavily on SQLite-specific SQL, a 2.10 GB single-file topology, JSON-in-TEXT fields, FTS5, filesystem publication and connection-local foreign-key enforcement. + +No active table was changed, no domain record was deleted and no destructive repair was attempted. A coherent backup was created with SQLite's `.backup`, then migrations and restore checks were performed only against temporary copies. + +## Copy and restore evidence + +| Artifact | Size | SHA-256 | Verification | +|---|---:|---|---| +| Staging immediately after backup | 2,100,543,488 B | `609208373d9a832c6d54e5d0a6679bed801bc35c59dc11847011d3c98b4f895d` | `quick_check=ok`, `integrity_check=ok`, FK violations 0 | +| Restore copy after staging migration exercise | 2,100,543,488 B | `6f665f81e900123b995f3dd92a0c47bcf379fe5241fa53a0d47bbfd94c360755` | `quick_check=ok`, row counts equal, FK violations 0 | + +The byte hashes differ because schema initialization/migration calls can change SQLite page layout. Logical checks and core row counts were equal. `create_tables(reset=False)` was executed twice on staging in 8.1 ms and 2.2 ms; both calls were idempotent and `validate_database(require_foreign_keys=True)` returned `valid`. + +## Engine settings and versioning + +| Setting | Observed value | Assessment | +|---|---:|---| +| File size | 2,100,543,488 B | Too large to package with application code; needs managed persistence/object storage separation. | +| Page size / count | 4,096 / 512,828 | Consistent with file size. | +| Freelist | 35 pages | Little free-page fragmentation. | +| Journal mode | `delete` | Single-host design; not a cloud HA strategy. | +| Synchronous | `2` (`FULL`) | Durable for local SQLite. | +| Auto-vacuum | `0` | No automatic reclamation. | +| `PRAGMA user_version` | `0` | Engine schema version is not used. | +| `metadata.schema_version` | `7` | Application migration version. | +| Registry schema version | `1` | Separate version domain. | +| Overview fact version | `2026-07-overview-facts-v5` | Separate materialization version. | +| Foreign keys on a fresh connection | `0` | Constraints exist but enforcement depends on each application connection enabling it. | + +The multiple version domains are understandable but need one migration ledger and explicit compatibility contract for PostgreSQL. + +## Tables and row counts + +| Table | Rows | Purpose / relationship | +|---|---:|---| +| `charities` | 373 | Curated organization profiles and grant-linked entities. | +| `charity_registry_organizations` | 397,469 | Charity Commission directory records. | +| `charity_registry_organizations_fts` | 397,469 | FTS5 search projection plus FTS shadow tables/triggers. | +| `exchange_rates` | 18,964 | Currency conversion observations. | +| `grants` | 302,546 | Core grant awards. | +| `grant_beneficiary_countries` | 104,309 | Many-to-many grant/country associations. | +| `grant_beneficiary_terms` | 556,719 | Geography evidence/terms. | +| `grant_programme_categories` | 358,883 | Grant/programme category associations. | +| `grant_overview_facts` | 302,546 | Precomputed overview facts. | +| `grant_source_funder_facts` | 104,309 | Funder/geography aggregates. | +| `grant_overview_cache` | 5 | Cached overview payloads. | +| `organization_registry_links` | 345 | Curated-to-registry organization links. | +| `source_funder_link_overrides` | 1 | Operator override/revision record. | +| `source_funder_profile_cache` | 1 | Cached/enriched funder profile. | +| `metadata` | small | Schema and materialization versions. | + +There are no application views. FTS5 creates internal tables and synchronization triggers. + +## Storage concentration + +`dbstat` shows the dominant objects: + +| Object | Approximate bytes | +|---|---:| +| `grants` table | 1,244,413,952 | +| `charity_registry_organizations` | 169,472,000 | +| `grant_overview_facts` | 106,946,560 | +| `grant_source_funder_facts` | 54,087,680 | +| FTS content/data | about 33 MB each | +| Large secondary indexes | several at 20–30 MB each | + +The database is primarily grant payload and duplicated/materialized lookup data. This is a candidate for normalized PostgreSQL operational tables plus S3/Parquet historical/raw storage, not an image layer. + +## Foreign keys and relationships + +- `grant_beneficiary_countries`, `grant_beneficiary_terms`, `grant_programme_categories`, `grant_overview_facts` and `grant_source_funder_facts` cascade to `grants`. +- Optional grant funder/recipient references point to `charities` with `NO ACTION` semantics. +- Registry links point to both registry records and curated charities. +- `PRAGMA foreign_key_check` returned **0 violations** on staging and restore copies. +- Orphan checks for the optional grant organization links returned **0**. +- Risk: `foreign_keys=OFF` is the SQLite default on a fresh connection. PostgreSQL will enforce constraints globally, so load order and deferred validation must be designed explicitly. + +## Keys, duplicates and identity + +| Check | Result | Interpretation / action | +|---|---:|---| +| Distinct grant IDs | 302,546 / 302,546 | Primary identity is unique. | +| Duplicate `(source, source_record_id)` | 0 | Source identity constraint is effective. | +| Exact business-key duplicate groups | 4,271 groups / 14,529 rows beyond one per group | Do not delete automatically. These can be distinct tranches/awards with similar business attributes. Review against source award IDs. | +| Registry IDs | 397,469 distinct | Unique registry record identity. | +| Registry duplicate charity numbers | 9,073 groups / 40,226 rows beyond one per charity number | Mostly constituent funds/linked registrations; examples show many legitimate records under umbrella numbers. Keep. | +| Organization registry links | 345 | All are accepted exact-identifier links with confidence 1.0. | + +## Grant completeness and numeric quality + +| Check | Result | Assessment | +|---|---:|---| +| Missing grant ID, recipient, amount, award date or currency | 0 | Pass for required core fields. | +| Invalid award-date shape | 0 | Pass for stored format. | +| Zero award amount | 2,101 | Retain; may represent corrections or source semantics. Review analytically. | +| Negative award amount | 2, min −10,000 | Likely corrections/reversals. Excluded from dashboard awarded-total logic; keep with explicit classification. | +| Maximum award | 465,755,576.5 | High-value outlier; source review recommended. | +| Date range | 1991-04-18 to 2026-10-30 | Includes one future-dated award. | +| Future-dated grant | `360G-TheSeafarersCharity-1756`, GBP 50,000 | Quarantine/review candidate, not deletion candidate. | + +## Currency conversion reconciliation + +| Currency/status | Grants | +|---|---:| +| GBP / `ecb_monthly_average` | 300,388 | +| GBP / `unavailable_missing_rate` | 419 | +| USD / `ecb_monthly_average` | 1,615 | +| USD / `unavailable_missing_rate` | 2 | +| EUR / native | 65 | +| ILS / monthly | 23 | +| CHF / monthly | 10 | +| CAD / monthly | 7 | +| ZAR / monthly | 6 | +| KES / missing rate | 10 | +| TZS / missing rate | 1 | + +- 302,114 grants have an EUR amount. +- 302,112 have an EUR amount and a non-negative source amount. +- Sum of every stored EUR amount: EUR 22,435,967,498.63. +- Sum used by the overview after excluding two negative corrections: EUR 22,435,986,707.70. +- API overview returned EUR 22,435,986,707.70 and 302,546 grants, exactly matching the implemented SQL semantics. +- 432 grants lack a usable conversion. +- The README describes daily award-date/previous-business-day ECB behavior, while stored statuses and tests implement monthly averages. Documentation and actual semantics have drifted. + +## Geography quality + +| Metric | Result | +|---|---:| +| Grants with no mapped beneficiary country | 198,355 | +| Grants with exactly one country | 104,128 | +| Multi-country grants | 63 | +| Maximum countries on one grant | 7 | +| Country associations | 104,309 | +| Distinct mapped grants | 104,191 | +| Overview country coverage | 34.44% | + +Methods: `unavailable` 180,682; source normalization 103,260 (135 reviews); deterministic regex 18,604 (236 reviews). The map discloses that multi-country full amounts are excluded from country funding totals. Preserve that rule and its evidence during migration. + +## Programme quality + +| Provenance/method | Result | +|---|---:| +| Unclassified provenance | 167,843 | +| Inferred provenance | 120,792 | +| Source provenance | 13,911 | +| Low-confidence records | 72 | +| Invalid source-label flag | 234,774 | +| Qualifying/classified grants in overview | 134,554 | +| Programme coverage | 44.54% | + +The category association table includes an `Unclassified` row, so its row count exceeds the number of classified awards. Rule versions, provenance and invalid-label flags must be migrated with the facts, not recomputed silently. + +## Organization and registry anomalies + +- Curated organizations: 299 Philea, 66 Charity Commission, and 8 records with blank/unknown source provenance. The blank-source records need provenance repair, not deletion. +- Philea's 299 organizations have no attached grants; that is consistent with organization-level source scope and must not be interpreted as failed linking. +- Registry status: 185,377 `Registered`, 212,092 `Removed`. +- Two registry records contain negative income. Retain as source evidence and place on a data-quality review queue. +- Registry name search uses FTS5. Benchmark on staging: `foundation` returned 50 rows in 651.171 ms; exact charity number search returned 3 rows in 1.997 ms; registered/income query returned 50 in 8.89 ms. +- Query plans used `idx_registry_charity_number` and covering `idx_registry_status_income` as intended. + +## PostgreSQL portability + +Static inventory found widespread SQLite coupling: approximately 34 `PRAGMA` references, 9 `sqlite_master`, 16 `INSERT OR REPLACE`, 6 `ON CONFLICT`, 9 `strftime`, 8 `rowid`, 9 `json_extract`, 16 `json_each`, 4 FTS5 and 15 `COLLATE NOCASE` references, plus 111 direct `sqlite3` references. Counts identify migration surface, not unique statements. + +Required rewrites include: + +- `?` placeholders and direct connection/row handling to a PostgreSQL driver/data-access layer; +- FTS5 virtual table and triggers to `tsvector`/GIN or a dedicated search service; +- JSON-in-TEXT to validated `jsonb` or normalized tables; +- `INSERT OR REPLACE` semantics to explicit `INSERT ... ON CONFLICT ... DO UPDATE` without accidental delete/reinsert behavior; +- `strftime`, `rowid`, case collation and SQLite-specific pragmas; +- file-copy atomic publish to transactionally versioned staging tables and view/schema switching; +- load ordering, sequences, numeric precision, timestamps/time zones and constraint validation; +- synchronous SQLite calls inside async endpoints. + +## Migration readiness gates + +1. Freeze and document schema v7 plus fact-version semantics. +2. Create PostgreSQL DDL with explicit keys, checks, numeric precision, time zones and JSON types. +3. Build a deterministic bulk loader from the coherent SQLite backup/raw artifacts. +4. Reconcile every table count, distinct key, FK/orphan count and the dashboard control totals above. +5. Run duplicate/anomaly policy checks without deleting records. +6. Run API golden responses and query-plan/performance tests against PostgreSQL. +7. Dual-write is optional; dual-read/shadow comparison is required before cutover. +8. Take a final source snapshot, stop writers, load delta, switch read traffic behind a flag. +9. Preserve the SQLite snapshot and application rollback path until production acceptance. + +## Retention, archive and quarantine candidates + +No listed item was deleted. + +- Preserve: active DB, raw source artifacts, source IDs, enrichment evidence, rule versions, negative/zero awards and registry constituent records until policy approval. +- Archive to S3 with lifecycle: versioned raw downloads, JSONL/preprocessed batches, superseded SQLite snapshots and generated audit exports. +- Quarantine for review: the one future-dated grant, two negative grants, two negative-income registry rows, 432 missing conversions and eight blank-provenance organizations. +- Potential technical deletion after explicit approval: temporary staging/restore DB copies, incomplete temporary virtualenv, browser profiles, test caches/build outputs and the 9.37 GB local Docker image. + +See `data-retention-and-deletion-candidates.md` for approval and risk details. diff --git a/docs/audits/feature-test-matrix.md b/docs/audits/feature-test-matrix.md new file mode 100644 index 0000000..f578187 --- /dev/null +++ b/docs/audits/feature-test-matrix.md @@ -0,0 +1,104 @@ +# Feature Test Matrix + +Audit date: 2026-07-28 +Revision: `97d9b02491866edc9b5d1aec1183dc73e2914626` + +`PASS` means the reachable behavior was exercised with satisfactory evidence. `PARTIAL` means the function exists and was exercised, but a material limitation or missing live/E2E proof remains. `FAIL` means the exercised behavior violates a central expectation. `NOT TESTABLE` is used where the environment prevented the requested verification. Mutating data functions were tested only with temporary databases/mocks, never against the active DB. + +## Frontend and user journeys + +| ID | Feature, description and entry point | Expected behavior | Actual test and test data | Evidence / result | Performance and error handling | Data correctness | AWS relevance | Status | +|---|---|---|---|---|---|---|---|---:| +| UI-01 | App bootstrap and sidebar navigation; `/`, React `App` | App loads and routes between main workspaces. | Vite dev start, browser at 1440/1024/768/390/1920, frontend tests. | App and sidebar rendered; navigation components present. Screenshots in `screenshots/`. | Dev server 200; initial data dependency is slow. | No contradiction observed in navigation labels. | Static build belongs on S3/CloudFront. | PASS | +| UI-02 | Overview KPI cards; `/` | Show funding, grants, donor scope and programme coverage. | Fully loaded 1440 screenshot and direct API/SQL reconciliation. | EUR 22.4bn, 302,546 grants, scope 571, coverage 44.54%. | Cold view about 28–35 s; warm overview API about 3.2 ms median. | Core totals match implemented SQL exactly. | Precomputed aggregates/cache required. | PARTIAL | +| UI-03 | Global grant map; `/` | Render mapped geography and switch value/connection modes. | Headless browser plus map/unit/backend tests on full DB. | Loaded choropleth; 104,191 mapped and 198,355 unmapped displayed. | Connection-heavy legacy map call 67.57 s and blocked peers; loader states captured. | Multi-country exclusion and EUR-conversion caveats displayed. | Materialized country facts and cache. | PARTIAL | +| UI-04 | Filters/reset/data-source controls; overview header | Apply URL-backed date, funder, recipient, geography, programme and currency filters. | Frontend filter tests and backend overview/filter tests; rendered controls inspected. | Filter state/queries exercised; reset button present. | Some filtered cold requests take seconds; invalid API inputs return 422. | Applied-filter metadata returned by APIs. | Query/index budget and cache-key design needed. | PARTIAL | +| UI-05 | Funding-over-time and programme charts; overview lower panels | Render trend and programme allocation with filter parity. | Backend trend/theme tests, API calls, loaded desktop screenshot. | Charts exist below map; programme facts reconcile. | Initial panels wait behind overview/map work. | Programme coverage and unknown categories preserved. | Preaggregate time/category facts. | PARTIAL | +| UI-06 | Overview drill-down | Selecting a segment returns underlying grants/entities. | Backend drill-down and filter tests on temp/full fixtures; route inventory. | Tests passed including paging/filter semantics. | Validation paths covered by FastAPI/tests. | Returned records preserve IDs/provenance. | Paginate and index in PostgreSQL. | PASS | +| UI-07 | Donor Directory; sidebar `/` donors view | Rank observed funders and open details. | Browser screenshot after 12 s; DE funder API and tests. | Ranking API correct, but fresh browser still showed loading/zero state at 12 s. | DE request 0.605–2.009 s; initial shared calls about 28 s. | Pears Foundation and Lankelly Chase ordering/amounts sampled. | Materialized ranking and independent loading. | PARTIAL | +| UI-08 | Advanced Charity Commission directory/research | Search, filter, paginate and inspect registry organizations. | Registry API, repository benchmark, cursor and backend tests. | Results/cursor stable; 397,469 rows searchable. | FTS `foundation` 651 ms; exact number 2 ms; validation 422 tested. | Duplicate charity numbers retained as distinct registry records. | PostgreSQL `tsvector`/GIN parity required. | PASS | +| UI-09 | Organization profile/detail modal | Show profile, source evidence, financials and provenance. | HTTP detail for charity 1075920 plus backend detail tests. | 200 response with source/enrichment/evidence fields. | 8.8 ms sampled; numeric missing ID returns 404. | Source and transaction coverage metadata present. | Public/private field classification needed. | PASS | +| UI-10 | Organization grants list | List grants and coverage for a profile. | HTTP `/1075920/grants`, 812 records; grant tests. | Status available, source 360Giving, limitation disclosed. | 108.9 ms sampled. | API record count equals metadata count 812. | Paginate/limit large profiles. | PASS | +| UI-11 | Sankey relationships | Render organization-to-theme/geography/funding flows. | HTTP `/1075920/sankey` and Sankey tests. | 31 nodes and 30 links returned. | 7.1 ms sampled; missing/empty cases covered in tests. | Link/node structure internally valid. | Relational aggregation should move to materialized queries. | PASS | +| UI-12 | Deterministic relevance score | Explain profile fit without presenting a prediction. | HTTP POST for charity 1075920 with education/Germany target and score tests. | Score 90.0, confidence 0.875, `not_a_prediction=true`. | 9.1 ms; invalid list type returns 422. | Components, assumptions and missing inputs returned. | Safe to keep as deterministic service; version config. | PASS | +| UI-13 | News briefing, lookback and PDF export | Search older/current news, summarize, stream and export. | News unit tests cover recent/older/all/empty/SSRF; existing runtime log showed real earlier request, but audit made no paid/live call. | Fallback beyond four weeks works; illustrative offline mode exists. | Network variability not benchmarked; 422 invalid lookback. | Completeness limited to RSS/feed results and article access. | Worker/cache, egress controls, provider secrets and retention needed. | PARTIAL | +| UI-14 | Favorites and saved news runs | Save selected organizations/runs locally. | Static code and frontend state tests/inventory. | LocalStorage-backed behavior present. | Browser persistence across sessions was not deeply automated. | Client-local only; no cross-device guarantees. | Decide tenant/user persistence before auth rollout. | PASS | +| UI-15 | Pipeline Monitor and manual refresh | View status/logs and trigger an allowed job. | Backend pipeline/admin tests with temp state/mocks; UI code inventory. | Function works in tests; active job was intentionally not triggered. | No auth; valid public trigger would mutate/use resources. | Atomic publish tests protect active data. | Private OIDC/RBAC control plane required. | PARTIAL | +| UI-16 | Responsive and accessibility behavior | Primary views fit mobile/tablet/desktop and support keyboard/assistive tech. | Screenshots at 390×844, 768×1024, 1024×768, 1440×900, 1920×1080; static ARIA review. | Mobile content/buttons/KPI grid overflow and crop; no axe/screen-reader/keyboard suite. | Loading overlays persist for long cold calls. | No data error, but content is inaccessible off-screen. | Add Playwright/axe and responsive release gate. | FAIL | + +## API operations + +| ID | Feature, description and entry point | Expected behavior | Actual test and test data | Evidence / result | Performance and error handling | Data correctness | AWS relevance | Status | +|---|---|---|---|---|---|---|---|---:| +| API-01 | Health; `GET /health` | Process health response. | 10 local calls and container calls. | 200 `healthy`. | Median about 1.18 ms warm; not DB-aware and can queue behind blocking work. | N/A. | Split liveness/readiness and include DB/dependency state. | PASS | +| API-02 | Charity list; `GET /api/charities` | Page/filter curated organizations. | HTTP five samples and backend tests. | 200; first 17 ms then ~4 ms warm. | Validation/error cases covered in tests. | Uses 373 curated organizations. | PostgreSQL pagination/indexes. | PASS | +| API-03 | Beneficiary geography options | Return filter vocabulary/counts. | Browser startup, direct API and backend tests. | 200 and values used by UI. | Cold request waited about 28 s when sharing event loop. | Counts based on association facts. | Cache as versioned vocabulary. | PARTIAL | +| API-04 | Registry list; directory GET | Search/filter/cursor page. | 10 HTTP calls, benchmark and tests. | 200; cursor schema stable. | Median ~2.14 ms warm for unfiltered 10; limit/negative income return 422. | Count/search plans verified. | Migrate FTS/search semantics. | PASS | +| API-05 | Registry detail GET | Return one registry record and source/profile information. | Backend tests with found/not-found fixtures. | Pass including 404 behavior. | Indexed identity lookup. | Preserves registry identity/status. | Field-level privacy classification. | PASS | +| API-06 | Charity statistics GET | Return totals/source/status statistics. | Browser/direct API and SQL reconciliation. | 373 total, 52 active, 18 removed, 302,546 grants. | Cold startup call about 28 s. | Source subtotals expose 8 blank-source records rather than hiding them. | Precompute and make readiness independent. | PARTIAL | +| API-07 | Legacy grants map GET | Return map points/connections and filters. | Real full-data call with GBP/connections plus backend tests. | Correct response eventually. | 67.5668 s; parallel reads waited ~55.4 s. | Coverage/exclusion semantics verified. | Critical query redesign before AWS. | FAIL | +| API-08 | Filtered grant overview GET | Return KPI/coverage aggregate. | Browser, direct HTTP and SQL control totals. | 200; exact total reconciliation. | Warm median ~3.24 ms; cold about 28–35 s. | Correct implemented non-negative/conversion semantics. | Versioned precompute/cache. | PARTIAL | +| API-09 | Entity suggestions GET | Autocomplete filter entities. | Runtime browser/API logs and backend tests. | Correct suggestions returned. | About 1.65–2.14 s observed. | Uses source/recipient/funder identities. | Search index/cache needed. | PARTIAL | +| API-10 | Overview trends GET | Return filtered time series. | Direct browser call/backend tests. | 200 and chart-compatible schema. | Warm response fast; queued behind heavy call in concurrency run. | Date/filter semantics tested. | Materialized period facts. | PASS | +| API-11 | Overview drill-down GET | Return records behind overview segments. | Backend pagination/filter tests. | Pass. | Invalid filters handled by validation/repository tests. | IDs and filters preserved. | Query limits and pagination. | PASS | +| API-12 | Source-funder enrich POST | Enrich selected observed funders. | Endpoint/service tests with mocks/temp DB only. | Functional tests pass. | Active/live paid mutation intentionally not run. Anonymous in current app. | Revision/source evidence logic tested. | Queue, OIDC/RBAC, idempotency, secrets. | PARTIAL | +| API-13 | Registry organization enrich POST | Enrich a registry organization. | Mocked endpoint/service tests only. | Pass without active mutation. | Provider/network behavior not live-tested; anonymous. | Cache/profile fields tested. | Private queued worker and audit log. | PARTIAL | +| API-14 | Reset source funder to observed POST | Remove enrichment/link state while preserving observed facts. | Temp-DB endpoint/repository tests; pre-audit user activity also logged. | Tests pass and revision semantics preserved. | Destructive logical operation is anonymous. | Observed source facts retained in tests. | Admin-only and reversible history required. | PARTIAL | +| API-15 | Relink source funder POST | Change registry/profile link. | Temp-DB endpoint/repository tests. | Pass including validation/revision behavior. | Anonymous mutation. | Link evidence/override semantics tested. | Admin RBAC and actor/reason audit. | PARTIAL | +| API-16 | Queue profile cache POST | Create/update funder profile cache. | Endpoint tests with mocks/temp DB; no audit mutation of active DB. | Pass. | Anonymous and currently process-local/background style. | Cache revision tested. | SQS/ECS worker and durable job state. | PARTIAL | +| API-17 | Profile cache GET | Read cached funder profile status/result. | Backend endpoint tests and active schema inspection. | Pass; one cache record present. | Fast lookup; not independently load-tested. | Cache/source metadata present. | TTL/version and privacy controls. | PASS | +| API-18 | Source funder list GET | Rank/page observed funders by beneficiary country. | DE HTTP request five times and backend tests. | 200; ordering sample verified. | 0.605–2.009 s after warmup; missing country returns 422. | Aggregates based on non-negative EUR/source facts. | Materialized/indexed ranking. | PARTIAL | +| API-19 | Source funder detail GET | Return observed/enriched profile and grants. | Backend tests plus earlier browser/runtime calls. | Pass for observed-only and linked/enriched cases. | Approximately 0.01–0.2 s seen. | Source-only state is explicit. | Privacy and cache strategy. | PASS | +| API-20 | Grant summary GET | Aggregate grants/recipients/funders. | Backend summary tests and API inventory. | Pass. | Heavy-query behavior covered functionally, not isolated load. | Totals/currencies preserve coverage fields. | Materialize common summaries. | PASS | +| API-21 | Grant trends GET | Return organization/global trends. | Backend tests and runtime calls. | Pass. | Queues behind synchronous DB work under concurrency. | Period totals tested. | Async DB pool/materialization. | PASS | +| API-22 | Grant themes GET | Return programme/theme distribution. | Backend tests and runtime calls. | Pass. | Queues behind heavy event-loop work. | Unclassified category retained. | Materialized categories. | PASS | +| API-23 | Charity detail GET | Return curated profile by registration number. | Real HTTP for 1075920 and tests. | 200 in 8.8 ms; numeric missing ID gives 404 in 3.6 ms. | Correct 404; non-numeric ID gives schema 422. | Provenance/evidence returned. | PII/field classification. | PASS | +| API-24 | Charity grants GET | Return organization grant list/coverage. | Real HTTP for 1075920 (812 grants) and tests. | 200 in 108.9 ms. | Empty/missing paths tested. | Metadata count equals list count. | Pagination and payload budgets. | PASS | +| API-25 | Charity Sankey GET | Return nodes/links. | Real HTTP and tests. | 31 nodes/30 links, 7.1 ms. | Empty/invalid cases tested. | Structural consistency observed. | PostgreSQL aggregation. | PASS | +| API-26 | Charity score POST | Deterministic relevance score. | Real valid/invalid requests and score tests. | 200, score 90; invalid input 422. | 9.1 ms. | Explicitly not predictive; explanations/components versioned. | Keep deterministic and observable. | PASS | +| API-27 | Core proxy GET/POST/PUT/PATCH/DELETE | Forward to fixed `CORE_API_URL`, preserve status safely. | Local unavailable downstream GET plus mocked method tests. | Returned sanitized 503; OpenAPI exposes all five methods. | No live core service integration; duplicate operation IDs in OpenAPI. | No domain data verified end-to-end. | Authenticate, restrict paths/headers, private networking and timeouts. | PARTIAL | +| API-28 | Pipeline status GET | Return current job status. | Backend tests and local status inspection. | Pass. | Local file is single-host and not a readiness/job history store. | No record-count reconciliation in status. | Durable jobs table/Step Functions. | PASS | +| API-29 | Pipeline trigger POST | Validate and launch a job. | Mocked/temp pipeline tests; no active valid trigger. | Functional tests pass. | Anonymous; long subprocess/background task and local lock. | Atomic publish/failure preservation tested. | Private queued control plane. | PARTIAL | +| API-30 | Pipeline logs GET | Tail recent pipeline logs. | Backend tests and runtime log inspection. | Returns last lines. | Anonymous and may expose names/emails/addresses. | Logs are not authoritative data lineage. | Redacted CloudWatch, RBAC and immutable run summary. | PARTIAL | +| API-31 | News summary GET | Fetch/filter/summarize news. | Mock tests for 4w/3m/12m/all, older fallback, empty and SSRF; invalid live request validation. | Functional pass under mocks; no paid/live audit call. | External latency/completeness unmeasured. | Source/date/limitations represented. | Async worker/cache, egress policy, secrets. | PARTIAL | +| API-32 | News summary SSE GET | Stream stages/results to UI. | SSE endpoint tests and existing runtime logs. | Stages and final/error events covered. | Disconnect/backpressure at scale not tested. | Same source limitations as summary. | API timeout/proxy buffering and async job design. | PARTIAL | + +## Ingestion, enrichment and background functions + +| ID | Feature, description and entry point | Expected behavior | Actual test and test data | Evidence / result | Performance and error handling | Data correctness | AWS relevance | Status | +|---|---|---|---|---|---|---|---|---:| +| DATA-01 | 360Giving scraper/import; `src/scrapers/360giving.py` | Fetch publisher data politely with pagination/retries and provenance. | Unit/integration tests with mocked responses; CLI help exit 0; cached full DB inspected. | Tests pass; all 302,546 current grants identify 360Giving. | Live rate/timeout not measured. | IDs and required fields complete; overall source coverage is not complete 360Giving. | ECS task, S3 raw, source quota. | PARTIAL | +| DATA-02 | Charity Commission register scraper | Download/page registry with polite behavior. | Mocked pagination/retry/user-agent tests; CLI help exit 0; cached 397,469-row registry. | Pass under fixtures. | Live API/key/network not exercised. | Unique registry IDs; constituent duplicates preserved. | ECS task, encrypted secret, S3 raw. | PARTIAL | +| DATA-03 | Philea organization scraper | Import member organization profiles. | Mocked scraper tests; CLI help exit 0; 299 cached organizations inspected. | Pass; organizations correctly have no grant transactions. | Live website behavior/terms not verified. | Org-level scope distinguished from grant data. | Source/legal review, ECS task. | PARTIAL | +| DATA-04 | Hinchilla scraper | Acquire configured foundation data. | Mocked tests and CLI help exit 0. | Interface/error paths pass. | No live request or current-volume proof. | No broad active grant contribution established. | Evaluate relevance before migration. | PARTIAL | +| DATA-05 | Impressum/contact extraction | Crawl allowed pages and extract contact evidence. | Unit tests/static review; CLI help exit 0. | Extraction safeguards/retries present. | Live crawling, robots/terms and accuracy not tested. | Contact data may be personal. | Separate opt-in worker, privacy classification and egress policy. | PARTIAL | +| DATA-06 | 360Giving publisher sampling CLI | Produce bounded publisher sample. | CLI help exit 0 and source tests. | Command surface starts. | Full live run not executed. | Sampling semantics tested in fixtures. | Useful preflight ECS task. | PARTIAL | +| DATA-07 | Extend observed 360Giving pilot CLI | Extend cached pilot deterministically. | CLI help exit 0; pipeline/import tests. | Pass under tests. | No full audit run because it writes large processed artifacts. | Source identity/idempotency covered. | Replace local files with versioned S3 outputs. | PASS | +| DATA-08 | Import observed grants CLI | Validate/load grants and child facts. | Import/idempotency/constraint tests on temp DB; CLI help exit 0. | Pass including duplicate and failure cases. | Full active import intentionally not run. | Core counts/constraints reconcile. | PostgreSQL bulk loader and reconciliation gate. | PASS | +| DATA-09 | Europe-tech curation pipeline | Create filtered/curated analytical dataset. | Curation tests and CLI help exit 0. | Pass on fixtures. | Full 759 MB artifact not regenerated. | Rule-derived scope needs version manifest. | Parquet/S3/Athena candidate. | PASS | +| DATA-10 | Geo/topic preprocessing | Extract deterministic evidence/categories. | Geo/topic tests and both CLI help paths. | Pass. | Accuracy measured only by fixtures/coverage, not external gold set. | Review flags and evidence persist. | Parallel batch worker; version outputs. | PARTIAL | +| DATA-11 | Reclassification/enrichment rules | Rebuild programme/geography facts without source re-fetch. | Reclassification and classification tests; DB distributions. | Deterministic behavior passes. | 234,774 invalid-source-label flags need investigation. | 44.54% programme and 34.44% country coverage. | Quality gate before cutover. | PARTIAL | +| DATA-12 | ECB exchange-rate backfill/conversion | Populate rates and EUR values. | Currency tests, CLI help exit 0 and SQL/API sum reconciliation. | Central total correct under implemented monthly-average semantics. | 432 missing; README documents a different daily policy. | Original amount/currency preserved. | Define rate contract, version and backfill in worker. | PARTIAL | +| DATA-13 | Overview cache prewarm | Precompute expensive overview payloads. | Cache/prewarm tests and CLI help exit 0; five cache rows present. | Functional pass. | Cache does not prevent all cold 28–35 s paths. | Fact/cache versions exist. | Scheduled materialization and cache invalidation. | PARTIAL | +| DATA-14 | End-to-end pipeline orchestration | Consolidate, enrich, stage, validate and publish. | Pipeline tests including modes, failures and CLI help exit 0. | Pass on temporary fixtures. | Full production-sized refresh not run. | Last active DB preserved on failure in tests. | Step Functions + ECS stages and run manifest. | PARTIAL | +| DATA-15 | Exclusive lock, atomic publish and recovery | Only one pipeline runs; failed build never replaces active DB. | 10 concurrent lock claims (one winner), corruption/missing/atomic-publish tests. | Pass. | Local `O_EXCL` and rename are single-filesystem only. | Staging counts/schema validation tested. | Replace with job state/lease plus transactional version switch. | PASS | +| DATA-16 | News acquisition older-than-four-weeks fallback | Return older results when recent window is empty and support `all`. | Mocked recent/older/all/undated/empty/SSRF tests. | Pass; no hard-coded four-week abort. | Google News RSS pagination/completeness and real availability not verified in audit. | Date/source limitations returned. | Source cache, bounded egress and retention. | PARTIAL | + +## Database, build and operations + +| ID | Feature, description and entry point | Expected behavior | Actual test and test data | Evidence / result | Performance and error handling | Data correctness | AWS relevance | Status | +|---|---|---|---|---|---|---|---|---:| +| OPS-01 | Schema initialization/migrations | Create/upgrade schema idempotently without reset. | `create_tables(reset=False)` twice on staging. | 8.1 ms then 2.2 ms; counts unchanged; schema 7 valid. | Corrupt/missing/locked DB paths covered by tests. | No FK violations. | Replace with Alembic/PostgreSQL migration ledger. | PASS | +| OPS-02 | Backup, integrity and restore | Produce coherent copy and prove restore. | SQLite `.backup`, quick/integrity/FK checks, second restore copy and count comparison. | All integrity checks `ok`, FK violations 0. | Integrity 39.08 s; quick check 15.15 s. | Logical restore matches baseline. | Automated RDS backup/PITR and restore drill. | PASS | +| OPS-03 | Registry FTS/index benchmark | Use intended indexes at realistic volume. | Benchmark on staging, 397,469 rows. | FTS 651 ms; number 2 ms; filtered 8.89 ms; plans use expected indexes. | FTS is the slowest query. | Result counts and identity retained. | Rebenchmark PostgreSQL text search. | PASS | +| OPS-04 | Docker image build | Build clean backend image from repository. | `docker-compose build bff` with empty audit Docker config. | Build succeeds, but context 8.81 GB and image 9.37 GB; active/registry DBs embedded. | About 80 s context transfer; no healthcheck/non-root/multistage. | Image contains stale snapshot risk. | Unacceptable for ECR/ECS as-is. | FAIL | +| OPS-05 | Docker Compose lifecycle | Start, healthcheck, restart and stop local service. | Compose build/up/restart/down. | 200 health after start/restart; clean stop/remove. | Uses `--reload`, root, bind-mounted `src`; Compose `version` warning. | Mounted local active data. | Development only; create production manifest. | PARTIAL | +| OPS-06 | Dockerfile default production-like start | Run image CMD without Compose override. | Local image on port 8001, health then stop/remove. | Started Uvicorn PID 1 and returned 200. | First immediate probe got empty response during startup; later 200 in 203 ms. | Uses DB embedded in image. | Startup works, packaging/security does not. | PARTIAL | +| OPS-07 | Frontend install/test/lint/build | Deterministically install and produce production assets. | `npm ci`, 8 tests, lint, Vite build. | All tests/build pass; lint has 5 hook warnings; 1.96 MB JS (612 KB gzip) chunk warning. | Bundle exceeds 500 KB warning; no E2E/a11y suite. | UI unit assertions pass. | CloudFront artifact and budgets/security scan. | PARTIAL | +| OPS-08 | Backend compile/lint/test/coverage | Catch syntax/runtime/regression failures. | `compileall`, blocking flake8 subset, full pytest/coverage. | 280 passed, 76.07% coverage; 3 deprecation warnings. | Advisory lint reports 1,071 issues and very high complexity. | Broad data/pipeline/API cases covered. | Gate migrations/integration/security in CI. | PASS | +| OPS-09 | Existing GitHub CI | Validate main/dev branches. | Workflow static review against local verified commands. | Backend tests/coverage and frontend build configured. | No frontend tests/lint, Docker, security, migration, deploy or rollback; only main/dev triggers. | No database reconciliation gate. | Not AWS-delivery-ready. | FAIL | +| OPS-10 | Dependency vulnerability/reproducibility audit | Install from clean host environment and scan Python/npm dependencies. | `npm audit` and isolated native pip install attempted; both blocked by restricted dependency-metadata/network access. Docker clean install succeeded. | Vulnerability status is unknown; Python ranges resolve different versions in host vs image. | No `pip-audit`, Trivy, OSV or gitleaks installed; escalation rejected. | No tracked high-confidence secret found, but dependency exposure unverified. | Add lockfiles/SBOM/scanners in network-enabled CI. | NOT TESTABLE | + +## Totals + +The table contains **74 identified feature/operation entries**: **32 PASS**, **37 PARTIAL**, **4 FAIL**, **1 NOT TESTABLE**. All 74 were inspected or exercised; the single `NOT TESTABLE` entry records the blocked dependency-vulnerability verification rather than claiming a result. diff --git a/docs/audits/performance/runtime-measurements.md b/docs/audits/performance/runtime-measurements.md new file mode 100644 index 0000000..4c44f63 --- /dev/null +++ b/docs/audits/performance/runtime-measurements.md @@ -0,0 +1,65 @@ +# Runtime Measurements + +Measured locally on 2026-07-28 on Apple Silicon/macOS, using the active FastAPI development process and the full 2.10 GB SQLite database. These are diagnostic samples, not a production load test. Browser sessions were fresh headless Chrome profiles unless labelled warm. + +## Warm sequential HTTP samples + +| Endpoint | Samples | Status | Median | Approx. sample p95/max | Notes | +|---|---:|---:|---:|---:|---| +| `GET /health` | 10 | 200 | 1.18 ms | 1.74 ms | Process-only health. | +| `GET /api/charities/grants/overview` | 10 | 200 | 3.24 ms | 4.28 ms | Warm application cache/materialization path. | +| `GET /api/charities/directory/organizations?limit=10` | 10 | 200 | 2.14 ms | 9.10 ms | First sample was slowest. | +| `GET /api/charities?limit=20` | 5 | 200 | 4.06 ms | 17.38 ms | First sample was slowest. | +| `GET /api/charities/grants/funders?beneficiary_country=DE&limit=20` | 5 | 200 | 1.071 s | 2.009 s | Remains expensive after warm-up. | + +The p95 column is the maximum for the small 5/10-sample set and must not be treated as a statistically robust production p95. + +## Cold browser and heavy query evidence + +| Scenario | Observed duration / state | Evidence | +|---|---|---| +| Fresh overview session | Shared `stats`, geography and overview calls about 27–35 s | `screenshots/overview-1440x900.png`, `overview-warm-1440x900.png`; runtime request logs. | +| Fully loaded overview | Complete at the 45 s screenshot budget | `screenshots/overview-loaded-1440x900.png`. | +| Fresh donor directory | Still displayed “Loading observed donors” and zero counts at 12 s | `screenshots/donor-directory-1440x900.png`. | +| Legacy map, GBP with connections | 67.5668 s | Uvicorn request-duration log. | +| Parallel requests during heavy map query | Trends, themes, charities, registry and funders each waited about 55.435 s | Concurrent curl/request log. This demonstrates event-loop blocking, not only DB contention. | +| Mobile overview, 390×844 | Still loading and horizontally cropped | `screenshots/overview-mobile-390x844.png`. | + +## Repository/database benchmark + +Executed against `/private/tmp/fip-audit-staging-20260728.db`, 397,469 registry rows: + +| Query | Latency | Result / plan | +|---|---:|---| +| FTS5 name search `foundation`, limit 50 | 651.171 ms | 50 rows; strategy `fts5`. | +| Exact charity number `200027`, limit 50 | 1.997 ms | 3 rows; `idx_registry_charity_number`. | +| Registered with income >= 100,000, sorted income desc, limit 50 | 8.890 ms | 50 rows; covering `idx_registry_status_income`. | + +SQLite `quick_check` took 15.15 s and full `integrity_check` 39.08 s on the 2.10 GB staging copy. + +## Frontend artifact + +- Vite build: 2,347 modules, about 755 ms. +- Main JavaScript: 1,963.84 KB, 612.12 KB gzip; Vite emitted its >500 KB chunk warning. +- CSS: 116.05 KB, 18.69 KB gzip. +- Lazy chunks observed: donor directory about 38.38 KB, registry about 19.42 KB. +- Lint completed with five `react-hooks/exhaustive-deps` warnings. + +## Docker artifact + +- Build context: 8.81 GB; transfer about 80.1 s. +- Image: 9,368,380,422 bytes. +- Largest layer: `COPY src/`, about 8.81 GB. +- Embedded DB files: 2.10 GB active DB and 1.70 GB processed registry SQLite, plus other source/processed artifacts. +- Tested architecture: Linux ARM64 only. +- Default-image startup returned health 200 after application initialization; an immediate probe during startup got an empty response. + +## Interpretation and required budgets + +Warm cache numbers conceal an unacceptable cold/concurrent path. FastAPI handlers are declared async but execute synchronous SQLite work, so a heavy aggregation blocks unrelated calls on the event loop. Before AWS staging exit: + +1. define representative cold/warm datasets and concurrency; +2. move blocking DB work off the event loop or use an async PostgreSQL data layer; +3. precompute/version overview, map, ranking and trend facts; +4. set a primary-view target (recommended p95 <= 3 s, cached API p95 <= 500 ms where feasible); +5. test p50/p95/p99, saturation, timeouts, cache invalidation and one-heavy-query isolation in CI/staging. diff --git a/docs/audits/requirements-traceability.md b/docs/audits/requirements-traceability.md new file mode 100644 index 0000000..f36d3c9 --- /dev/null +++ b/docs/audits/requirements-traceability.md @@ -0,0 +1,63 @@ +# Requirements Traceability + +Audit date: 2026-07-28 +Revision: `97d9b02491866edc9b5d1aec1183dc73e2914626` +Decision context: readiness to begin an AWS migration, not approval for public deployment. + +Status values are used exactly as requested: `PASS`, `PARTIAL`, `FAIL`, `NOT IMPLEMENTED`, `NOT TESTABLE`, `OUT OF CURRENT SCOPE`. Each requirement has exactly one status. + +## Epic requirements + +| ID | Requirement | Status | Evidence / affected files and components | Executed test and result | Risk | AWS impact | Recommended action | +|---|---|---:|---|---|---|---|---| +| EP-00 | Data Source Discovery & Evaluation | PARTIAL | Source clients exist in `src/scrapers/`; source metadata is present in `charities`, `grants` and registry records. README documents 360Giving, Charity Commission and Philea. No source register records licence, terms, owner, SLA, freshness or legal basis. | Repository/source inventory; scraper retry/pagination tests passed; cached source provenance queried in SQLite. Live third-party calls were not executed. | Unclear reuse rights, source coverage and operational ownership. | S3 retention classes, schedules and data contracts cannot be finalized safely. | Create an approved source register and data contract per source before production ingestion. | +| EP-01 | Data Collection & Ingestion | PARTIAL | `src/pipelines/run_pipeline.py`, source scrapers, JSON/JSONL staging and atomic SQLite publish exist. Scheduling is local/manual and status, locks and logs are filesystem-based. | Pipeline, scraper, pagination, retry, lock, atomic-publish and failure-preservation tests passed; every CLI returned `--help` exit 0. No live full refresh was run. | Manual operation and local state are fragile; external behavior remains unverified. | Long jobs do not fit Lambda; need orchestrated container workers and durable state. | Move jobs to ECS/Fargate tasks with EventBridge/Step Functions, S3 raw zones and SQS/DLQ. | +| EP-02 | Data Enrichment & Processing | PARTIAL | Deterministic geo/programme enrichment, deduplication, source evidence and ECB conversion exist in `src/preprocessing/`, `src/pipelines/` and repository code. | Backend suite passed 280 tests; SQL reconciled dashboard totals. Coverage is incomplete: 198,355 grants lack mapped geography, 167,843 are unclassified, 432 grants lack usable EUR conversion. | Partial classifications and conversion gaps can bias analysis. | Rules, versions and quality gates must survive PostgreSQL migration. | Define acceptance metrics, quarantine rules and versioned enrichment outputs; reconcile README currency semantics. | +| EP-03 | Analytics Dashboard & Visualization | PARTIAL | React/Vite dashboard, map, filters, trends, drill-down, donor directory, registry, profiles and Sankey exist in `frontend/src/`. | 8 frontend tests passed; browser screenshots at 390/768/1024/1440/1920 widths; loaded desktop data matches SQL. Cold primary view took about 28–35 s and mobile overflowed. | Primary UX misses the stated load objective and is unreliable under concurrent heavy queries. | API/cache design must be fixed before sizing public AWS services. | Precompute aggregates, remove event-loop blocking, add API caching and E2E/performance budgets. | +| EP-04 | Data Management & Governance | FAIL | SQLite schema v7, provenance fields, staging publish and backup are present. No retention policy, data owner matrix, deletion workflow, audit trail or comprehensive quality gate exists. | `quick_check` and `integrity_check` passed; zero FK violations; duplicate, anomaly and provenance queries executed. | Data can be retained, exposed or changed without governed lifecycle. | Blocks production S3/RDS lifecycle, backup and compliance design. | Approve retention/classification policy, ownership, quality gates, restore RTO/RPO and audit logging. | +| EP-05 | Infrastructure & Operations | FAIL | One backend Dockerfile and a development Compose file exist. CI only tests/builds; no IaC, deployment, monitoring, rollback, SBOM or vulnerability gate. | Docker build/start/restart/stop and default-image start passed, but image is 9.37 GB and embeds data. CI workflow inspected. | No repeatable or supportable AWS deployment path. | Fundamental migration blocker. | Add `.dockerignore`, multi-stage non-root images, frontend artifact build, IaC, staged delivery, observability and rollback gates. | +| EP-06 | Compliance & Data Protection | FAIL | Public registry/contact data and pipeline logs include addresses/emails. No authentication, authorization, retention policy, privacy review or data-subject workflow is present. | Route/security review, log sampling and tracked-secret scan. No high-confidence tracked secret found. | Public deployment could expose mutation controls and personal/contact data without safeguards. | Production AWS deployment must not proceed. | Threat model, OIDC/RBAC, private admin plane, WAF/rate limits, privacy/legal review, log redaction and deletion workflow. | + +## Functional requirements + +| ID | Requirement | Status | Evidence / affected files and components | Executed test and result | Risk | AWS impact | Recommended action | +|---|---|---:|---|---|---|---|---| +| FR-01 | automatische Datensammlung | PARTIAL | Source and orchestration CLIs exist; pipeline can run from cached or live inputs. There is no scheduler in the repository. | Pipeline/scraper tests passed; CLI surfaces start. Live scheduled execution was not tested. | Freshness depends on an operator. | Requires EventBridge scheduling and durable job orchestration. | Implement scheduled, idempotent ECS tasks with failure notifications. | +| FR-02 | relevante öffentliche Datenquellen | PARTIAL | 360Giving, Charity Commission, Philea and Hinchilla modules exist; current grants are exclusively 360Giving. | Source inventories and cached records inspected; source-specific tests passed. | Geographic/source coverage does not prove relevance or completeness. | Source-specific S3 prefixes and contracts are needed. | Approve source scope and measurable coverage targets. | +| FR-03 | Freshness- und Zeitfensterlogik | PARTIAL | Ingestion timestamps, date filters and news lookback modes exist. Source freshness SLAs and consistent last-success metadata do not. | Date filters and old-news fallback tests passed; DB date range checked. | Stale data may look current. | Needs central run metadata, watermarks and alarms. | Persist per-source watermarks and show freshness in UI/API. | +| FR-04 | Datenanreicherung | PARTIAL | Deterministic and optional Gemini-assisted enrichment are implemented with evidence/method/version fields. | Geo, topic, programme and enrichment tests passed; SQL distributions checked. | Large unavailable/unclassified cohorts and AI governance gaps remain. | Batch compute, model secrets and provenance need controlled services. | Establish quality thresholds and human-review queues; make AI optional and auditable. | +| FR-05 | Deduplizierung | PARTIAL | Source identity constraints, candidate tracking and conservative linking exist. | Duplicate/idempotency tests passed; SQL found no duplicate source IDs but 4,271 exact business-key groups. | Legitimate tranches may be merged or repeated awards may be double-counted if rules change. | PostgreSQL unique constraints must reproduce source semantics. | Define canonical award key and regression corpus before migration. | +| FR-06 | interaktives Dashboard | PARTIAL | Main React dashboard and navigation are implemented. | Browser and frontend tests passed functionally; initial render commonly exceeded 28 s. | Core user journey is too slow. | Requires backend concurrency/cache redesign before public traffic. | Add performance budgets and a loaded-state E2E suite. | +| FR-07 | interaktive Karte | PARTIAL | Choropleth, country selection, modes and connections are implemented. | Desktop map rendered; map tests passed; one connection-heavy request took 67.57 s. | Timeouts and blocked unrelated requests. | Precompute country facts and serve cacheable responses. | Replace dynamic full aggregation and add load tests. | +| FR-08 | Funding nach Sektoren | PASS | Programme categories/facts and dashboard allocation chart exist. | Programme tests passed; SQL/API reconciled 134,554 classified grants and 44.54% qualifying coverage. | Classification coverage is limited but disclosed. | Materialized aggregates are suitable for PostgreSQL/Athena. | Preserve rule version and coverage metadata in target model. | +| FR-09 | größte Donors | PASS | Source-funder ranking and detail APIs plus Donor Directory exist. | Germany ranking returned 200 with ordered amounts; related backend tests passed. | Rankings inherit source and conversion coverage. | Add indexed/materialized ranking views. | Reconcile rankings during dual-run migration. | +| FR-10 | größte Recipients | PARTIAL | Recipient fields, summary/trend APIs and organization profiles exist, but a clearly labelled global recipient-ranking journey is not evident. | Grant summary/detail tests and SQL recipient checks passed. | Users may infer ranking from incomplete UI paths. | Clarify target query/materialization. | Add an explicit recipient ranking contract and UI acceptance test. | +| FR-11 | Länder- und Regionenbeziehungen | PASS | Beneficiary-country associations, multi-country policy and map relationships are implemented and disclosed. | Geo/multi-country/map tests passed; 104,191 mapped grants reconciled. | 65.56% are unmapped. | Preserve many-to-many associations and disclosed exclusions. | Add coverage gates without fabricating geography. | +| FR-12 | Filter und Drill-down | PARTIAL | URL-backed filters, entity suggestions and drill-down APIs/UI exist. | Frontend filter tests, overview filter backend tests and API calls passed. Heavy calls block and no full browser automation exists. | Regressions and slow combinations may escape. | Query/index/cache work is required. | Add Playwright matrix and PostgreSQL `EXPLAIN ANALYZE` budgets. | +| FR-13 | Funding-Trends | PASS | Overview and organization trend endpoints/charts exist. | Backend trend tests and HTTP requests passed. | Historical meaning depends on source completeness. | Can be pre-aggregated by period in PostgreSQL/Athena. | Publish source coverage next to trends. | +| FR-14 | AI-basierte Prognosen, sofern bereits implementiert | NOT IMPLEMENTED | The score endpoint explicitly returns `not_a_prediction: true`; it is a deterministic relevance score. News uses LLM summarization, not forecasting. | Score call returned 90.0, confidence 0.875 and `not_a_prediction=true`. | Calling it forecasting would be misleading. | No forecast service should be migrated as if it existed. | Keep out of scope or create a separately governed forecasting requirement. | +| FR-15 | Retention Policy | NOT IMPLEMENTED | No approved retention/deletion configuration or workflow found. | Repository/documentation search and data inventory. | Privacy, cost and legal exposure. | Blocks S3 lifecycle and production data governance. | Approve policy before production migration. | +| FR-16 | konfigurierbare Refresh-Intervalle | NOT IMPLEMENTED | News lookback is configurable, but ingestion refresh schedules are not represented as application configuration. | Config/CLI/CI inventory. | Operations rely on manual convention. | EventBridge schedules cannot be derived from repository state. | Add per-source schedules, owners, SLAs and pause controls. | +| FR-17 | manueller Refresh | PARTIAL | Admin trigger API and Pipeline Monitor controls exist. | Admin/pipeline tests passed with temp files/mocks; no valid refresh was run against the active DB. Endpoint is unauthenticated. | Public use could exhaust resources or mutate data. | Must be private and authorized. | Require OIDC/RBAC, idempotency key, job queue and immutable run record. | + +## Non-functional requirements + +| ID | Requirement | Status | Evidence / affected files and components | Executed test and result | Risk | AWS impact | Recommended action | +|---|---|---:|---|---|---|---|---| +| NFR-01 | serverless-first | FAIL | Current design is a long-running FastAPI process with synchronous SQLite and long scraping/subprocess jobs. | Architecture/runtime review; 67.57 s map call and long batch workflows observed. | Direct Lambda lift-and-shift would time out or serialize work poorly. | Use managed services, but ECS/Fargate for API/workers rather than forcing Lambda. | Adopt hybrid managed architecture and document serverless exceptions. | +| NFR-02 | AWS und CI/CD | FAIL | No AWS IaC or deployment workflow; CI lacks security, Docker, migration, deploy and rollback gates. | `.github/workflows/ci.yml` inspected; local test/build commands passed. | Manual, unreproducible and unsafe delivery. | Fundamental blocker. | Implement Terraform/CDK, environments, OIDC deployment, staged migration and rollback. | +| NFR-03 | Open-Source-Präferenz | PARTIAL | Core stack is OSS; optional Gemini/Anthropic services are proprietary. No SBOM/licence inventory exists. | Dependency and licence-file inventory. | Licence obligations and vendor dependencies are not governed. | Image/SBOM scanning and model-provider choices needed. | Generate SBOM and third-party/source licence register; retain provider abstraction. | +| NFR-04 | Kosteneffizienz | FAIL | No cost model; 9.37 GB image, 8.81 GB build context and repeated full aggregations are expensive. | Docker image history and runtime timings measured. | Slow pulls, storage/egress and overprovisioning. | Direct deployment would create avoidable ECR/ECS/NAT/RDS cost. | Slim image, S3 data, pre-aggregation, load model and AWS budget alarms. | +| NFR-05 | Skalierbarkeit | FAIL | Single SQLite file, synchronous queries inside async handlers and filesystem locks/status. | Parallel calls queued for about 55 s behind map aggregation. | One heavy user stalls all users; one-host state is not horizontally safe. | Requires PostgreSQL, worker separation and caches. | Remove blocking I/O from event loop and introduce scalable persistence/queues. | +| NFR-06 | Datenschutz und Compliance | FAIL | No auth/RBAC, retention, privacy workflow or protected admin plane; logs/public datasets contain contact/address fields. | Security review, log sample and tracked-secret scan. | Unauthorized mutation and personal-data exposure. | Public deployment prohibited until controls exist. | Complete DPIA/legal review, OIDC/RBAC, redaction, encryption and retention workflows. | +| NFR-07 | Ladezeit der primären Ansichten | FAIL | Cold primary dashboard calls took roughly 28–35 s; map with connections took 67.57 s. | HTTP timing, concurrency test and 45 s headless-browser screenshot. | Far above a practical 3 s target. | Must be solved before capacity planning and public staging exit. | Precompute, cache, paginate and enforce p95 budgets in CI/staging. | +| NFR-08 | Rate-Limiting und polite crawling | PARTIAL | Source clients include retry, sleep/backoff and user-agent tests. API has no inbound rate limit; source terms/robots evidence is incomplete. | Scraper tests passed; middleware/config inventory found no API limiter. | Source blocking and public abuse remain possible. | Need WAF/API throttles and centrally enforced source quotas. | Add per-source policy configuration, robots/terms review and inbound throttling. | + +## Status totals + +| Scope | PASS | PARTIAL | FAIL | NOT IMPLEMENTED | NOT TESTABLE | OUT OF CURRENT SCOPE | Total | +|---|---:|---:|---:|---:|---:|---:|---:| +| EP | 0 | 4 | 3 | 0 | 0 | 0 | 7 | +| FR | 4 | 10 | 0 | 3 | 0 | 0 | 17 | +| NFR | 0 | 2 | 6 | 0 | 0 | 0 | 8 | +| **All** | **4** | **16** | **9** | **3** | **0** | **0** | **32** | diff --git a/docs/audits/screenshots/donor-directory-1440x900.png b/docs/audits/screenshots/donor-directory-1440x900.png new file mode 100644 index 0000000..15c470c Binary files /dev/null and b/docs/audits/screenshots/donor-directory-1440x900.png differ diff --git a/docs/audits/screenshots/overview-1440x900.png b/docs/audits/screenshots/overview-1440x900.png new file mode 100644 index 0000000..e4d8daa Binary files /dev/null and b/docs/audits/screenshots/overview-1440x900.png differ diff --git a/docs/audits/screenshots/overview-ipad-landscape-1024x768.png b/docs/audits/screenshots/overview-ipad-landscape-1024x768.png new file mode 100644 index 0000000..cf21d31 Binary files /dev/null and b/docs/audits/screenshots/overview-ipad-landscape-1024x768.png differ diff --git a/docs/audits/screenshots/overview-ipad-portrait-768x1024.png b/docs/audits/screenshots/overview-ipad-portrait-768x1024.png new file mode 100644 index 0000000..be57af7 Binary files /dev/null and b/docs/audits/screenshots/overview-ipad-portrait-768x1024.png differ diff --git a/docs/audits/screenshots/overview-large-1920x1080.png b/docs/audits/screenshots/overview-large-1920x1080.png new file mode 100644 index 0000000..9848bf1 Binary files /dev/null and b/docs/audits/screenshots/overview-large-1920x1080.png differ diff --git a/docs/audits/screenshots/overview-loaded-1440x900.png b/docs/audits/screenshots/overview-loaded-1440x900.png new file mode 100644 index 0000000..1224ac1 Binary files /dev/null and b/docs/audits/screenshots/overview-loaded-1440x900.png differ diff --git a/docs/audits/screenshots/overview-mobile-390x844.png b/docs/audits/screenshots/overview-mobile-390x844.png new file mode 100644 index 0000000..ba0237c Binary files /dev/null and b/docs/audits/screenshots/overview-mobile-390x844.png differ diff --git a/docs/audits/screenshots/overview-warm-1440x900.png b/docs/audits/screenshots/overview-warm-1440x900.png new file mode 100644 index 0000000..4885890 Binary files /dev/null and b/docs/audits/screenshots/overview-warm-1440x900.png differ diff --git a/docs/remediation/aws-postgres-architecture.md b/docs/remediation/aws-postgres-architecture.md new file mode 100644 index 0000000..3d0c263 --- /dev/null +++ b/docs/remediation/aws-postgres-architecture.md @@ -0,0 +1,109 @@ +# AWS/PostgreSQL Target Architecture Contract + +Status: Phase-0 design contract; no AWS resource has been created or changed. + +## Current-to-target boundary + +```mermaid +flowchart LR + subgraph Current[Current local runtime] + SRC[External sources] + PY[Python scraper/import processes] + FS[Local raw/processed files] + SQ[(SQLite 2.10 GB)] + API[FastAPI + synchronous sqlite3] + UI[React/Vite] + SRC --> PY --> FS --> SQ --> API --> UI + end + + subgraph Target[Target managed architecture] + EB[EventBridge Scheduler] + SF[Step Functions] + Q[SQS + DLQ] + WK[ECS Fargate workers] + S3[(S3 raw / validated / curated)] + PG[(RDS PostgreSQL)] + ATH[Athena / Glue] + ECS[ECS Fargate FastAPI] + ALB[ALB / API Gateway + WAF + OIDC] + CDN[CloudFront + private S3 frontend] + EB --> SF --> Q --> WK + WK --> S3 + WK --> PG + S3 --> ATH + PG --> ECS --> ALB --> CDN + end + + SQ -. read-only versioned migration .-> PG +``` + +## Runtime request path + +1. CloudFront serves immutable Vite assets from a private S3 origin. +2. API requests reach WAF and ALB/API Gateway. +3. OIDC establishes identity; application RBAC authorizes route/action. +4. ECS FastAPI uses bounded async SQLAlchemy/asyncpg sessions. +5. Interactive reads use normalized PostgreSQL tables and versioned materialisations. +6. Mutations use explicit transactions, idempotency keys and immutable audit events. +7. No request performs long scraping, full dataset rebuilds or local-file coordination. + +## Batch and activation path + +```mermaid +sequenceDiagram + participant Scheduler as EventBridge / operator + participant Orchestrator as Step Functions + participant Queue as SQS / DLQ + participant Worker as Fargate worker + participant ObjectStore as Versioned S3 + participant Database as PostgreSQL + + Scheduler->>Orchestrator: start source ingestion run + Orchestrator->>Queue: enqueue idempotent units + Queue->>Worker: deliver bounded work + Worker->>ObjectStore: write immutable raw object + checksum manifest + Worker->>Database: load versioned staging rows + Worker->>Database: write counts / quality issues / reconciliation + alt reconciliation passes + Orchestrator->>Database: transactionally activate dataset version + else validation fails + Orchestrator->>Database: reject candidate; preserve last active version + Orchestrator->>Queue: send terminal work to DLQ where applicable + end +``` + +## Data ownership + +| Data class | System of record | Notes | +|---|---|---| +| Raw source payload/version | S3 | Immutable, checksum-addressed manifest, KMS, no public access. | +| Validated/curated historical files | S3 Parquet | Partitioned by dataset/source/time; lifecycle remains non-destructive initially. | +| Operational organizations/grants/relationships | PostgreSQL | Typed, constrained and transactionally versioned. | +| Search | PostgreSQL `tsvector`/GIN and optional `pg_trgm` | Deterministic ranking/tie-break and cursor behavior. | +| Interactive aggregates | PostgreSQL materialized/versioned tables | Built before dataset activation. | +| Broad historical analytics/reconciliation | Athena/Glue over S3 | Never substitutes for transactional/low-latency PostgreSQL. | +| Job/run/audit/governance state | PostgreSQL | Durable, queryable and not tied to an API process filesystem. | +| User identity | External OIDC provider | Only subject/claims needed for authorization/audit are stored. | + +## Security boundaries + +- Public: static frontend and explicitly classified read-only API routes. +- Authenticated: analyst/profile/news reads where source cost or data policy requires identity. +- Operator: queued manual refresh and controlled data-quality actions. +- Administrator: configuration, relink/reset, retention approval and privileged operational actions. +- Internal service: worker callbacks and queue-driven state changes using task identities, not user cookies. +- Proxy: disabled by default; fixed host/path/method/header allowlists only. +- RDS and workers remain in private subnets. Data buckets block public access. Secrets enter tasks through Secrets Manager/SSM and task roles. + +## Failure and rollback contract + +- A candidate dataset is never visible before reconciliation succeeds. +- The last approved dataset stays readable during failed ingestion/migration. +- Activation is one transaction changing the active dataset reference. +- Rollback reactivates the prior approved PostgreSQL dataset and preserves the rejected target for diagnosis. +- The coherent SQLite source remains read-only during the validation window; it is not a permanent production fallback. +- No bidirectional replay occurs without a separately approved conflict policy. + +## Managed-service justification + +ECS Fargate is the justified serverless-container exception for the API and long-running/bulk workers. Lambda is not selected for the 2.10 GB migration, source crawling, large materialisation builds or latency-sensitive API with pooled database connections. EventBridge, Step Functions, SQS/DLQ, S3, RDS, CloudWatch and managed identity/secrets supply the managed control/storage plane. diff --git a/docs/remediation/aws-postgres-baseline.md b/docs/remediation/aws-postgres-baseline.md new file mode 100644 index 0000000..9efd13f --- /dev/null +++ b/docs/remediation/aws-postgres-baseline.md @@ -0,0 +1,130 @@ +# AWS/PostgreSQL Remediation Baseline + +Created: 2026-07-28 + +Target branch: `91-clean-up-code-for-aws-integration` + +Starting revision: `408eb879b05ec4d2caf92d9bbd782dda9b290e23` + +Immutable baseline commit: `19e84ba11dd3567fc871b3411166ae59a5b6eef0` + +## Safety contract + +- `src/data/charities.db` is a read-only migration source. +- `docs/audits/` is an immutable audit baseline. +- AWS resources, DNS, certificates, ECR, S3 uploads, RDS migrations, paid APIs, `terraform apply` and `terraform destroy` are prohibited without a later explicit approval. +- Local scoped commits are authorised. A Git push is not authorised until a later explicit confirmation. +- No existing uncommitted change may be discarded. + +## Initial Git state + +- Working directory: `/Users/manuelgrabmayer/netlight - github/FoundationIntelligencePlatform` +- Branch: `91-clean-up-code-for-aws-integration` +- Upstream: `origin/91-clean-up-code-for-aws-integration` +- HEAD: `408eb879b05ec4d2caf92d9bbd782dda9b290e23` +- Initial tracked diff: none. +- Initial untracked content: `docs/audits/` only. + +## Immutable audit baseline + +Exactly 16 files were found. No SQLite/database file, environment file, Terraform state, raw dataset or browser profile directory is present. A high-confidence private-key/token-pattern scan returned no match. + +| SHA-256 | Bytes | Path | +|---|---:|---| +| `21c75790c91fa7fcf401c66b1df38546645cc62ac9475b29ea8c0bae93a9b84d` | 17,644 | `docs/audits/audit-command-log.md` | +| `ab685f281528cf281d0bc5160e33a00057dd87f20f239ed546af63b043cb8199` | 14,636 | `docs/audits/aws-migration-plan.md` | +| `f7e02d8c65fc30a0a9d4929a678b98ee5d7d28e8480fb8381ea4dfadc466ed68` | 43,941 | `docs/audits/aws-readiness-audit-2026.md` | +| `0ff26c1acc11cb75ae9a6d86dd239ba2d22e512f722cdd2d62846ce02b6a4842` | 9,035 | `docs/audits/data-retention-and-deletion-candidates.md` | +| `d6e0c7c91e09a3c638601803de0933dd97f887420dd9347d8e733f4ca9bfd782` | 12,793 | `docs/audits/database-integrity-report.md` | +| `093abe5476c0c8687720121d7c0da28d2104a052c12c25d7fb3db3c30f1e05c6` | 26,725 | `docs/audits/feature-test-matrix.md` | +| `efd480cba706e60fbab8c15a51f5ce455f063cd6a517d0d3757d6ae19b5d9ad9` | 4,092 | `docs/audits/performance/runtime-measurements.md` | +| `4ba7216a7e84d0c8a3e00d47963d2d9178fcd32d346d940eeb62b96ef512a75b` | 15,646 | `docs/audits/requirements-traceability.md` | +| `a8438ed1e4e67a807182b96248cecd725c8e3438811f3ef2c551cd5d47f6d61a` | 75,230 | `docs/audits/screenshots/donor-directory-1440x900.png` | +| `f1fcaddd24733c232a832c48421754054cd3f550864bfcd19ee3f8d39c0e6dd3` | 112,667 | `docs/audits/screenshots/overview-1440x900.png` | +| `13c570b67679f5ecf94f65a0057d7c96f19ca84d1cf7992e8d543367001d4a8c` | 96,631 | `docs/audits/screenshots/overview-ipad-landscape-1024x768.png` | +| `ddb778f94c10562180a2acb791fbd8cfcc19ff62287884396abb195cda85d4bf` | 87,431 | `docs/audits/screenshots/overview-ipad-portrait-768x1024.png` | +| `c9d38dc7c2044680db9fdd423822e5ab8bdad20cd8f30747f1f8cdb57526476c` | 15,935 | `docs/audits/screenshots/overview-large-1920x1080.png` | +| `d068c1e99b1049cbf0e94984f84453c59f5b69cc4f81ca4bfaf183e2bd83610e` | 295,435 | `docs/audits/screenshots/overview-loaded-1440x900.png` | +| `70a8a3f784db8b99dd9f8bbe191d9fc835795578a0c0de8d8622211778bc6cb1` | 65,404 | `docs/audits/screenshots/overview-mobile-390x844.png` | +| `d7022a54c23ab08f68a37fbce2d1a6d6a89b5ac5e6e7e571e73ba02d05fb0067` | 114,303 | `docs/audits/screenshots/overview-warm-1440x900.png` | + +The checksums above are the immutability reference for all later phase and final checks. + +## Source database baseline + +The source was inspected read-only. A coherent backup was created with SQLite's backup mechanism at `/private/tmp/fip-remediation-baseline-20260728.db`; it is not a Git artifact. + +| Item | Verified result | +|---|---| +| Active path | `src/data/charities.db` | +| Active size / modified time | 2,100,543,488 bytes / 2026-07-28T17:23:28+0200 | +| Active file SHA-256 | `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` | +| Active read-only quick check | `ok` | +| Coherent backup size | 2,100,543,488 bytes | +| Coherent backup SHA-256 | `609208373d9a832c6d54e5d0a6679bed801bc35c59dc11847011d3c98b4f895d` | +| Backup integrity check | `ok` | +| Backup foreign-key violations | 0 | +| Application schema version | 7 | +| Registry schema version | 1 | +| Overview fact version | `2026-07-overview-facts-v5` | +| Host free space before backup | 69 GiB | + +The file and backup hashes differ because SQLite backup rebuilds page layout; logical integrity/counts are the migration acceptance evidence. The migration manifest records the coherent migration source checksum. + +### Reverified control values + +| Control | Value | +|---|---:| +| `charities` | 373 | +| `charity_registry_organizations` | 397,469 | +| `grants` | 302,546 | +| `grant_beneficiary_countries` | 104,309 | +| Distinct mapped grants | 104,191 | +| `grant_beneficiary_terms` | 556,719 | +| `grant_programme_categories` | 358,883 | +| `grant_overview_facts` | 302,546 | +| `grant_source_funder_facts` | 104,309 | +| `organization_registry_links` | 345 | +| Classified, conversion-eligible non-negative grants | 134,554 | +| Conversion-eligible non-negative grants | 302,112 | +| Overview total | EUR 22,435,986,707.70 | +| Duplicate source identity groups | 0 | +| Missing EUR conversions | 432 | +| Negative grants | 2 | +| Zero grants | 2,101 | +| Future-dated grants | 1 | +| Blank/unknown curated source provenance | 8 | +| Invalid programme source-label flags | 234,774 | +| Exact business-key duplicate groups / extra rows | 4,271 / 14,529 | +| Duplicate charity-number groups / extra rows | 9,073 / 40,226 | + +Currency, programme-provenance and geography-method distributions also match the immutable audit baseline. + +## Application baseline + +| Check | Verified result | +|---|---| +| Python compile | PASS | +| Blocking Flake8 `E9,F63,F7,F82` | PASS, 0 findings | +| Backend tests | PASS, 286 tests | +| BFF coverage | PASS, 76.29% (threshold 70%) | +| Backend warnings | 55 deprecation warnings; recorded, not hidden | +| Frontend install | PASS, `npm ci`, 74 packages | +| Frontend unit tests | PASS, 8 tests | +| Frontend lint | PARTIAL, exit 0 with five React hook dependency warnings | +| Frontend build | PASS with >500 KB chunk warning | +| Main JS | 1,963.60 KB / 612.00 KB gzip | +| Clean target-branch startup | PASS: backend and frontend reachable | +| OpenAPI | 33 paths / 38 operations | +| Anonymous charity/admin/proxy access | 401 after clean target-branch restart | +| Overview measured after persistent cache | 21.3 ms first sampled call, 4.2 ms warm; not a true empty-cache result | +| Registry FTS `foundation` | 808.9 ms | +| Docker build | FAIL: 8.81 GB context; `COPY src/` ended with `no space left on device` | + +An initial HTTP check accidentally reached the still-running process from the prior branch and showed 31/36 plus anonymous access. That process was stopped. The clean target-branch process produced the authoritative 33/38 and 401 baseline above. + +The Docker daemon is available (`23.0.5`, aarch64), but it held a 9.37 GB prior image and 12.74 GB build cache. The failure is evidence for Phase 2, not a passed Docker gate. + +## Phase-0 gate result + +`PASS`. The immutable evidence and remediation ledgers were committed separately from application and infrastructure code. A final checksum pass confirmed that the active SQLite source and all 16 audit files were unchanged. The target architecture and machine-readable migration-manifest contract are carried in the immediately following scoped Phase-0 commit. diff --git a/docs/remediation/aws-postgres-command-log.md b/docs/remediation/aws-postgres-command-log.md new file mode 100644 index 0000000..f76009e --- /dev/null +++ b/docs/remediation/aws-postgres-command-log.md @@ -0,0 +1,1056 @@ +# AWS/PostgreSQL Remediation Command Log + +Commands are recorded in execution order. Secret values are never recorded. Exit codes are included when known. + +## Phase 0 + +### Initial branch and working tree + +```zsh +pwd +git status --short --branch +git branch --show-current +git rev-parse HEAD +git log -5 --oneline +git remote -v +git diff --stat +git diff +git diff --cached +``` + +Result: exit 0. Correct branch `91-clean-up-code-for-aws-integration`, HEAD `408eb879b05ec4d2caf92d9bbd782dda9b290e23`, no tracked diff, `docs/audits/` untracked. + +### Immutable audit verification + +```zsh +find docs/audits -type f -print | LC_ALL=C sort +find docs/audits -type f -exec stat -f '%z %N' {} \; | LC_ALL=C sort -k2 +find docs/audits -type f -print0 | LC_ALL=C sort -z | xargs -0 shasum -a 256 +git status --short -- docs/audits +find docs/audits -type f \( -name '*.db' -o -name '*.sqlite' -o -name '*.sqlite3' -o -name '*.env' -o -name '*.tfstate*' \) -print +rg -n --hidden 'BEGIN (RSA |EC |OPENSSH )?PRIVATE KEY|AKIA[0-9A-Z]{16}|ASIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{30,}|sk-[A-Za-z0-9_-]{20,}|gh[pousr]_[A-Za-z0-9]{20,}|-----BEGIN [A-Z ]+ PRIVATE KEY-----' docs/audits || true +find docs/audits -type f | wc -l +``` + +Result: exit 0. Exactly 16 untracked files; no prohibited data/state file; no high-confidence secret-pattern match. Paths, sizes and hashes are in `aws-postgres-baseline.md`. + +### Repository modifications + +The first repository modification created only the four required remediation ledger/contract documents under `docs/remediation/`. No application, infrastructure or immutable audit file was changed. + +### Source database safety and coherent backup + +```zsh +df -h . /private/tmp +stat -f 'path=%N size=%z modified=%Sm' -t '%Y-%m-%dT%H:%M:%S%z' src/data/charities.db +shasum -a 256 src/data/charities.db +sqlite3 -readonly src/data/charities.db "SELECT key, value FROM metadata WHERE key IN ('schema_version','registry_schema_version','grant_overview_schema_version') ORDER BY key;" +sqlite3 -readonly src/data/charities.db 'PRAGMA quick_check;' +sqlite3 -readonly src/data/charities.db ".backup '/private/tmp/fip-remediation-baseline-20260728.db'" +stat -f 'path=%N size=%z modified=%Sm' -t '%Y-%m-%dT%H:%M:%S%z' /private/tmp/fip-remediation-baseline-20260728.db +shasum -a 256 /private/tmp/fip-remediation-baseline-20260728.db +sqlite3 -readonly /private/tmp/fip-remediation-baseline-20260728.db 'PRAGMA integrity_check;' +sqlite3 -readonly /private/tmp/fip-remediation-baseline-20260728.db 'PRAGMA foreign_key_check;' +``` + +Result: exit 0. Source quick check and backup integrity are `ok`; zero FK violations. Source and coherent-backup SHA-256 values are recorded in the baseline. + +### Database reconciliation baseline + +Read-only SQL was executed on `/private/tmp/fip-remediation-baseline-20260728.db` for table counts, distinct mapped grants, source identity duplicates, conversion eligibility/total, anomaly cohorts, exact business-key duplicate groups, duplicate charity numbers, currency/conversion status, programme provenance and geography methods. + +Result: exit 0. All mandatory controls match the immutable audit. The exact business key is `(funding_name, recipient_name, amount, currency, date, description)` and reproduces 4,271 groups / 14,529 extra rows. Classified count 134,554 is the non-unclassified cohort intersected with non-negative, EUR-convertible grants. + +### Backend and frontend baseline + +```zsh +venv/bin/python -m compileall -q src +venv/bin/python -m flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics +PYTHONPATH=src venv/bin/python -m pytest src/tests --cov=bff --cov-report=term-missing --cov-fail-under=70 +cd frontend && npm ci +cd frontend && npm run test +cd frontend && npm run lint +cd frontend && npm run build +``` + +Result: all commands exit 0. Backend 286 passed, 76.29% coverage, 55 deprecation warnings. Frontend 8 passed; lint has five hook warnings; build has a 1.96 MB main JavaScript chunk warning. + +### Architecture/config inventory + +Read-only `sed`, `find` and `rg` commands inspected FastAPI main/auth/config/proxy/admin/charity/news routes, Dockerfile, Compose, CI, dependency manifests, environment examples, source trees, SQLite-specific constructs and local background-process/file coordination. + +Result: current branch has local shared-secret/basic admin authentication, static credentials in frontend example, broad router authentication, unrestricted authenticated proxy forwarding, synchronous SQLite and filesystem jobs. These are remediation inputs, not accepted target controls. + +### Clean target-branch runtime baseline + +The old no-reload backend process from the prior branch was first queried accidentally, then explicitly stopped. The frontend process was also stopped. Clean target-branch processes were started with: + +```zsh +./start_backend.sh +cd frontend && npm run dev -- --host 127.0.0.1 --strictPort --port 5173 +``` + +Local `curl` checks then recorded health, OpenAPI counts, frontend reachability and anonymous charity/admin/proxy responses. A local Python request used configuration internally without printing credentials to verify login and cached API timing. + +Result: clean target branch has 33 paths/38 operations; health/frontend 200; anonymous protected routes 401; login 200; cached overview 21.3 ms then 4.2 ms; registry FTS 808.9 ms. + +### Legacy Docker baseline + +```zsh +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker info --format 'server={{.ServerVersion}} arch={{.Architecture}}' +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker-compose build bff +df -h . /private/tmp +DOCKER_CONFIG=/private/tmp/fip-audit-docker-config docker system df -v +``` + +Result: Docker server 23.0.5/aarch64. Build transferred an 8.81 GB context, installed unpinned dependencies, then failed at `COPY src/` with `no space left on device` (exit 1). Host free space fell from 69 GiB to 51 GiB; existing prior image is 9.37 GB and build cache reports 12.74 GB. No user-owned Docker artifact was deleted. + +### Baseline staging review + +The first `git diff --cached --check` exited 2 because the immutable audit Markdown contains intentional two-space CommonMark hard breaks. The audit files were not changed. The same check also identified removable whitespace in newly created remediation documents; those new documents were corrected and restaged. Future commits will pass `git diff --check` because the immutable audit is established in this dedicated baseline commit. + +### Immutable baseline commit and Phase-0 closeout + +```zsh +git commit -m "Document immutable AWS readiness baseline" +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 +shasum -a 256 src/data/charities.db +jq -e . docs/remediation/schemas/migration-manifest.schema.json +git diff --check +``` + +Result: commit `19e84ba11dd3567fc871b3411166ae59a5b6eef0` created with only the 16 immutable audit artifacts and four initial remediation documents. The final audit checksum set retained aggregate SHA-256 `d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`; the active database retained SHA-256 `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`. The manifest Schema parses successfully and the post-baseline working diff passes whitespace validation. + +## Phase 1 — Security hardening + +### Route, configuration and client inventory + +Read-only `rg`, `sed`, `find`, Git and OpenAPI inspection enumerated every route, authentication dependency, proxy behavior, configuration input, frontend login/fetch path and existing test assumption. The resulting classification is `aws-postgres-route-inventory.md`. + +One overly broad local environment inventory command printed values from the ignored developer `.env` into transient tool output. No value was copied into source, documentation, Git or a remote service. The user was notified immediately and rotation of the affected third-party credentials was recommended. The `.env` file was not changed. + +### Security implementation and test gate + +```zsh +venv/bin/python -m compileall -q src +venv/bin/python -m flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics +PYTHONPATH=src venv/bin/python -m pytest src/tests --cov=bff --cov-report=term-missing --cov-fail-under=70 +cd frontend && npm run test +cd frontend && npm run lint +cd frontend && npm run build +git diff --check +git grep -nE '' -- ':!docs/audits/**' +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +``` + +Result: compile and blocking Flake8 pass with zero findings. Backend has 297 passing tests plus 8 mutation-route subtests, 53 dependency/test-client warnings and 76.69% coverage. The 11 security test methods cover OIDC signature/claim validation, 401, 403, role success, audit fields, proxy/path/header allowlists, rate limiting, required idempotency/replay, payload limits, message/traceback redaction, public read-only behavior and production startup failure. Frontend has 8 passing tests; lint exits 0 with the five baseline hook warnings; production build passes with the baseline large-chunk warning. The tracked-source credential scan has no high-confidence match. + +### Default-runtime HTTP gate + +The first local start attempt inside the filesystem sandbox reached application startup but could not bind the loopback port (`operation not permitted`). It was repeated with the narrowly approved local start permission. No external network or service was contacted. + +```text +GET /health -> 200 + X-Request-ID +GET /api/charities -> 401 + audit event + X-Request-ID +GET /api/admin/pipeline/status -> 401 + audit event + X-Request-ID +GET /api/core/v1/data -> 401 + audit event + X-Request-ID +POST /api/auth/login -> 404 + audit event + X-Request-ID +``` + +Result: the application starts with authentication disabled, exposes only the public read surface, rejects protected/admin/proxy access and hides the local login until the development bypass is explicitly configured. The local process was stopped cleanly after the checks. + +### Phase-1 immutability and external-action check + +The active SQLite source retained SHA-256 `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`; the immutable audit checksum aggregate retained `d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`. No AWS mutation, paid API call, external upload, push or Docker artifact deletion occurred. + +## Phase 2 — Docker and local PostgreSQL foundation + +### Explicit registry authorization and dependency locks + +On 2026-07-29 the user explicitly authorized downloads only from PyPI, +`files.pythonhosted.org`, `registry.npmjs.org` and the Docker Hub registry/auth +endpoints. AWS, live scrapers, paid APIs, uploads and pushes remained forbidden. + +```zsh +venv/bin/python -m pip install --dry-run --ignore-installed --only-binary=:all: --report /private/tmp/fip-pip-tools-resolve.json pip-tools==7.5.2 +venv/bin/python -m pip install --index-url https://pypi.org/simple --only-binary=:all: --require-hashes -r requirements-locking.txt +venv/bin/pip-compile --index-url=https://pypi.org/simple --resolver=backtracking --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url --output-file=requirements-runtime.txt requirements-runtime.in +venv/bin/pip-compile --index-url=https://pypi.org/simple --resolver=backtracking --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url --output-file=requirements.txt requirements.in +``` + +The first compile attempts failed before modifying either output because +`pip-tools==7.5.2` is incompatible with `pip==26.1.2` +(`PackageFinder.allow_all_prereleases` is absent). The correction was resolved, +hashed and installed explicitly: + +```zsh +venv/bin/python -m pip install --dry-run --ignore-installed --only-binary=:all: --report /private/tmp/fip-pip-tools-resolve-pip25.json pip==25.3 pip-tools==7.5.2 +venv/bin/python -m pip install --index-url https://pypi.org/simple --only-binary=:all: --require-hashes -r requirements-locking.txt +venv/bin/pip-compile --index-url=https://pypi.org/simple --resolver=backtracking --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url --output-file=requirements-runtime.txt requirements-runtime.in +venv/bin/pip-compile --index-url=https://pypi.org/simple --resolver=backtracking --generate-hashes --strip-extras --allow-unsafe --no-emit-index-url --output-file=requirements.txt requirements.in +venv/bin/python -m pip install --index-url https://pypi.org/simple --require-hashes -r requirements.txt +venv/bin/python -m pip check +cd frontend && npm ci --ignore-scripts --registry=https://registry.npmjs.org +``` + +Result: both Python lockfiles contain exact transitive versions and accepted +SHA-256 artifact hashes. The npm lock contains exact tarball integrity values; +its only resolved host is `registry.npmjs.org`. `pip check` reports no broken +requirements. The authoritative version/digest inventory is +`aws-postgres-dependency-locks.md`. + +### Local application checks + +```zsh +venv/bin/python -m compileall -q src +venv/bin/python -m flake8 src --count --select=E9,F63,F7,F82 --show-source --statistics +PYTHONPATH=src venv/bin/python -m pytest src/tests --cov=bff --cov-report=term-missing --cov-fail-under=70 +cd frontend && npm run test +cd frontend && npm run lint +cd frontend && npm run build +``` + +Result: compile and blocking Flake8 pass; 300 backend tests pass with 76.57% +coverage and 53 warnings; 8 frontend tests, lint and the Vite production build +pass. The five existing hook warnings and 1.96 MB main chunk remain assigned to +Phase 7. + +### Base-image pulls and digest capture + +```zsh +docker pull python:3.12.13-slim-bookworm +docker pull node:22.22.2-alpine +docker pull nginxinc/nginx-unprivileged:1.30.4-alpine3.24 +docker pull postgres:16.14-alpine3.24 +docker image inspect --format '|||||' +``` + +The parallel Python and PostgreSQL pull clients reached their command timeouts; +the exact two pulls were resumed and completed. All selected local variants are +`linux/arm64`; the pinned values are multi-platform manifest-list digests. The +Dockerfile frontend image resolved to +`sha256:e87caa74dcb7d46cd820352bfea12591f3dba3ddc4285e19c7dcd13359f7cefd`. +No non-approved image source was contacted. + +### Image build and contract verification + +```zsh +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose config --quiet +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose --profile operations config --quiet +docker-compose build backend frontend +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose build backend frontend +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose build frontend +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config scripts/verify_container_image.sh foundation-intelligence-backend:local +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config scripts/verify_container_image.sh foundation-intelligence-frontend:local +docker buildx bake --print +``` + +The first build invocation stopped before Dockerfile execution because the +required Compose secret variable was absent. The second used the variable but +stopped before the build because the configured `docker-credential-desktop` +binary was unavailable. An isolated `/private/tmp` Docker config containing no +credential helper fixed that local CLI issue. The first frontend stage then +proved that production npm omission removed `tsc`; `npm ci --include=dev` fixed +the build stage while the runtime remained static-only. The final backend +context was 1.04 MB and frontend context 661 kB, compared with the 8.81 GB +baseline context. + +The final backend is `354092439` bytes and UID/GID `10001:10001`; the frontend +is `56230634` bytes and UID/GID `101:101`. Both have healthchecks and contain no +`.env`, SQLite file, application credential/key path, domain-data payload or +compiler. Both declare `linux/amd64` and `linux/arm64` in `docker-bake.hcl`. +The installed Docker CLI has no usable buildx plugin (`docker buildx bake +--print` exits 125 with `unknown flag: --print`), so local manifest assembly is +not claimed; the platform declaration is statically verified and CI will run +the real multi-platform build. + +### Gate-2 Compose start, health, HTTP and stop + +The first default-port start created containers but PostgreSQL did not start +because `127.0.0.1:5432` was already occupied. Host ports were parameterized; +containers and network from that attempt were removed without deleting the +volume. + +```zsh +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose up -d --no-build postgres backend frontend +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder docker-compose down --remove-orphans +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 BACKEND_HOST_PORT=58000 FRONTEND_HOST_PORT=58080 docker-compose up -d --no-build postgres backend frontend +docker inspect --format '' foundation-intelligence-postgres-1 foundation-intelligence-backend-1 foundation-intelligence-frontend-1 +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -tAc 'SELECT current_database(), current_user, current_setting('"'"'server_version'"'"');' +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:58000/health/live +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:58000/health/ready +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:58080/ +curl --fail --silent --show-error --max-time 10 http://127.0.0.1:58080/health/ready +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 BACKEND_HOST_PORT=58000 FRONTEND_HOST_PORT=58080 docker-compose stop +docker inspect --format '' foundation-intelligence-postgres-1 foundation-intelligence-backend-1 foundation-intelligence-frontend-1 +docker logs --tail 50 foundation-intelligence-backend-1 +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 BACKEND_HOST_PORT=58000 FRONTEND_HOST_PORT=58080 docker-compose down --remove-orphans +``` + +Result: PostgreSQL 16.14, backend and frontend all became healthy through +health-based dependencies. Backend readiness returned PostgreSQL `healthy`; +frontend static HTML loaded. An initial frontend `/health/ready` request exposed +a SPA fallback bug; the corrected prefix proxy then returned the backend JSON +readiness response. Backend/frontend roots are read-only, all Linux capabilities +are dropped and `no-new-privileges` is active. Uvicorn logged `Application +shutdown complete`; PostgreSQL/frontend exited 0 and the init-wrapped backend +reported the delivered SIGTERM as 143. Containers and network were removed; +the local PostgreSQL volume and images were retained. + +## Phase 3 — PostgreSQL schema + +### Schema and runtime implementation + +Read-only SQLite `pragma_table_info` and distinct-value queries captured every +source column plus observed classification, conversion, provenance and link +status before target constraints were written. The authoritative DDL was added +as Alembic revision `0001_postgresql_foundation`; no schema was generated from +mutable ORM metadata. + +```zsh +venv/bin/python -m compileall -q src alembic +venv/bin/python -m flake8 src alembic --count --select=E9,F63,F7,F82 --show-source --statistics +PYTHONPATH=src venv/bin/python -m pytest src/tests/test_postgres_schema.py -q +git diff --check +``` + +Result: static schema/search/cursor tests and the production import guard pass. +The normal suite later records 304 passed, one live test skipped and 76.41% +coverage. + +### Real PostgreSQL Alembic and integration gate + +PostgreSQL remained bound only to local port 55432 and used the non-production +Compose secret file. Commands passed the password by file path and never logged +its value. + +```zsh +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 docker-compose up -d --no-build postgres +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic upgrade head +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest src/tests/test_postgres_schema.py -q +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic downgrade base +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -tAc '' +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic upgrade head +docker-compose build backend +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic downgrade base +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 docker-compose --profile operations run --rm migration +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -tAc '' +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest src/tests/test_postgres_schema.py -q +``` + +The first real search test failed because asyncpg could not infer the SQL type +of a null optional status parameter. Casting that bind to PostgreSQL `text` +resolved the defect; the repeated real suite passes 4/4. Both host and +container zero-to-head upgrades pass. After downgrade, only `alembic_version` +remained. The final catalog reports 25 application tables, 30 FKs, zero +unvalidated FKs, 117 checks, the three search indexes and extensions `pg_trgm` +and `plpgsql`. + +### Production-mode boundary + +```zsh +APP_ENV=production AUTH_MODE=oidc OIDC_ISSUER=https://issuer.invalid OIDC_AUDIENCE=foundation-intelligence OIDC_JWKS_URL=https://issuer.invalid/jwks CORS_ORIGINS=https://app.invalid DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m uvicorn bff.main:app --host 127.0.0.1 --port 58001 --no-server-header +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:58001/health/live +curl --fail --silent --show-error --max-time 5 http://127.0.0.1:58001/health/ready +curl --silent --show-error --max-time 5 --output /dev/null --write-out '%{http_code}' 'http://127.0.0.1:58001/api/charities/directory/organizations?query=Alpha' +``` + +Result: the process logged PostgreSQL repository initialization, liveness and +readiness returned 200 with PostgreSQL healthy, anonymous search returned 401, +and Ctrl-C logged complete application shutdown. The `.invalid` OIDC values +were validation-only placeholders and were never contacted. The subprocess +import guard separately proves production imports no `sqlite3` module. + +## Phase 4 — deterministic SQLite-to-PostgreSQL migration + +### Source and capacity preflight + +```zsh +PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres preflight --source src/data/charities.db --expected-checksum 8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651 --expected-schema-version 7 +sqlite3 'file:src/data/charities.db?mode=ro&immutable=1' '' +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +``` + +The preflight returned schema `7`, `integrity_check=ok`, the approved checksum +and all 12 source counts. The source is 2,100,543,488 bytes; the conservative +minimum-free estimate is 23,340,679,168 bytes including a 10 GiB safety margin. +All capacity checks passed. The protected checksums remained +`8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` +and `d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`. + +### Implementation and real-PostgreSQL tests + +```zsh +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 docker-compose up -d --no-build postgres +PYTHONPATH=src venv/bin/python -m py_compile src/migration/sqlite_to_postgres.py src/tests/test_sqlite_to_postgres_migration.py +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest src/tests/test_sqlite_to_postgres_migration.py -q +PYTHONPATH=src venv/bin/python -m pytest -q +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic downgrade 0001_postgresql_foundation +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic downgrade 0002_exchange_rate_period +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +``` + +The fixture gate progressed from two unit passes plus one skip to a final five +real-PostgreSQL passes. It covers immutable source access, checksum rejection, +activation, repeated no-op, quarantine, failed-candidate retry, conflicting +override rollback, full global-record transaction rollback, dataset rollback +and restoration of any pre-existing active dataset. The final normal suite is +308 passed, 2 intentional skips, 8 subtests and 53 known warnings. + +Two source-fidelity defects were found by full reconciliation and fixed with +append-only Alembic revisions. `exchange_rate_date` contains 302,049 monthly +`YYYY-MM` periods, not malformed dates. In addition, 8,736 grant award values +contain full ISO timestamps; truncating them created 85 artificial duplicate +groups. Revisions `0002_exchange_rate_period` and +`0003_grant_award_timestamp` preserve both values exactly and passed their +downgrade/upgrade cycles. + +### Full migration attempts and failure isolation + +The migration command shape used throughout was: + +```zsh +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres migrate --source src/data/charities.db --expected-checksum 8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651 --expected-schema-version 7 --dataset-version --code-revision --actor-id codex-local-remediation --actor-type ci --output-directory --batch-size 10000 +``` + +Executed full checkpoints: + +- `262b4a894dd1f3c869ddfb43301b20990bf5f42d` stopped and quarantined the monthly exchange-period type mismatch before activation. +- One invocation containing an unverified SHA string was interrupted before PostgreSQL connection/mutation. The next invocation used `git rev-parse HEAD`; no record or candidate was created by the interrupted command. +- `9afbc4a18de5d52a159dccfe4fa55ef168069e99` loaded all rows but was rejected because truncated award dates produced 4,356 rather than 4,271 business-key duplicate groups. +- The rejected run had written 18,964 exchange rates and one override under the pre-staging implementation. With zero active datasets verified, exactly those derived rows were deleted in one local transaction (`DELETE 18964`, `DELETE 1`) before the atomic-staging correction was retested. +- `919aa96e835775b79a09aa639f5cc57826ec77c7` produced the first successful full snapshot `sqlite-v7-8fc0cce61c81`. +- An initial `r2` load under `a74d75c0772b6aaff839d9460302e8d3eca158de` was interrupted while inactive after a progress query exposed that a fixture cleanup had not restored the preceding active snapshot. `rollback --dataset-version sqlite-v7-8fc0cce61c81` restored it before the test-isolation fix. +- `d6d2b69d1f9ff7dd8bc6f58021060586b3c17757` produced the final full snapshot `sqlite-v7-8fc0cce61c81-r2` and manifest run `60af368e-c440-5521-9648-5ab272f9ddb6`. + +Read-only `docker-compose exec ... psql -Atc` queries repeatedly checked only +candidate status, active flags, schema revision, constraint validation and +aggregate table counts during these runs. No source rows, secrets or credential +values were printed. + +### Manifest, idempotency and rollback gate + +```zsh +python3 -c '' +jq '' /private/tmp/fip-phase4-full-report-d6d2b69/migration-sqlite-v7-8fc0cce61c81-r2.json + +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres rollback --dataset-version sqlite-v7-8fc0cce61c81 +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -Atc '' +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres rollback --dataset-version sqlite-v7-8fc0cce61c81-r2 +``` + +The committed JSON report validates against Draft 2020-12. All source/target +counts and controls pass, including 30 catalog-derived FK relationships with +zero violations. The exact repeated migration returns +`idempotent_noop=true` and the same run ID. Full rollback to the first snapshot +retained 302,546 active grants and 397,469 active registry rows; switching back +to `r2` succeeded and left exactly one active dataset. + +### Final container rebuild and inspection + +```zsh +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config docker build --pull=false --target backend-runtime -t foundation-intelligence-backend:local . +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config scripts/verify_container_image.sh foundation-intelligence-backend:local +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config docker run --rm --network none --read-only --tmpfs /tmp:rw,noexec,nosuid,size=32m,uid=10001,gid=10001 --entrypoint python foundation-intelligence-backend:local -c 'from migration.sqlite_to_postgres import MIGRATION_SCHEMA_VERSION; print(MIGRATION_SCHEMA_VERSION)' +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config docker run --rm --network none --read-only --tmpfs /tmp:rw,noexec,nosuid,size=32m,uid=10001,gid=10001 --workdir /app --entrypoint alembic foundation-intelligence-backend:local heads +docker image inspect --format '' foundation-intelligence-backend:local +``` + +The legacy builder re-downloaded only hash-locked PyPI artifacts from the +approved hosts and did not pull a base image. No dependency version changed. +The final image passes the complete contract, contains migration and Alembic +head `0003_grant_award_timestamp`, and has local image ID +`sha256:e43491e5e7080e0923b9d777aa1f985bfd3c4897482d662d0be7bf7364758b91`. +It is an unpushed local image and therefore has no repository digest. + +## Phase 5 — PostgreSQL application conversion + +### Runtime implementation and static gates + +```zsh +python3 -m py_compile src/bff/postgres/*.py src/bff/main.py src/bff/audit.py src/bff/schemas.py +git diff --check +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_security.py src/tests/test_database.py src/tests/test_postgres_schema.py -k 'not integration' +PYTHONPATH=src venv/bin/python -c '' +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_postgres_application.py -k 'not TestPostgreSQLApplicationIntegration' +``` + +The PostgreSQL router matches all 25 existing organization/grant contracts. +Production now selects PostgreSQL-backed admin routes in addition to the data +routes. Manual triggers create durable jobs rather than filesystem status, +locks, logs or subprocesses. Production audit middleware awaits the append-only +PostgreSQL sink. Static/security/database tests passed without network access. + +### Real PostgreSQL application gate + +The retained PostgreSQL 16.14 container was healthy on loopback port 55432. +Every invocation used `DATABASE_PASSWORD_FILE`; no password was printed. + +```zsh +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 docker-compose ps +APP_ENV=test AUTH_MODE=disabled RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_postgres_application.py -vv +``` + +The first repetitions exposed four PostgreSQL-specific defects: `grant` was a +reserved SQL alias, an integer interval bind was inferred as text, an aggregate +CTE retained the pre-CTE alias and joined detail columns were ambiguous. Each +was corrected and the affected real journey was repeated. The final Phase-5 +suite passes five tests: complete reads and Pydantic response validation, +transactional link/cache/job/audit mutations with outer rollback, missing-DB +startup failure, production PostgreSQL route selection and real readiness. + +### Regression and protected-state gates + +```zsh +PYTHONPATH=src venv/bin/python -m pytest -q +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +``` + +Result: 310 passed, five intentional skips, eight subtests and 53 known +deprecation warnings. The SQLite checksum remains +`8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`; +the aggregate audit checksum remains +`d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`. +No packages or images were downloaded in Phase 5. No AWS, paid/live API, +scraper/model, upload or push action occurred. + +## Phase 6 — performance and concurrency + +### Baseline, migration and materialization + +All PostgreSQL access remained on `127.0.0.1:55432` or through the retained local Compose container. Commands used the local password file; its value was never printed. + +```zsh +PYTHONPATH=src DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/python scripts/benchmark_postgres.py --samples 1 --concurrency 2 +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic upgrade head +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -Atc "SELECT refresh_analytics_materializations('')" +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic downgrade 0003_grant_award_timestamp +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder venv/bin/alembic upgrade head +``` + +The first `0004` upgrade rolled back transactionally because SQLAlchemy treated a colon in a function string literal as a bind marker. Removing that punctuation resolved the parser collision. After adding country connections and two grant source-ID indexes, the full `0004 -> 0003 -> 0004` cycle passed and the active materialization refreshed to 204,220 rows. No active dataset changed. + +The diagnostic single-sample baseline measured health 2.66 ms, organization list 324.29 ms, map 2,003.54 ms, trends 413.14 ms, themes 1,092.86 ms, summary 4,347.61 ms, dashboard 8,941.12 ms, two concurrent dashboards 16,025.45 ms, exact registry 15,038.99 ms, text registry 16,841.02 ms and country funders 800.02 ms. These values drove the aggregate/index work and are not represented as percentile evidence. + +### Static, PostgreSQL and performance tests + +```zsh +PYTHONPATH=src venv/bin/python -m py_compile src/bff/postgres/base.py src/bff/postgres/analytics_repository.py src/bff/postgres/funder_repository.py src/bff/postgres/registry_repository.py src/bff/postgres/routes.py src/migration/sqlite_to_postgres.py src/tests/test_postgres_application.py src/tests/test_postgres_performance.py src/tests/test_postgres_schema.py scripts/benchmark_postgres.py scripts/load_test_api.py alembic/versions/0004_versioned_analytics_materializations.py +venv/bin/python -m flake8 src alembic scripts/benchmark_postgres.py scripts/load_test_api.py --count --select=E9,F63,F7,F82 --show-source --statistics +git diff --check +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_postgres_performance.py +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_postgres_performance.py src/tests/test_postgres_application.py +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_postgres_application.py src/tests/test_postgres_schema.py src/tests/test_sqlite_to_postgres_migration.py +``` + +The dedicated gate finishes 5/5, performance plus application finishes 10/10, and application/schema/migration finishes 15/15. The latter exercises real candidate activation, materialization, rollback and cleanup. Static compilation, blocking Flake8 and diff whitespace checks pass. + +Recorded diagnostic failures were resolved and repeated: + +- An invocation with `TEST_DATABASE_URL=postgresql+asyncpg://foundation_app@127.0.0.1:55432/...` omitted the password and produced four authentication failures; the prescribed `RUN_POSTGRES_INTEGRATION=1` plus secret-file configuration then passed. +- The schema fixture attempted to insert a second active dataset. It now transactionally marks the prior dataset `rolled_back`, creates the fixture, deletes it and restores the exact prior active status; the repeated test passes. +- Ten simultaneous health requests alongside a heavy query exceeded the five-connection pool by construction. The isolation gate now matches the bounded five-request pool workload and passes below 100 ms. +- One combined run observed a 4,006 ms single cold dashboard outlier and 1,110.944 ms instrumented search plan. Cold dashboard evidence now uses 20 independent cache-cleared samples and checks p95; `EXPLAIN` retains index verification while endpoint p95 independently enforces the 1-second search SLO. +- A no-results search fixture contained common English tokens and legitimately produced trigram matches. A high-entropy token now verifies the corrected nonzero overall `registry_count` on an empty result page. + +### Final benchmark and authenticated in-process API load + +```zsh +PYTHONPATH=src APP_ENV=test DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_POOL_SIZE=5 DATABASE_MAX_OVERFLOW=5 DATABASE_POOL_TIMEOUT_SECONDS=5 DATABASE_CONNECT_TIMEOUT_SECONDS=5 DATABASE_STATEMENT_TIMEOUT_MS=30000 venv/bin/python scripts/benchmark_postgres.py --samples 10 --concurrency 5 +PYTHONPATH=src APP_ENV=production DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_POOL_SIZE=5 DATABASE_MAX_OVERFLOW=5 DATABASE_POOL_TIMEOUT_SECONDS=5 DATABASE_CONNECT_TIMEOUT_SECONDS=5 DATABASE_STATEMENT_TIMEOUT_MS=30000 venv/bin/python scripts/load_test_api.py --samples 20 --concurrency 5 +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -At -F '|' -c '' +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +``` + +The first API-load attempt used the normal 140-request application rate limit and correctly received HTTP 429 before the concurrent dashboard section. The local test harness was changed to 10,000 requests for this bounded run only and uses a generated in-memory RSA/JWKS/JWT plus `MemoryAuditSink`; it makes no identity, network or persistent audit calls. A missing `.venv/bin/python` invocation failed before execution and was repeated with the repository's existing `venv/bin/python`. + +Final repository cold-dashboard p95 is 255.31 ms. Final production-mode API p95 values are health 3.70 ms, organization list 245.73 ms, map 4.95 ms, lazy map connections 6.04 ms, overview 43.45 ms, yearly trends 5.09 ms, exact registry 18.01 ms, text registry 83.93 ms and country funder ranking 19.20 ms. Five concurrent dashboards complete in 472.80 ms at 10.575/s with zero errors. Endpoint error rates are all zero; cache hit ratio is 0.8333 and no pool connection remains checked out. + +The catalog query reports PostgreSQL 16.14, Alembic `0004_versioned_analytics`, exactly one active dataset and 204,220 controlled aggregate rows. It also reports 39 validated FKs, zero unvalidated FKs and 136 checks. Protected checksums remain `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` and `d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`. + +No dependency/image download, AWS access, paid/live external call, scraper/model invocation, upload or push occurred in Phase 6. + +### Final normal suite and container rebuild + +```zsh +PYTHONPATH=src venv/bin/python -m pytest -q +DOCKER_CONFIG=/private/tmp/fip-phase2-docker-config DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock docker build --pull=false --target backend-runtime -t foundation-intelligence-backend:local . +``` + +The normal suite passes 311 tests, skips nine explicit live-environment tests, passes eight route subtests and emits the same 53 dependency/test-client deprecation warnings. The first Docker request was rejected before process creation because the execution service's usage/approval limit had been reached. After the user explicitly reconfirmed the exact local action, the same command succeeded. The pinned base and hash-locked dependency layers all reported `Using cache`; no pull, package download or dependency resolution occurred. The final local arm64 image is 354,456,439 bytes, runs as `10001:10001` and has image ID `sha256:cf71388a8fc83cdc32632ea2cf8ea9b7b27d4d68b164f848cd6e97b49905af8a`. + +The scoped local checkpoint was then attempted: + +```zsh +git add +``` + +The ordinary sandbox failed with `Unable to create .git/index.lock: Operation not permitted`. The required escalated repetition was rejected before process creation by the same execution-service usage/approval limit. No index lock remained and no indirect `.git` write was attempted. The user then explicitly reconfirmed targeted staging and the local Phase-6 commit, while continuing to prohibit broad staging and push. + +## Phase 7 — frontend remediation + +All application and browser execution in this phase remained local. The +initial remediation used the existing `frontend/node_modules` and npm +lockfile. After the user separately approved the named test dependencies, +only `registry.npmjs.org` was contacted to resolve and install the exact +Playwright/axe packages documented below. No browser or image download, +external API call, AWS action, upload or push occurred. + +### Baseline and source inspection + +```zsh +cd frontend && npm test +cd frontend && npm run lint +cd frontend && npm run build +rg frontend/src +npm ls --depth=0 +``` + +The baseline passed eight tests but Oxlint reported five React hook warnings. +The production main chunk was 1,963.34 kB raw / 611.99 kB gzip and Vite emitted +the large-chunk warning. The stylesheet also contained a runtime Google Fonts +import. Playwright and axe packages were absent. + +### Static, unit and bundle gates + +```zsh +cd frontend && npm run lint +cd frontend && npm test +cd frontend && npm run build +cd frontend && npm run check:bundle +node --check frontend/scripts/check-runtime-layout.mjs +git diff --check +``` + +Final results: Oxlint emits zero warnings; all 13 tests pass; TypeScript and +Vite pass; initial JavaScript is 87.80 KiB gzip / 120 KiB, initial CSS is 18.59 +KiB gzip / 25 KiB and the largest deferred chunk is 392.37 KiB gzip / 425 KiB. +No generated `frontend/dist/` artifact is tracked. + +### Local browser journeys + +```zsh +cd frontend && npm run test:runtime +``` + +The script builds the current source, starts Vite Preview on `127.0.0.1:4173`, +launches the already installed Google Chrome 150 in headless mode with +background networking disabled, injects deterministic local API responses and +uses the Chrome DevTools Protocol. It stops both local processes and removes +its isolated temporary browser profile afterward. + +The final run passes at 320, 390, 768, 1024, 1440 and 1920 pixels. It checks +page overflow, visible control bounds/names, KPI cropping, map-first ordering, +wrapped map controls, one initial overview request, one interaction-triggered +connection request, filter drawer viewport bounds, Tab trapping, Escape/focus +restoration and browser console/runtime exceptions. At 320 and 1024 it also +navigates through Donor Directory and Advanced Charity Commission Search and +checks the Registry empty state and Registry filter drawer. + +Early diagnostic runs correctly failed because the first checker counted the +intentionally off-canvas closed mobile sidebar, checked focus before React's +asynchronous focus tick and once exercised a stale `dist/` build. The checker +now tests only intersecting visible controls, waits for focus and always builds +current source. A Strict Mode focus restoration issue was fixed by retaining +the trigger reference until the animation-frame focus completes. + +### Named-tool pre-approval check and immutable checks + +```zsh +cd frontend && npm ls @playwright/test playwright @axe-core/playwright axe-core --depth=0 +node --version +npm --version +"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" --version +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +git status --short +git diff --check +git diff --stat +``` + +The named Playwright/axe package query returned `(empty)`. At that point, +installing them required an npm-registry download and a lockfile update, so the +work stopped for explicit approval. The active SQLite checksum remained +`8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`; +the aggregate audit checksum remains +`d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a`. +No Phase-7 files were staged or committed before that approval. + +### Approved exact dependency install and named browser gate + +The user then explicitly approved downloads for the named Playwright and axe +dependencies. Network access was restricted to `registry.npmjs.org`, install +scripts and audits were disabled, and `PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1` +prevented browser downloads. The already installed Google Chrome remained the +only browser executable used. + +```zsh +cd frontend && npm view @playwright/test version --registry=https://registry.npmjs.org +cd frontend && npm view @axe-core/playwright version --registry=https://registry.npmjs.org +cd frontend && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install --package-lock-only --ignore-scripts --no-audit --no-fund --registry=https://registry.npmjs.org +cd frontend && PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm ci --ignore-scripts --no-audit --no-fund --registry=https://registry.npmjs.org +cd frontend && npm ls @playwright/test playwright playwright-core @axe-core/playwright axe-core --depth=1 +cd frontend && npm run test:e2e +``` + +The registry resolved exact direct versions `@playwright/test@1.62.0` and +`@axe-core/playwright@4.12.1`. Their locked transitive runtime packages are +`playwright@1.62.0`, `playwright-core@1.62.0` and `axe-core@4.12.1`; npm also +locked the platform-optional `fsevents@2.3.2`. The exact lockfile integrity +values are recorded in `docs/remediation/aws-postgres-dependency-locks.md`. +The lockfile-only diff showed only these named packages and their required +transitive entries; no existing package version changed. `npm ci` added 80 +local packages and performed no lifecycle script, audit or browser download. + +The first axe-enabled run found four genuine accessibility defects: invalid +SVG-group labelling, insufficient active-navigation contrast, a skipped +heading level and an unnamed compact-header disclosure. Each source issue was +corrected and the complete gate was repeated. The final named gate passes all +six viewport projects: eight tests pass and four deliberately redundant +secondary journeys are skipped. The Overview journey runs at 320, 390, 768, +1024, 1440 and 1920 pixels; the Donor/Registry journey additionally runs at +320 and 1024 pixels. Every executed page reports zero axe violations, browser +console errors, runtime errors and unexpected API requests. + +## Phase 8 — durable pipelines and storage contracts + +All execution remained local. No source scraper, news/model call, dependency +download, AWS API, S3/SQS/EventBridge/Step Functions action, upload or push +occurred. + +### Static and contract gates + +```zsh +PYTHONPATH=src venv/bin/python -m py_compile alembic/versions/0005_durable_pipeline.py src/pipelines/durable.py src/pipelines/durable_worker.py src/bff/postgres/idempotency_repository.py src/bff/postgres/job_repository.py src/bff/postgres/pipeline_repository.py src/bff/postgres/admin_routes.py src/bff/security.py src/bff/main.py src/tests/test_durable_pipeline.py +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_durable_pipeline.py +venv/bin/python -m flake8 src/bff/postgres/idempotency_repository.py src/bff/postgres/job_repository.py src/bff/postgres/pipeline_repository.py src/pipelines/durable.py src/pipelines/durable_worker.py src/tests/test_durable_pipeline.py alembic/versions/0005_durable_pipeline.py --count --select=E9,F63,F7,F82 --show-source --statistics +venv/bin/python -m json.tool config/source-pipelines.json +git diff --check +``` + +The first contract run correctly exposed two test-only expectation defects: +the immutable-store negative test supplied a differently sized payload and +therefore reached the length guard before its expected checksum guard, and a +text scan matched the explanatory word `subprocess` in a docstring. The +fixture now uses equal-length changed bytes and the coordination test checks +imports/calls rather than prose. The repeated local gate passes seven tests +with the one PostgreSQL-only test intentionally skipped. Python compilation, +blocking Flake8, JSON parsing and whitespace checks pass. + +### Local PostgreSQL migration and integration + +```zsh +docker-compose ps +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_durable_pipeline.py +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic downgrade 0004_versioned_analytics +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +PYTHONPATH=src venv/bin/python -m pytest -q +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_durable_pipeline.py src/tests/test_postgres_application.py src/tests/test_postgres_schema.py +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -At -F '|' -c '' +``` + +The retained PostgreSQL 16.14 container was already healthy; no image pull or +build occurred. Initial upgrade, `0005 -> 0004 -> 0005` and the repeated real +integration all pass. The dedicated run passes 8/8 and the combined +Phase-8/application/schema run passes 18/18. The normal suite passes 318 +tests, skips ten explicit live-environment tests, passes eight route subtests +and emits the existing 53 dependency/test-client deprecation warnings. + +The first catalog command omitted the Compose password-file variable and was +rejected by Compose before `psql` started; it was repeated with the existing +local secret-file path. Final catalog evidence is Alembic +`0005_durable_pipeline`, 40 application tables, 49 FKs, 161 checks, active +dataset `sqlite-v7-8fc0cce61c81-r2`, eight source configurations, zero enabled +schedules and eight governance blocks. Test transactions roll back their job, +idempotency, ingestion and object fixtures. The production-startup integration +persists only the authoritative eight disabled source-control rows. + +## Phase 9 — governance and retention + +All retention execution was dry-run/report-only. No row, object, source file, +backup or dataset was deleted or archived. + +### Configuration and local gates + +```zsh +PYTHONPATH=src venv/bin/python -m py_compile alembic/versions/0006_governance_retention.py src/governance/retention.py src/governance/exposure.py src/bff/postgres/governance_repository.py src/bff/postgres/governance_routes.py src/bff/postgres/admin_routes.py src/bff/utils/logging.py src/bff/main.py src/bff/schemas.py src/tests/test_governance_retention.py +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_governance_retention.py src/tests/test_security.py +venv/bin/python -m flake8 alembic/versions/0006_governance_retention.py src/governance src/bff/postgres/governance_repository.py src/bff/postgres/governance_routes.py src/tests/test_governance_retention.py --count --select=E9,F63,F7,F82 --show-source --statistics +venv/bin/python -m json.tool config/data-governance.json +git diff --check +``` + +The local governance/security run passes 19 tests, deliberately skips its one +PostgreSQL-only method and passes eight route/security subtests. Configuration +validation confirms 14 required classifications, unique retention classes, +no delete windows, no destructive/production activation, a complete role +register with unresolved named owners, a data-subject workflow and explicit +field policies. Compilation, blocking Flake8, JSON and whitespace pass. + +### Local PostgreSQL migration and evidence + +```zsh +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_governance_retention.py +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic downgrade 0005_durable_pipeline +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/alembic upgrade head +PYTHONPATH=src venv/bin/python -m pytest -q +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_governance_retention.py src/tests/test_durable_pipeline.py src/tests/test_postgres_application.py src/tests/test_postgres_schema.py +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -At -F '|' -c '' +``` + +The dedicated Phase-9 PostgreSQL run passes 9/9. It creates and releases an +incident hold, proves the hold overrides a due archive, records held/archive +dry-run manifests, proves deletion/restore evidence is immutable, reports an +expired export, creates a hashed data-subject request and rolls the entire +fixture back. The active dataset never changes. + +The `0006 -> 0005 -> 0006` cycle passes. The normal regression passes 326 +tests, skips 11 explicit live tests and passes eight subtests; combined +Phase-9/Phase-8/application/schema passes 27/27. Final catalog evidence is +revision `0006_governance_retention`, 45 tables, 55 FKs, 189 checks and 14 +policies with zero destructive flags, zero delete windows, zero active holds +and zero persisted deletion manifests. Production-startup configuration sync +persists only the 14 proposed, non-destructive policies. + +No dependency download, AWS call, paid/live API, upload or push occurred. + +## Phase 10 — observability + +All execution remained local. No CloudWatch/AWS operation, dependency +download, external API, upload or push occurred. + +```zsh +PYTHONPATH=src venv/bin/python -m py_compile src/observability/metrics.py src/bff/utils/logging.py src/bff/database.py src/bff/main.py src/bff/postgres/observability_routes.py src/pipelines/durable_worker.py src/tests/test_observability.py src/tests/test_database.py +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_observability.py src/tests/test_database.py src/tests/test_security.py +venv/bin/python -m flake8 src/observability src/bff/utils/logging.py src/bff/database.py src/bff/main.py src/bff/postgres/observability_routes.py src/pipelines/durable_worker.py src/tests/test_observability.py src/tests/test_database.py --count --select=E9,F63,F7,F82 --show-source --statistics +python3 -m json.tool config/observability.json +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_observability.py +PYTHONPATH=src venv/bin/python -m pytest -q +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_observability.py src/tests/test_governance_retention.py src/tests/test_durable_pipeline.py src/tests/test_postgres_application.py src/tests/test_postgres_schema.py +docker-compose exec -T postgres psql -U foundation_app -d foundation_intelligence -At -F '|' -c '' +shasum -a 256 src/data/charities.db +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +git diff --check +``` + +An initial static run exposed that the new JSON formatter had removed the +public `RedactingFormatter` compatibility class required by the Phase-1 +security test; the compatibility formatter was restored. The first real +PostgreSQL test correctly exhausted the application pool but expected the +standard-library timeout class instead of SQLAlchemy's timeout class. Only +the test expectation changed. The repeated Phase-10 PostgreSQL run passes +6/6, the combined regression passes 33/33 and the normal suite passes 331 +tests with 12 explicit live-environment skips, eight subtests and the existing +53 dependency/test-client warnings. + +Final catalog evidence is revision `0006_governance_retention`, active dataset +`sqlite-v7-8fc0cce61c81-r2`, eight source configurations, fourteen retention +policies, zero pending/failed outbox rows and zero dead-lettered jobs. + +## Phase 11 — Terraform AWS infrastructure definitions + +No AWS API, provider registry, remote state, DNS/certificate endpoint or paid +service was contacted. + +```zsh +command -v terraform tofu tflint checkov tfsec trivy semgrep syft grype +terraform version +docker image ls --format '{{.Repository}}:{{.Tag}} {{.Digest}} {{.ID}}' hashicorp/terraform +PYTHONPATH=src venv/bin/python scripts/validate_terraform_static.py +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_terraform_contract.py src/tests/test_database.py +PYTHONPATH=src venv/bin/python -m py_compile scripts/validate_terraform_static.py src/bff/database.py src/tests/test_terraform_contract.py src/tests/test_database.py +venv/bin/python -m flake8 scripts/validate_terraform_static.py src/bff/database.py src/tests/test_terraform_contract.py src/tests/test_database.py --count --select=E9,F63,F7,F82 --show-source --statistics +PYTHONPATH=src venv/bin/python -m pytest -q +git diff --check +``` + +`terraform version` failed with `zsh:1: command not found: terraform`. +The local Docker image query returned no Terraform image. No Terraform/OpenTofu +CLI, tflint, checkov, tfsec, trivy or semgrep is installed, and no provider +lock exists. Downloads from `registry.terraform.io` are not authorised. +Consequently fmt/init/validate/security scan/plan remain `NOT TESTED`. + +The offline checker reports 26 Terraform files, 101 resource blocks and 58 +AWS resource types across dev/staging. It validates balanced structure, +required services, private/deletion-protected RDS, public-access-blocked KMS +buckets without expiration, non-root/private/digest ECS, disabled schedules, +OIDC subject/audience and bounded wildcard contexts. The full regression +passes 334 tests with 12 explicit live skips, eight subtests and the existing +53 dependency/test-client warnings. + +## Phase 12 — CI/CD and supply chain + +Approved network activity was restricted to PyPI/files.pythonhosted.org for +the exact mypy lock/install and registry.npmjs.org for `npm audit`. No other +host, GitHub workflow, AWS API, image registry push or upload was used. + +```zsh +venv/bin/pip index versions mypy --index-url=https://pypi.org/simple +venv/bin/pip-compile --generate-hashes --allow-unsafe --strip-extras --no-emit-index-url --index-url=https://pypi.org/simple --output-file=requirements.txt requirements.in +venv/bin/python -m pip install --require-hashes -r requirements.txt +PYTHONPATH=src venv/bin/mypy --config-file mypy.ini +PYTHONPATH=src venv/bin/python scripts/generate_sbom.py +PYTHONPATH=src venv/bin/python scripts/check_licenses.py +ruby -e '' .github/workflows/*.yml +PYTHONPATH=src venv/bin/python scripts/validate_ci_workflows.py +PYTHONPATH=src venv/bin/python scripts/validate_terraform_static.py +PYTHONPATH=src venv/bin/python -m pytest -q '' +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_ci_performance_smoke.py +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder PYTHONPATH=src venv/bin/python -m migration.release_gate +cd frontend && npm run lint && npm test && npm run build +cd frontend && npm run test:e2e +PYTHONPATH=src venv/bin/python -m pytest -q +DOCKER_CONFIG=/private/tmp/docker-config-no-creds DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock docker build --pull=false --target backend-runtime --tag foundation-intelligence-backend:phase12 . +DOCKER_CONFIG=/private/tmp/docker-config-no-creds DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock scripts/verify_container_image.sh foundation-intelligence-backend:phase12 +DOCKER_CONFIG=/private/tmp/docker-config-no-creds DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock docker run --rm --network none --entrypoint python foundation-intelligence-backend:phase12 -c '' +cd frontend && npm audit --audit-level=high --ignore-scripts --registry=https://registry.npmjs.org +``` + +PyPI resolved `mypy==2.3.0` plus `ast-serialize==0.6.0`, `librt==0.13.0`, +`mypy-extensions==1.1.0` and `pathspec==1.1.1`; every hash is in +`requirements.txt` and no existing version changed. SBOM output is 70 Python +and 128 npm components. Licence scanning covers 155 installed components with +zero unknown/forbidden declarations. npm reports zero vulnerabilities. + +The first local Playwright attempt was blocked before tests by sandbox +`listen EPERM` on `127.0.0.1:4174`; the approved local-port repeat passes 8 +tests with 4 intentional skips. The first image import one-liner used the wrong +test attribute after successfully importing modules; the corrected isolated +smoke reports 14 governance policies, 21 metrics and eight sources. + +Final normal regression: 342 passing, 13 explicit live skips and eight +subtests. The data-free non-root image is 354,624,742 bytes with ID +`sha256:172dab7c1c7842b0b34f0991d97f8ae34391d36e6ece628db4a63672c36781e9`. + +## Phase 13 — Shadow comparison and cutover preparation + +No AWS API, production traffic, paid API, live news provider, upload or Git +remote was contacted. The final frontend rebuild/install contacted only the +explicitly approved `registry.npmjs.org` through deterministic `npm ci`. + +```zsh +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_shadow_transition.py src/tests/test_transition_golden.py +PYTHONPATH=src venv/bin/mypy --config-file mypy.ini +PYTHONPATH=src venv/bin/python -m flake8 src/transition scripts/verify_transition.py scripts/verify_local_rollback.py --count --select=E9,F63,F7,F82 --show-source --statistics +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python scripts/verify_transition.py --write-golden +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python scripts/verify_local_rollback.py +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder POSTGRES_HOST_PORT=55432 DOCKER_CONFIG=/private/tmp/docker-config-no-creds DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock scripts/verify_local_restore.sh +shasum -a 256 src/data/charities.db config/golden/transition-domain.json +find docs/audits -type f -print0 | sort -z | xargs -0 shasum -a 256 | shasum -a 256 +``` + +The first two transition-query executions failed on the date projection because +`asyncpg` required `date` objects rather than ISO strings. Explicit conversion +was added. The next full execution exposed only exact-representation differences: +SQLite integer minor units versus integral PostgreSQL `NUMERIC`; exact integral +normalization removed the type-only difference without tolerance or rounding. +The expanded final comparison passes 18 projections with zero differences. + +The first rollback execution stopped before activation because the migration +module still hard-coded schema `0004_versioned_analytics` while Alembic was at +`0006_governance_retention`. Migration and release gates now read the one +versioned expected schema contract. The repeat switches from +`sqlite-v7-8fc0cce61c81-r2` to `sqlite-v7-8fc0cce61c81` and back, with equal +counts and active materialization. + +The full logical restore produced a 247,509,368-byte custom archive (SHA-256 +`2c571954768ba4379f3e61160fb808cbc0bd35e6e13ec2f0b4d776c760ceae87`), +restored all 5.45 GB of database state into an isolated database, and matched +schema, active dataset, charity/registry/grant counts, eligible EUR minor total +and materialization state. The exact temporary database and archive were then +removed and their absence verified. + +Final local validation added: + +```zsh +docker build --pull=false --target backend-runtime --tag foundation-intelligence-backend:phase13 --tag foundation-intelligence-backend:local . +scripts/verify_container_image.sh foundation-intelligence-backend:phase13 +docker run --rm --network none --entrypoint python foundation-intelligence-backend:phase13 -c '' +docker-compose up -d --no-build backend frontend +curl --fail --silent --show-error http://127.0.0.1:8501/health/live +curl --fail --silent --show-error http://127.0.0.1:8501/health/ready +docker-compose restart backend +curl --retry 10 --retry-delay 2 --retry-connrefused --fail --silent --show-error http://127.0.0.1:8501/health/ready +PYTHONPATH=src venv/bin/coverage run --source=src/bff,src/migration,src/pipelines,src/transition -m pytest -q +venv/bin/coverage report --skip-empty +RUN_POSTGRES_INTEGRATION=1 DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder DATABASE_STATEMENT_TIMEOUT_MS=120000 PYTHONPATH=src venv/bin/python -m pytest -q '' +docker build --pull=false --target frontend-runtime --tag foundation-intelligence-frontend:phase13 --tag foundation-intelligence-frontend:local . +cd frontend && PHASE7_BASE_URL=http://127.0.0.1:8081 npm run test:e2e +cd frontend && npm ci --ignore-scripts --no-audit --no-fund --registry=https://registry.npmjs.org +cd frontend && npm run lint && npm test -- --run && npm run build +``` + +Normal regression passes 352 tests, 13 skips and eight subtests with measured +61% coverage across the selected BFF/migration/pipeline/transition source. The +combined PostgreSQL run passes 55. Frontend passes 13 unit tests and its bundle +budgets. The first E2E run against the pre-existing stale frontend container +failed eight tests. The current pinned frontend image was rebuilt, the container +recreated, and the repeat passes eight tests with four intentional skips. + +The frontend image build and final local install used the approved npm registry +through `npm ci`; 80 packages were installed locally on the final run. No other +host was contacted. Backend image `sha256:101071f6c750...338ee` is 354,658,326 +bytes/non-root; frontend image `sha256:8f665856111e...9ad1a` is 56,241,904 +bytes/non-root. Stack start, liveness, readiness and backend restart pass. + +## Final acceptance — exact CI coverage replay + +```zsh +PYTHONPATH=src venv/bin/pytest -q --cov=src --cov-report=term --cov-fail-under=70 +``` + +Result: exit 2 during collection. Four tests importing `scripts.*` failed with +`ModuleNotFoundError: No module named 'scripts'`. This reproduced the literal +workflow invocation and identified that the installed `pytest` entry point did +not retain the repository root when `PYTHONPATH=src` was set. + +The backend and PostgreSQL workflow steps were changed from bare `pytest` to +`python -m pytest`. The offline workflow validator now rejects a bare pytest +entry point. The corrected exact CI coverage gate was then run: + +```zsh +PYTHONPATH=src venv/bin/python -m pytest -q --cov=src --cov-report=term --cov-fail-under=70 +``` + +Result: exit 0; 352 passed, 13 skipped, eight subtests passed, 53 warnings and +72.73% total coverage across `src`, above the required 70% floor. No network, +AWS, database mutation, upload or push occurred. + +## Final acceptance — native Terraform formatting + +The approved Docker Hub-only pull resolved +`hashicorp/terraform:1.9.8` to manifest digest +`sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd` +and local image ID +`sha256:97aaea908f872c3c60b75e9bffd6eeae34386c0e9671d6b2a1e30418ea702269`. + +```zsh +docker run --rm --network none -v ':/workspace:ro' -w /workspace hashicorp/terraform:1.9.8 fmt -check -recursive infra/terraform +``` + +The first run failed on `edge.tf`: Terraform rejects nested `allow`, `none` +and `block` blocks written inside a single-line parent block. After expanding +those blocks, the parser listed 15 additional files requiring canonical +formatting. The pinned CLI formatted those files with no network, and the final +identical `fmt -check -recursive` exits 0. + +In an exact temporary copy, still with `--network none`, +`terraform init -backend=false -input=false` installed the local platform +module and then exited nonzero because `registry.terraform.io/hashicorp/aws` +could not be reached. `terraform validate` then reported `Missing required +provider`. The exact temporary directory was removed. No provider, AWS API, +state, plan, apply, destroy, upload or Git remote was accessed. + +The final evidence review corrected one report-only field: both environment +roots declare an intentionally coordinate-free S3 backend configuration point, +so `remote_backend_declared` is `true` while +`remote_backend_coordinates_committed` is `false`. No backend was initialized +and no state was read or changed. diff --git a/docs/remediation/aws-postgres-decisions.md b/docs/remediation/aws-postgres-decisions.md new file mode 100644 index 0000000..158d56b --- /dev/null +++ b/docs/remediation/aws-postgres-decisions.md @@ -0,0 +1,193 @@ +# AWS/PostgreSQL Architecture Decisions + +Created: 2026-07-28 + +Status: active architecture contract through Phase 2; implementation evidence is recorded separately. + +## ADR-001 — PostgreSQL is the operational database + +- Decision: PostgreSQL 16-compatible is the only normal staging/production runtime database. +- Application access: SQLAlchemy 2.x async sessions with `asyncpg`, bounded pools and explicit transactions. +- SQLite allowance: read-only migration/reconciliation source and explicit legacy fixtures only. +- Consequence: production configuration fails closed without PostgreSQL; runtime architectural tests prohibit `sqlite3` imports outside migration/legacy tooling. + +## ADR-002 — ECS Fargate for API and batch workloads + +- Decision: run the FastAPI API as an ECS Fargate service and long-running ingestion/enrichment jobs as one-shot Fargate tasks. +- Rationale: current query/job duration, subprocess behavior and source dependencies are poor Lambda fits. +- Serverless-container exception: managed orchestration/storage is preferred, while long-lived or resource-heavy compute uses containers. + +## ADR-003 — Separate serving, raw and analytical storage + +- Decision: PostgreSQL serves transactional and interactive reads/writes. S3 stores immutable raw/validated/curated versions. Parquet/Glue/Athena supports historical reconciliation and broad analytical scans. +- Constraint: Athena does not serve interactive profiles, admin/job state or transactional overrides. + +## ADR-004 — Preserve entity roles + +- Decision: source funders, grant recipients, curated profiles, registry organisations and registry constituent records keep explicit keys and relationship tables. +- Consequence: migration cannot collapse them into a generic organisation identity or delete apparent duplicates without a separate governed correction. + +## ADR-005 — Versioned materialisations + +- Decision: overview, map/country facts, funder/recipient rankings, programme allocation, filter vocabularies and monthly/yearly trends are dataset-versioned materialisations. +- Activation: build and reconcile a candidate dataset, then transactionally activate one version. A failed build cannot replace the last approved version. +- Cache keys include the active dataset/materialisation version. + +## ADR-006 — OIDC and default-deny RBAC + +- Decision: OIDC/Cognito-compatible JWT validation with viewer, analyst, operator and administrator roles. +- Public surface: explicitly classified read-only routes only. +- Control plane: admin/mutation/proxy/external-cost routes require roles and immutable audit events. +- Development bypass: explicit, local-only and prohibited in staging/production. +- Missing production auth configuration causes startup/readiness failure. +- Implementation: asymmetric OIDC algorithms, issuer, audience, expiry, subject and JWKS key ID are verified. No application shared password or symmetric production signing secret remains. +- Browser contract: production supplies a bearer token through the selected OIDC client/gateway pattern; only explicit local development may use the HttpOnly session cookie. + +## ADR-007 — Durable jobs and source ingestion state + +- Decision: PostgreSQL stores job/run/dataset state. SQS/DLQ supplies durable work delivery; Step Functions/EventBridge definitions orchestrate/schedule; Fargate workers execute. +- API manual refresh enqueues an idempotent job and returns a job ID; it never runs a large subprocess in the request lifecycle. +- Local filesystem locks/status/logs are not a production coordination mechanism. + +## ADR-008 — Migration activation and rollback + +- Decision: read SQLite in immutable/read-only mode, load versioned PostgreSQL staging structures efficiently, reconcile mandatory controls, and activate transactionally. +- Rollback: preserve the prior approved PostgreSQL dataset and coherent SQLite baseline during the validation window. Do not attempt bidirectional replay without an approved conflict policy. + +## ADR-009 — Retention defaults + +- Decision: destructive retention is disabled. Initial operation is dry-run/report-only with holds overriding every deletion candidate. +- Production activation requires approved durations, owners, legal/privacy review, restore verification and an audit manifest. + +## ADR-010 — Terraform and cloud execution boundary + +- Decision: Terraform is the single AWS IaC system for reusable development/staging definitions. +- Local scope: formatting, static/security validation and non-destructive plan when possible. +- Absolute boundary: no apply/destroy/import/remote-state mutation/AWS change/paid call without later explicit user approval. + +## ADR-011 — Frontend delivery and behavior + +- Decision: production frontend is a Vite static build for a private S3 origin behind CloudFront/WAF; no Vite dev server in production. +- Preserve the current visual identity and map-first structure while fixing responsiveness, accessibility, loading isolation, request duplication, stable keys and bundle size. + +## ADR-012 — Layered abuse controls and idempotency + +- Decision: application request IDs, payload/time bounds, per-actor rate limits and at-most-once mutation keys are mandatory even when edge controls are present. +- Local transition: process-local stores are permitted only while the application runs as one local task during remediation. +- Production target: WAF/API edge limits provide distributed abuse protection; PostgreSQL supplies durable idempotency records and security audit events before ECS scales past one task. +- Retry safety: a server failure/timeout retains the idempotency reservation because the side-effect outcome may be uncertain; validation/client failures release it for a corrected request. + +## ADR-013 — Reproducible dependency and image supply chain + +- Decision: direct Python inputs are exact pins; generated transitive locks carry accepted SHA-256 hashes and every installation/build uses `--require-hashes`. +- Decision: npm installs use the committed lockfile, `npm ci` and disabled lifecycle scripts. Build-stage development tools never enter the static Nginx runtime. +- Decision: the Dockerfile frontend and every base image use a concrete version plus a Docker Hub manifest-list digest. +- Network boundary: dependency resolution is limited to PyPI/files.pythonhosted.org, registry.npmjs.org and the Docker Hub registry/auth endpoints explicitly authorized on 2026-07-29. +- Multiarch: `linux/amd64` and `linux/arm64` are declared in one bake contract; CI must assemble/test both because the current local CLI has no working buildx plugin. + +## ADR-014 — Dataset-scoped relational schema and PostgreSQL search + +- Decision: serving rows use composite dataset/source keys so candidates and the prior approved dataset coexist; one partial unique index controls active selection. +- Decision: external grant, registry, source and dataset identifiers stay textual where their source owns identity. Internal UUIDs are limited to runs/events/actions/jobs. +- Decision: normal staging/production import selects PostgreSQL-only async repositories. Unported routes are absent until Phase 5 rather than using a hidden SQLite fallback. +- Search: stored `tsvector` plus GIN and `pg_trgm` indexes; rounded rank descending and registry ID ascending form the stable cursor order. +- Integrity: PostgreSQL FKs/checks are server-enforced, audit rows are append-only and override revisions must increment exactly once per update. + +## ADR-015 — Lossless temporal migration and atomic activation + +- Decision: migration preserves the source representation when it carries more precision than a normalized analytical field. Raw grant award values remain ISO date/timestamp text and ECB reference periods remain `YYYY-MM`; normalized fact dates stay typed separately. +- Decision: global exchange rates and operator overrides are staged in memory and written only inside the candidate activation transaction. A rejected, interrupted or quarantined candidate cannot change global operational state. +- Decision: every full migration report includes a catalog-driven anti-join result across all declared PostgreSQL foreign keys, in addition to relying on validated server constraints. +- Consequence: apparent duplicate-count drift caused by truncating source timestamps is a schema defect, not an approved data correction. The schema must preserve the original business key before reconciliation can pass. + +## ADR-016 — Domain-sized async repositories and queue-only control plane + +- Decision: production/staging organization, registry, grant analytics, source-funder, job and audit journeys use separate async PostgreSQL repositories behind small protocol interfaces. A single cross-domain repository is prohibited. +- Decision: the 25 pre-existing organization/grant method-and-path contracts remain stable while their production implementation changes. Route parity is an executable contract. +- Decision: application mutations own explicit transaction boundaries. Source-funder override revisions are locked and incremented in the same transaction that invalidates profile cache state. +- Decision: manual refresh and enrichment HTTP requests only create idempotent `job_runs` plus structured `job_events`; they never start a scraper or local subprocess. Job execution and delivery semantics belong to Phase 8. +- Decision: production security audit events are awaited and persisted to the append-only PostgreSQL table with actual HTTP status. Development/test may use structured-log or memory sinks without creating a production fallback. +- Consequence: production startup selects PostgreSQL application/admin routers before legacy modules can import and fails when PostgreSQL connection settings are incomplete. Legacy SQLite modules remain test/migration compatibility code, not a production route. + +## ADR-017 — Dataset-versioned serving aggregates and lazy relationships + +- Decision: the unfiltered dashboard reads transactionally refreshed, dataset-versioned scope, country, period, programme, entity and country-funder aggregates. Filtered requests continue to query versioned facts so an aggregate is never silently applied outside its declared dimensions. +- Decision: country-to-country and funder-to-recipient relationships are independent secondary journeys. Country connections are capped at 250 rows; each funder stores only its top 50 recipients and returns at most 25 in detail. The primary dashboard never waits for either relationship graph. +- Decision: the small application cache is TTL- and size-bounded, uses one in-flight loader per key, returns defensive copies and prefixes every key with the active dataset version. Dataset activation therefore invalidates by identity without mutable global flush coordination. +- Decision: candidate migration builds all aggregates and the materialization control row in the same transaction that activates the candidate. Rollback refuses an unapproved target and creates a missing target materialization before changing active status. +- Performance contract: local gates measure repository and authenticated production-mode API p50/p95/p99, cold dashboard p95, concurrent throughput/error count, cache hit ratio, pool recovery, timeout/cancellation and index-backed `EXPLAIN ANALYZE` plans. Performance claims are evidence, not substitutes for later production telemetry. +- Consequence: PostgreSQL remains the source of truth; the cache is disposable and no response depends on filesystem state or an external cache service. + +## ADR-018 — Transactional job outbox and PostgreSQL worker leases + +- Status: accepted and implemented locally in Phase 8. +- Decision: every API-triggered job, initial event and queue envelope are committed atomically in PostgreSQL. Workers claim with row locks, maintain bounded leases/heartbeats and persist retry/dead-letter transitions. Staging/production request idempotency also uses PostgreSQL. +- Reason: horizontally scaled API and worker tasks cannot coordinate reliably through filesystem locks or process-local dictionaries, and direct queue publication cannot be atomic with a database write. +- Consequences: a separate dispatcher publishes the durable outbox envelope to SQS by job-ID deduplication. The API never starts pipeline subprocesses. Actual SQS/DLQ execution remains deployment-gated and unclaimed. + +## ADR-019 — Immutable source objects and fail-closed schedules + +- Status: accepted and implemented locally in Phase 8. +- Decision: raw/validated/curated/export objects have versioned descriptors and checksums; raw descriptors and ingestion manifests are immutable. Source schedules are configuration-as-code and cannot be enabled while legal/licence status is unresolved or governance-blocked. +- Reason: reproducible ingestion requires byte identity, watermarks and counts, while unknown source rights must never be converted into implicit approval. +- Consequences: corrections create new objects/manifests. All current schedules remain disabled until owners approve governance. S3 is the production adapter, but no AWS object operation is part of this decision's local evidence. + +## ADR-020 — Non-destructive, hold-aware retention baseline + +- Status: accepted and implemented locally in Phase 9. +- Decision: retention policies are configuration-as-code by classification; archive windows create reports only. Destructive activation is globally false, delete windows are unset and there is no delete endpoint or worker. +- Decision: legal and incident holds override every candidate. Any future deletion must additionally reference successful immutable restore evidence, administrator approval and an approved policy revision. +- Reason: owner, legal, licence, privacy, backup-retention, RTO and RPO decisions remain unresolved; silently choosing a 12-month delete policy would be unsafe and unauthorised. +- Consequences: initial PostgreSQL constraints accept only dry-run report/archive manifests. Enabling deletion requires an explicit reviewed migration/configuration change plus production approval. + +## ADR-021 — Explicit exposure and recursive redaction + +- Status: accepted and implemented locally in Phase 9. +- Decision: typed response models or named allowlists define every exposed field; unknown policies fail. Generic job/event/source dictionaries never serialize all columns. +- Decision: structured values are recursively redacted before output, including credentials, connection strings, email/postal fields, raw payloads and article bodies. Plain text receives credential and email pattern redaction. +- Consequences: new generic serializers require an exposure-policy update and tests. Raw operational evidence stays internal even when its parent record is readable. + +## ADR-022 — Independent readiness and versioned observability contract + +- Status: accepted and implemented locally in Phase 10. +- Decision: liveness never waits for PostgreSQL or analytics. Readiness uses an independent `NullPool` PostgreSQL engine and one bounded query for schema, active dataset, synchronized governance configuration and durable queue state. +- Decision: structured application/worker logs are redacted JSON with route templates and pseudonymous actor IDs. Metric/alarm definitions and runbook links are versioned configuration; local process snapshots are bounded and administrator-only. +- Reason: exhausted analytical connections must not hide process viability or prevent an orchestrator from making a readiness decision, and high-cardinality or sensitive data must not enter operational telemetry. +- Consequences: Phase 11 may map the definitions to CloudWatch, but no deployed metric, alarm, dashboard or notification is claimed until AWS execution is separately authorized and evidenced. Threshold and owner approval remains an explicit production blocker. + +## ADR-023 — Isolated Terraform environments and fail-closed cloud controls + +- Status: accepted and implemented as unexecuted definitions in Phase 11. +- Decision: dev and staging instantiate one platform module with separate CIDRs, size/availability/cost settings and no shared state declaration. Development chooses one NAT; staging chooses one per AZ plus interface endpoints. Neither environment enables schedules or DNS by default. +- Decision: application data uses private bucket-owner-enforced KMS buckets without destructive lifecycle expiration. PostgreSQL is private, encrypted, PITR-backed, deletion-protected and protected from Terraform destroy. ECS uses private IPs, digest-only images, non-root/read-only containers and RDS-managed runtime credentials. +- Decision: GitHub trust is OIDC-only and pins repository environment subject plus audience. Artefact deployment rights are distinct from infrastructure bootstrap/state ownership; AWS credentials are never Terraform variables. +- Reason: environment isolation and deny-by-default switches preserve deployability without fabricating DNS, owner, secret, legal/source schedule or cloud execution approval. +- Consequences: the absent Terraform/provider/scanner toolchain blocks fmt/validate/plan evidence but not independent CI/CD definition work. Apply, state, DNS and AWS execution remain prohibited. + +## ADR-024 — Required PR gate and two-stage protected staging promotion + +- Status: accepted and implemented as unexecuted workflow definitions in Phase 12. +- Decision: one aggregate PR gate requires backend/frontend quality, security/dependency/licence/SAST, SBOM/container controls, locked Terraform validation and empty-PostgreSQL migration/integration/golden/performance evidence. Missing Terraform provider locks fail closed. +- Decision: staging separates immutable image publication/plan into protected `staging-publish` and exact-plan apply/deployment into independently protected `staging`. Both use exact-environment GitHub OIDC. Serving task definitions change only after private Alembic succeeds. +- Decision: a private release task checks schema, one active dataset, reconciliation, materialization, active quality blocks and DLQ. Prior task definitions are retained as rollback evidence. Production workflow remains hard-disabled. +- Reason: plan review must occur after digest resolution but before infrastructure/service mutation, and schema/data gates must precede traffic promotion. +- Consequences: GitHub branch/environment settings, action SHA pins, OIDC trust and staging execution remain external blockers. Workflow presence is not evidence of deployment success. + +## ADR-025 — PostgreSQL-authoritative transition and one-way shadow evidence + +- Status: accepted and implemented locally in Phase 13. +- Decision: `postgresql` is the default operational mode. `sqlite_migration_source` is local/test compatibility and is rejected in staging/production. `shadow_compare` returns only PostgreSQL results and requires a separate coherent SQLite snapshot. +- Decision: shadow reads are submitted only after the primary response is sent, with queue, byte, timeout and evidence bounds. Failures, timeouts and queue pressure are recorded but never trigger fallback or affect user latency. +- Decision: comparison records contain paths and fingerprints rather than raw profile values. Ordering may be ignored only for explicitly versioned non-semantic set paths; rankings and pagination are order-sensitive. +- Decision: cutover is one-way until an approved conflict policy exists. Rollback selects a retained approved PostgreSQL dataset or independently restored backup, retains the failed target and reconciles before writers resume. +- Reason: a hidden dual-read fallback would retain production SQLite coupling and mask PostgreSQL defects. Temporary asynchronous evidence provides transition confidence without changing authority. +- Consequences: the local full restore and dataset rollback satisfy Gate 13, but AWS traffic changes, RDS restore, production shadow observation and production cutover remain unapproved and `NOT TESTED`. + +## Open external decisions + +- OIDC issuer, audiences, role/group claims and identity owner. +- AWS accounts, region, DNS zone/domain and certificate ownership. +- Approved retention periods, legal/licence status and data owners. +- Staging/production RTO, RPO, availability tier and cost ceiling. + +Secure configuration interfaces and deny-by-default behavior will be implemented without fabricating these approvals. diff --git a/docs/remediation/aws-postgres-dependency-locks.md b/docs/remediation/aws-postgres-dependency-locks.md new file mode 100644 index 0000000..4141123 --- /dev/null +++ b/docs/remediation/aws-postgres-dependency-locks.md @@ -0,0 +1,100 @@ +# Dependency and Base-Image Lock Record + +Updated: 2026-07-29 + +## Python + +- Resolver bootstrap: `requirements-locking.txt`, including hashes for every + artifact selected by the bootstrap lock. +- Production input: `requirements-runtime.in`. +- Production transitive hash lock: `requirements-runtime.txt`. +- Development/test input: `requirements.in`. +- Development/test transitive hash lock: `requirements.txt`. +- Resolver: Python 3.12, `pip-tools==7.5.2`, `pip==25.3`. +- Install policy: `pip install --require-hashes`; the container wheel stage also + uses `--require-hashes`, `--no-deps` and `--no-build-isolation`. +- Index boundary: `https://pypi.org/simple` and artifacts from + `files.pythonhosted.org` only. + +The generated lockfiles are the authoritative inventory of every resolved +Python version and accepted artifact hash. `pip 26.1.2` was rejected after a +local resolver attempt because it is incompatible with `pip-tools 7.5.2`; +the bootstrap lock pins the compatible `pip 25.3` release. + +### Phase-12 type-check additions + +The approved PyPI-only resolution added `mypy==2.3.0` to the development +input. `pip-compile` retained every existing version and added only mypy plus +its locked transitive packages: `ast-serialize==0.6.0`, `librt==0.13.0`, +`mypy-extensions==1.1.0` and `pathspec==1.1.1`. Every accepted platform wheel +and source archive SHA-256 is recorded in `requirements.txt`; local +installation used `pip install --require-hashes`. No runtime/container lock +changed, because the type checker is not a production dependency. + +## npm + +- Lock: `frontend/package-lock.json`, lockfile version 3. +- Install command: `npm ci --ignore-scripts`. +- Registry boundary: `https://registry.npmjs.org` only. +- Lifecycle scripts are disabled during both the verified local install and + the Docker frontend build. + +The package lock is the authoritative inventory of all resolved npm versions, +tarball URLs and integrity hashes. + +### Phase-7 browser-test additions + +The user approved one bounded npm-registry resolution and install for the named +browser/a11y gate. Browser binaries were not downloaded; Playwright uses the +already installed local Google Chrome. `npm install --package-lock-only` created +the reviewed lock delta, followed by `npm ci --ignore-scripts` with +`PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1`. + +| Package | Version | Integrity | +|---|---:|---| +| `@playwright/test` | `1.62.0` | `sha512-9zOJ6ZQRAena31MpOH9VSzIz8Ou3YJ/wtY/eQm5T2uhfhG7/U3COrMS8xOtUrZrp9OgdmzEnIYODye3nY1VqzA==` | +| `playwright` | `1.62.0` | `sha512-Z14dG305dgaLu6foB1TXQagFiW8JfSUIUaUuPaKQ6NtBPKF1P/qXcqfh6c6K/icPqdy37JmjbiBXf6JNg6Sylw==` | +| `playwright-core` | `1.62.0` | `sha512-nsNRyq0r2zsG8AcRHWknc9QRA5XCueC7gWMrs+Gx2tlZn9hcl8zudfh00lhJPY1DE7NmZ6bDsT9g2yey8mXljA==` | +| `@axe-core/playwright` | `4.12.1` | `sha512-rMd7xriptqKpP+w5265i4Hdkv2X5kbu6uiBi/B2I7uf3hieRBM3qDCfaKPtxfiYb2mKXfF+yLODJwIx+Jv1GDw==` | +| `axe-core` | `4.12.1` | `sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==` | +| optional macOS `fsevents` | `2.3.2` | `sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==` | + +No pre-existing package entry changed version. All tarballs resolve under +`https://registry.npmjs.org`; the full transitive inventory remains in +`frontend/package-lock.json`. + +## OCI base images + +The following versioned multi-platform manifest digests were returned by +Docker Hub and are pinned directly in `Dockerfile` or `docker-compose.yml`: + +| Purpose | Tag | Manifest digest | +|---|---|---| +| Dockerfile frontend | `docker/dockerfile:1.8` | `sha256:e87caa74dcb7d46cd820352bfea12591f3dba3ddc4285e19c7dcd13359f7cefd` | +| Backend build/runtime | `python:3.12.13-slim-bookworm` | `sha256:d50fb7611f86d04a3b0471b46d7557818d88983fc3136726336b2a4c657aa30b` | +| Frontend build | `node:22.22.2-alpine` | `sha256:8ea2348b068a9544dae7317b4f3aafcdc032df1647bb7d768a05a5cad1a7683f` | +| Frontend runtime | `nginxinc/nginx-unprivileged:1.30.4-alpine3.24` | `sha256:44e36330f74d4f3a1d4e222acca9e23b401fb87811a7597024502bb759c4dd49` | +| Local PostgreSQL | `postgres:16.14-alpine3.24` | `sha256:57c72fd2a128e416c7fcc499958864df5301e940bca0a56f58fddf30ffc07777` | + +Local pull verification selected the `linux/arm64` variants. The pinned values +are manifest-list digests; `docker-bake.hcl` declares both `linux/amd64` and +`linux/arm64` targets. + +### Local Terraform validation tool + +The approved Docker Hub pull of `hashicorp/terraform:1.9.8` resolved manifest +digest `sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd` +and local arm64 image ID +`sha256:97aaea908f872c3c60b75e9bffd6eeae34386c0e9671d6b2a1e30418ea702269`. +It was run with `--network none`; no Terraform provider was downloaded. + +## Phase-4 rebuild record + +The Phase-4 backend rebuild reused the unchanged hashed Python locks and the +digest-pinned Python base above. The legacy builder downloaded only the exact +locked artifacts from PyPI/files.pythonhosted.org; no additional dependency +version was resolved. No base-image pull was requested. The resulting local +arm64 image is +`sha256:e43491e5e7080e0923b9d777aa1f985bfd3c4897482d662d0be7bf7364758b91` +at 354,209,929 bytes. It has no repository digest because it is an unpushed +local build; this is not represented as a registry artifact. diff --git a/docs/remediation/aws-postgres-progress.md b/docs/remediation/aws-postgres-progress.md new file mode 100644 index 0000000..d4cd3e9 --- /dev/null +++ b/docs/remediation/aws-postgres-progress.md @@ -0,0 +1,238 @@ +# AWS/PostgreSQL Remediation Progress + +This file is the durable continuation ledger. Read it before resuming interrupted work. + +## Overall status + +- Target branch: `91-clean-up-code-for-aws-integration` +- Starting commit: `408eb879b05ec4d2caf92d9bbd782dda9b290e23` +- Current phase: Phase 13 — shadow comparison and cutover preparation (next) +- Overall production status: `NO-GO` +- AWS mutations performed: none +- Paid external calls performed: none +- Push performed: none + +## Phase ledger + +### Phase 0 — Baseline and architecture contract + +- Status: `COMPLETED` +- Files changed: baseline, progress, decisions, command log, architecture contract and migration-manifest JSON Schema under `docs/remediation/`; immutable `docs/audits/` remains byte-for-byte unchanged. +- Commands executed: initial Git state; immutable audit verification; source/backup SQLite validation and control SQL; repository/architecture inventory; backend/frontend tests/build; clean runtime start and HTTP checks; Docker build and storage inspection. +- Tests executed: Python compile; blocking Flake8; 286 backend tests with 76.29% coverage; `npm ci`; 8 frontend tests; frontend lint/build; SQLite quick/integrity/FK/control/anomaly/distribution queries; clean runtime health/auth/OpenAPI; cached API timing; Docker build attempt. +- Tests not executed: true empty-cache/concurrent runtime timing and successful current Docker image start/inspection are intentionally assigned to later performance and container gates. +- Known failures: frontend has five hook warnings and a 1.96 MB main chunk; current Docker build failed at `COPY src/` with `no space left on device` after an 8.81 GB context transfer. +- Technical blockers: Docker storage pressure for the legacy broad-copy build. This does not block creation of the corrected data-free image; safe cleanup/rebuild will be handled in Phase 2. +- External blockers: none established. +- Gate evidence: the active database checksum and 16 audit checksums were unchanged at the final check; the migration-manifest Schema parses as JSON; architecture boundaries, ownership, failure handling and rollback are documented. +- Baseline commit: `19e84ba11dd3567fc871b3411166ae59a5b6eef0` (`Document immutable AWS readiness baseline`). +- Architecture commit: the scoped commit containing this completed ledger, `aws-postgres-architecture.md` and the manifest Schema. +- Remaining work: none for Gate 0. +- Next exact action: classify every route and implement fail-closed OIDC/RBAC, local-development restrictions, proxy allowlists, request IDs, mutation idempotency and security audit evidence. + +### Phase 1 — Security hardening + +- Status: `COMPLETED` +- Files changed: typed/fail-closed security configuration; OIDC JWT/JWKS validation; hierarchical RBAC; request safety and audit middleware; local-only development authentication; restricted proxy; route-specific role/idempotency dependencies; log/admin-output redaction; frontend credential removal and mutation headers; route inventory; security tests. +- Gate evidence: all routes are classified in `aws-postgres-route-inventory.md`; no protected mutation/admin route is anonymous; proxy is disabled by default and requires administrator plus destination/path/method/header allowlists; browser Authorization/Cookie headers are never forwarded; OIDC signature/issuer/audience/expiry/role behavior is locally tested. +- Abuse controls: generated/validated request IDs, bounded request bodies, request/proxy timeouts, per-actor sliding-window rate limiting, required at-most-once keys on side-effecting routes, fixed CORS origins and structured audit events with the required fields. +- Tests executed: compile and blocking Flake8; 297 backend tests plus 8 mutation-route subtests with 76.69% coverage; 11 dedicated security test methods; 8 frontend tests; frontend lint/build; tracked-source secret scan; default-runtime HTTP checks; source/audit immutability checks. +- Known non-blocking warnings: 53 backend dependency/test-client deprecation warnings; five existing React hook warnings; 1.96 MB frontend main chunk. The browser warnings/chunk are assigned to Phase 7. +- Transitional limits: the current single-process rate limiter and idempotency store are deterministic local controls. WAF/API edge rate limiting and PostgreSQL durable idempotency/audit/job records are mandatory before horizontally scaled ECS production. The runtime audit sink is append-only structured output for the managed log pipeline; durable relational audit storage follows the PostgreSQL schema phase. +- External decisions: actual OIDC issuer/audience/role claim ownership remains unresolved. Production fails closed until those values are supplied; no identity provider or paid service was contacted. +- Gate result: `PASS`. +- Commit hash: the scoped Phase-1 commit containing this completed ledger. +- Next exact action: create a strict data-excluding `.dockerignore`, multi-stage non-root image and local PostgreSQL Compose foundation before attempting a bounded image build. + +### Phase 2 — Docker and local PostgreSQL foundation + +- Status: `COMPLETED` +- Files changed: strict `.dockerignore`; digest-pinned multi-stage backend/frontend Dockerfile; hash-locked Python inputs/outputs; same-origin static Nginx frontend; health-based Compose services for PostgreSQL/backend/frontend/migration/worker; database readiness manager; image contract script; multiarch bake declaration; dependency/digest record. +- Supply-chain evidence: Python installs/builds require exact hashes; npm uses lockfile version 3, `npm ci`, the approved registry and disabled lifecycle scripts; Dockerfile frontend and four base images are pinned to Docker Hub manifest digests. +- Gate evidence: local PostgreSQL 16.14, backend and frontend all healthy; direct and same-origin readiness report PostgreSQL healthy; frontend HTML loads; controlled shutdown completes; no arbitrary dependency sleeps. +- Container evidence: backend UID/GID `10001:10001`, read-only root, all capabilities dropped, no-new-privileges, explicit tmpfs and healthcheck; no SQLite, `.env`, application credential/key path, domain payload or compiler; image size `354092439` bytes under the 500 MB target. Frontend is a 56.2 MB static Nginx image with equivalent isolation. +- Multiarch evidence: versioned manifest-list digests and `linux/amd64` plus `linux/arm64` bake targets are present. The local Docker CLI lacks a working buildx plugin, so actual dual-platform manifest assembly remains a CI responsibility and is not falsely claimed as locally tested. +- Tests executed: 300 backend tests with 76.57% coverage; Python compile/Flake8/pip check; 8 frontend tests, lint and build; Compose config for default and operations profiles; complete local image builds; image export/contract scans; PostgreSQL SQL query; container/HTTP health, start and stop checks. +- Resolved failures: incompatible pip 26.1.2 resolver bootstrap pinned back to pip 25.3; absent Docker credential helper isolated via temporary empty config; npm build-stage dev omission fixed with `--include=dev`; occupied default PostgreSQL host port fixed through host-port parameters; Nginx health SPA fallback fixed with an explicit prefix proxy. +- Known non-blocking warnings: 53 backend test warnings; five existing React hook warnings; 1.96 MB frontend main chunk. Frontend warnings/chunk remain assigned to Phase 7. +- External actions: approved package/image downloads only. No AWS call, paid API, scraper/live model call, upload or push. +- Gate result: `PASS`. +- Commit hash: the scoped Phase-2 commit containing this completed ledger. +- Next exact action: implement the authoritative PostgreSQL schema as Alembic migrations and validate upgrade/downgrade behavior against the retained local PostgreSQL volume. + +### Phase 3 — PostgreSQL schema + +- Status: `COMPLETED` +- Files changed: Alembic async environment and initial authoritative revision; PostgreSQL schema contract; async session factory; deterministic registry search repository/cursor; PostgreSQL-only production router; runtime import boundary and real schema integration tests; container now carries Alembic files. +- Schema evidence: 25 application tables cover every required table plus migration/idempotency and normalized charity geography/programme relationships. Catalog evidence reports 30 validated FKs, zero unvalidated FKs and 117 check constraints. +- Search evidence: `pg_trgm`, one stored `tsvector` GIN index and two trigram GIN indexes exist. Real asyncpg tests demonstrate deterministic rank/registry-ID cursor pagination. +- Runtime evidence: staging/production selects the async PostgreSQL router before any legacy module import. A subprocess import hook fails on any `sqlite3` load and passes. Development/test retains the legacy repository only as a transition; unported production routes are absent, not fallback-enabled, until Phase 5. +- Migration evidence: host `upgrade head`; `downgrade base` with only `alembic_version` remaining; second host zero-to-head upgrade; second downgrade; non-root/read-only Compose migration-service zero-to-head upgrade; catalog revision verification. +- Tests executed: 304 passing unit/static tests plus one intentionally skipped live-PostgreSQL test in the normal suite (76.41% coverage); the same PostgreSQL test passes separately with real PostgreSQL, including FK rejection and search. Production-mode local process starts, reports PostgreSQL ready, denies anonymous search with 401 and shuts down cleanly. +- Resolved failure: the first real search test exposed an ambiguous asyncpg type for an optional null status; an explicit PostgreSQL text cast fixed it and all repetitions pass. +- Transitional limit: Phase 5 must port every application journey/domain before production can become a GO. Phase 3 proves that normal production does not import or fall back to SQLite, not that all routes are already available. +- External actions: local PostgreSQL/container operations only. No AWS, paid API, live scraper/model, upload or push. +- Gate result: `PASS` for schema creation, PostgreSQL runtime boundary, FK/search enforcement and zero-to-head migration. +- Commit hash: the scoped Phase-3 commit containing this completed ledger. +- Next exact action: implement deterministic, read-only SQLite-to-versioned-PostgreSQL loading with checksum/schema/integrity preflight, quarantine, reconciliation, activation and rollback. + +### Phase 4 — SQLite to PostgreSQL migration + +- Status: `COMPLETED` +- Files changed: deterministic read-only migration CLI and integration tests; Alembic revisions preserving exchange-rate month and raw grant timestamp precision; atomic global-record staging; migration-manifest Schema extensions; data-free runtime-image inclusion; committed JSON and Markdown reconciliation evidence. +- Source safety: immutable URI open, `PRAGMA query_only`, SHA-256 `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`, schema version `7`, `integrity_check=ok` and capacity preflight all pass. The final manifest records 64,959,250,432 bytes available against 23,340,679,168 required. +- Reconciliation evidence: all 12 source/target table counts match. Mandatory controls pass exactly: 373 charities; 397,469 registry rows; 302,546 grants; 345 registry links; 104,191 mapped grants; 134,554 classified grants; EUR 22,435,986,707.70; zero source-identity duplicate groups; zero FK violations; 432 missing conversions; 2 negative and 2,101 zero grants; 1 future-dated grant; 4,271 business-key duplicate groups; 9,073 duplicate-charity-number groups. +- Fidelity corrections: source ECB periods remain exact `YYYY-MM` strings and raw grant dates retain full ISO timestamp precision. Normalized analytical award dates remain typed `DATE` in fact tables. Alembic revisions `0002_exchange_rate_period` and `0003_grant_award_timestamp` both pass downgrade/upgrade tests. +- Activation safety: candidates load under a dataset version; global exchange-rate/override records remain staged until the same transaction that activates the candidate. Quarantine, retry, conflicting override, failed-candidate isolation and prior-active restoration are exercised against real PostgreSQL. +- Idempotency and rollback: repeating the final full command returns `idempotent_noop=true` with the same run ID. Full rollback from `sqlite-v7-8fc0cce61c81-r2` to `sqlite-v7-8fc0cce61c81` retained 302,546 grants and 397,469 registry rows; switching back to `r2` succeeded. Exactly one dataset is active in the final local state. +- Report evidence: `evidence/phase4-migration-manifest.json` validates against the Draft 2020-12 Schema; `evidence/phase4-migration-report.md` is the human-readable companion. Final run ID is `60af368e-c440-5521-9648-5ab272f9ddb6`; code revision is `d6d2b69d1f9ff7dd8bc6f58021060586b3c17757`. +- Tests executed: 308 passing normal tests, 2 intentional skips, 8 subtests and 53 known warnings; 5/5 real PostgreSQL migration integration tests; manifest Schema validation; 30/30 FKs validated; container image contract and isolated in-image migration/Alembic imports. +- Container evidence: rebuilt backend is UID/GID `10001:10001`, 354,209,929 bytes, healthchecked and free of SQLite/domain data, environment/credential files and compiler tools. Image ID is `sha256:e43491e5e7080e0923b9d777aa1f985bfd3c4897482d662d0be7bf7364758b91`; the pinned Python base manifest digest is unchanged. +- Protected state: active SQLite and the aggregate `docs/audits/` checksum remain byte-for-byte equal to baseline. No AWS, paid/live external API, scraper/model call, upload or push occurred. +- Gate result: `PASS`. +- Local implementation checkpoints: `262b4a8`, `9afbc4a`, `919aa96`, `a74d75c`, `d6d2b69`; the scoped closeout commit contains this ledger and durable evidence. +- Next exact action: port every application journey and repository to async PostgreSQL, remove production SQLite route gaps and prove parity before starting the performance gate. + +### Phase 5 — Application conversion to PostgreSQL + +- Status: `COMPLETED` +- Runtime coverage: all 25 legacy organization/grant route contracts are present in the PostgreSQL router. Organization list/detail/stats/grants/Sankey/score, registry directory/detail, map/overview/filter suggestions/trends/themes/drill-down/summary, source-funder list/detail, link overrides and profile cache now use domain-sized async PostgreSQL repositories. +- Operational state: production/staging admin status, manual triggers, history and logs use `job_runs`/`job_events`. Requests enqueue bounded idempotent work and never start a local scraper/subprocess. Production security audits use the append-only `audit_events` table. +- Runtime boundary: production/staging imports only PostgreSQL application and admin routers; the production startup test proves `sqlite3`, the legacy router and the SQLite repository are absent. Startup fails before serving when PostgreSQL configuration is missing, and readiness passes only after a real database query. +- Transaction evidence: Funder relink/reset/cache, durable jobs and audit writes were exercised through real asyncpg sessions inside an outer transaction and fully rolled back. No test artifacts remain in the active dataset. +- Integration evidence: five Phase-5 tests pass against PostgreSQL 16.14, including all read journeys with response-model validation, mutation/idempotency/audit behavior and production startup/router selection. The normal suite passes 310 tests, skips five explicit live-environment tests, and passes eight route subtests. +- Resolved failures: real PostgreSQL caught and fixed a reserved `grant` alias, an incorrectly typed interval bind, a missing CTE alias and ambiguous joined columns. These failures could not have been detected by mocked repositories alone. +- Protected state: active SQLite and aggregate `docs/audits/` checksums remain exactly at baseline. No dependency download, AWS access, live scraper/model/API call, upload or push occurred. +- Gate result: `PASS`. +- Commit hash: the scoped Phase-5 commit containing this ledger. +- Next exact action: measure cold/warm and concurrent PostgreSQL journeys, capture query plans, then fix query/index/materialization and duplicate-request root causes for Gate 6. + +### Phase 6 — Performance and concurrency + +- Status: `COMPLETED` +- Runtime changes: default map, monthly/yearly trends, programme themes, network rankings, country funders and top-recipient relationships read dataset-versioned PostgreSQL aggregates. Country connections are a separate lazy endpoint capped at 250 rows. Filtered journeys retain bounded fact-table queries instead of returning stale aggregate results. +- Cache and pool controls: the in-process serving cache is dataset-keyed, TTL-bounded, copy-safe and single-flight. Activation/rollback changes the dataset key and stale entries are pruned. The async SQLAlchemy pool remains bounded at size 5 plus overflow 5; statement, pool and connect timeouts remain explicit. Cancellation releases the connection and a subsequent query succeeds. +- Performance evidence: 20 application-cache-cold repository dashboards have p50 `157.56 ms`, p95 `255.31 ms` and p99 `539.80 ms`. Production-mode API p95 is `3.70 ms` health, `245.73 ms` organization list, `4.95 ms` map, `43.45 ms` overview, `6.04 ms` lazy connections, `18.01 ms` exact registry and `83.93 ms` text registry. Five concurrent dashboards complete in `472.80 ms` at `10.575` dashboards/s with zero errors. +- Query-plan evidence: `EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` proves the default map uses `analytics_country_aggregates`, exact registry lookup uses `ix_registry_current_normalized_name`, and ranked text lookup uses a registry GIN/trigram index. The default paths no longer scan `grant_overview_facts` for every request. +- Materialization evidence: Alembic head `0004_versioned_analytics` contains nine aggregate tables and one transactional refresh function. The active `sqlite-v7-8fc0cce61c81-r2` materialization contains 204,220 rows; candidate activation builds it before switching the single active dataset, and rollback builds a missing prior materialization before reactivation. +- Tests executed: 311 passing normal tests, 9 explicit live-environment skips, 8 passing route subtests and 53 known warnings; five dedicated performance/cache/plan/concurrency/timeout tests; five PostgreSQL application tests; combined 15/15 PostgreSQL application/schema/migration tests; production-mode API load test and repository benchmark. All final measured request error rates are zero, the pool has zero checked-out connections after each run, and the final cache hit ratio is `0.8333` at the API layer. +- Resolved failures: the baseline default map, summary, dashboard and registry paths took 2–17 seconds. A first load script hit the application rate limit, an early test invocation omitted the database password, the schema fixture assumed an empty active-dataset state, and one cold timing/plan run exposed p95-versus-single-sample measurement ambiguity. The final harness uses local OIDC credentials, a test-only high request allowance, correct secret-file configuration, state restoration and percentile-based cold measurements. +- Evidence files: `evidence/phase6-performance-report.json` and `evidence/phase6-performance-report.md`. +- Protected state: active SQLite and aggregate `docs/audits/` checksums remain exactly at baseline. No dependency or image download, AWS call, paid/live API, scraper/model call, upload or push occurred. The final data-free backend rebuild succeeded with `--pull=false`; every dependency layer was cached. The local arm64 image is 354,456,439 bytes, runs as `10001:10001` and has ID `sha256:cf71388a8fc83cdc32632ea2cf8ea9b7b27d4d68b164f848cd6e97b49905af8a`. +- Gate result: `PASS`. +- Commit hash: the scoped Phase-6 commit containing this ledger and evidence. +- Next exact action: remove frontend request duplication and hook warnings, isolate loading/error states, implement responsive/accessibility fixes and split the oversized bundle without changing visual identity. + +### Phase 7 — Frontend remediation + +- Status: `COMPLETED` +- Runtime changes: the application shell no longer blocks on the health check; route, map and chart code is lazy; map connections are fetched only after explicit activation; obsolete requests are aborted and sequence-guarded; handled failures stay in panel-level UI rather than writing browser warnings/errors. +- Responsive/accessibility changes: the map remains the first major dashboard section; viewport shells and dense controls are bounded; KPI text wraps without cropping; mobile map controls wrap; filter drawers stay scrollable; skip navigation, active-page state, dialog focus traps, Escape handling and focus restoration cover Overview, Donor and Registry journeys. +- Bundle evidence: initial JavaScript is 87.81 KiB gzip against 120 KiB, initial CSS is 18.59 KiB against 25 KiB and the largest deferred JavaScript chunk is 392.36 KiB against 425 KiB. `npm run build` enforces these budgets. +- Tests executed: 13/13 frontend unit/contract tests; warn-free Oxlint; TypeScript/Vite production build; compressed bundle gate; local Chrome 150 runtime journeys at 320, 390, 768, 1024, 1440 and 1920 pixels. The runtime gate found no page overflow, clipped visible controls, cropped KPIs, unnamed visible controls, duplicate initial overview request, console warning/error or runtime exception. The named Playwright/axe suite passes eight journeys and intentionally skips four redundant secondary-journey viewport combinations. It reports zero axe violations across six Overview widths plus Donor/Registry journeys at 320 and 1024 pixels. +- Accessibility corrections from the named gate: active-state text now meets WCAG AA contrast, the mobile data-source disclosure has an explicit accessible name, the map SVG uses valid ARIA and the Overview heading hierarchy includes its H2 level. +- Supply-chain action: after explicit approval, exact `@playwright/test==1.62.0` and `@axe-core/playwright==4.12.1` development dependencies were resolved and downloaded only from `registry.npmjs.org`. The reviewed lock delta contains only those packages and their exact transitive dependencies. `npm ci --ignore-scripts` ran with browser download disabled; installed Chrome 150 is used locally. +- Protected state: active SQLite and aggregate `docs/audits/` checksums remain exactly at baseline. No AWS call, paid/live API, scraper/model call, browser download, upload or push occurred. +- Gate result: `PASS`. +- Commit hash: the scoped Phase-7 commit containing this ledger and evidence. +- Evidence files: `frontend-bundle-budget.md`, `evidence/phase7-frontend-report.json` and `evidence/phase7-frontend-report.md`. +- Next exact action: wait for explicit authorization before beginning Phase 8; do not perform S3/AWS, deployment, push or any other remote action. + +### Phase 8 — Pipelines, S3 and durable jobs + +- Status: `COMPLETED` +- Durable coordination: production/staging request idempotency is PostgreSQL-backed. Manual triggers transactionally create a job, first event and versioned dispatch-outbox envelope; API requests never launch a scraper or subprocess. Workers claim with `FOR UPDATE SKIP LOCKED`, publish leases/heartbeats and record bounded retry, timeout, success, failure and dead-letter transitions. +- Storage contract: `raw`, `validated`, `curated` and `export` object descriptors retain object version, SHA-256, byte length, content type and source/run ownership. Raw descriptors and canonical ingestion manifests are database-enforced immutable; corrections create new versions. +- Source controls: eight versioned configurations cover 360Giving, Charity Commission, Philea, Hinchilla, ECB, Google News RSS, bounded article content and optional Anthropic summaries. All legal/licence states remain `unresolved`; all schedules are disabled and governance-blocked. Application and database validation prevent activation before approval. +- Last-good preservation: jobs and ingestion runs retain the active dataset at enqueue/start. The retry/failure integration leaves `sqlite-v7-8fc0cce61c81-r2` active and never activates candidate data. +- Migration evidence: transactional `0005 -> 0004 -> 0005` cycle passes. The final catalog reports Alembic `0005_durable_pipeline`, 40 application tables, 49 FKs, 161 checks, eight source configs, zero enabled schedules and eight governance blocks. +- Tests executed: 318 passing normal tests, 10 explicit live-environment skips and 8 passing subtests; 8/8 dedicated Phase-8 PostgreSQL tests; 18/18 combined Phase-8/application/schema PostgreSQL tests; Python compile, blocking Flake8 and diff checks. +- AWS boundary: S3/SQS/DLQ/EventBridge/Step Functions interfaces and state contracts exist, but no AWS execution or deployment occurred and none is claimed. +- Evidence files: `pipeline-storage-contract.md`, `evidence/phase8-durable-pipeline-report.json` and `evidence/phase8-durable-pipeline-report.md`. +- Protected state: active SQLite and aggregate `docs/audits/` checksums remain exactly at baseline. No dependency download, live external call, paid API, upload or push occurred. +- Gate result: `PASS` for local code readiness; real AWS execution remains `NOT TESTED` by design. +- Next exact action: implement Phase-9 governance, explicit field exposure, configurable retention dry-run, holds, export expiry and auditable deletion manifests without enabling destructive deletion. + +### Phase 9 — Governance and retention + +- Status: `COMPLETED` +- Governance configuration: 14 required classifications, source/owner registers, privacy checklist, field allowlists, log-redaction keys, proposed backup/PITR policy and unresolved RTO/RPO are versioned under `config/`. Unknown owners and legal/licence states remain explicitly unresolved. +- Retention safety: automatic/destructive deletion and production activation are disabled; every destructive window is unset. Proposed archive windows generate dry-run reports only. PostgreSQL permits initial deletion manifests only for dry-run report/archive actions, and no destructive HTTP route exists. +- Hold/restore evidence: exact, retention-class and global legal/incident holds override retention. Deletion and restore-verification evidence is append-only; a restore record alone never enables deletion. +- Exposure/redaction: generic admin outputs use named allowlists and exclude job input, actor identity and credential references. Recursive redaction covers credentials, connection strings, email, postal address, raw payload and article content before serialization; plain log text also removes credential/email shapes. +- Workflow: export expiration is report-only; data-subject intake stores a hashed reference and requires identity verification, scoped review, hold/legal review, separate mutation approval and audit evidence. +- Migration/catalog: `0006 -> 0005 -> 0006` passes. Final catalog is revision `0006_governance_retention`, 45 tables, 55 FKs and 189 checks; 14 policies exist with zero destructive flags and zero delete windows. +- Tests executed: 326 normal tests, 11 explicit live skips and 8 subtests; 9/9 dedicated governance PostgreSQL tests; 27/27 combined Phase-9/Phase-8/application/schema tests; compile, blocking Flake8, JSON and diff checks. +- Evidence files: `data-governance-register.md`, `retention-privacy-guide.md`, `evidence/phase9-governance-report.json` and `evidence/phase9-governance-report.md`. +- Protected state/external boundary: SQLite and `docs/audits/` remain at baseline. No AWS, paid/live API, download, upload or push occurred. +- Gate result: `PASS` for the local governance framework; policy/owner/legal/licence/RTO/RPO approvals remain unresolved and production remains `NO-GO`. +- Next exact action: implement structured JSON request/job logging, richer readiness checks, metric/alarm definitions and operational runbooks without claiming deployed CloudWatch resources. + +### Phase 10 — Observability + +- Status: `COMPLETED` +- Health contract: `/health` and `/health/live` are dependency-free; `/health/ready` checks PostgreSQL, exact Alembic head, the approved active dataset, source/retention configuration and durable queue state. A dedicated `NullPool` health engine remains responsive while the bounded application pool is deliberately exhausted. +- Logging contract: application and durable-worker logs are redacted JSON with UTC time, service/environment, request/trace/job/dataset identifiers, pseudonymous actor, role, route-template operation, duration, status, error class, counts and retry state when applicable. Raw exception payloads and sensitive values are excluded. +- Telemetry contract: `config/observability.json` defines 21 metrics and 15 alarms for API, database, cache, freshness, pipelines, reconciliation, gaps/coverage, queue/DLQ, workers/restarts, RDS and cost. The administrator-only local evidence endpoint explicitly marks CloudWatch execution `not_tested`. +- Runbooks: 13 versioned procedures cover readiness, migration, ingestion, stale data, RDS, queue, DLQ replay, bad deployment, rollback, restore, security incidents, credential rotation and cost spikes. +- Tests executed: 331 normal tests, 12 explicit live skips and 8 subtests; 6/6 dedicated Phase-10 PostgreSQL tests; 33/33 combined observability/governance/pipeline/application/schema tests; compile, blocking Flake8, JSON and diff checks. +- Evidence files: `observability-contract.md`, `observability-runbooks.md`, `evidence/phase10-observability-report.json` and `evidence/phase10-observability-report.md`. +- Protected state/external boundary: SQLite and `docs/audits/` remain at baseline. No AWS, CloudWatch, paid/live API, download, upload or push occurred. +- Gate result: `PASS` for local observability readiness. Production alarm owners/destinations, SLO thresholds and cost ceiling remain unresolved; production remains `NO-GO`. +- Next exact action: define complete dev/staging Terraform modules, offline validation contracts and environment plans without provider downloads, AWS reads, apply/destroy/import or state mutation. + +### Phase 11 — Terraform AWS infrastructure definitions + +- Status: `COMPLETED_LOCAL`; native parsing/formatting passes, while + provider-dependent validation remains blocked because the AWS provider and + scanners are outside the authorised/available tool boundary. +- Execution restriction: no apply/destroy/import/state mutation or AWS resource change. +- Definitions: 26 `.tf` files, 103 resource blocks and 58 AWS resource types implement shared platform infrastructure plus isolated dev/staging roots. Coverage includes network tiers/NAT/endpoints/security groups, KMS, five private data buckets, CloudFront/WAF, ECR, ECS API/workers/ALB/autoscaling, private RDS/backups/PITR/managed secret, SQS/DLQ, disabled scheduler, Step Functions, logs/dashboard/15 alarms/budgets, least-privilege task roles, GitHub OIDC/deployment role and fail-closed ACM/DNS points. +- Security: RDS is private, encrypted, deletion-protected and `prevent_destroy`; data buckets block public access, enforce ownership/KMS and define transitions without expiration; ECS requires digest images, non-root/read-only tasks and private IPs; schedules and DNS remain disabled; no long-lived AWS keys exist. +- Runtime integration: ECS injects the RDS-managed password through Secrets Manager; the database manager now accepts this runtime-only secret interface as well as the local secret-file contract, without logging it. +- Documentation: four Mermaid diagrams, security/data/deployment boundaries, exact offline/provider validation procedure and dev/staging cost envelopes (not live price claims). +- Tests executed: offline validator passes 26 files/103 resources/58 types; 2/2 Terraform contract tests; native Terraform 1.9.8 parse/format and final `fmt -check -recursive` pass; 334 normal backend tests, 12 explicit live skips and 8 subtests; compile, blocking Flake8 and diff checks. +- Exact blocker: the approved Docker Hub CLI image is pinned by digest. Network-isolated `init -backend=false` installs the local module but stops because `registry.terraform.io/hashicorp/aws` is outside approved sources; `validate` reports that provider missing. Security scanners are absent, and AWS read/plan access is not approved. Provider lock/validate/security scan/plan remain `BLOCKED` or `NOT TESTED`. +- Evidence files: `terraform-aws-infrastructure.md`, `terraform-validation.md`, `evidence/phase11-terraform-report.json` and `evidence/phase11-terraform-report.md`. +- Gate result: infrastructure code review contract `PASS`; provider-dependent Gate-11 validation remains blocked/`NOT TESTED`, so production remains `NO-GO`. +- Next exact action: implement complete PR and protected staging workflow definitions, generate local backend/frontend SBOM evidence with already installed tools, and validate workflow contracts without executing any deployment. + +### Phase 12 — CI/CD + +- Status: `COMPLETED_LOCAL`; remote GitHub/AWS workflow execution is `NOT TESTED`. +- PR workflow: seven parallel jobs plus one aggregate required gate cover backend compile/blocking lint/mypy/test/coverage, frontend lint/test/build/Playwright/axe, secret/dependency/licence/SAST scans, backend/frontend/container SBOMs, container build/Trivy/non-root/size/no-data, Terraform offline/fmt/locked-init/validate/security and empty-PostgreSQL/Alembic/integration/reconciliation/golden/performance checks. +- Deployment workflow: exact manual confirmation, `staging-publish` approval before ECR mutation/plan, separate `staging` approval before applying the exact plan, private Alembic/reconciliation tasks, digest service update, frontend sync, smoke/load/E2E/security and rollback-readiness gates. OIDC only; no access-key inputs. Production is hard-disabled. +- Supply chain: added hash-locked `mypy==2.3.0` plus four transitives; 13 typed files pass. Deterministic CycloneDX SBOMs contain 70 Python and 128 npm components; 155 installed licence records have zero unknown/forbidden declarations; npm audit reports zero vulnerabilities. +- Runtime image: `--pull=false` build and contract pass at 354,624,742 bytes, `10001:10001`, healthchecked/data-free. Network-isolated imports prove 14 governance policies, 21 metrics, eight sources and release-gate schema `0006_governance_retention` are present. Image ID `sha256:172dab7c1c7842b0b34f0991d97f8ae34391d36e6ece628db4a63672c36781e9`. +- Tests executed: YAML parse and 24-marker workflow validator; mypy 13 files; 10 focused tests; 342 normal backend tests, 13 live skips and 8 subtests; local PostgreSQL performance/release gates; frontend lint, 13 unit tests, build/budget, 8 E2E/axe passes and 4 intentional skips; blocking Flake8/diff checks. +- External/tool blockers: provider locks, Terraform/Python/container/config scanners, branch protection, environment reviewers, OIDC trust, action commit-SHA resolution and actual GitHub workflow execution are unavailable/unapproved. The Terraform PR job deliberately fails closed until locks exist. +- Evidence files: `ci-cd-guide.md`, `evidence/phase12-cicd-report.json`, `evidence/phase12-cicd-report.md` and the two `evidence/sbom/*.cdx.json` files. +- Gate result: workflow definitions and available local checks `PASS`; remote PR/staging execution remains `NOT TESTED`, so production remains `NO-GO`. +- Next exact action: implement asynchronous, non-latency shadow comparison and complete domain golden fixtures, then prove local PostgreSQL restore/rollback and finish cutover/rollback runbooks without AWS execution. + +### Phase 13 — Shadow comparison and cutover preparation + +- Status: `COMPLETED_LOCAL`; AWS/production cutover remains explicitly unexecuted. +- Runtime boundary: PostgreSQL is the default and sole authoritative operational mode. SQLite migration-source mode is rejected in staging/production. Temporary shadow mode requires a separate coherent snapshot and cannot fall back to it. +- Shadow execution: the final PostgreSQL response is passed through before bounded background work starts. The queue is capped at 8, response capture at 2 MiB, timeout at 20 seconds and evidence at 100 fingerprint-only differences. Only three versioned set-like paths may ignore order. +- Domain gate: 21 journeys are registered. Eighteen executed SQLite/PostgreSQL semantic projections cover identities, totals, coverage, date/country/programme/donor/recipient filters, monthly/yearly trends, rankings, registry search/pagination, profiles, grant lists, Sankey, map relationships, score components and currency statuses. Result: zero differences. +- Golden fixture: `config/golden/transition-domain.json`, 110,590 bytes, SHA-256 `f2065b01fe6385357314e6db4bf61628f1755c9aae03be66246c327331639166`, bound to source checksum `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651`. +- Rollback proof: writers/jobs were frozen; active dataset switched from `sqlite-v7-8fc0cce61c81-r2` to approved prior `sqlite-v7-8fc0cce61c81` and back. All eight counts and materialization state matched; no bidirectional replay occurred. +- Restore proof: a full custom-format logical backup (247,509,368 bytes; SHA-256 `2c571954768ba4379f3e61160fb808cbc0bd35e6e13ec2f0b4d776c760ceae87`) was restored into an isolated database. Schema, active dataset, counts, EUR total and materialization matched; archive and temporary database were removed. +- Runbooks: transition mode, cutover, rollback, backup/restore and troubleshooting procedures are complete. They preserve the failed target/source snapshot and require reconciliation before resuming writers. +- Failed attempts recorded: two strict date-binding failures, exact `NUMERIC`/integer representation differences and one stale schema-revision rejection. Each stopped safely and was repeated successfully after a narrow fix. +- Protected state/external boundary: SQLite and `docs/audits/` remain at baseline. The final frontend rebuild/install used only the explicitly approved npm registry through `npm ci`; no other download host, AWS, production traffic, paid/live API, upload or push occurred. +- Gate result: Gate 13 `PASS` locally. AWS cutover, RDS restore and production execution remain `NOT TESTED`; production remains `NO-GO`. +- Evidence files: `runtime-transition-guide.md`, `cutover-runbook.md`, `rollback-runbook.md`, `backup-restore-guide.md`, `evidence/phase13-transition-report.json` and `.md`. +- Final CI replay: the literal workflow coverage command initially failed + during collection because the bare `pytest` entry point combined with + `PYTHONPATH=src` omitted the repository root and made four `scripts.*` + imports unavailable. Workflow test steps now invoke `python -m pytest`, and + the offline validator enforces that boundary. The exact corrected CI command + passes 352 tests, skips 13 explicit live cases, passes eight subtests and + measures 72.73% total `src` coverage against the 70% gate. + +### Push checkpoint + +- Status: `PENDING_EXPLICIT_APPROVAL` +- Rule: stop and request explicit user confirmation before the first push. diff --git a/docs/remediation/aws-postgres-route-inventory.md b/docs/remediation/aws-postgres-route-inventory.md new file mode 100644 index 0000000..af4ad29 --- /dev/null +++ b/docs/remediation/aws-postgres-route-inventory.md @@ -0,0 +1,77 @@ +# Phase-1 API Route and Authorization Inventory + +Status: implemented security contract. Role inheritance is `administrator > operator > analyst > viewer`; a higher role satisfies lower-role reads. All unlisted actions are denied by default. + +## Public and authentication surface + +| Method | Path | Classification | Required identity | Mutation/idempotency | Notes | +|---|---|---|---|---|---| +| GET | `/`, `/docs`, `/redoc`, `/openapi.json`, `/docs/oauth2-redirect` | public read | none | read-only | Documentation/redirect surface; production edge policy may further restrict it. | +| GET | `/health`, `/health/live` | public read | none | read-only | Process liveness only; no database or analytics dependency. | +| GET | `/health/ready` | public read | none | read-only | Non-sensitive PostgreSQL/schema/dataset/configuration/queue readiness states through an independent bounded connection. | +| POST | `/api/auth/login` | development authentication bootstrap | explicit local/test mode and allowed client host | authentication exception | Returns 404 unless `AUTH_MODE=development` and `DEV_AUTH_ENABLED=true`; unavailable in staging/production. | +| POST | `/api/auth/logout` | authenticated action | viewer or higher | session cleanup | Clears only the local-development cookie; production OIDC logout belongs to the identity provider. | + +## Organization and grant surface + +| Method | Path | Classification | Minimum role | Mutation/idempotency | +|---|---|---|---|---| +| GET | `/api/charities` | authenticated read | viewer | read-only | +| GET | `/api/charities/stats` | authenticated read | viewer | read-only | +| GET | `/api/charities/{reg_charity_number}` | authenticated read | viewer | read-only | +| GET | `/api/charities/{reg_charity_number}/grants` | authenticated read | viewer | read-only | +| GET | `/api/charities/{reg_charity_number}/sankey` | authenticated read | viewer | read-only | +| POST | `/api/charities/{reg_charity_number}/score` | authenticated calculation | analyst | pure calculation; no persisted mutation | +| GET | `/api/charities/directory/organizations` | authenticated read | viewer | read-only | +| GET | `/api/charities/directory/organizations/{registry_id}` | authenticated read | viewer | read-only | +| POST | `/api/charities/directory/organizations/enrich` | operator action | operator | required `Idempotency-Key` | +| GET | `/api/charities/grants/beneficiary-geographies` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/map` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/overview` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/overview/entity-suggestions` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/overview/trends` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/overview/drilldown` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/funders` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/funders/{source_funder_key}` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/funders/{source_funder_key}/profile-cache` | authenticated read | viewer | read-only | +| POST | `/api/charities/grants/funders/enrich` | operator action | operator | required `Idempotency-Key` | +| POST | `/api/charities/grants/funders/{source_funder_key}/profile-cache` | operator action | operator | required `Idempotency-Key` | +| POST | `/api/charities/grants/funders/{source_funder_key}/reset-to-observed` | administrator action | administrator | required `Idempotency-Key` | +| POST | `/api/charities/grants/funders/{source_funder_key}/relink` | administrator action | administrator | required `Idempotency-Key` | +| GET | `/api/charities/grants/summary` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/trends` | authenticated read | viewer | read-only | +| GET | `/api/charities/grants/themes` | authenticated read | viewer | read-only | + +## Operational, external-call and proxy surface + +| Method | Path | Classification | Minimum role | Additional controls | +|---|---|---|---|---| +| GET | `/api/admin/pipeline/status` | operator action/read | operator | Reads the latest durable PostgreSQL job state. | +| POST | `/api/admin/pipeline/trigger` | operator action | operator | Required `Idempotency-Key`; bounded mode validation; security audit event. | +| GET | `/api/admin/pipeline/jobs` | operator action/read | operator | Bounded durable PostgreSQL job history. | +| GET | `/api/admin/pipeline/logs` | administrator action/read | administrator | Last 100 structured PostgreSQL job events; output redacted; security audit event. | +| GET | `/api/admin/pipeline/sources` | operator action/read | operator | Reads governance-gated source schedules; credential references are masked. | +| PUT | `/api/admin/pipeline/sources/{source_name}/schedule` | administrator action | administrator | Required `Idempotency-Key`; unresolved legal/licence state blocks enablement. | +| GET | `/api/admin/governance/retention/policies` | governance read | administrator | Explicit proposed policies; destructive activation false. | +| POST | `/api/admin/governance/retention/dry-run` | governance action | administrator | Required `Idempotency-Key`; report/archive evidence only, never deletion. | +| GET/POST | `/api/admin/governance/holds` | governance read/action | administrator | Legal/incident hold list/create; create requires idempotency. | +| POST | `/api/admin/governance/holds/{hold_id}/release` | governance action | administrator | Required `Idempotency-Key`, actor and reason. | +| GET | `/api/admin/governance/exports/expiration-report` | governance read | administrator | Dry-run expiration report; no object mutation. | +| POST | `/api/admin/governance/data-subject-requests` | governance action | administrator | Required `Idempotency-Key`; accepts only a hashed subject reference. | +| GET | `/api/admin/observability/metrics` | observability read | administrator | Bounded process-local metric evidence and versioned definitions; no CloudWatch execution. | +| GET | `/api/news/{foundation_name}/summary` | authenticated resource-intensive read | analyst | Bounded parameters/timeouts; live paid-provider use remains separately approval-gated. | +| GET | `/api/news/{foundation_name}/summary/stream` | authenticated resource-intensive read | analyst | Same restrictions as the JSON endpoint. | +| configured allowlist only | `/api/core/{path}` | proxy action | administrator | Disabled by default; fixed destination host, exact/prefix path allowlist, method allowlist, request/response header allowlists, no browser credential forwarding, no redirects, timeout, request ID and idempotency for enabled mutations. Hidden from OpenAPI. | + +There is no internal-service route in the current application. Queue/task callbacks introduced in later phases must use workload identity and a separate internal-service authorization dependency; they must not reuse user cookies. + +## Phase boundary + +Production/staging application data, job state, request idempotency, link +overrides, profile caches and audit records use PostgreSQL. Manual refresh +routes only enqueue durable jobs and return a job ID. Phase-8 workers claim +through PostgreSQL leases and the transactional outbox supplies the SQS +delivery contract; neither path uses a production local lock or API +subprocess. Production audit events are append-only PostgreSQL rows; local +unit tests may replace the sink with deterministic memory state. +Edge/distributed rate limiting remains a Terraform and deployment concern. diff --git a/docs/remediation/aws-postgres-schema.md b/docs/remediation/aws-postgres-schema.md new file mode 100644 index 0000000..21edfbb --- /dev/null +++ b/docs/remediation/aws-postgres-schema.md @@ -0,0 +1,102 @@ +# PostgreSQL Schema Contract + +Status: implemented by Alembic revisions `0001_postgresql_foundation` through +`0006_governance_retention` on 2026-07-29. + +## Version and activation model + +`dataset_versions.dataset_version` is the preserved external version key. +`revision` is an identity-backed monotonic sequence. A partial unique index +permits exactly one active dataset, while a check constraint requires active +status and an activation timestamp to agree. Every serving row includes the +dataset version in its primary and foreign keys, so a candidate can be loaded +and reconciled without replacing the approved dataset. Prior versions remain +addressable for rollback. + +`migration_runs` records the immutable source checksum/schema/fact versions, +code revision, actor, counts, reconciliation results and errors. It owns no +activation side effect by itself; Phase 4 performs activation in one explicit +transaction after reconciliation. + +## Relational ownership + +| Domain | Tables and invariant | +|---|---| +| Organisations | `charities`, `charity_programme_categories`, `charity_geographic_areas`; source identity and dataset-scoped charity IDs are preserved. | +| Official registry | `charity_registry_organizations`; `registry_id` remains textual and constituent rows sharing a charity number remain distinct. | +| Grants | `grants`; textual grant/source IDs, original and EUR amounts/currency/conversion facts, negative/zero values, review evidence and dates remain typed fields. | +| Grant relationships | `grant_beneficiary_countries`, `grant_beneficiary_terms`, `grant_programme_categories`; frequently filtered many-to-many facts are relational, not JSON. | +| Serving facts | `grant_overview_facts`, `grant_source_funder_facts`; dataset/revision provenance and minor-unit money facts remain explicit. | +| Curated links | `organization_registry_links`, `source_funder_link_overrides`, `source_funder_profile_cache`; accepted links, overrides and revision sequence are constrained. | +| Currency | `exchange_rates`; ISO-style currency/date primary key, positive high-precision rate and typed retrieval timestamp. | +| Jobs | `job_runs`, `job_events`, `source_ingestion_runs`, `job_dispatch_outbox`, `worker_heartbeats`; bounded statuses, durable idempotency, leases, retry/dead-letter state and event sequence constraints. | +| Pipeline storage/configuration | `source_configurations`, `storage_objects`, `ingestion_run_manifests`; fail-closed schedules, immutable raw object descriptors, version/checksum ownership and append-only run evidence. | +| Audit/control | append-only `audit_events` and durable `idempotency_records`. | +| Quality/governance | `data_quality_issues`, `materialization_versions`, `retention_actions`, `export_jobs`; quarantines retain original JSON values while queryable control fields remain relational. | +| Governance evidence | `retention_policies`, `data_holds`, `restore_verifications`, `deletion_manifests`, `data_subject_requests`; destructive retention is fail-closed, holds override actions, and evidence is append-only. | +| Versioned analytics | `analytics_scope_totals`, `analytics_country_aggregates`, `analytics_country_connections`, `analytics_period_aggregates`, `analytics_programme_aggregates`, `analytics_entity_rankings`, `analytics_country_funder_rankings`, `analytics_funder_relationships`, `analytics_filter_values`; every row is dataset-scoped and deleted by cascade with its dataset. | + +Every relationship declares update/delete behavior. The local catalog contains +45 application tables, 55 validated foreign keys and 189 check constraints. PostgreSQL enforces these +server-side for every connection; there is no connection-local equivalent to a +SQLite FK pragma. + +## Types and constraints + +- Timestamps use `TIMESTAMPTZ`; normalized source/business dates use `DATE`. + `grants.award_date` preserves the source's date-or-full-timestamp text exactly, + while `grants.exchange_rate_date` preserves its monthly `YYYY-MM` precision. + Dataset-versioned fact tables retain normalized `DATE` columns for analysis. +- Grant and financial amounts use `NUMERIC(24,4)`; rates use + `NUMERIC(24,12)`; derived minor-unit facts use `BIGINT`. +- Currency and country codes have uppercase length/shape checks. +- Coordinates, confidence values, classification methods, conversion states, + job states, revision values and activation transitions are checked. +- JSONB is limited to raw payload/evidence, result manifests and flexible audit + details; query relationships and control/status fields remain typed columns. +- Audit events reject update and delete. Link-override updates require the + revision to advance by exactly one. + +## Search and cursor contract + +PostgreSQL `pg_trgm` is installed by the migration. Registry rows have a stored +`tsvector` over name, normalized name, charity number, postcode and activity. +The schema supplies a GIN vector index plus trigram GIN indexes for registered +and normalized names. + +`RegistrySearchRepository` combines `websearch_to_tsquery`, `ts_rank_cd` and +trigram similarity. Rank is rounded to eight decimal places and ordered +descending; `registry_id` is the deterministic ascending tie-break. The opaque +cursor contains exactly that rank/ID pair. Limits are bounded to 100 and all +queries use SQLAlchemy async sessions over asyncpg. + +## Runtime boundary + +Staging and production select the PostgreSQL-only router at module import and +never import the legacy SQLite repository. A subprocess architectural test +blocks `sqlite3` and imports the production application to prove this boundary. +Development/test may still load the legacy implementation while fixtures and +journeys are ported. Unported staging/production journeys are intentionally +absent rather than receiving a hidden SQLite fallback; Phase 5 must complete +all route/domain implementations before production becomes a GO. + +## Serving materialization contract + +Revision `0004_versioned_analytics` creates `refresh_analytics_materializations(dataset_version)`. It deletes and deterministically rebuilds only the named dataset's aggregates, then upserts the `dashboard_analytics` materialization control row. The migration activation transaction calls the function before changing the single active dataset. Rollback verifies the approved target and builds its aggregate set if absent before reactivation. + +The active local snapshot contains 204,220 aggregate rows: 10 scope totals, 241 countries, 176 country connections, 1,191 monthly/yearly periods, 68 programmes, 187,995 ranked entities, 1,155 country funders, 13,245 bounded funder-recipient relationships and 139 filter values. Original-currency scopes remain separate from converted EUR. Negative, missing and invalid monetary facts are counted for disclosure and excluded from additive totals. + +Default map, trend, theme, summary and funder queries use these tables. Arbitrarily filtered requests retain fact-table execution. Heavy country relationships are exposed through a separate endpoint capped at 250; funder-recipient materialization is capped at 50 per funder. Exact current-registry lookup adds a dataset/name/ID B-tree index, while text search retains the stored-vector/trigram GIN strategy. + +## Local gate evidence + +- Host-venv `alembic upgrade head` succeeded. +- `alembic downgrade base` left only `alembic_version`, then zero-to-head + upgrade succeeded again. +- The non-root, read-only Compose migration service independently upgraded from + base to head. +- The catalog reports revision `0004_versioned_analytics`, 34 application + tables, 39 validated FKs, zero unvalidated FKs, 136 checks, `pg_trgm`, the + three original search indexes and the exact-current-name B-tree index. +- A real asyncpg integration test demonstrates FK rejection, full-text search, + deterministic tie-breaking and cursor continuation. diff --git a/docs/remediation/aws-public-demo-runbook.md b/docs/remediation/aws-public-demo-runbook.md new file mode 100644 index 0000000..afb60e2 --- /dev/null +++ b/docs/remediation/aws-public-demo-runbook.md @@ -0,0 +1,212 @@ +# Temporary AWS public read-only demo + +Status: design and read-only preflight snapshot prepared on 2026-08-06. This is +a temporary non-production demo design. Its HTTP listener is not +production-ready and must never transport login cookies, tokens or credentials. +This document describes controlled phases; it is not evidence that a particular +phase has or has not been executed. + +## Confirmed read-only preflight + +- AWS profile: `netlight` +- Account: `208337080387` (exact match) +- Region/configured region: `eu-west-1` (exact match) +- Default VPC: `vpc-0348ad96c2afcf8dd`, `172.31.0.0/16` +- Internet gateway: `igw-08ecd35c2bbf38c46` +- Public ALB/ECS subnets: + - `subnet-04d4e3e5af84a7ea4`, `eu-west-1a`, `172.31.16.0/20` + - `subnet-076f66fcdd86cd28f`, `eu-west-1b`, `172.31.32.0/20` + - spare confirmed public subnet: `subnet-0f5b48470ee59b581`, + `eu-west-1c`, `172.31.0.0/20` +- All three default subnets map public IPs and inherit the active + `0.0.0.0/0 -> igw-08ecd35c2bbf38c46` route. +- No isolated RDS subnets exist. The template therefore creates + `172.31.240.0/28` in `eu-west-1a` and `172.31.240.16/28` in `eu-west-1b`, + associated only with a new route table whose sole route is VPC-local. +- Existing ECR repositories: + - `foundation-intelligence-platform/backend` + - `foundation-intelligence-platform/frontend` +- Both repositories currently contain zero images. The public AWS CLI + `2.27.41` manifest was checked and contains both linux/amd64 and linux/arm64. +- PostgreSQL `16.14`, gp3, `db.t4g.medium` and `db.t4g.small` are orderable in + `eu-west-1a`, `eu-west-1b` and `eu-west-1c`. +- No matching CloudFormation stacks, ECS clusters, RDS instances, ALBs, + Secrets Manager secrets, CloudWatch log groups, S3 buckets, IAM roles, + schedules or security groups were found. +- `iam:SimulatePrincipalPolicy` was denied for the current user. Per the + requested contract the IAM result is `UNKNOWN`; no create call was used as a + permission test. + +The existing ECR repositories are mutable-tag repositories with AES256 +encryption and scan-on-push. Deployment commands must use unique Git-SHA tags +and resolve the resulting image digests before the stack is updated; `latest` +is prohibited by the template input contract. + +## Resource and sizing contract + +The template is `infra/cloudformation/demo.yaml`. + +| Component | Exact demo configuration | +|---|---| +| Application task | Fargate Linux/ARM64, 512 CPU units, 2,048 MiB, default 20 GiB ephemeral storage, desired count 0 initially then 1 | +| Frontend | Same task, unprivileged port 8080 behind the ALB listener on port 80; Nginx proxies `/api` to `127.0.0.1:8000` | +| Backend | Same task, port 8000 not exposed by any security-group ingress rule | +| Migration task | Fargate Linux/ARM64, 1,024 CPU units, 4,096 MiB, default 20 GiB ephemeral storage, one-off only | +| RDS import | PostgreSQL 16.14, `db.t4g.medium`, Single-AZ, 30 GiB gp3, autoscaling maximum 100 GiB | +| RDS steady state | Change the same stack parameter to `db.t4g.small` after import and loaded smoke tests | +| RDS safety | private/isolated subnets, public access false, encrypted storage, forced TLS, one-day backup retention, snapshot on delete/replace | +| ALB | internet-facing, two public subnets, HTTP port 80, IP targets, `/health/live` | +| S3 | private, bucket-owner enforced, AES256, full public-access block, TLS-only policy, import objects expire after seven days | +| Logs | separate frontend, backend and migration log groups; seven-day retention | + +There is no NAT gateway, autoscaling, Multi-AZ database, replica, Performance +Insights or VPC endpoint. The application task receives a public IP solely for +outbound AWS/API access. Its inbound port 8080 accepts traffic only from the ALB +security group. RDS port 5432 accepts traffic only from the shared ECS security +group. + +The demo uses asyncpg's `ssl=require` mode for every PostgreSQL connection. It +encrypts transport to RDS but does not verify the server identity. Production +hardening should use `verify-full` with the AWS RDS CA certificate after that +mode and certificate distribution are added to the runtime contract. + +The RDS-managed master secret is injected only into the one-off migration task. +CloudFormation generates a second application secret. The migration bootstrap +applies Alembic and runs the existing migration as the master, then +creates/rotates `foundation_app` without superuser, role-creation or DDL +capabilities. The application role receives only database connect, schema usage +and table `SELECT`; its transactions default to read-only. Public application +startup also skips configuration synchronization and database-backed audit or +idempotency writes. The runtime task never receives the master secret. + +## Source and migration gate + +The approved local source was independently rechecked: + +- path: `src/data/charities.db` +- size: `2,100,543,488` bytes +- SHA-256: `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` +- source schema version: `7` +- SQLite quick check: `ok` +- PostgreSQL Alembic head: `0006_governance_retention` + +The migration task has three ordered containers: + +1. An AWS CLI init container downloads exactly `imports/charities.db` from the + stack's private S3 bucket into the shared 20 GiB task volume, creates the + evidence directory and hands the volume to the non-root migration UID. +2. `migration.aws_entrypoint` acquires a PostgreSQL advisory lock, applies + Alembic and invokes the existing `migration.sqlite_to_postgres.migrate` + workflow with the master secret, then configures the SELECT-only application + role and verifies the release gate through that role. SHA-256 and source + schema are checked before loading. The remote-RDS capacity mode counts only + the source plus a 4 GiB local safety margin; PostgreSQL data and WAL remain + on the 30–100 GiB RDS volume and are not incorrectly counted against Fargate. +3. Only after migration success, a final AWS CLI container uploads the generated + reconciliation evidence to `migration-evidence//`. + +The existing loader stages a versioned candidate, validates types and foreign +keys, builds analytics materializations, reconciles baseline counts and controls, +and activates in a transaction only if every reconciliation passes. On any +failure the candidate is failed/rejected and the previous active dataset remains +unchanged. The release gate must also report ready before the task succeeds. + +## Controlled phase sequence + +Values such as the reviewed Git SHA and created bucket/ARNs must be resolved at +execution time; they must not be guessed. + +1. Review, then locally commit only the listed repository changes. Do not push + to GitHub during Phase A. +2. Build `backend-runtime` and `frontend-runtime-ecs` for `linux/arm64`. +3. Authenticate Docker to the two existing private ECR repositories, tag each + image once with `demo-`, push, then read back and use immutable + image digests. No ECR repository creation is planned. +4. Create, inspect and stop at a CloudFormation change set using + `CAPABILITY_IAM`, the reviewed image digests, full Git SHA, + `ApplicationDesiredCount=0` and `DatabaseInstanceClass=db.t4g.medium`. + Do not execute the change set during Phase A. +5. In a separately approved later phase, execute the reviewed change set. Read + the generated `ImportBucketName`, then upload exactly + `src/data/charities.db` to `s3:///imports/charities.db`, with the + approved SHA-256 recorded as object metadata, then read back object metadata. +6. Run exactly one `MigrationTaskDefinition` task in the two confirmed public + subnets with `AssignPublicIp=ENABLED` and the stack ECS security group. Wait + for all containers to exit zero and inspect the migration logs/evidence. +7. Keep the web service at zero until release-gate, readiness and loaded API + smoke checks pass. Then execute a second reviewed stack update setting only + `ApplicationDesiredCount=1`. +8. After import and loaded browser/API tests pass, execute a third reviewed + stack update changing RDS from `db.t4g.medium` to `db.t4g.small`. Do not use + `ApplyImmediately`; inspect the planned maintenance behavior first. + +S3 upload, change-set execution, CloudFormation deployment, ECS `RunTask`, RDS +modification, GitHub push and resource deletion are outside Phase A. + +## Automatic refresh plan — intentionally not scheduled + +The repository currently has solid versioned migration, reconciliation, +transactional activation, durable PostgreSQL job/outbox/lease primitives and +source limits. It does not yet have a production executable that connects the +current filesystem-oriented scraper/consolidation commands to immutable S3 and +a PostgreSQL candidate dataset. All source configurations are also disabled and +governance-blocked with unresolved legal/licence status. Creating a task or +schedule that pretends otherwise would not be fail-closed. + +After source governance approval, implement and manually validate this separate +ARM64 Fargate refresh task; the web service must never run it in the background: + +`EventBridge Scheduler (disabled initially)` → `ECS Refresh Task` → acquire +PostgreSQL advisory lock → fetch bounded approved sources into immutable S3 keys +→ validate schema/checksum and deduplicate → build a new versioned candidate → +reconcile counts/quality/materializations → transactionally activate → release +lock. + +- Proposed task: 1,024 CPU units, 4,096 MiB, default 20 GiB ephemeral storage, + public subnet/public IP for bounded source egress, no inbound rule. +- Parallel runs: `pg_try_advisory_lock` plus the durable job lease/unique + idempotency key; a second run exits without scraping or changing data. +- Failure isolation: source and candidate identifiers are immutable/versioned; + active-dataset pointers change only in the final successful transaction. +- Retry: task-level maximum two retries with exponential backoff for transient + ECS/source failures; no retry for validation, governance or reconciliation + failures; CloudWatch records class/counts without raw payloads or credentials. +- Initial scheduler state: absent/disabled until a manual AWS refresh succeeds. +- Later weekly default: `cron(0 2 ? * MON *)` in UTC, maximum event age one hour, + maximum two delivery retries. Enable only in a separately reviewed stack + update after the manual gate. + +## Monthly cost estimate + +The estimate uses the AWS Price List API values returned for EU (Ireland) on +2026-08-06, 730 hours/month, on-demand pricing and a low-traffic assumption of +0.1 average ALB LCU. It excludes VAT, support, internet egress and unexpected +traffic. CloudWatch assumes 1 GiB/month of logs; ECR assumes roughly 2 GiB of +images. Public IPv4 assumes two ALB addresses plus one running application task. + +| Item | Initial import month | After RDS resize | +|---|---:|---:| +| RDS instance | $50.37 (`db.t4g.medium`) | $25.55 (`db.t4g.small`) | +| RDS 30 GiB gp3 | $3.81 | $3.81 | +| Fargate app 0.5 vCPU / 2 GiB | $17.02 | $17.02 | +| ALB hours | $18.40 | $18.40 | +| ALB 0.1 average LCU assumption | $0.58 | $0.58 | +| Three public IPv4 addresses | $10.95 | $10.95 | +| Two Secrets Manager secrets | $0.80 | $0.80 | +| CloudWatch, ECR and temporary S3 estimate | $0.78 | $0.78 | +| **Estimated total** | **about $103/month** | **about $78/month** | + +The one-off 1 vCPU/4 GiB migration task is approximately $0.04662 per task-hour +plus one public IPv4 at $0.005/hour; even a six-hour import adds less than $0.32. +At one full ALB LCU instead of 0.1, add approximately $5.26/month. Internet data +transfer and request-driven load remain variable and require a budget/alarm +before any broader demo distribution. + +## Hardening after the temporary demo + +Before any production claim or use of login credentials, add an approved domain, +ACM certificate and HTTPS listener; redirect HTTP to HTTPS; replace the public +read-only mode with OIDC; add edge abuse protection/budgets; complete privacy and +source-governance approval; test restore/rollback; and reassess Multi-AZ, backups, +deletion protection and private ECS networking. None of those production claims +apply to the temporary HTTP demo. diff --git a/docs/remediation/backup-restore-guide.md b/docs/remediation/backup-restore-guide.md new file mode 100644 index 0000000..374f5a2 --- /dev/null +++ b/docs/remediation/backup-restore-guide.md @@ -0,0 +1,38 @@ +# Backup and restore guide + +The supported database recovery proof is a full PostgreSQL logical archive, +restored into an isolated database before any promotion. Backups must be +encrypted at rest in approved environments, access-controlled, checksummed, +retained under the unresolved governance policy and accompanied by RPO/RTO +evidence. Retention deletion remains disabled. + +Local full proof: + +```bash +POSTGRES_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder \ +POSTGRES_HOST_PORT=55432 \ +DOCKER_CONFIG=/private/tmp/docker-config-no-creds \ +DOCKER_HOST=unix:///Users/manuelgrabmayer/.docker/run/docker.sock \ +scripts/verify_local_restore.sh +``` + +The script uses `pg_dump --format=custom --compress=zstd:3`, creates the exact +temporary database `fip_phase13_restore_check_20260729`, performs +`pg_restore --exit-on-error`, and compares schema revision, active dataset, +charity/registry/grant counts, EUR total and materialization state. A trap drops +only that prefixed temporary database and removes only the exact temporary +archive path. + +Local evidence on 2026-07-29: + +- Archive bytes: `247509368` +- Archive SHA-256: `2c571954768ba4379f3e61160fb808cbc0bd35e6e13ec2f0b4d776c760ceae87` +- Schema: `0006_governance_retention` +- Active dataset: `sqlite-v7-8fc0cce61c81-r2` +- Charities/registry/grants: `373 / 397469 / 302546` +- Eligible EUR minor units: `2243598670770` +- Active materialization control rows: `1` +- Temporary database and archive removal: `PASS` + +RDS automated backups, PITR and cross-account restore are defined in Terraform +but remain `NOT TESTED`; no AWS API was called. diff --git a/docs/remediation/ci-cd-guide.md b/docs/remediation/ci-cd-guide.md new file mode 100644 index 0000000..59dd0a0 --- /dev/null +++ b/docs/remediation/ci-cd-guide.md @@ -0,0 +1,99 @@ +# CI/CD and Protected Deployment Guide + +Status: workflow definitions and local contracts complete; no workflow, GitHub +setting, OIDC exchange, AWS action, image push, Terraform plan/apply or staging +deployment has been executed. + +## Pull request gate + +`.github/workflows/ci.yml` has seven required workstreams and one aggregate +gate: + +1. Backend hash-locked install, compile, blocking Flake8, mypy, tests and 70% + coverage floor. +2. Frontend `npm ci`, lint, unit tests, production build/bundle budgets and + Playwright/axe at all six viewports. +3. pip/npm vulnerability, licence and secret scanning. +4. Python/JavaScript CodeQL SAST. +5. Digest-pinned container build, CycloneDX SBOM, Trivy and non-root/size/ + no-data/health assertions. +6. Terraform offline contract, formatting, read-only provider locks, + `init -backend=false`, validation and Trivy configuration scan. +7. Empty PostgreSQL Alembic migration, PostgreSQL integration, deterministic + migration/reconciliation fixtures, API golden tests and pool/performance + smoke. + +Backend and frontend CycloneDX 1.5 SBOMs are generated deterministically from +the committed lockfiles. The committed evidence currently contains 70 Python +and 128 npm components. The local licence gate inspected 155 installed +components, found zero unknown declarations and no AGPL/SSPL/GPL declaration. + +The Terraform job deliberately requires committed provider locks and uses +`-lockfile=readonly`. Those locks cannot be generated under the current +registry authorization, so the future GitHub required gate must remain +unconfigured or expected red until the Phase-11 blocker is resolved. This is +not represented as a passing remote CI run. + +## Staging workflow + +`.github/workflows/deploy-staging.yml` is manual-only and requires the exact +confirmation phrase `I_APPROVE_STAGING`. It uses no access-key inputs. + +```mermaid +flowchart LR + Manual[Manual confirmation] --> Build[Local candidate build + scans] + Build --> PublishApproval[Protected staging-publish approval] + PublishApproval --> OIDCPlan[OIDC publish by tag, resolve digest, plan] + OIDCPlan --> Plan[Uploaded text + JSON + binary plan] + Plan --> DeployApproval[Protected staging approval] + DeployApproval --> Apply[Apply exact reviewed plan] + Apply --> Alembic[Private one-off Alembic task] + Alembic --> Services[Deploy API + workers by task definition] + Services --> Frontend[Sync frontend + invalidate CloudFront] + Frontend --> Gates[Smoke, load, reconcile, E2E, security, rollback readiness] +``` + +The application/worker services ignore task-definition drift in Terraform so +the reviewed infrastructure plan cannot move serving traffic before Alembic +passes. The workflow then updates the two services explicitly and waits for +stability. The private release-gate task verifies exact schema, one approved +active dataset, successful reconciliation, active analytics materialization, +no active quality block and an empty DLQ. + +The workflow assumes an already approved/bootstraped staging foundation. A +first deployment needs a separately reviewed zero-capacity/bootstrap sequence; +the artefact deployment role is intentionally not an infrastructure-admin +bootstrap role. + +## Required GitHub configuration + +These settings are external blockers and have not been configured or read: + +- Required checks for the aggregate PR gate and CodeQL/SARIF permissions. +- Protected environments `staging-publish`, `staging` and `production`, with + independent required reviewers and no self-approval. +- OIDC plan/publish/deployment roles and exact repository/environment trust. +- Staging account/region, encrypted state bucket, lock table, Route53/ACM, + API/frontend URL and repository variables. +- A short-lived staging test bearer token stored only in the protected + environment secret store. +- Retention rules for plan, image, SBOM, SARIF and deployment evidence. + +Action references use versioned upstream release tags because GitHub access +was not authorised to resolve and verify full action commit SHAs. Before +enabling required checks, resolve every action to a reviewed full commit SHA +and record the mapping; do not silently trust a moved tag. + +## Rollback and production + +Before apply, staging records the prior API and worker task-definition ARNs. +The gate proves they remain describable and the ECS deployment circuit breaker +is configured for rollback. Actual rollback is an explicit operator action +following the versioned rollback runbook; it is not silently performed from +an unapproved workflow. + +`.github/workflows/deploy-production.yml` declares the protected production +environment but its only job is hard-disabled with `if: ${{ false }}`. It must +not be enabled until provider locks, remote CI, AWS/DNS/identity/owner/legal/ +recovery approvals, staging evidence and an explicit production authorization +all exist. diff --git a/docs/remediation/cutover-runbook.md b/docs/remediation/cutover-runbook.md new file mode 100644 index 0000000..301ffef --- /dev/null +++ b/docs/remediation/cutover-runbook.md @@ -0,0 +1,57 @@ +# PostgreSQL cutover runbook + +This runbook is prepared but has not been executed in AWS or production. +Every production or AWS step requires a new explicit approval. + +## Preconditions and stop conditions + +- Approved change window, incident commander, data owner and rollback owner. +- Reviewed immutable application image digests and exact Terraform plan. +- Current full backup with checksum and a successful isolated restore proof. +- Current SQLite source checksum, source counts and frozen source snapshot. +- Alembic, reconciliation, golden, security, load, E2E and release gates green. +- PostgreSQL previous dataset and previous ECS task definitions retained. +- Stop if writers cannot be frozen, checksums drift, the final delta is not + coherent, reconciliation differs, readiness fails or approvals are absent. + +## Procedure + +1. Announce the window and freeze every SQLite writer, pipeline schedule, + worker, manual refresh and enrichment action. Confirm no queued/running job. +2. Create the final coherent SQLite backup with the application stopped. Record + byte size, SHA-256, schema version, source watermark and row counts. Preserve + the source snapshot read-only; never overwrite it. +3. Create a final PostgreSQL backup/restore point and verify the checksum and + restore instructions before changing the active dataset. +4. Load only the deterministic final SQLite delta into a new PostgreSQL dataset + version. Never update the currently active version in place. +5. Run FK/orphan checks, counts, totals, duplicate controls, conversion gaps, + Golden Fixtures and `scripts/verify_transition.py`. Any unexplained + difference is a stop condition. +6. Activate the approved PostgreSQL candidate transactionally. Retain the prior + dataset as `approved`/`rolled_back`; do not delete it. +7. Run `python -m migration.release_gate`. Only after it passes, switch the + application to `DATA_RUNTIME_MODE=postgresql`. Do not configure SQLite as a + fallback. +8. Run authenticated smoke tests for liveness/readiness, dashboard, map, + relationships, filters, registry, profiles, grants, Sankey, score, pipeline + status and operator-only manual refresh. +9. Observe API latency/errors, readiness, pool use, reconciliation, conversion + gaps, coverage, queue/DLQ, worker failures and cost alarms for the approved + monitoring window. +10. Preserve the SQLite source snapshot and all cutover evidence. Resume writers + only after reconciliation and monitoring acceptance. + +## Rollback triggers + +Rollback on readiness failure, auth/RBAC regression, unexplained semantic +difference, reconciliation failure, materialization absence, persistent error +or latency budget breach, writer inconsistency, or explicit incident-command +decision. Use `rollback-runbook.md`; do not improvise bidirectional replay. + +## Local evidence + +The local proof on 2026-07-29 compared 18 semantic projections with zero +differences, switched from `sqlite-v7-8fc0cce61c81-r2` to the approved prior +dataset and back, and fully restored a 247,509,368-byte logical archive into an +isolated database. No traffic, AWS resource or production mode was changed. diff --git a/docs/remediation/data-governance-register.md b/docs/remediation/data-governance-register.md new file mode 100644 index 0000000..4bb2eab --- /dev/null +++ b/docs/remediation/data-governance-register.md @@ -0,0 +1,89 @@ +# Data Governance Register + +Status: Phase-9 local governance baseline. Policy status is `proposed`. +No legal, licence, privacy or production approval is inferred by this file. + +## Owner register + +| Role | Owner | Status | +|---|---|---| +| Business owner | unassigned | unresolved | +| Data owner | unassigned | unresolved | +| Privacy owner | unassigned | unresolved | +| Legal owner | unassigned | unresolved | +| Security owner | unassigned | unresolved | +| Technical owner | data-platform | assigned role, named person unresolved | + +Unassigned owners block production approval. They are not replaced with the +developer, repository author or cloud account owner by assumption. + +## Source register + +`config/source-pipelines.json` is the executable source register. The database +retains its checksummed projection in `source_configurations`. + +| Source | Classification | Legal | Licence | Schedule | +|---|---|---|---|---| +| 360Giving | public organisation and grant data | unresolved | unresolved | disabled / blocked | +| Charity Commission | public organisation, contact and financial data | unresolved | unresolved | disabled / blocked | +| Philea | public organisation data | unresolved | unresolved | disabled / blocked | +| Hinchilla | public organisation data | unresolved | unresolved | disabled / blocked | +| ECB | public reference data | unresolved | unresolved | disabled / blocked | +| Google News RSS | article metadata | unresolved | unresolved | disabled / blocked | +| Article content | article content | unresolved | unresolved | disabled / blocked | +| Anthropic news summary | derived classification | unresolved | unresolved | disabled / blocked; paid use separately prohibited | + +Terms URLs intentionally remain unset until reviewed. Database and application +constraints prohibit enabling a source while legal/licence status is +unresolved or a governance block is present. + +## Classification and exposure register + +| Classification | Exposure | Proposed archive dry-run | Destructive window | +|---|---|---:|---| +| Public organisation data | authenticated | 365 days | unset | +| Contact data | restricted | 90 days | unset | +| Personal email addresses | restricted | 30 days | unset | +| Postal addresses | restricted | 90 days | unset | +| Raw source evidence | internal | 30 days | unset | +| Article metadata | authenticated | 30 days | unset | +| Article content | internal | 1 day | unset | +| Pipeline logs | restricted | 30 days | unset | +| Exports | restricted | 7 days; expiry report at 7 days | unset | +| Audit events | restricted | 365 days | unset | +| Derived classifications | authenticated | 365 days | unset | +| Enriched profiles | authenticated | 365 days | unset | +| Credentials | never | unset | unset | +| User identities | restricted | 90 days | unset | + +Archive windows create reports only; they are proposed operational review +points, not deletion approval. Holds override every window. + +## Field-level exposure + +Response models already name fields for typed domain endpoints. Generic +administrative dictionaries now additionally use policy allowlists: + +- health output: `status`, `service`, `checks`; +- organization summary: named public/analytical identity fields only; +- job history: lifecycle metadata only, excluding input and actor identity; +- job events: bounded event metadata plus recursively redacted details; +- source configuration: operational schedule fields, excluding credential + references and user-agent detail. + +There is no fallback policy that serializes every database column. An unknown +exposure policy raises an error. + +## Privacy checklist + +| Control | Status | +|---|---| +| Lawful-basis review | unresolved | +| Data protection impact assessment | not started | +| Data-subject contact | unassigned | +| Source terms review | unresolved | +| International-transfer review | unresolved | +| Processor register | not started | +| Breach-response owner | unassigned | + +These unresolved items keep the overall production decision at `NO-GO`. diff --git a/docs/remediation/documentation-index.md b/docs/remediation/documentation-index.md new file mode 100644 index 0000000..feca52a --- /dev/null +++ b/docs/remediation/documentation-index.md @@ -0,0 +1,28 @@ +# Remediation documentation index + +This index maps the required documentation without modifying `docs/audits/`. + +| Requirement | Remediation document | +|---|---| +| README and local development | `README.md`, `local-development-guide.md` | +| PostgreSQL setup/migration | `postgresql-migration-guide.md`, `aws-postgres-schema.md`, Phase-4 evidence | +| AWS architecture | `aws-postgres-architecture.md`, `terraform-aws-infrastructure.md` | +| Terraform deployment | `terraform-validation.md`, `ci-cd-guide.md` | +| Authentication/RBAC/security | `security-authentication-rbac-guide.md`, Phase-2/5 progress evidence | +| Data model/entity identity | `aws-postgres-schema.md`, `aws-postgres-architecture.md`, ADR-014/015 | +| Source register | `data-governance-register.md`, `pipeline-storage-contract.md` | +| Retention/privacy | `retention-privacy-guide.md`, Phase-9 evidence | +| Performance | Phase-6 performance JSON/Markdown and `frontend-bundle-budget.md` | +| Reconciliation | Phase-4 migration manifest/report and Phase-13 transition fixture/report | +| Operations/incidents | `observability-runbooks.md`, `runtime-transition-guide.md` | +| Backup/restore/rollback | `backup-restore-guide.md`, `rollback-runbook.md`, `cutover-runbook.md` | +| Cost estimate | `terraform-aws-infrastructure.md` (planning envelope, not live price claim) | +| CI/CD | `ci-cd-guide.md`, Phase-12 evidence | +| Environment variables | `environment-variable-reference.md` | +| Troubleshooting | `troubleshooting-guide.md` | +| Requirements traceability | `requirements-traceability.md` | +| Feature test matrix | `feature-test-matrix.md` | +| Remediation status | `aws-postgres-progress.md`, final acceptance report | + +The immutable audit versions of traceability and feature matrices remain in +`docs/audits/`; remediation status is maintained only in this directory. diff --git a/docs/remediation/environment-variable-reference.md b/docs/remediation/environment-variable-reference.md new file mode 100644 index 0000000..edc834e --- /dev/null +++ b/docs/remediation/environment-variable-reference.md @@ -0,0 +1,32 @@ +# Environment variable reference + +| Variable | Required | Boundary | +|---|---:|---| +| `APP_ENV` | Yes operationally | `development`, `test`, `staging`, `production` | +| `DATA_RUNTIME_MODE` | No | Defaults to `postgresql`; SQLite source is local/test only | +| `SHADOW_SQLITE_PATH` | Shadow only | Separate coherent snapshot; cannot equal active `DB_PATH` | +| `AUTH_MODE` | Yes outside test | Staging/production require `oidc` | +| `OIDC_ISSUER`, `OIDC_AUDIENCE` | OIDC | Exact identity contract | +| `OIDC_JWKS_URL` or `OIDC_JWKS_JSON` | OIDC | HTTPS URL outside development or injected static JWKS | +| `OIDC_ALGORITHMS` | No | Explicit asymmetric allowlist, default `RS256` | +| `CORS_ORIGINS` | Yes operationally | Explicit HTTPS origins in staging/production; no wildcard | +| `DATABASE_URL` | Alternative | Must be `postgresql+asyncpg` | +| `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_NAME`, `DATABASE_USER` | Alternative | PostgreSQL endpoint components | +| `DATABASE_PASSWORD_FILE` | Preferred | Mounted runtime-only secret file | +| `DATABASE_PASSWORD` | Alternative | Runtime-injected secret; never log/commit | +| `DATABASE_POOL_SIZE`, `DATABASE_MAX_OVERFLOW` | No | Positive bounded pool values | +| `DATABASE_POOL_TIMEOUT_SECONDS` | No | Positive checkout timeout | +| `DATABASE_CONNECT_TIMEOUT_SECONDS` | No | Positive connection/readiness timeout | +| `DATABASE_STATEMENT_TIMEOUT_MS` | No | Positive server statement limit | +| `RATE_LIMIT_REQUESTS`, `RATE_LIMIT_WINDOW_SECONDS` | No | Per-actor process/app limit | +| `MAX_REQUEST_BODY_BYTES`, `REQUEST_TIMEOUT_SECONDS` | No | Request safety bounds | +| `CORE_PROXY_ENABLED` | No | Default false | +| `CORE_API_URL`, `CORE_API_ALLOWED_HOSTS` | Proxy only | Exact absolute target and host allowlist | +| `CORE_PROXY_ALLOWED_PATHS`, `CORE_PROXY_ALLOWED_METHODS` | Proxy only | Required allowlists | +| `SCORE_CONFIG_PATH` | No | Reviewed scoring configuration | +| `DB_PATH` | Migration/local only | SQLite source path; never production fallback | +| `ANTHROPIC_AUTH_TOKEN`, `ANTHROPIC_BASE_URL` | Optional | Live news requires separate network/cost approval | + +Frontend `VITE_*` values are public build-time configuration and must never hold +secrets. Terraform variables do not accept AWS credentials; deployment identity +uses GitHub OIDC. diff --git a/docs/remediation/evidence/phase10-observability-report.json b/docs/remediation/evidence/phase10-observability-report.json new file mode 100644 index 0000000..4d6915d --- /dev/null +++ b/docs/remediation/evidence/phase10-observability-report.json @@ -0,0 +1,64 @@ +{ + "phase": 10, + "status": "passed", + "generated_on": "2026-07-29", + "alembic_revision": "0006_governance_retention", + "local_checks": { + "observability_security_contract_tests": { + "passed": 19, + "failed": 0, + "skipped": 1, + "subtests_passed": 8 + }, + "observability_postgresql_tests": { + "passed": 6, + "failed": 0 + }, + "combined_postgresql_tests": { + "passed": 33, + "failed": 0 + }, + "normal_backend_tests": { + "passed": 331, + "failed": 0, + "skipped": 12, + "subtests_passed": 8 + }, + "python_compile": "passed", + "blocking_flake8": "passed", + "configuration_json": "passed", + "diff_check": "passed" + }, + "contracts": { + "metric_definitions": 21, + "alarm_definitions": 15, + "runbooks": 13, + "structured_json_logs": true, + "recursive_redaction": true, + "pseudonymous_actor_ids": true, + "liveness_dependency_free": true, + "readiness_independent_pool": true, + "readiness_survives_application_pool_exhaustion": true, + "administrator_only_local_metrics": true, + "cloudwatch_execution": "not_tested" + }, + "postgresql_state": { + "active_dataset": "sqlite-v7-8fc0cce61c81-r2", + "source_configurations": 8, + "retention_policies": 14, + "pending_or_failed_outbox_rows": 0, + "dead_lettered_jobs": 0 + }, + "unresolved": [ + "production alarm owners and notification destinations", + "approved service-level thresholds and error budgets", + "approved cost ceiling", + "CloudWatch deployment and execution evidence" + ], + "aws_actions_performed": false, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase10-observability-report.md b/docs/remediation/evidence/phase10-observability-report.md new file mode 100644 index 0000000..3ca3600 --- /dev/null +++ b/docs/remediation/evidence/phase10-observability-report.md @@ -0,0 +1,39 @@ +# Phase 10 Observability Evidence + +## Result + +Gate 10 **passes for local code readiness**. Dependency-free liveness, +dependency-aware readiness, redacted structured JSON logs, bounded local +metric evidence, alarm definitions and incident runbooks are implemented and +tested. No CloudWatch resource or notification destination was created or +executed. + +## Verified locally + +- `/health` and `/health/live` do not query PostgreSQL. +- `/health/ready` checks PostgreSQL, exact Alembic revision, the approved + active dataset, synchronized source/retention configuration and the durable + queue contract. +- Readiness uses a dedicated `NullPool` connection and succeeds while the + one-connection application pool is deliberately exhausted. +- JSON logs carry request, trace, actor, role, operation, duration, status and + error-class context while credentials, email/contact data and raw exception + payloads are redacted. +- Twenty-one metric and fifteen alarm definitions cover API, database, cache, + freshness, pipeline, reconciliation, coverage, queue/DLQ, workers, restart, + RDS and cost signals. Every alarm links to one of thirteen versioned + runbooks. +- The local metrics endpoint is administrator-only and explicitly reports + CloudWatch execution as `not_tested`. + +The normal backend suite passes 331 tests, skips 12 explicit live-environment +tests and passes eight route subtests. Dedicated Phase-10 PostgreSQL execution +passes 6/6; the combined observability/governance/pipeline/application/schema +run passes 33/33. Compilation, blocking Flake8, JSON and whitespace checks +pass. + +The final local database remains on `0006_governance_retention` with active +dataset `sqlite-v7-8fc0cce61c81-r2`, eight source configurations, fourteen +retention policies, zero pending/failed outbox rows and zero dead-lettered +jobs. SQLite and `docs/audits/` remain byte-for-byte unchanged. No AWS, +external API, download, upload or push occurred. diff --git a/docs/remediation/evidence/phase11-terraform-report.json b/docs/remediation/evidence/phase11-terraform-report.json new file mode 100644 index 0000000..dd84d70 --- /dev/null +++ b/docs/remediation/evidence/phase11-terraform-report.json @@ -0,0 +1,72 @@ +{ + "phase": 11, + "status": "completed_local_provider_validation_not_tested", + "generated_on": "2026-07-29", + "terraform": { + "environments": ["dev", "staging"], + "files": 26, + "resource_blocks": 103, + "resource_types": 58, + "provider": "hashicorp/aws ~> 5.90", + "remote_backend_declared": true, + "remote_backend_coordinates_committed": false, + "schedules_enabled": false, + "terraform_cli": "docker hashicorp/terraform 1.9.8", + "local_terraform_image": "hashicorp/terraform@sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd", + "provider_lock": "not_generated", + "fmt": "passed", + "init_backend_false": "blocked_missing_unauthorised_aws_provider", + "validate": "blocked_missing_unauthorised_aws_provider", + "security_scanner": "not_installed", + "plan": "not_tested", + "apply": "prohibited_not_run" + }, + "offline_checks": { + "structure_and_security_contract": "passed", + "terraform_contract_tests": {"passed": 2, "failed": 0}, + "normal_backend_tests": { + "passed": 334, + "failed": 0, + "skipped": 12, + "subtests_passed": 8 + }, + "python_compile": "passed", + "blocking_flake8": "passed", + "diff_check": "passed" + }, + "controls": { + "private_rds": true, + "rds_encrypted": true, + "rds_deletion_protection": true, + "rds_prevent_destroy": true, + "rds_managed_secret": true, + "s3_public_access_blocked": true, + "s3_bucket_owner_enforced": true, + "s3_kms_encrypted": true, + "s3_destructive_expiration": false, + "ecs_private": true, + "ecs_non_root": true, + "ecs_read_only_root": true, + "ecs_digest_images_required": true, + "github_oidc_exact_subject": true, + "long_lived_aws_keys": false, + "dns_default_enabled": false + }, + "cost_envelopes_usd_per_month": { + "development": {"low": 180, "high": 300, "proposed_budget": 300}, + "staging": {"low": 550, "high": 900, "proposed_budget": 750}, + "live_pricing_claimed": false + }, + "blockers": [ + "Terraform providers cannot be fetched from the currently approved package sources", + "tflint/checkov/tfsec/trivy/semgrep are not installed", + "AWS read access and plan generation are not approved", + "DNS/ACM ownership and production secrets are unresolved" + ], + "aws_actions_performed": false, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase11-terraform-report.md b/docs/remediation/evidence/phase11-terraform-report.md new file mode 100644 index 0000000..b0855c4 --- /dev/null +++ b/docs/remediation/evidence/phase11-terraform-report.md @@ -0,0 +1,43 @@ +# Phase 11 Terraform Evidence + +## Result + +The Phase-11 infrastructure definitions are complete for local code review: +one reusable platform module and isolated dev/staging roots contain 26 +Terraform files, 103 resource blocks and 58 AWS resource types. The offline +structure/security gate and all application regressions pass. Terraform 1.9.8 +now parses and canonically formats the complete tree; `fmt -check -recursive` +passes after correcting one invalid nested WAF block and applying `terraform +fmt` to the files reported by the CLI. + +Provider-dependent Gate-11 execution remains **BLOCKED/NOT TESTED**. The pinned +Docker Hub image is +`hashicorp/terraform@sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd`. +Network-isolated `init -backend=false` installed the local module and stopped +when the unavailable `hashicorp/aws` provider would have required +`registry.terraform.io`; `validate` consequently reports the missing provider. +Security scanners and a non-destructive staging plan are not claimed. + +## Implemented definitions + +- Multi-AZ network tiers, cost-aware NAT and private service endpoints. +- KMS, private/versioned/non-public S3 stages, CloudFront and regional WAF. +- Immutable ECR, private non-root/read-only ECS API/workers, ALB, health checks + and autoscaling. +- Private encrypted RDS PostgreSQL with managed Secrets Manager credentials, + backup/PITR window, enhanced monitoring, deletion protection and + `prevent_destroy`. +- Encrypted FIFO SQS/DLQ, disabled scheduler, Step Functions, bounded task + roles and exact-subject GitHub OIDC deployment role. +- CloudWatch logs/dashboard/15 alarm definitions, SNS and environment budgets. +- Fail-closed ACM/Route53 configuration points and non-deployable example + variables; DNS remains disabled. +- Four architecture/security/data/deployment diagrams and planning cost + envelopes of USD 180–300/month for dev and USD 550–900/month for staging. + +The normal backend suite passes 334 tests, skips 12 explicit live tests and +passes eight subtests. The two dedicated Terraform contract tests, compile, +blocking Flake8 and whitespace checks pass. SQLite and `docs/audits/` remain +unchanged. The Terraform CLI image was downloaded only from approved Docker +Hub endpoints. No provider download, AWS call, state access, plan, apply, +destroy, import, DNS/certificate action, upload or push occurred. diff --git a/docs/remediation/evidence/phase12-cicd-report.json b/docs/remediation/evidence/phase12-cicd-report.json new file mode 100644 index 0000000..e4f8c0d --- /dev/null +++ b/docs/remediation/evidence/phase12-cicd-report.json @@ -0,0 +1,75 @@ +{ + "phase": 12, + "status": "completed_local_remote_execution_not_tested", + "generated_on": "2026-07-29", + "workflows": { + "files": 3, + "pr_gate_markers": 24, + "yaml_syntax": "passed", + "offline_contract": "passed", + "staging_environments": ["staging-publish", "staging"], + "staging_manual_confirmation": true, + "oidc_only": true, + "production_enabled": false, + "github_execution": "not_tested" + }, + "supply_chain": { + "mypy": "2.3.0", + "python_hash_lock": true, + "npm_ci": true, + "backend_sbom_components": 70, + "frontend_sbom_components": 128, + "backend_sbom_sha256": "bfaddc072cdf4d24fec612d0dd162690e512bc9fc762f11f523031438c704715", + "frontend_sbom_sha256": "c189f0ac2f92ddbb2df0381529ee2c2e5109551f87a168e7a8241001089644ac", + "licence_components_scanned": 155, + "unknown_licences": 0, + "forbidden_licences": 0, + "npm_vulnerabilities": 0, + "python_vulnerability_scan": "not_tested_local", + "container_vulnerability_scan": "not_tested_local" + }, + "local_checks": { + "typed_files": 13, + "mypy": "passed", + "focused_phase12_tests": {"passed": 10, "failed": 0}, + "normal_backend_tests": { + "passed": 342, + "failed": 0, + "skipped": 13, + "subtests_passed": 8 + }, + "postgresql_performance_smoke": {"passed": 1, "failed": 0}, + "postgresql_release_gate": "passed", + "frontend_unit_tests": 13, + "frontend_e2e_passed": 8, + "frontend_e2e_skipped": 4, + "frontend_lint": "passed", + "frontend_build": "passed", + "bundle_budget": "passed", + "blocking_flake8": "passed" + }, + "container": { + "build_pull": false, + "image_id": "sha256:172dab7c1c7842b0b34f0991d97f8ae34391d36e6ece628db4a63672c36781e9", + "size_bytes": 354624742, + "user": "10001:10001", + "healthcheck": true, + "data_free": true, + "governance_policies": 14, + "observability_metrics": 21, + "source_configurations": 8 + }, + "blockers": [ + "Terraform provider locks cannot be generated from currently approved sources", + "Terraform and local security/vulnerability scanners are not installed", + "GitHub branch protection, protected environments and OIDC settings are not configured or verified", + "workflow action commit SHAs cannot be resolved without GitHub access", + "no workflow or staging deployment has been executed" + ], + "aws_actions_performed": false, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase12-cicd-report.md b/docs/remediation/evidence/phase12-cicd-report.md new file mode 100644 index 0000000..43a2a81 --- /dev/null +++ b/docs/remediation/evidence/phase12-cicd-report.md @@ -0,0 +1,36 @@ +# Phase 12 CI/CD Evidence + +## Result + +Gate 12 passes for local workflow/code readiness. Three syntactically valid +workflows define 24 required PR gate markers, manual OIDC-only staging +publish/plan/deploy through two protected environments, and a hard-disabled +production environment. No workflow or remote GitHub setting was executed or +verified. + +## Local evidence + +- Mypy 2.3.0 and four transitive dependencies are exact/hash locked; 13 typed + production files pass. +- Deterministic CycloneDX SBOMs contain 70 Python and 128 npm components. +- The licence scan covers 155 installed components with no unknown or + forbidden declaration; npm reports zero known vulnerabilities. +- 342 backend tests pass, 13 explicit live tests skip and 8 subtests pass. +- Frontend lint, 13 unit tests, build/bundle budgets and 8 Playwright/axe tests + pass; 4 redundant viewport journeys intentionally skip. +- The local PostgreSQL performance smoke passes and the release gate confirms + schema, active dataset, reconciliation, materialization, quality and DLQ. +- The `--pull=false` runtime image is data-free, 354,624,742 bytes, non-root, + healthchecked and imports 14 governance policies, 21 metric definitions and + 8 source configurations. Image ID is + `sha256:172dab7c1c7842b0b34f0991d97f8ae34391d36e6ece628db4a63672c36781e9`. + +Python/container vulnerability scanners, Terraform/provider locks and remote +CI execution remain untested local/external blockers. The Terraform PR job +intentionally fails closed until reviewed provider locks exist. GitHub branch +rules, environment reviewers, OIDC trust and action commit-SHA pins are not +claimed. + +SQLite and `docs/audits/` remain unchanged. No AWS action, state access, +Terraform plan/apply, image push, frontend upload, deployment or Git push +occurred. diff --git a/docs/remediation/evidence/phase13-transition-report.json b/docs/remediation/evidence/phase13-transition-report.json new file mode 100644 index 0000000..d71cbdf --- /dev/null +++ b/docs/remediation/evidence/phase13-transition-report.json @@ -0,0 +1,97 @@ +{ + "phase": 13, + "status": "completed_local_aws_production_unexecuted", + "generated_on": "2026-07-29", + "runtime": { + "default": "postgresql", + "sqlite_migration_source_production_allowed": false, + "shadow_primary": "postgresql", + "sqlite_fallback": false, + "shadow_async_after_response": true, + "maximum_pending": 8, + "maximum_response_bytes": 2097152, + "timeout_seconds": 20, + "journeys_registered": 21 + }, + "shadow_comparison": { + "status": "passed", + "semantic_projections": 18, + "difference_count": 0, + "dataset_version": "sqlite-v7-8fc0cce61c81-r2", + "source_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "golden_sha256": "f2065b01fe6385357314e6db4bf61628f1755c9aae03be66246c327331639166", + "golden_bytes": 110590, + "live_news_calls": false + }, + "rollback": { + "status": "passed", + "writers_frozen": true, + "from": "sqlite-v7-8fc0cce61c81-r2", + "to": "sqlite-v7-8fc0cce61c81", + "restored_to": "sqlite-v7-8fc0cce61c81-r2", + "counts_equal": true, + "materialization_active": true, + "bidirectional_replay": false + }, + "restore": { + "status": "passed", + "type": "full_logical_pg_dump_pg_restore", + "archive_bytes": 247509368, + "archive_sha256": "2c571954768ba4379f3e61160fb808cbc0bd35e6e13ec2f0b4d776c760ceae87", + "schema_version": "0006_governance_retention", + "active_dataset": "sqlite-v7-8fc0cce61c81-r2", + "charities": 373, + "registry": 397469, + "grants": 302546, + "overview_total_eur_minor": 2243598670770, + "temporary_database_removed": true, + "archive_removed": true + }, + "final_local_checks": { + "normal_backend_tests": {"passed": 352, "skipped": 13, "subtests_passed": 8}, + "coverage_percent": 72.73, + "coverage_scope": "all src modules via the exact CI command", + "postgresql_combined_tests": {"passed": 55, "failed": 0}, + "frontend_unit_tests": 13, + "frontend_e2e_passed": 8, + "frontend_e2e_skipped": 4, + "mypy_files": 17, + "blocking_flake8": "passed", + "workflow_offline_validation": "passed", + "terraform_offline_validation": "passed", + "terraform_fmt_check": "passed", + "terraform_provider_validation": "blocked_missing_unauthorised_provider" + }, + "containers": { + "backend_image_id": "sha256:101071f6c750f6806408ed2f898a6a2496d24704a7f36ddcfbb679fcbcc338ee", + "backend_size_bytes": 354658326, + "backend_user": "10001:10001", + "frontend_image_id": "sha256:8f665856111eb532dae72c5c1c82d1b9ee4b90b7b73cd3975cd17d6cead9ad1a", + "frontend_size_bytes": 56241904, + "frontend_user": "101:101", + "stack_liveness": "passed", + "stack_readiness": "passed", + "backend_restart_readiness": "passed" + }, + "approved_downloads": { + "registry.npmjs.org": "npm ci in frontend image build and final local install", + "docker.io": "hashicorp/terraform:1.9.8 resolved to sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd", + "resolved_packages": 80, + "other_hosts": false + }, + "expected_failures_recorded": [ + "two initial transition query attempts rejected string parameters for PostgreSQL date bindings", + "the first rollback attempt rejected a stale hard-coded migration schema revision before switching datasets", + "integral PostgreSQL NUMERIC values initially produced representational type differences until exact integer normalization", + "the first container-backed E2E run used a stale local frontend image and failed 8 tests; rebuilding the current pinned image produced 8 passes and 4 intentional skips", + "the literal CI coverage command used a bare pytest entry point and failed four scripts imports; invoking python -m pytest restored the repository root and passed the 70 percent gate", + "the first native Terraform format check rejected invalid single-line nested WAF blocks; the syntax was corrected and the final network-isolated format check passed" + ], + "aws_cutover_performed": false, + "production_cutover_performed": false, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase13-transition-report.md b/docs/remediation/evidence/phase13-transition-report.md new file mode 100644 index 0000000..0d67e9d --- /dev/null +++ b/docs/remediation/evidence/phase13-transition-report.md @@ -0,0 +1,60 @@ +# Phase 13 transition evidence + +Status: `PASS` locally; AWS and production cutover `NOT TESTED`/unexecuted. + +PostgreSQL is the default authoritative runtime. SQLite migration-source mode is +rejected in staging/production, and temporary shadow mode requires a separate +snapshot. Shadow reads run after the primary response, are queue/size/time +bounded and record only privacy-safe difference fingerprints. + +The executed local source/target gate compared 18 semantic projections across +counts, totals, coverage, filters, map relationships, trends, donor/recipient +rankings, registry pagination/search, profiles, grant lists, Sankey, score +components and currency statuses. Result: `0` differences against active dataset +`sqlite-v7-8fc0cce61c81-r2` and SQLite SHA-256 `8fc0cce…7651`. + +The local rollback switched to approved prior dataset +`sqlite-v7-8fc0cce61c81` and restored `sqlite-v7-8fc0cce61c81-r2`, with equal +counts and an active materialization. No bidirectional replay occurred. + +The full logical restore created a 247,509,368-byte archive with SHA-256 +`2c571954…ae87`, restored it into an isolated temporary database and matched +schema `0006_governance_retention`, active dataset, counts, EUR total and +materialization. The temporary database and archive were removed. + +Three failed diagnostics remain explicit in the command log: strict date +binding, exact numeric representation and the stale migration-schema constant. +All were fixed and repeated successfully; none was hidden or described as a +successful first attempt. + +No AWS resource, production traffic, external API, live news provider or Git +remote was touched. + +Final regression evidence is 352 normal backend passes, 13 skips and eight +subtests at 72.73% coverage across all `src` modules, passing the exact 70% CI +gate; 55 combined PostgreSQL integration passes; +17 mypy files; 13 frontend unit passes; and 8 Playwright/axe passes with four +intentional skips. The first container E2E attempt used a stale local frontend +image and failed eight tests. After rebuilding the current pinned image, the +same command passed; both attempts are retained in the command log. + +The Phase-13 backend image is `sha256:101071…338ee`, 354,658,326 bytes and +`10001:10001`. The frontend image is `sha256:8f6658…9ad1a`, 56,241,904 bytes +and `101:101`. Stack liveness/readiness and backend restart readiness pass. +The frontend build and final deterministic install used only the explicitly +approved `registry.npmjs.org` via `npm ci`; no arbitrary download occurred. + +The final acceptance replay also exposed and fixed a workflow-only invocation +error: calling the installed `pytest` entry point with `PYTHONPATH=src` omitted +the repository root and prevented four `scripts.*` imports. All workflow test +steps now use `python -m pytest`; the offline workflow validator rejects a +regression to the bare entry point. The corrected exact CI coverage command +passes with the results above. + +The approved Docker Hub pull of Terraform 1.9.8 resolved manifest digest +`sha256:18f998…28cd`. The real parser exposed one invalid nested WAF block; +after correction and canonical formatting, network-isolated +`fmt -check -recursive` passes. A temporary, network-isolated +`init -backend=false` installed the local module and stopped at the unavailable, +unauthorised `registry.terraform.io/hashicorp/aws` provider; `validate` remains +blocked on that provider and no plan or AWS action occurred. diff --git a/docs/remediation/evidence/phase4-migration-manifest.json b/docs/remediation/evidence/phase4-migration-manifest.json new file mode 100644 index 0000000..c3a564f --- /dev/null +++ b/docs/remediation/evidence/phase4-migration-manifest.json @@ -0,0 +1,287 @@ +{ + "activation_status": "active", + "actor": { + "id": "codex-local-remediation", + "type": "ci" + }, + "capacity": { + "available_bytes": 64959250432, + "estimated_postgres_and_indexes_bytes": 6301630464, + "estimated_wal_and_temp_bytes": 4201086976, + "minimum_free_bytes": 23340679168, + "safety_margin_bytes": 10737418240, + "source_bytes": 2100543488, + "sufficient": true + }, + "code_revision": "d6d2b69d1f9ff7dd8bc6f58021060586b3c17757", + "completed_at": "2026-07-28T23:36:05.934003+00:00", + "controls": { + "business_key_duplicate_groups": 4271, + "classified_grants": 134554, + "distinct_mapped_grants": 104191, + "duplicate_charity_number_groups": 9073, + "duplicate_source_identity_groups": 0, + "foreign_key_violations": 0, + "future_dated_grants": 1, + "missing_eur_conversions": 432, + "negative_grants": 2, + "overview_total_eur": "22435986707.70", + "zero_grants": 2101 + }, + "dataset_version": "sqlite-v7-8fc0cce61c81-r2", + "errors": [], + "idempotent_noop": false, + "migration_run_id": "60af368e-c440-5521-9648-5ab272f9ddb6", + "reconciliation_results": { + "baseline_count.charities": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 373, + "status": "pass", + "target": 373 + }, + "baseline_count.charity_registry_organizations": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 397469, + "status": "pass", + "target": 397469 + }, + "baseline_count.grants": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 302546, + "status": "pass", + "target": 302546 + }, + "baseline_count.organization_registry_links": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 345, + "status": "pass", + "target": 345 + }, + "control.business_key_duplicate_groups": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 4271, + "status": "pass", + "target": 4271 + }, + "control.classified_grants": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 134554, + "status": "pass", + "target": 134554 + }, + "control.distinct_mapped_grants": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 104191, + "status": "pass", + "target": 104191 + }, + "control.duplicate_charity_number_groups": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 9073, + "status": "pass", + "target": 9073 + }, + "control.duplicate_source_identity_groups": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 0, + "status": "pass", + "target": 0 + }, + "control.foreign_key_violations": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 0, + "status": "pass", + "target": 0 + }, + "control.future_dated_grants": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 1, + "status": "pass", + "target": 1 + }, + "control.missing_eur_conversions": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 432, + "status": "pass", + "target": 432 + }, + "control.negative_grants": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 2, + "status": "pass", + "target": 2 + }, + "control.overview_total_eur": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": "22435986707.70", + "status": "pass", + "target": "22435986707.70" + }, + "control.zero_grants": { + "approval_reference": null, + "difference": null, + "reason": null, + "source": 2101, + "status": "pass", + "target": 2101 + }, + "count.charities": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 373, + "status": "pass", + "target": 373 + }, + "count.charity_registry_organizations": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 397469, + "status": "pass", + "target": 397469 + }, + "count.exchange_rates": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 18964, + "status": "pass", + "target": 18964 + }, + "count.grant_beneficiary_countries": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 104309, + "status": "pass", + "target": 104309 + }, + "count.grant_beneficiary_terms": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 556719, + "status": "pass", + "target": 556719 + }, + "count.grant_overview_facts": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 302546, + "status": "pass", + "target": 302546 + }, + "count.grant_programme_categories": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 358883, + "status": "pass", + "target": 358883 + }, + "count.grant_source_funder_facts": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 104309, + "status": "pass", + "target": 104309 + }, + "count.grants": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 302546, + "status": "pass", + "target": 302546 + }, + "count.organization_registry_links": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 345, + "status": "pass", + "target": 345 + }, + "count.source_funder_link_overrides": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 1, + "status": "pass", + "target": 1 + }, + "count.source_funder_profile_cache": { + "approval_reference": null, + "difference": 0, + "reason": null, + "source": 1, + "status": "pass", + "target": 1 + } + }, + "rollback_dataset_version": "sqlite-v7-8fc0cce61c81", + "source_counts": { + "charities": 373, + "charity_registry_organizations": 397469, + "exchange_rates": 18964, + "grant_beneficiary_countries": 104309, + "grant_beneficiary_terms": 556719, + "grant_overview_facts": 302546, + "grant_programme_categories": 358883, + "grant_source_funder_facts": 104309, + "grants": 302546, + "organization_registry_links": 345, + "source_funder_link_overrides": 1, + "source_funder_profile_cache": 1 + }, + "source_database_checksum": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "source_fact_version": "2026-07-overview-facts-v5", + "source_schema_version": "7", + "started_at": "2026-07-28T23:30:57.020098+00:00", + "target_counts": { + "charities": 373, + "charity_registry_organizations": 397469, + "exchange_rates": 18964, + "grant_beneficiary_countries": 104309, + "grant_beneficiary_terms": 556719, + "grant_overview_facts": 302546, + "grant_programme_categories": 358883, + "grant_source_funder_facts": 104309, + "grants": 302546, + "organization_registry_links": 345, + "source_funder_link_overrides": 1, + "source_funder_profile_cache": 1 + }, + "target_schema_version": "0003_grant_award_timestamp" +} diff --git a/docs/remediation/evidence/phase4-migration-report.md b/docs/remediation/evidence/phase4-migration-report.md new file mode 100644 index 0000000..40674ff --- /dev/null +++ b/docs/remediation/evidence/phase4-migration-report.md @@ -0,0 +1,41 @@ +# Migration report: sqlite-v7-8fc0cce61c81-r2 + +- Run: `60af368e-c440-5521-9648-5ab272f9ddb6` +- Source checksum: `8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651` +- Target schema: `0003_grant_award_timestamp` +- Activation: `active` +- Rollback dataset: `sqlite-v7-8fc0cce61c81` +- Reconciliation failures: `0` + +## Counts + +| Table | Source | Target | +|---|---:|---:| +| `charities` | 373 | 373 | +| `charity_registry_organizations` | 397469 | 397469 | +| `grants` | 302546 | 302546 | +| `grant_beneficiary_countries` | 104309 | 104309 | +| `grant_beneficiary_terms` | 556719 | 556719 | +| `grant_programme_categories` | 358883 | 358883 | +| `grant_overview_facts` | 302546 | 302546 | +| `grant_source_funder_facts` | 104309 | 104309 | +| `organization_registry_links` | 345 | 345 | +| `source_funder_link_overrides` | 1 | 1 | +| `source_funder_profile_cache` | 1 | 1 | +| `exchange_rates` | 18964 | 18964 | + +## Controls + +| Control | Value | +|---|---:| +| `business_key_duplicate_groups` | 4271 | +| `classified_grants` | 134554 | +| `distinct_mapped_grants` | 104191 | +| `duplicate_charity_number_groups` | 9073 | +| `duplicate_source_identity_groups` | 0 | +| `foreign_key_violations` | 0 | +| `future_dated_grants` | 1 | +| `missing_eur_conversions` | 432 | +| `negative_grants` | 2 | +| `overview_total_eur` | 22435986707.70 | +| `zero_grants` | 2101 | diff --git a/docs/remediation/evidence/phase6-performance-report.json b/docs/remediation/evidence/phase6-performance-report.json new file mode 100644 index 0000000..009f017 --- /dev/null +++ b/docs/remediation/evidence/phase6-performance-report.json @@ -0,0 +1,147 @@ +{ + "phase": 6, + "generated_at": "2026-07-29", + "environment": { + "execution": "local loopback only", + "postgresql_version": "16.14", + "alembic_revision": "0004_versioned_analytics", + "dataset_version": "sqlite-v7-8fc0cce61c81-r2", + "pool_size": 5, + "max_overflow": 5 + }, + "targets_ms": { + "health_p95": 100, + "cached_simple_p95": 500, + "cold_dashboard_p95": 3000, + "default_map_p95": 2000, + "heavy_relationship_p95": 5000, + "registry_exact_p95": 300, + "registry_text_p95": 1000 + }, + "baseline_single_sample_ms": { + "health": 2.66, + "organization_list": 324.29, + "default_map": 2003.54, + "monthly_trends": 413.14, + "programme_themes": 1092.86, + "network_summary": 4347.61, + "primary_dashboard": 8941.12, + "primary_dashboard_concurrent_2": 16025.45, + "registry_exact": 15038.99, + "registry_text": 16841.02, + "funder_ranking": 800.02 + }, + "repository_benchmark": { + "samples_per_warm_journey": 10, + "cold_dashboard_samples": 20, + "cold_dashboard": { + "p50_ms": 157.56, + "p95_ms": 255.31, + "p99_ms": 539.8 + }, + "warm_p95_ms": { + "health": 3.23, + "organization_list": 233.03, + "default_map": 3.34, + "map_connections": 6.05, + "monthly_trends": 4.0, + "yearly_trends": 7.32, + "programme_themes": 4.83, + "network_summary": 5.33, + "primary_dashboard": 205.23, + "registry_exact": 12.43, + "registry_text": 77.03, + "funder_ranking": 13.35 + }, + "concurrent_dashboard": { + "concurrency": 5, + "elapsed_ms": 453.23, + "throughput_rps": 11.032, + "errors": 0 + }, + "cache": { + "hits": 16, + "misses": 4, + "hit_ratio": 0.8 + }, + "pool_after_run": { + "checked_in": 5, + "checked_out": 0 + } + }, + "production_mode_api": { + "samples_per_endpoint": 20, + "metrics": { + "health": {"p50_ms": 3.47, "p95_ms": 3.7, "p99_ms": 4.26, "throughput_rps": 288.92, "error_rate": 0.0}, + "organization_list": {"p50_ms": 228.99, "p95_ms": 245.73, "p99_ms": 257.12, "throughput_rps": 4.328, "error_rate": 0.0}, + "default_map": {"p50_ms": 4.57, "p95_ms": 4.95, "p99_ms": 6.44, "throughput_rps": 213.391, "error_rate": 0.0}, + "map_connections": {"p50_ms": 5.12, "p95_ms": 6.04, "p99_ms": 7.13, "throughput_rps": 188.35, "error_rate": 0.0}, + "overview": {"p50_ms": 26.41, "p95_ms": 43.45, "p99_ms": 43.77, "throughput_rps": 35.284, "error_rate": 0.0}, + "yearly_trends": {"p50_ms": 3.54, "p95_ms": 5.09, "p99_ms": 5.57, "throughput_rps": 268.95, "error_rate": 0.0}, + "registry_exact": {"p50_ms": 8.88, "p95_ms": 18.01, "p99_ms": 21.19, "throughput_rps": 101.573, "error_rate": 0.0}, + "registry_text": {"p50_ms": 77.1, "p95_ms": 83.93, "p99_ms": 103.4, "throughput_rps": 12.735, "error_rate": 0.0}, + "funder_ranking": {"p50_ms": 9.95, "p95_ms": 19.2, "p99_ms": 19.5, "throughput_rps": 89.922, "error_rate": 0.0} + }, + "cold_dashboard_elapsed_ms": 454.2, + "concurrent_dashboard": { + "concurrency": 5, + "elapsed_ms": 472.8, + "throughput_rps": 10.575, + "errors": 0 + }, + "cache": {"hits": 20, "misses": 4, "hit_ratio": 0.8333}, + "pool_after_run": {"size": 5, "overflow": 0, "checked_out": 0} + }, + "query_plan_assertions": { + "map_relation": "analytics_country_aggregates", + "map_raw_fact_relation_absent": "grant_overview_facts", + "registry_exact_index": "ix_registry_current_normalized_name", + "registry_text_index_one_of": [ + "ix_registry_search_vector", + "ix_registry_normalized_name_trgm" + ], + "method": "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)" + }, + "materialization": { + "control_row_count": 204220, + "single_active_dataset_count": 1, + "table_rows": { + "analytics_scope_totals": 10, + "analytics_country_aggregates": 241, + "analytics_country_connections": 176, + "analytics_period_aggregates": 1191, + "analytics_programme_aggregates": 68, + "analytics_entity_rankings": 187995, + "analytics_country_funder_rankings": 1155, + "analytics_funder_relationships": 13245, + "analytics_filter_values": 139 + } + }, + "gates": { + "performance_cache_plan_concurrency_timeout": "5 passed", + "performance_plus_application": "10 passed", + "application_schema_migration": "15 passed", + "normal_regression": "311 passed, 9 skipped, 8 subtests passed", + "timeout_cancellation_pool_recovery": "passed", + "heavy_query_isolation": "passed", + "all_targets": "pass" + }, + "external_activity": { + "dependency_downloads": 0, + "image_downloads": 0, + "aws_calls": 0, + "paid_or_live_api_calls": 0, + "uploads": 0, + "git_pushes": 0, + "container_rebuild": { + "result": "success", + "pull": false, + "dependency_layers": "cached", + "image_id": "sha256:cf71388a8fc83cdc32632ea2cf8ea9b7b27d4d68b164f848cd6e97b49905af8a", + "size_bytes": 354456439, + "architecture": "arm64", + "os": "linux", + "user": "10001:10001" + } + } +} diff --git a/docs/remediation/evidence/phase6-performance-report.md b/docs/remediation/evidence/phase6-performance-report.md new file mode 100644 index 0000000..234f0b8 --- /dev/null +++ b/docs/remediation/evidence/phase6-performance-report.md @@ -0,0 +1,34 @@ +# Phase 6 Performance and Concurrency Evidence + +Date: 2026-07-29 +Environment: local PostgreSQL 16.14 on loopback, Alembic `0004_versioned_analytics`, dataset `sqlite-v7-8fc0cce61c81-r2` + +## Result + +Gate 6 passes. The structured companion is `phase6-performance-report.json`. + +| Journey | Target p95 | Repository p95 | Production-mode API p95 | +|---|---:|---:|---:| +| Health | <100 ms | 3.23 ms | 3.70 ms | +| Organization list | <500 ms | 233.03 ms | 245.73 ms | +| Default map | <2,000 ms | 3.34 ms | 4.95 ms | +| Lazy map relationships | <5,000 ms | 6.05 ms | 6.04 ms | +| Primary dashboard | <3,000 ms cold | 255.31 ms cold | 454.20 ms single cold run | +| Registry exact | <300 ms | 12.43 ms | 18.01 ms | +| Registry text | <1,000 ms | 77.03 ms | 83.93 ms | + +The repository cold-dashboard distribution uses 20 independent application-cache clears: p50 157.56 ms, p95 255.31 ms and p99 539.80 ms. Five authenticated production-mode dashboards complete in 472.80 ms at 10.575 dashboards/s with zero errors. API cache hit ratio is 0.8333; the five-connection pool finishes with zero checked-out connections and zero overflow. + +## Query and isolation evidence + +`EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON)` proves that the default map reads `analytics_country_aggregates`, exact current-name registry lookup reads `ix_registry_current_normalized_name`, and ranked text search reads a stored-vector or normalized-name GIN/trigram index. Timeout/cancellation aborts `pg_sleep(2)` within the application deadline, rolls back the session and then successfully executes `SELECT 1`. A simultaneous one-second heavy query does not push five health checks beyond the 100 ms gate. + +The active materialization control row reports 204,220 rows across nine aggregate tables. Monthly and yearly periods are both present. Map connections are independent and capped at 250; funder-recipient relationships are pre-ranked to 50 and detail returns at most 25. + +## Scope and safety + +The initial one-sample baseline was intentionally diagnostic, not statistically comparable: map 2.00 s, network summary 4.35 s, dashboard 8.94 s, exact registry 15.04 s and text registry 16.84 s. Final claims use the bounded repeatable harnesses in `scripts/benchmark_postgres.py` and `scripts/load_test_api.py`. + +No dependency/image download, AWS access, paid/live API, scraper/model call, upload or push occurred. The protected SQLite and aggregate audit checksums remain at their approved baselines. + +The normal regression suite passes 311 tests, skips nine explicit live-environment tests and passes eight route subtests. The final backend rebuild succeeded with `--pull=false`; every pinned base/dependency layer was cached and no download occurred. The local arm64 image is 354,456,439 bytes, runs as `10001:10001` and has ID `sha256:cf71388a8fc83cdc32632ea2cf8ea9b7b27d4d68b164f848cd6e97b49905af8a`. diff --git a/docs/remediation/evidence/phase7-frontend-report.json b/docs/remediation/evidence/phase7-frontend-report.json new file mode 100644 index 0000000..0c224f1 --- /dev/null +++ b/docs/remediation/evidence/phase7-frontend-report.json @@ -0,0 +1,51 @@ +{ + "phase": 7, + "status": "passed", + "generated_on": "2026-07-29", + "local_checks": { + "unit_contract_tests": { "passed": 13, "failed": 0 }, + "lint_warnings": 0, + "typescript_build": "passed", + "vite_build": "passed", + "bundle_budget": "passed", + "runtime_layout_gate": "passed", + "playwright_axe_gate": { "passed": 8, "failed": 0, "skipped": 4, "axe_violations": 0 } + }, + "bundle_gzip_kib": { + "initial_javascript": 87.81, + "initial_javascript_budget": 120, + "initial_css": 18.59, + "initial_css_budget": 25, + "largest_deferred_javascript_chunk": 392.36, + "deferred_javascript_chunk_budget": 425 + }, + "runtime": { + "browser": "Google Chrome 150.0.7871.187", + "viewports": [320, 390, 768, 1024, 1440, 1920], + "page_overflow_failures": 0, + "clipped_visible_control_failures": 0, + "cropped_kpi_failures": 0, + "unnamed_visible_control_failures": 0, + "console_warning_error_events": 0, + "runtime_exceptions": 0, + "initial_overview_requests_per_navigation": 1, + "lazy_connection_requests_after_activation": 1 + }, + "named_test_dependencies": { + "@playwright/test": "1.62.0", + "playwright": "1.62.0", + "playwright-core": "1.62.0", + "@axe-core/playwright": "4.12.1", + "axe-core": "4.12.1" + }, + "approved_network_actions": [ + "Version resolution and exact locked package downloads from registry.npmjs.org" + ], + "browser_download_performed": false, + "push_performed": false, + "aws_actions_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase7-frontend-report.md b/docs/remediation/evidence/phase7-frontend-report.md new file mode 100644 index 0000000..764f3c5 --- /dev/null +++ b/docs/remediation/evidence/phase7-frontend-report.md @@ -0,0 +1,45 @@ +# Phase 7 Frontend Evidence + +## Result + +Gate 7 **passes**. The unit, lint, production build, bundle, no-dependency +runtime and named Playwright/axe gates all complete locally. + +## Verified locally + +- 13 unit/contract tests pass; Oxlint has zero warnings. +- TypeScript and the Vite production build pass. +- Initial JavaScript is 87.81 KiB gzip (120 KiB budget). +- Initial CSS is 18.59 KiB gzip (25 KiB budget). +- The largest deferred JavaScript chunk is 392.36 KiB gzip (425 KiB budget). +- Headless Chrome passes at 320, 390, 768, 1024, 1440 and 1920 pixels. +- No page overflow, clipped visible control, cropped KPI, unnamed visible + control, browser console warning/error or runtime exception was observed. +- The map precedes analytics, map controls remain inside the viewport, the + Overview request runs once, and map connections run once only after the + interaction that enables them. +- Overview and Registry drawers trap Tab, close with Escape and restore focus. +- Donor and Registry navigation and the Registry empty state are covered at + 320 and 1024 pixels. +- Playwright passes eight journeys and intentionally skips four redundant + secondary-journey viewport combinations. +- axe reports zero violations across the six Overview widths and the two + representative Donor/Registry journeys. + +The runtime gate uses the already installed Chrome through the DevTools +Protocol, a local Vite Preview process and deterministic in-browser API mocks. +It disables background networking and performs no external request. + +## Named-tool supply chain + +After explicit user approval, `@playwright/test` and its Playwright runtime were +locked at `1.62.0`; `@axe-core/playwright` and `axe-core` were locked at +`4.12.1`. Resolution and tarball access were restricted to +`registry.npmjs.org`. The lock delta contains no pre-existing dependency +upgrade. `npm ci --ignore-scripts` ran with browser download disabled, and the +suite uses the already installed Chrome 150. + +The first axe run found and drove four corrections: active-state contrast, +mobile disclosure naming, valid SVG ARIA and Overview heading order. The +repeated full run has zero axe violations. No AWS, paid/live API, browser +download, upload or push occurred. diff --git a/docs/remediation/evidence/phase8-durable-pipeline-report.json b/docs/remediation/evidence/phase8-durable-pipeline-report.json new file mode 100644 index 0000000..09e488e --- /dev/null +++ b/docs/remediation/evidence/phase8-durable-pipeline-report.json @@ -0,0 +1,51 @@ +{ + "phase": 8, + "status": "passed", + "generated_on": "2026-07-29", + "alembic_revision": "0005_durable_pipeline", + "catalog": { + "application_tables": 40, + "foreign_keys": 49, + "check_constraints": 161, + "active_dataset": "sqlite-v7-8fc0cce61c81-r2" + }, + "source_configuration": { + "configured_sources": 8, + "enabled_sources": 0, + "governance_blocked_sources": 8, + "unresolved_status_preserved": true + }, + "local_checks": { + "phase8_contract_tests": { "passed": 7, "failed": 0, "skipped": 1 }, + "phase8_postgresql_tests": { "passed": 8, "failed": 0 }, + "combined_postgresql_tests": { "passed": 18, "failed": 0 }, + "normal_backend_tests": { "passed": 318, "failed": 0, "skipped": 10 }, + "normal_backend_subtests": 8, + "migration_downgrade_upgrade": "passed", + "blocking_flake8": "passed", + "python_compile": "passed" + }, + "contracts": { + "postgresql_request_idempotency": true, + "transactional_dispatch_outbox": true, + "worker_lease_and_heartbeat": true, + "retry_and_dead_letter_state": true, + "immutable_raw_object_descriptor": true, + "append_only_ingestion_manifest": true, + "last_good_dataset_preservation": true, + "api_subprocess_execution": false, + "production_local_lock_coordination": false + }, + "aws_execution": { + "s3": false, + "sqs": false, + "dlq": false, + "eventbridge": false, + "step_functions": false + }, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase8-durable-pipeline-report.md b/docs/remediation/evidence/phase8-durable-pipeline-report.md new file mode 100644 index 0000000..93631f3 --- /dev/null +++ b/docs/remediation/evidence/phase8-durable-pipeline-report.md @@ -0,0 +1,39 @@ +# Phase 8 Durable Pipeline Evidence + +## Result + +Gate 8 **passes for local code readiness**. PostgreSQL owns job coordination, +request idempotency, worker leases, outbox delivery state, source-ingestion +state and immutable manifest/object descriptors. The API only enqueues work. + +Real AWS S3, SQS, DLQ, EventBridge and Step Functions execution is **not +tested** and is not claimed. Those actions require a separately approved +deployment. + +## Verified locally + +- Alembic `0005_durable_pipeline` upgrades, downgrades to `0004` and upgrades + again without changing the active dataset. +- A queued job has a versioned, payload-minimal dispatch envelope; repeated + enqueue returns the same job ID. +- Workers claim through PostgreSQL row locking, refresh leases/heartbeats and + transition through retry, success and dead-letter states. +- Staging/production request idempotency uses PostgreSQL rather than a local + process dictionary. +- Raw storage descriptors and ingestion manifests reject mutation; corrections + require a new version. +- Failure/retry testing leaves `sqlite-v7-8fc0cce61c81-r2` active. +- All eight source configurations are disabled and blocked while their legal + and licence states remain honestly unresolved. +- Manual refresh remains operator-only, idempotent, queued and job-ID based; + logs are redacted before both text and structured event output. + +## Tests + +The normal backend run passes 318 tests with ten explicit live-environment +skips and eight route subtests. The dedicated real PostgreSQL Phase-8 suite +passes 8/8. The combined Phase-8/application/schema run passes 18/18. Python +compile, blocking Flake8 and whitespace checks pass. + +No dependency download, AWS call, live scraper/model/news call, paid API, +upload or push occurred. diff --git a/docs/remediation/evidence/phase9-governance-report.json b/docs/remediation/evidence/phase9-governance-report.json new file mode 100644 index 0000000..8586777 --- /dev/null +++ b/docs/remediation/evidence/phase9-governance-report.json @@ -0,0 +1,55 @@ +{ + "phase": 9, + "status": "passed", + "generated_on": "2026-07-29", + "alembic_revision": "0006_governance_retention", + "catalog": { + "application_tables": 45, + "foreign_keys": 55, + "check_constraints": 189, + "retention_policies": 14, + "destructive_policies_enabled": 0, + "destructive_delete_windows": 0, + "active_holds": 0, + "persisted_deletion_manifests": 0, + "active_dataset": "sqlite-v7-8fc0cce61c81-r2" + }, + "local_checks": { + "governance_security_contract_tests": { "passed": 19, "failed": 0, "skipped": 1 }, + "governance_postgresql_tests": { "passed": 9, "failed": 0 }, + "combined_postgresql_tests": { "passed": 27, "failed": 0 }, + "normal_backend_tests": { "passed": 326, "failed": 0, "skipped": 11 }, + "normal_backend_subtests": 8, + "migration_downgrade_upgrade": "passed", + "python_compile": "passed", + "blocking_flake8": "passed" + }, + "controls": { + "explicit_field_allowlists": true, + "recursive_log_redaction": true, + "archive_dry_run_only": true, + "legal_holds": true, + "incident_holds": true, + "append_only_deletion_manifests": true, + "append_only_restore_evidence": true, + "export_expiration_report_only": true, + "data_subject_workflow": true, + "destructive_api_route": false, + "automatic_deletion": false + }, + "unresolved": [ + "business/data/privacy/legal/security owners", + "source legal and licence approvals", + "privacy and DPIA checklist", + "backup retention and cross-account copy", + "PITR retention", + "RTO", + "RPO" + ], + "aws_actions_performed": false, + "push_performed": false, + "protected_checksums": { + "sqlite_sha256": "8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651", + "docs_audits_aggregate_sha256": "d40c8b0114f8c5ef728884dd0e8632ecc6f9f03912fdf8ba709556f9ba3c1f2a" + } +} diff --git a/docs/remediation/evidence/phase9-governance-report.md b/docs/remediation/evidence/phase9-governance-report.md new file mode 100644 index 0000000..8868e95 --- /dev/null +++ b/docs/remediation/evidence/phase9-governance-report.md @@ -0,0 +1,32 @@ +# Phase 9 Governance and Retention Evidence + +## Result + +Gate 9 **passes for local code readiness**. Governance configuration, +classification/owner/source registers, field allowlists, recursive redaction, +hold-aware retention dry-runs, append-only manifests, restore evidence, export +expiry reporting and data-subject control state exist and are locally tested. + +No destructive retention is enabled. Legal/licence/privacy/owner and service +recovery decisions remain explicitly unresolved, so production remains +`NO-GO`. + +## Verified locally + +- Fourteen required classifications map to unique retention classes. +- All 14 destructive flags are false and every deletion window is unset. +- Both legal and incident holds override an otherwise due archive report. +- Archive and export actions are report-only; no target bytes/rows are changed. +- Deletion manifests and restore-verification evidence reject update/delete. +- Unknown serializer policies fail; generic admin outputs expose named fields + and recursively redact credentials/contact/personal data. +- Data-subject intake stores a hashed subject reference and begins in identity + verification state. +- Active dataset `sqlite-v7-8fc0cce61c81-r2` remains unchanged. + +The normal suite passes 326 tests, skips 11 explicit live tests and passes +eight route subtests. Dedicated PostgreSQL governance passes 9/9; combined +Phase-9/Phase-8/application/schema passes 27/27. The `0006 -> 0005 -> 0006` +migration cycle, compile, blocking Flake8, JSON and whitespace checks pass. + +No AWS, external API, paid service, download, upload or push occurred. diff --git a/docs/remediation/evidence/sbom/backend.cdx.json b/docs/remediation/evidence/sbom/backend.cdx.json new file mode 100644 index 0000000..bce5d14 --- /dev/null +++ b/docs/remediation/evidence/sbom/backend.cdx.json @@ -0,0 +1,6034 @@ +{ + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "pkg:pypi/alembic@1.18.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "06d8ba9d04558022f5395e9317de03d270f3dced49cee01f89fe7a13c26f14bc" + }, + { + "alg": "SHA-256", + "content": "1554982221dd17e9a749b53902407578eb305e453f71999e8c7f0a48389fff8e" + } + ], + "name": "alembic", + "purl": "pkg:pypi/alembic@1.18.5", + "type": "library", + "version": "1.18.5" + }, + { + "bom-ref": "pkg:pypi/annotated-doc@0.0.5", + "hashes": [ + { + "alg": "SHA-256", + "content": "117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101" + }, + { + "alg": "SHA-256", + "content": "c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb" + } + ], + "name": "annotated-doc", + "purl": "pkg:pypi/annotated-doc@0.0.5", + "type": "library", + "version": "0.0.5" + }, + { + "bom-ref": "pkg:pypi/annotated-types@0.8.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7" + }, + { + "alg": "SHA-256", + "content": "f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0" + } + ], + "name": "annotated-types", + "purl": "pkg:pypi/annotated-types@0.8.0", + "type": "library", + "version": "0.8.0" + }, + { + "bom-ref": "pkg:pypi/anthropic@0.118.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "524d835869b8e374510b3bcc552e28dbdafc035a61d63fc0088c4035286ef8fe" + }, + { + "alg": "SHA-256", + "content": "acb3c43b7e7592cf635fa21930e67a99d32641d68ccd53f625b52f71d7870d55" + } + ], + "name": "anthropic", + "purl": "pkg:pypi/anthropic@0.118.0", + "type": "library", + "version": "0.118.0" + }, + { + "bom-ref": "pkg:pypi/anyio@4.14.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494" + }, + { + "alg": "SHA-256", + "content": "cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f" + } + ], + "name": "anyio", + "purl": "pkg:pypi/anyio@4.14.2", + "type": "library", + "version": "4.14.2" + }, + { + "bom-ref": "pkg:pypi/ast-serialize@0.6.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "0067b25fce104eaae5b88383de9ab803faeb671831e14ca698b771b356e2600f" + }, + { + "alg": "SHA-256", + "content": "085de7f62dc9cc247eb01e965a362707d1d90b1d89a82c5bf78301a60a3c417b" + }, + { + "alg": "SHA-256", + "content": "093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596" + }, + { + "alg": "SHA-256", + "content": "113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e" + }, + { + "alg": "SHA-256", + "content": "13b7769970a39983b0adf2f38917b1cd3b8946f76df045756c3d741bc689f089" + }, + { + "alg": "SHA-256", + "content": "2d947e45cafc4b09bd7528917fa84c517654a43de173c79785574b7b3068ac24" + }, + { + "alg": "SHA-256", + "content": "305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a" + }, + { + "alg": "SHA-256", + "content": "32ef62ec34cf6be20ad77d4799556638fbdf187f3ae10698dfb20ef9f2c89516" + }, + { + "alg": "SHA-256", + "content": "3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a" + }, + { + "alg": "SHA-256", + "content": "4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790" + }, + { + "alg": "SHA-256", + "content": "4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e" + }, + { + "alg": "SHA-256", + "content": "5337cb256dcea3df9288205213d1601581536526b8f4da44b6974f1180f3252a" + }, + { + "alg": "SHA-256", + "content": "577a6c189068686869f5f1ddc38363f3ae1808a4753b577266f9202071a7bb66" + }, + { + "alg": "SHA-256", + "content": "6e15ec740436e1a0d62de848641abe5f3a2f89a7f94907d534795ac91bbacf14" + }, + { + "alg": "SHA-256", + "content": "6f7a408601bb3edaefb3bc67a4c01f5235e3253653b6a5729a2ee2382b35341c" + }, + { + "alg": "SHA-256", + "content": "82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1" + }, + { + "alg": "SHA-256", + "content": "8670bfa51208a2c0c8d138928e40e998fab158f9200d53bb80c088b5b8eda7b8" + }, + { + "alg": "SHA-256", + "content": "897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b" + }, + { + "alg": "SHA-256", + "content": "9f8a8b78b13173de6a9ec22111d9be674874cd5bdccda04f14ae5ebc2bef403a" + }, + { + "alg": "SHA-256", + "content": "a14191beec7e0c078d2fc1f6edc0aee88bcd4db9f18e1bc9f8052b559c22dddc" + }, + { + "alg": "SHA-256", + "content": "a4826809eb8597a8cd59fd924b6d7c285b8969a1e0007e2cb652cab62376270f" + }, + { + "alg": "SHA-256", + "content": "a7520b672827885bafeae7501f684d14d47d17e5f45256f9df547686cca52264" + }, + { + "alg": "SHA-256", + "content": "aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe" + }, + { + "alg": "SHA-256", + "content": "b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b" + }, + { + "alg": "SHA-256", + "content": "c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32" + }, + { + "alg": "SHA-256", + "content": "c617417f9cbb0cb144f6283c3cbe0d2e0f01beaf9f608f662b21191058a626ec" + }, + { + "alg": "SHA-256", + "content": "c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081" + }, + { + "alg": "SHA-256", + "content": "c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b" + }, + { + "alg": "SHA-256", + "content": "ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e" + }, + { + "alg": "SHA-256", + "content": "cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77" + }, + { + "alg": "SHA-256", + "content": "cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b" + }, + { + "alg": "SHA-256", + "content": "dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7" + }, + { + "alg": "SHA-256", + "content": "e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad" + }, + { + "alg": "SHA-256", + "content": "f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554" + } + ], + "name": "ast-serialize", + "purl": "pkg:pypi/ast-serialize@0.6.0", + "type": "library", + "version": "0.6.0" + }, + { + "bom-ref": "pkg:pypi/asyncpg@0.31.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8" + }, + { + "alg": "SHA-256", + "content": "04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be" + }, + { + "alg": "SHA-256", + "content": "0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be" + }, + { + "alg": "SHA-256", + "content": "0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2" + }, + { + "alg": "SHA-256", + "content": "0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d" + }, + { + "alg": "SHA-256", + "content": "12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a" + }, + { + "alg": "SHA-256", + "content": "18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7" + }, + { + "alg": "SHA-256", + "content": "19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218" + }, + { + "alg": "SHA-256", + "content": "1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d" + }, + { + "alg": "SHA-256", + "content": "22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602" + }, + { + "alg": "SHA-256", + "content": "22be6e02381bab3101cd502d9297ac71e2f966c86e20e78caead9934c98a8af6" + }, + { + "alg": "SHA-256", + "content": "2657204552b75f8288de08ca60faf4a99a65deef3a71d1467454123205a88fab" + }, + { + "alg": "SHA-256", + "content": "2d076d42eb583601179efa246c5d7ae44614b4144bc1c7a683ad1222814ed095" + }, + { + "alg": "SHA-256", + "content": "334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5" + }, + { + "alg": "SHA-256", + "content": "37a58919cfef2448a920df00d1b2f821762d17194d0dbf355d6dde8d952c04f9" + }, + { + "alg": "SHA-256", + "content": "37fc6c00a814e18eef51833545d1891cac9aa69140598bb076b4cd29b3e010b9" + }, + { + "alg": "SHA-256", + "content": "3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c" + }, + { + "alg": "SHA-256", + "content": "3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec" + }, + { + "alg": "SHA-256", + "content": "3faa62f997db0c9add34504a68ac2c342cfee4d57a0c3062fcf0d86c7f9cb1e8" + }, + { + "alg": "SHA-256", + "content": "480c4befbdf079c14c9ca43c8c5e1fe8b6296c96f1f927158d4f1e750aacc047" + }, + { + "alg": "SHA-256", + "content": "54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e" + }, + { + "alg": "SHA-256", + "content": "5a4af56edf82a701aece93190cc4e094d2df7d33f6e915c222fb09efbb5afc24" + }, + { + "alg": "SHA-256", + "content": "6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31" + }, + { + "alg": "SHA-256", + "content": "72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186" + }, + { + "alg": "SHA-256", + "content": "795416369c3d284e1837461909f58418ad22b305f955e625a4b3a2521d80a5f3" + }, + { + "alg": "SHA-256", + "content": "831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61" + }, + { + "alg": "SHA-256", + "content": "8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a" + }, + { + "alg": "SHA-256", + "content": "8ea599d45c361dfbf398cb67da7fd052affa556a401482d3ff1ee99bd68808a1" + }, + { + "alg": "SHA-256", + "content": "9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2" + }, + { + "alg": "SHA-256", + "content": "98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2" + }, + { + "alg": "SHA-256", + "content": "9ea33213ac044171f4cac23740bed9a3805abae10e7025314cfbd725ec670540" + }, + { + "alg": "SHA-256", + "content": "a429e842a3a4b4ea240ea52d7fe3f82d5149853249306f7ff166cb9948faa46c" + }, + { + "alg": "SHA-256", + "content": "a8d758dac9d2e723e173d286ef5e574f0b350ec00e9186fce84d0fc5f6a8e6b8" + }, + { + "alg": "SHA-256", + "content": "aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671" + }, + { + "alg": "SHA-256", + "content": "b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad" + }, + { + "alg": "SHA-256", + "content": "ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d" + }, + { + "alg": "SHA-256", + "content": "bb223567dea5f47c45d347f2bde5486be8d9f40339f27217adb3fb1c3be51298" + }, + { + "alg": "SHA-256", + "content": "bc2b685f400ceae428f79f78b58110470d7b4466929a7f78d455964b17ad1008" + }, + { + "alg": "SHA-256", + "content": "bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3" + }, + { + "alg": "SHA-256", + "content": "bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20" + }, + { + "alg": "SHA-256", + "content": "bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2" + }, + { + "alg": "SHA-256", + "content": "bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4" + }, + { + "alg": "SHA-256", + "content": "c0807be46c32c963ae40d329b3a686356e417f674c976c07fa49f1b30303f109" + }, + { + "alg": "SHA-256", + "content": "c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403" + }, + { + "alg": "SHA-256", + "content": "c1a9c5b71d2371a2290bc93336cd05ba4ec781683cab292adbddc084f89443c6" + }, + { + "alg": "SHA-256", + "content": "c1e1ab5bc65373d92dd749d7308c5b26fb2dc0fbe5d3bf68a32b676aa3bcd24a" + }, + { + "alg": "SHA-256", + "content": "c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b" + }, + { + "alg": "SHA-256", + "content": "c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735" + }, + { + "alg": "SHA-256", + "content": "cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b" + }, + { + "alg": "SHA-256", + "content": "dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab" + }, + { + "alg": "SHA-256", + "content": "e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e" + }, + { + "alg": "SHA-256", + "content": "e5d5098f63beeae93512ee513d4c0c53dc12e9aa2b7a1af5a81cddf93fe4e4da" + }, + { + "alg": "SHA-256", + "content": "e6974f36eb9a224d8fb428bcf66bd411aa12cf57c2967463178149e73d4de366" + }, + { + "alg": "SHA-256", + "content": "ebb3cde58321a1f89ce41812be3f2a98dddedc1e76d0838aba1d724f1e4e1a95" + }, + { + "alg": "SHA-256", + "content": "eee690960e8ab85063ba93af2ce128c0f52fd655fdff9fdb1a28df01329f031d" + }, + { + "alg": "SHA-256", + "content": "f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44" + }, + { + "alg": "SHA-256", + "content": "f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696" + } + ], + "name": "asyncpg", + "purl": "pkg:pypi/asyncpg@0.31.0", + "type": "library", + "version": "0.31.0" + }, + { + "bom-ref": "pkg:pypi/beautifulsoup4@4.15.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7" + }, + { + "alg": "SHA-256", + "content": "d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9" + } + ], + "name": "beautifulsoup4", + "purl": "pkg:pypi/beautifulsoup4@4.15.0", + "type": "library", + "version": "4.15.0" + }, + { + "bom-ref": "pkg:pypi/certifi@2026.7.22", + "hashes": [ + { + "alg": "SHA-256", + "content": "62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775" + }, + { + "alg": "SHA-256", + "content": "741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55" + } + ], + "name": "certifi", + "purl": "pkg:pypi/certifi@2026.7.22", + "type": "library", + "version": "2026.7.22" + }, + { + "bom-ref": "pkg:pypi/cffi@2.1.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc" + }, + { + "alg": "SHA-256", + "content": "03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd" + }, + { + "alg": "SHA-256", + "content": "0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d" + }, + { + "alg": "SHA-256", + "content": "0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5" + }, + { + "alg": "SHA-256", + "content": "0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f" + }, + { + "alg": "SHA-256", + "content": "0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6" + }, + { + "alg": "SHA-256", + "content": "0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c" + }, + { + "alg": "SHA-256", + "content": "10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda" + }, + { + "alg": "SHA-256", + "content": "11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd" + }, + { + "alg": "SHA-256", + "content": "15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a" + }, + { + "alg": "SHA-256", + "content": "164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd" + }, + { + "alg": "SHA-256", + "content": "1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd" + }, + { + "alg": "SHA-256", + "content": "19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3" + }, + { + "alg": "SHA-256", + "content": "1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb" + }, + { + "alg": "SHA-256", + "content": "1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66" + }, + { + "alg": "SHA-256", + "content": "1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d" + }, + { + "alg": "SHA-256", + "content": "2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f" + }, + { + "alg": "SHA-256", + "content": "276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6" + }, + { + "alg": "SHA-256", + "content": "2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0" + }, + { + "alg": "SHA-256", + "content": "2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c" + }, + { + "alg": "SHA-256", + "content": "30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93" + }, + { + "alg": "SHA-256", + "content": "33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d" + }, + { + "alg": "SHA-256", + "content": "35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d" + }, + { + "alg": "SHA-256", + "content": "3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8" + }, + { + "alg": "SHA-256", + "content": "379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b" + }, + { + "alg": "SHA-256", + "content": "37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001" + }, + { + "alg": "SHA-256", + "content": "3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d" + }, + { + "alg": "SHA-256", + "content": "3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43" + }, + { + "alg": "SHA-256", + "content": "46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b" + }, + { + "alg": "SHA-256", + "content": "47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0" + }, + { + "alg": "SHA-256", + "content": "4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0" + }, + { + "alg": "SHA-256", + "content": "4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458" + }, + { + "alg": "SHA-256", + "content": "510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8" + }, + { + "alg": "SHA-256", + "content": "5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d" + }, + { + "alg": "SHA-256", + "content": "5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94" + }, + { + "alg": "SHA-256", + "content": "6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022" + }, + { + "alg": "SHA-256", + "content": "63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db" + }, + { + "alg": "SHA-256", + "content": "64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479" + }, + { + "alg": "SHA-256", + "content": "6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376" + }, + { + "alg": "SHA-256", + "content": "6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d" + }, + { + "alg": "SHA-256", + "content": "6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6" + }, + { + "alg": "SHA-256", + "content": "702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3" + }, + { + "alg": "SHA-256", + "content": "716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea" + }, + { + "alg": "SHA-256", + "content": "762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd" + }, + { + "alg": "SHA-256", + "content": "7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02" + }, + { + "alg": "SHA-256", + "content": "78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde" + }, + { + "alg": "SHA-256", + "content": "799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224" + }, + { + "alg": "SHA-256", + "content": "7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76" + }, + { + "alg": "SHA-256", + "content": "7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804" + }, + { + "alg": "SHA-256", + "content": "7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1" + }, + { + "alg": "SHA-256", + "content": "7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913" + }, + { + "alg": "SHA-256", + "content": "7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714" + }, + { + "alg": "SHA-256", + "content": "86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc" + }, + { + "alg": "SHA-256", + "content": "88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2" + }, + { + "alg": "SHA-256", + "content": "89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e" + }, + { + "alg": "SHA-256", + "content": "8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda" + }, + { + "alg": "SHA-256", + "content": "8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512" + }, + { + "alg": "SHA-256", + "content": "8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28" + }, + { + "alg": "SHA-256", + "content": "90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699" + }, + { + "alg": "SHA-256", + "content": "95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3" + }, + { + "alg": "SHA-256", + "content": "961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c" + }, + { + "alg": "SHA-256", + "content": "98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe" + }, + { + "alg": "SHA-256", + "content": "9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a" + }, + { + "alg": "SHA-256", + "content": "9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f" + }, + { + "alg": "SHA-256", + "content": "9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c" + }, + { + "alg": "SHA-256", + "content": "a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2" + }, + { + "alg": "SHA-256", + "content": "a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f" + }, + { + "alg": "SHA-256", + "content": "a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b" + }, + { + "alg": "SHA-256", + "content": "aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565" + }, + { + "alg": "SHA-256", + "content": "ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056" + }, + { + "alg": "SHA-256", + "content": "af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629" + }, + { + "alg": "SHA-256", + "content": "b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7" + }, + { + "alg": "SHA-256", + "content": "b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0" + }, + { + "alg": "SHA-256", + "content": "ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9" + }, + { + "alg": "SHA-256", + "content": "bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853" + }, + { + "alg": "SHA-256", + "content": "bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13" + }, + { + "alg": "SHA-256", + "content": "bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a" + }, + { + "alg": "SHA-256", + "content": "c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4" + }, + { + "alg": "SHA-256", + "content": "c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce" + }, + { + "alg": "SHA-256", + "content": "c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac" + }, + { + "alg": "SHA-256", + "content": "c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c" + }, + { + "alg": "SHA-256", + "content": "c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46" + }, + { + "alg": "SHA-256", + "content": "c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384" + }, + { + "alg": "SHA-256", + "content": "cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b" + }, + { + "alg": "SHA-256", + "content": "cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210" + }, + { + "alg": "SHA-256", + "content": "cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc" + }, + { + "alg": "SHA-256", + "content": "d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a" + }, + { + "alg": "SHA-256", + "content": "d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5" + }, + { + "alg": "SHA-256", + "content": "d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7" + }, + { + "alg": "SHA-256", + "content": "db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2" + }, + { + "alg": "SHA-256", + "content": "dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326" + }, + { + "alg": "SHA-256", + "content": "df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f" + }, + { + "alg": "SHA-256", + "content": "df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca" + }, + { + "alg": "SHA-256", + "content": "eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98" + }, + { + "alg": "SHA-256", + "content": "efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9" + }, + { + "alg": "SHA-256", + "content": "f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5" + }, + { + "alg": "SHA-256", + "content": "f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7" + }, + { + "alg": "SHA-256", + "content": "f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc" + }, + { + "alg": "SHA-256", + "content": "fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da" + }, + { + "alg": "SHA-256", + "content": "ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f" + } + ], + "name": "cffi", + "purl": "pkg:pypi/cffi@2.1.0", + "type": "library", + "version": "2.1.0" + }, + { + "bom-ref": "pkg:pypi/charset-normalizer@3.4.9", + "hashes": [ + { + "alg": "SHA-256", + "content": "0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380" + }, + { + "alg": "SHA-256", + "content": "03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62" + }, + { + "alg": "SHA-256", + "content": "04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c" + }, + { + "alg": "SHA-256", + "content": "0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226" + }, + { + "alg": "SHA-256", + "content": "0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5" + }, + { + "alg": "SHA-256", + "content": "0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833" + }, + { + "alg": "SHA-256", + "content": "16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b" + }, + { + "alg": "SHA-256", + "content": "16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99" + }, + { + "alg": "SHA-256", + "content": "19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501" + }, + { + "alg": "SHA-256", + "content": "1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec" + }, + { + "alg": "SHA-256", + "content": "21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698" + }, + { + "alg": "SHA-256", + "content": "231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4" + }, + { + "alg": "SHA-256", + "content": "253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a" + }, + { + "alg": "SHA-256", + "content": "280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3" + }, + { + "alg": "SHA-256", + "content": "2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2" + }, + { + "alg": "SHA-256", + "content": "304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a" + }, + { + "alg": "SHA-256", + "content": "32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e" + }, + { + "alg": "SHA-256", + "content": "33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4" + }, + { + "alg": "SHA-256", + "content": "375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419" + }, + { + "alg": "SHA-256", + "content": "3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84" + }, + { + "alg": "SHA-256", + "content": "3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da" + }, + { + "alg": "SHA-256", + "content": "40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519" + }, + { + "alg": "SHA-256", + "content": "416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe" + }, + { + "alg": "SHA-256", + "content": "432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381" + }, + { + "alg": "SHA-256", + "content": "43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29" + }, + { + "alg": "SHA-256", + "content": "440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614" + }, + { + "alg": "SHA-256", + "content": "45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0" + }, + { + "alg": "SHA-256", + "content": "476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe" + }, + { + "alg": "SHA-256", + "content": "4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29" + }, + { + "alg": "SHA-256", + "content": "4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0" + }, + { + "alg": "SHA-256", + "content": "4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917" + }, + { + "alg": "SHA-256", + "content": "51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9" + }, + { + "alg": "SHA-256", + "content": "51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32" + }, + { + "alg": "SHA-256", + "content": "58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94" + }, + { + "alg": "SHA-256", + "content": "5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63" + }, + { + "alg": "SHA-256", + "content": "5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd" + }, + { + "alg": "SHA-256", + "content": "609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198" + }, + { + "alg": "SHA-256", + "content": "60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde" + }, + { + "alg": "SHA-256", + "content": "611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012" + }, + { + "alg": "SHA-256", + "content": "6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1" + }, + { + "alg": "SHA-256", + "content": "65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15" + }, + { + "alg": "SHA-256", + "content": "673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b" + }, + { + "alg": "SHA-256", + "content": "67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993" + }, + { + "alg": "SHA-256", + "content": "68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4" + }, + { + "alg": "SHA-256", + "content": "68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5" + }, + { + "alg": "SHA-256", + "content": "69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8" + }, + { + "alg": "SHA-256", + "content": "75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35" + }, + { + "alg": "SHA-256", + "content": "78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642" + }, + { + "alg": "SHA-256", + "content": "78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2" + }, + { + "alg": "SHA-256", + "content": "79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d" + }, + { + "alg": "SHA-256", + "content": "7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9" + }, + { + "alg": "SHA-256", + "content": "83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c" + }, + { + "alg": "SHA-256", + "content": "84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33" + }, + { + "alg": "SHA-256", + "content": "871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db" + }, + { + "alg": "SHA-256", + "content": "898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf" + }, + { + "alg": "SHA-256", + "content": "8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9" + }, + { + "alg": "SHA-256", + "content": "8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee" + }, + { + "alg": "SHA-256", + "content": "90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84" + }, + { + "alg": "SHA-256", + "content": "9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44" + }, + { + "alg": "SHA-256", + "content": "920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f" + }, + { + "alg": "SHA-256", + "content": "93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9" + }, + { + "alg": "SHA-256", + "content": "9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9" + }, + { + "alg": "SHA-256", + "content": "9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177" + }, + { + "alg": "SHA-256", + "content": "9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8" + }, + { + "alg": "SHA-256", + "content": "9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b" + }, + { + "alg": "SHA-256", + "content": "a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39" + }, + { + "alg": "SHA-256", + "content": "a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41" + }, + { + "alg": "SHA-256", + "content": "a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0" + }, + { + "alg": "SHA-256", + "content": "aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616" + }, + { + "alg": "SHA-256", + "content": "ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d" + }, + { + "alg": "SHA-256", + "content": "ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba" + }, + { + "alg": "SHA-256", + "content": "b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2" + }, + { + "alg": "SHA-256", + "content": "bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b" + }, + { + "alg": "SHA-256", + "content": "bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9" + }, + { + "alg": "SHA-256", + "content": "c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b" + }, + { + "alg": "SHA-256", + "content": "c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209" + }, + { + "alg": "SHA-256", + "content": "c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48" + }, + { + "alg": "SHA-256", + "content": "c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046" + }, + { + "alg": "SHA-256", + "content": "cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632" + }, + { + "alg": "SHA-256", + "content": "cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a" + }, + { + "alg": "SHA-256", + "content": "cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2" + }, + { + "alg": "SHA-256", + "content": "d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1" + }, + { + "alg": "SHA-256", + "content": "ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b" + }, + { + "alg": "SHA-256", + "content": "df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990" + }, + { + "alg": "SHA-256", + "content": "df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5" + }, + { + "alg": "SHA-256", + "content": "e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b" + }, + { + "alg": "SHA-256", + "content": "e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9" + }, + { + "alg": "SHA-256", + "content": "ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534" + }, + { + "alg": "SHA-256", + "content": "f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81" + }, + { + "alg": "SHA-256", + "content": "f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a" + }, + { + "alg": "SHA-256", + "content": "f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d" + }, + { + "alg": "SHA-256", + "content": "fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf" + }, + { + "alg": "SHA-256", + "content": "fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115" + } + ], + "name": "charset-normalizer", + "purl": "pkg:pypi/charset-normalizer@3.4.9", + "type": "library", + "version": "3.4.9" + }, + { + "bom-ref": "pkg:pypi/click@8.4.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6" + }, + { + "alg": "SHA-256", + "content": "e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76" + } + ], + "name": "click", + "purl": "pkg:pypi/click@8.4.2", + "type": "library", + "version": "8.4.2" + }, + { + "bom-ref": "pkg:pypi/coverage@7.15.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2" + }, + { + "alg": "SHA-256", + "content": "0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e" + }, + { + "alg": "SHA-256", + "content": "094dd37f3ef7b2da8b068b583d1f4c40f91c65197e16c52a71962d5d537fc5db" + }, + { + "alg": "SHA-256", + "content": "09f5c6ec5901f667bd97dd140b5b9a2586b10efec66f46fb1e6d8135f8b95bdf" + }, + { + "alg": "SHA-256", + "content": "0e55510bc98ae943cece9e667a6c0fe94c6a92913720dea34243657a17993d0c" + }, + { + "alg": "SHA-256", + "content": "1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8" + }, + { + "alg": "SHA-256", + "content": "1268ac8fb9ddcd783d3948dbabaf80a5d53bfdaa0575e873e2139a692f797443" + }, + { + "alg": "SHA-256", + "content": "1473b3ba8e7ee0f076117b1a72c23f579a2b9e2bb742f48a8d86ea27ca93f91a" + }, + { + "alg": "SHA-256", + "content": "17c432b5f73ad52ef46fb06019f6fa7c66ce381961cf0f7dfd1d3a4bd3a98145" + }, + { + "alg": "SHA-256", + "content": "1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9" + }, + { + "alg": "SHA-256", + "content": "1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2" + }, + { + "alg": "SHA-256", + "content": "1d16e3a7104ea84f03e614611b3edbf6fb6892554b3ab0fe7fbb3f2b2ef04376" + }, + { + "alg": "SHA-256", + "content": "25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138" + }, + { + "alg": "SHA-256", + "content": "2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578" + }, + { + "alg": "SHA-256", + "content": "26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c" + }, + { + "alg": "SHA-256", + "content": "29c052f7c83ccfcc5c577eaae025d2e4a9bb80daf03c0ac31c996e83b000ce88" + }, + { + "alg": "SHA-256", + "content": "2f1ec6f304b156669cfde653b4e9a953f5de87e247ea02ac599bce0ab2744036" + }, + { + "alg": "SHA-256", + "content": "2fbeeeecea279727f8ac16c8e1133ddfeee793e985c86ae343d6a5ce744eef8c" + }, + { + "alg": "SHA-256", + "content": "2ff08701be2d1556fc78b326c80a3e8042da09352ecb3819105f8e386c8a3071" + }, + { + "alg": "SHA-256", + "content": "38c9518b7103826c403a461544e3c2e77151e8676d06eaed85911a97e962584a" + }, + { + "alg": "SHA-256", + "content": "3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d" + }, + { + "alg": "SHA-256", + "content": "3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b" + }, + { + "alg": "SHA-256", + "content": "40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a" + }, + { + "alg": "SHA-256", + "content": "42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b" + }, + { + "alg": "SHA-256", + "content": "434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050" + }, + { + "alg": "SHA-256", + "content": "44826758cfe73fcd0e6af5deb4ba6d5417cc1d13df3acb35c93484a11160f846" + }, + { + "alg": "SHA-256", + "content": "4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d" + }, + { + "alg": "SHA-256", + "content": "48ccc6395958eda89093ecdc35644c86f23a8b23a7f4d44958812b721aad67c1" + }, + { + "alg": "SHA-256", + "content": "4d3361879d736f469f45723c11ea1a5bbdaf1f6928f0e632c940378b5aa9b660" + }, + { + "alg": "SHA-256", + "content": "582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40" + }, + { + "alg": "SHA-256", + "content": "63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026" + }, + { + "alg": "SHA-256", + "content": "67d7602480a47bdf5b675635403625553ebaa70d5a62a657c035149fd401cea0" + }, + { + "alg": "SHA-256", + "content": "68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b" + }, + { + "alg": "SHA-256", + "content": "6aa28cfb6488e5453b5b762d65f73aa586380f6693a04d58078ce228a29b06c0" + }, + { + "alg": "SHA-256", + "content": "6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa" + }, + { + "alg": "SHA-256", + "content": "6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d" + }, + { + "alg": "SHA-256", + "content": "6fc448c377d6eeb00a47c673494bd9bae29280ca53987e1869e67ebedfe20658" + }, + { + "alg": "SHA-256", + "content": "728a33676d4c3f0db977990a4bd421dcaa3be3e53b5b6273036fff6666008e89" + }, + { + "alg": "SHA-256", + "content": "7466cc7ab6dc0db871d264bf99e8779f0917ee63d40730af0552f71535a6e072" + }, + { + "alg": "SHA-256", + "content": "77f091ea3a9cc611cd29f433565476bc1936c084ac8eee00ea0e7e70c27e4199" + }, + { + "alg": "SHA-256", + "content": "77f0ef5011df53a4bd1b35211ab122287f8d9b8d7aa1c4553e5c2deb24b1d446" + }, + { + "alg": "SHA-256", + "content": "7c63387e21ab21f512c69c9756a8c7dadd322c7275edb064064433c9a09c3743" + }, + { + "alg": "SHA-256", + "content": "7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7" + }, + { + "alg": "SHA-256", + "content": "7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1" + }, + { + "alg": "SHA-256", + "content": "7e8f27131dc7cd53de2c137dd207b3720919320b3c20d499dc30aa9ee6173287" + }, + { + "alg": "SHA-256", + "content": "81f382c5a94b434ec1f6da607edb904c76d7212e618cd4d1bc9f97bed4120ef5" + }, + { + "alg": "SHA-256", + "content": "835ec4e20b45f0a7f63ed78f94065aca00de033403df8377bfe8b9c6abc0a7be" + }, + { + "alg": "SHA-256", + "content": "8bb9f4b4279187560796a4cdaca3b0a93dd97e48ee667df005f4ed9a97403688" + }, + { + "alg": "SHA-256", + "content": "8c726b232659cbd2ae57ade46509eb068c9bd7a06df9fcbff6fe484870006934" + }, + { + "alg": "SHA-256", + "content": "913b6c56e110da40e035bbd168353bf7aaa2544a5eaccea5d98a4629aac156c7" + }, + { + "alg": "SHA-256", + "content": "97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440" + }, + { + "alg": "SHA-256", + "content": "9854ca62c152874b2060772503535be2e8f53f70b8aaa7686b094888d872f984" + }, + { + "alg": "SHA-256", + "content": "9911f31aad8906abe337c271343485cf20df5e70df5d2f57f9f136e7b55f26bc" + }, + { + "alg": "SHA-256", + "content": "9b5bd92ff1ec22e535eab0de75fa6db021992791f461a2aceb7822c625a1187d" + }, + { + "alg": "SHA-256", + "content": "9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098" + }, + { + "alg": "SHA-256", + "content": "9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6" + }, + { + "alg": "SHA-256", + "content": "9f4432898c4bf2fba0435bbe35dd4437d7264565e5a88a21f5b49d8662a6b629" + }, + { + "alg": "SHA-256", + "content": "a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b" + }, + { + "alg": "SHA-256", + "content": "a164b50081fc7357331c4024ef4d17b78ba325f8380d05f5a69599a7e05257ee" + }, + { + "alg": "SHA-256", + "content": "a29ec5305a7335aacee2d799e3422e91e1c8a12474986e2b3b07e315c91be82f" + }, + { + "alg": "SHA-256", + "content": "a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1" + }, + { + "alg": "SHA-256", + "content": "a4c46b247b5d4b78f613bd89fea926d32b25c6cc61a50bd1e99ba310348f3dad" + }, + { + "alg": "SHA-256", + "content": "a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3" + }, + { + "alg": "SHA-256", + "content": "a63b9e190711134d581c4d703df5df09851b1acf99792c7aacbbe9f41f0283c9" + }, + { + "alg": "SHA-256", + "content": "aaccad4129d735a8a4d526f26929894c9a4e8ef7034566f210b176749d6906e3" + }, + { + "alg": "SHA-256", + "content": "ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a" + }, + { + "alg": "SHA-256", + "content": "afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296" + }, + { + "alg": "SHA-256", + "content": "affd532502d34c0472d0cdb181325c89f1d2c44992fef0c17e88e7b1576259a1" + }, + { + "alg": "SHA-256", + "content": "b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd" + }, + { + "alg": "SHA-256", + "content": "b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73" + }, + { + "alg": "SHA-256", + "content": "b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f" + }, + { + "alg": "SHA-256", + "content": "bbc808daf4f5cd567af8075ecc72d21c6dfef9a254709a621a84c217c935ebc0" + }, + { + "alg": "SHA-256", + "content": "bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6" + }, + { + "alg": "SHA-256", + "content": "bcc0aae933921d03096f53b0b03eeb702129fd406dee59f08d2efacc68681fa5" + }, + { + "alg": "SHA-256", + "content": "bfd341ccf78128e72c094bc70cc25b3ef309c33c7c2c66ba3ed4309549e02de1" + }, + { + "alg": "SHA-256", + "content": "c6a98d698f9e2c8008d0370ec7fc452ebfcc530002ae2d0061170d768b992589" + }, + { + "alg": "SHA-256", + "content": "cb0fddaa6884be6aae36ced9544b5e90f7d5f03845a2853bf47a14953a4e8688" + }, + { + "alg": "SHA-256", + "content": "cee0f89f4767a6057c8fbf168f8135f18be651300496086bd873e3189fed0487" + }, + { + "alg": "SHA-256", + "content": "d17d7512151fedfcc64c1821a8977fc9be0dbf495754669afcab7b57abc98ae9" + }, + { + "alg": "SHA-256", + "content": "d46e62cb35d91e6e2589fda6d28074426b0e276422b5d2ebef2c6b11dc60dbfd" + }, + { + "alg": "SHA-256", + "content": "d50dd325e18ec25bfcc10cd7f99b04df1ab9ec76b0918c260e60817ad0643dee" + }, + { + "alg": "SHA-256", + "content": "db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d" + }, + { + "alg": "SHA-256", + "content": "dee88b1ed88587abd8c0269a1fc1f4cc77f7750d1dfde2869e2a123af420e67d" + }, + { + "alg": "SHA-256", + "content": "dfd3db045e95960ae3683059571e597fda7cc610106a8916f77c5839048c1deb" + }, + { + "alg": "SHA-256", + "content": "e26ff680768b8095e8874aabe0e9d3a47a2a9f176a8340d05f8604c56457c23a" + }, + { + "alg": "SHA-256", + "content": "e370c12133095ff18432de8c044962be85a5a96d90c6fcbce8e17e76236d2328" + }, + { + "alg": "SHA-256", + "content": "e38def96ad59853824c97953fdcd2c320a84ba3ce99b417db78af8bb6c3db635" + }, + { + "alg": "SHA-256", + "content": "e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188" + }, + { + "alg": "SHA-256", + "content": "eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c" + }, + { + "alg": "SHA-256", + "content": "f653e5d7248c1191ec988a85c72edeab46c3ff44f90639a4ed4874ec0be90243" + }, + { + "alg": "SHA-256", + "content": "fe41909c9515c3bfdb5f02c4d1f857dba322d9a9a1178069b91eea77889df63a" + } + ], + "name": "coverage", + "purl": "pkg:pypi/coverage@7.15.2", + "type": "library", + "version": "7.15.2" + }, + { + "bom-ref": "pkg:pypi/cryptography@49.0.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001" + }, + { + "alg": "SHA-256", + "content": "07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122" + }, + { + "alg": "SHA-256", + "content": "084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6" + }, + { + "alg": "SHA-256", + "content": "0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c" + }, + { + "alg": "SHA-256", + "content": "0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325" + }, + { + "alg": "SHA-256", + "content": "0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69" + }, + { + "alg": "SHA-256", + "content": "196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d" + }, + { + "alg": "SHA-256", + "content": "2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36" + }, + { + "alg": "SHA-256", + "content": "28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc" + }, + { + "alg": "SHA-256", + "content": "2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6" + }, + { + "alg": "SHA-256", + "content": "2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b" + }, + { + "alg": "SHA-256", + "content": "32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27" + }, + { + "alg": "SHA-256", + "content": "33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61" + }, + { + "alg": "SHA-256", + "content": "35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18" + }, + { + "alg": "SHA-256", + "content": "36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db" + }, + { + "alg": "SHA-256", + "content": "42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b" + }, + { + "alg": "SHA-256", + "content": "4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb" + }, + { + "alg": "SHA-256", + "content": "53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2" + }, + { + "alg": "SHA-256", + "content": "66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459" + }, + { + "alg": "SHA-256", + "content": "67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e" + }, + { + "alg": "SHA-256", + "content": "6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21" + }, + { + "alg": "SHA-256", + "content": "6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8" + }, + { + "alg": "SHA-256", + "content": "73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7" + }, + { + "alg": "SHA-256", + "content": "7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa" + }, + { + "alg": "SHA-256", + "content": "8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9" + }, + { + "alg": "SHA-256", + "content": "966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db" + }, + { + "alg": "SHA-256", + "content": "9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64" + }, + { + "alg": "SHA-256", + "content": "b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505" + }, + { + "alg": "SHA-256", + "content": "b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5" + }, + { + "alg": "SHA-256", + "content": "b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615" + }, + { + "alg": "SHA-256", + "content": "b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f" + }, + { + "alg": "SHA-256", + "content": "b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866" + }, + { + "alg": "SHA-256", + "content": "be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6" + }, + { + "alg": "SHA-256", + "content": "c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561" + }, + { + "alg": "SHA-256", + "content": "c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838" + }, + { + "alg": "SHA-256", + "content": "cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9" + }, + { + "alg": "SHA-256", + "content": "ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7" + }, + { + "alg": "SHA-256", + "content": "d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68" + }, + { + "alg": "SHA-256", + "content": "d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8" + }, + { + "alg": "SHA-256", + "content": "e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3" + }, + { + "alg": "SHA-256", + "content": "e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e" + }, + { + "alg": "SHA-256", + "content": "ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a" + }, + { + "alg": "SHA-256", + "content": "f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d" + }, + { + "alg": "SHA-256", + "content": "f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4" + }, + { + "alg": "SHA-256", + "content": "f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493" + }, + { + "alg": "SHA-256", + "content": "fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b" + } + ], + "name": "cryptography", + "purl": "pkg:pypi/cryptography@49.0.0", + "type": "library", + "version": "49.0.0" + }, + { + "bom-ref": "pkg:pypi/distro@1.9.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed" + }, + { + "alg": "SHA-256", + "content": "7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2" + } + ], + "name": "distro", + "purl": "pkg:pypi/distro@1.9.0", + "type": "library", + "version": "1.9.0" + }, + { + "bom-ref": "pkg:pypi/docstring-parser@0.18.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015" + }, + { + "alg": "SHA-256", + "content": "b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b" + } + ], + "name": "docstring-parser", + "purl": "pkg:pypi/docstring-parser@0.18.0", + "type": "library", + "version": "0.18.0" + }, + { + "bom-ref": "pkg:pypi/ecdsa@0.19.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "62635b0ac1ca2e027f82122b5b81cb706edc38cd91c63dda28e4f3455a2bf930" + }, + { + "alg": "SHA-256", + "content": "840f5dc5e375c68f36c1a7a5b9caad28f95daa65185c9253c0c08dd952bb7399" + } + ], + "name": "ecdsa", + "purl": "pkg:pypi/ecdsa@0.19.2", + "type": "library", + "version": "0.19.2" + }, + { + "bom-ref": "pkg:pypi/fastapi@0.139.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e" + }, + { + "alg": "SHA-256", + "content": "b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c" + } + ], + "name": "fastapi", + "purl": "pkg:pypi/fastapi@0.139.2", + "type": "library", + "version": "0.139.2" + }, + { + "bom-ref": "pkg:pypi/feedparser@6.0.12", + "hashes": [ + { + "alg": "SHA-256", + "content": "64f76ce90ae3e8ef5d1ede0f8d3b50ce26bcce71dd8ae5e82b1cd2d4a5f94228" + }, + { + "alg": "SHA-256", + "content": "6bbff10f5a52662c00a2e3f86a38928c37c48f77b3c511aedcd51de933549324" + } + ], + "name": "feedparser", + "purl": "pkg:pypi/feedparser@6.0.12", + "type": "library", + "version": "6.0.12" + }, + { + "bom-ref": "pkg:pypi/flake8@7.3.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e" + }, + { + "alg": "SHA-256", + "content": "fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872" + } + ], + "name": "flake8", + "purl": "pkg:pypi/flake8@7.3.0", + "type": "library", + "version": "7.3.0" + }, + { + "bom-ref": "pkg:pypi/google-auth@2.56.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6" + }, + { + "alg": "SHA-256", + "content": "e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051" + } + ], + "name": "google-auth", + "purl": "pkg:pypi/google-auth@2.56.2", + "type": "library", + "version": "2.56.2" + }, + { + "bom-ref": "pkg:pypi/google-genai@2.14.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "a9d1f4f362d76280f1be1340fcb3c86e63dbca128f6a4ae09d86ab47ff7148e8" + }, + { + "alg": "SHA-256", + "content": "ae7172cdd35695189b516b33a878e4132e5daa2dbc03a5b44cddfa8a82fad664" + } + ], + "name": "google-genai", + "purl": "pkg:pypi/google-genai@2.14.0", + "type": "library", + "version": "2.14.0" + }, + { + "bom-ref": "pkg:pypi/googlenewsdecoder@0.1.7", + "hashes": [ + { + "alg": "SHA-256", + "content": "0fcd8fa2aa6a145f84798921aeef0ec45d40bcf5ceebb467742f0efd068d3902" + }, + { + "alg": "SHA-256", + "content": "a8be897bb41864a82d81da6e05777d8c795989cf65f14b41e8e772b3f5a8354d" + } + ], + "name": "googlenewsdecoder", + "purl": "pkg:pypi/googlenewsdecoder@0.1.7", + "type": "library", + "version": "0.1.7" + }, + { + "bom-ref": "pkg:pypi/greenlet@3.5.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20" + }, + { + "alg": "SHA-256", + "content": "07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c" + }, + { + "alg": "SHA-256", + "content": "08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994" + }, + { + "alg": "SHA-256", + "content": "0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8" + }, + { + "alg": "SHA-256", + "content": "123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d" + }, + { + "alg": "SHA-256", + "content": "12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9" + }, + { + "alg": "SHA-256", + "content": "13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f" + }, + { + "alg": "SHA-256", + "content": "178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809" + }, + { + "alg": "SHA-256", + "content": "1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c" + }, + { + "alg": "SHA-256", + "content": "188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c" + }, + { + "alg": "SHA-256", + "content": "1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72" + }, + { + "alg": "SHA-256", + "content": "1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3" + }, + { + "alg": "SHA-256", + "content": "24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02" + }, + { + "alg": "SHA-256", + "content": "27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c" + }, + { + "alg": "SHA-256", + "content": "2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c" + }, + { + "alg": "SHA-256", + "content": "2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7" + }, + { + "alg": "SHA-256", + "content": "2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec" + }, + { + "alg": "SHA-256", + "content": "305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c" + }, + { + "alg": "SHA-256", + "content": "32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686" + }, + { + "alg": "SHA-256", + "content": "3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861" + }, + { + "alg": "SHA-256", + "content": "39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8" + }, + { + "alg": "SHA-256", + "content": "394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0" + }, + { + "alg": "SHA-256", + "content": "3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4" + }, + { + "alg": "SHA-256", + "content": "3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9" + }, + { + "alg": "SHA-256", + "content": "41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3" + }, + { + "alg": "SHA-256", + "content": "42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9" + }, + { + "alg": "SHA-256", + "content": "4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7" + }, + { + "alg": "SHA-256", + "content": "57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7" + }, + { + "alg": "SHA-256", + "content": "58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd" + }, + { + "alg": "SHA-256", + "content": "60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3" + }, + { + "alg": "SHA-256", + "content": "60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2" + }, + { + "alg": "SHA-256", + "content": "69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616" + }, + { + "alg": "SHA-256", + "content": "70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df" + }, + { + "alg": "SHA-256", + "content": "73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf" + }, + { + "alg": "SHA-256", + "content": "77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0" + }, + { + "alg": "SHA-256", + "content": "791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a" + }, + { + "alg": "SHA-256", + "content": "7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f" + }, + { + "alg": "SHA-256", + "content": "7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22" + }, + { + "alg": "SHA-256", + "content": "870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356" + }, + { + "alg": "SHA-256", + "content": "89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353" + }, + { + "alg": "SHA-256", + "content": "91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e" + }, + { + "alg": "SHA-256", + "content": "9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7" + }, + { + "alg": "SHA-256", + "content": "99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5" + }, + { + "alg": "SHA-256", + "content": "9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8" + }, + { + "alg": "SHA-256", + "content": "9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde" + }, + { + "alg": "SHA-256", + "content": "a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52" + }, + { + "alg": "SHA-256", + "content": "ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190" + }, + { + "alg": "SHA-256", + "content": "adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05" + }, + { + "alg": "SHA-256", + "content": "ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937" + }, + { + "alg": "SHA-256", + "content": "b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867" + }, + { + "alg": "SHA-256", + "content": "b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d" + }, + { + "alg": "SHA-256", + "content": "b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf" + }, + { + "alg": "SHA-256", + "content": "bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f" + }, + { + "alg": "SHA-256", + "content": "bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd" + }, + { + "alg": "SHA-256", + "content": "c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da" + }, + { + "alg": "SHA-256", + "content": "c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071" + }, + { + "alg": "SHA-256", + "content": "c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88" + }, + { + "alg": "SHA-256", + "content": "c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17" + }, + { + "alg": "SHA-256", + "content": "ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c" + }, + { + "alg": "SHA-256", + "content": "cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66" + }, + { + "alg": "SHA-256", + "content": "cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb" + }, + { + "alg": "SHA-256", + "content": "cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c" + }, + { + "alg": "SHA-256", + "content": "d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25" + }, + { + "alg": "SHA-256", + "content": "d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0" + }, + { + "alg": "SHA-256", + "content": "d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927" + }, + { + "alg": "SHA-256", + "content": "dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6" + }, + { + "alg": "SHA-256", + "content": "dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c" + }, + { + "alg": "SHA-256", + "content": "e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59" + }, + { + "alg": "SHA-256", + "content": "e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb" + }, + { + "alg": "SHA-256", + "content": "e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606" + }, + { + "alg": "SHA-256", + "content": "ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef" + }, + { + "alg": "SHA-256", + "content": "ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3" + }, + { + "alg": "SHA-256", + "content": "ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da" + }, + { + "alg": "SHA-256", + "content": "ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132" + }, + { + "alg": "SHA-256", + "content": "f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7" + }, + { + "alg": "SHA-256", + "content": "f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f" + }, + { + "alg": "SHA-256", + "content": "f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2" + }, + { + "alg": "SHA-256", + "content": "f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f" + }, + { + "alg": "SHA-256", + "content": "f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667" + } + ], + "name": "greenlet", + "purl": "pkg:pypi/greenlet@3.5.4", + "type": "library", + "version": "3.5.4" + }, + { + "bom-ref": "pkg:pypi/h11@0.16.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1" + }, + { + "alg": "SHA-256", + "content": "63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86" + } + ], + "name": "h11", + "purl": "pkg:pypi/h11@0.16.0", + "type": "library", + "version": "0.16.0" + }, + { + "bom-ref": "pkg:pypi/httpcore@1.0.9", + "hashes": [ + { + "alg": "SHA-256", + "content": "2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55" + }, + { + "alg": "SHA-256", + "content": "6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8" + } + ], + "name": "httpcore", + "purl": "pkg:pypi/httpcore@1.0.9", + "type": "library", + "version": "1.0.9" + }, + { + "bom-ref": "pkg:pypi/httpx@0.28.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc" + }, + { + "alg": "SHA-256", + "content": "d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad" + } + ], + "name": "httpx", + "purl": "pkg:pypi/httpx@0.28.1", + "type": "library", + "version": "0.28.1" + }, + { + "bom-ref": "pkg:pypi/idna@3.18", + "hashes": [ + { + "alg": "SHA-256", + "content": "7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2" + }, + { + "alg": "SHA-256", + "content": "ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848" + } + ], + "name": "idna", + "purl": "pkg:pypi/idna@3.18", + "type": "library", + "version": "3.18" + }, + { + "bom-ref": "pkg:pypi/iniconfig@2.3.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730" + }, + { + "alg": "SHA-256", + "content": "f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12" + } + ], + "name": "iniconfig", + "purl": "pkg:pypi/iniconfig@2.3.0", + "type": "library", + "version": "2.3.0" + }, + { + "bom-ref": "pkg:pypi/jiter@0.16.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536" + }, + { + "alg": "SHA-256", + "content": "0758ab7747a984797cf048e8eedea1d8ef39d7994b25611daf5b48fc903e8873" + }, + { + "alg": "SHA-256", + "content": "0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f" + }, + { + "alg": "SHA-256", + "content": "0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3" + }, + { + "alg": "SHA-256", + "content": "0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce" + }, + { + "alg": "SHA-256", + "content": "10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26" + }, + { + "alg": "SHA-256", + "content": "12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e" + }, + { + "alg": "SHA-256", + "content": "17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85" + }, + { + "alg": "SHA-256", + "content": "1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7" + }, + { + "alg": "SHA-256", + "content": "1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f" + }, + { + "alg": "SHA-256", + "content": "2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207" + }, + { + "alg": "SHA-256", + "content": "26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5" + }, + { + "alg": "SHA-256", + "content": "26d122613ada2b708eb714695446f40fce5bdf2edb4b02116dec62faa62dfab3" + }, + { + "alg": "SHA-256", + "content": "2a0d46ef67cc58d906a6132dd3040ca70ae4f0b0d7c9c052fe432c658a69b3f6" + }, + { + "alg": "SHA-256", + "content": "2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056" + }, + { + "alg": "SHA-256", + "content": "2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131" + }, + { + "alg": "SHA-256", + "content": "2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331" + }, + { + "alg": "SHA-256", + "content": "3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03" + }, + { + "alg": "SHA-256", + "content": "31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93" + }, + { + "alg": "SHA-256", + "content": "3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290" + }, + { + "alg": "SHA-256", + "content": "3b21b412b899fd8bd51a3046934b59a3bb068b79f70a5c6010053ac77cc53f0c" + }, + { + "alg": "SHA-256", + "content": "3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a" + }, + { + "alg": "SHA-256", + "content": "3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284" + }, + { + "alg": "SHA-256", + "content": "3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c" + }, + { + "alg": "SHA-256", + "content": "41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195" + }, + { + "alg": "SHA-256", + "content": "44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730" + }, + { + "alg": "SHA-256", + "content": "46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48" + }, + { + "alg": "SHA-256", + "content": "49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69" + }, + { + "alg": "SHA-256", + "content": "491e7d072a253b156fff46b78bceac4652a697aa8d7082c9c18c03d7b7917d24" + }, + { + "alg": "SHA-256", + "content": "4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c" + }, + { + "alg": "SHA-256", + "content": "51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274" + }, + { + "alg": "SHA-256", + "content": "560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7" + }, + { + "alg": "SHA-256", + "content": "561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5" + }, + { + "alg": "SHA-256", + "content": "57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106" + }, + { + "alg": "SHA-256", + "content": "5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b" + }, + { + "alg": "SHA-256", + "content": "5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00" + }, + { + "alg": "SHA-256", + "content": "5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9" + }, + { + "alg": "SHA-256", + "content": "5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b" + }, + { + "alg": "SHA-256", + "content": "5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b" + }, + { + "alg": "SHA-256", + "content": "616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818" + }, + { + "alg": "SHA-256", + "content": "62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c" + }, + { + "alg": "SHA-256", + "content": "63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2" + }, + { + "alg": "SHA-256", + "content": "63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077" + }, + { + "alg": "SHA-256", + "content": "64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037" + }, + { + "alg": "SHA-256", + "content": "64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2" + }, + { + "alg": "SHA-256", + "content": "659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84" + }, + { + "alg": "SHA-256", + "content": "6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a" + }, + { + "alg": "SHA-256", + "content": "67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a" + }, + { + "alg": "SHA-256", + "content": "67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3" + }, + { + "alg": "SHA-256", + "content": "6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb" + }, + { + "alg": "SHA-256", + "content": "70a490b55634dc0d2606ce8a8e01b1d62459011beb368d15d76e1eaf62460e3d" + }, + { + "alg": "SHA-256", + "content": "710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe" + }, + { + "alg": "SHA-256", + "content": "741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84" + }, + { + "alg": "SHA-256", + "content": "7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c" + }, + { + "alg": "SHA-256", + "content": "7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126" + }, + { + "alg": "SHA-256", + "content": "850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad" + }, + { + "alg": "SHA-256", + "content": "8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee" + }, + { + "alg": "SHA-256", + "content": "8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053" + }, + { + "alg": "SHA-256", + "content": "8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929" + }, + { + "alg": "SHA-256", + "content": "8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450" + }, + { + "alg": "SHA-256", + "content": "8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244" + }, + { + "alg": "SHA-256", + "content": "96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3" + }, + { + "alg": "SHA-256", + "content": "9acf1b2faec82d998811ecce7ae84d9005e53410773e9d37d61cdc424ba4581b" + }, + { + "alg": "SHA-256", + "content": "9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9" + }, + { + "alg": "SHA-256", + "content": "9ec553a99b0987efd7a3645a1a825cf29c224e494db267a83369fcc8da9aeda5" + }, + { + "alg": "SHA-256", + "content": "a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734" + }, + { + "alg": "SHA-256", + "content": "a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585" + }, + { + "alg": "SHA-256", + "content": "a5c54ef4ff776d9675837ef535b3308d6e31c208d43ebc44a0f7ab8a208c68f7" + }, + { + "alg": "SHA-256", + "content": "adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29" + }, + { + "alg": "SHA-256", + "content": "adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91" + }, + { + "alg": "SHA-256", + "content": "b1e7923093a376d93c6eb507c77045ae258d689ba577392846a1b3f10d0b09a9" + }, + { + "alg": "SHA-256", + "content": "b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee" + }, + { + "alg": "SHA-256", + "content": "b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9" + }, + { + "alg": "SHA-256", + "content": "b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e" + }, + { + "alg": "SHA-256", + "content": "b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5" + }, + { + "alg": "SHA-256", + "content": "b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db" + }, + { + "alg": "SHA-256", + "content": "baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd" + }, + { + "alg": "SHA-256", + "content": "bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620" + }, + { + "alg": "SHA-256", + "content": "bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567" + }, + { + "alg": "SHA-256", + "content": "bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898" + }, + { + "alg": "SHA-256", + "content": "c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01" + }, + { + "alg": "SHA-256", + "content": "c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea" + }, + { + "alg": "SHA-256", + "content": "c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c" + }, + { + "alg": "SHA-256", + "content": "c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026" + }, + { + "alg": "SHA-256", + "content": "c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e" + }, + { + "alg": "SHA-256", + "content": "c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e" + }, + { + "alg": "SHA-256", + "content": "cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8" + }, + { + "alg": "SHA-256", + "content": "cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883" + }, + { + "alg": "SHA-256", + "content": "d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702" + }, + { + "alg": "SHA-256", + "content": "d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09" + }, + { + "alg": "SHA-256", + "content": "d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72" + }, + { + "alg": "SHA-256", + "content": "d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8" + }, + { + "alg": "SHA-256", + "content": "d8f80521644426d451e70f00c7974240cab8f6ee088aedaa9af2697153ab7805" + }, + { + "alg": "SHA-256", + "content": "da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af" + }, + { + "alg": "SHA-256", + "content": "dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3" + }, + { + "alg": "SHA-256", + "content": "de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf" + }, + { + "alg": "SHA-256", + "content": "e03a5f21a5ce96a9441b8cb32719a8b88ed5388f53e0f339c5bcf54f1317f9d0" + }, + { + "alg": "SHA-256", + "content": "e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f" + }, + { + "alg": "SHA-256", + "content": "e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1" + }, + { + "alg": "SHA-256", + "content": "eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e" + }, + { + "alg": "SHA-256", + "content": "ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e" + }, + { + "alg": "SHA-256", + "content": "f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127" + }, + { + "alg": "SHA-256", + "content": "f3bd327cdfa118bc1ce69c214c2678571d5bd39b8ccd0ebf43a54db00541ba9a" + }, + { + "alg": "SHA-256", + "content": "f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e" + }, + { + "alg": "SHA-256", + "content": "fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de" + } + ], + "name": "jiter", + "purl": "pkg:pypi/jiter@0.16.0", + "type": "library", + "version": "0.16.0" + }, + { + "bom-ref": "pkg:pypi/librt@0.13.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "05d96b80b95d3a2721b619f8982b8558848b04875bb4772fd54842b59f61dd97" + }, + { + "alg": "SHA-256", + "content": "0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b" + }, + { + "alg": "SHA-256", + "content": "091b60a4d2174fc1ec5c34cdc0b72efb6224753d76b7da61ebeab7a191aec8bd" + }, + { + "alg": "SHA-256", + "content": "0b795f5fc70fbbb787ceaf79bb3a0d627bcc33c53de51741755263ec406b775a" + }, + { + "alg": "SHA-256", + "content": "0d38604854e8d22faadf683ec6c02bb0f886e2ba56ef981a1c36ee275f21ea22" + }, + { + "alg": "SHA-256", + "content": "109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6" + }, + { + "alg": "SHA-256", + "content": "1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7" + }, + { + "alg": "SHA-256", + "content": "17221a7569f8f292aa0014226e48aa25b8c2b08da18088cd230953d0ea0f9cd1" + }, + { + "alg": "SHA-256", + "content": "1b5a7bbff495baedbd9b916c367d66854008f8f3b575908ded477c499dc60082" + }, + { + "alg": "SHA-256", + "content": "1ce61b3746545029d4f5c17d6bd74b676254ad98433086c846ffb5e8fa73f007" + }, + { + "alg": "SHA-256", + "content": "1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781" + }, + { + "alg": "SHA-256", + "content": "21b7ac084f701a9cdff6139745a6620579d65a9379ac2d9d50a86368b109e63c" + }, + { + "alg": "SHA-256", + "content": "22034924f5b42d5a56371cf271771bfeaabf235a7a8b6264bef2d20013f786c6" + }, + { + "alg": "SHA-256", + "content": "25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71" + }, + { + "alg": "SHA-256", + "content": "2608d3b39f9e0b4a66a130d9150c615cba40a5090d25eeeaa225e0e46de8c0ac" + }, + { + "alg": "SHA-256", + "content": "260c33e92263fa629b4f6d3c51967a1c2158fe6c33237aaa3ebeac586b085259" + }, + { + "alg": "SHA-256", + "content": "2e56ea4ee4df77585a6b5c138f6538680886024fa559f5b55bd14b12e98e67b2" + }, + { + "alg": "SHA-256", + "content": "2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99" + }, + { + "alg": "SHA-256", + "content": "301067672387902c55f94b51d5022304b36c966ea9fe1f21caab99a9bef487c9" + }, + { + "alg": "SHA-256", + "content": "30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0" + }, + { + "alg": "SHA-256", + "content": "32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c" + }, + { + "alg": "SHA-256", + "content": "34bc7938b9fdf14fe32a406c19c71faf894c5cee7e7474bd0be2f17200b82d14" + }, + { + "alg": "SHA-256", + "content": "34e47058fcc69a313293d6dee94216a4f30c929ae6f2476e58c5ba635aa639d5" + }, + { + "alg": "SHA-256", + "content": "3657346f867469e962549435aa05fd15330b1d6a92829f8e27988e194382d005" + }, + { + "alg": "SHA-256", + "content": "36b306a623aaad96fe4b378692b54f9c0789fccd833b9851753d5fbf6138cfde" + }, + { + "alg": "SHA-256", + "content": "371f7ce73026815dafd51c50ce38416e91428b28c4b2ec97cd39271164b0045c" + }, + { + "alg": "SHA-256", + "content": "375f5af8f99cbaa99dd293af986e3d57caabc9ba81a5d3f021603764854197a1" + }, + { + "alg": "SHA-256", + "content": "387e2f1d27e89bffe0d3f520f0da0662c973fd607ca16c1808f8a5085419485e" + }, + { + "alg": "SHA-256", + "content": "3aaedf52171bee90860704c560bc798fe83b76247df47568e0197e9b13c735a0" + }, + { + "alg": "SHA-256", + "content": "3dbb2a31882456cadc7053378e81ad7ed7693db4ac9f98ab5f81ef034aa8ec9f" + }, + { + "alg": "SHA-256", + "content": "4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628" + }, + { + "alg": "SHA-256", + "content": "40ccd13c252d3fe473ffc8a57be7565abc8b64cf1b108344c859d5164f7f3e0c" + }, + { + "alg": "SHA-256", + "content": "46c330e82565962c761dbce7941be2cff7db674ee807455a8d0cadc5f9b759b0" + }, + { + "alg": "SHA-256", + "content": "4f6db193d2e5e0ed60359b9a5a682cd67205d0d3b1e459a867dd4b5c4e7eaa7a" + }, + { + "alg": "SHA-256", + "content": "531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db" + }, + { + "alg": "SHA-256", + "content": "54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650" + }, + { + "alg": "SHA-256", + "content": "5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b" + }, + { + "alg": "SHA-256", + "content": "5f31b0aa13c9b04370d4da6be1ab7779776b3a075cceb6747a39a4be85fe1e40" + }, + { + "alg": "SHA-256", + "content": "5fdcf34f86de8fb66d7dc7589f96ba91c4aa46671200d400e6fd6f109a483f18" + }, + { + "alg": "SHA-256", + "content": "66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348" + }, + { + "alg": "SHA-256", + "content": "66cb1138f384a191a6d75f986064841fcfdc0cea98f7bd9c9ab9b38049917588" + }, + { + "alg": "SHA-256", + "content": "68a5faee4bba381cb93b5961f684a514cf0053cb92308ff9c792c2fea0b174c6" + }, + { + "alg": "SHA-256", + "content": "6bf6a559ffe4a93bbea6cf31ddf01a7fd9ba342ef51f27beb178e318b74acd61" + }, + { + "alg": "SHA-256", + "content": "70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd" + }, + { + "alg": "SHA-256", + "content": "791aa18a373b90da8ac3c44fc77544f33fdf53ae403acdce9b39f1c26b4a3b94" + }, + { + "alg": "SHA-256", + "content": "79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927" + }, + { + "alg": "SHA-256", + "content": "7db9a3ff32ef5f7d1703d93831a3316cdf0b537de6a1cc03cc8fdd09b9194e89" + }, + { + "alg": "SHA-256", + "content": "860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3" + }, + { + "alg": "SHA-256", + "content": "9320d34c3376ae204b2cd176e8d4883a013934e0aef822f1aed9c536490c275d" + }, + { + "alg": "SHA-256", + "content": "93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5" + }, + { + "alg": "SHA-256", + "content": "94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9" + }, + { + "alg": "SHA-256", + "content": "96bad8725a4f196a798366c25ce075d1f7543a4ec045ffc13e6a7ec095cdab04" + }, + { + "alg": "SHA-256", + "content": "9af313c66157a69dc69ea0059a66961692250e0dc95af9c385a48ffb770a0d16" + }, + { + "alg": "SHA-256", + "content": "9c5d02b89de5acd0379a51ec44a89476fb03df6145442e1c8ecd6bee2f91b176" + }, + { + "alg": "SHA-256", + "content": "9e786428f291dd2d2f1cbfc0e0caa45a2e395fab0ad3e2c9314daa8873414390" + }, + { + "alg": "SHA-256", + "content": "9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a" + }, + { + "alg": "SHA-256", + "content": "9fd35e95ab5e45c3901d37110263c7db85a961110f5460588fe37f8c131f88a7" + }, + { + "alg": "SHA-256", + "content": "a001519c315d5db40710f2665d32c4791f1d4779fc96a9423fd18d92c8b9ac7b" + }, + { + "alg": "SHA-256", + "content": "a3762e75fcac8c9e4dacaaf438bffd9003e2ca2c531b756f3c0035deefa674c8" + }, + { + "alg": "SHA-256", + "content": "a38fb81d8376dfa2f8963b265fec07637802b0d01e2a127c19c66cb070fb24f5" + }, + { + "alg": "SHA-256", + "content": "a3dfe4edf10e8ed7e55b026a8bfc2c2a8704218b659cd4bffdf604fab966dc39" + }, + { + "alg": "SHA-256", + "content": "a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f" + }, + { + "alg": "SHA-256", + "content": "a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3" + }, + { + "alg": "SHA-256", + "content": "a6e556d6aba31c93dd97ce661d66614d2429c0a3923f9dc8f0af7e8df10223a4" + }, + { + "alg": "SHA-256", + "content": "ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7" + }, + { + "alg": "SHA-256", + "content": "ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c" + }, + { + "alg": "SHA-256", + "content": "b15e26cc0fe622d0c67e98bee6ef6bc8f792e20ee3006aa12627a00463d9399f" + }, + { + "alg": "SHA-256", + "content": "b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03" + }, + { + "alg": "SHA-256", + "content": "c3cd253cf32fe4f4662960d6bf7d55cb8be0c31a5d644a4d48aeafebaff3409a" + }, + { + "alg": "SHA-256", + "content": "c6014e3c80f9c1fe268ef8b0e0ef113bac672cc032f2f93866e7ddad4f3e663d" + }, + { + "alg": "SHA-256", + "content": "c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1" + }, + { + "alg": "SHA-256", + "content": "c7897db4e95e22468bdda33d8e012ceacd0182abf001e6389d763f0def6286b9" + }, + { + "alg": "SHA-256", + "content": "cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9" + }, + { + "alg": "SHA-256", + "content": "cc99dfb62b23c9207c33d0be8a2e2af7a42e21e6ea388b380a0c948c7b88953b" + }, + { + "alg": "SHA-256", + "content": "d4c8d9bd5abce34b2e75edb3bf37ab0f34e49b1f915a40ae8468eb7c85bc5b46" + }, + { + "alg": "SHA-256", + "content": "d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566" + }, + { + "alg": "SHA-256", + "content": "d63bae12a8aeb51380be3438e4dc4bd27354d0f8e19166b2f44e3e94d6f552dc" + }, + { + "alg": "SHA-256", + "content": "d6fb0eaa108814581c4d3bfbd068c3fb6757812a81415008d1bae08267cca360" + }, + { + "alg": "SHA-256", + "content": "d9188caac26e47671b52836a5e2a49873a7fc11c673b0c122d22515f98bc14e1" + }, + { + "alg": "SHA-256", + "content": "db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82" + }, + { + "alg": "SHA-256", + "content": "dbdd5b6509d0c2a8fe72cf494c299a61dbd58142a90a4190664ae159e4a7b547" + }, + { + "alg": "SHA-256", + "content": "e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1" + }, + { + "alg": "SHA-256", + "content": "e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa" + }, + { + "alg": "SHA-256", + "content": "f125f5d46b20f89dc5587a55cc416b4ba2a5b2ffda36d048ee120e17598a653a" + }, + { + "alg": "SHA-256", + "content": "f19e181de5b3a1148bb3420b8c4b0b0ea0fce6950099724ad151d6cea5acc180" + }, + { + "alg": "SHA-256", + "content": "f1f9cc4d09a46d9cb3c2063ae100629d3f52a6517c3c08c2f4c9828261883929" + }, + { + "alg": "SHA-256", + "content": "f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6" + }, + { + "alg": "SHA-256", + "content": "f2a7253458e34f33543551394ae4fe104b497ec2a65ac266074de64c1df82e37" + }, + { + "alg": "SHA-256", + "content": "f40e56b61b41be5f7dec938cfeffd660668cf4b5e72c78e7bd671d66b7bc2c79" + }, + { + "alg": "SHA-256", + "content": "f442e3954b1addc759faae22a7c9a3f1e16d7d1db3f484279dc27d62e06968fa" + }, + { + "alg": "SHA-256", + "content": "fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e" + }, + { + "alg": "SHA-256", + "content": "fc67741da44c6eaa90e01eafb586bbba9b51eb5b6ed381ee6f5ae72eb3316d21" + } + ], + "name": "librt", + "purl": "pkg:pypi/librt@0.13.0", + "type": "library", + "version": "0.13.0" + }, + { + "bom-ref": "pkg:pypi/mako@1.3.12", + "hashes": [ + { + "alg": "SHA-256", + "content": "8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9" + }, + { + "alg": "SHA-256", + "content": "9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a" + } + ], + "name": "mako", + "purl": "pkg:pypi/mako@1.3.12", + "type": "library", + "version": "1.3.12" + }, + { + "bom-ref": "pkg:pypi/markupsafe@3.0.3", + "hashes": [ + { + "alg": "SHA-256", + "content": "0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f" + }, + { + "alg": "SHA-256", + "content": "068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a" + }, + { + "alg": "SHA-256", + "content": "0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf" + }, + { + "alg": "SHA-256", + "content": "0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19" + }, + { + "alg": "SHA-256", + "content": "0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf" + }, + { + "alg": "SHA-256", + "content": "0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c" + }, + { + "alg": "SHA-256", + "content": "1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175" + }, + { + "alg": "SHA-256", + "content": "116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219" + }, + { + "alg": "SHA-256", + "content": "12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb" + }, + { + "alg": "SHA-256", + "content": "133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6" + }, + { + "alg": "SHA-256", + "content": "1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab" + }, + { + "alg": "SHA-256", + "content": "15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26" + }, + { + "alg": "SHA-256", + "content": "177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1" + }, + { + "alg": "SHA-256", + "content": "1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce" + }, + { + "alg": "SHA-256", + "content": "1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218" + }, + { + "alg": "SHA-256", + "content": "1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634" + }, + { + "alg": "SHA-256", + "content": "1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695" + }, + { + "alg": "SHA-256", + "content": "1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad" + }, + { + "alg": "SHA-256", + "content": "218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73" + }, + { + "alg": "SHA-256", + "content": "26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c" + }, + { + "alg": "SHA-256", + "content": "2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe" + }, + { + "alg": "SHA-256", + "content": "2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa" + }, + { + "alg": "SHA-256", + "content": "2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559" + }, + { + "alg": "SHA-256", + "content": "32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa" + }, + { + "alg": "SHA-256", + "content": "3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37" + }, + { + "alg": "SHA-256", + "content": "3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758" + }, + { + "alg": "SHA-256", + "content": "35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f" + }, + { + "alg": "SHA-256", + "content": "38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8" + }, + { + "alg": "SHA-256", + "content": "3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d" + }, + { + "alg": "SHA-256", + "content": "3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c" + }, + { + "alg": "SHA-256", + "content": "457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97" + }, + { + "alg": "SHA-256", + "content": "4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a" + }, + { + "alg": "SHA-256", + "content": "4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19" + }, + { + "alg": "SHA-256", + "content": "4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9" + }, + { + "alg": "SHA-256", + "content": "509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9" + }, + { + "alg": "SHA-256", + "content": "5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc" + }, + { + "alg": "SHA-256", + "content": "591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2" + }, + { + "alg": "SHA-256", + "content": "5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4" + }, + { + "alg": "SHA-256", + "content": "69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354" + }, + { + "alg": "SHA-256", + "content": "6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50" + }, + { + "alg": "SHA-256", + "content": "722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698" + }, + { + "alg": "SHA-256", + "content": "729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9" + }, + { + "alg": "SHA-256", + "content": "77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b" + }, + { + "alg": "SHA-256", + "content": "795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc" + }, + { + "alg": "SHA-256", + "content": "7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115" + }, + { + "alg": "SHA-256", + "content": "7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e" + }, + { + "alg": "SHA-256", + "content": "7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485" + }, + { + "alg": "SHA-256", + "content": "83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f" + }, + { + "alg": "SHA-256", + "content": "8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12" + }, + { + "alg": "SHA-256", + "content": "8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025" + }, + { + "alg": "SHA-256", + "content": "8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009" + }, + { + "alg": "SHA-256", + "content": "915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d" + }, + { + "alg": "SHA-256", + "content": "949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b" + }, + { + "alg": "SHA-256", + "content": "94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a" + }, + { + "alg": "SHA-256", + "content": "9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5" + }, + { + "alg": "SHA-256", + "content": "9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f" + }, + { + "alg": "SHA-256", + "content": "a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d" + }, + { + "alg": "SHA-256", + "content": "a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1" + }, + { + "alg": "SHA-256", + "content": "ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287" + }, + { + "alg": "SHA-256", + "content": "b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6" + }, + { + "alg": "SHA-256", + "content": "bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f" + }, + { + "alg": "SHA-256", + "content": "bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581" + }, + { + "alg": "SHA-256", + "content": "bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed" + }, + { + "alg": "SHA-256", + "content": "be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b" + }, + { + "alg": "SHA-256", + "content": "c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c" + }, + { + "alg": "SHA-256", + "content": "c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026" + }, + { + "alg": "SHA-256", + "content": "c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8" + }, + { + "alg": "SHA-256", + "content": "ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676" + }, + { + "alg": "SHA-256", + "content": "d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6" + }, + { + "alg": "SHA-256", + "content": "d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e" + }, + { + "alg": "SHA-256", + "content": "d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d" + }, + { + "alg": "SHA-256", + "content": "d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d" + }, + { + "alg": "SHA-256", + "content": "de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01" + }, + { + "alg": "SHA-256", + "content": "df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7" + }, + { + "alg": "SHA-256", + "content": "e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419" + }, + { + "alg": "SHA-256", + "content": "e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795" + }, + { + "alg": "SHA-256", + "content": "e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1" + }, + { + "alg": "SHA-256", + "content": "e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5" + }, + { + "alg": "SHA-256", + "content": "e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d" + }, + { + "alg": "SHA-256", + "content": "e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42" + }, + { + "alg": "SHA-256", + "content": "eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe" + }, + { + "alg": "SHA-256", + "content": "ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda" + }, + { + "alg": "SHA-256", + "content": "f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e" + }, + { + "alg": "SHA-256", + "content": "f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737" + }, + { + "alg": "SHA-256", + "content": "f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523" + }, + { + "alg": "SHA-256", + "content": "f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591" + }, + { + "alg": "SHA-256", + "content": "f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc" + }, + { + "alg": "SHA-256", + "content": "f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a" + }, + { + "alg": "SHA-256", + "content": "fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50" + } + ], + "name": "markupsafe", + "purl": "pkg:pypi/markupsafe@3.0.3", + "type": "library", + "version": "3.0.3" + }, + { + "bom-ref": "pkg:pypi/mccabe@0.7.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325" + }, + { + "alg": "SHA-256", + "content": "6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" + } + ], + "name": "mccabe", + "purl": "pkg:pypi/mccabe@0.7.0", + "type": "library", + "version": "0.7.0" + }, + { + "bom-ref": "pkg:pypi/mypy@2.3.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491" + }, + { + "alg": "SHA-256", + "content": "094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762" + }, + { + "alg": "SHA-256", + "content": "09abd66d8685e73f8f7d17b847c3e104d9a7b164a8706ea87d6c96a3d45816d5" + }, + { + "alg": "SHA-256", + "content": "0b025a93cffb9781d231f232be07a17912f35f10a313c24f301c81e842870654" + }, + { + "alg": "SHA-256", + "content": "13b1b16e2fa39f3b2e33fb1c468abc7a69369fa2e886b4b87b5afc81472325cd" + }, + { + "alg": "SHA-256", + "content": "1c6c6bf687b17f90dbfcad95b960d32eaa0154c00da45f03ab50bf8952e047fe" + }, + { + "alg": "SHA-256", + "content": "1fa8d916ac3b705af733c4c1e6c9ebe38fd0d52beb15b105c3e8355b55e6ecdc" + }, + { + "alg": "SHA-256", + "content": "28e1e2af8cd8fff551fd30f2fe4b03fb76764ac8b1ba6c6a1bd00ad32b412db3" + }, + { + "alg": "SHA-256", + "content": "2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5" + }, + { + "alg": "SHA-256", + "content": "3419d00717afbc5265b50dd14b1278f29ea4884dd398ab67873489ac093fd329" + }, + { + "alg": "SHA-256", + "content": "37fa4de896a84e2dc9200d91e614c22563b43d1a266789d4bbac7b22ebe6192b" + }, + { + "alg": "SHA-256", + "content": "3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3" + }, + { + "alg": "SHA-256", + "content": "3dd0bed92c4bdec57c42505b96416fb9e6a5aa7be84d2809bcd5f2ecec2860d7" + }, + { + "alg": "SHA-256", + "content": "3e77244df3843048c3f927182916730e40c124cbaa43905c1fb86cb382aa0805" + }, + { + "alg": "SHA-256", + "content": "4359424140d985192c778c1ce2c114a10c1ca58a381ed79cfa70d37df94b299f" + }, + { + "alg": "SHA-256", + "content": "465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e" + }, + { + "alg": "SHA-256", + "content": "56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7" + }, + { + "alg": "SHA-256", + "content": "5e91adad1ca81742ac7ef9893959911df867752206b37135185e88dfb3c89494" + }, + { + "alg": "SHA-256", + "content": "691fdc37132b1ae628d834f672e74de83462d9fb4aff621835767fb43a8dd373" + }, + { + "alg": "SHA-256", + "content": "6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88" + }, + { + "alg": "SHA-256", + "content": "6f99ec626e3c3a2f7c0b22c5b90ddb5dabb1c18729c971e9bdaca1f1766d2cee" + }, + { + "alg": "SHA-256", + "content": "7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2" + }, + { + "alg": "SHA-256", + "content": "75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757" + }, + { + "alg": "SHA-256", + "content": "75cbb4b9ef04a0c84a957f07abc4504fbf64b8dcc145675101f2d3a78a4b1d6a" + }, + { + "alg": "SHA-256", + "content": "7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b" + }, + { + "alg": "SHA-256", + "content": "85c5385b93012ffa3b31479ab579aef5415f4f3a32c6cf1ae07a984d2a0ff461" + }, + { + "alg": "SHA-256", + "content": "91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97" + }, + { + "alg": "SHA-256", + "content": "944c665d984157cb96a679dfb7a4a81dd1d36b24b9c284b699514e6e626b82d4" + }, + { + "alg": "SHA-256", + "content": "9559ab18a9c9957dfa3004ab57cd4bac5f26a724329a9584e583367f0c2e1117" + }, + { + "alg": "SHA-256", + "content": "982e3d53dd23d0a4cef67dd66791fdbede0cf38f9eb617bf47663554c51e1e36" + }, + { + "alg": "SHA-256", + "content": "99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac" + }, + { + "alg": "SHA-256", + "content": "adebc76aab4f3495a88b41d48aa4aff0c03f2822501da76625afcca5975f19e5" + }, + { + "alg": "SHA-256", + "content": "aec15d465d477558fd842757b487849007311cf3897849cdda0e3162ac0ac556" + }, + { + "alg": "SHA-256", + "content": "b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c" + }, + { + "alg": "SHA-256", + "content": "b352b7e49f5e6576009e8df730e1ff4f915cb565b851b396d2ffe2f5a6f5da88" + }, + { + "alg": "SHA-256", + "content": "b5cd2f027a972a4a5f2278a11fac9747f5f81a53a30b714d74950b6807e55568" + }, + { + "alg": "SHA-256", + "content": "be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595" + }, + { + "alg": "SHA-256", + "content": "cfca8ee88544090f86b6dcce05ec55d66eb48a762412ac2507810ba4bd793b6f" + }, + { + "alg": "SHA-256", + "content": "d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1" + }, + { + "alg": "SHA-256", + "content": "de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef" + }, + { + "alg": "SHA-256", + "content": "de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6" + }, + { + "alg": "SHA-256", + "content": "ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db" + }, + { + "alg": "SHA-256", + "content": "f1b3a98dfd21058bc759bb3337d5d1f61d0fdf9f3cf9c00f4291790fb5427bff" + }, + { + "alg": "SHA-256", + "content": "f4ed18f111bfe2d599bca7468e7f9251042c1c2118f762c8de2766a56d773c60" + }, + { + "alg": "SHA-256", + "content": "fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c" + } + ], + "name": "mypy", + "purl": "pkg:pypi/mypy@2.3.0", + "type": "library", + "version": "2.3.0" + }, + { + "bom-ref": "pkg:pypi/mypy-extensions@1.1.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505" + }, + { + "alg": "SHA-256", + "content": "52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558" + } + ], + "name": "mypy-extensions", + "purl": "pkg:pypi/mypy-extensions@1.1.0", + "type": "library", + "version": "1.1.0" + }, + { + "bom-ref": "pkg:pypi/packaging@26.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e" + }, + { + "alg": "SHA-256", + "content": "ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661" + } + ], + "name": "packaging", + "purl": "pkg:pypi/packaging@26.2", + "type": "library", + "version": "26.2" + }, + { + "bom-ref": "pkg:pypi/pathspec@1.1.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a" + }, + { + "alg": "SHA-256", + "content": "a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189" + } + ], + "name": "pathspec", + "purl": "pkg:pypi/pathspec@1.1.1", + "type": "library", + "version": "1.1.1" + }, + { + "bom-ref": "pkg:pypi/pluggy@1.6.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3" + }, + { + "alg": "SHA-256", + "content": "e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746" + } + ], + "name": "pluggy", + "purl": "pkg:pypi/pluggy@1.6.0", + "type": "library", + "version": "1.6.0" + }, + { + "bom-ref": "pkg:pypi/psycopg@3.3.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a" + }, + { + "alg": "SHA-256", + "content": "e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc" + } + ], + "name": "psycopg", + "purl": "pkg:pypi/psycopg@3.3.4", + "type": "library", + "version": "3.3.4" + }, + { + "bom-ref": "pkg:pypi/psycopg-binary@3.3.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070" + }, + { + "alg": "SHA-256", + "content": "0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c" + }, + { + "alg": "SHA-256", + "content": "136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc" + }, + { + "alg": "SHA-256", + "content": "13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e" + }, + { + "alg": "SHA-256", + "content": "17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68" + }, + { + "alg": "SHA-256", + "content": "1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae" + }, + { + "alg": "SHA-256", + "content": "1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829" + }, + { + "alg": "SHA-256", + "content": "22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097" + }, + { + "alg": "SHA-256", + "content": "26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c" + }, + { + "alg": "SHA-256", + "content": "276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992" + }, + { + "alg": "SHA-256", + "content": "28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97" + }, + { + "alg": "SHA-256", + "content": "2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6" + }, + { + "alg": "SHA-256", + "content": "32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a" + }, + { + "alg": "SHA-256", + "content": "41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d" + }, + { + "alg": "SHA-256", + "content": "46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228" + }, + { + "alg": "SHA-256", + "content": "47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e" + }, + { + "alg": "SHA-256", + "content": "494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4" + }, + { + "alg": "SHA-256", + "content": "514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41" + }, + { + "alg": "SHA-256", + "content": "574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949" + }, + { + "alg": "SHA-256", + "content": "580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9" + }, + { + "alg": "SHA-256", + "content": "5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089" + }, + { + "alg": "SHA-256", + "content": "5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf" + }, + { + "alg": "SHA-256", + "content": "612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da" + }, + { + "alg": "SHA-256", + "content": "6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578" + }, + { + "alg": "SHA-256", + "content": "6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014" + }, + { + "alg": "SHA-256", + "content": "71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e" + }, + { + "alg": "SHA-256", + "content": "7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e" + }, + { + "alg": "SHA-256", + "content": "75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31" + }, + { + "alg": "SHA-256", + "content": "773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652" + }, + { + "alg": "SHA-256", + "content": "77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b" + }, + { + "alg": "SHA-256", + "content": "7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839" + }, + { + "alg": "SHA-256", + "content": "8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277" + }, + { + "alg": "SHA-256", + "content": "94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7" + }, + { + "alg": "SHA-256", + "content": "ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4" + }, + { + "alg": "SHA-256", + "content": "ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e" + }, + { + "alg": "SHA-256", + "content": "b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70" + }, + { + "alg": "SHA-256", + "content": "b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf" + }, + { + "alg": "SHA-256", + "content": "b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a" + }, + { + "alg": "SHA-256", + "content": "bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429" + }, + { + "alg": "SHA-256", + "content": "c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8" + }, + { + "alg": "SHA-256", + "content": "c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3" + }, + { + "alg": "SHA-256", + "content": "cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13" + }, + { + "alg": "SHA-256", + "content": "cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007" + }, + { + "alg": "SHA-256", + "content": "d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc" + }, + { + "alg": "SHA-256", + "content": "dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38" + }, + { + "alg": "SHA-256", + "content": "df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9" + }, + { + "alg": "SHA-256", + "content": "e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95" + }, + { + "alg": "SHA-256", + "content": "e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d" + }, + { + "alg": "SHA-256", + "content": "eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16" + }, + { + "alg": "SHA-256", + "content": "eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d" + }, + { + "alg": "SHA-256", + "content": "ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260" + }, + { + "alg": "SHA-256", + "content": "f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744" + }, + { + "alg": "SHA-256", + "content": "f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28" + }, + { + "alg": "SHA-256", + "content": "fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765" + }, + { + "alg": "SHA-256", + "content": "fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7" + } + ], + "name": "psycopg-binary", + "purl": "pkg:pypi/psycopg-binary@3.3.4", + "type": "library", + "version": "3.3.4" + }, + { + "bom-ref": "pkg:pypi/pyasn1@0.6.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81" + }, + { + "alg": "SHA-256", + "content": "deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b" + } + ], + "name": "pyasn1", + "purl": "pkg:pypi/pyasn1@0.6.4", + "type": "library", + "version": "0.6.4" + }, + { + "bom-ref": "pkg:pypi/pyasn1-modules@0.4.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a" + }, + { + "alg": "SHA-256", + "content": "677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6" + } + ], + "name": "pyasn1-modules", + "purl": "pkg:pypi/pyasn1-modules@0.4.2", + "type": "library", + "version": "0.4.2" + }, + { + "bom-ref": "pkg:pypi/pycodestyle@2.14.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783" + }, + { + "alg": "SHA-256", + "content": "dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d" + } + ], + "name": "pycodestyle", + "purl": "pkg:pypi/pycodestyle@2.14.0", + "type": "library", + "version": "2.14.0" + }, + { + "bom-ref": "pkg:pypi/pycountry@26.2.16", + "hashes": [ + { + "alg": "SHA-256", + "content": "115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a" + }, + { + "alg": "SHA-256", + "content": "5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801" + } + ], + "name": "pycountry", + "purl": "pkg:pypi/pycountry@26.2.16", + "type": "library", + "version": "26.2.16" + }, + { + "bom-ref": "pkg:pypi/pycparser@3.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29" + }, + { + "alg": "SHA-256", + "content": "b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992" + } + ], + "name": "pycparser", + "purl": "pkg:pypi/pycparser@3.0", + "type": "library", + "version": "3.0" + }, + { + "bom-ref": "pkg:pypi/pydantic@2.13.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba" + }, + { + "alg": "SHA-256", + "content": "c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6" + } + ], + "name": "pydantic", + "purl": "pkg:pypi/pydantic@2.13.4", + "type": "library", + "version": "2.13.4" + }, + { + "bom-ref": "pkg:pypi/pydantic-core@2.46.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0" + }, + { + "alg": "SHA-256", + "content": "0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262" + }, + { + "alg": "SHA-256", + "content": "041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda" + }, + { + "alg": "SHA-256", + "content": "0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0" + }, + { + "alg": "SHA-256", + "content": "0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e" + }, + { + "alg": "SHA-256", + "content": "0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b" + }, + { + "alg": "SHA-256", + "content": "0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594" + }, + { + "alg": "SHA-256", + "content": "10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29" + }, + { + "alg": "SHA-256", + "content": "133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2" + }, + { + "alg": "SHA-256", + "content": "14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c" + }, + { + "alg": "SHA-256", + "content": "14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d" + }, + { + "alg": "SHA-256", + "content": "17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398" + }, + { + "alg": "SHA-256", + "content": "184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d" + }, + { + "alg": "SHA-256", + "content": "18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3" + }, + { + "alg": "SHA-256", + "content": "19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f" + }, + { + "alg": "SHA-256", + "content": "1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb" + }, + { + "alg": "SHA-256", + "content": "1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7" + }, + { + "alg": "SHA-256", + "content": "2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5" + }, + { + "alg": "SHA-256", + "content": "228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9" + }, + { + "alg": "SHA-256", + "content": "23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462" + }, + { + "alg": "SHA-256", + "content": "2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4" + }, + { + "alg": "SHA-256", + "content": "29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b" + }, + { + "alg": "SHA-256", + "content": "2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d" + }, + { + "alg": "SHA-256", + "content": "3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df" + }, + { + "alg": "SHA-256", + "content": "3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2" + }, + { + "alg": "SHA-256", + "content": "3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0" + }, + { + "alg": "SHA-256", + "content": "372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519" + }, + { + "alg": "SHA-256", + "content": "395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd" + }, + { + "alg": "SHA-256", + "content": "3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7" + }, + { + "alg": "SHA-256", + "content": "3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac" + }, + { + "alg": "SHA-256", + "content": "3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6" + }, + { + "alg": "SHA-256", + "content": "3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565" + }, + { + "alg": "SHA-256", + "content": "3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898" + }, + { + "alg": "SHA-256", + "content": "428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb" + }, + { + "alg": "SHA-256", + "content": "432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928" + }, + { + "alg": "SHA-256", + "content": "4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6" + }, + { + "alg": "SHA-256", + "content": "4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3" + }, + { + "alg": "SHA-256", + "content": "4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a" + }, + { + "alg": "SHA-256", + "content": "4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596" + }, + { + "alg": "SHA-256", + "content": "56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987" + }, + { + "alg": "SHA-256", + "content": "5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e" + }, + { + "alg": "SHA-256", + "content": "5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d" + }, + { + "alg": "SHA-256", + "content": "5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712" + }, + { + "alg": "SHA-256", + "content": "5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008" + }, + { + "alg": "SHA-256", + "content": "617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd" + }, + { + "alg": "SHA-256", + "content": "62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1" + }, + { + "alg": "SHA-256", + "content": "633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be" + }, + { + "alg": "SHA-256", + "content": "66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea" + }, + { + "alg": "SHA-256", + "content": "6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292" + }, + { + "alg": "SHA-256", + "content": "6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33" + }, + { + "alg": "SHA-256", + "content": "7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3" + }, + { + "alg": "SHA-256", + "content": "7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4" + }, + { + "alg": "SHA-256", + "content": "7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b" + }, + { + "alg": "SHA-256", + "content": "7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826" + }, + { + "alg": "SHA-256", + "content": "811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac" + }, + { + "alg": "SHA-256", + "content": "8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7" + }, + { + "alg": "SHA-256", + "content": "82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d" + }, + { + "alg": "SHA-256", + "content": "8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf" + }, + { + "alg": "SHA-256", + "content": "85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4" + }, + { + "alg": "SHA-256", + "content": "86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc" + }, + { + "alg": "SHA-256", + "content": "8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15" + }, + { + "alg": "SHA-256", + "content": "8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3" + }, + { + "alg": "SHA-256", + "content": "8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b" + }, + { + "alg": "SHA-256", + "content": "8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914" + }, + { + "alg": "SHA-256", + "content": "9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04" + }, + { + "alg": "SHA-256", + "content": "905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c" + }, + { + "alg": "SHA-256", + "content": "90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b" + }, + { + "alg": "SHA-256", + "content": "91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9" + }, + { + "alg": "SHA-256", + "content": "926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce" + }, + { + "alg": "SHA-256", + "content": "9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4" + }, + { + "alg": "SHA-256", + "content": "9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a" + }, + { + "alg": "SHA-256", + "content": "962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f" + }, + { + "alg": "SHA-256", + "content": "97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424" + }, + { + "alg": "SHA-256", + "content": "9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894" + }, + { + "alg": "SHA-256", + "content": "9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9" + }, + { + "alg": "SHA-256", + "content": "9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76" + }, + { + "alg": "SHA-256", + "content": "9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201" + }, + { + "alg": "SHA-256", + "content": "9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb" + }, + { + "alg": "SHA-256", + "content": "a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109" + }, + { + "alg": "SHA-256", + "content": "a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4" + }, + { + "alg": "SHA-256", + "content": "aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848" + }, + { + "alg": "SHA-256", + "content": "ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526" + }, + { + "alg": "SHA-256", + "content": "af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0" + }, + { + "alg": "SHA-256", + "content": "b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01" + }, + { + "alg": "SHA-256", + "content": "b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458" + }, + { + "alg": "SHA-256", + "content": "b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e" + }, + { + "alg": "SHA-256", + "content": "bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba" + }, + { + "alg": "SHA-256", + "content": "bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a" + }, + { + "alg": "SHA-256", + "content": "c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39" + }, + { + "alg": "SHA-256", + "content": "c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c" + }, + { + "alg": "SHA-256", + "content": "c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000" + }, + { + "alg": "SHA-256", + "content": "c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b" + }, + { + "alg": "SHA-256", + "content": "c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf" + }, + { + "alg": "SHA-256", + "content": "c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4" + }, + { + "alg": "SHA-256", + "content": "cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd" + }, + { + "alg": "SHA-256", + "content": "cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28" + }, + { + "alg": "SHA-256", + "content": "ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9" + }, + { + "alg": "SHA-256", + "content": "d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30" + }, + { + "alg": "SHA-256", + "content": "d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983" + }, + { + "alg": "SHA-256", + "content": "d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1" + }, + { + "alg": "SHA-256", + "content": "d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76" + }, + { + "alg": "SHA-256", + "content": "da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5" + }, + { + "alg": "SHA-256", + "content": "daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4" + }, + { + "alg": "SHA-256", + "content": "db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7" + }, + { + "alg": "SHA-256", + "content": "e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c" + }, + { + "alg": "SHA-256", + "content": "e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066" + }, + { + "alg": "SHA-256", + "content": "e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3" + }, + { + "alg": "SHA-256", + "content": "e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02" + }, + { + "alg": "SHA-256", + "content": "e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89" + }, + { + "alg": "SHA-256", + "content": "ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50" + }, + { + "alg": "SHA-256", + "content": "f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76" + }, + { + "alg": "SHA-256", + "content": "f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49" + }, + { + "alg": "SHA-256", + "content": "f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b" + }, + { + "alg": "SHA-256", + "content": "f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d" + }, + { + "alg": "SHA-256", + "content": "f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7" + }, + { + "alg": "SHA-256", + "content": "f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4" + }, + { + "alg": "SHA-256", + "content": "fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c" + }, + { + "alg": "SHA-256", + "content": "fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e" + }, + { + "alg": "SHA-256", + "content": "fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff" + }, + { + "alg": "SHA-256", + "content": "fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae" + } + ], + "name": "pydantic-core", + "purl": "pkg:pypi/pydantic-core@2.46.4", + "type": "library", + "version": "2.46.4" + }, + { + "bom-ref": "pkg:pypi/pyflakes@3.4.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58" + }, + { + "alg": "SHA-256", + "content": "f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f" + } + ], + "name": "pyflakes", + "purl": "pkg:pypi/pyflakes@3.4.0", + "type": "library", + "version": "3.4.0" + }, + { + "bom-ref": "pkg:pypi/pygments@2.20.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f" + }, + { + "alg": "SHA-256", + "content": "81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176" + } + ], + "name": "pygments", + "purl": "pkg:pypi/pygments@2.20.0", + "type": "library", + "version": "2.20.0" + }, + { + "bom-ref": "pkg:pypi/pysocks@1.7.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299" + }, + { + "alg": "SHA-256", + "content": "2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5" + }, + { + "alg": "SHA-256", + "content": "3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0" + } + ], + "name": "pysocks", + "purl": "pkg:pypi/pysocks@1.7.1", + "type": "library", + "version": "1.7.1" + }, + { + "bom-ref": "pkg:pypi/pytest@9.1.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313" + }, + { + "alg": "SHA-256", + "content": "37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c" + } + ], + "name": "pytest", + "purl": "pkg:pypi/pytest@9.1.1", + "type": "library", + "version": "9.1.1" + }, + { + "bom-ref": "pkg:pypi/pytest-cov@7.1.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2" + }, + { + "alg": "SHA-256", + "content": "a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678" + } + ], + "name": "pytest-cov", + "purl": "pkg:pypi/pytest-cov@7.1.0", + "type": "library", + "version": "7.1.0" + }, + { + "bom-ref": "pkg:pypi/python-jose@3.5.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771" + }, + { + "alg": "SHA-256", + "content": "fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b" + } + ], + "name": "python-jose", + "purl": "pkg:pypi/python-jose@3.5.0", + "type": "library", + "version": "3.5.0" + }, + { + "bom-ref": "pkg:pypi/python-multipart@0.0.32", + "hashes": [ + { + "alg": "SHA-256", + "content": "be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e" + }, + { + "alg": "SHA-256", + "content": "ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23" + } + ], + "name": "python-multipart", + "purl": "pkg:pypi/python-multipart@0.0.32", + "type": "library", + "version": "0.0.32" + }, + { + "bom-ref": "pkg:pypi/requests@2.34.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0" + }, + { + "alg": "SHA-256", + "content": "f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed" + } + ], + "name": "requests", + "purl": "pkg:pypi/requests@2.34.2", + "type": "library", + "version": "2.34.2" + }, + { + "bom-ref": "pkg:pypi/rsa@4.9.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762" + }, + { + "alg": "SHA-256", + "content": "e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75" + } + ], + "name": "rsa", + "purl": "pkg:pypi/rsa@4.9.1", + "type": "library", + "version": "4.9.1" + }, + { + "bom-ref": "pkg:pypi/selectolax@0.4.11", + "hashes": [ + { + "alg": "SHA-256", + "content": "086ca6f7e4c475bfff871ec1448ae5d342d43d6a2ca2cea65160d01b3a6a75ec" + }, + { + "alg": "SHA-256", + "content": "168b34466dc4f998d7ceb04f218693546d141543e7c5d327f9e006c0799cd62c" + }, + { + "alg": "SHA-256", + "content": "1a6deb4464198ac67f32e56c4463aedf3e1d834b458eaac5b5b5b1ef02dcf15e" + }, + { + "alg": "SHA-256", + "content": "1da54e42ab99b9191269306e13c0fd67ada1c6654e8dc8d74fac615931dd3c62" + }, + { + "alg": "SHA-256", + "content": "21359cf4d1261b314895c07321e76aa2334b1967956efba04cc0d223d7817f48" + }, + { + "alg": "SHA-256", + "content": "224682039ca13eb822be626e49a03592ee2b8557bcdc6381e49417a995170c94" + }, + { + "alg": "SHA-256", + "content": "25d633cddca0fc769e7d890e9e838908fb4a7326eec5e3b23ae42c27f457541d" + }, + { + "alg": "SHA-256", + "content": "28915b8fa90c1c3cb585858a6d24d433a3f38ea514aea59013bdb0930d9f6025" + }, + { + "alg": "SHA-256", + "content": "2b565ddabce6c9a7b73fa28a39acf8f411a084fa2f169234ec2470f552d4421d" + }, + { + "alg": "SHA-256", + "content": "2b842c829f916fecb51f0f55882eca3e2ad49e85388178f14ae6fe0912be0a57" + }, + { + "alg": "SHA-256", + "content": "2e8d01d9688e9670b7ed3f042dc7ba379f844fb7fcb8e85191885ed75d6178fb" + }, + { + "alg": "SHA-256", + "content": "3066be35f630e4c4a00cf2c829548574fcba5963735411e6a9ac78bd4ca830ce" + }, + { + "alg": "SHA-256", + "content": "36183968e5c122da414db0dd7d22d4a6ca808e9b9e4837532dfbe55df6f26bff" + }, + { + "alg": "SHA-256", + "content": "3637d21f7fe60fbd6ca3dbc67a1747f6a55a9389114d72f06b5d69ba2beddf01" + }, + { + "alg": "SHA-256", + "content": "386494779e5464e587ed4dc076e1c48c24ebaf2da1e3a249690551d1f97fe8ed" + }, + { + "alg": "SHA-256", + "content": "3abf5aecfd0c314b62c10ce6e9baee2094cb385e2ec154bd1743febd2fcc9bdb" + }, + { + "alg": "SHA-256", + "content": "41f388c26304c1d840f5ee5e07c06bb9388ec834d10fec60dc148f22f98efd38" + }, + { + "alg": "SHA-256", + "content": "466daca0599408c9d2cad7658a68490facc5c9b8d0f41ac5d17948914f57306f" + }, + { + "alg": "SHA-256", + "content": "47be0e591125484de14ff0c9aaaa814dd4a4019de35eabe360e88169a263a2b5" + }, + { + "alg": "SHA-256", + "content": "48545b0351b6f92c4ad2fff835e832768d0cb37834766a82f40242e51fc9901d" + }, + { + "alg": "SHA-256", + "content": "4c3d4fd42deabfd03cc5c4fce9517e7226054f9de3984759476e6c4a7b7fe187" + }, + { + "alg": "SHA-256", + "content": "5153157ed60d968ed303acbfd2c8762fa0c0462e2663bd04466471c565deb88a" + }, + { + "alg": "SHA-256", + "content": "57e71c021d9950113e7d49892eba56014c072c9d86b71c0b3b7cb846ee913e20" + }, + { + "alg": "SHA-256", + "content": "597b8e065978be200c598ae6d682496d96fbce14d34b5d519e93cf5b6be5fb60" + }, + { + "alg": "SHA-256", + "content": "5c7a91fbe1a94849d85228897c416ab9b4518bea6b04dce8ef8acd825ec80e9d" + }, + { + "alg": "SHA-256", + "content": "5ff73122f5fee3350528235349f1b9e5626a1a0ef1887fef82d7fddac18f6605" + }, + { + "alg": "SHA-256", + "content": "634db1b7ae1b1f10ee0b0adbef0eee1aec65a3d01f09b75132bc5b043b7623d1" + }, + { + "alg": "SHA-256", + "content": "663ff792f92ed749cfcf452ac19aff5da74b05521e7daacb3b74388deb14d117" + }, + { + "alg": "SHA-256", + "content": "75c13f519ecb85185e149049bd4c4937aa0b953bd21e1748eb711c6cdcee571e" + }, + { + "alg": "SHA-256", + "content": "75eeddbd3db7e82293a325c210cf4d218d25961870dbb468383c7516e3f577aa" + }, + { + "alg": "SHA-256", + "content": "80af1c7345701934769679a83fcc86526c56eef590423cc2d55cc1eb81adfa5f" + }, + { + "alg": "SHA-256", + "content": "8166bb8cad8f2eabed664f689b708851078b74dd50dd93e426b291095badc2cb" + }, + { + "alg": "SHA-256", + "content": "81f21972788df01b83a5940d9b7d62e6e222f80872c2cfaf67927276e8ef4975" + }, + { + "alg": "SHA-256", + "content": "83515cc0fe9b8c4a1ca8bfcd83cf48408bcdd3f2fccd088f04635619e93695c9" + }, + { + "alg": "SHA-256", + "content": "8da39a07a589fd181b5e8d25f695d7d40d3a1d89e47c2e00c08ece7fff5ddd3d" + }, + { + "alg": "SHA-256", + "content": "8f0223a83051ece2591a1d0cf487f1d0719cb76660d7c5de4c0a201b761c5c23" + }, + { + "alg": "SHA-256", + "content": "8ff49b24d4e5c81bcc96b25e2cf23d9af2d06e4d473865c812487be6665416c0" + }, + { + "alg": "SHA-256", + "content": "903286e6d9752b63b3b37f9ee98a8ff7db403b23416b469b065cc992b2556b80" + }, + { + "alg": "SHA-256", + "content": "9077fa36e99ef4bb801194ff8f492f67279c0562e7cdfa9b4d06f5c010131969" + }, + { + "alg": "SHA-256", + "content": "96de39e2f28c359cf930547193b2cacf0ab6b9187309e2515461fff9d5a5aa3c" + }, + { + "alg": "SHA-256", + "content": "9862c1d0435e89e419e0c595084eaa8f6f71c3fb7a968bdff51db595cec07161" + }, + { + "alg": "SHA-256", + "content": "991866e2f45ce85e3ce7a31714e522f620d83b1eca15fe530055108a94e6087e" + }, + { + "alg": "SHA-256", + "content": "a532f1993f08f627300891751982dd9641ce1bd29618b535232aec9fa023a1c8" + }, + { + "alg": "SHA-256", + "content": "ad4bbf08af4e2c89f78dd12a7b8265f0924d0434705bd955b5a262297a924452" + }, + { + "alg": "SHA-256", + "content": "b0f56c49161b18621ac452e42e02b0c5c61ba4c21095cfff3990e040bd9a043c" + }, + { + "alg": "SHA-256", + "content": "b530a2c4fad7400af27b2b7e0333c1318ecb5f5dc38e8a141dbe3bd81b398fdf" + }, + { + "alg": "SHA-256", + "content": "bd843540a197a33049a08fd80e59bfeafbaa688e632d53a05a9b65af5e88296f" + }, + { + "alg": "SHA-256", + "content": "bde121202b33e6041e9d2db1d62e7466b5883fd1c441eb96ff68d3ea015cfcbd" + }, + { + "alg": "SHA-256", + "content": "c3f6fac361b5f415c409dfd1a07dd0e9a5899d10daba8d88ce16bd552b0e06f2" + }, + { + "alg": "SHA-256", + "content": "d00753a5f832625f69b585dda7a7ddc1a7d2803c31aa8710c5d209fca064d774" + }, + { + "alg": "SHA-256", + "content": "d33e2ed75cc33e7af3fd50521c33e7d8634fae23bc197a6cee6a5015e056eef6" + }, + { + "alg": "SHA-256", + "content": "d5ce592a92fceeca2694b369a83ad72891a9c356f668718fe7e1c83eea407bb4" + }, + { + "alg": "SHA-256", + "content": "daf7a841d1baa795f940200bec45019c4a31020def16e3a4e35485e82ed64167" + }, + { + "alg": "SHA-256", + "content": "dbee8998d426c1b6b78aab7a051009f9de38d77b93d073025ae19f7d71b3ead3" + }, + { + "alg": "SHA-256", + "content": "dc8d449adfdef675bcbfc67d5548513271c7ea869e9b71a2e898138bf34a976a" + }, + { + "alg": "SHA-256", + "content": "dcd24bfc4899e4df2bdcbabc973384367686f391e5a2d8ebb229c043f43be82a" + }, + { + "alg": "SHA-256", + "content": "dff45ee85409e6126e4900e0db6950a59a7b58b7ff9d667a6a275b0405b87692" + }, + { + "alg": "SHA-256", + "content": "e2d7e1a2b67a5e8d251733b2c3f978ed26a56d04552472e0a2fb5ea41fa4e663" + }, + { + "alg": "SHA-256", + "content": "e5929cbe3eedfaf51a09ec89642ab5355b703486d43bcf3c8f0c27d6043a488d" + }, + { + "alg": "SHA-256", + "content": "f4894d4cb2b9cd4828dc9d9cbed683db051748fde66f5937aa823f2afff16b36" + }, + { + "alg": "SHA-256", + "content": "f69c377298a09f571c89af5916ac85f69a9aaedd7312b0bb803a46d2ca2f9048" + }, + { + "alg": "SHA-256", + "content": "f8121f4cbfe870d9ad24ae418a735d918d55844e905c3270077e97f4e579770f" + }, + { + "alg": "SHA-256", + "content": "f8f014f328f6f79a364293bca54d43af1cec776dc10a5302054a54b5fb2d8675" + }, + { + "alg": "SHA-256", + "content": "fad5b1065f73eeaa07ea343cbc548aaa9f9a5c359c3bdd8d98f5d80b61550d1c" + } + ], + "name": "selectolax", + "purl": "pkg:pypi/selectolax@0.4.11", + "type": "library", + "version": "0.4.11" + }, + { + "bom-ref": "pkg:pypi/sgmllib3k@1.0.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9" + } + ], + "name": "sgmllib3k", + "purl": "pkg:pypi/sgmllib3k@1.0.0", + "type": "library", + "version": "1.0.0" + }, + { + "bom-ref": "pkg:pypi/six@1.17.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274" + }, + { + "alg": "SHA-256", + "content": "ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81" + } + ], + "name": "six", + "purl": "pkg:pypi/six@1.17.0", + "type": "library", + "version": "1.17.0" + }, + { + "bom-ref": "pkg:pypi/sniffio@1.3.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2" + }, + { + "alg": "SHA-256", + "content": "f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc" + } + ], + "name": "sniffio", + "purl": "pkg:pypi/sniffio@1.3.1", + "type": "library", + "version": "1.3.1" + }, + { + "bom-ref": "pkg:pypi/soupsieve@2.9.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c" + }, + { + "alg": "SHA-256", + "content": "c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba" + } + ], + "name": "soupsieve", + "purl": "pkg:pypi/soupsieve@2.9.1", + "type": "library", + "version": "2.9.1" + }, + { + "bom-ref": "pkg:pypi/sqlalchemy@2.0.51", + "hashes": [ + { + "alg": "SHA-256", + "content": "0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23" + }, + { + "alg": "SHA-256", + "content": "0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1" + }, + { + "alg": "SHA-256", + "content": "08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8" + }, + { + "alg": "SHA-256", + "content": "0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72" + }, + { + "alg": "SHA-256", + "content": "0e8203d2fbd5c6254692ef0a72c740d75b2f3c7ca345404f4c1a4604813c77c0" + }, + { + "alg": "SHA-256", + "content": "0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5" + }, + { + "alg": "SHA-256", + "content": "0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e" + }, + { + "alg": "SHA-256", + "content": "111604e637da87031255ddc26c7d7bc22bc6af6f5d459ccff3af1b4660233a85" + }, + { + "alg": "SHA-256", + "content": "1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d" + }, + { + "alg": "SHA-256", + "content": "159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2" + }, + { + "alg": "SHA-256", + "content": "1aa10c0daee6705294d181daadaa793221e1a59ed55000a3fab1d42b088ce4ba" + }, + { + "alg": "SHA-256", + "content": "1af05726b3d0cdba1c55284bf408fd3b792e690fe2399bfb8304565551cda652" + }, + { + "alg": "SHA-256", + "content": "1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f" + }, + { + "alg": "SHA-256", + "content": "1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9" + }, + { + "alg": "SHA-256", + "content": "1e47b1199c2e832e325eacabc8d32d2487f58c9358f97e9a00f5eb93c5680d84" + }, + { + "alg": "SHA-256", + "content": "247acaa29ccef6250dfd6a3eedf8f94ddf23564180a39fe362e32ae9dbdbde46" + }, + { + "alg": "SHA-256", + "content": "2a97eaad21c84b4ef8010b11eeba9fe6153eb0b3df3ff8b6abc309df1b978ef7" + }, + { + "alg": "SHA-256", + "content": "2cf39aabdf48e87c1c2c2ed6d20d33ffa0733b3071ce9c5f66357947dd009080" + }, + { + "alg": "SHA-256", + "content": "2e54ff2dd657f2e3e0fbf2b097db1182f7bfea263eca4353f00065bae2a67c3d" + }, + { + "alg": "SHA-256", + "content": "39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d" + }, + { + "alg": "SHA-256", + "content": "4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54" + }, + { + "alg": "SHA-256", + "content": "436728ce18a80f6951a1e11cc6112c2ede9faf20766f1a26195a7c441ca12dbd" + }, + { + "alg": "SHA-256", + "content": "483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195" + }, + { + "alg": "SHA-256", + "content": "4a011ea4510683319ce4ed274b56ee05194b39b6da9d09ca7a39388f0fa84dcc" + }, + { + "alg": "SHA-256", + "content": "581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e" + }, + { + "alg": "SHA-256", + "content": "59cab3686b1bc039dd9cded2f8d0c08a246e84e76bd4ab5b4f18c7cdae293825" + }, + { + "alg": "SHA-256", + "content": "6b588fd681ddf0c196b8df1ea49a8913514894b2b8f945a9511b4b48871f99c8" + }, + { + "alg": "SHA-256", + "content": "6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522" + }, + { + "alg": "SHA-256", + "content": "6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491" + }, + { + "alg": "SHA-256", + "content": "72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400" + }, + { + "alg": "SHA-256", + "content": "740cf6f35351b1ac3d82369152acf1d51d37e3dcf85d4dc0a22ca01410eabe2a" + }, + { + "alg": "SHA-256", + "content": "7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07" + }, + { + "alg": "SHA-256", + "content": "7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7" + }, + { + "alg": "SHA-256", + "content": "7d78702b26ba1c18b2d0fb2ea940ba7f17a9581b42e8361ff93920ebbee1235a" + }, + { + "alg": "SHA-256", + "content": "804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9" + }, + { + "alg": "SHA-256", + "content": "9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7" + }, + { + "alg": "SHA-256", + "content": "96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499" + }, + { + "alg": "SHA-256", + "content": "9f380393be5abeb6815f68fd39271b95127173511b6706b0a630a9995d53f8f5" + }, + { + "alg": "SHA-256", + "content": "a42ad6afcbaaa777241e347aa2e29155993045a0d6b7db74da61053ffe875fe0" + }, + { + "alg": "SHA-256", + "content": "a5b2ed6d828f1f09bd812861f4f59ca3bc3803f9df871f4555187f0faf018604" + }, + { + "alg": "SHA-256", + "content": "a6d26094615306d116dd5e4a51b0304c99dd2356fc569eed6922a80a6bd3b265" + }, + { + "alg": "SHA-256", + "content": "aa18ae738b5170e253ad0bb6c4b0f07585081e8a6e50893e4d911d47b39a0904" + }, + { + "alg": "SHA-256", + "content": "ad30ae663711786303fbcd46a47516302d201ee49a877cb3fac61f672895110a" + }, + { + "alg": "SHA-256", + "content": "b21f0e7efc7a5c509e953784e9d1575ebb8b4318960e7e7d7a93bb803626cf64" + }, + { + "alg": "SHA-256", + "content": "b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d" + }, + { + "alg": "SHA-256", + "content": "b7f08588854bbb724041d9ae9d980d40040c922382e1d9a2ecb390edc4fd5032" + }, + { + "alg": "SHA-256", + "content": "b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b" + }, + { + "alg": "SHA-256", + "content": "bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5" + }, + { + "alg": "SHA-256", + "content": "bb1f5062f98b0b3290e72b707747fdd7e0f22d6956b236ba7ca7f5c9971d2da2" + }, + { + "alg": "SHA-256", + "content": "c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d" + }, + { + "alg": "SHA-256", + "content": "c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389" + }, + { + "alg": "SHA-256", + "content": "c68568f3facf8f66fa76c60e0ced69b67666ffa9941d1d0a3756fda196049080" + }, + { + "alg": "SHA-256", + "content": "c95ef01f53233a305a874a44a63fbfb1d81cd79b49de0f8529b3548cde437e37" + }, + { + "alg": "SHA-256", + "content": "ca216e8af5c05e326efc7e28716ac2381a7cf9791749f5ee1849dccdc99c9b00" + }, + { + "alg": "SHA-256", + "content": "ca8435d13829b92f4a97362d91975154a4015db3a2634154e1754e9a915e6b86" + }, + { + "alg": "SHA-256", + "content": "dc261707bf5739aea8a541593f3cc1d463c2701fb05fbcbba0ce031b69a21260" + }, + { + "alg": "SHA-256", + "content": "e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de" + }, + { + "alg": "SHA-256", + "content": "fa268106c8987639a17a18514cfe0cd9bf17420ab887e1e1bf486da8836135b1" + } + ], + "name": "sqlalchemy", + "purl": "pkg:pypi/sqlalchemy@2.0.51", + "type": "library", + "version": "2.0.51" + }, + { + "bom-ref": "pkg:pypi/starlette@1.3.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0" + }, + { + "alg": "SHA-256", + "content": "c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6" + } + ], + "name": "starlette", + "purl": "pkg:pypi/starlette@1.3.1", + "type": "library", + "version": "1.3.1" + }, + { + "bom-ref": "pkg:pypi/tenacity@9.1.4", + "hashes": [ + { + "alg": "SHA-256", + "content": "6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55" + }, + { + "alg": "SHA-256", + "content": "adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a" + } + ], + "name": "tenacity", + "purl": "pkg:pypi/tenacity@9.1.4", + "type": "library", + "version": "9.1.4" + }, + { + "bom-ref": "pkg:pypi/typing-extensions@4.16.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8" + }, + { + "alg": "SHA-256", + "content": "dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5" + } + ], + "name": "typing-extensions", + "purl": "pkg:pypi/typing-extensions@4.16.0", + "type": "library", + "version": "4.16.0" + }, + { + "bom-ref": "pkg:pypi/typing-inspection@0.4.2", + "hashes": [ + { + "alg": "SHA-256", + "content": "4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7" + }, + { + "alg": "SHA-256", + "content": "ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464" + } + ], + "name": "typing-inspection", + "purl": "pkg:pypi/typing-inspection@0.4.2", + "type": "library", + "version": "0.4.2" + }, + { + "bom-ref": "pkg:pypi/urllib3@2.7.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c" + }, + { + "alg": "SHA-256", + "content": "9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897" + } + ], + "name": "urllib3", + "purl": "pkg:pypi/urllib3@2.7.0", + "type": "library", + "version": "2.7.0" + }, + { + "bom-ref": "pkg:pypi/uvicorn@0.51.0", + "hashes": [ + { + "alg": "SHA-256", + "content": "5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b" + }, + { + "alg": "SHA-256", + "content": "f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0" + } + ], + "name": "uvicorn", + "purl": "pkg:pypi/uvicorn@0.51.0", + "type": "library", + "version": "0.51.0" + }, + { + "bom-ref": "pkg:pypi/websockets@16.1.1", + "hashes": [ + { + "alg": "SHA-256", + "content": "01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175" + }, + { + "alg": "SHA-256", + "content": "024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a" + }, + { + "alg": "SHA-256", + "content": "04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d" + }, + { + "alg": "SHA-256", + "content": "056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985" + }, + { + "alg": "SHA-256", + "content": "0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1" + }, + { + "alg": "SHA-256", + "content": "0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573" + }, + { + "alg": "SHA-256", + "content": "1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb" + }, + { + "alg": "SHA-256", + "content": "125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9" + }, + { + "alg": "SHA-256", + "content": "130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87" + }, + { + "alg": "SHA-256", + "content": "1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22" + }, + { + "alg": "SHA-256", + "content": "195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328" + }, + { + "alg": "SHA-256", + "content": "1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747" + }, + { + "alg": "SHA-256", + "content": "1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab" + }, + { + "alg": "SHA-256", + "content": "1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499" + }, + { + "alg": "SHA-256", + "content": "23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d" + }, + { + "alg": "SHA-256", + "content": "249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62" + }, + { + "alg": "SHA-256", + "content": "29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512" + }, + { + "alg": "SHA-256", + "content": "2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7" + }, + { + "alg": "SHA-256", + "content": "2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf" + }, + { + "alg": "SHA-256", + "content": "2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa" + }, + { + "alg": "SHA-256", + "content": "2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57" + }, + { + "alg": "SHA-256", + "content": "30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e" + }, + { + "alg": "SHA-256", + "content": "34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3" + }, + { + "alg": "SHA-256", + "content": "38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0" + }, + { + "alg": "SHA-256", + "content": "387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc" + }, + { + "alg": "SHA-256", + "content": "39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43" + }, + { + "alg": "SHA-256", + "content": "3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe" + }, + { + "alg": "SHA-256", + "content": "414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31" + }, + { + "alg": "SHA-256", + "content": "41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b" + }, + { + "alg": "SHA-256", + "content": "42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383" + }, + { + "alg": "SHA-256", + "content": "42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217" + }, + { + "alg": "SHA-256", + "content": "43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499" + }, + { + "alg": "SHA-256", + "content": "443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8" + }, + { + "alg": "SHA-256", + "content": "46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51" + }, + { + "alg": "SHA-256", + "content": "496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509" + }, + { + "alg": "SHA-256", + "content": "49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d" + }, + { + "alg": "SHA-256", + "content": "4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428" + }, + { + "alg": "SHA-256", + "content": "4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead" + }, + { + "alg": "SHA-256", + "content": "4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc" + }, + { + "alg": "SHA-256", + "content": "5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1" + }, + { + "alg": "SHA-256", + "content": "53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3" + }, + { + "alg": "SHA-256", + "content": "536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0" + }, + { + "alg": "SHA-256", + "content": "54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f" + }, + { + "alg": "SHA-256", + "content": "56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5" + }, + { + "alg": "SHA-256", + "content": "5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731" + }, + { + "alg": "SHA-256", + "content": "5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be" + }, + { + "alg": "SHA-256", + "content": "5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df" + }, + { + "alg": "SHA-256", + "content": "61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb" + }, + { + "alg": "SHA-256", + "content": "6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293" + }, + { + "alg": "SHA-256", + "content": "69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e" + }, + { + "alg": "SHA-256", + "content": "69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7" + }, + { + "alg": "SHA-256", + "content": "6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3" + }, + { + "alg": "SHA-256", + "content": "6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3" + }, + { + "alg": "SHA-256", + "content": "6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3" + }, + { + "alg": "SHA-256", + "content": "7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a" + }, + { + "alg": "SHA-256", + "content": "7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9" + }, + { + "alg": "SHA-256", + "content": "79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56" + }, + { + "alg": "SHA-256", + "content": "7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562" + }, + { + "alg": "SHA-256", + "content": "7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0" + }, + { + "alg": "SHA-256", + "content": "8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15" + }, + { + "alg": "SHA-256", + "content": "820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869" + }, + { + "alg": "SHA-256", + "content": "8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7" + }, + { + "alg": "SHA-256", + "content": "84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999" + }, + { + "alg": "SHA-256", + "content": "86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01" + }, + { + "alg": "SHA-256", + "content": "8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57" + }, + { + "alg": "SHA-256", + "content": "90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838" + }, + { + "alg": "SHA-256", + "content": "9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4" + }, + { + "alg": "SHA-256", + "content": "92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1" + }, + { + "alg": "SHA-256", + "content": "952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458" + }, + { + "alg": "SHA-256", + "content": "97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3" + }, + { + "alg": "SHA-256", + "content": "9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392" + }, + { + "alg": "SHA-256", + "content": "9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3" + }, + { + "alg": "SHA-256", + "content": "9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a" + }, + { + "alg": "SHA-256", + "content": "a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9" + }, + { + "alg": "SHA-256", + "content": "a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785" + }, + { + "alg": "SHA-256", + "content": "a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648" + }, + { + "alg": "SHA-256", + "content": "aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49" + }, + { + "alg": "SHA-256", + "content": "ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1" + }, + { + "alg": "SHA-256", + "content": "b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7" + }, + { + "alg": "SHA-256", + "content": "b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d" + }, + { + "alg": "SHA-256", + "content": "b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a" + }, + { + "alg": "SHA-256", + "content": "bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6" + }, + { + "alg": "SHA-256", + "content": "bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231" + }, + { + "alg": "SHA-256", + "content": "cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00" + }, + { + "alg": "SHA-256", + "content": "cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac" + }, + { + "alg": "SHA-256", + "content": "cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea" + }, + { + "alg": "SHA-256", + "content": "d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c" + }, + { + "alg": "SHA-256", + "content": "d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81" + }, + { + "alg": "SHA-256", + "content": "d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f" + }, + { + "alg": "SHA-256", + "content": "d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751" + }, + { + "alg": "SHA-256", + "content": "d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68" + }, + { + "alg": "SHA-256", + "content": "da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2" + }, + { + "alg": "SHA-256", + "content": "dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c" + }, + { + "alg": "SHA-256", + "content": "db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57" + }, + { + "alg": "SHA-256", + "content": "dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b" + }, + { + "alg": "SHA-256", + "content": "dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165" + }, + { + "alg": "SHA-256", + "content": "dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737" + }, + { + "alg": "SHA-256", + "content": "e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b" + }, + { + "alg": "SHA-256", + "content": "e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854" + }, + { + "alg": "SHA-256", + "content": "ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87" + }, + { + "alg": "SHA-256", + "content": "f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2" + }, + { + "alg": "SHA-256", + "content": "f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847" + }, + { + "alg": "SHA-256", + "content": "f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d" + }, + { + "alg": "SHA-256", + "content": "f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4" + }, + { + "alg": "SHA-256", + "content": "f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8" + }, + { + "alg": "SHA-256", + "content": "f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d" + }, + { + "alg": "SHA-256", + "content": "fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29" + }, + { + "alg": "SHA-256", + "content": "fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051" + }, + { + "alg": "SHA-256", + "content": "fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a" + } + ], + "name": "websockets", + "purl": "pkg:pypi/websockets@16.1.1", + "type": "library", + "version": "16.1.1" + } + ], + "metadata": { + "component": { + "name": "foundation-intelligence-backend", + "type": "application", + "version": "1.0.0" + }, + "properties": [ + { + "name": "generation.source", + "value": "committed-lockfile" + }, + { + "name": "generation.reproducible", + "value": "true" + } + ] + }, + "specVersion": "1.5", + "version": 1 +} diff --git a/docs/remediation/evidence/sbom/frontend.cdx.json b/docs/remediation/evidence/sbom/frontend.cdx.json new file mode 100644 index 0000000..d2ffb0a --- /dev/null +++ b/docs/remediation/evidence/sbom/frontend.cdx.json @@ -0,0 +1,1767 @@ +{ + "bomFormat": "CycloneDX", + "components": [ + { + "bom-ref": "pkg:npm/@axe-core/playwright@4.12.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "acc77bc6b8a9b6a2a93fec39dbae62e07764bf65f991bbbaba2062fc1d88eee7f786279104cdea0c27da28fb717e261bda62977c5fb22ce0c9c08c7e26fd460f" + } + ], + "name": "@axe-core/playwright", + "purl": "pkg:npm/@axe-core/playwright@4.12.1", + "scope": "optional", + "type": "library", + "version": "4.12.1" + }, + { + "bom-ref": "pkg:npm/@emnapi/core@1.11.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "452bdb4261f374accdb0b61aff01eb6dcdca378b182ca01d3d9c6a88cd87013aaffd2064dbf10d487a6f5c668b38c72c032cf4a68106aa49a62981b739688911" + } + ], + "name": "@emnapi/core", + "purl": "pkg:npm/@emnapi/core@1.11.1", + "scope": "optional", + "type": "library", + "version": "1.11.1" + }, + { + "bom-ref": "pkg:npm/@emnapi/runtime@1.11.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "be08fb477cb75a0c76e0841a18f03f47a6055cb1d530e674b951322103da5acfab7750337c43179400b6d856303b55e4291e8d3ecabb9946a7747f2821175917" + } + ], + "name": "@emnapi/runtime", + "purl": "pkg:npm/@emnapi/runtime@1.11.1", + "scope": "optional", + "type": "library", + "version": "1.11.1" + }, + { + "bom-ref": "pkg:npm/@emnapi/wasi-threads@1.2.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "73de6a39790777274d2a1b1c05379ba840b5095019a72a8e7d57c1cd0d6a833ca5de07de95d5232208036c86600cab072e09ec33e8a01fb4c9fde01ab1a56e30" + } + ], + "name": "@emnapi/wasi-threads", + "purl": "pkg:npm/@emnapi/wasi-threads@1.2.2", + "scope": "optional", + "type": "library", + "version": "1.2.2" + }, + { + "bom-ref": "pkg:npm/@napi-rs/wasm-runtime@1.1.6", + "hashes": [ + { + "alg": "SHA-512", + "content": "64bbff25d51f92f3b2f5e0a79c16867e23be5e299b8de6c078ef8c450a83fc1f85475b6744dd2da4a4891d16c4f2c15f4ba6aab1767aed34237f07ecc542d56e" + } + ], + "name": "@napi-rs/wasm-runtime", + "purl": "pkg:npm/@napi-rs/wasm-runtime@1.1.6", + "scope": "optional", + "type": "library", + "version": "1.1.6" + }, + { + "bom-ref": "pkg:npm/@oxc-project/types@0.139.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "afd807a61b42b3ed4cec9d29c3a4a7fe187f5a96bf890ace3a4acd02554b17f807abefc22661c858a2945217568dc0fa0886bc89d6abb290ac012897097bc553" + } + ], + "name": "@oxc-project/types", + "purl": "pkg:npm/@oxc-project/types@0.139.0", + "scope": "optional", + "type": "library", + "version": "0.139.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-android-arm-eabi@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "fa01ddd769ae548f592c16962cf907b77163ee38a1163810d4c1ad05a44bf2f5ab5ab2343fb74b52dcbf707ac74b4a2a3d8211810509b0fbb6084c50b8c73037" + } + ], + "name": "@oxlint/binding-android-arm-eabi", + "purl": "pkg:npm/@oxlint/binding-android-arm-eabi@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-android-arm64@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "c6329da0c07e1fe442381cafffb97b9df2065bd9a53a2b2a61d7320bbe54a98b9079c2e945e01e11851fd8235e0c42372ad994831a51c3ff9ceb7cb801e17f07" + } + ], + "name": "@oxlint/binding-android-arm64", + "purl": "pkg:npm/@oxlint/binding-android-arm64@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-darwin-arm64@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "8942bbc2f73ab1e8cc2ac0be3edebb9a7b68179c1e15cc84ba78597cb25c794ea02f8d7d99ec73e700644b1fc49e414113130b36d3a2f5f9c349ce717cad08cd" + } + ], + "name": "@oxlint/binding-darwin-arm64", + "purl": "pkg:npm/@oxlint/binding-darwin-arm64@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-darwin-x64@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "82029cfed9f9489d6edb21b68b30ba54a3837d8295f0c43602270994dcce8eecab0b6f6e76f39e7fafc9ccadabdf6c6a0a70432c5a2e475542923cc4234fcdf5" + } + ], + "name": "@oxlint/binding-darwin-x64", + "purl": "pkg:npm/@oxlint/binding-darwin-x64@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-freebsd-x64@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "bbef9d1ffe378f2f614cb6e779a5a54b48256bf069d49770276ce07af0a5f270c5521eaa44218cc5c2f47f495bec1cb703da7f2df16bfbb706e07535846f39ea" + } + ], + "name": "@oxlint/binding-freebsd-x64", + "purl": "pkg:npm/@oxlint/binding-freebsd-x64@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-arm-gnueabihf@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "4a3d739ad1435533de21b233e197dc5c06c51ea0a62825dd094940273bd317b236d0d4c7d510e9a05d8f864a8d383bad2735612589a6de8cf41b0818fadbc4da" + } + ], + "name": "@oxlint/binding-linux-arm-gnueabihf", + "purl": "pkg:npm/@oxlint/binding-linux-arm-gnueabihf@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-arm-musleabihf@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "fff3cac906ffb505dc1c0af1d9fef3fa8548fde312d89a6ff9e74db80b4eae022159d19ca47c6882f780c73985daba47d40207a781ead38445feb8097622029d" + } + ], + "name": "@oxlint/binding-linux-arm-musleabihf", + "purl": "pkg:npm/@oxlint/binding-linux-arm-musleabihf@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-arm64-gnu@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "fe4d4c7be697dad8ee1f5d0aeb698b4b4cbc70b9090475fa09ed313caf9e59e7a5e1b49d10667c76fe3e8583338341ab4a68f0cb8c8062c0cf78478a05fb7fdb" + } + ], + "name": "@oxlint/binding-linux-arm64-gnu", + "purl": "pkg:npm/@oxlint/binding-linux-arm64-gnu@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-arm64-musl@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "ded1528c1c5ce43f3fcef8c4b8bbcea9c03c657283d3ee8db9a54efde75e6a635ce3d32802c6df682f6cd548b0383c2017ab2519a17cc890364cf92e91defbb6" + } + ], + "name": "@oxlint/binding-linux-arm64-musl", + "purl": "pkg:npm/@oxlint/binding-linux-arm64-musl@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-ppc64-gnu@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "f50820b4f9123d73824eef12ce2b3b6ae38afec0bb29d41a37efd3ba33fb615561cc200e8607517e0bfcb84cf4af6b64e71760bace6b2d852b083a1936d89153" + } + ], + "name": "@oxlint/binding-linux-ppc64-gnu", + "purl": "pkg:npm/@oxlint/binding-linux-ppc64-gnu@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-riscv64-gnu@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "54ce553d42780c921688af80667f0549c52a32be8e16b0806326cc6231048bb5b5dcf6ab23ae0c0728974e428ca99a419af43dcf197d11bab65543a265125852" + } + ], + "name": "@oxlint/binding-linux-riscv64-gnu", + "purl": "pkg:npm/@oxlint/binding-linux-riscv64-gnu@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-riscv64-musl@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "49a0d8d6087dace039f76279e20fa0bb9864385150059b0c988607b3e3511c6f94ab43b1b6eb825cc590defbb5f37d04ba0bf9b8c5f28c6f9bbf667dcb82178f" + } + ], + "name": "@oxlint/binding-linux-riscv64-musl", + "purl": "pkg:npm/@oxlint/binding-linux-riscv64-musl@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-s390x-gnu@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "6404d0787642cabe8c6c3bde834a1b0f9b312c714e821b4e742e63c15c1896f156aad14ec5cb601441ba11fffae2162f66b5b287209c901d7401e96a2e8a5e0c" + } + ], + "name": "@oxlint/binding-linux-s390x-gnu", + "purl": "pkg:npm/@oxlint/binding-linux-s390x-gnu@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-x64-gnu@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "f9a22f272add783ecbc024365982cc5163669db7834523dbe3468162d3e30fdf8f4ea5308099e4f872b9c8b7d231ea97accac3844f6e4e6b6ddb2e74b6f8e11f" + } + ], + "name": "@oxlint/binding-linux-x64-gnu", + "purl": "pkg:npm/@oxlint/binding-linux-x64-gnu@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-linux-x64-musl@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "5f292d691f2584adaa5a208ad1393ca180fefdc827fa81c0e9d7519f14925d42a49288e457bf02992859531405fb2ac116cd12b96f8904e3c6ea179cc9cfe266" + } + ], + "name": "@oxlint/binding-linux-x64-musl", + "purl": "pkg:npm/@oxlint/binding-linux-x64-musl@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-openharmony-arm64@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "9b36e3acf9789de6955222757d48845314c6692641a222ad6a807a8e6229cfd4be54e036bc36262698a143cdb0eb5efc5798f1ce27254e0f02823e7217ab530e" + } + ], + "name": "@oxlint/binding-openharmony-arm64", + "purl": "pkg:npm/@oxlint/binding-openharmony-arm64@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-win32-arm64-msvc@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "bd401ef689294b639ae7e957ebb96a1ccb8d52f7e495e44ac2b50327f58906c82675dbd9d66aec8761d59ae143473a8510b85a26115a08d3ae47a6bb2f7aed20" + } + ], + "name": "@oxlint/binding-win32-arm64-msvc", + "purl": "pkg:npm/@oxlint/binding-win32-arm64-msvc@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-win32-ia32-msvc@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "cb25d72726184972f823c2bc8c05a326cf89ddf6bd807d899846e8e1fe5a766fb5b4d0bd8adb1e89c04dbb02bb0431efa90e49e77e2cfb20d42eef3dbd82f665" + } + ], + "name": "@oxlint/binding-win32-ia32-msvc", + "purl": "pkg:npm/@oxlint/binding-win32-ia32-msvc@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@oxlint/binding-win32-x64-msvc@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "5530bd2184c832b55493f8ba32cd68873cc32991645a7d0a5363816cf073826559d95df4d7ae53ff32b82f3b53af45e0a7d7d9d6a419d6bb1902efeb126c9294" + } + ], + "name": "@oxlint/binding-win32-x64-msvc", + "purl": "pkg:npm/@oxlint/binding-win32-x64-msvc@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/@playwright/test@1.62.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "f73389e9941101e9dadf5329387f554b3233f0ebb7609ff0b58fde426e53dae85f846eff53708eacc4bcc4eb54ad9ae9f4e81d9b3127218383c9ede763556acc" + } + ], + "name": "@playwright/test", + "purl": "pkg:npm/@playwright/test@1.62.0", + "scope": "optional", + "type": "library", + "version": "1.62.0" + }, + { + "bom-ref": "pkg:npm/@react-leaflet/core@3.0.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "dc45a67a4878373fa919cafec637f428dc987c2dd4d898e7916b21d3372a69ec58aa6981e59847dfb933e352571a62b3a5a3190a73e87c1066eb88be62b12f19" + } + ], + "name": "@react-leaflet/core", + "purl": "pkg:npm/@react-leaflet/core@3.0.0", + "type": "library", + "version": "3.0.0" + }, + { + "bom-ref": "pkg:npm/@reduxjs/toolkit@2.12.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "2a24fe47365ba7a990113f8c83e87673defef63d6c35f954c509081c8ed8bb37fa3debbe398a66927ea76c75a62cb5a3fb564e0d4245c0667b831dcbf51f443b" + } + ], + "name": "@reduxjs/toolkit", + "purl": "pkg:npm/@reduxjs/toolkit@2.12.0", + "type": "library", + "version": "2.12.0" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-android-arm64@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "95983c7ea22fdafec5176dfb6f0320cc664426f18befdfece649c9fe2e859ac185e175e5cdc719e236fe4eb148c6d492c45b48a5db20acf65e324d48f4015cc9" + } + ], + "name": "@rolldown/binding-android-arm64", + "purl": "pkg:npm/@rolldown/binding-android-arm64@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-darwin-arm64@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "e75067c7da4d88c44a49436d05fc9290d27dbcc53d1e1dc8d68cc377a8323cf63369709f9e9b547046715c66059feba5d9db5c3148aa191d586a2d8ad74ba84f" + } + ], + "name": "@rolldown/binding-darwin-arm64", + "purl": "pkg:npm/@rolldown/binding-darwin-arm64@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-darwin-x64@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "4e6fa06df0b4687bb5b4103f26f290877d92d0ae988021e4880178fd6eb15f42b446636e73de1578ae35f5d26813ae51e5a4719a8fa7a194125ab00c17ac9bea" + } + ], + "name": "@rolldown/binding-darwin-x64", + "purl": "pkg:npm/@rolldown/binding-darwin-x64@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-freebsd-x64@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "24ccc3282097abddd871c1b9833de1bceb35a1744a01fd176297ce4bcf1efb066b0bc22e823ea3ebcf3aeefad8664be90c3a4a9ff2a828e45386672132917a4c" + } + ], + "name": "@rolldown/binding-freebsd-x64", + "purl": "pkg:npm/@rolldown/binding-freebsd-x64@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-arm-gnueabihf@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "b8c2f6d63d8ae537cf1aeb4ac6e6fe33e9cb8d922b5a35d0e46af1e2509efe78a64e3f41e0beb7cc7a635cb978cb42f799c9b686d11021bd3aa021bfb337ab37" + } + ], + "name": "@rolldown/binding-linux-arm-gnueabihf", + "purl": "pkg:npm/@rolldown/binding-linux-arm-gnueabihf@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-arm64-gnu@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "9dabd28ae4cca20be742866833fbfe977656a39d3f353c121d2ce1780071fd10a79943da2b0abda92a3806bd8e611b36d7e173f2e16a21364cdc7e28a4e074fd" + } + ], + "name": "@rolldown/binding-linux-arm64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-arm64-gnu@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-arm64-musl@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "940af2a87ec8b5eced9825d05e4d1eb4a8f8c0143b1b1e52e8b8ca86c829f736fc2396ecbaf53fda5947d610d0723b057aa22ca2f315377dfdffca540c3057c4" + } + ], + "name": "@rolldown/binding-linux-arm64-musl", + "purl": "pkg:npm/@rolldown/binding-linux-arm64-musl@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-ppc64-gnu@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "7ec2bfb0d067c730652f83b524dad96a4550c4fb29aa9103e5d2ed36c652f6838a9ad4a974d233c47da49289791d84d624de3bb04db4ced3093f17d9f3da863a" + } + ], + "name": "@rolldown/binding-linux-ppc64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-ppc64-gnu@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-s390x-gnu@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "80b61be0121a7657d33984f980ee74de7f334235df960ce90f415cc8a874333c77aea0992a71e825657dc5ed4a5d4279971d897dc487aff9a1cd2d0f0bf31c00" + } + ], + "name": "@rolldown/binding-linux-s390x-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-s390x-gnu@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-x64-gnu@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "16372910a53227280782cd68e7455836f92de7eecb7bf544758b7402446990bdf7327c907f0afc979997c0c99f9936f230faf9bc92c2fbcfc677d8179f053541" + } + ], + "name": "@rolldown/binding-linux-x64-gnu", + "purl": "pkg:npm/@rolldown/binding-linux-x64-gnu@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-linux-x64-musl@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "31ef8f7cf2364cc78e424d206167cb419b5392dae6cdbafc7036e8a97f375ca73b52b8008b9e6017ed9d52459dc5dd7d9f9e44b2ca76c9e71af8ef4de6b0711e" + } + ], + "name": "@rolldown/binding-linux-x64-musl", + "purl": "pkg:npm/@rolldown/binding-linux-x64-musl@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-openharmony-arm64@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "c9ce56acbcd792ceb30907e7f4ec6bf293912b297fa45f908c7996fd0c77aaed28cabacd0becb624b4d4d44dab7166009b3967aa78165c7423cb9d55cd6ef457" + } + ], + "name": "@rolldown/binding-openharmony-arm64", + "purl": "pkg:npm/@rolldown/binding-openharmony-arm64@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-wasm32-wasi@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "55b4063d7d9be2be3c4c0308336723825b883351d8bad9b8a5c4c426c95eee210feec075745aad3cb0556dd2c0642c72d6dc4270fc5fe1015fe2ff2ebe5b4fa8" + } + ], + "name": "@rolldown/binding-wasm32-wasi", + "purl": "pkg:npm/@rolldown/binding-wasm32-wasi@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-win32-arm64-msvc@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "807bfcda4eb7cf8aa9579f90d72ff5d8aacad25b5606e9150c8f2765c6d3ed3b7f665388570a696b39deab417dde80f14e8dc88003040c8a108771369fa995b3" + } + ], + "name": "@rolldown/binding-win32-arm64-msvc", + "purl": "pkg:npm/@rolldown/binding-win32-arm64-msvc@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/binding-win32-x64-msvc@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "b5366e0c13f0f39b443793d08b5a67101cc3cb4678f47b5270b01b0f9b7a872794f7603de69456692330d466598bf470812814225d31ad2957213ffd433bd848" + } + ], + "name": "@rolldown/binding-win32-x64-msvc", + "purl": "pkg:npm/@rolldown/binding-win32-x64-msvc@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/@rolldown/pluginutils@1.0.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "da3f5b1ade4987c863faf3ed8333ed97bda3d324711c0cae9a8a3a4cd7c08ec2c1d3852da52bcf6cf703701331cfb9fef42601d1cd46c501716118368e29aa1b" + } + ], + "name": "@rolldown/pluginutils", + "purl": "pkg:npm/@rolldown/pluginutils@1.0.1", + "scope": "optional", + "type": "library", + "version": "1.0.1" + }, + { + "bom-ref": "pkg:npm/@standard-schema/spec@1.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "976685cb98c02e19e21b91e0aab0fa8d72e2feb516acabea37fa89c7aca826c80a85b95577e8aaa94e110976af9bf8cf8adc83a394c2bca327a632a73ab8b2d3" + } + ], + "name": "@standard-schema/spec", + "purl": "pkg:npm/@standard-schema/spec@1.1.0", + "type": "library", + "version": "1.1.0" + }, + { + "bom-ref": "pkg:npm/@standard-schema/utils@0.3.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "7bb31ec3af3aa3031a3c954d34bb39e4f52f833dfbd672a0c2c738bf1138f73b0e0e92449f1831468db2fce2abaae79abd781934c0d58f70dd1c595a480794f2" + } + ], + "name": "@standard-schema/utils", + "purl": "pkg:npm/@standard-schema/utils@0.3.0", + "type": "library", + "version": "0.3.0" + }, + { + "bom-ref": "pkg:npm/@svg-maps/world@2.0.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "5666189f2ca0279661af8f313eaba4141a2ae1a0818153488c1cea06758a5d274786b44ade99ed76f6c26bc94cb31c15085cc4c3d6beb2620c106236b02d7171" + } + ], + "name": "@svg-maps/world", + "purl": "pkg:npm/@svg-maps/world@2.0.0", + "type": "library", + "version": "2.0.0" + }, + { + "bom-ref": "pkg:npm/@tybys/wasm-util@0.10.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "1777e8d4c62b44960bdf3111d0e50e9a4bad8ebd55a76de6eceb12829ee7ab848fe8ea97e82ff9e971483c09796eddf3681463996ed21b3deeffa2f016961c3a" + } + ], + "name": "@tybys/wasm-util", + "purl": "pkg:npm/@tybys/wasm-util@0.10.3", + "scope": "optional", + "type": "library", + "version": "0.10.3" + }, + { + "bom-ref": "pkg:npm/@types/d3-array@3.2.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "84e2d655b9bbb91cdad01617a48216e69c5faca7b45be0f996b162004611fa96fac373764b048c6896d77547d212ff9d4f831f1c12ed9f98ecd0b01668eea8b7" + } + ], + "name": "@types/d3-array", + "purl": "pkg:npm/@types/d3-array@3.2.2", + "type": "library", + "version": "3.2.2" + }, + { + "bom-ref": "pkg:npm/@types/d3-color@3.1.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "88ef74b1cb61f5601b9a0bfba20a2ae7b3bd6292a61416e6a04a021c3076c4c058d3efca56ba806820f2084d7a754b2978ebc8c4515123ed2c12da83ab2d9be0" + } + ], + "name": "@types/d3-color", + "purl": "pkg:npm/@types/d3-color@3.1.3", + "type": "library", + "version": "3.1.3" + }, + { + "bom-ref": "pkg:npm/@types/d3-ease@3.0.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "35c5752633b9a03ce82b6ea83336c82c4e875bbb955ce1cb42f1ec841516e1431d6467e263abf905e43087d6bdb42ceff8279e7d940726de524602b5e7846d88" + } + ], + "name": "@types/d3-ease", + "purl": "pkg:npm/@types/d3-ease@3.0.2", + "type": "library", + "version": "3.0.2" + }, + { + "bom-ref": "pkg:npm/@types/d3-interpolate@3.0.4", + "hashes": [ + { + "alg": "SHA-512", + "content": "9a02cf11396ba55575611248825af8133e3b83b6318e5d658fb60ab223026f6ed5247f56f0d54ce816fd77c924a46fee0104b90266c0e3cab6200a2528aa3530" + } + ], + "name": "@types/d3-interpolate", + "purl": "pkg:npm/@types/d3-interpolate@3.0.4", + "type": "library", + "version": "3.0.4" + }, + { + "bom-ref": "pkg:npm/@types/d3-path@3.1.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "54c64163242f6c6996c9655e6b4107b3f0702e0c5cf8c2a2d732c308e364b28cc8e1824c713b7c644b8847849bd4c31313c30c5b8f6fd08c08e7e6fb4667d696" + } + ], + "name": "@types/d3-path", + "purl": "pkg:npm/@types/d3-path@3.1.1", + "type": "library", + "version": "3.1.1" + }, + { + "bom-ref": "pkg:npm/@types/d3-scale@4.0.9", + "hashes": [ + { + "alg": "SHA-512", + "content": "74b9adc01f3390078efe3b8031f9d5fac22d2a396cc3694a759555cba2d1af470199e8314800622c4a4656648927c3b4f22e3eb0647aa90b5be96b6ec0976f57" + } + ], + "name": "@types/d3-scale", + "purl": "pkg:npm/@types/d3-scale@4.0.9", + "type": "library", + "version": "4.0.9" + }, + { + "bom-ref": "pkg:npm/@types/d3-shape@3.1.8", + "hashes": [ + { + "alg": "SHA-512", + "content": "95a7b48967dc0de47baadeeb03cf01362a9dbcf4b9a4554fa68e4e7e3125c0d693db2c9e91b3340bdbcafb2a81a84987afa9439119d83684c162502025aedae3" + } + ], + "name": "@types/d3-shape", + "purl": "pkg:npm/@types/d3-shape@3.1.8", + "type": "library", + "version": "3.1.8" + }, + { + "bom-ref": "pkg:npm/@types/d3-time@3.0.4", + "hashes": [ + { + "alg": "SHA-512", + "content": "caecd9ba0d6790001a0650418a42994e0cc2780fa4d6ecb8645c1600d39f2b0e73e4b46157480d03b80d90a9bb1e82be1d1374c17dc49311a4d1fa5b5a1981ee" + } + ], + "name": "@types/d3-time", + "purl": "pkg:npm/@types/d3-time@3.0.4", + "type": "library", + "version": "3.0.4" + }, + { + "bom-ref": "pkg:npm/@types/d3-timer@3.0.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "3ecdd3f04f1d6436a6e9f5323623247a42b75d4b1a5048a4fa274ef7f6233ed7e3daaaee17cb450574bb5e1b44e2221704bc6198b8cfdd25e92e155fdb523d2f" + } + ], + "name": "@types/d3-timer", + "purl": "pkg:npm/@types/d3-timer@3.0.2", + "type": "library", + "version": "3.0.2" + }, + { + "bom-ref": "pkg:npm/@types/geojson@7946.0.16", + "hashes": [ + { + "alg": "SHA-512", + "content": "e82f27a96babde3f7c53afa55c37d351621f82f654f84ba6be91ca7188ee8ca1fbc286322e3dac5267dfd2d461aea33b068854c3b3f3dd90758e3da05bd16f9a" + } + ], + "name": "@types/geojson", + "purl": "pkg:npm/@types/geojson@7946.0.16", + "scope": "optional", + "type": "library", + "version": "7946.0.16" + }, + { + "bom-ref": "pkg:npm/@types/leaflet@1.9.21", + "hashes": [ + { + "alg": "SHA-512", + "content": "4db01df4368f1929f3a7a42fb589e09ed31981c464fa280510bc11d8df7d5d99fb45751d2a0b17311fb6f1b50ed2b3ec5a9f0c22efdfe3b96e2c842e48b62fff" + } + ], + "name": "@types/leaflet", + "purl": "pkg:npm/@types/leaflet@1.9.21", + "scope": "optional", + "type": "library", + "version": "1.9.21" + }, + { + "bom-ref": "pkg:npm/@types/node@24.13.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "0e1f2f02c577ea2839c1af4e5f8a57bcc73d0f755e89b7f0db08b1d0253060e0cb0fc9e48fd52c2e3012af8f673e0fb678acf184157ebfb2fca57bd3e1eeb7f5" + } + ], + "name": "@types/node", + "purl": "pkg:npm/@types/node@24.13.3", + "scope": "optional", + "type": "library", + "version": "24.13.3" + }, + { + "bom-ref": "pkg:npm/@types/react@19.2.17", + "hashes": [ + { + "alg": "SHA-512", + "content": "3177e6a9a54f115824053fda6346860a44565ad072898417a37c5d43caf9473b85acf8919fc19aaf6b5075749443618a5776e45dc91e93cf55f3040163643f03" + } + ], + "name": "@types/react", + "purl": "pkg:npm/@types/react@19.2.17", + "type": "library", + "version": "19.2.17" + }, + { + "bom-ref": "pkg:npm/@types/react-dom@19.2.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "8e9d8bfde63a7e7f8a81555000ea9822d6c5d1563f600a5ee4ccf61746b29123bc831df56d8099caf49e6310872afcc71b97998dcfb3c9a4b906b056c9adbe91" + } + ], + "name": "@types/react-dom", + "purl": "pkg:npm/@types/react-dom@19.2.3", + "scope": "optional", + "type": "library", + "version": "19.2.3" + }, + { + "bom-ref": "pkg:npm/@types/use-sync-external-store@0.0.6", + "hashes": [ + { + "alg": "SHA-512", + "content": "cc50c00feb65a5fdabe1ab2e1c48f45c7ea963a8b483935e0073e7fb5e70937055e8903af5e1111570b58321938439156b5cc2c8aaf98e8c75fb62db57f00c2e" + } + ], + "name": "@types/use-sync-external-store", + "purl": "pkg:npm/@types/use-sync-external-store@0.0.6", + "type": "library", + "version": "0.0.6" + }, + { + "bom-ref": "pkg:npm/@vitejs/plugin-react@6.0.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "be616f728e7f42e0b67fd3a3fb04e4d3ef577831641f149a9b064a61ceccc58c0a2027ef52f94c86a288d15b8808f96d1aa8759dea81283bcee247acd726bcbe" + } + ], + "name": "@vitejs/plugin-react", + "purl": "pkg:npm/@vitejs/plugin-react@6.0.3", + "scope": "optional", + "type": "library", + "version": "6.0.3" + }, + { + "bom-ref": "pkg:npm/axe-core@4.12.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "b3b8867f919a54cc441b410d37dc7ec53afb1856426f564fff5b804d4a4210ad97efc9c30774706ed142a3da4601ff6bbbe570a10e3ae0391a2c4791334f3024" + } + ], + "name": "axe-core", + "purl": "pkg:npm/axe-core@4.12.1", + "scope": "optional", + "type": "library", + "version": "4.12.1" + }, + { + "bom-ref": "pkg:npm/clsx@2.1.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "7989b441606d52b0566561b4777f3a386030d7a67df793e2395a3607b6e35926c779d1a5e5ed1959aabae6438681448d7ac1080e407d2126d383f24af5d84264" + } + ], + "name": "clsx", + "purl": "pkg:npm/clsx@2.1.1", + "type": "library", + "version": "2.1.1" + }, + { + "bom-ref": "pkg:npm/csstype@3.2.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "cf51c629c632db103c00641fc2b9f43c0cbe3c1ed7fc64a3dd45495bda8aca7e37c566be825e675e6538aaa2cc4735952c50bc2aeb145fc4ffd240a2398a4021" + } + ], + "name": "csstype", + "purl": "pkg:npm/csstype@3.2.3", + "type": "library", + "version": "3.2.3" + }, + { + "bom-ref": "pkg:npm/d3-array@3.2.4", + "hashes": [ + { + "alg": "SHA-512", + "content": "b5d4009b2035f22e09ef0a6ba58abc0a5731672dd20b7d5031e072c8217246dec1547751110679969ce87b998511865459efa8abc1c1be4f498e989bc8457826" + } + ], + "name": "d3-array", + "purl": "pkg:npm/d3-array@3.2.4", + "type": "library", + "version": "3.2.4" + }, + { + "bom-ref": "pkg:npm/d3-color@3.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "ce0fdc85b5f2781b4c4352db0ff592a16d83a42dc8d26a663dd5beca74538ffc760c0598ac863ba9e6481e2768cf0576e26e226afaf5e653702302f14663b184" + } + ], + "name": "d3-color", + "purl": "pkg:npm/d3-color@3.1.0", + "type": "library", + "version": "3.1.0" + }, + { + "bom-ref": "pkg:npm/d3-ease@3.0.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "c11fd72b70f75dc2c8670a5bbd0c10e5f2bef1891db358a9ec0d93c5ed32c6771749dab52fdb24706edb95c79d90e5fe65c83118098b5f516b4466c00f0ce2d3" + } + ], + "name": "d3-ease", + "purl": "pkg:npm/d3-ease@3.0.1", + "type": "library", + "version": "3.0.1" + }, + { + "bom-ref": "pkg:npm/d3-format@3.1.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "0090dd60e767c910d5e5be80ae297308f3f07357a3907728c856abaa53ea4fbcd162385abdc4f7b92aea70cbec821d828283db2b7442c321da572c5870fd80ae" + } + ], + "name": "d3-format", + "purl": "pkg:npm/d3-format@3.1.2", + "type": "library", + "version": "3.1.2" + }, + { + "bom-ref": "pkg:npm/d3-interpolate@3.0.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "ddb62cd6b383df7ba8f1aa897ca3f72563c089b830f199b6f8bf6f04a107276460faf89347ba39326bf999972278df854586803965f94890135fa9353d6cfbda" + } + ], + "name": "d3-interpolate", + "purl": "pkg:npm/d3-interpolate@3.0.1", + "type": "library", + "version": "3.0.1" + }, + { + "bom-ref": "pkg:npm/d3-path@3.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "a7728fe4709ffdbbe305248ab9789de99aa28f1ef021f356f89fe668fb3e8b0477e5ab792426cb513d0bcc5d5c9e36c21d686acd04c83762697bca5179b20415" + } + ], + "name": "d3-path", + "purl": "pkg:npm/d3-path@3.1.0", + "type": "library", + "version": "3.1.0" + }, + { + "bom-ref": "pkg:npm/d3-scale@4.0.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "1995b8eb8835487eda83763b8578dff11a14b80148aa494e02adcc465e0e69669b4c5258f4f37f1356249615cb87e390ddf33dc92da73a40a84be58b67a92fc5" + } + ], + "name": "d3-scale", + "purl": "pkg:npm/d3-scale@4.0.2", + "type": "library", + "version": "4.0.2" + }, + { + "bom-ref": "pkg:npm/d3-shape@3.2.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "49a2c1bb01a6dcc395891ab600193778ba31c1910ba47eb3865dc56c0a09ed59b58287cac7a125d486f9cf6dcd504845f40b0697bcbe7732dee42c36000ac64c" + } + ], + "name": "d3-shape", + "purl": "pkg:npm/d3-shape@3.2.0", + "type": "library", + "version": "3.2.0" + }, + { + "bom-ref": "pkg:npm/d3-time@3.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "56a2a3cc12de8db48c4f82206e65600e3a6462b3565182676c21a8f3be2eecc30a216b082d15fe3a95ff81393c32a8e94f503f73a1d8d9d080efb64dd25910d9" + } + ], + "name": "d3-time", + "purl": "pkg:npm/d3-time@3.1.0", + "type": "library", + "version": "3.1.0" + }, + { + "bom-ref": "pkg:npm/d3-time-format@4.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "749c4f065cc2ecdba00763c32f0a3d43c2624d1dccddee3f5c0364ade29253117cbef5caaa6d587eae10e5d97c6ee765ba745595451a0d4805b7b780f03e8d2e" + } + ], + "name": "d3-time-format", + "purl": "pkg:npm/d3-time-format@4.1.0", + "type": "library", + "version": "4.1.0" + }, + { + "bom-ref": "pkg:npm/d3-timer@3.0.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "9dd7c9fc9c7131dde7c37d6ec8aa18da76a2bc5fabdbd57e2dcd2cbd9c5ed49bef2119a2f2152caccbdd3b0812d68eae0479a2cfdd60790d3294f3f46a3d5550" + } + ], + "name": "d3-timer", + "purl": "pkg:npm/d3-timer@3.0.1", + "type": "library", + "version": "3.0.1" + }, + { + "bom-ref": "pkg:npm/decimal.js-light@2.5.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "a88305a533199a7cbe30c222b4007a0fb8953c4a2b570e984115a4bdaad3913e2d05e48b2e21f37308faab4326612142895a62a8f253244608ae970fcd5108be" + } + ], + "name": "decimal.js-light", + "purl": "pkg:npm/decimal.js-light@2.5.1", + "type": "library", + "version": "2.5.1" + }, + { + "bom-ref": "pkg:npm/detect-libc@2.1.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "06d8f604e38ef37a375b21f9f5ef0c817b3111055c6ab9143a9118aee6c1d2eaf09cdd74c90dfae2bb22072535d67665a966199b4e62fe87fb8a8e26ce2841b5" + } + ], + "name": "detect-libc", + "purl": "pkg:npm/detect-libc@2.1.2", + "scope": "optional", + "type": "library", + "version": "2.1.2" + }, + { + "bom-ref": "pkg:npm/es-toolkit@1.49.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "1b9899e8f73f14d458feca0a6470be4f11830fcdeb1d40d7c73696846097e38bc0bffb4cb39e9632eb279bf28c34afa5b943ec800f54dbc706af8443952ccbda" + } + ], + "name": "es-toolkit", + "purl": "pkg:npm/es-toolkit@1.49.0", + "type": "library", + "version": "1.49.0" + }, + { + "bom-ref": "pkg:npm/eventemitter3@5.0.4", + "hashes": [ + { + "alg": "SHA-512", + "content": "9a5b1347219a3c18cf79d93a06fc3e6aa6ec5c3b680320339b930eec9814fb2551c8c4393bc6c3e0a71c8bb052f397fddef79e81e08f90bf11e062c29678cb17" + } + ], + "name": "eventemitter3", + "purl": "pkg:npm/eventemitter3@5.0.4", + "type": "library", + "version": "5.0.4" + }, + { + "bom-ref": "pkg:npm/fdir@6.5.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "b486d8b596ee70eb340511aa3c992c84951874bf920c7edd54cf208f2f84469dd60148cb105244fb4da46a7c87b708d63a7c2b298062c0098cd29e242c90275e" + } + ], + "name": "fdir", + "purl": "pkg:npm/fdir@6.5.0", + "scope": "optional", + "type": "library", + "version": "6.5.0" + }, + { + "bom-ref": "pkg:npm/fsevents@2.3.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "c62a8c411e3101e1d3b81f6e5a6f9f1517083a02813223813fe7978b24fb8ec8150aad5b915ca0b74d28012a3007b11db6938769a3e02adf35d8ff5a6fe0c328" + } + ], + "name": "fsevents", + "purl": "pkg:npm/fsevents@2.3.2", + "scope": "optional", + "type": "library", + "version": "2.3.2" + }, + { + "bom-ref": "pkg:npm/fsevents@2.3.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "e71a037d7f9f2fb7da0139da82658fa5b16dc21fd1efb5a630caaa1c64bae42defbc1d181eb805f81d58999df8e35b4c8f99fade4d36d765cda09c339617df43" + } + ], + "name": "fsevents", + "purl": "pkg:npm/fsevents@2.3.3", + "scope": "optional", + "type": "library", + "version": "2.3.3" + }, + { + "bom-ref": "pkg:npm/immer@11.1.15", + "hashes": [ + { + "alg": "SHA-512", + "content": "56b3403659a75909e1e423972083905ccf68389c3b9da18a9414fbe1938e47a96955773780512ef4524b0c5729089da3f8d5abf13230b560fffd3e995fa4ca89" + } + ], + "name": "immer", + "purl": "pkg:npm/immer@11.1.15", + "type": "library", + "version": "11.1.15" + }, + { + "bom-ref": "pkg:npm/internmap@2.0.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "e4787b635c106ef639a281a03db0da2f98982c03f331352b8ccba5b241cb1fac27bff03ed6ae6b8046a2baa12307ea3119721566cd4517e47f28f37765241862" + } + ], + "name": "internmap", + "purl": "pkg:npm/internmap@2.0.3", + "type": "library", + "version": "2.0.3" + }, + { + "bom-ref": "pkg:npm/leaflet@1.9.4", + "hashes": [ + { + "alg": "SHA-512", + "content": "9f14b5ca7cc93a63a51e9fa22f71725aa2bcf46b4d2fc53caef94c3ac41d4d3b2cc59c025e1f0dd8d07718340e2fe611dd79d6c99031c108b15116fe140ef83c" + } + ], + "name": "leaflet", + "purl": "pkg:npm/leaflet@1.9.4", + "type": "library", + "version": "1.9.4" + }, + { + "bom-ref": "pkg:npm/lightningcss@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "357601ce29cdadb95fada3c6cab6cfa03d7d0b587d95f23fd66ce0598bd75137b8d781b3fd7d450f65c165264ceeb453acc03c24bdceb4068689fac82a1439c9" + } + ], + "name": "lightningcss", + "purl": "pkg:npm/lightningcss@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-android-arm64@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "60aeff0a54ede2400ad2fa3ac375fe3e79b40f671fdaf3c76e139776836d8b519ad1a9753f84c1661c23013be33702c40429cabe325cda34201d71f4344c2502" + } + ], + "name": "lightningcss-android-arm64", + "purl": "pkg:npm/lightningcss-android-arm64@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-darwin-arm64@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "473786f49bb96da83606fd7f97095526f044deae93b57b247592cb0b27e0e69b7e1cbcfd06a94808eecb64ced51cd4d39ffe4f4611c50528e4e65738726b1c3d" + } + ], + "name": "lightningcss-darwin-arm64", + "purl": "pkg:npm/lightningcss-darwin-arm64@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-darwin-x64@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "53e42c069da6fecdb0aa95184ffeb09e56a07596ed65d9dd4a6badfcd26a94270c2d35a9e66b82ac80fe2b9509ea3a83d8116c85e8c26179e23c36cd87bdd5f3" + } + ], + "name": "lightningcss-darwin-x64", + "purl": "pkg:npm/lightningcss-darwin-x64@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-freebsd-x64@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "2424e281e74492c664ded1d34ed86731d55f19feb5164cbc262d84e188d44c4417d78c62cbf953cd79eed6fc2265eddb61ed2af92a6c487fc24de0d72ba5878a" + } + ], + "name": "lightningcss-freebsd-x64", + "purl": "pkg:npm/lightningcss-freebsd-x64@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-linux-arm-gnueabihf@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "c7aae79e945ad862f4cd03a4b7aaedb376033f376e2e95afc0017a10c8571556570f8b4fac190416acc6a30cc2b085ac3e3a922beb723443835015de7951d293" + } + ], + "name": "lightningcss-linux-arm-gnueabihf", + "purl": "pkg:npm/lightningcss-linux-arm-gnueabihf@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-linux-arm64-gnu@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "d279ccca8c8e2d12577db30e8a569245c2c7dc9c39cfd1c33467d3fe0c023e06838e7c748bcc3bbc1cc52c54757fa08c2ca17c8156de6e69143777dafe4409a5" + } + ], + "name": "lightningcss-linux-arm64-gnu", + "purl": "pkg:npm/lightningcss-linux-arm64-gnu@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-linux-arm64-musl@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "529424a1e9ebe14244ce054862923cd250c5bd198f560ea8a9ba0d1dfa07e024087cd03e1cead9ecca3b2993f4d9d0ba2e38213d025e06cbd7849a1dff09c806" + } + ], + "name": "lightningcss-linux-arm64-musl", + "purl": "pkg:npm/lightningcss-linux-arm64-musl@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-linux-x64-gnu@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "57b42be7622166674a3d5afe56dc3ca3e58bb1025809377c96821fa4368c45619465f04e60425ec89224a862033193f03f1db8a5431fc12c7123ca61aff31b38" + } + ], + "name": "lightningcss-linux-x64-gnu", + "purl": "pkg:npm/lightningcss-linux-x64-gnu@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-linux-x64-musl@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "6d870ba7e55bd1ac2c89783ff34b8245ecc2607360d7f9779add20cc79d657d5cfd56e6c29ae7f4c274659a47fcc13363de17f1dbb10bff8f651134e895bb15a" + } + ], + "name": "lightningcss-linux-x64-musl", + "purl": "pkg:npm/lightningcss-linux-x64-musl@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-win32-arm64-msvc@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "f126c2f01478d294ba6da08cf2c6ed6034b011541de099454ce95a0f78161877d385370006734305d6ba79365ea9ba1f6a5209845c74a8ace01c999c3d39c677" + } + ], + "name": "lightningcss-win32-arm64-msvc", + "purl": "pkg:npm/lightningcss-win32-arm64-msvc@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lightningcss-win32-x64-msvc@1.32.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "026abd07f4a86587438b5905ae88e7a2a3cbc58850e16a395e22fc11526b56c07c011a02d4f596e951ad4f458a09e9a3cbc682fa5a2e2678d2ed4d7cc776f4e9" + } + ], + "name": "lightningcss-win32-x64-msvc", + "purl": "pkg:npm/lightningcss-win32-x64-msvc@1.32.0", + "scope": "optional", + "type": "library", + "version": "1.32.0" + }, + { + "bom-ref": "pkg:npm/lucide-react@1.25.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "fe67494d16e2c1c2ce43535964ad5a99917dac8672bced7c0faafd4e7804e931b5366a87805b93e1e13b5eb926f54b1731b0490f53657f01d596d083587ace4f" + } + ], + "name": "lucide-react", + "purl": "pkg:npm/lucide-react@1.25.0", + "type": "library", + "version": "1.25.0" + }, + { + "bom-ref": "pkg:npm/nanoid@3.3.16", + "hashes": [ + { + "alg": "SHA-512", + "content": "6f394a4f2349efe2dd188230cbc8a316922a11022f69f6a157b798ca427c0af878d8474978e0e827a814257a5026f7a3d4175d1fc3aa4d764d13f29f6d602ef1" + } + ], + "name": "nanoid", + "purl": "pkg:npm/nanoid@3.3.16", + "scope": "optional", + "type": "library", + "version": "3.3.16" + }, + { + "bom-ref": "pkg:npm/oxlint@1.74.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "a1d1a5dacdb1e48389a23dc0d2fd64d0f1815d514165e676f802bff8ad8c26eafb1a18b76e4c97e4db8b5161caa9ae23b35c237a9de125eb934093a5afee3e74" + } + ], + "name": "oxlint", + "purl": "pkg:npm/oxlint@1.74.0", + "scope": "optional", + "type": "library", + "version": "1.74.0" + }, + { + "bom-ref": "pkg:npm/picocolors@1.1.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "c5c787dac9e1b5be4cf658aa0ec984c39ea57b7efa993664117fe311bfd1c4d1727a036e97b78db250973fd1438ff2dcbb45fc284c8c71e3f69eda5a1eb0c454" + } + ], + "name": "picocolors", + "purl": "pkg:npm/picocolors@1.1.1", + "scope": "optional", + "type": "library", + "version": "1.1.1" + }, + { + "bom-ref": "pkg:npm/picomatch@4.0.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "46fc3072bb8d8c8d67713e7145a91ec92f4b7fc95c22dbf7e0a0fe6a27fe547f6476e0327d8062a46875db6ef8c6d7a720f675d7dfd1f4175305af200304a1d0" + } + ], + "name": "picomatch", + "purl": "pkg:npm/picomatch@4.0.5", + "scope": "optional", + "type": "library", + "version": "4.0.5" + }, + { + "bom-ref": "pkg:npm/playwright@1.62.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "675e1d1b7d3976068bbba7e80754d741a805896f097d250851a52e3da290e8db413ca1753ffa9772a7e1e9ce8afe270fa9dcb7ec99a36e20577fa24d83a4b297" + } + ], + "name": "playwright", + "purl": "pkg:npm/playwright@1.62.0", + "scope": "optional", + "type": "library", + "version": "1.62.0" + }, + { + "bom-ref": "pkg:npm/playwright-core@1.62.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "9ec351caad2bdb3b06f007111d692773d4110395c2b9e0bb81632bb3e1b1dad9599fd85c97ccee75f874d258493d8d4313b36667a6c3b13f60db27b2f265e58c" + } + ], + "name": "playwright-core", + "purl": "pkg:npm/playwright-core@1.62.0", + "scope": "optional", + "type": "library", + "version": "1.62.0" + }, + { + "bom-ref": "pkg:npm/postcss@8.5.20", + "hashes": [ + { + "alg": "SHA-512", + "content": "956eb5ea5f39b9c2102fe16870c98bee94053ea066c1e8eb08c83e88fc7222696b00d3491bd347abf464c82668a4385df02dcb034dcf1d12438e46c6bbcbefba" + } + ], + "name": "postcss", + "purl": "pkg:npm/postcss@8.5.20", + "scope": "optional", + "type": "library", + "version": "8.5.20" + }, + { + "bom-ref": "pkg:npm/react@19.2.7", + "hashes": [ + { + "alg": "SHA-512", + "content": "1cd7bd5ac9536d79852bca3c726c2001e245481bedd5b3dd1c254ab5a695f96940377ea6a53e21711a706dfddf639e9aaf6a085f3b01a4e0222111c0758d0e95" + } + ], + "name": "react", + "purl": "pkg:npm/react@19.2.7", + "type": "library", + "version": "19.2.7" + }, + { + "bom-ref": "pkg:npm/react-dom@19.2.7", + "hashes": [ + { + "alg": "SHA-512", + "content": "b74051557bdb884fe8db41dfc3aebdacb6cc0835ad6192ef9898a0cb67f4331b1717eef5a71851df13a4b299ac3bc877665373ca26c09ad09ade5e2560a93a81" + } + ], + "name": "react-dom", + "purl": "pkg:npm/react-dom@19.2.7", + "type": "library", + "version": "19.2.7" + }, + { + "bom-ref": "pkg:npm/react-is@19.2.7", + "hashes": [ + { + "alg": "SHA-512", + "content": "919167a2ec95bfb78ffcf866ae5a3d14afb370076b89926fcf15c71754a5d4fdfbed64867b61bf2715689614eb07f8de578ee52889a1354b228e31c001b725fc" + } + ], + "name": "react-is", + "purl": "pkg:npm/react-is@19.2.7", + "type": "library", + "version": "19.2.7" + }, + { + "bom-ref": "pkg:npm/react-leaflet@5.0.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "0966d3a6be6f707c396edf62e334a53c455075355c30bdfdd138de0c6d1c2b9f73d7295ec69a82e8cd4f2458d5f230fb085f8008116c2c80ece8344ca0b1287f" + } + ], + "name": "react-leaflet", + "purl": "pkg:npm/react-leaflet@5.0.0", + "type": "library", + "version": "5.0.0" + }, + { + "bom-ref": "pkg:npm/react-redux@9.3.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "290a2982a168fe9fdf82602ce6acfaa7945668d033ab8d1602eedf2485e742963114f6c962db093d6bc6785dab381698fe412e57bec056c5fc4ec4332a6f80fe" + } + ], + "name": "react-redux", + "purl": "pkg:npm/react-redux@9.3.0", + "type": "library", + "version": "9.3.0" + }, + { + "bom-ref": "pkg:npm/recharts@3.9.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "1b87f2f8f938e9169783058c87e373872a3f95b14056a5e8f60b5eb65c9e85ee848607bd0ac803b8e4f04030fe8b5fa595a2e4b4d06236ae1b8671a50d15c5b7" + } + ], + "name": "recharts", + "purl": "pkg:npm/recharts@3.9.2", + "type": "library", + "version": "3.9.2" + }, + { + "bom-ref": "pkg:npm/redux@5.0.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "33dfc42ea17a7f2f05c269299c5d12dd828ea8cca8589e3e092e447e0d9cb77a18f5d690bddfcf73bd45a4666c56c6e5dc2a5bf8821c8c10d49e7c817506eae3" + } + ], + "name": "redux", + "purl": "pkg:npm/redux@5.0.1", + "type": "library", + "version": "5.0.1" + }, + { + "bom-ref": "pkg:npm/redux-thunk@3.1.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "356dabe53ea4b1429709a6f384bf73fa1db4e87430fcd26470b9b518f226450f08cdf5f0446aa356128919ab878ab4f40c0bb2cba863767319691a00732d0a97" + } + ], + "name": "redux-thunk", + "purl": "pkg:npm/redux-thunk@3.1.0", + "type": "library", + "version": "3.1.0" + }, + { + "bom-ref": "pkg:npm/reselect@5.2.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "02067750e666dd89dd7eb2783988e0ad3edb9829bfd62aa48ef11f1ffa188f387a3c3daac3842e4f78e39d722ba5db78313a4c5dc94c4f79576e6458f9745a93" + } + ], + "name": "reselect", + "purl": "pkg:npm/reselect@5.2.0", + "type": "library", + "version": "5.2.0" + }, + { + "bom-ref": "pkg:npm/rolldown@1.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "b7dcf6f5c2635dffefc50f1dca1092a6de87e9a4b01d393c713e48de2cba48c5ee169939981e8f2fa5df0bc3c2c2b3d3c7ddee7702949bd1d1b5e0254c604b88" + } + ], + "name": "rolldown", + "purl": "pkg:npm/rolldown@1.1.5", + "scope": "optional", + "type": "library", + "version": "1.1.5" + }, + { + "bom-ref": "pkg:npm/scheduler@0.27.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "78dbfe5ab55b2aed5fdef6d8253ff1b62179b320391cf20cb5ff48818fe72a0d2c5aacc0504bea63fc66ece71973fa9a7cbc7f88ef4580e99e480a78bf9b62fd" + } + ], + "name": "scheduler", + "purl": "pkg:npm/scheduler@0.27.0", + "type": "library", + "version": "0.27.0" + }, + { + "bom-ref": "pkg:npm/source-map-js@1.2.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "51758c2a12cec1529bef6f0852d40f5f17d853ebac7726ed52b2bff2e184f0240cbeb84ea70bf30c1c23d108522fb31073bbc8b084811bc550f3e203431a5f40" + } + ], + "name": "source-map-js", + "purl": "pkg:npm/source-map-js@1.2.1", + "scope": "optional", + "type": "library", + "version": "1.2.1" + }, + { + "bom-ref": "pkg:npm/tiny-invariant@1.3.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "f856c13c4d68f50018bef89abbfa82e5213771ac36d6adf192f58a06d8dae6f82a3962071c9de2a1aab554f7e7fd2cea72dcf66d4fe861e29df7fcf904bf8f56" + } + ], + "name": "tiny-invariant", + "purl": "pkg:npm/tiny-invariant@1.3.3", + "type": "library", + "version": "1.3.3" + }, + { + "bom-ref": "pkg:npm/tinyglobby@0.2.17", + "hashes": [ + { + "alg": "SHA-512", + "content": "c1747f758a5ca8a99f5a911d66388a24ec023459dd0f40cc9eb5bf7188d51adb449017d581c2c51e836b963e3b9a339589cf72c8dbbae5a96c805e0d4324dada" + } + ], + "name": "tinyglobby", + "purl": "pkg:npm/tinyglobby@0.2.17", + "scope": "optional", + "type": "library", + "version": "0.2.17" + }, + { + "bom-ref": "pkg:npm/tslib@2.8.1", + "hashes": [ + { + "alg": "SHA-512", + "content": "a0916ef781d06fe29576e49440bef09e99aa9df98bb0e03f9c087a6fa107d30084a0ad3f98f79753a737c0a0d5f373243ae1cf447b525ca294f7d2016b34bfdb" + } + ], + "name": "tslib", + "purl": "pkg:npm/tslib@2.8.1", + "scope": "optional", + "type": "library", + "version": "2.8.1" + }, + { + "bom-ref": "pkg:npm/typescript@6.0.3", + "hashes": [ + { + "alg": "SHA-512", + "content": "cb64efbb14993c3c906a490544f6472859be28a56a222b1d83dfc26709bd7edbca5cb3fc3515a3dfcfce0e335baf8dd2b285ea36e022b047f519d0b1a9671d07" + } + ], + "name": "typescript", + "purl": "pkg:npm/typescript@6.0.3", + "scope": "optional", + "type": "library", + "version": "6.0.3" + }, + { + "bom-ref": "pkg:npm/undici-types@7.18.2", + "hashes": [ + { + "alg": "SHA-512", + "content": "02cb82cdf7c61c9c9b49a46b9abe5e1ebf359b0254de48f0e8cfaea6b5af09788d78df52386c10dc99fc8dbf26dd9ea2cd58249e7d51d054c21003f642b2d8ff" + } + ], + "name": "undici-types", + "purl": "pkg:npm/undici-types@7.18.2", + "scope": "optional", + "type": "library", + "version": "7.18.2" + }, + { + "bom-ref": "pkg:npm/use-sync-external-store@1.6.0", + "hashes": [ + { + "alg": "SHA-512", + "content": "3e9e864b018ffcdacf22bc551402243907b2c3c9457a73878a34169144eb0efac5e002eaca53f60bf28291e4bd76950cdcabd84508676b9bedec82fde7e650f7" + } + ], + "name": "use-sync-external-store", + "purl": "pkg:npm/use-sync-external-store@1.6.0", + "type": "library", + "version": "1.6.0" + }, + { + "bom-ref": "pkg:npm/victory-vendor@37.3.6", + "hashes": [ + { + "alg": "SHA-512", + "content": "49b3c33dd0c1629fb93091e104203223bc0a3377798af7a48a07360e4dacee981b67dc08808058195c38cc61c19a5fea4c56dec6ba1f5d6e86bb89e81b1acec1" + } + ], + "name": "victory-vendor", + "purl": "pkg:npm/victory-vendor@37.3.6", + "type": "library", + "version": "37.3.6" + }, + { + "bom-ref": "pkg:npm/vite@8.1.5", + "hashes": [ + { + "alg": "SHA-512", + "content": "ed42cbc2c09d631fe7472ae9884c2fa9be53147acc559c81b7eae0fce0174fbae08ffcfe0ed4c3c8a15e2c074392e6c3543283f233ac9dd9b2ee6e795dc7d4b3" + } + ], + "name": "vite", + "purl": "pkg:npm/vite@8.1.5", + "scope": "optional", + "type": "library", + "version": "8.1.5" + } + ], + "metadata": { + "component": { + "name": "foundation-intelligence-frontend", + "type": "application", + "version": "1.0.0" + }, + "properties": [ + { + "name": "generation.source", + "value": "committed-lockfile" + }, + { + "name": "generation.reproducible", + "value": "true" + } + ] + }, + "specVersion": "1.5", + "version": 1 +} diff --git a/docs/remediation/feature-test-matrix.md b/docs/remediation/feature-test-matrix.md new file mode 100644 index 0000000..cfcef6c --- /dev/null +++ b/docs/remediation/feature-test-matrix.md @@ -0,0 +1,28 @@ +# Remediation feature test matrix + +| Journey/control | Unit/contract | Local PostgreSQL | Browser/container | Status | +|---|---:|---:|---:|---| +| Dashboard overview/totals | Yes | Golden projection | E2E | PASS | +| Beneficiary map | Yes | Country projection | E2E | PASS | +| Map relationships | Yes | 100-flow projection | E2E lazy-load | PASS | +| Date/country/programme filters | Yes | Exact projections | E2E URL state | PASS | +| Donor/recipient filters | Yes | Exact projections | E2E URL state | PASS | +| Monthly/yearly trends | Yes | Exact full periods | E2E | PASS | +| Donor/recipient rankings | Yes | Top-25 exact identity/order | UI tests | PASS | +| Registry search/pagination | Yes | Exact first page | E2E | PASS | +| Profile detail | Yes | Exact selected fields | E2E | PASS | +| Grant list/drill-down | Yes | Exact projection/repository | E2E | PASS | +| Sankey | Yes | Exact top flows | E2E | PASS | +| Experimental score | Yes | Golden components | E2E contract | PASS | +| News | Yes, empty/error paths | No live call | Network disabled | PARTIAL | +| Pipeline status | Yes | Durable job repository | Auth E2E | PASS locally | +| Manual refresh permissions | Yes | Transactional enqueue | Operator-only E2E | PASS locally | +| Auth/RBAC/rate/proxy | Yes | Audit/idempotency integration | API tests | PASS locally | +| Migration/reconciliation | Yes | Full active dataset | N/A | PASS | +| Shadow non-latency/differences | Yes | 18 projections, zero diff | Runtime middleware contract | PASS locally | +| Dataset rollback | Yes | Prior and original activated | N/A | PASS | +| Full logical restore | Script contract | Isolated full restore | Docker PostgreSQL | PASS | +| Docker no-data/non-root/health | Contract | Stack health | Image inspect | PASS locally | +| Terraform provider validation | Offline only | N/A | No provider tool | NOT TESTED | +| AWS staging deployment | Workflow only | N/A | No AWS execution | NOT TESTED | +| Production deployment | Hard-disabled | N/A | No execution | NOT TESTED | diff --git a/docs/remediation/frontend-bundle-budget.md b/docs/remediation/frontend-bundle-budget.md new file mode 100644 index 0000000..c36cfb2 --- /dev/null +++ b/docs/remediation/frontend-bundle-budget.md @@ -0,0 +1,20 @@ +# Frontend bundle budget + +Phase 7 keeps the map as the first major dashboard section while preventing its +geometry and charting dependencies from blocking the initial application shell. +The production build enforces these compressed limits locally: + +| Asset scope | gzip budget | +| --- | ---: | +| Initial JavaScript, including module preloads | 120 KiB | +| Initial CSS | 25 KiB | +| Any deferred JavaScript chunk | 425 KiB | + +`npm run build` runs `scripts/check-bundle-budget.mjs` after Vite. The checker +reads the generated `dist/index.html`, measures the referenced initial assets, +measures every lazy JavaScript chunk, and exits non-zero when a budget is +exceeded. The larger map geometry is intentionally deferred behind React +Suspense; charts and directory workspaces are separate lazy chunks. + +No CDN fonts, scripts, or stylesheets are allowed. All production assets must be +resolved from the pinned npm lockfile and emitted into `frontend/dist/`. diff --git a/docs/remediation/local-development-guide.md b/docs/remediation/local-development-guide.md new file mode 100644 index 0000000..15e05d5 --- /dev/null +++ b/docs/remediation/local-development-guide.md @@ -0,0 +1,52 @@ +# Local development guide + +## Prerequisites + +- Python 3.12 +- Node.js 22 and npm +- Docker Desktop with Compose +- A local secret file containing the PostgreSQL password + +Install only from the committed locks: + +```bash +python3.12 -m venv venv +venv/bin/python -m pip install --require-hashes -r requirements-locking.txt +venv/bin/python -m pip install --require-hashes -r requirements.txt +cd frontend && npm ci --ignore-scripts --no-audit --no-fund && cd .. +``` + +Start PostgreSQL and the complete stack: + +```bash +export POSTGRES_PASSWORD_FILE=/absolute/path/to/local-postgres-password +export POSTGRES_HOST_PORT=55432 +docker-compose up -d postgres +docker-compose --profile operations run --rm migration upgrade head +docker-compose up -d backend frontend +docker-compose ps +curl --fail http://127.0.0.1:8000/health/live +curl --fail http://127.0.0.1:8000/health/ready +``` + +Compose explicitly sets `DATA_RUNTIME_MODE=postgresql`. The image is data-free; +it cannot silently serve from SQLite. To work only on legacy migration-source +compatibility outside Compose, set `DATA_RUNTIME_MODE=sqlite_migration_source` +and `APP_ENV=development`; this mode is rejected in staging/production. + +Run local checks: + +```bash +PYTHONPATH=src venv/bin/mypy --config-file mypy.ini +PYTHONPATH=src venv/bin/python -m pytest -q +cd frontend && npm run lint && npm test && npm run build +``` + +Stop cleanly: + +```bash +docker-compose stop frontend backend postgres +``` + +No local development command authorizes AWS, deployment, live source refresh, +paid APIs or Git push. diff --git a/docs/remediation/observability-contract.md b/docs/remediation/observability-contract.md new file mode 100644 index 0000000..8c618b9 --- /dev/null +++ b/docs/remediation/observability-contract.md @@ -0,0 +1,46 @@ +# Observability Contract + +## Health + +`GET /health/live` checks process viability only. `GET /health/ready` uses an +independent, no-pool PostgreSQL connection so exhausted analytical connections +cannot block the probe. Readiness requires: + +- PostgreSQL query success; +- exact expected Alembic revision; +- one approved active dataset; +- synchronized source and retention configuration; +- durable outbox/queue contract availability. + +The public response contains check states only, not database identifiers or +connection details. The legacy `/health` route remains a liveness alias. + +## Structured logs + +Every application log line is one redacted JSON object containing timestamp, +level, service, environment and message. Request completion adds request ID, +trace ID, pseudonymous actor ID, role, route-template operation, duration, +status and error class. Worker events add job/dataset IDs, operation, counts +and retry state where available. + +The formatter excludes raw exception payloads and applies the Phase-9 +recursive redaction policy to structured fields. Request paths use route +templates to avoid high-cardinality IDs. + +## Metrics and alarms + +`config/observability.json` is the executable definition. It covers API +latency/errors, DB pool, query latency, cache hit rate, dataset/source age, +pipeline duration/failures/counts, conversion gaps, programme/geography +coverage, queue age, DLQ, worker failures, task restarts and estimated cost. + +Alarm definitions cover readiness, 5xx/latency budgets, stale data, pipeline +failure, reconciliation, DLQ/backlog, gap/coverage regression, cost and RDS +CPU/connections/storage. Baseline coverage/gap thresholds derive from the +locally reconciled dataset; the USD 500 cost threshold is a proposed fail-safe, +not an approved budget. Owners must approve production thresholds. + +`/api/admin/observability/metrics` is administrator-only and returns definitions +plus bounded local-process evidence. Terraform maps the same definitions to +CloudWatch later. No CloudWatch metric, dashboard, alarm or notification has +been created or tested in AWS. diff --git a/docs/remediation/observability-runbooks.md b/docs/remediation/observability-runbooks.md new file mode 100644 index 0000000..adba3c4 --- /dev/null +++ b/docs/remediation/observability-runbooks.md @@ -0,0 +1,133 @@ +# Observability Runbooks + +Status: local code-readiness procedures. AWS/CloudWatch execution is not +tested. Any production command must follow environment approval and change +control; examples here are decision steps, not authorization. + +## readiness-failure + +Signals: `/health/ready` returns 503 or the readiness alarm fires. + +1. Read the five non-sensitive checks: PostgreSQL, schema version, active + dataset, critical configuration and queue contract. +2. Correlate request/trace IDs in structured logs; do not log connection + strings or credentials. +3. If schema mismatches, stop rollout and follow `migration-failure`. +4. If no active dataset exists, stop writers and follow `stale-dataset` or + `rollback` using an approved version. +5. If PostgreSQL is unavailable, follow `rds-outage`. If queue state is + unavailable, stop manual/scheduled triggers and follow `queue-backlog`. + +## migration-failure + +1. Stop application promotion and pipeline writers; retain the failed target. +2. Record Alembic revision, migration run ID, dataset version and error class. +3. Verify source checksum and last approved dataset remain unchanged. +4. Use the revision's tested downgrade only when the migration owner approves; + do not improvise DDL or modify Alembic state directly. +5. Re-run empty-database migration, reconciliation and restore tests before a + new attempt. + +## ingestion-failure + +1. Pause the affected source schedule and preserve its last-good dataset. +2. Inspect job/ingestion events, immutable manifest, checksum, watermark and + accepted/rejected/quarantined counts. +3. Classify retryability; allow only bounded configured retries. +4. Dead-letter exhausted work, correct through a new source/object version and + reconcile before activation. +5. Do not call paid/news/model sources during incident diagnosis without + separate approval. + +## stale-dataset + +1. Confirm dataset age and per-source freshness, accounting for paused or + governance-blocked schedules. +2. Compare conversion gaps and programme/geography coverage with the last + approved snapshot; never lower thresholds merely to silence an alarm. +3. Trigger only an authorised, idempotent queued refresh. +4. Keep the last-good dataset active on failure and communicate known coverage + gaps. + +## rds-outage + +1. Fail readiness and stop write/pipeline traffic while liveness remains up. +2. Check RDS health, CPU, connections, free storage, maintenance and network + path using approved read-only operational access. +3. Do not expose connection strings or rotate credentials as a speculative + fix. +4. If restore/failover is approved, verify checksum/revision/active dataset and + run reconciliation before reopening traffic. + +## queue-backlog + +1. Check oldest-message age, pending outbox rows, worker heartbeat age and + worker failure/restart rates. +2. Pause schedules/manual triggers if age continues to rise. +3. Verify worker capacity, lease expiry, poison-message repetition and database + health before scaling. +4. Preserve idempotency keys; never duplicate-publish by manually rewriting + outbox status. + +## dlq-replay + +1. Classify every DLQ message and record its original job ID/attempt history. +2. Fix the root cause and verify the handler against a local/recorded fixture. +3. Replay only through an approved tool that preserves job identity and + deduplication; cap the batch and monitor failures. +4. Stop immediately on repeated failure and retain the message/evidence. + +## bad-deployment + +1. Stop promotion when 5xx, latency, readiness, E2E or reconciliation budgets + fail. +2. Compare immutable image digest, configuration revision, schema and frontend + artifact with the last approved release. +3. Follow `rollback`; do not combine a rollback with an unreviewed migration. +4. Re-run smoke, security, E2E, load and rollback-readiness gates before retry. + +## rollback + +1. Route traffic to the last approved immutable application artifacts. +2. Stop incompatible writers and retain the failed target for diagnosis. +3. Reactivate only an approved PostgreSQL dataset/materialization version. +4. Reconcile totals, identities, filters and critical journeys before traffic + resumes. Do not perform bidirectional replay without a conflict policy. + +## restore + +1. Identify the approved backup/PITR point and source checksum. +2. Restore into isolation; never overwrite the only working environment. +3. Verify encryption, database integrity, Alembic revision, FK/check constraints, + active dataset and reconciliation. +4. Persist append-only restore evidence. Restore success does not itself approve + deletion or cutover. + +## security-incident + +1. Preserve evidence, restrict access and notify the assigned security/privacy + owners; currently unassigned ownership is an escalation blocker. +2. Revoke affected sessions/credentials through approved systems and block + suspect routes/sources at the edge. +3. Search only redacted logs by request/trace/job IDs. Do not copy raw personal + payloads into tickets or chat. +4. Apply incident holds before any retention action and document scope/timing. + +## credential-rotation + +1. Identify consumers and secret reference, not secret value. +2. Create a new version in the approved secret manager, update consumers and + verify readiness without printing credentials. +3. Revoke the prior version after all healthy tasks use the new version. +4. Review audit logs and repository scans; never commit `.env` or static AWS + keys. + +## cost-spike + +1. Break down RDS, NAT, Fargate, CloudWatch, S3, Athena, CloudFront, WAF and + transfer costs by environment/tag. +2. Check runaway schedules, log cardinality/retention, query scans, NAT routing + and task scaling. +3. Pause non-critical schedules safely; preserve last-good data and evidence. +4. Any capacity reduction must retain readiness, backup/PITR and performance + budgets. diff --git a/docs/remediation/pipeline-storage-contract.md b/docs/remediation/pipeline-storage-contract.md new file mode 100644 index 0000000..80df8a7 --- /dev/null +++ b/docs/remediation/pipeline-storage-contract.md @@ -0,0 +1,74 @@ +# Durable Pipeline and Storage Contract + +Status: implemented locally by Alembic revision `0005_durable_pipeline`. +No AWS queue, object-store or scheduler operation has been executed. + +## Coordination boundary + +Production API requests create a `job_runs` record, its first `job_events` +entry and one `job_dispatch_outbox` envelope in the same PostgreSQL +transaction. The API never starts a scraper or subprocess. Request-level +idempotency is also PostgreSQL-backed in staging/production, so horizontally +scaled API tasks do not depend on process-local locks or dictionaries. + +Workers claim one queued job with `FOR UPDATE SKIP LOCKED`, publish heartbeats +and bounded leases, and record success, retry, timeout, failure or dead-letter +events. A failed worker cannot activate a candidate dataset: every job and +ingestion run records the dataset that was active at enqueue/start time as its +last-good version. Retry and lease-reaping transitions leave that version +active. + +The transactional outbox is the SQS delivery contract. Its versioned envelope +contains only job identity, type, queue, request time and bounded attempt +metadata; the worker obtains job input from PostgreSQL. Publishing is +idempotently keyed by job ID. AWS delivery remains unexecuted until separately +approved. + +## Object zones and immutability + +`storage_objects` models the `raw`, `validated`, `curated` and `export` bucket +aliases, object keys, provider version IDs, SHA-256 checksums, byte length, +content type and source/run ownership. Raw objects must be immutable. Database +triggers reject update or deletion of immutable object descriptors. +Corrections therefore create new object versions and new ingestion runs. + +`ingestion_run_manifests` is append-only and records: + +- source, source version, dataset and manifest schema version; +- raw, validated and curated object identities; +- before/after watermarks; +- source, accepted, rejected and quarantined record counts; +- retry count and a canonical SHA-256 manifest checksum. + +The Python object-store protocol is intentionally SDK-neutral. Local tests use +an in-memory immutable substitute. Terraform supplies encrypted/versioned S3 +zones later; this phase does not claim an S3 upload. + +## Source schedules and governance + +`config/source-pipelines.json` defines bounded schedules and limits for +360Giving, Charity Commission, Philea, Hinchilla, ECB, Google News RSS, +bounded article content and the optional Anthropic summary integration. Each +entry includes owners, legal/licence status, terms URL, rate limit, user agent, +freshness SLA, schedule, enabled state, watermark, classification, retention +class, schema version, credential reference, retry policy, timeout and record/ +page limits. + +Unknown approval is represented as `unresolved`, never inferred. All eight +sources are disabled and governance-blocked. Both the configuration validator +and PostgreSQL check constraint reject an enabled schedule unless legal and +licence states are `approved` and the governance block has been removed. +Enabling a blocked schedule through the administrator endpoint returns a +conflict. Paid AI/news access remains separately approval-gated. + +## Local gate + +- `0005 -> 0004 -> 0005` migration cycle passes transactionally. +- Seven local contract tests and one real PostgreSQL integration test pass. +- The combined Phase-8/application/schema PostgreSQL run passes 18 tests. +- The normal backend suite passes 318 tests, skips ten explicit live tests and + passes eight route subtests. +- The catalog reports 40 application tables, 49 foreign keys and 161 checks. +- Exactly eight source configurations exist; zero are enabled and eight are + governance-blocked. +- The active dataset remains `sqlite-v7-8fc0cce61c81-r2`. diff --git a/docs/remediation/postgresql-migration-guide.md b/docs/remediation/postgresql-migration-guide.md new file mode 100644 index 0000000..849191a --- /dev/null +++ b/docs/remediation/postgresql-migration-guide.md @@ -0,0 +1,49 @@ +# PostgreSQL setup and migration guide + +The schema is managed exclusively by Alembic. PostgreSQL holds versioned source +facts, materialized analytics, durable jobs/outbox, audit events, governance and +observability controls. One partial unique constraint permits exactly one active +dataset while prior approved datasets remain available for rollback. + +## Empty schema + +```bash +export POSTGRES_PASSWORD_FILE=/absolute/path/to/local-postgres-password +export POSTGRES_HOST_PORT=55432 +docker-compose up -d postgres +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 \ +DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app \ +DATABASE_PASSWORD_FILE="$POSTGRES_PASSWORD_FILE" \ +venv/bin/alembic upgrade head +``` + +## Deterministic full source migration + +Keep the source immutable and invoke the migration with a reviewed output +directory outside `docs/audits/`: + +```bash +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 \ +DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app \ +DATABASE_PASSWORD_FILE="$POSTGRES_PASSWORD_FILE" \ +PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres migrate \ + --source src/data/charities.db \ + --output-dir /private/tmp/fip-migration-evidence \ + --enforce-baseline +``` + +The command preflights disk/capacity and the read-only source, stages a candidate +dataset, preserves duplicate cohorts and anomalies, validates every declared FK, +builds materializations, reconciles counts/controls, and activates only a fully +passing candidate. It is idempotent by source checksum and code revision. + +Post-migration gates: + +```bash +PYTHONPATH=src venv/bin/python scripts/verify_transition.py +PYTHONPATH=src venv/bin/python -m migration.release_gate +``` + +Use `rollback-runbook.md` for exact rollback boundaries. Never replay writes in +both directions without an approved conflict policy. Never run `terraform apply` +or an AWS migration from this local guide. diff --git a/docs/remediation/requirements-traceability.md b/docs/remediation/requirements-traceability.md new file mode 100644 index 0000000..1c0b89d --- /dev/null +++ b/docs/remediation/requirements-traceability.md @@ -0,0 +1,24 @@ +# Remediation requirements traceability + +The immutable source audit remains in `docs/audits/requirements-traceability.md`. +This table records implemented remediation status and does not overwrite it. + +| Requirement | Status | Implementation/evidence | Remaining limitation | AWS/production | +|---|---|---|---|---| +| PostgreSQL default runtime | PASS | transition config, Compose, production route tests | None locally | AWS unexecuted | +| No production SQLite dependency/fallback | PASS | fail-closed modes, data-free image/import boundary | Legacy code remains local migration compatibility | Production unexecuted | +| Lossless deterministic migration | PASS | Phase-4 manifest and 18 Phase-13 projections | New source deltas need rerun | No AWS impact | +| Preserve duplicates/anomalies | PASS | migration controls and reports | Business-owner review remains | No AWS impact | +| Authentication/RBAC/admin plane | PASS locally | security tests and security guide | Real IdP/settings unverified | External blocker | +| Proxy/rate limit/audit/redaction | PASS locally | security/observability tests | Distributed edge behavior untested | AWS unexecuted | +| Performance/materializations | PASS locally | Phase-6 metrics and integration tests | Production load unknown | AWS unexecuted | +| Durable pipelines/jobs | PASS locally | Phase-8 outbox/worker tests | SQS/S3 execution untested | AWS unexecuted | +| Governance/retention/privacy | PARTIAL | Phase-9 controls; deletion disabled | Owners/legal/licence/RPO/RTO unresolved | Production NO-GO | +| Observability/runbooks | PASS locally | Phase-10 contract and runbooks | CloudWatch destinations/threshold approval absent | AWS unexecuted | +| Terraform definitions | PARTIAL | 26 files/103 blocks/58 resource types offline pass | fmt/init/validate/plan/scans/provider locks not tested | No resources changed | +| CI/CD definitions | PASS locally | Phase-12 workflows/offline validation | GitHub settings/OIDC/workflow execution untested | Staging NO-GO | +| Shadow comparison/goldens | PASS locally | 18 projections, zero differences, golden fixture | Production observation unexecuted | AWS unexecuted | +| Backup/restore/rollback | PASS locally | full pg_dump/restore and dataset switch-back | RDS/PITR restore untested | AWS unexecuted | +| Frontend responsive/accessibility | PASS locally | Phase-7 build/E2E/axe evidence | Production/browser matrix limited | Production unexecuted | +| Immutable audit baseline | PASS | exact SQLite/audit checksums | Must recheck before push | No AWS impact | +| Git push | NOT TESTED | 19 local commits ahead at Phase-13 checkpoint | Explicit approval required | No push performed | diff --git a/docs/remediation/retention-privacy-guide.md b/docs/remediation/retention-privacy-guide.md new file mode 100644 index 0000000..8e20611 --- /dev/null +++ b/docs/remediation/retention-privacy-guide.md @@ -0,0 +1,81 @@ +# Retention, Privacy and Data-Subject Guide + +## Safety baseline + +`config/data-governance.json` is authoritative for the current proposed +configuration. Destructive deletion and production activation are both +disabled. Every `delete_after_days` value is `null`. PostgreSQL independently +requires approval, successful restore evidence and no holds for any future +destructive retention action; the initial deletion-manifest table additionally +accepts only dry-run report/archive actions. + +There is deliberately no destructive HTTP route, cleanup command or scheduled +delete worker. + +## Dry-run lifecycle + +1. Identify a target, retention class, last-modified time, record/object/byte + counts and available SHA-256 checksums. +2. Check exact, retention-class and global legal/incident holds. +3. Produce `report` or `archive` with `dry_run=true`; a hold produces `held`. +4. Store an auditable `retention_actions` record and append-only + `deletion_manifests` evidence. +5. Do not change or delete the target. + +Archive correction means a new report/manifest; evidence cannot be edited. +Export expiry likewise produces a report. `hold_until` suppresses expiry +reporting while active. + +## Holds + +Legal and incident holds contain type, scoped target, reason, actor and time. +They may scope one target, a retention class or all data. Only an +administrator can create or release a hold, and both actions require an +idempotency key and audit reason. Holds override retention even after a window +is reached. + +## Restore-before-delete + +Any later destructive design must reference an append-only +`restore_verifications` record with target, backup reference, backup checksum, +verifier, result and evidence. The current planner rejects deletion before +evaluating this prerequisite because global deletion is disabled. Recording a +restore test never enables deletion by itself. + +## Data-subject workflow + +1. Receive and classify access, correction, deletion, restriction or objection. +2. Persist only a SHA-256 subject reference at intake. +3. Verify identity using an approved private process. +4. Locate scoped records without exporting unrelated data. +5. Check legal/incident holds and source/legal constraints. +6. Record an approval or rejection and the responsible handler. +7. Execute a mutation only under a separately approved, audited procedure. +8. Retain the audit reference and notify the requester through an approved + private channel. + +This repository implements intake/control state, not an identity-verification +service or an approved deletion executor. + +## Log redaction + +Structured redaction replaces credentials, Authorization/cookies, connection +strings, emails, postal addresses, raw payloads and article bodies. It recurses +through objects and arrays before administrative output is serialized. Plain +log text additionally removes credential-shaped values, database passwords, +AWS-key shapes, model-key shapes and email addresses. + +Logs must never contain full third-party article bodies, raw Authorization +headers or unredacted personal/contact fields. + +## Backup, PITR, RTO and RPO + +- Backup policy: proposed; encryption and restore tests required. Retention and + cross-account copy remain unresolved. +- PostgreSQL PITR: required by proposed policy; retention period unresolved. +- RTO: unresolved. +- RPO: unresolved. + +Terraform may encode fail-safe defaults, but these values must not be presented +as approved service objectives until owners decide them and deployment evidence +exists. diff --git a/docs/remediation/rollback-runbook.md b/docs/remediation/rollback-runbook.md new file mode 100644 index 0000000..1798883 --- /dev/null +++ b/docs/remediation/rollback-runbook.md @@ -0,0 +1,41 @@ +# PostgreSQL rollback runbook + +This is a controlled rollback, not a bidirectional synchronization design. + +1. Declare the rollback and route traffic to the last known-good immutable + application task definition. If the fault is data-only and the current + binary is compatible, keep traffic drained until dataset activation finishes. +2. Stop pipeline workers, schedules and every mutation endpoint. Confirm no + queued/running/retrying writers and retain the failed target unchanged for + investigation. +3. Record current task definitions, schema revision, active dataset, database + backup identifier, counts, checksums, reconciliation output and error traces. +4. Select only a retained `approved` or `rolled_back` PostgreSQL dataset with an + active or reproducible dashboard materialization. Never select an unknown or + rejected candidate. +5. Execute the transactional local form only after resolving the exact target: + + ```bash + DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 \ + DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app \ + DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder \ + PYTHONPATH=src venv/bin/python -m migration.sqlite_to_postgres \ + rollback --dataset-version EXACT_APPROVED_DATASET_VERSION + ``` + +6. Run readiness, release, FK/orphan, reconciliation, golden and authenticated + journey smoke tests. Resume traffic, then writers, only after they pass. +7. If logical dataset activation cannot recover the service, restore the prior + PostgreSQL backup into a separate database, validate it, and promote it by + the approved database recovery procedure. Never overwrite the failed target + before evidence is retained. +8. Do not replay changes in both directions unless an approved conflict policy, + ownership rule, idempotency model and reconciliation design exists. No such + policy is currently approved. + +The local proof uses `scripts/verify_local_rollback.py`. It refuses active jobs, +verifies counts/materialization on the prior version and restores the original +version in a `finally` path if an intermediate assertion fails. + +AWS traffic changes, ECS rollbacks, RDS restores and production execution are +`NOT TESTED` and require explicit approval. diff --git a/docs/remediation/runtime-transition-guide.md b/docs/remediation/runtime-transition-guide.md new file mode 100644 index 0000000..e68fcc6 --- /dev/null +++ b/docs/remediation/runtime-transition-guide.md @@ -0,0 +1,49 @@ +# Runtime transition guide + +Date: 2026-07-29 + +PostgreSQL is the default and only authoritative operational datastore. The +transition configuration has three explicit modes: + +| `DATA_RUNTIME_MODE` | Purpose | May serve user responses | Environment boundary | +|---|---|---:|---| +| `postgresql` | Normal runtime | Yes, from PostgreSQL | Default everywhere | +| `sqlite_migration_source` | Local/test migration compatibility | Local/test only | Rejected in staging/production | +| `shadow_compare` | Temporary PostgreSQL-primary comparison | Yes, PostgreSQL only | Requires a separate coherent SQLite snapshot | + +There is no SQLite failover. A failed, timed-out, oversized or queue-rejected +shadow read cannot change the primary response. `shadow_compare` schedules the +SQLite read only after the final PostgreSQL response body has been passed to the +client. Work is bounded to eight pending comparisons, 2 MiB per response, a +20-second shadow timeout and 100 recorded differences. + +Differences contain a JSON path, difference kind and SHA-256-derived value +fingerprints; values themselves are not logged. Ordering is ignored only for +the three allowlisted set-like paths in `config/runtime-transition.json`. +Rankings, pages, grant lists and Sankey links retain semantic order. + +The SQLite adapter opens only an explicitly supplied snapshot copy. The +configuration rejects `SHADOW_SQLITE_PATH` when it resolves to the active +`DB_PATH`, preventing the compatibility repository from changing the preserved +source. Production images contain no SQLite database or source dataset. + +Local activation example (temporary, not a production fallback): + +```bash +cp src/data/charities.db /private/tmp/fip-shadow-snapshot.db +DATA_RUNTIME_MODE=shadow_compare \ +SHADOW_SQLITE_PATH=/private/tmp/fip-shadow-snapshot.db \ +DATABASE_HOST=127.0.0.1 DATABASE_PORT=55432 \ +DATABASE_NAME=foundation_intelligence DATABASE_USER=foundation_app \ +DATABASE_PASSWORD_FILE=/private/tmp/fip-compose-secret-placeholder \ +PYTHONPATH=src venv/bin/uvicorn bff.main:app --host 127.0.0.1 --port 8000 +``` + +Delete only the temporary snapshot after evidence has been reviewed. Never +mount the preserved active SQLite source into a production task. + +The versioned journey register contains dashboard, maps and relationships, +date/country/programme/donor/recipient filters, monthly/yearly trends, +donor/recipient rankings, registry search, profile, grant list, drill-down, +Sankey, score, news, pipeline status and manual-refresh authorization. Live +news is deliberately not invoked during local acceptance. diff --git a/docs/remediation/schemas/migration-manifest.schema.json b/docs/remediation/schemas/migration-manifest.schema.json new file mode 100644 index 0000000..1baeeb5 --- /dev/null +++ b/docs/remediation/schemas/migration-manifest.schema.json @@ -0,0 +1,166 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://foundation-intelligence.local/schemas/migration-manifest.schema.json", + "title": "Foundation Intelligence PostgreSQL Migration Manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "migration_run_id", + "source_database_checksum", + "source_schema_version", + "source_fact_version", + "target_schema_version", + "dataset_version", + "code_revision", + "started_at", + "completed_at", + "source_counts", + "target_counts", + "reconciliation_results", + "activation_status", + "rollback_dataset_version", + "errors", + "actor" + ], + "properties": { + "migration_run_id": { + "type": "string", + "format": "uuid" + }, + "source_database_checksum": { + "type": "string", + "pattern": "^[a-f0-9]{64}$" + }, + "source_schema_version": { + "type": "string", + "minLength": 1 + }, + "source_fact_version": { + "type": "string", + "minLength": 1 + }, + "target_schema_version": { + "type": "string", + "minLength": 1 + }, + "dataset_version": { + "type": "string", + "minLength": 1 + }, + "code_revision": { + "type": "string", + "pattern": "^[a-f0-9]{40}$" + }, + "started_at": { + "type": "string", + "format": "date-time" + }, + "completed_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "source_counts": { + "$ref": "#/$defs/counts" + }, + "target_counts": { + "$ref": "#/$defs/counts" + }, + "controls": { + "type": "object", + "additionalProperties": { + "type": ["integer", "number", "string"] + } + }, + "reconciliation_results": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/reconciliationResult" + } + }, + "activation_status": { + "enum": [ + "created", + "loading", + "validating", + "rejected", + "approved", + "active", + "rolled_back", + "failed" + ] + }, + "rollback_dataset_version": { + "type": ["string", "null"] + }, + "errors": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["code", "message"], + "properties": { + "code": {"type": "string", "minLength": 1}, + "message": {"type": "string", "minLength": 1}, + "record_identity": {"type": ["string", "null"]}, + "details": {"type": "object"} + } + } + }, + "actor": { + "type": "object", + "additionalProperties": false, + "required": ["id", "type"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "type": {"enum": ["human", "service", "ci"]} + } + }, + "capacity": { + "type": "object", + "additionalProperties": false, + "required": [ + "source_bytes", + "estimated_postgres_and_indexes_bytes", + "estimated_wal_and_temp_bytes", + "safety_margin_bytes", + "minimum_free_bytes", + "available_bytes", + "sufficient" + ], + "properties": { + "source_bytes": {"type": "integer", "minimum": 0}, + "estimated_postgres_and_indexes_bytes": {"type": "integer", "minimum": 0}, + "estimated_wal_and_temp_bytes": {"type": "integer", "minimum": 0}, + "safety_margin_bytes": {"type": "integer", "minimum": 0}, + "minimum_free_bytes": {"type": "integer", "minimum": 0}, + "available_bytes": {"type": "integer", "minimum": 0}, + "sufficient": {"type": "boolean"} + } + }, + "idempotent_noop": { + "type": "boolean" + } + }, + "$defs": { + "counts": { + "type": "object", + "additionalProperties": { + "type": "integer", + "minimum": 0 + } + }, + "reconciliationResult": { + "type": "object", + "additionalProperties": false, + "required": ["status", "source", "target"], + "properties": { + "status": {"enum": ["pass", "fail", "approved_difference", "not_tested"]}, + "source": {}, + "target": {}, + "difference": {}, + "reason": {"type": ["string", "null"]}, + "approval_reference": {"type": ["string", "null"]} + } + } + } +} diff --git a/docs/remediation/security-authentication-rbac-guide.md b/docs/remediation/security-authentication-rbac-guide.md new file mode 100644 index 0000000..5f6fca5 --- /dev/null +++ b/docs/remediation/security-authentication-rbac-guide.md @@ -0,0 +1,78 @@ +# Authentication, RBAC and administration guide + +Staging and production fail closed unless `AUTH_MODE=oidc`. The issuer, +audience, JWKS and asymmetric algorithm allowlist are explicit. Browser sessions +use secure cookies; wildcard CORS, browser credential forwarding and anonymous +mutations are forbidden. Local development authentication requires an explicit +opt-in and localhost allowlist. + +Roles are ordered by capability: + +| Role | Read data | Score/news | Queue refresh/enrichment | Governance/admin | +|---|---:|---:|---:|---:| +| viewer | Yes | No | No | No | +| analyst | Yes | Yes | No | No | +| operator | Yes | Yes | Yes | Limited operational reads | +| administrator | Yes | Yes | Yes | Yes | + +Mutation routes require role authorization, bounded request bodies, rate limits, +an idempotency key where declared, structured audit evidence and safe error +responses. Manual refresh only enqueues a durable job; it never starts a scraper +inside the API process. Pipeline workers and dispatchers are separate. + +The optional core proxy is disabled by default. If enabled, its destination +host, path, method and headers must match explicit allowlists; browser +`Authorization` and `Cookie` headers are never forwarded. + +## Temporary public read-only demo + +`AUTH_MODE=public_readonly` is accepted only together with `APP_ENV=demo`. +Any other environment fails during startup. This mode is exclusively for a +temporary public, non-production demonstration. It creates no login, session, +cookie or shared token and requires the development login and core proxy to be +disabled. It also requires `DATA_RUNTIME_MODE=postgresql`; the legacy SQLite +response path cannot be exposed by this mode. + +Anonymous access is a route-template and method allowlist in +`bff.security.PUBLIC_READONLY_ROUTE_ALLOWLIST`. Only `GET` and `HEAD` can use +the anonymous viewer principal. The reviewed allowlist contains the normal +read-only UI surfaces: + +- `/api/charities` +- `/api/charities/stats` +- `/api/charities/{reg_charity_number}` +- `/api/charities/{reg_charity_number}/grants` +- `/api/charities/{reg_charity_number}/sankey` +- `/api/charities/{reg_charity_number}/score` +- `/api/charities/directory/organizations` +- `/api/charities/directory/organizations/{registry_id}` +- `/api/charities/grants/beneficiary-geographies` +- `/api/charities/grants/funders` +- `/api/charities/grants/funders/{source_funder_key}` +- `/api/charities/grants/map` +- `/api/charities/grants/overview` +- `/api/charities/grants/overview/trends` +- `/api/charities/grants/overview/entity-suggestions` +- `/api/charities/grants/overview/drilldown` +- `/api/charities/grants/map/connections` +- `/api/charities/grants/summary` +- `/api/charities/grants/trends` +- `/api/charities/grants/themes` + +These routes provide the dashboard, statistics, maps, filters, rankings, +organization/registry directories, donor directory, details, grant evidence +and drill-downs shown by the browser. Profile-cache hydration, live news +research, exports, authentication, administration, system configuration, +jobs/pipelines, governance/retention, observability and all mutation routes +remain protected. Adding a viewer route does not make it public; every new +public route requires a separate schema, response and data-classification +review before it can be added to the exact allowlist. + +Secrets must enter through mounted files, runtime secret stores or OIDC. Do not +commit `.env`, passwords, tokens, Terraform state/plans or generated cloud +secrets. Logs recursively redact credentials, connection strings, emails, +postal details, raw payloads and article bodies. + +GitHub/AWS OIDC trust, branch protection and protected-environment reviewers +are defined/required but externally `NOT TESTED`. No long-lived AWS access key +is accepted by the deployment workflows. diff --git a/docs/remediation/terraform-aws-infrastructure.md b/docs/remediation/terraform-aws-infrastructure.md new file mode 100644 index 0000000..b9bf0d5 --- /dev/null +++ b/docs/remediation/terraform-aws-infrastructure.md @@ -0,0 +1,182 @@ +# Terraform AWS Infrastructure Contract + +Status: implemented and offline-checked for development and staging. No AWS +API call, Terraform provider download, state access, plan or apply has occurred. + +## Architecture + +```mermaid +flowchart LR + User[Browser] --> CF[CloudFront] + CF --> Frontend[(Private frontend S3)] + User --> WAF[Regional WAF] + WAF --> ALB[Public ALB] + ALB --> API[ECS Fargate API] + API --> RDS[(Private RDS PostgreSQL)] + API --> SQS[SQS FIFO] + SQS --> Workers[ECS Fargate workers] + Workers --> RDS + Workers --> Raw[(Raw S3)] + Workers --> Validated[(Validated S3)] + Workers --> Curated[(Curated S3)] + Workers --> Exports[(Exports S3)] + Scheduler[Disabled EventBridge schedule] --> SFN[Step Functions] + SFN --> Workers +``` + +The ALB occupies public subnets only. API and workers have no public IP and +run in private application subnets. PostgreSQL occupies isolated private +database subnets with no internet route. Development uses one NAT gateway and +omits interface endpoints to cap fixed cost; staging uses one NAT per zone and +private ECR/logs/secrets/SQS endpoints. Both use the S3 gateway endpoint. + +## Deployment topology + +```mermaid +flowchart TB + subgraph GitHub[GitHub protected environment] + Build[Build + scan immutable artefacts] + Approval[Required environment approval] + OIDC[Short-lived OIDC token] + end + subgraph AWS[One approved AWS environment] + Role[Exact-subject deployment role] + ECR[Immutable ECR repositories] + ECS[ECS API + workers] + S3[Private frontend bucket] + CDN[CloudFront] + end + Build --> Approval --> OIDC --> Role + Role --> ECR + Role --> ECS + Role --> S3 + Role --> CDN +``` + +The role trust pins `aud=sts.amazonaws.com` and exactly +`repo:/:environment:`. No AWS access key is a +Terraform input. Infrastructure bootstrap/state ownership is deliberately not +granted to the artefact deployment role. + +## Data flow + +```mermaid +flowchart LR + Source[Approved source] --> Raw[Versioned raw object] + Raw --> Validate[Validate + quarantine] + Validate --> Validated[Versioned validated object] + Validated --> Curate[Curate + reconcile] + Curate --> Curated[Versioned curated object] + Curated --> Candidate[(Candidate PostgreSQL dataset)] + Candidate --> Gate{Reconciliation gate} + Gate -->|pass| Active[(Single active dataset)] + Gate -->|fail| Quarantine[Retain last-good dataset] + Active --> API[Read-only serving journeys] + API --> Export[Versioned exports] +``` + +Every data bucket blocks public access, enforces bucket-owner ownership and +KMS encryption. Raw/validated/curated/export buckets are versioned. Lifecycle +rules only transition storage classes; no object or version expiration is +defined. Schedule state remains `DISABLED` in both environments. + +## Security boundaries + +```mermaid +flowchart TB + Internet((Internet)) -->|80 redirect / 443 only after DNS approval| ALBSG[ALB security group] + ALBSG -->|TCP 8000| ECSSG[ECS security group] + ECSSG -->|TCP 5432| RDSSG[RDS security group] + ECSSG -->|TCP 443| Endpoints[AWS endpoints / NAT] + Endpoints --> Services[ECR, logs, secrets, SQS, S3] + RDSSG --> RDS[(Encrypted private RDS)] + subgraph Identity[Identity boundary] + ExecRole[ECS execution role] + APIRole[API task role] + WorkerRole[Worker task role] + DeployRole[GitHub deployment role] + end +``` + +Cross-tier rules reference security groups rather than CIDRs. RDS has no +public endpoint, is deletion-protected and has a Terraform `prevent_destroy` +guard. ECS tasks run as `10001:10001`, drop all Linux capabilities, use a +read-only root filesystem and accept only digest-addressed images. Database +credentials are generated by RDS, encrypted by KMS and injected from Secrets +Manager at runtime. Application code accepts the injected password without +writing a secret file. + +The only wildcard IAM resources are AWS API necessities: ECR authorization, +KMS key policy scope, Step Functions log-delivery/task-observation APIs and +tag-constrained task-definition registration. No statement grants wildcard +actions. Bucket wildcard actions occur only in an explicit deny for insecure +transport. + +## Environment profile and cost envelope + +These are planning envelopes, not live AWS quotes or approved budgets. They +must be replaced with an approved AWS Pricing Calculator export before apply. + +| Driver | Development assumption | Staging assumption | +|---|---:|---:| +| RDS PostgreSQL | Single-AZ `db.t4g.small`, 30–100 GiB | Multi-AZ `db.t4g.medium`, 100–500 GiB | +| NAT | One gateway, low transfer | One per AZ, production-like transfer | +| Fargate | One 0.5-vCPU API + one worker | Two 0.5-vCPU API + two workers, autoscaling | +| Endpoints | S3 gateway only | S3 plus ECR/logs/secrets/SQS endpoints | +| Logs/metrics | 30-day logs, low ingest | 90-day logs, container insights and higher ingest | +| S3/Athena | Limited fixtures; Athena usage not provisioned | Versioned data; Athena scans remain optional/unapproved | +| CloudFront/WAF/ALB | Low request volume | Pre-production request/load-test volume | +| Data transfer | Low | Moderate; source and E2E tests dominate | +| Estimated monthly range | USD 180–300 | USD 550–900 | +| Proposed budget alarm | USD 300 | USD 750 | + +The largest uncertain drivers are NAT bytes, RDS Multi-AZ/runtime, Fargate +worker duty cycle, CloudWatch log cardinality/retention, S3 volume/class +transitions, optional Athena scans, WAF requests, CloudFront egress and +cross-AZ/data-transfer traffic. + +## DNS, ACM and state configuration points + +`manage_dns=false` is the default and is fixed in both examples. Enabling it +requires an approved domain and Route53 zone ID; Terraform then defines DNS +validation, an ACM certificate, HTTPS listener and ALB alias. No DNS or +certificate action has been executed. CloudFront uses its AWS TLS hostname +until a separately reviewed us-east-1 edge certificate/DNS alias is approved. + +Each environment now contains an encrypted, coordinate-free S3 backend +configuration point for the protected deployment workflow. Bucket, key, +region and lock-table coordinates are supplied only by protected environment +variables. An approved operator must separately define the state account, +encrypted/versioned state bucket, lock mechanism, recovery owner and bootstrap +procedure. This repository does not create or mutate that state foundation +under the current authorization. + +## Validation and deployment boundary + +The offline validation command is: + +```zsh +PYTHONPATH=src venv/bin/python scripts/validate_terraform_static.py +PYTHONPATH=src venv/bin/python -m pytest -q src/tests/test_terraform_contract.py +``` + +Terraform 1.9.8 parsing and `fmt -check -recursive` have now passed in a +network-isolated container. Once the AWS provider source and an isolated +non-AWS validation context are approved, continue for each environment: + +```zsh +terraform fmt -check -recursive infra/terraform +terraform -chdir=infra/terraform/environments/staging init -backend=false +terraform -chdir=infra/terraform/environments/staging validate +``` + +Only after separate approval for AWS read access and safe plan credentials: + +```zsh +terraform -chdir=infra/terraform/environments/staging plan -refresh=false -lock=false -input=false -var-file=terraform.tfvars -out=reviewed.tfplan +terraform -chdir=infra/terraform/environments/staging show -json reviewed.tfplan +``` + +Do not run `apply`, `destroy`, `import`, state mutation, DNS changes or a real +plan under the current authorization. The example image/account values are +non-deployable placeholders and all schedules remain disabled. diff --git a/docs/remediation/terraform-validation.md b/docs/remediation/terraform-validation.md new file mode 100644 index 0000000..fd4b428 --- /dev/null +++ b/docs/remediation/terraform-validation.md @@ -0,0 +1,69 @@ +# Terraform Validation Status + +Date: 2026-07-29 + +## Local result + +- Terraform configuration: 26 `.tf` files, 103 resource blocks and 58 AWS + resource types across one platform module plus dev/staging roots. +- Offline bracket/string/comment structure check: `PASS`. +- Offline required-resource and environment coverage check: `PASS`. +- Offline security contract: `PASS` for private/deletion-protected RDS, + encrypted/non-public/non-destructive S3, immutable ECR/images, private and + non-root ECS, fail-closed schedules, exact GitHub OIDC subject/audience and + bounded wildcard-resource contexts. +- Python contract tests: `PASS`. +- `terraform fmt` and `fmt -check -recursive`: `PASS` with Terraform `1.9.8` + in a network-isolated Docker container. +- `init -backend=false`: `BLOCKED` after installing the local platform module; + the AWS provider cannot be resolved from the authorised registries. +- `validate`: `BLOCKED` on the missing AWS provider after local module + initialization; no provider-dependent success is claimed. +- Provider lock generation: `NOT TESTED`; no `.terraform.lock.hcl` is claimed. +- Checkov/tfsec/tflint/trivy: `NOT TESTED` because none is installed locally. +- Terraform plan: `NOT TESTED` because Terraform/providers are unavailable and + AWS read access is outside the approved boundary. +- AWS changes/state/DNS/certificate/provider downloads: none. + +## Exact blockers + +The host still has no Terraform binary, but the explicitly approved Docker Hub +download resolved `hashicorp/terraform:1.9.8` to manifest digest +`sha256:18f9986038bbaf02cf49db9c09261c778161c51dcc7fb7e355ae8938459428cd` +(local image ID `sha256:97aaea908f872c3c60b75e9bffd6eeae34386c0e9671d6b2a1e30418ea702269`). +The first real parser run rejected an invalid single-line nested WAF block; +that syntax and the remaining canonical formatting were corrected. The final +network-isolated `fmt -check -recursive` passes. + +`terraform init -backend=false -input=false` was then attempted in an isolated +temporary copy with `--network none`. It installed the local module and stopped +at: + +```text +Failed to query available provider packages +Could not retrieve the list of available versions for provider hashicorp/aws +``` + +The container had no network and made no successful external contact. A +follow-up `validate` reported `Missing required provider`. The temporary copy +was removed. `tofu`, `tflint`, `checkov`, `tfsec`, `trivy` and `semgrep` remain +absent. Downloading Terraform providers would contact `registry.terraform.io`, +which is outside the approved dependency sources. + +## Remaining exact validation + +After explicit approval for the Terraform provider source, generate and commit +provider locks for supported platforms, then run: + +```zsh +terraform fmt -check -recursive infra/terraform +terraform -chdir=infra/terraform/environments/dev init -backend=false +terraform -chdir=infra/terraform/environments/dev validate +terraform -chdir=infra/terraform/environments/staging init -backend=false +terraform -chdir=infra/terraform/environments/staging validate +``` + +Run an approved, version-pinned Terraform security scanner and resolve every +high/critical finding. A staging plan additionally requires explicit AWS +read-access approval and real non-secret variables; it must use no remote +state/lock under the current workflow. Apply remains prohibited. diff --git a/docs/remediation/troubleshooting-guide.md b/docs/remediation/troubleshooting-guide.md new file mode 100644 index 0000000..ea434b7 --- /dev/null +++ b/docs/remediation/troubleshooting-guide.md @@ -0,0 +1,46 @@ +# Troubleshooting guide + +## Readiness is unavailable + +Run liveness first, then inspect the independent readiness checks. Verify the +database secret file, host/port, Alembic revision, exactly one active dataset, +source/governance configuration and durable outbox tables. Do not relax +readiness to route traffic around a missing schema or dataset. + +## Migration refuses the schema + +Run `alembic current` and `alembic heads`. The migration gate reads the expected +revision from the versioned observability configuration. Upgrade with Alembic; +never edit `alembic_version` manually. + +## Shadow differences appear + +Preserve the evidence, identify the JSON paths, reproduce with +`scripts/verify_transition.py`, and classify the cause. Only versioned +allowlisted set ordering may be ignored. Do not add broad tolerances, remove +fields or switch responses to SQLite. + +## Rollback cannot select a target + +The target must exist with `approved` or `rolled_back` status and a valid +materialization. Retain the failed target, freeze writers and follow the +rollback runbook. Never force status flags in SQL. + +## Docker build/start fails + +Use `docker build --pull=false` only when pinned base images are present. The +runtime image intentionally contains no SQLite database or raw data. Compose +requires a real local password file and explicitly selects PostgreSQL. + +## Terraform validation is unavailable + +Do not claim success. Terraform/provider locks and scanners are not locally +available from approved sources. Use the offline validator, record `NOT TESTED`, +and later run the exact non-destructive commands in `terraform-validation.md`. +Never run apply/destroy to diagnose validation. + +## GitHub/AWS workflow cannot run + +Branch protection, environments, reviewers, OIDC trust and action SHA resolution +are external blockers. Workflow files alone do not prove configuration. Stop +before push/AWS and request explicit approval. diff --git a/frontend/.env.example b/frontend/.env.example index 9f6c237..be10e24 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -2,5 +2,5 @@ # By default the UI uses its current browser hostname on port 8000. Override only # when the API genuinely runs elsewhere; mixing localhost and 127.0.0.1 breaks cookies. # VITE_API_BASE_URL=http://127.0.0.1:8000 -VITE_BFF_USERNAME=admin -VITE_BFF_PASSWORD=password +# Authentication is handled by the deployment OIDC flow. Never add credentials +# or shared secrets to VITE_* variables because they are bundled into JavaScript. diff --git a/frontend/.gitignore b/frontend/.gitignore index a2c5b94..2a045eb 100644 --- a/frontend/.gitignore +++ b/frontend/.gitignore @@ -11,6 +11,8 @@ node_modules dist dist-ssr .test-dist +playwright-report +test-results *.local # Editor directories and files diff --git a/frontend/e2e/phase7.spec.ts b/frontend/e2e/phase7.spec.ts new file mode 100644 index 0000000..b6240ae --- /dev/null +++ b/frontend/e2e/phase7.spec.ts @@ -0,0 +1,240 @@ +import { expect, test, type Page } from "@playwright/test"; +import AxeBuilder from "@axe-core/playwright"; + +const map = { + status: "available", + geographic_dimension: "beneficiary", + items: [{ + region_or_country_code: "GB", + region_or_country_name: "United Kingdom", + grant_count: 1250, + total_amount: 123456789, + currency: "EUR", + distinct_funders: 84, + distinct_recipients: 720, + top_programme_areas: [], + top_funders: [], + top_recipients: [], + original_geographies: ["United Kingdom"], + funding_grant_count: 1200, + excluded_multi_country_grant_count: 0, + excluded_invalid_amount_grant_count: 0, + }], + known_geography_count: 1250, + unknown_geography_count: 50, + coverage_percentage: 96.2, + currencies: ["EUR"], + selected_currency: "EUR", + funding_status: "available", + funding_mode_available: true, + grant_country_association_count: 1250, + multi_country_grant_count: 0, + funding_excluded_multi_country_count: 0, + funding_excluded_multi_country_amount: 0, + funding_excluded_currency_count: 0, + funding_excluded_invalid_amount_count: 0, + connections: [], + connection_grant_count: 0, + connection_excluded_no_headquarters_count: 0, + connection_same_country_count: 0, + minimum_coverage_threshold: 0, + metadata: { data_mode: "playwright_test", source: ["360Giving"], record_count: 1300, limitations: [] }, +}; + +const overview = { + status: "available", + kpis: { + awarded_funding: 123456789, + currency: "EUR", + grants_monitored: 1300, + country_coverage_percentage: 96.2, + mapped_grant_count: 1250, + unmapped_grant_count: 50, + programme_coverage_percentage: 87.5, + classified_grant_count: 1138, + qualifying_programme_grant_count: 1300, + }, + map, + trends: { + status: "available", + currency: "EUR", + available_currencies: ["EUR"], + granularity: "monthly", + period: null, + items: [], + excluded: {}, + last_refreshed_at: null, + }, + themes: { + status: "available", + currency: "EUR", + items: [], + classification_coverage: { classified_percentage: 87.5, classified_grant_count: 1138, unclassified_grant_count: 162 }, + }, + available_date_range: { from: "2024-01-01", to: "2026-06-30" }, +}; + +const responses = { + health: { status: "ok" }, + stats: { + total_charities: 42, + active_charities: 40, + removed_charities: 2, + average_income: 1_000_000, + average_expenditure: 800_000, + source: ["360Giving", "Charity Commission for England and Wales", "Philea"], + }, + overview, + geographies: ["United Kingdom", "Germany", "France"], + entitySuggestions: { status: "available", donors: [], recipients: [] }, + connections: { status: "available", connections: [], connection_grant_count: 0 }, + registry: { results: [], next_cursor: null, has_more: false, page_size: 50, search_strategy: "indexed_directory" }, +}; + +async function installApiRoutes(page: Page) { + const requestCounts = new Map(); + await page.route("**/health", async route => { + const key = new URL(route.request().url()).pathname; + requestCounts.set(key, (requestCounts.get(key) || 0) + 1); + await route.fulfill({ json: responses.health }); + }); + await page.route("**/api/**", async route => { + const url = new URL(route.request().url()); + const key = `${url.pathname}${url.search}`; + requestCounts.set(key, (requestCounts.get(key) || 0) + 1); + let body: unknown; + if (url.pathname === "/api/charities/stats") body = responses.stats; + else if (url.pathname === "/api/charities/grants/overview") body = responses.overview; + else if (url.pathname === "/api/charities/grants/beneficiary-geographies") body = responses.geographies; + else if (url.pathname === "/api/charities/grants/overview/entity-suggestions") body = responses.entitySuggestions; + else if (url.pathname === "/api/charities/grants/map/connections") body = responses.connections; + else if (url.pathname === "/api/charities/directory/organizations") body = responses.registry; + else body = { detail: `Unexpected local test route: ${url.pathname}` }; + await route.fulfill({ status: body === responses.registry || !Object.hasOwn(body as object, "detail") ? 200 : 404, json: body }); + }); + return requestCounts; +} + +function captureBrowserProblems(page: Page) { + const problems: string[] = []; + page.on("console", message => { + if (message.type() === "warning" || message.type() === "error") problems.push(`${message.type()}: ${message.text()}`); + }); + page.on("pageerror", error => problems.push(`pageerror: ${error.message}`)); + return problems; +} + +async function expectNoHorizontalOverflow(page: Page) { + const dimensions = await page.evaluate(() => ({ + content: Math.max(document.documentElement.scrollWidth, document.body.scrollWidth), + viewport: window.innerWidth, + })); + expect(dimensions.content).toBeLessThanOrEqual(dimensions.viewport + 1); +} + +test("map-first overview is responsive, accessible and request-bounded", async ({ page }, testInfo) => { + const requestCounts = await installApiRoutes(page); + const browserProblems = captureBrowserProblems(page); + await page.goto("/"); + await expect(page.locator(".world-map-card")).toBeVisible(); + + await expectNoHorizontalOverflow(page); + const layout = await page.evaluate(() => { + const viewportWidth = window.innerWidth; + const inside = (element: Element) => { + const bounds = element.getBoundingClientRect(); + return bounds.left >= -1 && bounds.right <= viewportWidth + 1; + }; + const visibleControls = [...document.querySelectorAll("button, input, select, summary, a")] + .filter(element => { + const bounds = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return bounds.width > 0 && bounds.height > 0 && bounds.right > 0 && bounds.left < viewportWidth + && bounds.bottom > 0 && bounds.top < window.innerHeight && style.visibility !== "hidden"; + }); + const accessibleName = (element: Element) => { + const labels = "labels" in element + ? [...((element as HTMLInputElement).labels || [])].map(label => label.textContent || "").join(" ").trim() + : ""; + return element.getAttribute("aria-label") || labels || element.textContent?.trim() || element.getAttribute("title") || ""; + }; + const mapCard = document.querySelector(".world-map-card"); + const analytics = document.querySelector(".overview-analytics-grid"); + return { + clippedControls: visibleControls.filter(element => !inside(element)).length, + unnamedControls: visibleControls.filter(element => !accessibleName(element)).length, + croppedKpis: [...document.querySelectorAll(".overview-kpi")] + .filter(element => element.scrollWidth > element.clientWidth + 1 || element.scrollHeight > element.clientHeight + 1).length, + mapBeforeAnalytics: Boolean(mapCard && analytics && (mapCard.compareDocumentPosition(analytics) & Node.DOCUMENT_POSITION_FOLLOWING)), + mapControlsInside: [...document.querySelectorAll(".map-mode-control button")].every(inside), + }; + }); + expect(layout).toEqual({ + clippedControls: 0, + unnamedControls: 0, + croppedKpis: 0, + mapBeforeAnalytics: true, + mapControlsInside: true, + }); + + const overviewRequestCount = [...requestCounts.entries()] + .filter(([key]) => key.startsWith("/api/charities/grants/overview?")) + .reduce((total, [, count]) => total + count, 0); + expect(overviewRequestCount).toBe(1); + + if (testInfo.project.name === "chrome-320") { + await page.getByRole("button", { name: "Connections" }).click(); + await expect.poll(() => [...requestCounts.entries()] + .filter(([key]) => key.startsWith("/api/charities/grants/map/connections?")) + .reduce((total, [, count]) => total + count, 0)).toBe(1); + } + + const filterTrigger = page.locator(".app-header-filter"); + await filterTrigger.focus(); + await filterTrigger.click(); + const drawer = page.locator(".overview-filter-drawer"); + await expect(drawer).toBeVisible(); + expect(await drawer.evaluate(panel => panel.contains(document.activeElement))).toBe(true); + const drawerControls = drawer.locator("button:not([disabled]), input:not([disabled]), select:not([disabled])"); + await drawerControls.last().focus(); + await page.keyboard.press("Tab"); + await expect(drawerControls.first()).toBeFocused(); + await page.keyboard.press("Escape"); + await expect(drawer).toBeHidden(); + await expect(filterTrigger).toBeFocused(); + + const axe = await new AxeBuilder({ page }).analyze(); + expect(axe.violations, JSON.stringify(axe.violations, null, 2)).toEqual([]); + expect(browserProblems).toEqual([]); +}); + +test("Donor and Registry empty-state journey remains keyboard-safe", async ({ page }, testInfo) => { + test.skip(!["chrome-320", "chrome-1024"].includes(testInfo.project.name), "Representative mobile and tablet/desktop widths cover this secondary journey."); + await installApiRoutes(page); + const browserProblems = captureBrowserProblems(page); + await page.goto("/"); + await expect(page.locator(".world-map-card")).toBeVisible(); + + if (testInfo.project.name === "chrome-320") await page.getByRole("button", { name: "Open navigation" }).click(); + await page.getByTitle("Donor Directory").click(); + await expect(page.locator(".donor-directory-page")).toBeVisible(); + await expectNoHorizontalOverflow(page); + await page.getByRole("button", { name: /Advanced Charity Commission Search/ }).click(); + await expect(page.locator(".registry-directory")).toBeVisible(); + await expect(page.getByRole("heading", { name: "No registry organizations found" })).toBeVisible(); + await expectNoHorizontalOverflow(page); + + const filterTrigger = page.locator(".app-header-filter"); + await filterTrigger.focus(); + await filterTrigger.click(); + const drawer = page.locator(".registry-filter-drawer"); + await expect(drawer).toBeVisible(); + expect(await drawer.evaluate(panel => panel.contains(document.activeElement))).toBe(true); + await page.keyboard.press("Escape"); + await expect(drawer).toBeHidden(); + await expect(filterTrigger).toBeFocused(); + + const axe = await new AxeBuilder({ page }).analyze(); + expect(axe.violations, JSON.stringify(axe.violations, null, 2)).toEqual([]); + expect(browserProblems).toEqual([]); +}); diff --git a/frontend/e2e/public-readonly.spec.ts b/frontend/e2e/public-readonly.spec.ts new file mode 100644 index 0000000..0cff6b6 --- /dev/null +++ b/frontend/e2e/public-readonly.spec.ts @@ -0,0 +1,28 @@ +import { expect, test } from "@playwright/test"; + +test("anonymous demo exposes the reviewed read-only UI routes", async ({ context, page }) => { + const overviewResponsePromise = page.waitForResponse(response => { + const url = new URL(response.url()); + return url.pathname === "/api/charities/grants/overview" + && response.request().method() === "GET"; + }); + + await page.goto("/"); + const overviewResponse = await overviewResponsePromise; + expect(overviewResponse.status()).toBe(200); + expect(overviewResponse.headers()["set-cookie"]).toBeUndefined(); + await expect(page.locator(".world-map-card")).toBeVisible(); + + const adminResponse = await page.request.get("/api/admin/pipeline/status"); + expect(adminResponse.status()).toBe(401); + const directoryResponse = await page.request.get("/api/charities"); + expect(directoryResponse.status()).toBe(200); + expect(directoryResponse.headers()["set-cookie"]).toBeUndefined(); + const mutationResponse = await page.request.post("/api/admin/pipeline/trigger", { + data: { source: "quick_consolidate" }, + headers: { "Idempotency-Key": "public-demo-browser-test" }, + }); + expect(mutationResponse.status()).toBe(401); + + expect(await context.cookies()).toEqual([]); +}); diff --git a/frontend/package-lock.json b/frontend/package-lock.json index f31b376..c256d3d 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -17,6 +17,8 @@ "recharts": "^3.9.2" }, "devDependencies": { + "@axe-core/playwright": "4.12.1", + "@playwright/test": "1.62.0", "@types/leaflet": "^1.9.21", "@types/node": "^24.13.2", "@types/react": "^19.2.17", @@ -27,6 +29,19 @@ "vite": "^8.1.1" } }, + "node_modules/@axe-core/playwright": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/@axe-core/playwright/-/playwright-4.12.1.tgz", + "integrity": "sha512-rMd7xriptqKpP+w5265i4Hdkv2X5kbu6uiBi/B2I7uf3hieRBM3qDCfaKPtxfiYb2mKXfF+yLODJwIx+Jv1GDw==", + "dev": true, + "license": "MPL-2.0", + "dependencies": { + "axe-core": "~4.12.1" + }, + "peerDependencies": { + "playwright-core": ">= 1.0.0" + } + }, "node_modules/@emnapi/core": { "version": "1.11.1", "resolved": "https://registry.npmjs.org/@emnapi/core/-/core-1.11.1.tgz", @@ -413,6 +428,22 @@ "node": "^20.19.0 || >=22.12.0" } }, + "node_modules/@playwright/test": { + "version": "1.62.0", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.0.tgz", + "integrity": "sha512-9zOJ6ZQRAena31MpOH9VSzIz8Ou3YJ/wtY/eQm5T2uhfhG7/U3COrMS8xOtUrZrp9OgdmzEnIYODye3nY1VqzA==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@react-leaflet/core": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/@react-leaflet/core/-/core-3.0.0.tgz", @@ -885,6 +916,16 @@ } } }, + "node_modules/axe-core": { + "version": "4.12.1", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.12.1.tgz", + "integrity": "sha512-s7iGf5GaVMxEG0ENN9x+xTr7GFZCb1ZP/1uATUpCEK2X78nDB3RwbtFCo9pGAf9ru+VwoQ464DkaLEeRM08wJA==", + "dev": true, + "license": "MPL-2.0", + "engines": { + "node": ">=4" + } + }, "node_modules/clsx": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz", @@ -1470,6 +1511,53 @@ "url": "https://github.com/sponsors/jonschlinkert" } }, + "node_modules/playwright": { + "version": "1.62.0", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.0.tgz", + "integrity": "sha512-Z14dG305dgaLu6foB1TXQagFiW8JfSUIUaUuPaKQ6NtBPKF1P/qXcqfh6c6K/icPqdy37JmjbiBXf6JNg6Sylw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.0" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.0", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.0.tgz", + "integrity": "sha512-nsNRyq0r2zsG8AcRHWknc9QRA5XCueC7gWMrs+Gx2tlZn9hcl8zudfh00lhJPY1DE7NmZ6bDsT9g2yey8mXljA==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/postcss": { "version": "8.5.20", "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.20.tgz", diff --git a/frontend/package.json b/frontend/package.json index be465ad..6dfe9fc 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,9 +5,12 @@ "type": "module", "scripts": { "dev": "vite", - "build": "tsc -b && vite build", + "build": "tsc -b && vite build && npm run check:bundle", + "check:bundle": "node scripts/check-bundle-budget.mjs", "lint": "oxlint", "test": "tsc -p tsconfig.tests.json && node --test .test-dist/tests/*.test.js", + "test:e2e": "npm run build && env -u NO_COLOR playwright test", + "test:runtime": "npm run build && node scripts/check-runtime-layout.mjs", "preview": "vite preview" }, "dependencies": { @@ -20,6 +23,8 @@ "recharts": "^3.9.2" }, "devDependencies": { + "@axe-core/playwright": "4.12.1", + "@playwright/test": "1.62.0", "@types/leaflet": "^1.9.21", "@types/node": "^24.13.2", "@types/react": "^19.2.17", diff --git a/frontend/playwright.config.ts b/frontend/playwright.config.ts new file mode 100644 index 0000000..7379b56 --- /dev/null +++ b/frontend/playwright.config.ts @@ -0,0 +1,51 @@ +import { existsSync } from "node:fs"; +import { defineConfig } from "@playwright/test"; + +const defaultChrome = process.platform === "darwin" + ? "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome" + : "/usr/bin/google-chrome"; +const chromeExecutable = process.env.PHASE7_CHROME_PATH || defaultChrome; +const externalBaseURL = process.env.PHASE7_BASE_URL; + +if (!existsSync(chromeExecutable)) { + throw new Error(`Local Chrome executable not found at ${chromeExecutable}. Set PHASE7_CHROME_PATH to an installed Chrome/Chromium binary.`); +} + +const viewports = [320, 390, 768, 1024, 1440, 1920]; + +export default defineConfig({ + testDir: "./e2e", + fullyParallel: false, + workers: 1, + timeout: 30_000, + expect: { timeout: 5_000 }, + reporter: "line", + use: { + baseURL: externalBaseURL || "http://127.0.0.1:4174", + browserName: "chromium", + headless: true, + launchOptions: { + executablePath: chromeExecutable, + args: [ + "--disable-background-networking", + "--disable-component-update", + "--disable-default-apps", + "--disable-extensions", + "--disable-sync", + "--metrics-recording-only", + "--no-first-run", + "--no-default-browser-check", + ], + }, + }, + projects: viewports.map(width => ({ + name: `chrome-${width}`, + use: { viewport: { width, height: 1000 } }, + })), + webServer: externalBaseURL ? undefined : { + command: "npm run preview -- --host 127.0.0.1 --port 4174 --strictPort", + url: "http://127.0.0.1:4174", + reuseExistingServer: false, + timeout: 15_000, + }, +}); diff --git a/frontend/scripts/check-bundle-budget.mjs b/frontend/scripts/check-bundle-budget.mjs new file mode 100644 index 0000000..17642fc --- /dev/null +++ b/frontend/scripts/check-bundle-budget.mjs @@ -0,0 +1,50 @@ +import { gzipSync } from "node:zlib"; +import { readFileSync, readdirSync } from "node:fs"; +import { basename, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const DIST = new URL("../dist/", import.meta.url); +const ASSETS = fileURLToPath(new URL("../dist/assets/", import.meta.url)); +const KIB = 1024; +const BUDGETS = { + initialJavaScriptGzip: 120 * KIB, + initialCssGzip: 25 * KIB, + deferredJavaScriptChunkGzip: 425 * KIB, +}; + +const gzipSize = path => gzipSync(readFileSync(path)).byteLength; +const format = bytes => `${(bytes / KIB).toFixed(2)} KiB gzip`; +const indexHtml = readFileSync(new URL("index.html", DIST), "utf8"); +const initialJavaScript = Array.from(indexHtml.matchAll(/(?:src|href)="\/assets\/([^"]+\.js)"/g), match => match[1]); +const initialCss = Array.from(indexHtml.matchAll(/href="\/assets\/([^"]+\.css)"/g), match => match[1]); +const javascriptChunks = readdirSync(ASSETS).filter(file => file.endsWith(".js")); + +const initialJavaScriptBytes = initialJavaScript.reduce((total, file) => total + gzipSize(join(ASSETS, file)), 0); +const initialCssBytes = initialCss.reduce((total, file) => total + gzipSize(join(ASSETS, file)), 0); +const oversizedChunks = javascriptChunks + .map(file => ({ file, bytes: gzipSize(join(ASSETS, file)) })) + .filter(chunk => chunk.bytes > BUDGETS.deferredJavaScriptChunkGzip) + .sort((left, right) => right.bytes - left.bytes); + +const failures = []; +if (!initialJavaScript.length) failures.push("No initial JavaScript entry was found in dist/index.html."); +if (initialJavaScriptBytes > BUDGETS.initialJavaScriptGzip) { + failures.push(`Initial JavaScript is ${format(initialJavaScriptBytes)}; budget is ${format(BUDGETS.initialJavaScriptGzip)}.`); +} +if (initialCssBytes > BUDGETS.initialCssGzip) { + failures.push(`Initial CSS is ${format(initialCssBytes)}; budget is ${format(BUDGETS.initialCssGzip)}.`); +} +for (const chunk of oversizedChunks) { + failures.push(`${basename(chunk.file)} is ${format(chunk.bytes)}; deferred JavaScript chunk budget is ${format(BUDGETS.deferredJavaScriptChunkGzip)}.`); +} + +console.log(`Initial JavaScript: ${format(initialJavaScriptBytes)} / ${format(BUDGETS.initialJavaScriptGzip)}`); +console.log(`Initial CSS: ${format(initialCssBytes)} / ${format(BUDGETS.initialCssGzip)}`); +console.log(`Largest deferred JavaScript chunk: ${format(Math.max(0, ...javascriptChunks.map(file => gzipSize(join(ASSETS, file)))))} / ${format(BUDGETS.deferredJavaScriptChunkGzip)}`); + +if (failures.length) { + for (const failure of failures) console.error(`Bundle budget failure: ${failure}`); + process.exitCode = 1; +} else { + console.log("Bundle budget passed."); +} diff --git a/frontend/scripts/check-runtime-layout.mjs b/frontend/scripts/check-runtime-layout.mjs new file mode 100644 index 0000000..04db5cc --- /dev/null +++ b/frontend/scripts/check-runtime-layout.mjs @@ -0,0 +1,382 @@ +import { spawn } from "node:child_process"; +import { existsSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const WIDTHS = process.env.PHASE7_WIDTHS + ? process.env.PHASE7_WIDTHS.split(",").map(Number) + : [320, 390, 768, 1024, 1440, 1920]; +const HEIGHT = 1000; +const BASE_URL = "http://127.0.0.1:4173"; +const chromeCandidates = [ + process.env.CHROME_PATH, + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + "/usr/bin/google-chrome", + "/usr/bin/chromium", +].filter(Boolean); +const chromePath = chromeCandidates.find(candidate => existsSync(candidate)); + +if (!chromePath) throw new Error("A local Chrome or Chromium binary is required for the runtime layout gate."); + +const sleep = milliseconds => new Promise(resolve => setTimeout(resolve, milliseconds)); +const children = []; +const chromeProfile = mkdtempSync(join(tmpdir(), "fip-phase7-chrome-")); + +function stopChildren() { + for (const child of children) { + if (!child.killed) child.kill("SIGTERM"); + } + rmSync(chromeProfile, { recursive: true, force: true }); +} + +process.once("exit", stopChildren); +process.once("SIGINT", () => process.exit(130)); +process.once("SIGTERM", () => process.exit(143)); + +async function waitForPreview() { + for (let attempt = 0; attempt < 80; attempt += 1) { + try { + const response = await fetch(BASE_URL); + if (response.ok) return; + } catch { + // The local preview process is still starting. + } + await sleep(50); + } + throw new Error("The local Vite preview did not become ready."); +} + +class CdpClient { + constructor(socket) { + this.socket = socket; + this.nextId = 1; + this.pending = new Map(); + this.events = []; + socket.addEventListener("message", event => { + const message = JSON.parse(String(event.data)); + if (message.id) { + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.error) pending.reject(new Error(message.error.message)); + else pending.resolve(message.result); + return; + } + this.events.push(message); + }); + } + + send(method, params = {}) { + const id = this.nextId; + this.nextId += 1; + return new Promise((resolve, reject) => { + this.pending.set(id, { resolve, reject }); + this.socket.send(JSON.stringify({ id, method, params })); + }); + } +} + +async function evaluate(client, expression) { + const result = await client.send("Runtime.evaluate", { + expression, + awaitPromise: true, + returnByValue: true, + }); + if (result.exceptionDetails) throw new Error(result.exceptionDetails.text || "Browser evaluation failed."); + return result.result.value; +} + +async function waitFor(client, expression, message) { + for (let attempt = 0; attempt < 100; attempt += 1) { + if (await evaluate(client, expression)) return; + await sleep(50); + } + throw new Error(message); +} + +const map = { + status: "available", + geographic_dimension: "beneficiary", + items: [{ + region_or_country_code: "GB", + region_or_country_name: "United Kingdom", + grant_count: 1250, + total_amount: 123456789, + currency: "EUR", + distinct_funders: 84, + distinct_recipients: 720, + top_programme_areas: [], + top_funders: [], + top_recipients: [], + original_geographies: ["United Kingdom"], + funding_grant_count: 1200, + excluded_multi_country_grant_count: 0, + excluded_invalid_amount_grant_count: 0, + }], + known_geography_count: 1250, + unknown_geography_count: 50, + coverage_percentage: 96.2, + currencies: ["EUR"], + selected_currency: "EUR", + funding_status: "available", + funding_mode_available: true, + grant_country_association_count: 1250, + multi_country_grant_count: 0, + funding_excluded_multi_country_count: 0, + funding_excluded_multi_country_amount: 0, + funding_excluded_currency_count: 0, + funding_excluded_invalid_amount_count: 0, + connections: [], + connection_grant_count: 0, + connection_excluded_no_headquarters_count: 0, + connection_same_country_count: 0, + minimum_coverage_threshold: 0, + metadata: { data_mode: "runtime_test", source: ["360Giving"], record_count: 1300, limitations: [] }, +}; + +const overview = { + status: "available", + kpis: { + awarded_funding: 123456789, + currency: "EUR", + grants_monitored: 1300, + country_coverage_percentage: 96.2, + mapped_grant_count: 1250, + unmapped_grant_count: 50, + programme_coverage_percentage: 87.5, + classified_grant_count: 1138, + qualifying_programme_grant_count: 1300, + }, + map, + trends: { + status: "available", + currency: "EUR", + available_currencies: ["EUR"], + granularity: "monthly", + period: null, + items: [], + excluded: {}, + last_refreshed_at: null, + }, + themes: { + status: "available", + currency: "EUR", + items: [], + classification_coverage: { classified_percentage: 87.5, classified_grant_count: 1138, unclassified_grant_count: 162 }, + }, + available_date_range: { from: "2024-01-01", to: "2026-06-30" }, +}; + +const browserMock = `(() => { + const responses = { + health: ${JSON.stringify({ status: "ok" })}, + stats: ${JSON.stringify({ total_charities: 42, active_charities: 40, removed_charities: 2, average_income: 1000000, average_expenditure: 800000, source: ["360Giving", "Charity Commission for England and Wales", "Philea"] })}, + overview: ${JSON.stringify(overview)}, + geographies: ${JSON.stringify(["United Kingdom", "Germany", "France"])}, + connections: ${JSON.stringify({ status: "available", connections: [], connection_grant_count: 0 })}, + registry: ${JSON.stringify({ results: [], next_cursor: null, has_more: false, page_size: 50, search_strategy: "indexed_directory" })}, + }; + window.__phase7Requests = {}; + const nativeFetch = window.fetch.bind(window); + window.fetch = (input, init) => { + const url = new URL(typeof input === "string" ? input : input.url, window.location.href); + const key = url.pathname + url.search; + window.__phase7Requests[key] = (window.__phase7Requests[key] || 0) + 1; + let body; + if (url.pathname === "/health") body = responses.health; + else if (url.pathname === "/api/charities/stats") body = responses.stats; + else if (url.pathname === "/api/charities/grants/overview") body = responses.overview; + else if (url.pathname === "/api/charities/grants/beneficiary-geographies") body = responses.geographies; + else if (url.pathname === "/api/charities/grants/map/connections") body = responses.connections; + else if (url.pathname === "/api/charities/directory/organizations") body = responses.registry; + else return nativeFetch(input, init); + return Promise.resolve(new Response(JSON.stringify(body), { status: 200, headers: { "Content-Type": "application/json" } })); + }; +})();`; + +let socket; +try { + const preview = spawn(process.execPath, ["node_modules/vite/bin/vite.js", "preview", "--host", "127.0.0.1", "--port", "4173", "--strictPort"], { + stdio: ["ignore", "pipe", "pipe"], + }); + children.push(preview); + await waitForPreview(); + + const chrome = spawn(chromePath, [ + "--headless=new", + "--disable-background-networking", + "--disable-component-update", + "--disable-default-apps", + "--disable-extensions", + "--disable-sync", + "--metrics-recording-only", + "--no-first-run", + "--no-default-browser-check", + "--remote-debugging-port=0", + `--user-data-dir=${chromeProfile}`, + "about:blank", + ], { stdio: ["ignore", "ignore", "pipe"] }); + children.push(chrome); + + const browserWebSocket = await new Promise((resolve, reject) => { + let output = ""; + const timeout = setTimeout(() => reject(new Error("Chrome DevTools did not become ready.")), 10_000); + chrome.stderr.on("data", chunk => { + output += chunk.toString(); + const match = output.match(/DevTools listening on (ws:\/\/[^\s]+)/); + if (!match) return; + clearTimeout(timeout); + resolve(match[1]); + }); + chrome.once("exit", code => reject(new Error(`Chrome exited before testing (code ${code}).`))); + }); + + const debugPort = new URL(browserWebSocket).port; + const targets = await (await fetch(`http://127.0.0.1:${debugPort}/json/list`)).json(); + const pageTarget = targets.find(target => target.type === "page"); + if (!pageTarget) throw new Error("Chrome did not expose a page target."); + socket = new WebSocket(pageTarget.webSocketDebuggerUrl); + await new Promise((resolve, reject) => { + socket.addEventListener("open", resolve, { once: true }); + socket.addEventListener("error", reject, { once: true }); + }); + const client = new CdpClient(socket); + await client.send("Page.enable"); + await client.send("Runtime.enable"); + await client.send("Log.enable"); + await client.send("Page.addScriptToEvaluateOnNewDocument", { source: browserMock }); + + const failures = []; + for (const width of WIDTHS) { + const failureCountBeforeWidth = failures.length; + client.events.length = 0; + await client.send("Emulation.setDeviceMetricsOverride", { + width, + height: HEIGHT, + deviceScaleFactor: 1, + mobile: width < 768, + }); + await client.send("Page.navigate", { url: BASE_URL }); + await waitFor(client, "document.readyState === 'complete' && Boolean(document.querySelector('.world-map-card'))", `Overview did not render at ${width}px.`); + await sleep(150); + + const layout = await evaluate(client, `(() => { + const viewportWidth = window.innerWidth; + const rectInside = element => { + const rect = element.getBoundingClientRect(); + return rect.left >= -1 && rect.right <= viewportWidth + 1; + }; + const visibleControls = [...document.querySelectorAll('button, input, select, summary, a')] + .filter(element => { + const rect = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return rect.width > 0 && rect.height > 0 && rect.right > 0 && rect.left < viewportWidth && rect.bottom > 0 && rect.top < window.innerHeight && style.visibility !== 'hidden'; + }); + const accessibleName = element => { + const labelledBy = (element.getAttribute('aria-labelledby') || '').split(' ').filter(Boolean) + .map(id => document.getElementById(id)?.textContent || '').join(' ').trim(); + return element.getAttribute('aria-label') + || labelledBy + || [...(element.labels || [])].map(label => label.textContent || '').join(' ').trim() + || element.textContent.trim() + || element.getAttribute('title') + || ''; + }; + const map = document.querySelector('.world-map-card'); + const analytics = document.querySelector('.overview-analytics-grid'); + return { + scrollWidth: Math.max(document.documentElement.scrollWidth, document.body.scrollWidth), + viewportWidth, + clippedControls: visibleControls.filter(element => !rectInside(element)).map(element => element.getAttribute('aria-label') || element.textContent.trim().slice(0, 50)), + unnamedControls: visibleControls.filter(element => !accessibleName(element)).map(element => element.outerHTML.slice(0, 100)), + croppedKpis: [...document.querySelectorAll('.overview-kpi')].filter(element => element.scrollWidth > element.clientWidth + 1 || element.scrollHeight > element.clientHeight + 1).length, + mapBeforeAnalytics: Boolean(map && analytics && (map.compareDocumentPosition(analytics) & Node.DOCUMENT_POSITION_FOLLOWING)), + mapControls: [...document.querySelectorAll('.map-mode-control button')].every(rectInside), + overviewRequests: Object.entries(window.__phase7Requests).filter(([key]) => key.startsWith('/api/charities/grants/overview?')).reduce((total, [, count]) => total + count, 0), + }; + })()`); + + if (layout.scrollWidth > layout.viewportWidth + 1) failures.push(`${width}px: page scroll width ${layout.scrollWidth}px exceeds viewport ${layout.viewportWidth}px.`); + if (layout.clippedControls.length) failures.push(`${width}px: clipped visible controls: ${layout.clippedControls.join(", ")}.`); + if (layout.unnamedControls.length) failures.push(`${width}px: controls without accessible names: ${layout.unnamedControls.join(", ")}.`); + if (layout.croppedKpis) failures.push(`${width}px: ${layout.croppedKpis} KPI card(s) crop their content.`); + if (!layout.mapBeforeAnalytics) failures.push(`${width}px: map is not before the analytics panels.`); + if (!layout.mapControls) failures.push(`${width}px: map controls leave the viewport.`); + if (layout.overviewRequests !== 1) failures.push(`${width}px: expected one initial overview request, observed ${layout.overviewRequests}.`); + + if (width === 320) { + await evaluate(client, "[...document.querySelectorAll('.map-mode-control button')].find(button => button.textContent.includes('Connections'))?.click()"); + await waitFor(client, "Object.entries(window.__phase7Requests).some(([key, count]) => key.startsWith('/api/charities/grants/map/connections?') && count === 1)", "The lazy map connection request did not run exactly once."); + } + + await evaluate(client, "(() => { const trigger = document.querySelector('.app-header-filter'); trigger?.focus(); trigger?.click(); })()"); + await waitFor(client, "Boolean(document.querySelector('.overview-filter-drawer'))", `Filter drawer did not open at ${width}px.`); + const drawer = await evaluate(client, `(() => { + const panel = document.querySelector('.overview-filter-drawer'); + const rect = panel.getBoundingClientRect(); + const controls = [...panel.querySelectorAll('button, input, select')]; + return { + insideViewport: rect.left >= -1 && rect.right <= window.innerWidth + 1 && rect.top >= -1 && rect.bottom <= window.innerHeight + 1, + controlsInside: controls.every(control => { const bounds = control.getBoundingClientRect(); return bounds.left >= -1 && bounds.right <= window.innerWidth + 1; }), + focusContained: panel.contains(document.activeElement), + }; + })()`); + if (!drawer.insideViewport || !drawer.controlsInside) failures.push(`${width}px: filter drawer or a drawer control leaves the viewport.`); + if (!drawer.focusContained) failures.push(`${width}px: focus did not move into the filter drawer.`); + await evaluate(client, "(() => { const controls = [...document.querySelectorAll('.overview-filter-drawer button:not([disabled]), .overview-filter-drawer input:not([disabled]), .overview-filter-drawer select:not([disabled])')]; controls.at(-1)?.focus(); })()"); + await client.send("Input.dispatchKeyEvent", { type: "keyDown", key: "Tab", code: "Tab" }); + await client.send("Input.dispatchKeyEvent", { type: "keyUp", key: "Tab", code: "Tab" }); + const focusWrapped = await evaluate(client, "(() => { const controls = [...document.querySelectorAll('.overview-filter-drawer button:not([disabled]), .overview-filter-drawer input:not([disabled]), .overview-filter-drawer select:not([disabled])')]; return document.activeElement === controls[0]; })()"); + if (!focusWrapped) failures.push(`${width}px: Tab did not wrap within the modal filter drawer.`); + await client.send("Input.dispatchKeyEvent", { type: "keyDown", key: "Escape", code: "Escape" }); + await client.send("Input.dispatchKeyEvent", { type: "keyUp", key: "Escape", code: "Escape" }); + await waitFor(client, "!document.querySelector('.overview-filter-drawer')", `Filter drawer did not close with Escape at ${width}px.`); + try { + await waitFor(client, "document.activeElement?.classList.contains('app-header-filter')", `Focus did not return to the filter trigger at ${width}px.`); + } catch { + const activeElement = await evaluate(client, "({ tag: document.activeElement?.tagName, className: document.activeElement?.className || '', label: document.activeElement?.getAttribute('aria-label') || document.activeElement?.textContent?.trim().slice(0, 40) || '' })"); + failures.push(`${width}px: focus did not return to the filter trigger (active: ${activeElement.tag}.${activeElement.className} “${activeElement.label}”).`); + } + + if (width === 320 || width === 1024) { + await evaluate(client, "document.querySelector('.nav-item[title=\"Donor Directory\"]')?.click()"); + await waitFor(client, "Boolean(document.querySelector('.donor-directory-page'))", `Donor Directory did not render at ${width}px.`); + const donorWidth = await evaluate(client, "Math.max(document.documentElement.scrollWidth, document.body.scrollWidth)"); + if (donorWidth > width + 1) failures.push(`${width}px: Donor Directory scroll width ${donorWidth}px exceeds the viewport.`); + await evaluate(client, "[...document.querySelectorAll('.donor-directory-secondary-links button')].find(button => button.textContent.includes('Advanced Charity Commission Search'))?.click()"); + await waitFor(client, "Boolean(document.querySelector('.registry-directory'))", `Registry Directory did not render at ${width}px.`); + await waitFor(client, "Boolean(document.querySelector('.directory-empty-state'))", `Registry empty state did not render at ${width}px.`); + const registryWidth = await evaluate(client, "Math.max(document.documentElement.scrollWidth, document.body.scrollWidth)"); + if (registryWidth > width + 1) failures.push(`${width}px: Registry Directory scroll width ${registryWidth}px exceeds the viewport.`); + await evaluate(client, "(() => { const trigger = document.querySelector('.app-header-filter'); trigger?.focus(); trigger?.click(); })()"); + await waitFor(client, "Boolean(document.querySelector('.registry-filter-drawer'))", `Registry filter drawer did not open at ${width}px.`); + const registryDrawerInside = await evaluate(client, "(() => { const bounds = document.querySelector('.registry-filter-drawer').getBoundingClientRect(); return bounds.left >= -1 && bounds.right <= window.innerWidth + 1; })()"); + if (!registryDrawerInside) failures.push(`${width}px: Registry filter drawer leaves the viewport.`); + await client.send("Input.dispatchKeyEvent", { type: "keyDown", key: "Escape", code: "Escape" }); + await client.send("Input.dispatchKeyEvent", { type: "keyUp", key: "Escape", code: "Escape" }); + await waitFor(client, "!document.querySelector('.registry-filter-drawer')", `Registry filter drawer did not close with Escape at ${width}px.`); + await waitFor(client, "document.activeElement?.classList.contains('app-header-filter')", `Registry filter focus did not return at ${width}px.`); + } + + const browserProblems = client.events.filter(event => ( + (event.method === "Runtime.exceptionThrown") + || (event.method === "Runtime.consoleAPICalled" && ["error", "warning"].includes(event.params.type)) + || (event.method === "Log.entryAdded" && ["error", "warning"].includes(event.params.entry.level)) + )); + if (browserProblems.length) failures.push(`${width}px: ${browserProblems.length} browser console warning/error event(s).`); + if (failures.length === failureCountBeforeWidth) { + console.log(`${width}px: no page overflow; map, filters, controls, focus, requests and console passed.`); + } + } + + if (failures.length) { + for (const failure of failures) console.error(`Runtime layout failure: ${failure}`); + process.exitCode = 1; + } else { + console.log("Runtime layout gate passed at 320, 390, 768, 1024, 1440 and 1920 pixels."); + } +} finally { + socket?.close(); + stopChildren(); +} diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 983fba4..0dd7ab8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -1,4 +1,12 @@ -import { lazy, Suspense, useState, useEffect, useMemo, useRef } from "react"; +import { + lazy, + Suspense, + useEffect, + useEffectEvent, + useMemo, + useRef, + useState, +} from "react"; import { Building2, ArrowLeft, @@ -18,20 +26,11 @@ import { Star, X, } from "lucide-react"; -import { - BarChart, - Bar, - XAxis, - YAxis, - CartesianGrid, - Tooltip, - ResponsiveContainer, - AreaChart, - Area, -} from "recharts"; -import GrantWorldMap from "./components/GrantWorldMap"; -import OverviewDashboard, { type FavoriteGrantExplorerPayload, type OverviewFilters } from "./components/OverviewDashboard"; import AppHeader from "./components/AppHeader"; +import type { + FavoriteGrantExplorerPayload, + OverviewFilters, +} from "./components/OverviewDashboard"; import type { FavoriteDonorPayload, FavoriteDonorRequestPayload, @@ -42,6 +41,7 @@ import { grantScopeFromUrl, type GrantScope, } from "./lib/grantScope"; +import { mutationHeaders } from "./lib/http"; import amplifyLogo from "./assets/amplify-logo.svg"; import type { GrantMapFilters, @@ -49,14 +49,17 @@ import type { SourceFunderCountrySelection, } from "./components/GrantWorldMap"; +const GrantWorldMap = lazy(() => import("./components/GrantWorldMap")); +const OverviewDashboard = lazy(() => import("./components/OverviewDashboard")); +const GrantAwardsChart = lazy(() => import("./components/DataCharts").then(module => ({ default: module.GrantAwardsChart }))); +const ProgrammeAllocationChart = lazy(() => import("./components/DataCharts").then(module => ({ default: module.ProgrammeAllocationChart }))); +const FinancialHistoryChart = lazy(() => import("./components/DataCharts").then(module => ({ default: module.FinancialHistoryChart }))); const RegistryDirectory = lazy(() => import("./components/RegistryDirectory")); const DonorDirectoryPage = lazy(() => import("./components/DonorDirectoryPage")); // Configuration for API requests const API_BASE = import.meta.env.VITE_API_BASE_URL - || `${window.location.protocol}//${window.location.hostname}:8000`; -const DEMO_USERNAME = import.meta.env.VITE_BFF_USERNAME || "admin"; -const DEMO_PASSWORD = import.meta.env.VITE_BFF_PASSWORD || "password"; + || ""; const SHOW_LEGACY_OVERVIEW = import.meta.env.VITE_LEGACY_OVERVIEW === "true"; const DEFAULT_DATA_SOURCES = ["360Giving", "Charity Commission for England and Wales", "Philea"]; // Interface definitions @@ -791,12 +794,14 @@ export default function App() { const [mapData, setMapData] = useState(EMPTY_MAP); const [mapLoading, setMapLoading] = useState(false); const [mapError, setMapError] = useState(null); + const mapRequestRef = useRef(null); const [mapFilters] = useState(EMPTY_GRANT_MAP_FILTERS); const [grantTrends, setGrantTrends] = useState(null); const [grantThemes, setGrantThemes] = useState(null); const [grantAnalyticsLoading, setGrantAnalyticsLoading] = useState(false); const [grantAnalyticsError, setGrantAnalyticsError] = useState(null); const [grantAnalyticsCurrency, setGrantAnalyticsCurrency] = useState(""); + const grantAnalyticsRequestRef = useRef(null); const [selectedCharity, setSelectedCharity] = useState(null); const [selectedCharityDetail, setSelectedCharityDetail] = useState(null); const [selectedSourceFunderProfileKey, setSelectedSourceFunderProfileKey] = useState(null); @@ -970,6 +975,9 @@ export default function App() { && newsSummary && newsSummary.generated_at === savedNewsRun.generated_at, ); + const visibleNewsSources = useMemo(() => Array.from(new Map( + (newsSummary?.sources || []).map(source => [`${source.link}\u001f${source.title}`, source]), + ).values()), [newsSummary?.sources]); const financialHistoryData = useMemo(() => { const history = selectedCharityDetail?.financial_history; if (!Array.isArray(history)) return []; @@ -1024,21 +1032,57 @@ export default function App() { const [pipelineIds, setPipelineIds] = useState(""); const logsEndRef = useRef(null); + const runHealthCheck = useEffectEvent((signal: AbortSignal) => { + void checkBffHealth(signal); + }); + const refreshStatistics = useEffectEvent((signal?: AbortSignal) => { + void fetchStats(undefined, signal); + }); + const refreshDirectory = useEffectEvent(() => { + void fetchCharities(); + }); + const loadProfileSections = useEffectEvent(( + id: number, + sourceFunderKey: string | null, + requestId: number, + signal: AbortSignal, + ) => { + void fetchCharityDetail(id, requestId, signal); + void fetchCharityGrants(id, requestId, signal); + void fetchSankeyData(id, requestId, signal); + void fetchScoreData(id, requestId, signal); + if (sourceFunderKey) { + void hydrateSourceFunderProfile(sourceFunderKey, id, requestId, signal); + } + }); + const refreshPipelineStatus = useEffectEvent((signal: AbortSignal) => { + void fetchPipelineStatus(signal); + }); + const refreshAfterPipelineSuccess = useEffectEvent((signal: AbortSignal) => { + void fetchStats(undefined, signal); + void fetchCharities(); + window.dispatchEvent(new Event("overview-refresh")); + }); // Fetch initial configuration on mount useEffect(() => { - checkBffHealth(); + const controller = new AbortController(); + runHealthCheck(controller.signal); + return () => controller.abort(); }, []); useEffect(() => { - fetchStats(); + if (!isBffOnline) return; + const controller = new AbortController(); + refreshStatistics(controller.signal); + return () => controller.abort(); }, [isBffOnline]); useEffect(() => { if (activeTab !== "directory" || directoryMode !== "profiles") return; const delay = searchTerm.trim() || maxAnnualGivingInput || maxAvgGrantSizeInput ? 250 : 0; const debounce = window.setTimeout(() => { - fetchCharities(); + refreshDirectory(); }, delay); return () => { window.clearTimeout(debounce); @@ -1078,7 +1122,7 @@ export default function App() { }) .catch(error => { if ((error as Error).name !== "AbortError") { - console.warn("Beneficiary geography options unavailable", error); + // Keep the bundled fallback options; this panel is independently usable. } }); return () => controller.abort(); @@ -1114,7 +1158,6 @@ export default function App() { setOrganizationSuggestions(await response.json()); } catch (error) { if ((error as Error).name !== "AbortError") { - console.warn("Organization suggestions unavailable", error); setOrganizationSuggestions([]); } } finally { @@ -1175,18 +1218,12 @@ export default function App() { : profileSectionState("idle"), }); - void fetchCharityDetail(selectedCharityId, requestId, controller.signal); - void fetchCharityGrants(selectedCharityId, requestId, controller.signal); - void fetchSankeyData(selectedCharityId, requestId, controller.signal); - void fetchScoreData(selectedCharityId, requestId, controller.signal); - if (selectedSourceFunderProfileKey) { - void hydrateSourceFunderProfile( - selectedSourceFunderProfileKey, - selectedCharityId, - requestId, - controller.signal, - ); - } + loadProfileSections( + selectedCharityId, + selectedSourceFunderProfileKey, + requestId, + controller.signal, + ); return () => { controller.abort(); @@ -1198,22 +1235,26 @@ export default function App() { }, [selectedCharityId, selectedSourceFunderProfileKey, isBffOnline]); useEffect(() => { - if (activeTab === "admin") { - fetchPipelineStatus(); - const interval = setInterval(fetchPipelineStatus, 3000); - return () => clearInterval(interval); - } + if (activeTab !== "admin") return; + let controller: AbortController | null = null; + const refresh = () => { + controller?.abort(); + controller = new AbortController(); + refreshPipelineStatus(controller.signal); + }; + refresh(); + const interval = window.setInterval(refresh, 3000); + return () => { + window.clearInterval(interval); + controller?.abort(); + }; }, [activeTab]); useEffect(() => { - if (pipelineStatus.status === "success") { - fetchStats(); - fetchCharities(); - // The active Overview owns one aggregated request for its complete grant - // population. Notify it after ingestion instead of reviving legacy map - // and chart calls in the background. - window.dispatchEvent(new Event("overview-refresh")); - } + if (pipelineStatus.status !== "success") return; + const controller = new AbortController(); + refreshAfterPipelineSuccess(controller.signal); + return () => controller.abort(); }, [pipelineStatus.status]); useEffect(() => { @@ -1271,69 +1312,49 @@ export default function App() { return () => window.removeEventListener("active-source-funder-change", receiveActiveSourceFunder); }, []); - const autoLogin = async () => { + const checkBffHealth = async (signal?: AbortSignal) => { + setInitialLoading(true); try { - const resp = await fetch(`${API_BASE}/api/auth/login`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ username: DEMO_USERNAME, password: DEMO_PASSWORD }), - credentials: "include" - }); + const resp = await fetch(`${API_BASE}/health`, { signal }); + if (signal?.aborted) return; if (resp.ok) { setAuthError(null); setIsBffOnline(true); - console.log("Logged in to BFF successfully."); - return true; - } - setAuthError("The backend is reachable, but automatic demo authentication failed."); - } catch (e) { - console.error("Auto login failed", e); - setAuthError("The backend is reachable, but automatic demo authentication failed."); - } - return false; - }; - - const checkBffHealth = async () => { - setInitialLoading(true); - try { - const resp = await fetch(`${API_BASE}/health`); - if (resp.ok) { - console.log("BFF Backend is online. Authenticating..."); - const loggedIn = await autoLogin(); - if (loggedIn) { - setIsBffOnline(true); - setApiError("health", null); - // The online-state effects load the live dataset once. Calling the - // same fetches here as well used to race the first directory render. - return; - } + setApiError("health", null); + // Authentication is established by the deployment OIDC boundary. + // The browser never contains or posts a shared application password. + return; } setIsBffOnline(false); setApiError("health", "Backend unavailable. Values marked as illustrative are local prototype data."); - console.warn("BFF offline. Falling back to mock dataset."); - } catch { + } catch (error) { + if ((error as Error).name === "AbortError") return; setIsBffOnline(false); setApiError("health", "Backend unavailable. Values marked as illustrative are local prototype data."); - console.warn("BFF offline. Falling back to mock dataset."); } finally { - setInitialLoading(false); + if (!signal?.aborted) setInitialLoading(false); } }; - const fetchStats = async (forceOnline?: boolean) => { + const fetchStats = async (forceOnline?: boolean, signal?: AbortSignal) => { const isOnline = forceOnline !== undefined ? forceOnline : isBffOnline; if (!isOnline) return; try { - const resp = await fetch(`${API_BASE}/api/charities/stats`, { credentials: "include" }); + const resp = await fetch(`${API_BASE}/api/charities/stats`, { + credentials: "include", + signal, + }); + if (signal?.aborted) return; if (resp.ok) { const data = await resp.json(); + if (signal?.aborted) return; setStats(data); setApiError("statistics", null); } else { setApiError("statistics", `Statistics request failed (${resp.status}).`); } } catch (e) { - console.error("Failed to fetch stats", e); + if ((e as Error).name === "AbortError") return; setApiError("statistics", "Statistics are temporarily unavailable."); } }; @@ -1440,7 +1461,6 @@ export default function App() { } } catch (e) { if ((e as Error).name === "AbortError") return; - console.error("Failed to fetch charities", e); if (requestSequence === directoryRequestSequenceRef.current) { setApiError("directory", "The organization directory is temporarily unavailable."); } @@ -1518,7 +1538,6 @@ export default function App() { } } catch (e) { if ((e as Error).name === "AbortError" || !isCurrentProfileRequest(requestId)) return; - console.error("Failed to fetch charity details", e); const message = "Organization details are temporarily unavailable."; setSelectedCharityDetail(null); setProfileSection(requestId, "detail", "error", message); @@ -1543,7 +1562,12 @@ export default function App() { try { const queued = await fetch( `${API_BASE}/api/charities/grants/funders/${encodeURIComponent(sourceFunderKey)}/profile-cache`, - { method: "POST", credentials: "include", signal }, + { + method: "POST", + credentials: "include", + headers: mutationHeaders("hydrate source-funder profile"), + signal, + }, ); if (!requestIsCurrent()) return; if (!queued.ok && queued.status !== 409) { @@ -1601,7 +1625,6 @@ export default function App() { } } catch (error) { if ((error as Error).name === "AbortError" || !requestIsCurrent()) return; - console.error("Could not hydrate source-funder profile", error); setProfileSection( requestId, "source_record", @@ -1793,7 +1816,6 @@ export default function App() { if (!completed) throw new Error("News research ended before a briefing was returned."); } catch (e: any) { if ((e as Error).name === "AbortError" || !isCurrentNewsRequest()) return; - console.error("Failed to fetch news summary", e); setNewsError(e?.message || "An error occurred while connecting to the news service."); } finally { if (isCurrentNewsRequest()) { @@ -1811,10 +1833,14 @@ export default function App() { ) => { const isOnline = forceOnline !== undefined ? forceOnline : isBffOnline; if (!isOnline) { + mapRequestRef.current?.abort(); setMapData(EMPTY_MAP); setMapLoading(false); return; } + mapRequestRef.current?.abort(); + const controller = new AbortController(); + mapRequestRef.current = controller; setMapLoading(true); setMapError(null); const requestedCurrency = currencyOverride ?? grantAnalyticsCurrency; @@ -1839,8 +1865,9 @@ export default function App() { try { const resp = await fetch( `${API_BASE}/api/charities/grants/map${query ? `?${query}` : ""}`, - { credentials: "include" }, + { credentials: "include", signal: controller.signal }, ); + if (controller.signal.aborted || mapRequestRef.current !== controller) return; if (resp.ok) { const data: GrantMapResponse = await resp.json(); setMapData(data); @@ -1848,11 +1875,14 @@ export default function App() { setMapError(`Map-data request failed (${resp.status}).`); } } catch (e) { - console.error("Failed to fetch map metrics", e); + if ((e as Error).name === "AbortError" || mapRequestRef.current !== controller) return; setMapData(EMPTY_MAP); setMapError("Map data is temporarily unavailable."); } finally { - setMapLoading(false); + if (mapRequestRef.current === controller) { + mapRequestRef.current = null; + setMapLoading(false); + } } }; @@ -2141,11 +2171,15 @@ export default function App() { const fetchGrantAnalytics = async (forceOnline?: boolean, currencyOverride?: string) => { const isOnline = forceOnline !== undefined ? forceOnline : isBffOnline; if (!isOnline) { + grantAnalyticsRequestRef.current?.abort(); setGrantTrends(null); setGrantThemes(null); setGrantAnalyticsError("Grant analytics require the cached SQLite transaction database."); return; } + grantAnalyticsRequestRef.current?.abort(); + const controller = new AbortController(); + grantAnalyticsRequestRef.current = controller; setGrantAnalyticsLoading(true); setGrantAnalyticsError(null); const requestedCurrency = currencyOverride || grantAnalyticsCurrency; @@ -2155,12 +2189,15 @@ export default function App() { try { const [trendsResponse, themesResponse] = await Promise.all([ fetch(`${API_BASE}/api/charities/grants/trends?months=24${currencyQuery}`, { - credentials: "include" + credentials: "include", + signal: controller.signal, }), fetch(`${API_BASE}/api/charities/grants/themes?${currencyQuery.slice(1)}`, { - credentials: "include" + credentials: "include", + signal: controller.signal, }) ]); + if (controller.signal.aborted || grantAnalyticsRequestRef.current !== controller) return; if (!trendsResponse.ok || !themesResponse.ok) { setGrantAnalyticsError( `Grant analytics request failed (${trendsResponse.status}/${themesResponse.status}).` @@ -2174,12 +2211,15 @@ export default function App() { const resolvedCurrency = trends.currency || themes.currency; if (resolvedCurrency) setGrantAnalyticsCurrency(resolvedCurrency); } catch (error) { - console.error("Failed to fetch grant analytics", error); + if ((error as Error).name === "AbortError" || grantAnalyticsRequestRef.current !== controller) return; setGrantAnalyticsError("Grant analytics are temporarily unavailable."); setGrantTrends(null); setGrantThemes(null); } finally { - setGrantAnalyticsLoading(false); + if (grantAnalyticsRequestRef.current === controller) { + grantAnalyticsRequestRef.current = null; + setGrantAnalyticsLoading(false); + } } }; @@ -2210,7 +2250,6 @@ export default function App() { } } catch (e) { if ((e as Error).name === "AbortError" || !isCurrentProfileRequest(requestId)) return; - console.error("Failed to fetch grants", e); const message = "Grant transactions are temporarily unavailable."; setCharityGrants([]); setGrantStatus("request_failed"); @@ -2266,7 +2305,6 @@ export default function App() { } } catch (e) { if ((e as Error).name === "AbortError" || !isCurrentProfileRequest(requestId)) return; - console.error("Failed to fetch sankey metrics", e); const message = "Grant-flow data is temporarily unavailable."; setSankeyData({ status: "request_failed", nodes: [], links: [], currency: null, excludedCount: 0 }); setProfileSection(requestId, "relationships", "error", message); @@ -2282,10 +2320,8 @@ export default function App() { return; } const resp = await fetch(`${API_BASE}/api/charities/${id}/score`, { - method: "POST", - headers: { "Content-Type": "application/json" }, + method: "GET", credentials: "include", - body: JSON.stringify({}), signal, }); if (!isCurrentProfileRequest(requestId)) return; @@ -2300,7 +2336,6 @@ export default function App() { } } catch (e) { if ((e as Error).name === "AbortError" || !isCurrentProfileRequest(requestId)) return; - console.error("Failed to fetch experimental relevance score", e); const message = "The experimental relevance score is temporarily unavailable."; setScoreData(null); setProfileSection(requestId, "score", "error", message); @@ -2365,7 +2400,11 @@ export default function App() { try { const response = await fetch( `${API_BASE}/api/charities/grants/funders/${encodeURIComponent(target.sourceFunderKey)}/reset-to-observed`, - { method: "POST", credentials: "include" }, + { + method: "POST", + credentials: "include", + headers: mutationHeaders("reset source funder to observed-only"), + }, ); const body = await response.json(); if (!response.ok) { @@ -2383,29 +2422,37 @@ export default function App() { setActiveSourceFunder(null); clearActiveProfileSafely(); } catch (error) { - console.error("Could not reset source-funder profile link", error); setApiError("source_reset", (error as Error).message || `Could not reset ${target.displayName} to observed-only.`); } finally { setSourceFunderResetPending(false); } }; - const fetchPipelineStatus = async () => { + const fetchPipelineStatus = async (signal?: AbortSignal) => { if (!isBffOnline) return; try { - const resp = await fetch(`${API_BASE}/api/admin/pipeline/status`, { credentials: "include" }); + const resp = await fetch(`${API_BASE}/api/admin/pipeline/status`, { + credentials: "include", + signal, + }); + if (signal?.aborted) return; if (resp.ok) { const data = await resp.json(); setPipelineStatus(data); } - const logResp = await fetch(`${API_BASE}/api/admin/pipeline/logs`, { credentials: "include" }); + const logResp = await fetch(`${API_BASE}/api/admin/pipeline/logs`, { + credentials: "include", + signal, + }); + if (signal?.aborted) return; if (logResp.ok) { const logData = await logResp.json(); setLogs(logData.logs || "System idle.\n"); } } catch (e) { - console.error("Failed to get pipeline metrics", e); + if ((e as Error).name === "AbortError") return; + setLogs(previous => `${previous}\n[Error] Pipeline metrics are temporarily unavailable.\n`); } }; @@ -2439,7 +2486,7 @@ export default function App() { try { const resp = await fetch(`${API_BASE}/api/admin/pipeline/trigger`, { method: "POST", - headers: { "Content-Type": "application/json" }, + headers: mutationHeaders(`trigger ${source} pipeline`, true), body: JSON.stringify({ source: source, limit: source === "quick_consolidate" ? undefined : pipelineLimit, @@ -2459,7 +2506,7 @@ export default function App() { alert(`Failed to trigger: ${errorData.detail || "Unknown error"}`); } } catch (e) { - console.error("Failed to trigger pipeline run", e); + setLogs(previous => `${previous}\n[Error] Pipeline trigger failed: ${(e as Error).message || "Unknown error"}\n`); } finally { setIsTriggering(false); } @@ -2545,77 +2592,9 @@ export default function App() { } }; - if (initialLoading) { - return ( -
- {/* Modern glowing spinner */} -
-
-
-
- - {/* Loading text */} -
- Connecting to Foundation Intelligence... -
- - {/* Style injection for spin & pulse keyframes */} - -
- ); - } - return (
+ Skip to main content {/* Sidebar Navigation */}
}> + openLinkedDirectoryProfile({ charity_id: profileId, name: profileName })} + onSearchOrganization={(organizationName) => openOrganizationDirectoryFromMap({ ...EMPTY_GRANT_MAP_FILTERS, search: organizationName })} + onExploreSourceFunders={openSourceFundersFromMap} + favoriteGrantExplorerKeys={favorites.grantExplorers.map(favorite => favorite.key)} + onToggleFavoriteGrantExplorer={toggleFavoriteGrantExplorer} + presentation="favorite-explorer" + initialDrilldown={favoriteGrantExplorer.selection} + onBackToFavorites={closeFavoriteGrantExplorer} + /> + ) : favoriteDonorWorkspace ? ( Loading saved donor view…
}>
- + Loading world map…}> + +
{/* Monthly source-derived grant awards */} @@ -2977,51 +2971,9 @@ export default function App() { ) : grantTrends?.status === "available" && grantTrends.items.length > 0 ? ( <>
- - - - - - - - - - String(month).slice(2)} /> - formatCurrency(Number(value), grantTrends.currency || "GBP")} /> - { - if (!active || !payload?.length) return null; - const item = payload[0]?.payload as GrantTrendItem; - return ( -
- {label} - {item.coverage_status === "unknown" ? ( -
No source coverage established; not a confirmed zero.
- ) : item.coverage_status === "partial" ? ( -
Source records exist, but no valid amount can be aggregated.
- ) : ( - <> -
{formatCurrency(item.total_amount, grantTrends.currency || "GBP")}
-
{item.grant_count} recorded grants
- - )} -
- ); - }} - /> - - - +
Loading chart…
}> + +
Active currency: {grantTrends.currency} · Period {grantTrends.period?.from}–{grantTrends.period?.to}, anchored to the latest available award month. {unknownTrendMonths} month(s) have unknown source coverage. @@ -3057,19 +3009,9 @@ export default function App() { ) : grantThemes?.status === "available" && grantThemes.items.length > 0 ? ( <>
- - - - formatCurrency(Number(value), grantThemes.currency || "GBP")} /> - - formatCurrency(Number(value), grantThemes.currency || "GBP")} - labelFormatter={(label) => String(label)} - contentStyle={{ backgroundColor: "var(--bg-surface-opaque)", borderColor: "var(--border-glass)" }} - /> - - - +
Loading chart…
}> + +
Classified: {grantThemes.classification_coverage.classified_grant_count} / {grantThemes.classification_coverage.qualifying_grant_count} grants ({classificationPercentage}%). Unclassified remains visible. Multi-category amounts and counts are split equally to preserve totals. @@ -3841,15 +3783,15 @@ export default function App() { {newsRunMode === "illustrative" ? "Illustrative local briefing" : "Live external research"} Generated {formatNewsDate(newsSummary.generated_at)} Coverage: last {newsSummary.searched_weeks} weeks - {newsSummary.sources.length} cited source{newsSummary.sources.length === 1 ? "" : "s"} + {visibleNewsSources.length} cited source{visibleNewsSources.length === 1 ? "" : "s"}
{renderMarkdown(newsSummary.summary)}
- {newsSummary.sources.length > 0 && ( + {visibleNewsSources.length > 0 && (
Evidence

Articles used in this briefing

- {newsSummary.sources.map((source, index) => ( - + {visibleNewsSources.map(source => ( + {formatNewsDate(source.published)} {source.title} {source.source}{source.note ? ` · ${source.note}` : ""} @@ -4090,28 +4032,9 @@ export default function App() {

Financial History Trends

- - - - - - - - - - - - - - formatCurrency(v).replace("€", "")} /> - formatCurrency(Number(value))} contentStyle={{ backgroundColor: "var(--bg-surface-opaque)", borderColor: "var(--border-glass)" }} /> - - - - +
Loading chart…
}> + +
)} @@ -4138,8 +4061,8 @@ export default function App() { - ) : charityGrants.map((gr, idx) => ( - + ) : charityGrants.map(gr => ( + {gr.grant_id} {gr.funding_charity_id === selectedCharity.registered_charity_number ? gr.recipient_name : (gr.funding_name || "Unknown funder")} diff --git a/frontend/src/components/AppHeader.tsx b/frontend/src/components/AppHeader.tsx index 7f23266..fd94ea6 100644 --- a/frontend/src/components/AppHeader.tsx +++ b/frontend/src/components/AppHeader.tsx @@ -61,7 +61,7 @@ export default function AppHeader({ Reset
- + diff --git a/frontend/src/components/DataCharts.tsx b/frontend/src/components/DataCharts.tsx new file mode 100644 index 0000000..dd90d65 --- /dev/null +++ b/frontend/src/components/DataCharts.tsx @@ -0,0 +1,142 @@ +import { + Area, + AreaChart, + Bar, + BarChart, + CartesianGrid, + ResponsiveContainer, + Tooltip, + XAxis, + YAxis, +} from "recharts"; + +type CurrencyFormatter = (value: number | null | undefined, currency?: string) => string; + +interface GrantTrendPoint { + month: string; + grant_count: number | null; + total_amount: number | null; + coverage_status: "observed" | "partial" | "unknown"; +} + +interface ProgrammeAllocationPoint { + programme_area: string; + allocated_amount: number; +} + +interface FinancialHistoryPoint { + year: string | number; + Income: number | null; + Expenditure: number | null; +} + +export function GrantAwardsChart({ + items, + currency, + formatCurrency, +}: { + items: GrantTrendPoint[]; + currency: string; + formatCurrency: CurrencyFormatter; +}) { + return ( + + + + + + + + + + String(month).slice(2)} /> + formatCurrency(Number(value), currency)} /> + { + if (!active || !payload?.length) return null; + const item = payload[0]?.payload as GrantTrendPoint; + return ( +
+ {label} + {item.coverage_status === "unknown" ? ( + No source coverage established; not a confirmed zero. + ) : item.coverage_status === "partial" ? ( + Source records exist, but no valid amount can be aggregated. + ) : ( + <>{formatCurrency(item.total_amount, currency)}{item.grant_count} recorded grants + )} +
+ ); + }} + /> + + + + ); +} + +export function ProgrammeAllocationChart({ + items, + currency, + formatCurrency, +}: { + items: ProgrammeAllocationPoint[]; + currency: string; + formatCurrency: CurrencyFormatter; +}) { + return ( + + + + formatCurrency(Number(value), currency)} /> + + formatCurrency(Number(value), currency)} + labelFormatter={label => String(label)} + contentStyle={{ backgroundColor: "var(--bg-surface-opaque)", borderColor: "var(--border-glass)" }} + /> + + + + ); +} + +export function FinancialHistoryChart({ + data, + formatCurrency, +}: { + data: FinancialHistoryPoint[]; + formatCurrency: CurrencyFormatter; +}) { + return ( + + + + + + + + + + + + + + formatCurrency(Number(value)).replace("€", "")} /> + formatCurrency(Number(value))} contentStyle={{ backgroundColor: "var(--bg-surface-opaque)", borderColor: "var(--border-glass)" }} /> + + + + + ); +} diff --git a/frontend/src/components/DonorDirectoryPage.tsx b/frontend/src/components/DonorDirectoryPage.tsx index d689a5a..932b733 100644 --- a/frontend/src/components/DonorDirectoryPage.tsx +++ b/frontend/src/components/DonorDirectoryPage.tsx @@ -30,6 +30,7 @@ import { type DonorDirectoryState, type GrantScope, } from "../lib/grantScope"; +import { mutationHeaders } from "../lib/http"; type ProfileLink = | { status: "none" } @@ -327,6 +328,16 @@ function evidenceLinkType(evidence: EvidenceLink): "website" | "json" { return "website"; } +function evidenceKey(evidence: EvidenceLink) { + return [ + evidence.kind, + evidence.url, + evidence.role || "", + evidence.organization_name || "", + evidence.link_type || "", + ].join("\u001f"); +} + function evidenceOrganizationName(evidence: EvidenceLink): string { if (evidence.organization_name?.trim()) return evidence.organization_name; const role = evidenceRole(evidence); @@ -785,7 +796,7 @@ export default function DonorDirectoryPage({ fetch(`${apiBase}/api/charities/grants/funders/enrich`, { method: "POST", credentials: "include", - headers: { "Content-Type": "application/json" }, + headers: mutationHeaders("enrich source funders", true), body: JSON.stringify({ reg_numbers: queue.map(item => item.charityNumber) }), }), new Promise(resolve => window.setTimeout(resolve, LOCAL_PROFILE_LINK_DURATION_MS)), @@ -954,7 +965,10 @@ export default function DonorDirectoryPage({ && enrichmentIsActive(enrichmentRun) && enrichmentRun?.donorKeys.includes(detail.funder.source_funder_key), ); - const visibleEvidence = useMemo(() => (detail?.source_evidence || []) + const uniqueEvidence = useMemo(() => Array.from(new Map( + (detail?.source_evidence || []).map(evidence => [evidenceKey(evidence), evidence]), + ).values()), [detail?.source_evidence]); + const visibleEvidence = useMemo(() => uniqueEvidence .filter(evidence => ( evidenceRoleVisibility[evidenceRole(evidence)] && evidenceTypeVisibility[evidenceLinkType(evidence)] @@ -963,7 +977,7 @@ export default function DonorDirectoryPage({ const typeOrder = Number(evidenceLinkType(left) === "json") - Number(evidenceLinkType(right) === "json"); if (typeOrder) return typeOrder; return evidenceOrganizationName(left).localeCompare(evidenceOrganizationName(right)); - }), [detail?.source_evidence, evidenceRoleVisibility, evidenceTypeVisibility]); + }), [evidenceRoleVisibility, evidenceTypeVisibility, uniqueEvidence]); return (
@@ -1291,7 +1305,7 @@ export default function DonorDirectoryPage({
{detail.source_evidence.length ? <>
- {visibleEvidence.length} of {detail.source_evidence.length} links shown + {visibleEvidence.length} of {uniqueEvidence.length} links shown
{evidenceSettingsOpen &&
@@ -1302,10 +1316,10 @@ export default function DonorDirectoryPage({ setEvidenceTypeVisibility({ website: true, json: true }); }}>Show all
} - {visibleEvidence.length ? visibleEvidence.map((evidence, index) => { + {visibleEvidence.length ? visibleEvidence.map(evidence => { const role = evidenceRole(evidence); const linkType = evidenceLinkType(evidence); - return + return {evidenceOrganizationName(evidence)}{role[0].toUpperCase() + role.slice(1)} {linkType === "json" ? "record" : "website"} · stored source record {linkType === "json" ? "JSON" : "Website"} ; diff --git a/frontend/src/components/GrantWorldMap.tsx b/frontend/src/components/GrantWorldMap.tsx index 528feb0..d6dc794 100644 --- a/frontend/src/components/GrantWorldMap.tsx +++ b/frontend/src/components/GrantWorldMap.tsx @@ -95,6 +95,10 @@ interface GrantWorldMapProps { selectedCountryCode?: string | null; onCountrySelectionChange?: (selection: MapCountrySelection | null) => void; refreshing?: boolean; + connectionsVisible?: boolean; + connectionsLoading?: boolean; + connectionsError?: string | null; + connectionsDisabledReason?: string | null; onConnectionsVisibilityChange?: (visible: boolean) => void; onResetScope?: () => void; } @@ -165,9 +169,10 @@ function buildQuantileScale(values: number[]) { function RankingList({ items }: { items: MapRankingItem[] }) { if (!items.length) return Unavailable; + const uniqueItems = Array.from(new Map(items.map(item => [item.name, item])).values()); return (
    - {items.map(item => ( + {uniqueItems.map(item => (
  • {item.name} {item.count} @@ -227,6 +232,10 @@ export default function GrantWorldMap({ selectedCountryCode, onCountrySelectionChange, refreshing = false, + connectionsVisible, + connectionsLoading = false, + connectionsError = null, + connectionsDisabledReason = null, onConnectionsVisibilityChange, onResetScope, }: GrantWorldMapProps) { @@ -234,6 +243,7 @@ export default function GrantWorldMap({ const [hoveredCode, setHoveredCode] = useState(null); const [uncontrolledSelectedCode, setUncontrolledSelectedCode] = useState(null); const [showConnections, setShowConnections] = useState(false); + const connectionsShown = connectionsVisible ?? showConnections; const [connectionGeometry, setConnectionGeometry] = useState([]); const [loadingProgress, setLoadingProgress] = useState(0); const pathRefs = useRef(new Map()); @@ -246,12 +256,17 @@ export default function GrantWorldMap({ : "Number of grants"; const visibleConnections = useMemo( - () => (data.connections || []).slice(0, MAX_VISIBLE_CONNECTIONS), + () => Array.from(new Map( + (data.connections || []).map(connection => [ + `${connection.origin_country_code}:${connection.destination_country_code}`, + connection, + ]), + ).values()).slice(0, MAX_VISIBLE_CONNECTIONS), [data.connections], ); useLayoutEffect(() => { - if (!showConnections || !visibleConnections.length) { + if (!connectionsShown || !visibleConnections.length) { setConnectionGeometry([]); return; } @@ -295,7 +310,7 @@ export default function GrantWorldMap({ window.cancelAnimationFrame(animationFrame); window.removeEventListener("resize", calculateGeometry); }; - }, [showConnections, visibleConnections]); + }, [connectionsShown, visibleConnections]); const itemByCode = useMemo(() => new Map( data.items @@ -411,10 +426,12 @@ export default function GrantWorldMap({
{refreshing && } + {connectionsShown && connectionsLoading && ( +
+ Loading bounded country connections… +
+ )} + {connectionsShown && connectionsError && ( +
{connectionsError}
+ )}
{totalFiltered.toLocaleString("en-GB")} filtered grants @@ -525,8 +550,8 @@ export default function GrantWorldMap({ ); })} - {showConnections && ( - + {connectionsShown && ( + {connectionGeometry.map(({ connection, path, strokeWidth, opacity }) => ( - {showConnections && ( + {connectionsShown && (
{CONNECTION_DISCLOSURE} @@ -581,7 +606,7 @@ export default function GrantWorldMap({
Quantile scale {legendLabels.map((label, index) => ( - + {label} ))} @@ -649,7 +674,7 @@ export default function GrantWorldMap({ {data.metadata.limitations.length > 0 && (
Coverage limitations -
    {data.metadata.limitations.map(limitation =>
  • {limitation}
    • )}
+
    {Array.from(new Set(data.metadata.limitations)).map(limitation =>
  • {limitation}
    • )}
)} +

Official Charity Commission records. Registry presence does not imply funding activity.

@@ -482,12 +567,12 @@ export default function RegistryDirectory({ apiBase, online, initialQuery = "", {page && {page.search_strategy === "fts5" ? "Indexed name search" : "Indexed directory"}}
{rangeValidationError &&
{rangeValidationError} Results are not refreshed until the range is valid.
} - {error &&
{error}
} + {error &&
{error}
} {loading && !page ?
Loading organization directory…
: ( <>
{(page?.results || []).map(organization => ( - +
+
event.stopPropagation()}> + {detailLoading || !detail ?
Loading registry detail…
: <> Charity Commission · #{detail.charity_number}

{detail.registered_name}

diff --git a/frontend/src/index.css b/frontend/src/index.css index 1b2dbc5..b656604 100644 --- a/frontend/src/index.css +++ b/frontend/src/index.css @@ -1,5 +1,3 @@ -@import url('https://fonts.googleapis.com/css2?family=Golos+Text:wght@400;500;600;700&family=Inter:wght@300;400;500;600;700&display=swap'); - :root { /* Netlight light-mode palette: one primary, restrained semantic accents. */ --bg-dark: #ffffff; @@ -79,13 +77,16 @@ body { /* App Core Container Layout */ #root { + width: 100%; + max-width: 100%; min-height: 100vh; display: flex; } .app-container { display: flex; - width: 100vw; + width: 100%; + max-width: 100%; min-height: 100vh; background: #ffffff; } @@ -152,7 +153,7 @@ body { } .nav-item.active { - color: var(--nl-unicorn); + color: var(--nl-unicorn-active); background-color: var(--nl-unicorn-glow); border-color: rgba(102, 100, 241, 0.28); } @@ -180,7 +181,7 @@ body { display: flex; align-items: center; justify-content: center; - color: var(--nl-unicorn); + color: var(--nl-unicorn-active); font-weight: 600; border: 1px solid rgba(102, 100, 241, 0.32); } @@ -233,6 +234,7 @@ body { .main-content { flex: 1 1 0; width: 0; + max-width: 100%; min-width: 0; display: flex; flex-direction: column; @@ -368,6 +370,8 @@ body { .page-container { flex-grow: 1; + width: 100%; + max-width: 100%; min-width: 0; padding: 32px; } @@ -386,6 +390,33 @@ body { border: 1px solid rgba(220, 38, 38, 0.2); } +.data-notice-info { + color: var(--semantic-info); + background: rgba(0, 125, 167, 0.08); + border: 1px solid rgba(0, 125, 167, 0.2); +} + +.connection-status { + display: flex; + align-items: center; + gap: 8px; +} + +.skip-link { + position: fixed; + z-index: 1000; + top: 8px; + left: 8px; + padding: 10px 14px; + border-radius: var(--radius-sm); + color: #fff; + background: var(--nl-unicorn-active); + transform: translateY(-160%); +} + +.skip-link:focus { transform: translateY(0); } +.main-content:focus { outline: none; } + .status-badge { display: inline-flex; align-items: center; @@ -2802,7 +2833,7 @@ body { .chart-actions { display: inline-flex; flex-wrap: wrap; justify-content: flex-end; gap: 4px; } .chart-segmented button, .chart-actions button { min-height: 32px; border: 1px solid var(--border-glass); border-radius: 7px; padding: 6px 8px; color: var(--text-secondary); background: #fff; font: inherit; font-size: 11px; font-weight: 600; cursor: pointer; } -.chart-segmented button.active { border-color: var(--nl-unicorn); color: var(--nl-unicorn); background: var(--nl-unicorn-glow); } +.chart-segmented button.active { border-color: var(--nl-unicorn); color: var(--nl-unicorn-active); background: var(--nl-unicorn-glow); } .trend-card-header { position: relative; z-index: 3; } .trend-controls { position: relative; display: flex; align-items: center; flex: 0 0 auto; gap: 8px; } .trend-segmented { flex-wrap: nowrap; } @@ -3813,3 +3844,77 @@ body { .donor-detail-metrics { grid-template-columns: 1fr 1fr; } .donor-detail-content { padding: 12px; } } + +/* Phase 7 viewport safety: constrain every page shell to its actual viewport + while allowing dense controls and data tables to scroll inside their panel. */ +html, +body, +#root, +.app-container, +.main-content, +.page-container { + width: 100%; + max-width: 100%; +} + +.page-container > *, +.overview-dashboard, +.world-map-card, +.world-map-header > *, +.analytics-chart-card, +.registry-directory, +.donor-directory-page, +.donor-directory-workspace, +.donor-detail-content, +.filter-drawer, +.overview-filter-drawer { + min-width: 0; + max-width: 100%; +} + +img, +svg, +canvas, +input, +select, +textarea { + max-width: 100%; +} + +.table-container { max-width: 100%; overflow-x: auto; } +.map-mode-control { max-width: 100%; } +.map-secondary-status { margin: 0 0 10px; color: var(--text-secondary); font-size: 11px; } +.map-secondary-status.is-error { color: var(--semantic-error); } + +@media (max-width: 520px) { + .overview-kpi-grid.compact, + .landscape-metrics { grid-template-columns: repeat(2, minmax(0, 1fr)) !important; } + .overview-kpi { min-width: 0; } + .overview-kpi strong, + .overview-kpi small { + overflow: visible; + text-overflow: clip; + white-space: normal; + overflow-wrap: anywhere; + } + .overview-kpi small { line-height: 1.3; } + .map-mode-control { display: flex; flex-wrap: wrap; overflow: visible; } + .map-mode-control button { flex: 1 1 calc(50% - 4px); min-width: 0; white-space: normal; } + .map-mode-divider { display: none; } + .filter-drawer-header, + .filter-drawer-body, + .overview-filter-drawer-header, + .overview-filter-drawer-body { padding-right: 14px; padding-left: 14px; } + .filter-drawer-footer, + .overview-filter-drawer-footer { flex-wrap: wrap; padding-right: 14px; padding-left: 14px; } + .filter-drawer-footer > *, + .overview-filter-drawer-footer > * { flex: 1 1 128px; min-height: 44px; } + .world-map-header, + .map-controls-anchor { min-width: 0; max-width: 100%; } +} + +@media (max-width: 350px) { + .overview-kpi-grid.compact, + .landscape-metrics { grid-template-columns: 1fr !important; } + .header-title { max-width: 104px; } +} diff --git a/frontend/src/lib/http.ts b/frontend/src/lib/http.ts new file mode 100644 index 0000000..62d9275 --- /dev/null +++ b/frontend/src/lib/http.ts @@ -0,0 +1,10 @@ +export function mutationHeaders(reason: string, json = false): Record { + const key = typeof globalThis.crypto?.randomUUID === "function" + ? globalThis.crypto.randomUUID() + : `${Date.now()}-${Math.random().toString(36).slice(2)}`; + return { + ...(json ? { "Content-Type": "application/json" } : {}), + "Idempotency-Key": key, + "X-Action-Reason": reason, + }; +} diff --git a/frontend/tests/phase7Contracts.test.ts b/frontend/tests/phase7Contracts.test.ts new file mode 100644 index 0000000..95c2b74 --- /dev/null +++ b/frontend/tests/phase7Contracts.test.ts @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import test from "node:test"; + +const source = (relativePath: string) => readFileSync(join(process.cwd(), relativePath), "utf8"); + +test("the frontend has no runtime CDN stylesheet or console warning/error paths", () => { + const css = source("src/index.css"); + const app = source("src/App.tsx"); + const components = [ + "src/components/OverviewDashboard.tsx", + "src/components/GrantWorldMap.tsx", + "src/components/DonorDirectoryPage.tsx", + "src/components/RegistryDirectory.tsx", + ].map(source).join("\n"); + + assert.doesNotMatch(css, /@import\s+url\(["']?https?:/i); + assert.doesNotMatch(`${app}\n${components}`, /console\.(?:warn|error)\s*\(/); +}); + +test("heavy map, overview and chart code remains behind lazy boundaries", () => { + const app = source("src/App.tsx"); + + assert.match(app, /lazy\(\(\) => import\("\.\/components\/GrantWorldMap"\)\)/); + assert.match(app, /lazy\(\(\) => import\("\.\/components\/OverviewDashboard"\)\)/); + assert.match(app, /import\("\.\/components\/DataCharts"\)/); + assert.doesNotMatch(app, /from\s+["']recharts["']/); +}); + +test("request lifecycles cancel stale work and load map connections on demand", () => { + const app = source("src/App.tsx"); + const overview = source("src/components/OverviewDashboard.tsx"); + + assert.match(app, /mapRequestRef\.current\?\.abort\(\)/); + assert.match(app, /grantAnalyticsRequestRef\.current\?\.abort\(\)/); + assert.match(overview, /new AbortController\(\)/); + assert.match(overview, /api\/charities\/grants\/map\/connections/); + assert.doesNotMatch(overview, /include_connections/); + assert.match(overview, /return \(\) => controller\.abort\(\)/); +}); + +test("stable semantic keys replace index-derived evidence and legend keys", () => { + const app = source("src/App.tsx"); + const donorDirectory = source("src/components/DonorDirectoryPage.tsx"); + const worldMap = source("src/components/GrantWorldMap.tsx"); + + assert.doesNotMatch(app, /source\.link\}.*index/); + assert.doesNotMatch(donorDirectory, /evidence\.url\}.*index/); + assert.match(donorDirectory, /key=\{evidenceKey\(evidence\)\}/); + assert.doesNotMatch(worldMap, /key=\{`\$\{label\}-\$\{index\}`\}/); +}); + +test("the responsive shell preserves the map-first layout and viewport constraints", () => { + const app = source("src/App.tsx"); + const overview = source("src/components/OverviewDashboard.tsx"); + const css = source("src/index.css"); + + assert.match(app, /className="skip-link" href="#main-content"/); + assert.match(app, /id="main-content" className="main-content" tabIndex=\{-1\}/); + assert.ok(overview.indexOf("- + Cost-conscious, non-production Foundation Intelligence public read-only demo. + HTTP is temporary and not production-ready. DesiredCount defaults to zero and + no data, image or stack operation is performed by this template file itself. + +Parameters: + VpcId: + Type: AWS::EC2::VPC::Id + Default: vpc-0348ad96c2afcf8dd + Description: Existing VPC confirmed in account 208337080387, eu-west-1. + PublicSubnetIds: + Type: List + Default: subnet-04d4e3e5af84a7ea4,subnet-076f66fcdd86cd28f + Description: Two existing public subnets in distinct Availability Zones. + DatabaseAvailabilityZoneA: + Type: AWS::EC2::AvailabilityZone::Name + Default: eu-west-1a + DatabaseAvailabilityZoneB: + Type: AWS::EC2::AvailabilityZone::Name + Default: eu-west-1b + DatabaseSubnetCidrA: + Type: String + Default: 172.31.240.0/28 + AllowedPattern: '^([0-9]{1,3}\.){3}[0-9]{1,3}/28$' + DatabaseSubnetCidrB: + Type: String + Default: 172.31.240.16/28 + AllowedPattern: '^([0-9]{1,3}\.){3}[0-9]{1,3}/28$' + BackendImageUri: + Type: String + Description: Immutable ARM64 backend ECR URI, preferably pinned by digest. + AllowedPattern: '^208337080387\.dkr\.ecr\.eu-west-1\.amazonaws\.com/foundation-intelligence-platform/backend:(?!latest$)[A-Za-z0-9._-]+(@sha256:[a-f0-9]{64})?$' + FrontendImageUri: + Type: String + Description: Immutable ARM64 frontend-runtime-ecs ECR URI, preferably pinned by digest. + AllowedPattern: '^208337080387\.dkr\.ecr\.eu-west-1\.amazonaws\.com/foundation-intelligence-platform/frontend:(?!latest$)[A-Za-z0-9._-]+(@sha256:[a-f0-9]{64})?$' + AwsCliImageUri: + Type: String + Default: public.ecr.aws/aws-cli/aws-cli:2.27.41 + Description: Multi-architecture AWS CLI image used only by migration init/evidence containers. + ApplicationDesiredCount: + Type: Number + Default: 0 + AllowedValues: [0, 1] + Description: Keep zero until migration and release-gate evidence pass. + DatabaseInstanceClass: + Type: String + Default: db.t4g.medium + AllowedValues: [db.t4g.small, db.t4g.medium] + DatabaseEngineVersion: + Type: String + Default: '16.14' + AllowedPattern: '^16\.[0-9]+$' + ApplicationDatabaseUser: + Type: String + Default: foundation_app + AllowedPattern: '^[a-z_][a-z0-9_]{0,62}$' + SourceObjectKey: + Type: String + Default: imports/charities.db + AllowedPattern: '^[A-Za-z0-9!_.*()/''-]+$' + SourceSha256: + Type: String + Default: 8fc0cce61c81d54869a3cc9a61d9378e1cb03f2b9607a70c2836b52fba257651 + AllowedPattern: '^[a-f0-9]{64}$' + SourceSchemaVersion: + Type: String + Default: '7' + AllowedPattern: '^[0-9]+$' + DatasetVersion: + Type: String + Default: demo-initial-2026-08-06 + AllowedPattern: '^[A-Za-z0-9._-]{1,200}$' + CodeRevision: + Type: String + Description: Full Git commit SHA used to build the approved backend image. + AllowedPattern: '^[a-f0-9]{40}$' + +Resources: + DatabaseSubnetA: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VpcId + AvailabilityZone: !Ref DatabaseAvailabilityZoneA + CidrBlock: !Ref DatabaseSubnetCidrA + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-database-a' + - Key: Environment + Value: demo + + DatabaseSubnetB: + Type: AWS::EC2::Subnet + Properties: + VpcId: !Ref VpcId + AvailabilityZone: !Ref DatabaseAvailabilityZoneB + CidrBlock: !Ref DatabaseSubnetCidrB + MapPublicIpOnLaunch: false + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-database-b' + - Key: Environment + Value: demo + + DatabaseRouteTable: + Type: AWS::EC2::RouteTable + Properties: + VpcId: !Ref VpcId + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-database-isolated' + + DatabaseSubnetRouteAssociationA: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref DatabaseRouteTable + SubnetId: !Ref DatabaseSubnetA + + DatabaseSubnetRouteAssociationB: + Type: AWS::EC2::SubnetRouteTableAssociation + Properties: + RouteTableId: !Ref DatabaseRouteTable + SubnetId: !Ref DatabaseSubnetB + + AlbSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Temporary public HTTP ingress for the non-production demo + VpcId: !Ref VpcId + SecurityGroupIngress: + - Description: Temporary HTTP demo only + IpProtocol: tcp + FromPort: 80 + ToPort: 80 + CidrIp: 0.0.0.0/0 + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-alb' + + EcsSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: Shared application and one-off task ENIs + VpcId: !Ref VpcId + SecurityGroupEgress: + - Description: TLS to ECR, S3, Secrets Manager, logs and approved sources + IpProtocol: tcp + FromPort: 443 + ToPort: 443 + CidrIp: 0.0.0.0/0 + - Description: PostgreSQL to the private RDS security group + IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + DestinationSecurityGroupId: !GetAtt DatabaseSecurityGroup.GroupId + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-ecs' + + DatabaseSecurityGroup: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: PostgreSQL only from the ECS task security group + VpcId: !Ref VpcId + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-rds' + + FrontendIngressFromAlb: + Type: AWS::EC2::SecurityGroupIngress + Properties: + Description: ALB to the ECS frontend only + GroupId: !GetAtt EcsSecurityGroup.GroupId + IpProtocol: tcp + FromPort: 8080 + ToPort: 8080 + SourceSecurityGroupId: !GetAtt AlbSecurityGroup.GroupId + + DatabaseIngressFromEcs: + Type: AWS::EC2::SecurityGroupIngress + Properties: + Description: ECS application and one-off tasks to PostgreSQL + GroupId: !GetAtt DatabaseSecurityGroup.GroupId + IpProtocol: tcp + FromPort: 5432 + ToPort: 5432 + SourceSecurityGroupId: !GetAtt EcsSecurityGroup.GroupId + + ImportBucket: + Type: AWS::S3::Bucket + DeletionPolicy: Retain + UpdateReplacePolicy: Retain + Properties: + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: AES256 + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + OwnershipControls: + Rules: + - ObjectOwnership: BucketOwnerEnforced + VersioningConfiguration: + Status: Enabled + LifecycleConfiguration: + Rules: + - Id: ExpireTemporaryImports + Status: Enabled + Prefix: imports/ + ExpirationInDays: 7 + NoncurrentVersionExpiration: + NoncurrentDays: 7 + Tags: + - Key: Environment + Value: demo + - Key: Purpose + Value: one-off-private-import + + ImportBucketPolicy: + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref ImportBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: DenyInsecureTransport + Effect: Deny + Principal: '*' + Action: s3:* + Resource: + - !GetAtt ImportBucket.Arn + - !Sub '${ImportBucket.Arn}/*' + Condition: + Bool: + aws:SecureTransport: 'false' + + FrontendLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/frontend' + RetentionInDays: 7 + + BackendLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/backend' + RetentionInDays: 7 + + MigrationLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub '/foundation-intelligence/demo/${AWS::StackName}/migration' + RetentionInDays: 7 + + AppDatabaseSecret: + Type: AWS::SecretsManager::Secret + Properties: + Description: Restricted application database role for the public demo + GenerateSecretString: + SecretStringTemplate: !Sub '{"username":"${ApplicationDatabaseUser}"}' + GenerateStringKey: password + PasswordLength: 40 + ExcludePunctuation: true + Tags: + - Key: Environment + Value: demo + + DatabaseSubnetGroup: + Type: AWS::RDS::DBSubnetGroup + Properties: + DBSubnetGroupDescription: Two isolated database subnets without internet routes + SubnetIds: + - !Ref DatabaseSubnetA + - !Ref DatabaseSubnetB + Tags: + - Key: Name + Value: !Sub '${AWS::StackName}-postgresql' + + DatabaseParameterGroup: + Type: AWS::RDS::DBParameterGroup + Properties: + Description: PostgreSQL 16 encrypted transport for the demo + Family: postgres16 + Parameters: + rds.force_ssl: '1' + + Database: + Type: AWS::RDS::DBInstance + DeletionPolicy: Snapshot + UpdateReplacePolicy: Snapshot + Properties: + DBInstanceIdentifier: !Sub '${AWS::StackName}-postgresql' + Engine: postgres + EngineVersion: !Ref DatabaseEngineVersion + DBInstanceClass: !Ref DatabaseInstanceClass + DBName: foundation_intelligence + MasterUsername: foundation_admin + ManageMasterUserPassword: true + Port: 5432 + AllocatedStorage: '30' + MaxAllocatedStorage: 100 + StorageType: gp3 + StorageEncrypted: true + MultiAZ: false + PubliclyAccessible: false + BackupRetentionPeriod: 1 + CopyTagsToSnapshot: true + AutoMinorVersionUpgrade: true + EnablePerformanceInsights: false + DeletionProtection: false + DBSubnetGroupName: !Ref DatabaseSubnetGroup + DBParameterGroupName: !Ref DatabaseParameterGroup + VPCSecurityGroups: + - !GetAtt DatabaseSecurityGroup.GroupId + Tags: + - Key: Environment + Value: demo + - Key: ResizeAfterImport + Value: db.t4g.small + + EcsCluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + ClusterSettings: + - Name: containerInsights + Value: disabled + Tags: + - Key: Environment + Value: demo + + EcsExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy + Policies: + - PolicyName: ReadDemoDatabaseSecrets + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: secretsmanager:GetSecretValue + Resource: + - !Ref AppDatabaseSecret + - !GetAtt Database.MasterUserSecret.SecretArn + + ApplicationTaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + + MigrationTaskRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: ecs-tasks.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: OneOffImportObjectAndEvidence + PolicyDocument: + Version: '2012-10-17' + Statement: + - Sid: ReadApprovedSourceObject + Effect: Allow + Action: + - s3:GetObject + - s3:GetObjectVersion + Resource: !Sub '${ImportBucket.Arn}/${SourceObjectKey}' + - Sid: WriteMigrationEvidence + Effect: Allow + Action: s3:PutObject + Resource: !Sub '${ImportBucket.Arn}/migration-evidence/${DatasetVersion}/*' + + ApplicationTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub '${AWS::StackName}-application' + Cpu: '512' + Memory: '2048' + NetworkMode: awsvpc + RequiresCompatibilities: [FARGATE] + ExecutionRoleArn: !GetAtt EcsExecutionRole.Arn + TaskRoleArn: !GetAtt ApplicationTaskRole.Arn + RuntimePlatform: + CpuArchitecture: ARM64 + OperatingSystemFamily: LINUX + ContainerDefinitions: + - Name: backend + Image: !Ref BackendImageUri + Essential: true + User: '10001:10001' + StopTimeout: 30 + Environment: + - Name: APP_ENV + Value: demo + - Name: AUTH_MODE + Value: public_readonly + - Name: DEV_AUTH_ENABLED + Value: 'false' + - Name: CORE_PROXY_ENABLED + Value: 'false' + - Name: CORS_ORIGINS + Value: '' + - Name: DATA_RUNTIME_MODE + Value: postgresql + - Name: DATABASE_HOST + Value: !GetAtt Database.Endpoint.Address + - Name: DATABASE_PORT + Value: '5432' + - Name: DATABASE_NAME + Value: foundation_intelligence + - Name: DATABASE_USER + Value: !Ref ApplicationDatabaseUser + - Name: DATABASE_SSL_MODE + Value: require + - Name: DATABASE_POOL_SIZE + Value: '5' + - Name: DATABASE_MAX_OVERFLOW + Value: '5' + Secrets: + - Name: DATABASE_PASSWORD + ValueFrom: !Sub '${AppDatabaseSecret}:password::' + HealthCheck: + Command: + - CMD + - python + - -c + - >- + import urllib.request; + urllib.request.urlopen('http://127.0.0.1:8000/health/ready', timeout=3).read() + Interval: 30 + Timeout: 5 + Retries: 3 + StartPeriod: 120 + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref BackendLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: backend + - Name: frontend + Image: !Ref FrontendImageUri + Essential: true + User: '101:101' + DependsOn: + - ContainerName: backend + Condition: HEALTHY + PortMappings: + - Name: http + ContainerPort: 8080 + HostPort: 8080 + Protocol: tcp + AppProtocol: http + HealthCheck: + Command: + - CMD + - wget + - -q + - -O + - /dev/null + - http://127.0.0.1:8080/ + Interval: 30 + Timeout: 5 + Retries: 3 + StartPeriod: 30 + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref FrontendLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: frontend + Tags: + - Key: Environment + Value: demo + - Key: EphemeralStorageGiB + Value: '20-default' + + MigrationTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub '${AWS::StackName}-migration' + Cpu: '1024' + Memory: '4096' + NetworkMode: awsvpc + RequiresCompatibilities: [FARGATE] + ExecutionRoleArn: !GetAtt EcsExecutionRole.Arn + TaskRoleArn: !GetAtt MigrationTaskRole.Arn + RuntimePlatform: + CpuArchitecture: ARM64 + OperatingSystemFamily: LINUX + Volumes: + - Name: migration-work + ContainerDefinitions: + - Name: download-source + Image: !Ref AwsCliImageUri + Essential: false + EntryPoint: [sh, -c] + Command: + - >- + aws s3 cp "$SOURCE_URI" /migration-work/charities.db --only-show-errors + && mkdir -p /migration-work/evidence + && chown -R 10001:10001 /migration-work + Environment: + - Name: SOURCE_URI + Value: !Sub 's3://${ImportBucket}/${SourceObjectKey}' + MountPoints: + - SourceVolume: migration-work + ContainerPath: /migration-work + ReadOnly: false + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref MigrationLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: download + - Name: migrate + Image: !Ref BackendImageUri + Essential: false + User: '10001:10001' + EntryPoint: [python, -m] + Command: [migration.aws_entrypoint] + DependsOn: + - ContainerName: download-source + Condition: SUCCESS + Environment: + - Name: APP_ENV + Value: demo + - Name: DATABASE_HOST + Value: !GetAtt Database.Endpoint.Address + - Name: DATABASE_PORT + Value: '5432' + - Name: DATABASE_NAME + Value: foundation_intelligence + - Name: DATABASE_APP_USER + Value: !Ref ApplicationDatabaseUser + - Name: DATABASE_SSL_MODE + Value: require + - Name: MIGRATION_SOURCE_PATH + Value: /migration-work/charities.db + - Name: MIGRATION_OUTPUT_DIRECTORY + Value: /migration-work/evidence + - Name: MIGRATION_EXPECTED_CHECKSUM + Value: !Ref SourceSha256 + - Name: MIGRATION_EXPECTED_SCHEMA_VERSION + Value: !Ref SourceSchemaVersion + - Name: MIGRATION_DATASET_VERSION + Value: !Ref DatasetVersion + - Name: MIGRATION_CODE_REVISION + Value: !Ref CodeRevision + Secrets: + - Name: DATABASE_ADMIN_USER + ValueFrom: !Sub '${Database.MasterUserSecret.SecretArn}:username::' + - Name: DATABASE_ADMIN_PASSWORD + ValueFrom: !Sub '${Database.MasterUserSecret.SecretArn}:password::' + - Name: DATABASE_APP_PASSWORD + ValueFrom: !Sub '${AppDatabaseSecret}:password::' + MountPoints: + - SourceVolume: migration-work + ContainerPath: /migration-work + ReadOnly: false + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref MigrationLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: migrate + - Name: upload-evidence + Image: !Ref AwsCliImageUri + Essential: true + EntryPoint: [aws] + Command: + - s3 + - cp + - /migration-work/evidence/ + - !Sub 's3://${ImportBucket}/migration-evidence/${DatasetVersion}/' + - --recursive + - --only-show-errors + DependsOn: + - ContainerName: migrate + Condition: SUCCESS + MountPoints: + - SourceVolume: migration-work + ContainerPath: /migration-work + ReadOnly: true + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref MigrationLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: evidence + Tags: + - Key: Environment + Value: demo + - Key: EphemeralStorageGiB + Value: '20-default' + + LoadBalancer: + Type: AWS::ElasticLoadBalancingV2::LoadBalancer + Properties: + Scheme: internet-facing + Type: application + IpAddressType: ipv4 + SecurityGroups: + - !GetAtt AlbSecurityGroup.GroupId + Subnets: !Ref PublicSubnetIds + Tags: + - Key: Environment + Value: demo + - Key: TransportNotice + Value: temporary-http-not-production-ready + + FrontendTargetGroup: + Type: AWS::ElasticLoadBalancingV2::TargetGroup + Properties: + TargetType: ip + Protocol: HTTP + Port: 80 + VpcId: !Ref VpcId + HealthCheckEnabled: true + HealthCheckProtocol: HTTP + HealthCheckPath: /health/live + HealthCheckPort: traffic-port + Matcher: + HttpCode: '200' + Tags: + - Key: Environment + Value: demo + + HttpListener: + Type: AWS::ElasticLoadBalancingV2::Listener + Properties: + LoadBalancerArn: !Ref LoadBalancer + Port: 80 + Protocol: HTTP + DefaultActions: + - Type: forward + TargetGroupArn: !Ref FrontendTargetGroup + + ApplicationService: + Type: AWS::ECS::Service + DependsOn: HttpListener + Properties: + ServiceName: application + Cluster: !Ref EcsCluster + TaskDefinition: !Ref ApplicationTaskDefinition + LaunchType: FARGATE + PlatformVersion: LATEST + DesiredCount: !Ref ApplicationDesiredCount + EnableExecuteCommand: false + HealthCheckGracePeriodSeconds: 180 + DeploymentConfiguration: + MinimumHealthyPercent: 0 + MaximumPercent: 100 + NetworkConfiguration: + AwsvpcConfiguration: + AssignPublicIp: ENABLED + SecurityGroups: + - !GetAtt EcsSecurityGroup.GroupId + Subnets: !Ref PublicSubnetIds + LoadBalancers: + - ContainerName: frontend + ContainerPort: 8080 + TargetGroupArn: !Ref FrontendTargetGroup + Tags: + - Key: Environment + Value: demo + +Outputs: + TemporaryDemoUrl: + Description: HTTP-only temporary demo URL; do not treat as production-ready. + Value: !Sub 'http://${LoadBalancer.DNSName}' + ClusterArn: + Value: !GetAtt EcsCluster.Arn + ApplicationServiceArn: + Value: !Ref ApplicationService + MigrationTaskDefinitionArn: + Value: !Ref MigrationTaskDefinition + ImportBucketName: + Value: !Ref ImportBucket + DatabaseEndpoint: + Value: !GetAtt Database.Endpoint.Address + DatabaseInstanceClass: + Value: !Ref DatabaseInstanceClass + ApplicationDesiredCount: + Value: !Ref ApplicationDesiredCount + DatabaseSubnetIds: + Value: !Join [',', [!Ref DatabaseSubnetA, !Ref DatabaseSubnetB]] diff --git a/infra/cloudformation/parameters.demo.example.json b/infra/cloudformation/parameters.demo.example.json new file mode 100644 index 0000000..baa19fd --- /dev/null +++ b/infra/cloudformation/parameters.demo.example.json @@ -0,0 +1,22 @@ +[ + { + "ParameterKey": "BackendImageUri", + "ParameterValue": "208337080387.dkr.ecr.eu-west-1.amazonaws.com/foundation-intelligence-platform/backend:demo-ARM64-GIT_SHA" + }, + { + "ParameterKey": "FrontendImageUri", + "ParameterValue": "208337080387.dkr.ecr.eu-west-1.amazonaws.com/foundation-intelligence-platform/frontend:demo-ARM64-GIT_SHA" + }, + { + "ParameterKey": "CodeRevision", + "ParameterValue": "FULL_40_CHARACTER_GIT_SHA_AFTER_REVIEWED_COMMIT" + }, + { + "ParameterKey": "ApplicationDesiredCount", + "ParameterValue": "0" + }, + { + "ParameterKey": "DatabaseInstanceClass", + "ParameterValue": "db.t4g.medium" + } +] diff --git a/infra/terraform/environments/dev/main.tf b/infra/terraform/environments/dev/main.tf new file mode 100644 index 0000000..481572e --- /dev/null +++ b/infra/terraform/environments/dev/main.tf @@ -0,0 +1,39 @@ +module "platform" { + source = "../../modules/platform" + + environment = "dev" + aws_region = var.aws_region + + availability_zones = ["${var.aws_region}a", "${var.aws_region}b"] + vpc_cidr = "10.40.0.0/16" + public_subnet_cidrs = ["10.40.0.0/24", "10.40.1.0/24"] + application_subnet_cidrs = ["10.40.10.0/24", "10.40.11.0/24"] + database_subnet_cidrs = ["10.40.20.0/24", "10.40.21.0/24"] + single_nat_gateway = true + enable_interface_endpoints = false + + api_image = var.api_image + worker_image = var.worker_image + frontend_origin_image_digest = var.frontend_origin_image_digest + api_desired_count = 1 + worker_desired_count = 1 + + rds_instance_class = "db.t4g.small" + rds_allocated_storage_gib = 30 + rds_max_storage_gib = 100 + rds_backup_retention_days = 7 + rds_multi_az = false + + github_repository = var.github_repository + github_oidc_provider_arn = var.github_oidc_provider_arn + domain_name = var.domain_name + hosted_zone_id = var.hosted_zone_id + manage_dns = var.manage_dns + enable_schedules = false + monthly_budget_usd = 300 + + tags = { + CostCentre = "unapproved-development" + Owner = "unassigned" + } +} diff --git a/infra/terraform/environments/dev/outputs.tf b/infra/terraform/environments/dev/outputs.tf new file mode 100644 index 0000000..5cd1a13 --- /dev/null +++ b/infra/terraform/environments/dev/outputs.tf @@ -0,0 +1,12 @@ +output "deployment_contract" { + value = { + api_dns = module.platform.api_load_balancer_dns_name + frontend_dns = module.platform.frontend_distribution_domain_name + deployment_role = module.platform.deployment_role_arn + schedules = module.platform.schedules_enabled + } +} + +output "release_gate" { + value = module.platform.release_gate +} diff --git a/infra/terraform/environments/dev/providers.tf b/infra/terraform/environments/dev/providers.tf new file mode 100644 index 0000000..86cc150 --- /dev/null +++ b/infra/terraform/environments/dev/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Application = "foundation-intelligence" + Environment = "dev" + ManagedBy = "terraform" + } + } +} diff --git a/infra/terraform/environments/dev/terraform.tfvars.example b/infra/terraform/environments/dev/terraform.tfvars.example new file mode 100644 index 0000000..6e9a150 --- /dev/null +++ b/infra/terraform/environments/dev/terraform.tfvars.example @@ -0,0 +1,11 @@ +aws_region = "eu-central-1" + +api_image = "000000000000.dkr.ecr.eu-central-1.amazonaws.com/foundation-intelligence-dev/api@sha256:0000000000000000000000000000000000000000000000000000000000000000" +worker_image = "000000000000.dkr.ecr.eu-central-1.amazonaws.com/foundation-intelligence-dev/worker@sha256:0000000000000000000000000000000000000000000000000000000000000000" +frontend_origin_image_digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" +github_repository = "replace-owner/FoundationIntelligencePlatform" + +# DNS and certificate creation stay disabled until ownership is approved. +manage_dns = false +domain_name = null +hosted_zone_id = null diff --git a/infra/terraform/environments/dev/variables.tf b/infra/terraform/environments/dev/variables.tf new file mode 100644 index 0000000..f42a3a6 --- /dev/null +++ b/infra/terraform/environments/dev/variables.tf @@ -0,0 +1,42 @@ +variable "aws_region" { + type = string +} + +variable "api_image" { + type = string +} + +variable "worker_image" { + type = string +} + +variable "frontend_origin_image_digest" { + type = string +} + +variable "github_repository" { + type = string +} + +variable "github_oidc_provider_arn" { + type = string + default = null + nullable = true +} + +variable "domain_name" { + type = string + default = null + nullable = true +} + +variable "hosted_zone_id" { + type = string + default = null + nullable = true +} + +variable "manage_dns" { + type = bool + default = false +} diff --git a/infra/terraform/environments/dev/versions.tf b/infra/terraform/environments/dev/versions.tf new file mode 100644 index 0000000..24cc0b3 --- /dev/null +++ b/infra/terraform/environments/dev/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.7.0, < 2.0.0" + + backend "s3" { + encrypt = true + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.90" + } + } +} diff --git a/infra/terraform/environments/staging/main.tf b/infra/terraform/environments/staging/main.tf new file mode 100644 index 0000000..4c85277 --- /dev/null +++ b/infra/terraform/environments/staging/main.tf @@ -0,0 +1,39 @@ +module "platform" { + source = "../../modules/platform" + + environment = "staging" + aws_region = var.aws_region + + availability_zones = ["${var.aws_region}a", "${var.aws_region}b"] + vpc_cidr = "10.50.0.0/16" + public_subnet_cidrs = ["10.50.0.0/24", "10.50.1.0/24"] + application_subnet_cidrs = ["10.50.10.0/24", "10.50.11.0/24"] + database_subnet_cidrs = ["10.50.20.0/24", "10.50.21.0/24"] + single_nat_gateway = false + enable_interface_endpoints = true + + api_image = var.api_image + worker_image = var.worker_image + frontend_origin_image_digest = var.frontend_origin_image_digest + api_desired_count = 2 + worker_desired_count = 2 + + rds_instance_class = "db.t4g.medium" + rds_allocated_storage_gib = 100 + rds_max_storage_gib = 500 + rds_backup_retention_days = 14 + rds_multi_az = true + + github_repository = var.github_repository + github_oidc_provider_arn = var.github_oidc_provider_arn + domain_name = var.domain_name + hosted_zone_id = var.hosted_zone_id + manage_dns = var.manage_dns + enable_schedules = false + monthly_budget_usd = 750 + + tags = { + CostCentre = "unapproved-staging" + Owner = "unassigned" + } +} diff --git a/infra/terraform/environments/staging/outputs.tf b/infra/terraform/environments/staging/outputs.tf new file mode 100644 index 0000000..5cd1a13 --- /dev/null +++ b/infra/terraform/environments/staging/outputs.tf @@ -0,0 +1,12 @@ +output "deployment_contract" { + value = { + api_dns = module.platform.api_load_balancer_dns_name + frontend_dns = module.platform.frontend_distribution_domain_name + deployment_role = module.platform.deployment_role_arn + schedules = module.platform.schedules_enabled + } +} + +output "release_gate" { + value = module.platform.release_gate +} diff --git a/infra/terraform/environments/staging/providers.tf b/infra/terraform/environments/staging/providers.tf new file mode 100644 index 0000000..a8e7fe8 --- /dev/null +++ b/infra/terraform/environments/staging/providers.tf @@ -0,0 +1,11 @@ +provider "aws" { + region = var.aws_region + + default_tags { + tags = { + Application = "foundation-intelligence" + Environment = "staging" + ManagedBy = "terraform" + } + } +} diff --git a/infra/terraform/environments/staging/terraform.tfvars.example b/infra/terraform/environments/staging/terraform.tfvars.example new file mode 100644 index 0000000..67c76a5 --- /dev/null +++ b/infra/terraform/environments/staging/terraform.tfvars.example @@ -0,0 +1,11 @@ +aws_region = "eu-central-1" + +api_image = "000000000000.dkr.ecr.eu-central-1.amazonaws.com/foundation-intelligence-staging/api@sha256:0000000000000000000000000000000000000000000000000000000000000000" +worker_image = "000000000000.dkr.ecr.eu-central-1.amazonaws.com/foundation-intelligence-staging/worker@sha256:0000000000000000000000000000000000000000000000000000000000000000" +frontend_origin_image_digest = "sha256:0000000000000000000000000000000000000000000000000000000000000000" +github_repository = "replace-owner/FoundationIntelligencePlatform" + +# DNS and certificate creation stay disabled until ownership is approved. +manage_dns = false +domain_name = null +hosted_zone_id = null diff --git a/infra/terraform/environments/staging/variables.tf b/infra/terraform/environments/staging/variables.tf new file mode 100644 index 0000000..f42a3a6 --- /dev/null +++ b/infra/terraform/environments/staging/variables.tf @@ -0,0 +1,42 @@ +variable "aws_region" { + type = string +} + +variable "api_image" { + type = string +} + +variable "worker_image" { + type = string +} + +variable "frontend_origin_image_digest" { + type = string +} + +variable "github_repository" { + type = string +} + +variable "github_oidc_provider_arn" { + type = string + default = null + nullable = true +} + +variable "domain_name" { + type = string + default = null + nullable = true +} + +variable "hosted_zone_id" { + type = string + default = null + nullable = true +} + +variable "manage_dns" { + type = bool + default = false +} diff --git a/infra/terraform/environments/staging/versions.tf b/infra/terraform/environments/staging/versions.tf new file mode 100644 index 0000000..24cc0b3 --- /dev/null +++ b/infra/terraform/environments/staging/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 1.7.0, < 2.0.0" + + backend "s3" { + encrypt = true + } + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.90" + } + } +} diff --git a/infra/terraform/modules/platform/compute.tf b/infra/terraform/modules/platform/compute.tf new file mode 100644 index 0000000..942e634 --- /dev/null +++ b/infra/terraform/modules/platform/compute.tf @@ -0,0 +1,436 @@ +resource "aws_ecr_repository" "api" { + name = "${local.name}/api" + image_tag_mutability = "IMMUTABLE" + force_delete = false + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.platform.arn + } + + image_scanning_configuration { + scan_on_push = true + } + + tags = local.common_tags +} + +resource "aws_ecr_repository" "worker" { + name = "${local.name}/worker" + image_tag_mutability = "IMMUTABLE" + force_delete = false + + encryption_configuration { + encryption_type = "KMS" + kms_key = aws_kms_key.platform.arn + } + + image_scanning_configuration { + scan_on_push = true + } + + tags = local.common_tags +} + +resource "aws_ecr_lifecycle_policy" "api" { + repository = aws_ecr_repository.api.name + policy = jsonencode({ + rules = [{ + rulePriority = 1 + description = "Retain the most recent 30 immutable images" + selection = { + tagStatus = "any" + countType = "imageCountMoreThan" + countNumber = 30 + } + action = { type = "expire" } + }] + }) +} + +resource "aws_ecr_lifecycle_policy" "worker" { + repository = aws_ecr_repository.worker.name + policy = aws_ecr_lifecycle_policy.api.policy +} + +resource "aws_ecs_cluster" "this" { + name = local.name + + setting { + name = "containerInsights" + value = "enabled" + } + + tags = local.common_tags +} + +resource "aws_cloudwatch_log_group" "api" { + name = "/foundation-intelligence/${var.environment}/api" + retention_in_days = var.environment == "staging" ? 90 : 30 + kms_key_id = aws_kms_key.logs.arn + + tags = local.common_tags +} + +resource "aws_cloudwatch_log_group" "worker" { + name = "/foundation-intelligence/${var.environment}/worker" + retention_in_days = var.environment == "staging" ? 90 : 30 + kms_key_id = aws_kms_key.logs.arn + + tags = local.common_tags +} + +resource "aws_ecs_task_definition" "api" { + family = "${local.name}-api" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = tostring(var.api_cpu) + memory = tostring(var.api_memory) + execution_role_arn = aws_iam_role.ecs_execution.arn + task_role_arn = aws_iam_role.api_task.arn + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + + container_definitions = jsonencode([{ + name = "api" + image = var.api_image + essential = true + user = "10001:10001" + readonlyRootFilesystem = true + stopTimeout = 60 + linuxParameters = { + initProcessEnabled = true + capabilities = { drop = ["ALL"] } + } + portMappings = [{ + name = "http" + containerPort = 8000 + hostPort = 8000 + protocol = "tcp" + appProtocol = "http" + }] + environment = [ + { name = "APP_ENV", value = var.environment }, + { name = "DATABASE_HOST", value = aws_db_instance.postgresql.address }, + { name = "DATABASE_PORT", value = tostring(aws_db_instance.postgresql.port) }, + { name = "DATABASE_NAME", value = aws_db_instance.postgresql.db_name }, + { name = "DATABASE_USER", value = aws_db_instance.postgresql.username }, + { name = "PIPELINE_QUEUE_URL", value = aws_sqs_queue.pipeline.url }, + { name = "RAW_BUCKET", value = aws_s3_bucket.data["raw"].id }, + { name = "VALIDATED_BUCKET", value = aws_s3_bucket.data["validated"].id }, + { name = "CURATED_BUCKET", value = aws_s3_bucket.data["curated"].id }, + { name = "EXPORTS_BUCKET", value = aws_s3_bucket.data["exports"].id }, + ] + secrets = [{ + name = "DATABASE_PASSWORD" + valueFrom = "${aws_db_instance.postgresql.master_user_secret[0].secret_arn}:password::" + }] + healthCheck = { + command = ["CMD-SHELL", "python -c 'import urllib.request; urllib.request.urlopen(\"http://127.0.0.1:8000/health/ready\", timeout=3)' || exit 1"] + interval = 30 + timeout = 5 + retries = 3 + startPeriod = 60 + } + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.api.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "api" + } + } + }]) + + ephemeral_storage { + size_in_gib = 21 + } + + tags = local.common_tags +} + +resource "aws_ecs_task_definition" "worker" { + family = "${local.name}-worker" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = tostring(var.worker_cpu) + memory = tostring(var.worker_memory) + execution_role_arn = aws_iam_role.ecs_execution.arn + task_role_arn = aws_iam_role.worker_task.arn + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + + container_definitions = jsonencode([{ + name = "worker" + image = var.worker_image + essential = true + user = "10001:10001" + readonlyRootFilesystem = true + stopTimeout = 120 + linuxParameters = { + initProcessEnabled = true + capabilities = { drop = ["ALL"] } + } + environment = [ + { name = "APP_ENV", value = var.environment }, + { name = "DATABASE_HOST", value = aws_db_instance.postgresql.address }, + { name = "DATABASE_PORT", value = tostring(aws_db_instance.postgresql.port) }, + { name = "DATABASE_NAME", value = aws_db_instance.postgresql.db_name }, + { name = "DATABASE_USER", value = aws_db_instance.postgresql.username }, + { name = "PIPELINE_QUEUE_URL", value = aws_sqs_queue.pipeline.url }, + { name = "PIPELINE_DLQ_URL", value = aws_sqs_queue.pipeline_dlq.url }, + { name = "RAW_BUCKET", value = aws_s3_bucket.data["raw"].id }, + { name = "VALIDATED_BUCKET", value = aws_s3_bucket.data["validated"].id }, + { name = "CURATED_BUCKET", value = aws_s3_bucket.data["curated"].id }, + ] + secrets = [{ + name = "DATABASE_PASSWORD" + valueFrom = "${aws_db_instance.postgresql.master_user_secret[0].secret_arn}:password::" + }] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.worker.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "worker" + } + } + }]) + + ephemeral_storage { + size_in_gib = 21 + } + + tags = local.common_tags +} + +resource "aws_ecs_task_definition" "release_gate" { + family = "${local.name}-release-gate" + requires_compatibilities = ["FARGATE"] + network_mode = "awsvpc" + cpu = "256" + memory = "512" + execution_role_arn = aws_iam_role.ecs_execution.arn + task_role_arn = aws_iam_role.release_gate_task.arn + + runtime_platform { + operating_system_family = "LINUX" + cpu_architecture = "ARM64" + } + + container_definitions = jsonencode([{ + name = "release-gate" + image = var.api_image + essential = true + user = "10001:10001" + readonlyRootFilesystem = true + stopTimeout = 60 + entryPoint = ["python", "-m"] + command = ["migration.release_gate"] + linuxParameters = { + initProcessEnabled = true + capabilities = { drop = ["ALL"] } + } + environment = [ + { name = "APP_ENV", value = var.environment }, + { name = "DATABASE_HOST", value = aws_db_instance.postgresql.address }, + { name = "DATABASE_PORT", value = tostring(aws_db_instance.postgresql.port) }, + { name = "DATABASE_NAME", value = aws_db_instance.postgresql.db_name }, + { name = "DATABASE_USER", value = aws_db_instance.postgresql.username }, + ] + secrets = [{ + name = "DATABASE_PASSWORD" + valueFrom = "${aws_db_instance.postgresql.master_user_secret[0].secret_arn}:password::" + }] + logConfiguration = { + logDriver = "awslogs" + options = { + awslogs-group = aws_cloudwatch_log_group.api.name + awslogs-region = var.aws_region + awslogs-stream-prefix = "release-gate" + } + } + }]) + + tags = local.common_tags +} + +resource "aws_lb" "api" { + name = substr("${local.name}-api", 0, 32) + internal = false + load_balancer_type = "application" + security_groups = [aws_security_group.alb.id] + subnets = values(aws_subnet.public)[*].id + drop_invalid_header_fields = true + enable_deletion_protection = true + + tags = local.common_tags +} + +resource "aws_lb_target_group" "api" { + name = substr("${local.name}-api", 0, 32) + port = 8000 + protocol = "HTTP" + target_type = "ip" + vpc_id = aws_vpc.this.id + + deregistration_delay = 60 + + health_check { + enabled = true + path = "/health/ready" + matcher = "200" + interval = 30 + timeout = 5 + healthy_threshold = 2 + unhealthy_threshold = 3 + } + + tags = local.common_tags +} + +resource "aws_lb_listener" "http" { + count = var.manage_dns && var.domain_name != null ? 1 : 0 + + load_balancer_arn = aws_lb.api.arn + port = 80 + protocol = "HTTP" + + default_action { + type = "redirect" + redirect { + port = "443" + protocol = "HTTPS" + status_code = "HTTP_301" + } + } +} + +resource "aws_ecs_service" "api" { + name = "api" + cluster = aws_ecs_cluster.this.id + task_definition = aws_ecs_task_definition.api.arn + desired_count = var.api_desired_count + launch_type = "FARGATE" + platform_version = "1.4.0" + deployment_minimum_healthy_percent = 100 + deployment_maximum_percent = 200 + health_check_grace_period_seconds = 120 + enable_execute_command = false + wait_for_steady_state = false + + network_configuration { + assign_public_ip = false + subnets = values(aws_subnet.application)[*].id + security_groups = [aws_security_group.ecs.id] + } + + load_balancer { + target_group_arn = aws_lb_target_group.api.arn + container_name = "api" + container_port = 8000 + } + + deployment_circuit_breaker { + enable = true + rollback = true + } + + tags = local.common_tags + + depends_on = [aws_lb_listener.https] + + lifecycle { + ignore_changes = [task_definition] + } +} + +resource "aws_ecs_service" "worker" { + name = "worker" + cluster = aws_ecs_cluster.this.id + task_definition = aws_ecs_task_definition.worker.arn + desired_count = var.worker_desired_count + launch_type = "FARGATE" + platform_version = "1.4.0" + enable_execute_command = false + + network_configuration { + assign_public_ip = false + subnets = values(aws_subnet.application)[*].id + security_groups = [aws_security_group.ecs.id] + } + + deployment_circuit_breaker { + enable = true + rollback = true + } + + tags = local.common_tags + + lifecycle { + ignore_changes = [task_definition] + } +} + +resource "aws_appautoscaling_target" "api" { + max_capacity = var.environment == "staging" ? 6 : 2 + min_capacity = var.api_desired_count + resource_id = "service/${aws_ecs_cluster.this.name}/${aws_ecs_service.api.name}" + scalable_dimension = "ecs:service:DesiredCount" + service_namespace = "ecs" +} + +resource "aws_appautoscaling_policy" "api_cpu" { + name = "${local.name}-api-cpu" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.api.resource_id + scalable_dimension = aws_appautoscaling_target.api.scalable_dimension + service_namespace = aws_appautoscaling_target.api.service_namespace + + target_tracking_scaling_policy_configuration { + predefined_metric_specification { + predefined_metric_type = "ECSServiceAverageCPUUtilization" + } + target_value = 60 + scale_in_cooldown = 300 + scale_out_cooldown = 60 + } +} + +resource "aws_appautoscaling_target" "worker" { + max_capacity = var.environment == "staging" ? 8 : 2 + min_capacity = var.worker_desired_count + resource_id = "service/${aws_ecs_cluster.this.name}/${aws_ecs_service.worker.name}" + scalable_dimension = "ecs:service:DesiredCount" + service_namespace = "ecs" +} + +resource "aws_appautoscaling_policy" "worker_queue" { + name = "${local.name}-worker-queue" + policy_type = "TargetTrackingScaling" + resource_id = aws_appautoscaling_target.worker.resource_id + scalable_dimension = aws_appautoscaling_target.worker.scalable_dimension + service_namespace = aws_appautoscaling_target.worker.service_namespace + + target_tracking_scaling_policy_configuration { + customized_metric_specification { + namespace = "AWS/SQS" + metric_name = "ApproximateNumberOfMessagesVisible" + statistic = "Average" + unit = "Count" + dimensions { + name = "QueueName" + value = aws_sqs_queue.pipeline.name + } + } + target_value = 5 + scale_in_cooldown = 300 + scale_out_cooldown = 60 + } +} diff --git a/infra/terraform/modules/platform/edge.tf b/infra/terraform/modules/platform/edge.tf new file mode 100644 index 0000000..18ad6ec --- /dev/null +++ b/infra/terraform/modules/platform/edge.tf @@ -0,0 +1,234 @@ +resource "aws_cloudfront_origin_access_control" "frontend" { + name = "${local.name}-frontend" + description = "SigV4-only access to the private frontend bucket" + origin_access_control_origin_type = "s3" + signing_behavior = "always" + signing_protocol = "sigv4" +} + +resource "aws_cloudfront_response_headers_policy" "security" { + name = "${local.name}-security-headers" + + security_headers_config { + content_type_options { override = true } + frame_options { + frame_option = "DENY" + override = true + } + referrer_policy { + referrer_policy = "strict-origin-when-cross-origin" + override = true + } + strict_transport_security { + access_control_max_age_sec = 31536000 + include_subdomains = true + preload = true + override = true + } + xss_protection { + protection = true + mode_block = true + override = true + } + } +} + +resource "aws_cloudfront_distribution" "frontend" { + enabled = true + is_ipv6_enabled = true + comment = "${local.name} private frontend" + price_class = "PriceClass_100" + http_version = "http2and3" + + origin { + domain_name = aws_s3_bucket.data["frontend"].bucket_regional_domain_name + origin_id = "private-frontend" + origin_access_control_id = aws_cloudfront_origin_access_control.frontend.id + } + + default_cache_behavior { + allowed_methods = ["GET", "HEAD", "OPTIONS"] + cached_methods = ["GET", "HEAD", "OPTIONS"] + target_origin_id = "private-frontend" + viewer_protocol_policy = "redirect-to-https" + compress = true + + cache_policy_id = "658327ea-f89d-4fab-a63d-7e88639e58f6" + response_headers_policy_id = aws_cloudfront_response_headers_policy.security.id + } + + custom_error_response { + error_code = 403 + response_code = 200 + response_page_path = "/index.html" + error_caching_min_ttl = 10 + } + + custom_error_response { + error_code = 404 + response_code = 200 + response_page_path = "/index.html" + error_caching_min_ttl = 10 + } + + restrictions { + geo_restriction { + restriction_type = "none" + } + } + + viewer_certificate { + cloudfront_default_certificate = true + minimum_protocol_version = "TLSv1.2_2021" + } + + tags = merge(local.common_tags, { + FrontendBuildImageDigest = var.frontend_origin_image_digest + }) +} + +resource "aws_wafv2_web_acl" "api" { + name = "${local.name}-api" + description = "Managed-rule and bounded-rate protection for the API ALB" + scope = "REGIONAL" + + default_action { + allow {} + } + + rule { + name = "aws-common-rules" + priority = 10 + override_action { + none {} + } + statement { + managed_rule_group_statement { + name = "AWSManagedRulesCommonRuleSet" + vendor_name = "AWS" + } + } + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "${local.name}-common" + sampled_requests_enabled = false + } + } + + rule { + name = "known-bad-inputs" + priority = 20 + override_action { + none {} + } + statement { + managed_rule_group_statement { + name = "AWSManagedRulesKnownBadInputsRuleSet" + vendor_name = "AWS" + } + } + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "${local.name}-bad-inputs" + sampled_requests_enabled = false + } + } + + rule { + name = "per-ip-rate-limit" + priority = 30 + action { + block {} + } + statement { + rate_based_statement { + aggregate_key_type = "IP" + limit = 1200 + } + } + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "${local.name}-rate-limit" + sampled_requests_enabled = false + } + } + + visibility_config { + cloudwatch_metrics_enabled = true + metric_name = "${local.name}-api" + sampled_requests_enabled = false + } + + tags = local.common_tags +} + +resource "aws_wafv2_web_acl_association" "api" { + resource_arn = aws_lb.api.arn + web_acl_arn = aws_wafv2_web_acl.api.arn +} + +resource "aws_acm_certificate" "api" { + count = var.domain_name == null ? 0 : 1 + + domain_name = var.domain_name + validation_method = "DNS" + + lifecycle { + create_before_destroy = true + } + + tags = merge(local.common_tags, { Name = var.domain_name }) +} + +resource "aws_route53_record" "certificate_validation" { + for_each = var.manage_dns && var.domain_name != null ? { + for option in aws_acm_certificate.api[0].domain_validation_options : + option.domain_name => { + name = option.resource_record_name + record = option.resource_record_value + type = option.resource_record_type + } + } : {} + + zone_id = var.hosted_zone_id + name = each.value.name + type = each.value.type + records = [each.value.record] + ttl = 60 +} + +resource "aws_acm_certificate_validation" "api" { + count = var.manage_dns && var.domain_name != null ? 1 : 0 + + certificate_arn = aws_acm_certificate.api[0].arn + validation_record_fqdns = [for record in aws_route53_record.certificate_validation : record.fqdn] +} + +resource "aws_lb_listener" "https" { + count = var.manage_dns && var.domain_name != null ? 1 : 0 + + load_balancer_arn = aws_lb.api.arn + port = 443 + protocol = "HTTPS" + ssl_policy = "ELBSecurityPolicy-TLS13-1-2-2021-06" + certificate_arn = aws_acm_certificate_validation.api[0].certificate_arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.api.arn + } +} + +resource "aws_route53_record" "api" { + count = var.manage_dns && var.domain_name != null ? 1 : 0 + + zone_id = var.hosted_zone_id + name = var.domain_name + type = "A" + + alias { + name = aws_lb.api.dns_name + zone_id = aws_lb.api.zone_id + evaluate_target_health = true + } +} diff --git a/infra/terraform/modules/platform/github_oidc.tf b/infra/terraform/modules/platform/github_oidc.tf new file mode 100644 index 0000000..443a91d --- /dev/null +++ b/infra/terraform/modules/platform/github_oidc.tf @@ -0,0 +1,156 @@ +resource "aws_iam_openid_connect_provider" "github" { + count = var.github_oidc_provider_arn == null ? 1 : 0 + + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = ["6938fd4d98bab03faadb97b34396831e3780aea1"] + + tags = local.common_tags +} + +locals { + github_oidc_provider_arn = coalesce( + var.github_oidc_provider_arn, + try(aws_iam_openid_connect_provider.github[0].arn, null), + ) +} + +data "aws_iam_policy_document" "github_deployment_trust" { + statement { + sid = "GitHubProtectedEnvironmentOnly" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + principals { + type = "Federated" + identifiers = [local.github_oidc_provider_arn] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:aud" + values = ["sts.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "token.actions.githubusercontent.com:sub" + values = [local.repository_environment_subject] + } + } +} + +resource "aws_iam_role" "github_deployment" { + name = "${local.name}-github-deployment" + assume_role_policy = data.aws_iam_policy_document.github_deployment_trust.json + max_session_duration = 3600 + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "github_deployment" { + name = "immutable-artifact-and-service-deployment" + role = aws_iam_role.github_deployment.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AuthenticateToECR" + Effect = "Allow" + Action = ["ecr:GetAuthorizationToken"] + Resource = "*" + }, + { + Sid = "PublishImmutableImages" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:CompleteLayerUpload", + "ecr:GetDownloadUrlForLayer", + "ecr:InitiateLayerUpload", + "ecr:PutImage", + "ecr:UploadLayerPart", + ] + Resource = [aws_ecr_repository.api.arn, aws_ecr_repository.worker.arn] + }, + { + Sid = "InspectAndDeployServices" + Effect = "Allow" + Action = [ + "ecs:DescribeServices", + "ecs:DescribeTaskDefinition", + "ecs:UpdateService", + ] + Resource = [ + aws_ecs_cluster.this.arn, + aws_ecs_service.api.id, + aws_ecs_service.worker.id, + aws_ecs_task_definition.api.arn, + aws_ecs_task_definition.worker.arn, + aws_ecs_task_definition.release_gate.arn, + ] + }, + { + Sid = "RegisterTaggedTaskDefinitions" + Effect = "Allow" + Action = ["ecs:RegisterTaskDefinition"] + Resource = "*" + Condition = { + StringEquals = { + "aws:RequestTag/Environment" = var.environment + "aws:RequestTag/Application" = var.project + } + } + }, + { + Sid = "PassExactTaskRoles" + Effect = "Allow" + Action = ["iam:PassRole"] + Resource = [ + aws_iam_role.ecs_execution.arn, + aws_iam_role.api_task.arn, + aws_iam_role.worker_task.arn, + aws_iam_role.release_gate_task.arn, + ] + }, + { + Sid = "RunReleaseGate" + Effect = "Allow" + Action = ["ecs:RunTask"] + Resource = [aws_ecs_task_definition.release_gate.arn] + Condition = { + ArnEquals = { + "ecs:cluster" = aws_ecs_cluster.this.arn + } + } + }, + { + Sid = "ObserveReleaseGate" + Effect = "Allow" + Action = ["ecs:DescribeTasks"] + Resource = "*" + Condition = { + ArnEquals = { + "ecs:cluster" = aws_ecs_cluster.this.arn + } + } + }, + { + Sid = "PublishFrontend" + Effect = "Allow" + Action = ["s3:DeleteObject", "s3:GetObject", "s3:ListBucket", "s3:PutObject"] + Resource = [ + aws_s3_bucket.data["frontend"].arn, + "${aws_s3_bucket.data["frontend"].arn}/*", + ] + }, + { + Sid = "InvalidateFrontendCache" + Effect = "Allow" + Action = ["cloudfront:CreateInvalidation", "cloudfront:GetDistribution"] + Resource = [aws_cloudfront_distribution.frontend.arn] + }, + ] + }) +} diff --git a/infra/terraform/modules/platform/iam.tf b/infra/terraform/modules/platform/iam.tf new file mode 100644 index 0000000..886b515 --- /dev/null +++ b/infra/terraform/modules/platform/iam.tf @@ -0,0 +1,185 @@ +resource "aws_iam_role" "ecs_execution" { + name_prefix = "${local.name}-ecs-execution-" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "ecs_execution" { + name = "runtime-image-logs-and-secrets" + role = aws_iam_role.ecs_execution.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "PullImages" + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken", + ] + Resource = "*" + }, + { + Sid = "ReadImageLayers" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + Resource = [ + aws_ecr_repository.api.arn, + aws_ecr_repository.worker.arn, + ] + }, + { + Sid = "WriteTaskLogs" + Effect = "Allow" + Action = [ + "logs:CreateLogStream", + "logs:PutLogEvents", + ] + Resource = [ + "${aws_cloudwatch_log_group.api.arn}:*", + "${aws_cloudwatch_log_group.worker.arn}:*", + ] + }, + { + Sid = "ReadDatabaseSecret" + Effect = "Allow" + Action = ["secretsmanager:GetSecretValue"] + Resource = [aws_db_instance.postgresql.master_user_secret[0].secret_arn] + }, + { + Sid = "DecryptRuntimeSecrets" + Effect = "Allow" + Action = ["kms:Decrypt"] + Resource = [aws_kms_key.platform.arn] + }, + ] + }) +} + +resource "aws_iam_role" "api_task" { + name_prefix = "${local.name}-api-task-" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "ecs-tasks.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "api_task" { + name = "curated-export-and-queue-access" + role = aws_iam_role.api_task.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "ListCuratedAndExports" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = [aws_s3_bucket.data["curated"].arn, aws_s3_bucket.data["exports"].arn] + }, + { + Sid = "ReadCuratedObjects" + Effect = "Allow" + Action = ["s3:GetObject"] + Resource = ["${aws_s3_bucket.data["curated"].arn}/*"] + }, + { + Sid = "ReadWriteExports" + Effect = "Allow" + Action = ["s3:GetObject", "s3:PutObject"] + Resource = ["${aws_s3_bucket.data["exports"].arn}/*"] + }, + { + Sid = "EnqueuePipelineJobs" + Effect = "Allow" + Action = ["sqs:GetQueueAttributes", "sqs:SendMessage"] + Resource = [aws_sqs_queue.pipeline.arn] + }, + { + Sid = "UseDataKey" + Effect = "Allow" + Action = ["kms:Decrypt", "kms:Encrypt", "kms:GenerateDataKey"] + Resource = [aws_kms_key.platform.arn] + }, + ] + }) +} + +resource "aws_iam_role" "worker_task" { + name_prefix = "${local.name}-worker-task-" + + assume_role_policy = aws_iam_role.api_task.assume_role_policy + tags = local.common_tags +} + +resource "aws_iam_role" "release_gate_task" { + name_prefix = "${local.name}-release-gate-task-" + + assume_role_policy = aws_iam_role.api_task.assume_role_policy + tags = local.common_tags +} + +resource "aws_iam_role_policy" "worker_task" { + name = "pipeline-storage-and-queue-access" + role = aws_iam_role.worker_task.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "ListPipelineBuckets" + Effect = "Allow" + Action = ["s3:ListBucket"] + Resource = [for name, bucket in aws_s3_bucket.data : bucket.arn if contains(local.pipeline_storage_classes, name)] + }, + { + Sid = "ReadWriteVersionedPipelineObjects" + Effect = "Allow" + Action = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + ] + Resource = [for name, bucket in aws_s3_bucket.data : "${bucket.arn}/*" if contains(local.pipeline_storage_classes, name)] + }, + { + Sid = "ConsumePipelineQueue" + Effect = "Allow" + Action = [ + "sqs:ChangeMessageVisibility", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:ReceiveMessage", + "sqs:SendMessage", + ] + Resource = [aws_sqs_queue.pipeline.arn, aws_sqs_queue.pipeline_dlq.arn] + }, + { + Sid = "UseDataKey" + Effect = "Allow" + Action = ["kms:Decrypt", "kms:Encrypt", "kms:GenerateDataKey"] + Resource = [aws_kms_key.platform.arn] + }, + ] + }) +} diff --git a/infra/terraform/modules/platform/kms.tf b/infra/terraform/modules/platform/kms.tf new file mode 100644 index 0000000..e1cdbfd --- /dev/null +++ b/infra/terraform/modules/platform/kms.tf @@ -0,0 +1,60 @@ +data "aws_caller_identity" "current" {} + +data "aws_partition" "current" {} + +resource "aws_kms_key" "platform" { + description = "${local.name} application data encryption" + enable_key_rotation = true + deletion_window_in_days = 30 + + tags = merge(local.common_tags, { Name = "${local.name}-data" }) +} + +resource "aws_kms_alias" "platform" { + name = "alias/${local.name}-data" + target_key_id = aws_kms_key.platform.key_id +} + +resource "aws_kms_key" "logs" { + description = "${local.name} CloudWatch log encryption" + enable_key_rotation = true + deletion_window_in_days = 30 + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AccountAdministration" + Effect = "Allow" + Principal = { AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" } + Action = "kms:*" + Resource = "*" + }, + { + Sid = "CloudWatchLogsEncryption" + Effect = "Allow" + Principal = { Service = "logs.${var.aws_region}.amazonaws.com" } + Action = [ + "kms:Decrypt", + "kms:Encrypt", + "kms:GenerateDataKey*", + "kms:DescribeKey", + "kms:ReEncrypt*", + ] + Resource = "*" + Condition = { + ArnLike = { + "kms:EncryptionContext:aws:logs:arn" = "arn:${data.aws_partition.current.partition}:logs:${var.aws_region}:${data.aws_caller_identity.current.account_id}:log-group:/foundation-intelligence/${var.environment}/*" + } + } + }, + ] + }) + + tags = merge(local.common_tags, { Name = "${local.name}-logs" }) +} + +resource "aws_kms_alias" "logs" { + name = "alias/${local.name}-logs" + target_key_id = aws_kms_key.logs.key_id +} diff --git a/infra/terraform/modules/platform/kms_policy.tf b/infra/terraform/modules/platform/kms_policy.tf new file mode 100644 index 0000000..65f5b44 --- /dev/null +++ b/infra/terraform/modules/platform/kms_policy.tf @@ -0,0 +1,46 @@ +resource "aws_kms_key_policy" "platform" { + key_id = aws_kms_key.platform.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "AccountAdministration" + Effect = "Allow" + Principal = { AWS = "arn:${data.aws_partition.current.partition}:iam::${data.aws_caller_identity.current.account_id}:root" } + Action = "kms:*" + Resource = "*" + }, + { + Sid = "CloudFrontReadFrontendObjects" + Effect = "Allow" + Principal = { Service = "cloudfront.amazonaws.com" } + Action = [ + "kms:Decrypt", + "kms:DescribeKey", + ] + Resource = "*" + Condition = { + StringEquals = { + "AWS:SourceArn" = aws_cloudfront_distribution.frontend.arn + } + } + }, + { + Sid = "OperationalNotificationEncryption" + Effect = "Allow" + Principal = { Service = "sns.amazonaws.com" } + Action = [ + "kms:Decrypt", + "kms:GenerateDataKey*", + ] + Resource = "*" + Condition = { + StringEquals = { + "AWS:SourceAccount" = data.aws_caller_identity.current.account_id + } + } + }, + ] + }) +} diff --git a/infra/terraform/modules/platform/locals.tf b/infra/terraform/modules/platform/locals.tf new file mode 100644 index 0000000..74789bf --- /dev/null +++ b/infra/terraform/modules/platform/locals.tf @@ -0,0 +1,42 @@ +locals { + name = "${var.project}-${var.environment}" + + common_tags = merge(var.tags, { + Application = var.project + Environment = var.environment + ManagedBy = "terraform" + DataClass = "foundation-intelligence" + }) + + public_subnets = { + for index, cidr in var.public_subnet_cidrs : tostring(index) => { + cidr = cidr + az = var.availability_zones[index] + } + } + application_subnets = { + for index, cidr in var.application_subnet_cidrs : tostring(index) => { + cidr = cidr + az = var.availability_zones[index] + } + } + database_subnets = { + for index, cidr in var.database_subnet_cidrs : tostring(index) => { + cidr = cidr + az = var.availability_zones[index] + } + } + + nat_gateway_keys = var.single_nat_gateway ? ["0"] : sort(keys(local.public_subnets)) + storage_classes = toset(["raw", "validated", "curated", "exports", "frontend"]) + pipeline_storage_classes = toset(["raw", "validated", "curated", "exports"]) + versioned_storage_classes = toset([ + "raw", + "validated", + "curated", + "exports", + ]) + + repository_environment_subject = "repo:${var.github_repository}:environment:${var.environment}" + schedule_state = var.enable_schedules ? "ENABLED" : "DISABLED" +} diff --git a/infra/terraform/modules/platform/network.tf b/infra/terraform/modules/platform/network.tf new file mode 100644 index 0000000..a7d89b4 --- /dev/null +++ b/infra/terraform/modules/platform/network.tf @@ -0,0 +1,200 @@ +resource "aws_vpc" "this" { + cidr_block = var.vpc_cidr + enable_dns_hostnames = true + enable_dns_support = true + + tags = merge(local.common_tags, { Name = local.name }) + + lifecycle { + precondition { + condition = ( + length(var.public_subnet_cidrs) == length(var.availability_zones) && + length(var.application_subnet_cidrs) == length(var.availability_zones) && + length(var.database_subnet_cidrs) == length(var.availability_zones) + ) + error_message = "Every availability zone must have public, application and database subnets." + } + precondition { + condition = ( + !var.manage_dns || (var.domain_name != null && var.hosted_zone_id != null) + ) + error_message = "manage_dns requires an approved domain_name and hosted_zone_id." + } + } +} + +resource "aws_internet_gateway" "this" { + vpc_id = aws_vpc.this.id + tags = merge(local.common_tags, { Name = "${local.name}-igw" }) +} + +resource "aws_subnet" "public" { + for_each = local.public_subnets + + vpc_id = aws_vpc.this.id + availability_zone = each.value.az + cidr_block = each.value.cidr + map_public_ip_on_launch = false + + tags = merge(local.common_tags, { + Name = "${local.name}-public-${each.key}" + Tier = "public-load-balancer" + }) +} + +resource "aws_subnet" "application" { + for_each = local.application_subnets + + vpc_id = aws_vpc.this.id + availability_zone = each.value.az + cidr_block = each.value.cidr + map_public_ip_on_launch = false + + tags = merge(local.common_tags, { + Name = "${local.name}-application-${each.key}" + Tier = "private-application" + }) +} + +resource "aws_subnet" "database" { + for_each = local.database_subnets + + vpc_id = aws_vpc.this.id + availability_zone = each.value.az + cidr_block = each.value.cidr + map_public_ip_on_launch = false + + tags = merge(local.common_tags, { + Name = "${local.name}-database-${each.key}" + Tier = "private-database" + }) +} + +resource "aws_eip" "nat" { + for_each = toset(local.nat_gateway_keys) + domain = "vpc" + + tags = merge(local.common_tags, { Name = "${local.name}-nat-${each.key}" }) + + depends_on = [aws_internet_gateway.this] +} + +resource "aws_nat_gateway" "this" { + for_each = toset(local.nat_gateway_keys) + + allocation_id = aws_eip.nat[each.key].id + subnet_id = aws_subnet.public[each.key].id + + tags = merge(local.common_tags, { Name = "${local.name}-nat-${each.key}" }) + depends_on = [aws_internet_gateway.this] +} + +resource "aws_route_table" "public" { + vpc_id = aws_vpc.this.id + tags = merge(local.common_tags, { Name = "${local.name}-public" }) +} + +resource "aws_route" "public_internet" { + route_table_id = aws_route_table.public.id + destination_cidr_block = "0.0.0.0/0" + gateway_id = aws_internet_gateway.this.id +} + +resource "aws_route_table_association" "public" { + for_each = aws_subnet.public + + subnet_id = each.value.id + route_table_id = aws_route_table.public.id +} + +resource "aws_route_table" "application" { + for_each = aws_subnet.application + + vpc_id = aws_vpc.this.id + tags = merge(local.common_tags, { Name = "${local.name}-application-${each.key}" }) +} + +resource "aws_route" "application_egress" { + for_each = aws_route_table.application + + route_table_id = each.value.id + destination_cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.this[ + var.single_nat_gateway ? "0" : each.key + ].id +} + +resource "aws_route_table_association" "application" { + for_each = aws_subnet.application + + subnet_id = each.value.id + route_table_id = aws_route_table.application[each.key].id +} + +resource "aws_route_table" "database" { + for_each = aws_subnet.database + + vpc_id = aws_vpc.this.id + tags = merge(local.common_tags, { Name = "${local.name}-database-${each.key}" }) +} + +resource "aws_route_table_association" "database" { + for_each = aws_subnet.database + + subnet_id = each.value.id + route_table_id = aws_route_table.database[each.key].id +} + +resource "aws_vpc_endpoint" "s3" { + vpc_id = aws_vpc.this.id + service_name = "com.amazonaws.${var.aws_region}.s3" + vpc_endpoint_type = "Gateway" + route_table_ids = values(aws_route_table.application)[*].id + + tags = merge(local.common_tags, { Name = "${local.name}-s3" }) +} + +resource "aws_security_group" "endpoints" { + count = var.enable_interface_endpoints ? 1 : 0 + + name_prefix = "${local.name}-endpoints-" + description = "TLS from private application tasks to VPC endpoints" + vpc_id = aws_vpc.this.id + + ingress { + description = "TLS from ECS tasks" + protocol = "tcp" + from_port = 443 + to_port = 443 + security_groups = [aws_security_group.ecs.id] + } + + egress { + description = "Endpoint response traffic" + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = [var.vpc_cidr] + } + + tags = merge(local.common_tags, { Name = "${local.name}-endpoints" }) +} + +resource "aws_vpc_endpoint" "interface" { + for_each = var.enable_interface_endpoints ? toset([ + "ecr.api", + "ecr.dkr", + "logs", + "secretsmanager", + "sqs", + ]) : toset([]) + + vpc_id = aws_vpc.this.id + service_name = "com.amazonaws.${var.aws_region}.${each.value}" + vpc_endpoint_type = "Interface" + private_dns_enabled = true + subnet_ids = values(aws_subnet.application)[*].id + security_group_ids = [aws_security_group.endpoints[0].id] + + tags = merge(local.common_tags, { Name = "${local.name}-${replace(each.value, ".", "-")}" }) +} diff --git a/infra/terraform/modules/platform/observability.tf b/infra/terraform/modules/platform/observability.tf new file mode 100644 index 0000000..dc93c7d --- /dev/null +++ b/infra/terraform/modules/platform/observability.tf @@ -0,0 +1,302 @@ +locals { + custom_alarms = { + readiness-failure = { + metric = "readiness_success" + comparison = "LessThanThreshold" + threshold = 1 + periods = 2 + period = 60 + statistic = "Minimum" + runbook = "readiness-failure" + } + api-5xx-budget = { + metric = "api_errors_total" + comparison = "GreaterThanThreshold" + threshold = 5 + periods = 5 + period = 60 + statistic = "Sum" + runbook = "bad-deployment" + } + api-latency-budget = { + metric = "api_request_duration_ms" + comparison = "GreaterThanThreshold" + threshold = 1000 + periods = 5 + period = 60 + statistic = "p95" + runbook = "bad-deployment" + } + stale-data = { + metric = "dataset_age_seconds" + comparison = "GreaterThanThreshold" + threshold = 691200 + periods = 2 + period = 3600 + statistic = "Maximum" + runbook = "stale-dataset" + } + pipeline-failure = { + metric = "pipeline_failures_total" + comparison = "GreaterThanThreshold" + threshold = 0 + periods = 1 + period = 300 + statistic = "Sum" + runbook = "ingestion-failure" + } + reconciliation-mismatch = { + metric = "reconciliation_mismatch_count" + comparison = "GreaterThanThreshold" + threshold = 0 + periods = 1 + period = 300 + statistic = "Maximum" + runbook = "migration-failure" + } + conversion-gap-increase = { + metric = "conversion_gap_count" + comparison = "GreaterThanThreshold" + threshold = 432 + periods = 1 + period = 3600 + statistic = "Maximum" + runbook = "stale-dataset" + } + programme-coverage-decrease = { + metric = "programme_coverage_ratio" + comparison = "LessThanThreshold" + threshold = 0.44 + periods = 1 + period = 3600 + statistic = "Minimum" + runbook = "stale-dataset" + } + geography-coverage-decrease = { + metric = "geography_coverage_ratio" + comparison = "LessThanThreshold" + threshold = 0.34 + periods = 1 + period = 3600 + statistic = "Minimum" + runbook = "stale-dataset" + } + cost-threshold = { + metric = "estimated_cost_usd" + comparison = "GreaterThanThreshold" + threshold = 500 + periods = 1 + period = 86400 + statistic = "Maximum" + runbook = "cost-spike" + } + } + + rds_alarms = { + rds-cpu = { + metric = "CPUUtilization" + comparison = "GreaterThanThreshold" + threshold = 80 + periods = 5 + period = 60 + statistic = "Average" + } + rds-connections = { + metric = "DatabaseConnections" + comparison = "GreaterThanThreshold" + threshold = 80 + periods = 5 + period = 60 + statistic = "Average" + } + rds-storage = { + metric = "FreeStorageSpace" + comparison = "LessThanThreshold" + threshold = 10737418240 + periods = 3 + period = 300 + statistic = "Minimum" + } + } +} + +resource "aws_sns_topic" "operations" { + name = "${local.name}-operations" + kms_master_key_id = aws_kms_key.platform.id + + tags = local.common_tags + + depends_on = [aws_kms_key_policy.platform] +} + +resource "aws_sns_topic_subscription" "email" { + count = var.notification_email == null ? 0 : 1 + + topic_arn = aws_sns_topic.operations.arn + protocol = "email" + endpoint = var.notification_email +} + +resource "aws_cloudwatch_metric_alarm" "custom" { + for_each = local.custom_alarms + + alarm_name = "${local.name}-${each.key}" + alarm_description = "Runbook: docs/remediation/observability-runbooks.md#${each.value.runbook}" + namespace = "FoundationIntelligence" + metric_name = each.value.metric + comparison_operator = each.value.comparison + threshold = each.value.threshold + evaluation_periods = each.value.periods + period = each.value.period + statistic = contains(["p90", "p95", "p99"], each.value.statistic) ? null : each.value.statistic + extended_statistic = contains(["p90", "p95", "p99"], each.value.statistic) ? each.value.statistic : null + treat_missing_data = "breaching" + alarm_actions = [aws_sns_topic.operations.arn] + ok_actions = [aws_sns_topic.operations.arn] + + dimensions = { + Service = "foundation-intelligence-api" + Environment = var.environment + } + + tags = local.common_tags +} + +resource "aws_cloudwatch_metric_alarm" "queue_backlog" { + alarm_name = "${local.name}-queue-backlog" + alarm_description = "Runbook: docs/remediation/observability-runbooks.md#queue-backlog" + namespace = "AWS/SQS" + metric_name = "ApproximateAgeOfOldestMessage" + comparison_operator = "GreaterThanThreshold" + threshold = 900 + evaluation_periods = 3 + period = 60 + statistic = "Maximum" + treat_missing_data = "notBreaching" + alarm_actions = [aws_sns_topic.operations.arn] + + dimensions = { QueueName = aws_sqs_queue.pipeline.name } + tags = local.common_tags +} + +resource "aws_cloudwatch_metric_alarm" "dlq" { + alarm_name = "${local.name}-dlq-messages" + alarm_description = "Runbook: docs/remediation/observability-runbooks.md#dlq-replay" + namespace = "AWS/SQS" + metric_name = "ApproximateNumberOfMessagesVisible" + comparison_operator = "GreaterThanThreshold" + threshold = 0 + evaluation_periods = 1 + period = 60 + statistic = "Maximum" + treat_missing_data = "notBreaching" + alarm_actions = [aws_sns_topic.operations.arn] + + dimensions = { QueueName = aws_sqs_queue.pipeline_dlq.name } + tags = local.common_tags +} + +resource "aws_cloudwatch_metric_alarm" "rds" { + for_each = local.rds_alarms + + alarm_name = "${local.name}-${each.key}" + alarm_description = "Runbook: docs/remediation/observability-runbooks.md#rds-outage" + namespace = "AWS/RDS" + metric_name = each.value.metric + comparison_operator = each.value.comparison + threshold = each.value.threshold + evaluation_periods = each.value.periods + period = each.value.period + statistic = each.value.statistic + treat_missing_data = "breaching" + alarm_actions = [aws_sns_topic.operations.arn] + + dimensions = { DBInstanceIdentifier = aws_db_instance.postgresql.id } + tags = local.common_tags +} + +resource "aws_cloudwatch_dashboard" "platform" { + dashboard_name = local.name + + dashboard_body = jsonencode({ + widgets = [ + { + type = "metric" + x = 0 + y = 0 + width = 12 + height = 6 + properties = { + title = "API readiness, errors and latency" + region = var.aws_region + metrics = [ + ["FoundationIntelligence", "readiness_success", "Environment", var.environment], + [".", "api_errors_total", ".", "."], + [".", "api_request_duration_ms", ".", ".", { stat = "p95" }], + ] + } + }, + { + type = "metric" + x = 12 + y = 0 + width = 12 + height = 6 + properties = { + title = "RDS health" + region = var.aws_region + metrics = [ + ["AWS/RDS", "CPUUtilization", "DBInstanceIdentifier", aws_db_instance.postgresql.id], + [".", "DatabaseConnections", ".", "."], + [".", "FreeStorageSpace", ".", "."], + ] + } + }, + { + type = "metric" + x = 0 + y = 6 + width = 24 + height = 6 + properties = { + title = "Pipeline queue and DLQ" + region = var.aws_region + metrics = [ + ["AWS/SQS", "ApproximateAgeOfOldestMessage", "QueueName", aws_sqs_queue.pipeline.name], + [".", "ApproximateNumberOfMessagesVisible", ".", aws_sqs_queue.pipeline.name], + [".", ".", ".", aws_sqs_queue.pipeline_dlq.name], + ] + } + }, + ] + }) +} + +resource "aws_budgets_budget" "monthly" { + name = "${local.name}-monthly" + budget_type = "COST" + limit_amount = tostring(var.monthly_budget_usd) + limit_unit = "USD" + time_unit = "MONTHLY" + + cost_filter { + name = "TagKeyValue" + values = [format("user:Environment$%s", var.environment)] + } + + notification { + comparison_operator = "GREATER_THAN" + threshold = 80 + threshold_type = "PERCENTAGE" + notification_type = "FORECASTED" + subscriber_sns_topic_arns = [aws_sns_topic.operations.arn] + } + + notification { + comparison_operator = "GREATER_THAN" + threshold = 100 + threshold_type = "PERCENTAGE" + notification_type = "ACTUAL" + subscriber_sns_topic_arns = [aws_sns_topic.operations.arn] + } +} diff --git a/infra/terraform/modules/platform/orchestration.tf b/infra/terraform/modules/platform/orchestration.tf new file mode 100644 index 0000000..d0139f7 --- /dev/null +++ b/infra/terraform/modules/platform/orchestration.tf @@ -0,0 +1,237 @@ +resource "aws_sqs_queue" "pipeline_dlq" { + name = "${local.name}-pipeline-dlq.fifo" + fifo_queue = true + content_based_deduplication = false + kms_master_key_id = aws_kms_key.platform.arn + kms_data_key_reuse_period_seconds = 300 + message_retention_seconds = 1209600 + visibility_timeout_seconds = 900 + + tags = local.common_tags +} + +resource "aws_sqs_queue" "pipeline" { + name = "${local.name}-pipeline.fifo" + fifo_queue = true + content_based_deduplication = false + deduplication_scope = "messageGroup" + fifo_throughput_limit = "perMessageGroupId" + kms_master_key_id = aws_kms_key.platform.arn + kms_data_key_reuse_period_seconds = 300 + message_retention_seconds = 345600 + visibility_timeout_seconds = 900 + receive_wait_time_seconds = 20 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.pipeline_dlq.arn + maxReceiveCount = 5 + }) + + tags = local.common_tags +} + +resource "aws_sqs_queue_redrive_allow_policy" "pipeline_dlq" { + queue_url = aws_sqs_queue.pipeline_dlq.id + redrive_allow_policy = jsonencode({ + redrivePermission = "byQueue" + sourceQueueArns = [aws_sqs_queue.pipeline.arn] + }) +} + +resource "aws_cloudwatch_log_group" "step_functions" { + name = "/foundation-intelligence/${var.environment}/step-functions" + retention_in_days = var.environment == "staging" ? 90 : 30 + kms_key_id = aws_kms_key.logs.arn + + tags = local.common_tags +} + +resource "aws_iam_role" "step_functions" { + name_prefix = "${local.name}-step-functions-" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "states.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "step_functions" { + name = "run-bounded-worker-task" + role = aws_iam_role.step_functions.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "RunWorkerTask" + Effect = "Allow" + Action = ["ecs:RunTask"] + Resource = [aws_ecs_task_definition.worker.arn] + Condition = { + ArnEquals = { + "ecs:cluster" = aws_ecs_cluster.this.arn + } + } + }, + { + Sid = "PassWorkerRoles" + Effect = "Allow" + Action = ["iam:PassRole"] + Resource = [aws_iam_role.ecs_execution.arn, aws_iam_role.worker_task.arn] + }, + { + Sid = "ManageSynchronousTaskEvents" + Effect = "Allow" + Action = [ + "events:PutRule", + "events:PutTargets", + "events:DescribeRule", + ] + Resource = [ + "arn:${data.aws_partition.current.partition}:events:${var.aws_region}:${data.aws_caller_identity.current.account_id}:rule/StepFunctionsGetEventsForECSTaskRule" + ] + }, + { + Sid = "ManageSynchronousWorkerTask" + Effect = "Allow" + Action = ["ecs:DescribeTasks", "ecs:StopTask"] + Resource = "*" + Condition = { + ArnEquals = { + "ecs:cluster" = aws_ecs_cluster.this.arn + } + } + }, + { + Sid = "WriteExecutionLogs" + Effect = "Allow" + Action = [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + "logs:DescribeLogGroups", + ] + Resource = "*" + }, + ] + }) +} + +resource "aws_sfn_state_machine" "pipeline" { + name = "${local.name}-pipeline" + role_arn = aws_iam_role.step_functions.arn + type = "STANDARD" + + definition = jsonencode({ + Comment = "Run one immutable worker task; durable job state remains PostgreSQL/SQS owned." + StartAt = "RunWorker" + States = { + RunWorker = { + Type = "Task" + Resource = "arn:${data.aws_partition.current.partition}:states:::ecs:runTask.sync" + Parameters = { + LaunchType = "FARGATE" + Cluster = aws_ecs_cluster.this.arn + TaskDefinition = aws_ecs_task_definition.worker.arn + PlatformVersion = "1.4.0" + EnableExecuteCommand = false + NetworkConfiguration = { + AwsvpcConfiguration = { + AssignPublicIp = "DISABLED" + Subnets = values(aws_subnet.application)[*].id + SecurityGroups = [aws_security_group.ecs.id] + } + } + } + TimeoutSeconds = 7200 + Retry = [{ + ErrorEquals = ["ECS.AmazonECSException", "ECS.ThrottlingException", "States.Timeout"] + IntervalSeconds = 30 + BackoffRate = 2 + MaxAttempts = 3 + }] + End = true + } + } + }) + + logging_configuration { + include_execution_data = false + level = "ERROR" + log_destination = "${aws_cloudwatch_log_group.step_functions.arn}:*" + } + + tags = local.common_tags +} + +resource "aws_iam_role" "scheduler" { + name_prefix = "${local.name}-scheduler-" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { Service = "scheduler.amazonaws.com" } + Action = "sts:AssumeRole" + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy" "scheduler" { + name = "start-pipeline-state-machine" + role = aws_iam_role.scheduler.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Action = ["states:StartExecution"] + Resource = aws_sfn_state_machine.pipeline.arn + }] + }) +} + +resource "aws_scheduler_schedule_group" "pipeline" { + name = "${local.name}-pipeline" + tags = local.common_tags +} + +resource "aws_scheduler_schedule" "weekly_pipeline" { + name = "weekly-curation" + group_name = aws_scheduler_schedule_group.pipeline.name + state = local.schedule_state + + flexible_time_window { + mode = "OFF" + } + + schedule_expression = "cron(0 2 ? * SUN *)" + schedule_expression_timezone = "UTC" + + target { + arn = aws_sfn_state_machine.pipeline.arn + role_arn = aws_iam_role.scheduler.arn + input = jsonencode({ + job_type = "scheduled_curation" + source = "governance-gated" + }) + + retry_policy { + maximum_event_age_in_seconds = 3600 + maximum_retry_attempts = 2 + } + + } +} diff --git a/infra/terraform/modules/platform/outputs.tf b/infra/terraform/modules/platform/outputs.tf new file mode 100644 index 0000000..29a67d5 --- /dev/null +++ b/infra/terraform/modules/platform/outputs.tf @@ -0,0 +1,54 @@ +output "vpc_id" { + value = aws_vpc.this.id +} + +output "api_load_balancer_dns_name" { + value = aws_lb.api.dns_name +} + +output "frontend_distribution_domain_name" { + value = aws_cloudfront_distribution.frontend.domain_name +} + +output "database_endpoint" { + value = aws_db_instance.postgresql.endpoint + sensitive = true +} + +output "database_secret_arn" { + value = aws_db_instance.postgresql.master_user_secret[0].secret_arn + sensitive = true +} + +output "storage_bucket_arns" { + value = { for name, bucket in aws_s3_bucket.data : name => bucket.arn } +} + +output "api_repository_url" { + value = aws_ecr_repository.api.repository_url +} + +output "worker_repository_url" { + value = aws_ecr_repository.worker.repository_url +} + +output "deployment_role_arn" { + value = aws_iam_role.github_deployment.arn +} + +output "schedules_enabled" { + value = var.enable_schedules +} + +output "release_gate" { + value = { + cluster_name = aws_ecs_cluster.this.name + task_definition_arn = aws_ecs_task_definition.release_gate.arn + application_subnets = values(aws_subnet.application)[*].id + security_group_id = aws_security_group.ecs.id + frontend_bucket = aws_s3_bucket.data["frontend"].id + cloudfront_id = aws_cloudfront_distribution.frontend.id + api_task_definition = aws_ecs_task_definition.api.arn + worker_task_definition = aws_ecs_task_definition.worker.arn + } +} diff --git a/infra/terraform/modules/platform/rds.tf b/infra/terraform/modules/platform/rds.tf new file mode 100644 index 0000000..125fd99 --- /dev/null +++ b/infra/terraform/modules/platform/rds.tf @@ -0,0 +1,106 @@ +resource "aws_db_subnet_group" "postgresql" { + name = "${local.name}-postgresql" + subnet_ids = values(aws_subnet.database)[*].id + + tags = merge(local.common_tags, { Name = "${local.name}-postgresql" }) +} + +resource "aws_db_parameter_group" "postgresql" { + name_prefix = "${local.name}-postgresql16-" + family = "postgres16" + + parameter { + name = "log_connections" + value = "1" + } + + parameter { + name = "log_disconnections" + value = "1" + } + + parameter { + name = "log_min_duration_statement" + value = "1000" + } + + parameter { + name = "rds.force_ssl" + value = "1" + } + + tags = merge(local.common_tags, { Name = "${local.name}-postgresql16" }) +} + +resource "aws_iam_role" "rds_monitoring" { + name_prefix = "${local.name}-rds-monitoring-" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Effect = "Allow" + Principal = { + Service = "monitoring.rds.amazonaws.com" + } + Action = "sts:AssumeRole" + }] + }) + + tags = local.common_tags +} + +resource "aws_iam_role_policy_attachment" "rds_monitoring" { + role = aws_iam_role.rds_monitoring.name + policy_arn = "arn:${data.aws_partition.current.partition}:iam::aws:policy/service-role/AmazonRDSEnhancedMonitoringRole" +} + +resource "aws_db_instance" "postgresql" { + identifier = "${local.name}-postgresql" + + engine = "postgres" + engine_version = "16.4" + instance_class = var.rds_instance_class + allocated_storage = var.rds_allocated_storage_gib + max_allocated_storage = var.rds_max_storage_gib + storage_type = "gp3" + storage_encrypted = true + kms_key_id = aws_kms_key.platform.arn + db_name = "foundation_intelligence" + username = "foundation_admin" + manage_master_user_password = true + master_user_secret_kms_key_id = aws_kms_key.platform.arn + + db_subnet_group_name = aws_db_subnet_group.postgresql.name + parameter_group_name = aws_db_parameter_group.postgresql.name + vpc_security_group_ids = [aws_security_group.rds.id] + publicly_accessible = false + multi_az = var.rds_multi_az + + backup_retention_period = var.rds_backup_retention_days + backup_window = "02:00-03:00" + maintenance_window = "sun:03:30-sun:04:30" + copy_tags_to_snapshot = true + deletion_protection = true + skip_final_snapshot = false + final_snapshot_identifier = "${local.name}-final" + + auto_minor_version_upgrade = true + enabled_cloudwatch_logs_exports = ["postgresql", "upgrade"] + monitoring_interval = 60 + monitoring_role_arn = aws_iam_role.rds_monitoring.arn + performance_insights_enabled = true + performance_insights_kms_key_id = aws_kms_key.platform.arn + performance_insights_retention_period = 7 + + apply_immediately = false + + tags = merge(local.common_tags, { + Name = "${local.name}-postgresql" + BackupRetention = tostring(var.rds_backup_retention_days) + DeletionProtected = "true" + }) + + lifecycle { + prevent_destroy = true + } +} diff --git a/infra/terraform/modules/platform/security.tf b/infra/terraform/modules/platform/security.tf new file mode 100644 index 0000000..6bad8b0 --- /dev/null +++ b/infra/terraform/modules/platform/security.tf @@ -0,0 +1,86 @@ +resource "aws_security_group" "alb" { + name_prefix = "${local.name}-alb-" + description = "Public HTTPS ingress to the application load balancer" + vpc_id = aws_vpc.this.id + + tags = merge(local.common_tags, { Name = "${local.name}-alb" }) +} + +resource "aws_security_group" "ecs" { + name_prefix = "${local.name}-ecs-" + description = "Private API and worker tasks" + vpc_id = aws_vpc.this.id + + tags = merge(local.common_tags, { Name = "${local.name}-ecs" }) +} + +resource "aws_security_group" "rds" { + name_prefix = "${local.name}-rds-" + description = "Private PostgreSQL from application tasks only" + vpc_id = aws_vpc.this.id + + tags = merge(local.common_tags, { Name = "${local.name}-rds" }) +} + +resource "aws_vpc_security_group_ingress_rule" "alb_http" { + security_group_id = aws_security_group.alb.id + description = "HTTP redirect or approved dev ingress" + ip_protocol = "tcp" + from_port = 80 + to_port = 80 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_ingress_rule" "alb_https" { + security_group_id = aws_security_group.alb.id + description = "HTTPS" + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "alb_to_api" { + security_group_id = aws_security_group.alb.id + description = "API target traffic only" + ip_protocol = "tcp" + from_port = 8000 + to_port = 8000 + referenced_security_group_id = aws_security_group.ecs.id +} + +resource "aws_vpc_security_group_ingress_rule" "api_from_alb" { + security_group_id = aws_security_group.ecs.id + description = "API traffic from ALB" + ip_protocol = "tcp" + from_port = 8000 + to_port = 8000 + referenced_security_group_id = aws_security_group.alb.id +} + +resource "aws_vpc_security_group_egress_rule" "ecs_https" { + security_group_id = aws_security_group.ecs.id + description = "TLS to AWS endpoints and approved external sources" + ip_protocol = "tcp" + from_port = 443 + to_port = 443 + cidr_ipv4 = "0.0.0.0/0" +} + +resource "aws_vpc_security_group_egress_rule" "ecs_postgresql" { + security_group_id = aws_security_group.ecs.id + description = "PostgreSQL" + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 + referenced_security_group_id = aws_security_group.rds.id +} + +resource "aws_vpc_security_group_ingress_rule" "postgresql_from_ecs" { + security_group_id = aws_security_group.rds.id + description = "PostgreSQL from ECS" + ip_protocol = "tcp" + from_port = 5432 + to_port = 5432 + referenced_security_group_id = aws_security_group.ecs.id +} diff --git a/infra/terraform/modules/platform/storage.tf b/infra/terraform/modules/platform/storage.tf new file mode 100644 index 0000000..0098621 --- /dev/null +++ b/infra/terraform/modules/platform/storage.tf @@ -0,0 +1,128 @@ +resource "aws_s3_bucket" "data" { + for_each = local.storage_classes + + bucket_prefix = "${local.name}-${each.key}-" + force_destroy = false + + tags = merge(local.common_tags, { + Name = "${local.name}-${each.key}" + DataStage = each.key + DestructiveRules = "disabled" + }) +} + +resource "aws_s3_bucket_public_access_block" "data" { + for_each = aws_s3_bucket.data + + bucket = each.value.id + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_ownership_controls" "data" { + for_each = aws_s3_bucket.data + + bucket = each.value.id + rule { + object_ownership = "BucketOwnerEnforced" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "data" { + for_each = aws_s3_bucket.data + + bucket = each.value.id + rule { + apply_server_side_encryption_by_default { + kms_master_key_id = aws_kms_key.platform.arn + sse_algorithm = "aws:kms" + } + bucket_key_enabled = true + } +} + +resource "aws_s3_bucket_versioning" "data" { + for_each = aws_s3_bucket.data + + bucket = each.value.id + versioning_configuration { + status = contains(local.versioned_storage_classes, each.key) ? "Enabled" : "Suspended" + } +} + +resource "aws_s3_bucket_lifecycle_configuration" "data" { + for_each = toset(["raw", "validated", "curated", "exports"]) + + bucket = aws_s3_bucket.data[each.key].id + + rule { + id = "non-destructive-storage-transition" + status = "Enabled" + + filter {} + + transition { + days = each.key == "exports" ? 30 : 90 + storage_class = "STANDARD_IA" + } + + transition { + days = each.key == "exports" ? 90 : 180 + storage_class = "GLACIER_IR" + } + } +} + +data "aws_iam_policy_document" "bucket_transport" { + for_each = aws_s3_bucket.data + + statement { + sid = "DenyInsecureTransport" + effect = "Deny" + + principals { + type = "*" + identifiers = ["*"] + } + + actions = ["s3:*"] + resources = [each.value.arn, "${each.value.arn}/*"] + + condition { + test = "Bool" + variable = "aws:SecureTransport" + values = ["false"] + } + } + + dynamic "statement" { + for_each = each.key == "frontend" ? [1] : [] + content { + sid = "AllowCloudFrontOriginAccess" + effect = "Allow" + actions = ["s3:GetObject"] + resources = ["${each.value.arn}/*"] + + principals { + type = "Service" + identifiers = ["cloudfront.amazonaws.com"] + } + + condition { + test = "StringEquals" + variable = "AWS:SourceArn" + values = [aws_cloudfront_distribution.frontend.arn] + } + } + } + +} + +resource "aws_s3_bucket_policy" "data" { + for_each = aws_s3_bucket.data + + bucket = each.value.id + policy = data.aws_iam_policy_document.bucket_transport[each.key].json +} diff --git a/infra/terraform/modules/platform/variables.tf b/infra/terraform/modules/platform/variables.tf new file mode 100644 index 0000000..3aee34f --- /dev/null +++ b/infra/terraform/modules/platform/variables.tf @@ -0,0 +1,209 @@ +variable "project" { + description = "Lowercase project identifier used in resource names and tags." + type = string + default = "foundation-intelligence" + + validation { + condition = can(regex("^[a-z][a-z0-9-]{2,30}$", var.project)) + error_message = "project must be a short lowercase identifier." + } +} + +variable "environment" { + description = "Isolated deployment environment." + type = string + + validation { + condition = contains(["dev", "staging"], var.environment) + error_message = "Only dev and staging are defined by this remediation." + } +} + +variable "aws_region" { + description = "AWS region selected by the approved deployment operator." + type = string +} + +variable "availability_zones" { + description = "Two or more approved availability zones." + type = list(string) + + validation { + condition = length(var.availability_zones) >= 2 + error_message = "At least two availability zones are required." + } +} + +variable "vpc_cidr" { + type = string +} + +variable "public_subnet_cidrs" { + type = list(string) +} + +variable "application_subnet_cidrs" { + type = list(string) +} + +variable "database_subnet_cidrs" { + type = list(string) +} + +variable "single_nat_gateway" { + description = "Cost-aware dev mode; staging should use one NAT per AZ." + type = bool + default = false +} + +variable "enable_interface_endpoints" { + description = "Create private endpoints for ECR, logs, secrets and SQS." + type = bool + default = true +} + +variable "api_image" { + description = "Immutable ECR image reference including an sha256 digest." + type = string + + validation { + condition = can(regex("@sha256:[0-9a-f]{64}$", var.api_image)) + error_message = "api_image must be immutable and end with @sha256:<64 hex>." + } +} + +variable "worker_image" { + description = "Immutable ECR image reference including an sha256 digest." + type = string + + validation { + condition = can(regex("@sha256:[0-9a-f]{64}$", var.worker_image)) + error_message = "worker_image must be immutable and end with @sha256:<64 hex>." + } +} + +variable "frontend_origin_image_digest" { + description = "Audit metadata for the immutable frontend build image." + type = string + + validation { + condition = can(regex("^sha256:[0-9a-f]{64}$", var.frontend_origin_image_digest)) + error_message = "frontend_origin_image_digest must be sha256:<64 hex>." + } +} + +variable "api_cpu" { + type = number + default = 512 +} + +variable "api_memory" { + type = number + default = 1024 +} + +variable "api_desired_count" { + type = number + default = 1 +} + +variable "worker_cpu" { + type = number + default = 512 +} + +variable "worker_memory" { + type = number + default = 1024 +} + +variable "worker_desired_count" { + type = number + default = 1 +} + +variable "rds_instance_class" { + type = string +} + +variable "rds_allocated_storage_gib" { + type = number +} + +variable "rds_max_storage_gib" { + type = number +} + +variable "rds_backup_retention_days" { + type = number + + validation { + condition = var.rds_backup_retention_days >= 7 + error_message = "At least seven days of PITR backups are required." + } +} + +variable "rds_multi_az" { + type = bool +} + +variable "domain_name" { + description = "Optional approved application DNS name. Null avoids DNS/certificate resources." + type = string + default = null + nullable = true +} + +variable "hosted_zone_id" { + description = "Approved Route53 zone ID. Required only when manage_dns is true." + type = string + default = null + nullable = true +} + +variable "manage_dns" { + description = "Explicit opt-in for certificate validation and DNS records." + type = bool + default = false +} + +variable "github_repository" { + description = "GitHub owner/repository allowed to assume deployment roles." + type = string + + validation { + condition = can(regex("^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$", var.github_repository)) + error_message = "github_repository must be owner/repository." + } +} + +variable "github_oidc_provider_arn" { + description = "Existing GitHub OIDC provider ARN; null creates one in this account." + type = string + default = null + nullable = true +} + +variable "enable_schedules" { + description = "Fail-closed schedule switch; keep false until source governance approval." + type = bool + default = false +} + +variable "notification_email" { + description = "Optional approved alarm/budget destination. Null creates no subscription." + type = string + default = null + nullable = true + sensitive = true +} + +variable "monthly_budget_usd" { + description = "Proposed environment budget; must be approved before deployment." + type = number +} + +variable "tags" { + type = map(string) + default = {} +} diff --git a/infra/terraform/modules/platform/versions.tf b/infra/terraform/modules/platform/versions.tf new file mode 100644 index 0000000..d827e70 --- /dev/null +++ b/infra/terraform/modules/platform/versions.tf @@ -0,0 +1,10 @@ +terraform { + required_version = ">= 1.7.0, < 2.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.90" + } + } +} diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..adbffc4 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,26 @@ +[mypy] +python_version = 3.12 +explicit_package_bases = True +ignore_missing_imports = True +check_untyped_defs = True +show_error_codes = True +warn_unused_configs = True + +files = + scripts/validate_terraform_static.py, + scripts/verify_transition.py, + scripts/verify_local_rollback.py, + src/bff/config.py, + src/bff/database.py, + src/bff/security.py, + src/bff/postgres/governance_repository.py, + src/bff/postgres/job_repository.py, + src/bff/postgres/pipeline_repository.py, + src/governance/exposure.py, + src/governance/retention.py, + src/migration/sqlite_to_postgres.py, + src/observability/metrics.py, + src/transition/runtime.py, + src/transition/shadow.py, + src/pipelines/durable.py, + src/pipelines/durable_worker.py diff --git a/requirements-locking.txt b/requirements-locking.txt new file mode 100644 index 0000000..7859108 --- /dev/null +++ b/requirements-locking.txt @@ -0,0 +1,19 @@ +# Bootstrap lock for the reproducible Python dependency resolver. +# Resolved from PyPI for Python 3.12 on 2026-07-29. +build==1.5.0 \ + --hash=sha256:13f3eecb844759ab66efec90ca17639bbf14dc06cb2fdf37a9010322d9c50a6f +click==8.4.2 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e +# pip 25.3 is the newest release compatible with pip-tools 7.5.2. +pip==25.3 \ + --hash=sha256:9655943313a94722b7774661c21049070f6bbb0a1516bf02f7c8d5d9201514cd +pip-tools==7.5.2 \ + --hash=sha256:2fe16db727bbe5bf28765aeb581e792e61be51fc275545ef6725374ad720a1ce +pyproject-hooks==1.2.0 \ + --hash=sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913 +setuptools==83.0.0 \ + --hash=sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3 +wheel==0.47.0 \ + --hash=sha256:212281cab4dff978f6cedd499cd893e1f620791ca6ff7107cf270781e587eced diff --git a/requirements-runtime.in b/requirements-runtime.in new file mode 100644 index 0000000..b5a7cb6 --- /dev/null +++ b/requirements-runtime.in @@ -0,0 +1,19 @@ +# Direct production dependencies. Exact pins constrain resolver input; the +# generated requirements-runtime.txt contains the full transitive hash lock. +alembic==1.18.5 +anthropic==0.118.0 +asyncpg==0.31.0 +beautifulsoup4==4.15.0 +fastapi==0.139.2 +feedparser==6.0.12 +google-genai==2.14.0 +googlenewsdecoder==0.1.7 +httpx==0.28.1 +psycopg[binary]==3.3.4 +pycountry==26.2.16 +pydantic==2.13.4 +python-jose[cryptography]==3.5.0 +python-multipart==0.0.32 +requests==2.34.2 +SQLAlchemy[asyncio]==2.0.51 +uvicorn==0.51.0 diff --git a/requirements-runtime.txt b/requirements-runtime.txt new file mode 100644 index 0000000..c390447 --- /dev/null +++ b/requirements-runtime.txt @@ -0,0 +1,1222 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --index-url=https://pypi.org/simple --no-emit-index-url --output-file=requirements-runtime.txt --strip-extras requirements-runtime.in +# +alembic==1.18.5 \ + --hash=sha256:06d8ba9d04558022f5395e9317de03d270f3dced49cee01f89fe7a13c26f14bc \ + --hash=sha256:1554982221dd17e9a749b53902407578eb305e453f71999e8c7f0a48389fff8e + # via -r requirements-runtime.in +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via fastapi +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anthropic==0.118.0 \ + --hash=sha256:524d835869b8e374510b3bcc552e28dbdafc035a61d63fc0088c4035286ef8fe \ + --hash=sha256:acb3c43b7e7592cf635fa21930e67a99d32641d68ccd53f625b52f71d7870d55 + # via -r requirements-runtime.in +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via + # anthropic + # google-genai + # httpx + # starlette +asyncpg==0.31.0 \ + --hash=sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8 \ + --hash=sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be \ + --hash=sha256:0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be \ + --hash=sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2 \ + --hash=sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d \ + --hash=sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a \ + --hash=sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7 \ + --hash=sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218 \ + --hash=sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d \ + --hash=sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602 \ + --hash=sha256:22be6e02381bab3101cd502d9297ac71e2f966c86e20e78caead9934c98a8af6 \ + --hash=sha256:2657204552b75f8288de08ca60faf4a99a65deef3a71d1467454123205a88fab \ + --hash=sha256:2d076d42eb583601179efa246c5d7ae44614b4144bc1c7a683ad1222814ed095 \ + --hash=sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5 \ + --hash=sha256:37a58919cfef2448a920df00d1b2f821762d17194d0dbf355d6dde8d952c04f9 \ + --hash=sha256:37fc6c00a814e18eef51833545d1891cac9aa69140598bb076b4cd29b3e010b9 \ + --hash=sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c \ + --hash=sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec \ + --hash=sha256:3faa62f997db0c9add34504a68ac2c342cfee4d57a0c3062fcf0d86c7f9cb1e8 \ + --hash=sha256:480c4befbdf079c14c9ca43c8c5e1fe8b6296c96f1f927158d4f1e750aacc047 \ + --hash=sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e \ + --hash=sha256:5a4af56edf82a701aece93190cc4e094d2df7d33f6e915c222fb09efbb5afc24 \ + --hash=sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31 \ + --hash=sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186 \ + --hash=sha256:795416369c3d284e1837461909f58418ad22b305f955e625a4b3a2521d80a5f3 \ + --hash=sha256:831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61 \ + --hash=sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a \ + --hash=sha256:8ea599d45c361dfbf398cb67da7fd052affa556a401482d3ff1ee99bd68808a1 \ + --hash=sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2 \ + --hash=sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2 \ + --hash=sha256:9ea33213ac044171f4cac23740bed9a3805abae10e7025314cfbd725ec670540 \ + --hash=sha256:a429e842a3a4b4ea240ea52d7fe3f82d5149853249306f7ff166cb9948faa46c \ + --hash=sha256:a8d758dac9d2e723e173d286ef5e574f0b350ec00e9186fce84d0fc5f6a8e6b8 \ + --hash=sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671 \ + --hash=sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad \ + --hash=sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d \ + --hash=sha256:bb223567dea5f47c45d347f2bde5486be8d9f40339f27217adb3fb1c3be51298 \ + --hash=sha256:bc2b685f400ceae428f79f78b58110470d7b4466929a7f78d455964b17ad1008 \ + --hash=sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3 \ + --hash=sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20 \ + --hash=sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2 \ + --hash=sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4 \ + --hash=sha256:c0807be46c32c963ae40d329b3a686356e417f674c976c07fa49f1b30303f109 \ + --hash=sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403 \ + --hash=sha256:c1a9c5b71d2371a2290bc93336cd05ba4ec781683cab292adbddc084f89443c6 \ + --hash=sha256:c1e1ab5bc65373d92dd749d7308c5b26fb2dc0fbe5d3bf68a32b676aa3bcd24a \ + --hash=sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b \ + --hash=sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735 \ + --hash=sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b \ + --hash=sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab \ + --hash=sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e \ + --hash=sha256:e5d5098f63beeae93512ee513d4c0c53dc12e9aa2b7a1af5a81cddf93fe4e4da \ + --hash=sha256:e6974f36eb9a224d8fb428bcf66bd411aa12cf57c2967463178149e73d4de366 \ + --hash=sha256:ebb3cde58321a1f89ce41812be3f2a98dddedc1e76d0838aba1d724f1e4e1a95 \ + --hash=sha256:eee690960e8ab85063ba93af2ce128c0f52fd655fdff9fdb1a28df01329f031d \ + --hash=sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44 \ + --hash=sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696 + # via -r requirements-runtime.in +beautifulsoup4==4.15.0 \ + --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ + --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 + # via -r requirements-runtime.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx + # requests +cffi==2.1.0 \ + --hash=sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc \ + --hash=sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd \ + --hash=sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d \ + --hash=sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5 \ + --hash=sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f \ + --hash=sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6 \ + --hash=sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c \ + --hash=sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda \ + --hash=sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd \ + --hash=sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a \ + --hash=sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd \ + --hash=sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd \ + --hash=sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3 \ + --hash=sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb \ + --hash=sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66 \ + --hash=sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d \ + --hash=sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f \ + --hash=sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6 \ + --hash=sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0 \ + --hash=sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c \ + --hash=sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93 \ + --hash=sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d \ + --hash=sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d \ + --hash=sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8 \ + --hash=sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b \ + --hash=sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001 \ + --hash=sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d \ + --hash=sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43 \ + --hash=sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b \ + --hash=sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0 \ + --hash=sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0 \ + --hash=sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458 \ + --hash=sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8 \ + --hash=sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d \ + --hash=sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94 \ + --hash=sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022 \ + --hash=sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db \ + --hash=sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479 \ + --hash=sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376 \ + --hash=sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d \ + --hash=sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6 \ + --hash=sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3 \ + --hash=sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea \ + --hash=sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd \ + --hash=sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02 \ + --hash=sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde \ + --hash=sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224 \ + --hash=sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76 \ + --hash=sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804 \ + --hash=sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1 \ + --hash=sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913 \ + --hash=sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714 \ + --hash=sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e \ + --hash=sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda \ + --hash=sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512 \ + --hash=sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28 \ + --hash=sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699 \ + --hash=sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3 \ + --hash=sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c \ + --hash=sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe \ + --hash=sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a \ + --hash=sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f \ + --hash=sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c \ + --hash=sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2 \ + --hash=sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f \ + --hash=sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 \ + --hash=sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056 \ + --hash=sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629 \ + --hash=sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7 \ + --hash=sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0 \ + --hash=sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9 \ + --hash=sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853 \ + --hash=sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13 \ + --hash=sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a \ + --hash=sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4 \ + --hash=sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce \ + --hash=sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac \ + --hash=sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c \ + --hash=sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46 \ + --hash=sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384 \ + --hash=sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b \ + --hash=sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210 \ + --hash=sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc \ + --hash=sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a \ + --hash=sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5 \ + --hash=sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7 \ + --hash=sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2 \ + --hash=sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326 \ + --hash=sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f \ + --hash=sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca \ + --hash=sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98 \ + --hash=sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9 \ + --hash=sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5 \ + --hash=sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7 \ + --hash=sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc \ + --hash=sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da \ + --hash=sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f + # via cryptography +charset-normalizer==3.4.9 \ + --hash=sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380 \ + --hash=sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62 \ + --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ + --hash=sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226 \ + --hash=sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5 \ + --hash=sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833 \ + --hash=sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b \ + --hash=sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99 \ + --hash=sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501 \ + --hash=sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec \ + --hash=sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698 \ + --hash=sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4 \ + --hash=sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a \ + --hash=sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3 \ + --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 \ + --hash=sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a \ + --hash=sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e \ + --hash=sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4 \ + --hash=sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419 \ + --hash=sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84 \ + --hash=sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da \ + --hash=sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519 \ + --hash=sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe \ + --hash=sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381 \ + --hash=sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29 \ + --hash=sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614 \ + --hash=sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0 \ + --hash=sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe \ + --hash=sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29 \ + --hash=sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0 \ + --hash=sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917 \ + --hash=sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9 \ + --hash=sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32 \ + --hash=sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94 \ + --hash=sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63 \ + --hash=sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd \ + --hash=sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198 \ + --hash=sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde \ + --hash=sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012 \ + --hash=sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1 \ + --hash=sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15 \ + --hash=sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b \ + --hash=sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993 \ + --hash=sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4 \ + --hash=sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5 \ + --hash=sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8 \ + --hash=sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35 \ + --hash=sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642 \ + --hash=sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2 \ + --hash=sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d \ + --hash=sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9 \ + --hash=sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c \ + --hash=sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33 \ + --hash=sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db \ + --hash=sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf \ + --hash=sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9 \ + --hash=sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee \ + --hash=sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84 \ + --hash=sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44 \ + --hash=sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f \ + --hash=sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9 \ + --hash=sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9 \ + --hash=sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177 \ + --hash=sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 \ + --hash=sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b \ + --hash=sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39 \ + --hash=sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41 \ + --hash=sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0 \ + --hash=sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616 \ + --hash=sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d \ + --hash=sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba \ + --hash=sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2 \ + --hash=sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b \ + --hash=sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9 \ + --hash=sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b \ + --hash=sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209 \ + --hash=sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48 \ + --hash=sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046 \ + --hash=sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632 \ + --hash=sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a \ + --hash=sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2 \ + --hash=sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1 \ + --hash=sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b \ + --hash=sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990 \ + --hash=sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5 \ + --hash=sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b \ + --hash=sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9 \ + --hash=sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534 \ + --hash=sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81 \ + --hash=sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a \ + --hash=sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d \ + --hash=sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf \ + --hash=sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115 + # via requests +click==8.4.2 \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via uvicorn +cryptography==49.0.0 \ + --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ + --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ + --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ + --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ + --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ + --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ + --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ + --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ + --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ + --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ + --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ + --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ + --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ + --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ + --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ + --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ + --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ + --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ + --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ + --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ + --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ + --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ + --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ + --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ + --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ + --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ + --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ + --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ + --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ + --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ + --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ + --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ + --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ + --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ + --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ + --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ + --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ + --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ + --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ + --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ + --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ + --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ + --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ + --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ + --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ + --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b + # via + # google-auth + # python-jose +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 + # via + # anthropic + # google-genai +docstring-parser==0.18.0 \ + --hash=sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015 \ + --hash=sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b + # via anthropic +ecdsa==0.19.2 \ + --hash=sha256:62635b0ac1ca2e027f82122b5b81cb706edc38cd91c63dda28e4f3455a2bf930 \ + --hash=sha256:840f5dc5e375c68f36c1a7a5b9caad28f95daa65185c9253c0c08dd952bb7399 + # via python-jose +fastapi==0.139.2 \ + --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ + --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c + # via -r requirements-runtime.in +feedparser==6.0.12 \ + --hash=sha256:64f76ce90ae3e8ef5d1ede0f8d3b50ce26bcce71dd8ae5e82b1cd2d4a5f94228 \ + --hash=sha256:6bbff10f5a52662c00a2e3f86a38928c37c48f77b3c511aedcd51de933549324 + # via -r requirements-runtime.in +google-auth==2.56.2 \ + --hash=sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6 \ + --hash=sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051 + # via google-genai +google-genai==2.14.0 \ + --hash=sha256:a9d1f4f362d76280f1be1340fcb3c86e63dbca128f6a4ae09d86ab47ff7148e8 \ + --hash=sha256:ae7172cdd35695189b516b33a878e4132e5daa2dbc03a5b44cddfa8a82fad664 + # via -r requirements-runtime.in +googlenewsdecoder==0.1.7 \ + --hash=sha256:0fcd8fa2aa6a145f84798921aeef0ec45d40bcf5ceebb467742f0efd068d3902 \ + --hash=sha256:a8be897bb41864a82d81da6e05777d8c795989cf65f14b41e8e772b3f5a8354d + # via -r requirements-runtime.in +greenlet==3.5.4 \ + --hash=sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20 \ + --hash=sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c \ + --hash=sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994 \ + --hash=sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8 \ + --hash=sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d \ + --hash=sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9 \ + --hash=sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f \ + --hash=sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809 \ + --hash=sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c \ + --hash=sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c \ + --hash=sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72 \ + --hash=sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3 \ + --hash=sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02 \ + --hash=sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c \ + --hash=sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c \ + --hash=sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7 \ + --hash=sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec \ + --hash=sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c \ + --hash=sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686 \ + --hash=sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861 \ + --hash=sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8 \ + --hash=sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0 \ + --hash=sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4 \ + --hash=sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9 \ + --hash=sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3 \ + --hash=sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9 \ + --hash=sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7 \ + --hash=sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7 \ + --hash=sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd \ + --hash=sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3 \ + --hash=sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2 \ + --hash=sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616 \ + --hash=sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df \ + --hash=sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf \ + --hash=sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0 \ + --hash=sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a \ + --hash=sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f \ + --hash=sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22 \ + --hash=sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356 \ + --hash=sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353 \ + --hash=sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e \ + --hash=sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7 \ + --hash=sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5 \ + --hash=sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8 \ + --hash=sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde \ + --hash=sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52 \ + --hash=sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190 \ + --hash=sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05 \ + --hash=sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937 \ + --hash=sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867 \ + --hash=sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d \ + --hash=sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf \ + --hash=sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f \ + --hash=sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd \ + --hash=sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da \ + --hash=sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071 \ + --hash=sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88 \ + --hash=sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17 \ + --hash=sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c \ + --hash=sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66 \ + --hash=sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb \ + --hash=sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c \ + --hash=sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25 \ + --hash=sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0 \ + --hash=sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927 \ + --hash=sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6 \ + --hash=sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c \ + --hash=sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59 \ + --hash=sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb \ + --hash=sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606 \ + --hash=sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef \ + --hash=sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3 \ + --hash=sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da \ + --hash=sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132 \ + --hash=sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7 \ + --hash=sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f \ + --hash=sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2 \ + --hash=sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f \ + --hash=sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667 + # via sqlalchemy +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via + # -r requirements-runtime.in + # anthropic + # google-genai +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx + # requests +jiter==0.16.0 \ + --hash=sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536 \ + --hash=sha256:0758ab7747a984797cf048e8eedea1d8ef39d7994b25611daf5b48fc903e8873 \ + --hash=sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f \ + --hash=sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3 \ + --hash=sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce \ + --hash=sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26 \ + --hash=sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e \ + --hash=sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85 \ + --hash=sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7 \ + --hash=sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f \ + --hash=sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207 \ + --hash=sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5 \ + --hash=sha256:26d122613ada2b708eb714695446f40fce5bdf2edb4b02116dec62faa62dfab3 \ + --hash=sha256:2a0d46ef67cc58d906a6132dd3040ca70ae4f0b0d7c9c052fe432c658a69b3f6 \ + --hash=sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056 \ + --hash=sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131 \ + --hash=sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331 \ + --hash=sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03 \ + --hash=sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93 \ + --hash=sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290 \ + --hash=sha256:3b21b412b899fd8bd51a3046934b59a3bb068b79f70a5c6010053ac77cc53f0c \ + --hash=sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a \ + --hash=sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284 \ + --hash=sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c \ + --hash=sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195 \ + --hash=sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730 \ + --hash=sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48 \ + --hash=sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69 \ + --hash=sha256:491e7d072a253b156fff46b78bceac4652a697aa8d7082c9c18c03d7b7917d24 \ + --hash=sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c \ + --hash=sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274 \ + --hash=sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7 \ + --hash=sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5 \ + --hash=sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106 \ + --hash=sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b \ + --hash=sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00 \ + --hash=sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9 \ + --hash=sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b \ + --hash=sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b \ + --hash=sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818 \ + --hash=sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c \ + --hash=sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2 \ + --hash=sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077 \ + --hash=sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037 \ + --hash=sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2 \ + --hash=sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84 \ + --hash=sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a \ + --hash=sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a \ + --hash=sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3 \ + --hash=sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb \ + --hash=sha256:70a490b55634dc0d2606ce8a8e01b1d62459011beb368d15d76e1eaf62460e3d \ + --hash=sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe \ + --hash=sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84 \ + --hash=sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c \ + --hash=sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126 \ + --hash=sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad \ + --hash=sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee \ + --hash=sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053 \ + --hash=sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929 \ + --hash=sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450 \ + --hash=sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244 \ + --hash=sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3 \ + --hash=sha256:9acf1b2faec82d998811ecce7ae84d9005e53410773e9d37d61cdc424ba4581b \ + --hash=sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9 \ + --hash=sha256:9ec553a99b0987efd7a3645a1a825cf29c224e494db267a83369fcc8da9aeda5 \ + --hash=sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734 \ + --hash=sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585 \ + --hash=sha256:a5c54ef4ff776d9675837ef535b3308d6e31c208d43ebc44a0f7ab8a208c68f7 \ + --hash=sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29 \ + --hash=sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91 \ + --hash=sha256:b1e7923093a376d93c6eb507c77045ae258d689ba577392846a1b3f10d0b09a9 \ + --hash=sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee \ + --hash=sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9 \ + --hash=sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e \ + --hash=sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5 \ + --hash=sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db \ + --hash=sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd \ + --hash=sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620 \ + --hash=sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567 \ + --hash=sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898 \ + --hash=sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01 \ + --hash=sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea \ + --hash=sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c \ + --hash=sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026 \ + --hash=sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e \ + --hash=sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e \ + --hash=sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8 \ + --hash=sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883 \ + --hash=sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702 \ + --hash=sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09 \ + --hash=sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72 \ + --hash=sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8 \ + --hash=sha256:d8f80521644426d451e70f00c7974240cab8f6ee088aedaa9af2697153ab7805 \ + --hash=sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af \ + --hash=sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3 \ + --hash=sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf \ + --hash=sha256:e03a5f21a5ce96a9441b8cb32719a8b88ed5388f53e0f339c5bcf54f1317f9d0 \ + --hash=sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f \ + --hash=sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1 \ + --hash=sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e \ + --hash=sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e \ + --hash=sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127 \ + --hash=sha256:f3bd327cdfa118bc1ce69c214c2678571d5bd39b8ccd0ebf43a54db00541ba9a \ + --hash=sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e \ + --hash=sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de + # via anthropic +mako==1.3.12 \ + --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ + --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a + # via alembic +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via mako +psycopg==3.3.4 \ + --hash=sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a \ + --hash=sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc + # via -r requirements-runtime.in +psycopg-binary==3.3.4 \ + --hash=sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070 \ + --hash=sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c \ + --hash=sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc \ + --hash=sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e \ + --hash=sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68 \ + --hash=sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae \ + --hash=sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829 \ + --hash=sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097 \ + --hash=sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c \ + --hash=sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992 \ + --hash=sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97 \ + --hash=sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6 \ + --hash=sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a \ + --hash=sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d \ + --hash=sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228 \ + --hash=sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e \ + --hash=sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4 \ + --hash=sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41 \ + --hash=sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949 \ + --hash=sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9 \ + --hash=sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089 \ + --hash=sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf \ + --hash=sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da \ + --hash=sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578 \ + --hash=sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014 \ + --hash=sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e \ + --hash=sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e \ + --hash=sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31 \ + --hash=sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652 \ + --hash=sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b \ + --hash=sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839 \ + --hash=sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277 \ + --hash=sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7 \ + --hash=sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4 \ + --hash=sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e \ + --hash=sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70 \ + --hash=sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf \ + --hash=sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a \ + --hash=sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429 \ + --hash=sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8 \ + --hash=sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3 \ + --hash=sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13 \ + --hash=sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007 \ + --hash=sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc \ + --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 \ + --hash=sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9 \ + --hash=sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95 \ + --hash=sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d \ + --hash=sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16 \ + --hash=sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d \ + --hash=sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260 \ + --hash=sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744 \ + --hash=sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28 \ + --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ + --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 + # via psycopg +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b + # via + # pyasn1-modules + # python-jose + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pycountry==26.2.16 \ + --hash=sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a \ + --hash=sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801 + # via -r requirements-runtime.in +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 + # via + # -r requirements-runtime.in + # anthropic + # fastapi + # google-genai +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae + # via pydantic +pysocks==1.7.1 \ + --hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \ + --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \ + --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 + # via googlenewsdecoder +python-jose==3.5.0 \ + --hash=sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771 \ + --hash=sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b + # via -r requirements-runtime.in +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via -r requirements-runtime.in +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -r requirements-runtime.in + # google-auth + # google-genai + # googlenewsdecoder +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via python-jose +selectolax==0.4.11 \ + --hash=sha256:086ca6f7e4c475bfff871ec1448ae5d342d43d6a2ca2cea65160d01b3a6a75ec \ + --hash=sha256:168b34466dc4f998d7ceb04f218693546d141543e7c5d327f9e006c0799cd62c \ + --hash=sha256:1a6deb4464198ac67f32e56c4463aedf3e1d834b458eaac5b5b5b1ef02dcf15e \ + --hash=sha256:1da54e42ab99b9191269306e13c0fd67ada1c6654e8dc8d74fac615931dd3c62 \ + --hash=sha256:21359cf4d1261b314895c07321e76aa2334b1967956efba04cc0d223d7817f48 \ + --hash=sha256:224682039ca13eb822be626e49a03592ee2b8557bcdc6381e49417a995170c94 \ + --hash=sha256:25d633cddca0fc769e7d890e9e838908fb4a7326eec5e3b23ae42c27f457541d \ + --hash=sha256:28915b8fa90c1c3cb585858a6d24d433a3f38ea514aea59013bdb0930d9f6025 \ + --hash=sha256:2b565ddabce6c9a7b73fa28a39acf8f411a084fa2f169234ec2470f552d4421d \ + --hash=sha256:2b842c829f916fecb51f0f55882eca3e2ad49e85388178f14ae6fe0912be0a57 \ + --hash=sha256:2e8d01d9688e9670b7ed3f042dc7ba379f844fb7fcb8e85191885ed75d6178fb \ + --hash=sha256:3066be35f630e4c4a00cf2c829548574fcba5963735411e6a9ac78bd4ca830ce \ + --hash=sha256:36183968e5c122da414db0dd7d22d4a6ca808e9b9e4837532dfbe55df6f26bff \ + --hash=sha256:3637d21f7fe60fbd6ca3dbc67a1747f6a55a9389114d72f06b5d69ba2beddf01 \ + --hash=sha256:386494779e5464e587ed4dc076e1c48c24ebaf2da1e3a249690551d1f97fe8ed \ + --hash=sha256:3abf5aecfd0c314b62c10ce6e9baee2094cb385e2ec154bd1743febd2fcc9bdb \ + --hash=sha256:41f388c26304c1d840f5ee5e07c06bb9388ec834d10fec60dc148f22f98efd38 \ + --hash=sha256:466daca0599408c9d2cad7658a68490facc5c9b8d0f41ac5d17948914f57306f \ + --hash=sha256:47be0e591125484de14ff0c9aaaa814dd4a4019de35eabe360e88169a263a2b5 \ + --hash=sha256:48545b0351b6f92c4ad2fff835e832768d0cb37834766a82f40242e51fc9901d \ + --hash=sha256:4c3d4fd42deabfd03cc5c4fce9517e7226054f9de3984759476e6c4a7b7fe187 \ + --hash=sha256:5153157ed60d968ed303acbfd2c8762fa0c0462e2663bd04466471c565deb88a \ + --hash=sha256:57e71c021d9950113e7d49892eba56014c072c9d86b71c0b3b7cb846ee913e20 \ + --hash=sha256:597b8e065978be200c598ae6d682496d96fbce14d34b5d519e93cf5b6be5fb60 \ + --hash=sha256:5c7a91fbe1a94849d85228897c416ab9b4518bea6b04dce8ef8acd825ec80e9d \ + --hash=sha256:5ff73122f5fee3350528235349f1b9e5626a1a0ef1887fef82d7fddac18f6605 \ + --hash=sha256:634db1b7ae1b1f10ee0b0adbef0eee1aec65a3d01f09b75132bc5b043b7623d1 \ + --hash=sha256:663ff792f92ed749cfcf452ac19aff5da74b05521e7daacb3b74388deb14d117 \ + --hash=sha256:75c13f519ecb85185e149049bd4c4937aa0b953bd21e1748eb711c6cdcee571e \ + --hash=sha256:75eeddbd3db7e82293a325c210cf4d218d25961870dbb468383c7516e3f577aa \ + --hash=sha256:80af1c7345701934769679a83fcc86526c56eef590423cc2d55cc1eb81adfa5f \ + --hash=sha256:8166bb8cad8f2eabed664f689b708851078b74dd50dd93e426b291095badc2cb \ + --hash=sha256:81f21972788df01b83a5940d9b7d62e6e222f80872c2cfaf67927276e8ef4975 \ + --hash=sha256:83515cc0fe9b8c4a1ca8bfcd83cf48408bcdd3f2fccd088f04635619e93695c9 \ + --hash=sha256:8da39a07a589fd181b5e8d25f695d7d40d3a1d89e47c2e00c08ece7fff5ddd3d \ + --hash=sha256:8f0223a83051ece2591a1d0cf487f1d0719cb76660d7c5de4c0a201b761c5c23 \ + --hash=sha256:8ff49b24d4e5c81bcc96b25e2cf23d9af2d06e4d473865c812487be6665416c0 \ + --hash=sha256:903286e6d9752b63b3b37f9ee98a8ff7db403b23416b469b065cc992b2556b80 \ + --hash=sha256:9077fa36e99ef4bb801194ff8f492f67279c0562e7cdfa9b4d06f5c010131969 \ + --hash=sha256:96de39e2f28c359cf930547193b2cacf0ab6b9187309e2515461fff9d5a5aa3c \ + --hash=sha256:9862c1d0435e89e419e0c595084eaa8f6f71c3fb7a968bdff51db595cec07161 \ + --hash=sha256:991866e2f45ce85e3ce7a31714e522f620d83b1eca15fe530055108a94e6087e \ + --hash=sha256:a532f1993f08f627300891751982dd9641ce1bd29618b535232aec9fa023a1c8 \ + --hash=sha256:ad4bbf08af4e2c89f78dd12a7b8265f0924d0434705bd955b5a262297a924452 \ + --hash=sha256:b0f56c49161b18621ac452e42e02b0c5c61ba4c21095cfff3990e040bd9a043c \ + --hash=sha256:b530a2c4fad7400af27b2b7e0333c1318ecb5f5dc38e8a141dbe3bd81b398fdf \ + --hash=sha256:bd843540a197a33049a08fd80e59bfeafbaa688e632d53a05a9b65af5e88296f \ + --hash=sha256:bde121202b33e6041e9d2db1d62e7466b5883fd1c441eb96ff68d3ea015cfcbd \ + --hash=sha256:c3f6fac361b5f415c409dfd1a07dd0e9a5899d10daba8d88ce16bd552b0e06f2 \ + --hash=sha256:d00753a5f832625f69b585dda7a7ddc1a7d2803c31aa8710c5d209fca064d774 \ + --hash=sha256:d33e2ed75cc33e7af3fd50521c33e7d8634fae23bc197a6cee6a5015e056eef6 \ + --hash=sha256:d5ce592a92fceeca2694b369a83ad72891a9c356f668718fe7e1c83eea407bb4 \ + --hash=sha256:daf7a841d1baa795f940200bec45019c4a31020def16e3a4e35485e82ed64167 \ + --hash=sha256:dbee8998d426c1b6b78aab7a051009f9de38d77b93d073025ae19f7d71b3ead3 \ + --hash=sha256:dc8d449adfdef675bcbfc67d5548513271c7ea869e9b71a2e898138bf34a976a \ + --hash=sha256:dcd24bfc4899e4df2bdcbabc973384367686f391e5a2d8ebb229c043f43be82a \ + --hash=sha256:dff45ee85409e6126e4900e0db6950a59a7b58b7ff9d667a6a275b0405b87692 \ + --hash=sha256:e2d7e1a2b67a5e8d251733b2c3f978ed26a56d04552472e0a2fb5ea41fa4e663 \ + --hash=sha256:e5929cbe3eedfaf51a09ec89642ab5355b703486d43bcf3c8f0c27d6043a488d \ + --hash=sha256:f4894d4cb2b9cd4828dc9d9cbed683db051748fde66f5937aa823f2afff16b36 \ + --hash=sha256:f69c377298a09f571c89af5916ac85f69a9aaedd7312b0bb803a46d2ca2f9048 \ + --hash=sha256:f8121f4cbfe870d9ad24ae418a735d918d55844e905c3270077e97f4e579770f \ + --hash=sha256:f8f014f328f6f79a364293bca54d43af1cec776dc10a5302054a54b5fb2d8675 \ + --hash=sha256:fad5b1065f73eeaa07ea343cbc548aaa9f9a5c359c3bdd8d98f5d80b61550d1c + # via googlenewsdecoder +sgmllib3k==1.0.0 \ + --hash=sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9 + # via feedparser +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via ecdsa +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # anthropic + # google-genai +soupsieve==2.9.1 \ + --hash=sha256:4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c \ + --hash=sha256:c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba + # via beautifulsoup4 +sqlalchemy==2.0.51 \ + --hash=sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23 \ + --hash=sha256:0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1 \ + --hash=sha256:08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8 \ + --hash=sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72 \ + --hash=sha256:0e8203d2fbd5c6254692ef0a72c740d75b2f3c7ca345404f4c1a4604813c77c0 \ + --hash=sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5 \ + --hash=sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e \ + --hash=sha256:111604e637da87031255ddc26c7d7bc22bc6af6f5d459ccff3af1b4660233a85 \ + --hash=sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d \ + --hash=sha256:159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2 \ + --hash=sha256:1aa10c0daee6705294d181daadaa793221e1a59ed55000a3fab1d42b088ce4ba \ + --hash=sha256:1af05726b3d0cdba1c55284bf408fd3b792e690fe2399bfb8304565551cda652 \ + --hash=sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f \ + --hash=sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9 \ + --hash=sha256:1e47b1199c2e832e325eacabc8d32d2487f58c9358f97e9a00f5eb93c5680d84 \ + --hash=sha256:247acaa29ccef6250dfd6a3eedf8f94ddf23564180a39fe362e32ae9dbdbde46 \ + --hash=sha256:2a97eaad21c84b4ef8010b11eeba9fe6153eb0b3df3ff8b6abc309df1b978ef7 \ + --hash=sha256:2cf39aabdf48e87c1c2c2ed6d20d33ffa0733b3071ce9c5f66357947dd009080 \ + --hash=sha256:2e54ff2dd657f2e3e0fbf2b097db1182f7bfea263eca4353f00065bae2a67c3d \ + --hash=sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d \ + --hash=sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54 \ + --hash=sha256:436728ce18a80f6951a1e11cc6112c2ede9faf20766f1a26195a7c441ca12dbd \ + --hash=sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195 \ + --hash=sha256:4a011ea4510683319ce4ed274b56ee05194b39b6da9d09ca7a39388f0fa84dcc \ + --hash=sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e \ + --hash=sha256:59cab3686b1bc039dd9cded2f8d0c08a246e84e76bd4ab5b4f18c7cdae293825 \ + --hash=sha256:6b588fd681ddf0c196b8df1ea49a8913514894b2b8f945a9511b4b48871f99c8 \ + --hash=sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522 \ + --hash=sha256:6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491 \ + --hash=sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400 \ + --hash=sha256:740cf6f35351b1ac3d82369152acf1d51d37e3dcf85d4dc0a22ca01410eabe2a \ + --hash=sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07 \ + --hash=sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7 \ + --hash=sha256:7d78702b26ba1c18b2d0fb2ea940ba7f17a9581b42e8361ff93920ebbee1235a \ + --hash=sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9 \ + --hash=sha256:9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7 \ + --hash=sha256:96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499 \ + --hash=sha256:9f380393be5abeb6815f68fd39271b95127173511b6706b0a630a9995d53f8f5 \ + --hash=sha256:a42ad6afcbaaa777241e347aa2e29155993045a0d6b7db74da61053ffe875fe0 \ + --hash=sha256:a5b2ed6d828f1f09bd812861f4f59ca3bc3803f9df871f4555187f0faf018604 \ + --hash=sha256:a6d26094615306d116dd5e4a51b0304c99dd2356fc569eed6922a80a6bd3b265 \ + --hash=sha256:aa18ae738b5170e253ad0bb6c4b0f07585081e8a6e50893e4d911d47b39a0904 \ + --hash=sha256:ad30ae663711786303fbcd46a47516302d201ee49a877cb3fac61f672895110a \ + --hash=sha256:b21f0e7efc7a5c509e953784e9d1575ebb8b4318960e7e7d7a93bb803626cf64 \ + --hash=sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d \ + --hash=sha256:b7f08588854bbb724041d9ae9d980d40040c922382e1d9a2ecb390edc4fd5032 \ + --hash=sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b \ + --hash=sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5 \ + --hash=sha256:bb1f5062f98b0b3290e72b707747fdd7e0f22d6956b236ba7ca7f5c9971d2da2 \ + --hash=sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d \ + --hash=sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389 \ + --hash=sha256:c68568f3facf8f66fa76c60e0ced69b67666ffa9941d1d0a3756fda196049080 \ + --hash=sha256:c95ef01f53233a305a874a44a63fbfb1d81cd79b49de0f8529b3548cde437e37 \ + --hash=sha256:ca216e8af5c05e326efc7e28716ac2381a7cf9791749f5ee1849dccdc99c9b00 \ + --hash=sha256:ca8435d13829b92f4a97362d91975154a4015db3a2634154e1754e9a915e6b86 \ + --hash=sha256:dc261707bf5739aea8a541593f3cc1d463c2701fb05fbcbba0ce031b69a21260 \ + --hash=sha256:e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de \ + --hash=sha256:fa268106c8987639a17a18514cfe0cd9bf17420ab887e1e1bf486da8836135b1 + # via + # -r requirements-runtime.in + # alembic + # sqlalchemy +starlette==1.3.1 \ + --hash=sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0 \ + --hash=sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6 + # via fastapi +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ + --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a + # via google-genai +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # alembic + # anthropic + # anyio + # beautifulsoup4 + # fastapi + # google-genai + # psycopg + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 + # via + # fastapi + # pydantic +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via requests +uvicorn==0.51.0 \ + --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ + --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 + # via -r requirements-runtime.in +websockets==16.1.1 \ + --hash=sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175 \ + --hash=sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a \ + --hash=sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d \ + --hash=sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985 \ + --hash=sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1 \ + --hash=sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573 \ + --hash=sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb \ + --hash=sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9 \ + --hash=sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87 \ + --hash=sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22 \ + --hash=sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328 \ + --hash=sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747 \ + --hash=sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab \ + --hash=sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499 \ + --hash=sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d \ + --hash=sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62 \ + --hash=sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512 \ + --hash=sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7 \ + --hash=sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf \ + --hash=sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa \ + --hash=sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57 \ + --hash=sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e \ + --hash=sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3 \ + --hash=sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0 \ + --hash=sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc \ + --hash=sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43 \ + --hash=sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe \ + --hash=sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31 \ + --hash=sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b \ + --hash=sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383 \ + --hash=sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217 \ + --hash=sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499 \ + --hash=sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8 \ + --hash=sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51 \ + --hash=sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509 \ + --hash=sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d \ + --hash=sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428 \ + --hash=sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead \ + --hash=sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc \ + --hash=sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1 \ + --hash=sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3 \ + --hash=sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0 \ + --hash=sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f \ + --hash=sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5 \ + --hash=sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731 \ + --hash=sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be \ + --hash=sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df \ + --hash=sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb \ + --hash=sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293 \ + --hash=sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e \ + --hash=sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7 \ + --hash=sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3 \ + --hash=sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3 \ + --hash=sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3 \ + --hash=sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a \ + --hash=sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9 \ + --hash=sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56 \ + --hash=sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562 \ + --hash=sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0 \ + --hash=sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15 \ + --hash=sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869 \ + --hash=sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7 \ + --hash=sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999 \ + --hash=sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01 \ + --hash=sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57 \ + --hash=sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838 \ + --hash=sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4 \ + --hash=sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1 \ + --hash=sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458 \ + --hash=sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3 \ + --hash=sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392 \ + --hash=sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3 \ + --hash=sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a \ + --hash=sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9 \ + --hash=sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785 \ + --hash=sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648 \ + --hash=sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49 \ + --hash=sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1 \ + --hash=sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7 \ + --hash=sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d \ + --hash=sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a \ + --hash=sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6 \ + --hash=sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231 \ + --hash=sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00 \ + --hash=sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac \ + --hash=sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea \ + --hash=sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c \ + --hash=sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81 \ + --hash=sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f \ + --hash=sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751 \ + --hash=sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68 \ + --hash=sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2 \ + --hash=sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c \ + --hash=sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57 \ + --hash=sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b \ + --hash=sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165 \ + --hash=sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737 \ + --hash=sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b \ + --hash=sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854 \ + --hash=sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87 \ + --hash=sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2 \ + --hash=sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847 \ + --hash=sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d \ + --hash=sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4 \ + --hash=sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8 \ + --hash=sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d \ + --hash=sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29 \ + --hash=sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051 \ + --hash=sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a + # via google-genai diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..5ea58b6 --- /dev/null +++ b/requirements.in @@ -0,0 +1,6 @@ +-r requirements-runtime.in + +flake8==7.3.0 +mypy==2.3.0 +pytest==9.1.1 +pytest-cov==7.1.0 diff --git a/requirements.txt b/requirements.txt index 6160ba5..f21e4dc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,1545 @@ -requests>=2.31.0 -beautifulsoup4>=4.12.0 -google-genai>=2.0.0 -pydantic>=2.0.0 -flake8>=7.0.0 -fastapi>=0.110.0 -uvicorn>=0.28.0 -httpx>=0.27.0 -python-jose[cryptography]>=3.3.0 -python-multipart>=0.0.9 -pytest>=8.0.0 -pytest-cov>=4.1.0 -pycountry>=24.6.1 -feedparser>=6.0 -anthropic>=0.40 -googlenewsdecoder>=0.1.7 +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --index-url=https://pypi.org/simple --no-emit-index-url --output-file=requirements.txt --strip-extras requirements.in +# +alembic==1.18.5 \ + --hash=sha256:06d8ba9d04558022f5395e9317de03d270f3dced49cee01f89fe7a13c26f14bc \ + --hash=sha256:1554982221dd17e9a749b53902407578eb305e453f71999e8c7f0a48389fff8e + # via -r requirements-runtime.in +annotated-doc==0.0.5 \ + --hash=sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101 \ + --hash=sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb + # via fastapi +annotated-types==0.8.0 \ + --hash=sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7 \ + --hash=sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0 + # via pydantic +anthropic==0.118.0 \ + --hash=sha256:524d835869b8e374510b3bcc552e28dbdafc035a61d63fc0088c4035286ef8fe \ + --hash=sha256:acb3c43b7e7592cf635fa21930e67a99d32641d68ccd53f625b52f71d7870d55 + # via -r requirements-runtime.in +anyio==4.14.2 \ + --hash=sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494 \ + --hash=sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f + # via + # anthropic + # google-genai + # httpx + # starlette +ast-serialize==0.6.0 \ + --hash=sha256:0067b25fce104eaae5b88383de9ab803faeb671831e14ca698b771b356e2600f \ + --hash=sha256:085de7f62dc9cc247eb01e965a362707d1d90b1d89a82c5bf78301a60a3c417b \ + --hash=sha256:093cb8bb91b720d8523580498d031791bb1bbaa048599c3d21085d380e11a596 \ + --hash=sha256:113b58346f9ceb664352032770caca817d4a3c86f611c6088e6ef65ddaa70f0e \ + --hash=sha256:13b7769970a39983b0adf2f38917b1cd3b8946f76df045756c3d741bc689f089 \ + --hash=sha256:2d947e45cafc4b09bd7528917fa84c517654a43de173c79785574b7b3068ac24 \ + --hash=sha256:305802f2ce2a7c4e87835078ea85c58b586ddda8095b92fe2ead9364ae19c80a \ + --hash=sha256:32ef62ec34cf6be20ad77d4799556638fbdf187f3ae10698dfb20ef9f2c89516 \ + --hash=sha256:3ae22a366b752ab4496191525b78b097b5b72d531752e3c1dd7e383a8f2c8a1a \ + --hash=sha256:4d6ef91590258ada18909b9caea344dac4de2013906b035473cd674a43f4b790 \ + --hash=sha256:4ed29121da8b3fdc291002801a1de0f76248fa07dce89157a5f277842cf6126e \ + --hash=sha256:5337cb256dcea3df9288205213d1601581536526b8f4da44b6974f1180f3252a \ + --hash=sha256:577a6c189068686869f5f1ddc38363f3ae1808a4753b577266f9202071a7bb66 \ + --hash=sha256:6e15ec740436e1a0d62de848641abe5f3a2f89a7f94907d534795ac91bbacf14 \ + --hash=sha256:6f7a408601bb3edaefb3bc67a4c01f5235e3253653b6a5729a2ee2382b35341c \ + --hash=sha256:82c312a7844d2fdeb4d5c48bd3d215bf940dafd4704e1a9bcf252a99010a99b1 \ + --hash=sha256:8670bfa51208a2c0c8d138928e40e998fab158f9200d53bb80c088b5b8eda7b8 \ + --hash=sha256:897ac47b5637be41c0c07061c8a912fafa967ef1dc73fa115e4bfa70882a093b \ + --hash=sha256:9f8a8b78b13173de6a9ec22111d9be674874cd5bdccda04f14ae5ebc2bef403a \ + --hash=sha256:a14191beec7e0c078d2fc1f6edc0aee88bcd4db9f18e1bc9f8052b559c22dddc \ + --hash=sha256:a4826809eb8597a8cd59fd924b6d7c285b8969a1e0007e2cb652cab62376270f \ + --hash=sha256:a7520b672827885bafeae7501f684d14d47d17e5f45256f9df547686cca52264 \ + --hash=sha256:aadd3ffcf4858c9726bf3515f7b199c7eadbe504f96028e4a87172c0da65a8fe \ + --hash=sha256:b1dac4e09d341c1300ba69cdcbe62867b32a8c75d90db9bf4d083bec3b039f0b \ + --hash=sha256:c4af9a1386166e40ed01464991806f89038a2d89782576c7774876fa77034e32 \ + --hash=sha256:c617417f9cbb0cb144f6283c3cbe0d2e0f01beaf9f608f662b21191058a626ec \ + --hash=sha256:c7b8b8f0c42f752ea00b2b7d7c090b3f80d9c1c5c75cadf16423790a0cc74081 \ + --hash=sha256:c901adbd750029b9ac4ad3d6aa56853e0ad4875119fbf52b7b8298afc223828b \ + --hash=sha256:ccd132fe8db56f61fe743b1f644d01b8d65b83248a8da506f3132bda86d6ed5e \ + --hash=sha256:cd5b91b9e6f2356ace3a556963b0cd783b395fbbb0bb17b4defc283415466e77 \ + --hash=sha256:cdc4e6f930b9090c2f92c9036ad12ffb8e6e44d4a5ba06f1458a05d60f203f7b \ + --hash=sha256:dcbed41e9386059fc0261d602445ede0976c2ecec2939688bcbcb9ed0b6f28b7 \ + --hash=sha256:e61580a69faf47e3689795367ed211f2a10fd741478cc0f36a0f128793360aad \ + --hash=sha256:f2ff3baffc3a29c1f15bc9098aa0c09763410262d5e6cef42116f7356c184554 + # via mypy +asyncpg==0.31.0 \ + --hash=sha256:027eaa61361ec735926566f995d959ade4796f6a49d3bde17e5134b9964f9ba8 \ + --hash=sha256:04d19392716af6b029411a0264d92093b6e5e8285ae97a39957b9a9c14ea72be \ + --hash=sha256:0b17c89312c2f4ccea222a3a6571f7df65d4ba2c0e803339bfc7bed46a96d3be \ + --hash=sha256:0bfbcc5b7ffcd9b75ab1558f00db2ae07db9c80637ad1b2469c43df79d7a5ae2 \ + --hash=sha256:0c89ccf741c067614c9b5fc7f1fc6f3b61ab05ae4aaa966e6fd6b93097c7d20d \ + --hash=sha256:12b3b2e39dc5470abd5e98c8d3373e4b1d1234d9fbdedf538798b2c13c64460a \ + --hash=sha256:18c83b03bc0d1b23e6230f5bf8d4f217dc9bc08644ce0502a9d91dc9e634a9c7 \ + --hash=sha256:19857a358fc811d82227449b7ca40afb46e75b33eb8897240c3839dd8b744218 \ + --hash=sha256:1b41f1afb1033f2b44f3234993b15096ddc9cd71b21a42dbd87fc6a57b43d65d \ + --hash=sha256:22bc525ebbdc24d1261ecbf6f504998244d4e3be1721784b5f64664d61fbe602 \ + --hash=sha256:22be6e02381bab3101cd502d9297ac71e2f966c86e20e78caead9934c98a8af6 \ + --hash=sha256:2657204552b75f8288de08ca60faf4a99a65deef3a71d1467454123205a88fab \ + --hash=sha256:2d076d42eb583601179efa246c5d7ae44614b4144bc1c7a683ad1222814ed095 \ + --hash=sha256:334dec28cf20d7f5bb9e45b39546ddf247f8042a690bff9b9573d00086e69cb5 \ + --hash=sha256:37a58919cfef2448a920df00d1b2f821762d17194d0dbf355d6dde8d952c04f9 \ + --hash=sha256:37fc6c00a814e18eef51833545d1891cac9aa69140598bb076b4cd29b3e010b9 \ + --hash=sha256:3b1fbcb0e396a5ca435a8826a87e5c2c2cc0c8c68eb6fadf82168056b0e53a8c \ + --hash=sha256:3df118d94f46d85b2e434fd62c84cb66d5834d5a890725fe625f498e72e4d5ec \ + --hash=sha256:3faa62f997db0c9add34504a68ac2c342cfee4d57a0c3062fcf0d86c7f9cb1e8 \ + --hash=sha256:480c4befbdf079c14c9ca43c8c5e1fe8b6296c96f1f927158d4f1e750aacc047 \ + --hash=sha256:54a64f91839ba59008eccf7aad2e93d6e3de688d796f35803235ea1c4898ae1e \ + --hash=sha256:5a4af56edf82a701aece93190cc4e094d2df7d33f6e915c222fb09efbb5afc24 \ + --hash=sha256:6d11b198111a72f47154fa03b85799f9be63701e068b43f84ac25da0bda9cb31 \ + --hash=sha256:72d6bdcbc93d608a1158f17932de2321f68b1a967a13e014998db87a72ed3186 \ + --hash=sha256:795416369c3d284e1837461909f58418ad22b305f955e625a4b3a2521d80a5f3 \ + --hash=sha256:831712dd3cf117eec68575a9b50da711893fd63ebe277fc155ecae1c6c9f0f61 \ + --hash=sha256:8df714dba348efcc162d2adf02d213e5fab1bd9f557e1305633e851a61814a7a \ + --hash=sha256:8ea599d45c361dfbf398cb67da7fd052affa556a401482d3ff1ee99bd68808a1 \ + --hash=sha256:9322b563e2661a52e3cdbc93eed3be7748b289f792e0011cb2720d278b366ce2 \ + --hash=sha256:98cc158c53f46de7bb677fd20c417e264fc02b36d901cc2a43bd6cb0dc6dbfd2 \ + --hash=sha256:9ea33213ac044171f4cac23740bed9a3805abae10e7025314cfbd725ec670540 \ + --hash=sha256:a429e842a3a4b4ea240ea52d7fe3f82d5149853249306f7ff166cb9948faa46c \ + --hash=sha256:a8d758dac9d2e723e173d286ef5e574f0b350ec00e9186fce84d0fc5f6a8e6b8 \ + --hash=sha256:aad7a33913fb8bcb5454313377cc330fbb19a0cd5faa7272407d8a0c4257b671 \ + --hash=sha256:b44c31e1efc1c15188ef183f287c728e2046abb1d26af4d20858215d50d91fad \ + --hash=sha256:ba5f8886e850882ff2c2ace5732300e99193823e8107e2c53ef01c1ebfa1e85d \ + --hash=sha256:bb223567dea5f47c45d347f2bde5486be8d9f40339f27217adb3fb1c3be51298 \ + --hash=sha256:bc2b685f400ceae428f79f78b58110470d7b4466929a7f78d455964b17ad1008 \ + --hash=sha256:bd4107bb7cdd0e9e65fae66a62afd3a249663b844fa34d479f6d5b3bef9c04c3 \ + --hash=sha256:bd5b6efff3c17c3202d4b37189969acf8927438a238c6257f66be3c426beba20 \ + --hash=sha256:bdb957706da132e982cc6856bb2f7b740603472b54c3ebc77fe60ea3e57e1bd2 \ + --hash=sha256:bef056aa502ee34204c161c72ca1f3c274917596877f825968368b2c33f585f4 \ + --hash=sha256:c0807be46c32c963ae40d329b3a686356e417f674c976c07fa49f1b30303f109 \ + --hash=sha256:c0e0822b1038dc7253b337b0f3f676cadc4ac31b126c5d42691c39691962e403 \ + --hash=sha256:c1a9c5b71d2371a2290bc93336cd05ba4ec781683cab292adbddc084f89443c6 \ + --hash=sha256:c1e1ab5bc65373d92dd749d7308c5b26fb2dc0fbe5d3bf68a32b676aa3bcd24a \ + --hash=sha256:c204fab1b91e08b0f47e90a75d1b3c62174dab21f670ad6c5d0f243a228f015b \ + --hash=sha256:c989386c83940bfbd787180f2b1519415e2d3d6277a70d9d0f0145ac73500735 \ + --hash=sha256:cea3a0b2a14f95834cee29432e4ddc399b95700eb1d51bbc5bfee8f31fa07b2b \ + --hash=sha256:dc5f2fa9916f292e5c5c8b2ac2813763bcd7f58e130055b4ad8a0531314201ab \ + --hash=sha256:e009abc333464ff18b8f6fd146addffd9aaf63e79aa3bb40ab7a4c332d0c5e9e \ + --hash=sha256:e5d5098f63beeae93512ee513d4c0c53dc12e9aa2b7a1af5a81cddf93fe4e4da \ + --hash=sha256:e6974f36eb9a224d8fb428bcf66bd411aa12cf57c2967463178149e73d4de366 \ + --hash=sha256:ebb3cde58321a1f89ce41812be3f2a98dddedc1e76d0838aba1d724f1e4e1a95 \ + --hash=sha256:eee690960e8ab85063ba93af2ce128c0f52fd655fdff9fdb1a28df01329f031d \ + --hash=sha256:f6b56b91bb0ffc328c4e3ed113136cddd9deefdf5f79ab448598b9772831df44 \ + --hash=sha256:f890de5e1e4f7e14023619399a471ce4b71f5418cd67a51853b9910fdfa73696 + # via -r requirements-runtime.in +beautifulsoup4==4.15.0 \ + --hash=sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7 \ + --hash=sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9 + # via -r requirements-runtime.in +certifi==2026.7.22 \ + --hash=sha256:62f22742b58a1a33014a2b6b706588a8d7e2a88ae7bd1a6ebe8c992928483775 \ + --hash=sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55 + # via + # httpcore + # httpx + # requests +cffi==2.1.0 \ + --hash=sha256:02cb7ff33ded4f1532476731f89ede53e2e488a8e6205515a82144246ffa7dcc \ + --hash=sha256:03e9810d18c646077e501f661b682fbf5dee4676048527ca3cffe66faa9960dd \ + --hash=sha256:0520e1f4c35f44e209cbbb421b67eec42e6a157f59444dfb6058874ff3610e5d \ + --hash=sha256:0582a58f3051372229ca8e7f5f589f9e5632678208d8636fea3676711fdf7fe5 \ + --hash=sha256:0611e7ebf90573a535ebdc33ae9da222d037853983e13359f580fab781ca017f \ + --hash=sha256:0a42c688d19fca6e095a53c6a6e2295a5b050a8b289f109adab02a9e61a25de6 \ + --hash=sha256:0a96b74cda968eebbad56d973efe5098974f0a9fb323865bf99ea1fd24e3e64c \ + --hash=sha256:10537b1df4967ca26d21e5072d7d54188354483b91dc75058968d3f0cf13fbda \ + --hash=sha256:11b3fb55f4f8ad92274ed26705f65d8f91457de71f5380061eb6d125a768fecd \ + --hash=sha256:15faec4adfff450819f3aee0e2e02c812de6edb88203aa58807955db2003472a \ + --hash=sha256:164bff1657b2a74f0b6d54e11c9b375bc97b931f2ca9c43fcf875838da1570dd \ + --hash=sha256:1854b724d00f6654c742097d5387569021be12d3a0f770eae1df8f8acfcc6acd \ + --hash=sha256:19c54ac121cad98450b4896fa9a43ee0180d57bc4bc911a33db6cab1efab6cd3 \ + --hash=sha256:1b96bfe2c4bd825681b7d311ad6d9b7280a091f43e8f63da5729638083cd3bfb \ + --hash=sha256:1e9f50d192a3e525b15a75ab5114e442d83d657b7ec29182a991bc9a88fd3a66 \ + --hash=sha256:1ff3456eab0d889592d1936d6125bbfbc7ae4d3354a700f8bd80450a66445d4d \ + --hash=sha256:2282cd5e38aa8accd03e99d1256af8411c84cdbee6a89d841b563fdbd1f3e50f \ + --hash=sha256:276f20fffd7b396e12516ba8edf9509210ac248cbbc5acbc39cd512f9f59ebe6 \ + --hash=sha256:2b71d409cccee78310ab5dec549aed052aaea483346e282c7b02362596e01bb0 \ + --hash=sha256:2e9dabb9abcb7ad15938c7196ad5c1718a4e6d33cc79b4c0209bdb64c4a54a5c \ + --hash=sha256:30b65779d598c370374fefabf138d456fd6f3216bfa7bedfab1ba82025b0cd93 \ + --hash=sha256:33eb1ad83ebe8f313e0df035c406227d55a79456704a863fad9842136af5ad7d \ + --hash=sha256:35aaea0c7ee0e58a5cd8c2fd1a48fdf7ece0d2699b7ecdda08194e9ce5dd9b3d \ + --hash=sha256:3681e031db29958a7502f5c0c9d6bbc4c36cb20f7b104086fa642d1799631ff8 \ + --hash=sha256:379de10ce1ba048b1448599d1b37b24caee16309d1ac98d3982fc997f768700b \ + --hash=sha256:37f525a7e7e50c017fdebe58b787be310ad59357ae43a053943a6e1a6c526001 \ + --hash=sha256:3b926723c13eba9f81d2ef3820d63aeceec3b2d4639906047bf675cb8a7a500d \ + --hash=sha256:3d7f118b5adbfdfead90c25822690b02bc8074fba949bb7858bec4ebd55adb43 \ + --hash=sha256:46b1c8db8f6122420f32d02fffb924c2fe9bc772d228c7c711748fff56aabb2b \ + --hash=sha256:47ff3a8bfd8cb9da1af7524b965127095055654c177fcfc7578debcb015eecd0 \ + --hash=sha256:4d433a51f1870e43a13b6732f92aaf540ff77c2015097c78556f75a2d6c030e0 \ + --hash=sha256:4f26194e3d95e06501b942642855aed4f953d55e95d7d01b7c4483db3ecff458 \ + --hash=sha256:510aeeeac94811b138077451da1fb18b308a5feab47dd2b603af55804155e1c8 \ + --hash=sha256:5972433ad71a9e46516584ef60a0fda12d9dc459938d1539c3ddecf9bdc1368d \ + --hash=sha256:5ecbd0499275d57506d397eebe1981cee87b47fcd9ef5c22cab7ed7644a39a94 \ + --hash=sha256:6274dcb2d15cef48daa73ed1be5a40d501d74dccd0cd6db364776d12cb6ba022 \ + --hash=sha256:63960549e4f8dc41e31accb97b975abaecfc44c03e396c093a6436763c2ea7db \ + --hash=sha256:64c753a0f87a256020004f37a1c8c02c480e725f910f0b2a0f3f07debd1b2479 \ + --hash=sha256:6af371f3767faeffc6ac1ef57cdfd25844403e9d3f476c5537caee499de96376 \ + --hash=sha256:6ca4919c6e4f89aa99c42510b42cf54596892c00b3f9077f6bdd1505e24b9c8d \ + --hash=sha256:6d194185eabd279f1c05ebe3504265ddfc5ad2b58d0714f7db9f01da592e9eb6 \ + --hash=sha256:702c436735fbe99d59ada02a1f65cfc0d31c0ee8b7290912f8fbc5cd1e4b16c3 \ + --hash=sha256:716ff8ec22f20b4d988b12884086bcef0fc99737043e503f7a3935a6be99b1ea \ + --hash=sha256:762f99479dcb369f60ab9017ad4ab97a36a1dd7c1ee5a3b15db0f4b8659120cd \ + --hash=sha256:7762faa47e8ff7eb80bd261d9a7d8eea2d8baa69de5e95b70c1f338bbe712f02 \ + --hash=sha256:78474632761faa0fb96f30b1c928c84ebcf68713cbb80d15bab09dfe61640fde \ + --hash=sha256:799416bae98336e400981ff6e532d67d5c709cfb30afb79865a1315f94b0e224 \ + --hash=sha256:7d034dcffa09e9a46c93fa3a3be402096cb5354ac6e41ab8e5cc9cd8b642ad76 \ + --hash=sha256:7d28dff1db6764108bc30788d85d61c876beff416d9a49cb9dd7c5a9f34f5804 \ + --hash=sha256:7d3538f9c0e50670f4deb93dbb696576e60590369cae2faf7de681e597a8a1f1 \ + --hash=sha256:7d5980a3433d4b71a5e120f9dd551403d7824e31e2e67124fe2769c404c06913 \ + --hash=sha256:7ea6b3e2c4250ff1de21c630fe72d0f63eb95c2c32ffbf64a358cf4a8836d714 \ + --hash=sha256:86cf8755a791f72c85dc287128cc62d4f24d392e3f1e15837245623f4a33cccc \ + --hash=sha256:88023dfe18799507b73f1dbb0d14326a17465de1bc9c9c7655c22845e9ddc3a2 \ + --hash=sha256:89095c1968b4ba8285840e131bf2891b09ae137fe2146905acae0354fbce1b5e \ + --hash=sha256:8d35c139744adb3e727cd51b1a18324bbe44b8bd41bf8322bca4d41289f48eda \ + --hash=sha256:8e74a6135550c4748af665b1b1118b6aab33b1fc6a16f9aff630af107c3b4512 \ + --hash=sha256:8f9ec95b8a043d3dfbc74d9abc6f7baf524dd27a8dc160b0a32ff9cdab650c28 \ + --hash=sha256:90bec57cf82089383bd06a605b3eb8daebf7e5a668520beaf6e327a83a947699 \ + --hash=sha256:95f2954c2c9473d892eca6e0409f3568b37ab62a8eedb122461f73cc273476e3 \ + --hash=sha256:961be50688f7fba2fa65f63712d3b9b341a22311f5253460ce933f52f0de1c8c \ + --hash=sha256:98fff996e983a36d3aa2eca83af40c5821202e7e6f32d13ae94e3d2286f10cfe \ + --hash=sha256:9b8f0f26ca4e7513c534d351eca551947d053fac438f2a04ac96d882909b0d3a \ + --hash=sha256:9d72af0cf10a76a600a9690078fe31c63b9588c8e86bf9fd353f713c84b5db0f \ + --hash=sha256:9d8272c0e483b024e1b9ad029821470ed8ec65631dbd90217469da0e7cd89f1c \ + --hash=sha256:a016194dbe13d14ee9556e734b772d8d67b947092b268d757fd4290e3ba2dfc2 \ + --hash=sha256:a5781494d4d400a3f47f8f1da94b324f6e6b440a53387774002890a2a2f4b50f \ + --hash=sha256:a95b05f9baf29b91171b3a8bd2020b028835243e7b0ff6bb23e2a3c228518b1b \ + --hash=sha256:aa7a1b53a2a4452ada2d1b5dade9960b2522f1e61293a811a077439e39029565 \ + --hash=sha256:ac0f1a2d0cfa7eea3f2aaf006ab6e70e8feeb16b75d65b7e5939982ca2f11056 \ + --hash=sha256:af5e2915d41fe6c961694d7bfdc8562942638200f3ce2765dfb8b745cf997629 \ + --hash=sha256:b6422532152adf4e59b110cb2808cee7a033800952f5c036b4af047ee43199e7 \ + --hash=sha256:b65f590ef2a44640f9a05dbb548a429b4ade77913ce683ac8b1480777658a6c0 \ + --hash=sha256:ba00f661f8ba35d075c937174e27c2c421cec3942fd2e0ea3e66996757c0fdd9 \ + --hash=sha256:bccbbb5ee76a61f9d99b5bf3846a51d7fca4b6a732fe46f89295610edaf41853 \ + --hash=sha256:bf01d8c84cbea96b944c73b22182e6c7c432b3475632b8111dbfdc95ddad6e13 \ + --hash=sha256:bf5c6cf48238b0eb4c086978c492ad1cbc22373fc5b2d7353b3a598ce6db887a \ + --hash=sha256:c16914df9fb7f500e440e6875fa23ff5e0b31db01fa9c06af98d59a91f0dc2e4 \ + --hash=sha256:c351efb95e832a853a29361675f33a7ce53de1a109cd73fd47af0712213aa4ce \ + --hash=sha256:c4165821e131d6d4ca444347c2b694e2311bcfa3fe5a861cc72968f28867beac \ + --hash=sha256:c5f5df567f6eb216de69be06ce55c8b714090fae02b18a3b40da8163b8c5fa9c \ + --hash=sha256:c941bb58d5a6e1c3892d86e42927ed6c180302f07e6d395d08c416e594b98b46 \ + --hash=sha256:c97f080ea627e2863524c5af3836e2270b5f5dfff1f104392b959f8df0c5d384 \ + --hash=sha256:cb96698e3c7413d906ce83f8ffd245ec1bd94707541f299d0ce4d6b0193e982b \ + --hash=sha256:cbb7640ce37159548d2147b5b8c241f962143d4c71231431820783f4dc78f210 \ + --hash=sha256:cdf2448aab5f661c9315308ec8b93f4e8a1a67a3c733f8631067a2b67d5913dc \ + --hash=sha256:d2117334c3af3bdcb9a88522b844a2bdb5efdc4f71c6c822df55486ae1c3347a \ + --hash=sha256:d53d10f7da99ae46f7373b9150393e9c5eab9b224909982b43832668de4779f5 \ + --hash=sha256:d9fafc5aa2e2a39aaf7f8cc0c1f044a9b07fca12e558dca53a3cc5c654ad67a7 \ + --hash=sha256:db3eb7d46527159a878ec3460e9d40615bc25ba337d477db681aea6e4f05c5d2 \ + --hash=sha256:dbf7c7a88e2bac086f06d14577332760bdeecc42bdec8ac4077f6260557d9326 \ + --hash=sha256:df2b82571a1b30f58a87bf4e5a9e78d2b1eff6c6ce8fd3aa3757221f93f0863f \ + --hash=sha256:df92f2aba50eb4d96718b68ef76f2e57a57b54f2fa62333496d16c6d585a85ca \ + --hash=sha256:eb4e8997a49aa2c08a3e43c9045d224448b8941d88e7ac163c7d383e560cbf98 \ + --hash=sha256:efc1cdd798b1aaf39b4610bba7aad28c9bea9b910f25c784ccf9ec1fa719d1f9 \ + --hash=sha256:f146d154428a2523f9cc7936c02353c2459b8f6cf07d3cd1ee1c0a611109c5d5 \ + --hash=sha256:f5bce581e6b8c235e566a14768a943b172ada3ed73537bb0c0be1edee312d4e7 \ + --hash=sha256:f9912624a0c0b834b7520d7769b3644453aabc0a7e1c839da7359f050750e9bc \ + --hash=sha256:fb62edb5bb52cca65fab91a63afa7561607120d26090a7e8fda6fb9f064726da \ + --hash=sha256:ff067a8d8d880e7809e4ac88eb009bb848870115317b306666502ccad30b147f + # via cryptography +charset-normalizer==3.4.9 \ + --hash=sha256:0327fcd59a935777d83410750c50600ee9571af2846f71ce40f25b13da1ef380 \ + --hash=sha256:03d07803992c6c7bbc976327f34b18b6160327fc81cb82c9d504720ac0be3b62 \ + --hash=sha256:04ce310cb89c15df659582aee80a0603788732a5e017d5bd5c81158106ce249c \ + --hash=sha256:0d861473f743244d349b50f850d10eb87aeb22bbdcc8e64f79273c94af5a8226 \ + --hash=sha256:0e94703ec9684807f20cfb5eed95c70f67f2a8f21ad620146d7b5a13677b93e5 \ + --hash=sha256:0fa1aec2d32bcc03c8fa0f6f1712caad1adc38509f31142112e5c9daf5b9c833 \ + --hash=sha256:16b65ea0f2465b6fb52aa22de5eca612aa964ddfec00a912e26f4656cbef890b \ + --hash=sha256:16d10d789dd9bcca1173c95af82c58433122564b7bc39385124be735a35cbe99 \ + --hash=sha256:19ac87f93086ce37b86e098888555c4b4bc48102279bae3350098c0ed664b501 \ + --hash=sha256:1d22856ffbe153a602df38e4a5464f0b748a54002e0d69ac6d2ad0a197cc99ec \ + --hash=sha256:21e764fd1e70b6a3e205a0e46f3051701f98a8cb3fad66eeb80e48bb502f8698 \ + --hash=sha256:231ddcbb35e2ff8973e1365db41fe0572662893b99a05deb183b68ad4c0c8bd4 \ + --hash=sha256:253a4a220747e8b5faf57ec320c4f5efb0cef05f647420bf267143ec15dba10a \ + --hash=sha256:280081916dc341820640489a66e4696049401ef1cf6dd672f672e70ad915aca3 \ + --hash=sha256:2a441ea71902098ffe78c5abe6c494f44160b4af614ed16c3d9a3b1d17fd8ee2 \ + --hash=sha256:304b13570067b2547562e308af560b3963857b1fa90bd6afd978130130fe2d6a \ + --hash=sha256:32286a2c8d167e897177b673176c1e3e00d4057caf5d2b64eef9a3666b03018e \ + --hash=sha256:33bdcc2a32c0a0e861f60841a512c8acc658c87c2ac59d89e3a46dacf7d866e4 \ + --hash=sha256:375b83ed0aecfce76c16d198fbc21f3b11b337d68662bea0a995046682a11419 \ + --hash=sha256:3c09a49d6cde137258beb3d551994a2927fd35ad5cf96aed573f61bbd67c5f84 \ + --hash=sha256:3d92613ec25e43b05f042302531ec0f00b8445190e43325880cbd6ab7c2581da \ + --hash=sha256:40a126142a56b2dfc0aacbad1de8310cbf60da7656db0e6b16eebd48e3e93519 \ + --hash=sha256:416c229f77e5ea25b3dfd4b582f8d73d7e43c22320302b9ab128a2d3a0b38efe \ + --hash=sha256:432786d3561e69aeeae6c7e8648964ce0ad05736120135601f87ac26b9c83381 \ + --hash=sha256:43b9e366a31fdd1c87d0eb08f579b4a82b723ea54338f040d6b4e518a026ea29 \ + --hash=sha256:440eede837960000d74978f0eba527be106b5b9aee0daf779d395276ed0b0614 \ + --hash=sha256:45b0cc4e3556cd875e09102988d1ab8356c998b596c9fced84547c8138b487a0 \ + --hash=sha256:476743fe6dfe14a2da12e3ac79125dc84a3b2cf8094369a47a1529b0cd8549fe \ + --hash=sha256:4773092f8019072343a7447203308b176e10199920eb02d6195e81bbb3274c29 \ + --hash=sha256:4b3dac63058cc36820b0dd072f89898604e2d39686fe05321729d00d8ac185a0 \ + --hash=sha256:4d1c96a7a18b9690a4d46df09e3e3382406ae3213727cd1019ebade1c4a81917 \ + --hash=sha256:51307f5c71007673a2bf8232ad973483d281e74cb99c8c5a990af1eefa6277d9 \ + --hash=sha256:51447e9aa2684679af07ca5021c3db526e0284347ebf4ffcec1154c3350cfe32 \ + --hash=sha256:58150c9f9b9a552505912d182ccdf26f6396fb6094816ceebcbb20eecabaed94 \ + --hash=sha256:5b10cd92fc5c498b35a8635df6d5a100207f88b63a4dc1de7ef9a548e1e2cd63 \ + --hash=sha256:5e226f6218febc71f6c1fc2fafb91c226f75bdc1d8fb12d66823716e891608fd \ + --hash=sha256:609b3ba8fcc0fb5ab7af00719d0fb6ad0cb518e48e7712d12fd68f1327951198 \ + --hash=sha256:60f44ade2cf573dad7a277e6f8ca9a51a21dda572b13bd7d8539bb3cd5dbedde \ + --hash=sha256:611057cc5d5c0afc743ba8be6bd828c17e0aaa8643f9d0a9b9bb7dea80eb8012 \ + --hash=sha256:6366a16e1a25018694d6a5d784d09b046edc9eac40ea2b54065c3052672516a1 \ + --hash=sha256:65a7ff3f705e57d392f7261b6d0550fe137c3019477431f1c355e0db0a7d3e15 \ + --hash=sha256:673611bbd43f0810bec0b0f028ddeaaa501190339cac411f347ac76917c3ae7b \ + --hash=sha256:67830fc78e67501f47bb950471b2dcb9b35b140084429318e862895a8e89c993 \ + --hash=sha256:68ce9f4d6b26d5ccbf7fd4459bf75f74a0a146677ebba80597df60cbdb20e6f4 \ + --hash=sha256:68e5f26a1ad57ded6d1cfb85331d1c1a195314756471d97758c48498bb4dcdf5 \ + --hash=sha256:69b157c5d3292bcd443faca052f3096f637f1e074b98212a933c074ae23dc3b8 \ + --hash=sha256:75286256590a6320cf106a0d28970d3560aad9ee09aa7b34fb40524792436d35 \ + --hash=sha256:78841cccf1af7b40f6f716338d50c0902dbe88d9f800b3c973b7a9a0a693a642 \ + --hash=sha256:78fa18e436a1a0e58dbd7e02fc4473f3f32cceb12df9dfca542d075961c307d2 \ + --hash=sha256:79580094b00d1789d1f93ea55bc43cb2f611910c72235b7657f3482ddcc1b22d \ + --hash=sha256:7b86a2b16095d250c6f58b3d9b2eee6f4147754344f3dab0922f7c9bf7d226c9 \ + --hash=sha256:83aed2c10721ddd90f68140685391b50811a880af20654c59af6b6c66c40513c \ + --hash=sha256:84fd18bcc17526fc2b3c1af7d2b9217d32c9c04448c16ec693b9b4f1985c3d33 \ + --hash=sha256:871ff67ea1aad4dfd91736464934d56b32dac49f9fbe16cddba36198a7b3a0db \ + --hash=sha256:898f0e9068ca27d37f8e83a5b962821df851532e6c4a7d615c1c033f9da6eedf \ + --hash=sha256:8a79d9f4d8001473a30c163556b3c3bfebec837495a412dde78b51672f6134f9 \ + --hash=sha256:8c041122946b7ba21bb32c45b1aa57b1be35527690aeb3c5c234521085632eee \ + --hash=sha256:90c44bc373b7687f6948b693cceaea1348ae0975d7474746559494468e3c1d84 \ + --hash=sha256:9104ed0bd76a429d46f9ec0dbc9b08ad1d2dcdf2b00a5a0daa1c145329b35b44 \ + --hash=sha256:920079c3f7456fa213e0829ed2073aaa727fd39d889ead5b4f35d0de5460d04f \ + --hash=sha256:93d59d504b230e83c7a843251681959a0b6a9cd76f6e146ce1b8a80eb8739af9 \ + --hash=sha256:9b2aff1c7b3884512b9512c3eaadd9bab39fb45042ffaaa1dd08ff2b9f8109d9 \ + --hash=sha256:9b8e0f3107e2200b76f6054de99016eac3ee6762713587b36baaa7e4bd2ae177 \ + --hash=sha256:9bb41182d93ea91f60b4bc8fbf4c820c69ef8a12ab2d917f3f1834f1acad07e8 \ + --hash=sha256:9cdef90ae47919cae358d8ab15797a800ed41da7aba5d72419fb510729e2ed4b \ + --hash=sha256:a1786910334ed46ab1dd73222f2cd1e05c2c3bb39f6dddb4f8b36fc382058a39 \ + --hash=sha256:a4cfde78a9f2880208d16a93b795726a3017d5977e08d1e162a7a31322479c41 \ + --hash=sha256:a4fbdde9dd4a9ce5fd52c2b3a347bb50cc89483ef783f1cb00d408c13f7a96c0 \ + --hash=sha256:aa99adc8f081b475a12843953db36831eaf83ec33eb46a90629ca6a5de45a616 \ + --hash=sha256:ac351b3b8014eead140e77e9717e2992c6bbe30b63bc3422422eb84865412e3d \ + --hash=sha256:ad41ba96094304aa090f5a30cb6e4fb3b3f1c264c523394b4c39bbacc4dc92ba \ + --hash=sha256:b5314963fce9b0b12743891de876e724997864ee22aa496f903f426c7e2fa5b2 \ + --hash=sha256:bcf74c1df76758a395bf0af608c04c82257523f55c9868b334f06270d0f2112b \ + --hash=sha256:bd47ba7fc3ca94896759ea0109775132d3e7ab921fbf54038e1bab2e46c313c9 \ + --hash=sha256:c0323c9daef75ef2e5083624b4585018a0c9d5e3b40f607eed81a311270b934b \ + --hash=sha256:c1225416b463483160e4af85d5fc3a9690ccb53fd4b1865a6437825f5ede3209 \ + --hash=sha256:c1c948747b03be832dceed96ca815cef7360de9aa19d37c730f8e3f6101aca48 \ + --hash=sha256:c25fe15c70c59eb7c5ce8c06a1f3fa1da0ecc5ea1e7a5922c40fd2fa9b0d5046 \ + --hash=sha256:cc1b0fff8ead343dae06305f954eb8468ba0ec1a97881f42489d198e4ce3c632 \ + --hash=sha256:cd6280cf040f233bd7d3407b743b4b4c74f70e8e1c4199cb112a62c941c0772a \ + --hash=sha256:cd6c3d4b783c556fa00bf540854e42f135e2f256abd29669fcd0da0f2dec79c2 \ + --hash=sha256:d4d6fcde76f94f5cb9e43e9e9a61f16dacefd228cbbf6f1a09bd9b219a92f1a1 \ + --hash=sha256:ddf4af30b417d9fe16481e9b81c27ab2a7cde1ff7ba3e85653b02db7d145dc7b \ + --hash=sha256:df115d4d83168fdf2cae48ef1ff6d1cb4c466364e30861b37121de0f3bf1b990 \ + --hash=sha256:df7276909358e5635ae203673ab7e509ddd224225a8d6b0790bf13eb2bde1cc5 \ + --hash=sha256:e4fd89cc178bced6ad29cb3e6dd4aa63fa5017c3524dbd0b25998fb64a87cc8b \ + --hash=sha256:e9701d0049d92c16703a42771b98d560b95248949f23f8cf7b4eddd201814fb9 \ + --hash=sha256:ee2f2a527e3c1a6e6411eb4209642e138b544a2d72fe5d0d76daf77b24063534 \ + --hash=sha256:f7fb7d750cfa0a070d2c24e831fd3481019a60dd317ea2b39acbcebc08b6ed81 \ + --hash=sha256:f840ed6d8ecba8255df8c42b87fadeda98ddfc6eeec05e2dc66e26d46dd6f58a \ + --hash=sha256:f86c6358749bd4fda175388691e3ba8c46e24c5347d0afd20f9b7edfc9faf07d \ + --hash=sha256:fa36ec09ef71d158186bc79e359ff5fdd6e7996fe8ab638f00d6b93139ba4fcf \ + --hash=sha256:fe2c7201c642b7c308f1675355ad7ff7b66acfe3541625efe5a3ad38f29d6115 + # via requests +click==8.4.2 \ + --hash=sha256:9a6cea6e60b17ebe0a44c5cc636d94f09bd66142c1cd7d8b4cd731c4917a15f6 \ + --hash=sha256:e6f9f66136c816745b9d65817da91d61d957fb16e02e4dcd0552553c5a197b76 + # via uvicorn +coverage==7.15.2 \ + --hash=sha256:075560438765b7a2ef43bf7aa7758661b53d889df47f062a31bda6c1ade553a2 \ + --hash=sha256:0901cfe6c13bcd2302da4f83e884555d2a22bda6e4c476f09ef204ba20ca536e \ + --hash=sha256:094dd37f3ef7b2da8b068b583d1f4c40f91c65197e16c52a71962d5d537fc5db \ + --hash=sha256:09f5c6ec5901f667bd97dd140b5b9a2586b10efec66f46fb1e6d8135f8b95bdf \ + --hash=sha256:0e55510bc98ae943cece9e667a6c0fe94c6a92913720dea34243657a17993d0c \ + --hash=sha256:1121caa19159a38b5463eaae4b1e1fde81e525b15ecc5e000cd5b1a108f743a8 \ + --hash=sha256:1268ac8fb9ddcd783d3948dbabaf80a5d53bfdaa0575e873e2139a692f797443 \ + --hash=sha256:1473b3ba8e7ee0f076117b1a72c23f579a2b9e2bb742f48a8d86ea27ca93f91a \ + --hash=sha256:17c432b5f73ad52ef46fb06019f6fa7c66ce381961cf0f7dfd1d3a4bd3a98145 \ + --hash=sha256:1adac78e5abc7c5438f7a209c9ca69d06542f0bf481d728b6989ea80b813fdf9 \ + --hash=sha256:1cd7a5beb7af3e864a13b1f0fb26efd3695da43ef0daf71e586adfffaf34d5b2 \ + --hash=sha256:1d16e3a7104ea84f03e614611b3edbf6fb6892554b3ab0fe7fbb3f2b2ef04376 \ + --hash=sha256:25fd15dd40a0a2c51a500d664ca29053c09c3259d998407bf982b6e114696138 \ + --hash=sha256:2617f8799d268fabdeef42a7e89ac3a23e1deee9025427db2df970f99a89a578 \ + --hash=sha256:26c3b04a6377fd7c09800921fa934e3a17c0020439cd59df73e73ae1d4b6a78c \ + --hash=sha256:29c052f7c83ccfcc5c577eaae025d2e4a9bb80daf03c0ac31c996e83b000ce88 \ + --hash=sha256:2f1ec6f304b156669cfde653b4e9a953f5de87e247ea02ac599bce0ab2744036 \ + --hash=sha256:2fbeeeecea279727f8ac16c8e1133ddfeee793e985c86ae343d6a5ce744eef8c \ + --hash=sha256:2ff08701be2d1556fc78b326c80a3e8042da09352ecb3819105f8e386c8a3071 \ + --hash=sha256:38c9518b7103826c403a461544e3c2e77151e8676d06eaed85911a97e962584a \ + --hash=sha256:3df60dc267f0a2ca23cb7a9ab1109c62b9335ffbf519fcfe167157c28c09b81d \ + --hash=sha256:3ed010aa1b69cda8e827aabfca9866216c980e2dca82ab9a78c5f83689964c8b \ + --hash=sha256:40f633c5c5fc783732f6312280122e859538fa24461235597c13d803ea9a108a \ + --hash=sha256:42ec3d989421b174a2ab607c1539f24127ad362757b7f1c0c0d7a2993f7eb37b \ + --hash=sha256:434e68d531858205895eb0d74b73d20b84260de426387d53c422a5acda2cf050 \ + --hash=sha256:44826758cfe73fcd0e6af5deb4ba6d5417cc1d13df3acb35c93484a11160f846 \ + --hash=sha256:4510fb9cdf6bb02dfa6af0be4a534b8102d086e22e4a33f8836df663da3d660d \ + --hash=sha256:48ccc6395958eda89093ecdc35644c86f23a8b23a7f4d44958812b721aad67c1 \ + --hash=sha256:4d3361879d736f469f45723c11ea1a5bbdaf1f6928f0e632c940378b5aa9b660 \ + --hash=sha256:582edc45c2040543fef83341be23c43024a3ab3ae0c2d8bc498a06282905ad40 \ + --hash=sha256:63022c4c8dec1d0342f05c3ede99842fe3d007689acc45e86f123a1746e4a026 \ + --hash=sha256:67d7602480a47bdf5b675635403625553ebaa70d5a62a657c035149fd401cea0 \ + --hash=sha256:68af907f595ab01a78f794932ff3bdf929c316d3000810d38dbc247129e26f8b \ + --hash=sha256:6aa28cfb6488e5453b5b762d65f73aa586380f6693a04d58078ce228a29b06c0 \ + --hash=sha256:6c0be82b4d4aa5b2704e08518e2252f3e3d110164bcca826816801052e48a7aa \ + --hash=sha256:6f6966fc30e6f06ca8f98fb0ce51eda6b111b3ee8d066a8b1ec9e77fa06ab55d \ + --hash=sha256:6fc448c377d6eeb00a47c673494bd9bae29280ca53987e1869e67ebedfe20658 \ + --hash=sha256:728a33676d4c3f0db977990a4bd421dcaa3be3e53b5b6273036fff6666008e89 \ + --hash=sha256:7466cc7ab6dc0db871d264bf99e8779f0917ee63d40730af0552f71535a6e072 \ + --hash=sha256:77f091ea3a9cc611cd29f433565476bc1936c084ac8eee00ea0e7e70c27e4199 \ + --hash=sha256:77f0ef5011df53a4bd1b35211ab122287f8d9b8d7aa1c4553e5c2deb24b1d446 \ + --hash=sha256:7c63387e21ab21f512c69c9756a8c7dadd322c7275edb064064433c9a09c3743 \ + --hash=sha256:7d29ca7bd67af6e12e74632d65f026eabc1364da5c254494cd914446a28a3ef7 \ + --hash=sha256:7dc2950a2992cd676d35c20ae63522836deeb034f08874699d14068710af3dc1 \ + --hash=sha256:7e8f27131dc7cd53de2c137dd207b3720919320b3c20d499dc30aa9ee6173287 \ + --hash=sha256:81f382c5a94b434ec1f6da607edb904c76d7212e618cd4d1bc9f97bed4120ef5 \ + --hash=sha256:835ec4e20b45f0a7f63ed78f94065aca00de033403df8377bfe8b9c6abc0a7be \ + --hash=sha256:8bb9f4b4279187560796a4cdaca3b0a93dd97e48ee667df005f4ed9a97403688 \ + --hash=sha256:8c726b232659cbd2ae57ade46509eb068c9bd7a06df9fcbff6fe484870006934 \ + --hash=sha256:913b6c56e110da40e035bbd168353bf7aaa2544a5eaccea5d98a4629aac156c7 \ + --hash=sha256:97a5c5457a9fb1d6c4e06cfb5dc835871fbfb6a6a51addc9e925bdeff5ef7440 \ + --hash=sha256:9854ca62c152874b2060772503535be2e8f53f70b8aaa7686b094888d872f984 \ + --hash=sha256:9911f31aad8906abe337c271343485cf20df5e70df5d2f57f9f136e7b55f26bc \ + --hash=sha256:9b5bd92ff1ec22e535eab0de75fa6db021992791f461a2aceb7822c625a1187d \ + --hash=sha256:9deddf09eecb717b7f980414b43d90a5b22ff3967d2949ab29cb0aa83d9e9098 \ + --hash=sha256:9e36686f7a442185db2400b3df171aac520869faf9deb59df687d28659eda2a6 \ + --hash=sha256:9f4432898c4bf2fba0435bbe35dd4437d7264565e5a88a21f5b49d8662a6b629 \ + --hash=sha256:a0f47002c6eeb7c280228467a4cb0cc15ca2103a8421b986b2d3ec04a0f9bd8b \ + --hash=sha256:a164b50081fc7357331c4024ef4d17b78ba325f8380d05f5a69599a7e05257ee \ + --hash=sha256:a29ec5305a7335aacee2d799e3422e91e1c8a12474986e2b3b07e315c91be82f \ + --hash=sha256:a300c6934e0989c327b9e8a1e110329da4641149f872bbe9f70168be66da76c1 \ + --hash=sha256:a4c46b247b5d4b78f613bd89fea926d32b25c6cc61a50bd1e99ba310348f3dad \ + --hash=sha256:a638db90c61cd219aeee65e83a24fdaa57269a741ae0cf773309208ac862cee3 \ + --hash=sha256:a63b9e190711134d581c4d703df5df09851b1acf99792c7aacbbe9f41f0283c9 \ + --hash=sha256:aaccad4129d735a8a4d526f26929894c9a4e8ef7034566f210b176749d6906e3 \ + --hash=sha256:ae901f7e55ba405c84ee1cab3d3e962e4e871e4a2bcb9c90911adbd69b42ac5a \ + --hash=sha256:afa29e2eff3d5729267e2cb2fd4ce9d61c952932fb2694e34ccb5d9540c6a296 \ + --hash=sha256:affd532502d34c0472d0cdb181325c89f1d2c44992fef0c17e88e7b1576259a1 \ + --hash=sha256:b171bdd71cb7ff792bf32e376173b0ace7e7963e7e57c58dfc42063a6a7174cd \ + --hash=sha256:b868acc62aa5de3be7a9d05c2333bf8359ca987e43f9cb30ff8fbda6a024ab73 \ + --hash=sha256:b9a6367e4aff723e8ee8190836836124284e8fcd4265e307c844010cfa074f3f \ + --hash=sha256:bbc808daf4f5cd567af8075ecc72d21c6dfef9a254709a621a84c217c935ebc0 \ + --hash=sha256:bbf44513ceb1589e31948e20eafbde9deaface90e1a1afa5f5f77b4423d17ce6 \ + --hash=sha256:bcc0aae933921d03096f53b0b03eeb702129fd406dee59f08d2efacc68681fa5 \ + --hash=sha256:bfd341ccf78128e72c094bc70cc25b3ef309c33c7c2c66ba3ed4309549e02de1 \ + --hash=sha256:c6a98d698f9e2c8008d0370ec7fc452ebfcc530002ae2d0061170d768b992589 \ + --hash=sha256:cb0fddaa6884be6aae36ced9544b5e90f7d5f03845a2853bf47a14953a4e8688 \ + --hash=sha256:cee0f89f4767a6057c8fbf168f8135f18be651300496086bd873e3189fed0487 \ + --hash=sha256:d17d7512151fedfcc64c1821a8977fc9be0dbf495754669afcab7b57abc98ae9 \ + --hash=sha256:d46e62cb35d91e6e2589fda6d28074426b0e276422b5d2ebef2c6b11dc60dbfd \ + --hash=sha256:d50dd325e18ec25bfcc10cd7f99b04df1ab9ec76b0918c260e60817ad0643dee \ + --hash=sha256:db9c8438057e5b0f6a22a0af99c0c1d26b57fbbdbd1be5861ddb8f897fcc3a2d \ + --hash=sha256:dee88b1ed88587abd8c0269a1fc1f4cc77f7750d1dfde2869e2a123af420e67d \ + --hash=sha256:dfd3db045e95960ae3683059571e597fda7cc610106a8916f77c5839048c1deb \ + --hash=sha256:e26ff680768b8095e8874aabe0e9d3a47a2a9f176a8340d05f8604c56457c23a \ + --hash=sha256:e370c12133095ff18432de8c044962be85a5a96d90c6fcbce8e17e76236d2328 \ + --hash=sha256:e38def96ad59853824c97953fdcd2c320a84ba3ce99b417db78af8bb6c3db635 \ + --hash=sha256:e8f91bce78e32343af184c3b7fa28fcf5a9e2641f4b6623d392038f804939188 \ + --hash=sha256:eb6bcae8d1a9d305351ecb108232441d11c5cfe9de840a04388ba5d2db8d735c \ + --hash=sha256:f653e5d7248c1191ec988a85c72edeab46c3ff44f90639a4ed4874ec0be90243 \ + --hash=sha256:fe41909c9515c3bfdb5f02c4d1f857dba322d9a9a1178069b91eea77889df63a + # via pytest-cov +cryptography==49.0.0 \ + --hash=sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001 \ + --hash=sha256:07cab27cc7b7e0fd28e5e26bb9eeedde5c135c868b46de4a27845abe94af6122 \ + --hash=sha256:084ef1af862eb07ec46d25f68689f2102a9fc0e05ce7b80f14f5fe51e4eef0f6 \ + --hash=sha256:0b82e28ee398a386f0807bba7884d30f25218855690f45115831bcce5d90822c \ + --hash=sha256:0e959b578856a3924bc0cbb710fc12c387b9412a951389f3ca61704a9e25f325 \ + --hash=sha256:0f21641cf4b30fca7aee061ced0ec7ad7b073518088b7c9969a297c0ae796c69 \ + --hash=sha256:196ecd6a36e4e9aa10270393bb98d8df88fccee0bf1e5128b91ae4eb4375896d \ + --hash=sha256:2400ef9c9e2299a25614eb1dea3db54a69b1349efd043bfac9c67630d136df36 \ + --hash=sha256:28d8b15e6275f12c8a207dc309dfa957903c927d08d0cc937ee3f63f200693cc \ + --hash=sha256:2afe9051da7ae7bd5905da5a949280c7d2bb75682e188f650a9d0f2756b834c6 \ + --hash=sha256:2eda353d8a27bcbcaa4cbed18994a74ab4d19a2ca897db188ea269ab9b71419b \ + --hash=sha256:32703d93296f5c1f4b53349ad3a250c2cae0fdecd3a3dd5d47e616d8d616af27 \ + --hash=sha256:33cd0565932807baddb67b96dbee92f2c374b5c89dee09fd74079aeb8c8dba61 \ + --hash=sha256:35b151772baff2c74cba7fa290ceaff4c3b11c0c881eb93eb5dbc05a7cfbba18 \ + --hash=sha256:36d1709f992593689b45bda411498d62c6e365f2ca00b84657d4dadd24de16db \ + --hash=sha256:42b0684e0e40cf26122427802486f6d93aea593612603a94fbf260c7eb1e9c1b \ + --hash=sha256:4ae387c9cb68ea569ca17e490d66d8142b81c3cc814bf179974b7d146e490bbb \ + --hash=sha256:53ecee2e23f7169b6117e99fc8a944e5e50f79e69758a83b52a00cb98ab2b2d2 \ + --hash=sha256:66ec79c3904820572d7e987abdf304281f141d37ad9a489b8e97066e7b9b6459 \ + --hash=sha256:67e1d20ad9ef3a563c59ef22e7a8a0b8210bd26604369ea4a30a7c66aefe504e \ + --hash=sha256:6f2debedf9ca60cf1d5bd466475638af5130f89965605cd818484d19987d3a21 \ + --hash=sha256:6fc361c34fb6aac015ce19435876635e5c6d21db31998b0920f675f131e043b8 \ + --hash=sha256:73a205dce83953d131a4aa1e0fd917a2fd1c5b1eef251e9d7152efefcbf5caf7 \ + --hash=sha256:7abcee80084cda3f7691f3eb1ce480d8df49cec637b429aa35986c1de71738aa \ + --hash=sha256:8c25ceb16df5b9435f3f6a9829204985b0e0cbee3b48aacd432c7d2c850b44d9 \ + --hash=sha256:966fe0e9c67490071f14c0d2b1cb2dfb3023c5ce39457343931415f08382f2db \ + --hash=sha256:9e82dcc8e56052715fb18b2429e3bca4823b1629136a2084fc45a9a5cecb9b64 \ + --hash=sha256:b20133d204d2bb56ba047642199603876c872026ca53e79c35b83772ab2cc505 \ + --hash=sha256:b39efa323140595abd3ecca8529d321ae50f55f3aa3ba9cc81ea56a6011953d5 \ + --hash=sha256:b47db11c2c3525083296069b98ac5221907455e989ae0c2e3008bde851921615 \ + --hash=sha256:b87e65d263b3e5d3bb92a57e2a6638e2f31110fa7aa890c7b2dbba42248d0a3f \ + --hash=sha256:b970c6da94d5bb18629db453d14f2a1300f6bf59b61e9b82377931ef95504866 \ + --hash=sha256:be9fcb48a55f023493482827d4f459bd263cc20efde64f204b97c123201850c6 \ + --hash=sha256:c2bc30226390d60ea19d9f82b19db005fe0452154a23c1c410c12ea801e43561 \ + --hash=sha256:c83782480a4a9da4d0feb51950131ba32e12e70813848b3343f6e18c28a66838 \ + --hash=sha256:cbc77da8c523d5abd028635ba850a6966fcee2c82e2bf65a41d1d8afe0f98be9 \ + --hash=sha256:ccac2bfebc306b862133e3bb71f3f6ee8bb525240089b2d952e4144b3a6d5da7 \ + --hash=sha256:d0527ce944105f257f605a827d6ebead966c752038b6e8656abb9c5edee6fc68 \ + --hash=sha256:d8ecde755e2e91bf773fc94e8c9d730cd7f2007004cb492263a794ec3899a1c8 \ + --hash=sha256:e3fb64c420688e5319ae25113a354015abbd8dffbfbc41781a1ea66fc7622ac3 \ + --hash=sha256:e5dfc1e64de5677cec922ffa8da89c546d0415bf6efdf081842e5d44c84e1f0e \ + --hash=sha256:ec5e529fb80935c94fe7b729f9972b50e351a0e6b50aa294fd5cabb109fcc29a \ + --hash=sha256:f37d847238971164fdbc68ade6f6574aecc9c0af714190e2083429ff68f4ce9d \ + --hash=sha256:f78ff2c9ed8dc2d036b0f4d640e22522213d047c1b14e61205a7e55c80a494d4 \ + --hash=sha256:f89660a348f4f78a92366240a61404e337586ef7f5909a2fef59ca88ef505493 \ + --hash=sha256:fc1e275c2f1d97b1a6450b8b0ea3ebfa6e087a611c2b26cb2404d48588abab7b + # via + # google-auth + # python-jose +distro==1.9.0 \ + --hash=sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed \ + --hash=sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2 + # via + # anthropic + # google-genai +docstring-parser==0.18.0 \ + --hash=sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015 \ + --hash=sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b + # via anthropic +ecdsa==0.19.2 \ + --hash=sha256:62635b0ac1ca2e027f82122b5b81cb706edc38cd91c63dda28e4f3455a2bf930 \ + --hash=sha256:840f5dc5e375c68f36c1a7a5b9caad28f95daa65185c9253c0c08dd952bb7399 + # via python-jose +fastapi==0.139.2 \ + --hash=sha256:333145a6891e9b5b3cfceb69baf817e8240cde4d4588ae5a10bf56ffacb6255e \ + --hash=sha256:b9ad015a835173d59865e2f5d8296fbc2b317bf56a2ba1a5bfbdd03de2fd4b1c + # via -r requirements-runtime.in +feedparser==6.0.12 \ + --hash=sha256:64f76ce90ae3e8ef5d1ede0f8d3b50ce26bcce71dd8ae5e82b1cd2d4a5f94228 \ + --hash=sha256:6bbff10f5a52662c00a2e3f86a38928c37c48f77b3c511aedcd51de933549324 + # via -r requirements-runtime.in +flake8==7.3.0 \ + --hash=sha256:b9696257b9ce8beb888cdbe31cf885c90d31928fe202be0889a7cdafad32f01e \ + --hash=sha256:fe044858146b9fc69b551a4b490d69cf960fcb78ad1edcb84e7fbb1b4a8e3872 + # via -r requirements.in +google-auth==2.56.2 \ + --hash=sha256:c8270ea95b2697b74e3d8438ae9c5b898e38b623b915c7b5c5635921e7de68a6 \ + --hash=sha256:e28f103ca8091fb7012b99c44243d7366c29863713b8e34a220c3322b7a07051 + # via google-genai +google-genai==2.14.0 \ + --hash=sha256:a9d1f4f362d76280f1be1340fcb3c86e63dbca128f6a4ae09d86ab47ff7148e8 \ + --hash=sha256:ae7172cdd35695189b516b33a878e4132e5daa2dbc03a5b44cddfa8a82fad664 + # via -r requirements-runtime.in +googlenewsdecoder==0.1.7 \ + --hash=sha256:0fcd8fa2aa6a145f84798921aeef0ec45d40bcf5ceebb467742f0efd068d3902 \ + --hash=sha256:a8be897bb41864a82d81da6e05777d8c795989cf65f14b41e8e772b3f5a8354d + # via -r requirements-runtime.in +greenlet==3.5.4 \ + --hash=sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20 \ + --hash=sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c \ + --hash=sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994 \ + --hash=sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8 \ + --hash=sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d \ + --hash=sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9 \ + --hash=sha256:13b980043cb1b3134e81ea469da1250ddcc6bfe6d245bbaa59168d9cdc8f228f \ + --hash=sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809 \ + --hash=sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c \ + --hash=sha256:188e4d142f243051d92a1f5c244a741da02dddc070a0620c842804d7b56d008c \ + --hash=sha256:1e1a4a684b16c45ba324e60b32a4386a87722bcb815d2a149d2182f9b401ca72 \ + --hash=sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3 \ + --hash=sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02 \ + --hash=sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c \ + --hash=sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c \ + --hash=sha256:2cdaadc3d31445a8f782bde3cd37e49a2c2a9c6da6daf76a3e34c683b271a3c7 \ + --hash=sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec \ + --hash=sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c \ + --hash=sha256:32802705c2c1ff25e8237b3bdacf2594fa02be80af8a66703eb7853ea7e68686 \ + --hash=sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861 \ + --hash=sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8 \ + --hash=sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0 \ + --hash=sha256:3d66250e8b09f182ede05490998c818b5961f7a3640332d44c4927caec7bbfe4 \ + --hash=sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9 \ + --hash=sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3 \ + --hash=sha256:42afdc1ab5f66da8c586c32af9224a74a706b4f0ea0dc3a4188a0860a09c65c9 \ + --hash=sha256:4ab9f0704bccf6d3b38e0d2130b7b33271cff11453690da074fa280c3aa8e8e7 \ + --hash=sha256:57aa201b351f7c7c75627c60d29e4d5b97a07d37efeb62b903466fca42c097d7 \ + --hash=sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd \ + --hash=sha256:60149df8f462d1b230038e6590c23c3b4768bb5d6c022b3b6e82532b34b0b8a3 \ + --hash=sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2 \ + --hash=sha256:69173331fbc5d64bfac0065d7e22c39cfcd089e9b18d125bdcd5079363b09616 \ + --hash=sha256:70bdfacdc183dac838b2a0aaff2dd6134a457c52fe68a9c6bbab435483d2b9df \ + --hash=sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf \ + --hash=sha256:77d6ce04fed0d9aeed42e0f37923cc43eba9b027bdd9c34546bb4ccd143d0fe0 \ + --hash=sha256:791fdfeeb9c6e0c7b10fa151bf110d2a6974866f13dcb5b1c7efae698245893a \ + --hash=sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f \ + --hash=sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22 \ + --hash=sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356 \ + --hash=sha256:89f3738167bab8c1084b94e23023d41d247117ac149fa0fbcb5bd4cf6262b353 \ + --hash=sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e \ + --hash=sha256:9667862a2e38ad379f11b845daeda22c8989186def44f06962c9c4c05e556da7 \ + --hash=sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5 \ + --hash=sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8 \ + --hash=sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde \ + --hash=sha256:a2d614cb2372c7101a12ea8b96dd56f81c986d247c5a73db67063f3ed1ca4a52 \ + --hash=sha256:ac5bf81d79d2c8eeb2ef6359b2e1687a1e9ebf46c2b1f970da9a9255df51d190 \ + --hash=sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05 \ + --hash=sha256:ae53534b5dec0f4c2ec26f898f538dc8ea1ca3ef2927d597a9439e40a09da937 \ + --hash=sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867 \ + --hash=sha256:b7a5f095767c4493afcd06067f2bb3b8716e3f3f9e92b99c88e7e99f885b3d4d \ + --hash=sha256:b7c895310363f310361e0fe2072af85269d2a2a285cd04c0c59e79a5e3670dcf \ + --hash=sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f \ + --hash=sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd \ + --hash=sha256:c38c902a0986eba1f6e7ba1ab39ad5195926abde90f3fe080e08212db62176da \ + --hash=sha256:c3fe76c2cac86b4f7a1e92865ac0a54384deb05c92986287c1a7110d9bd53071 \ + --hash=sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88 \ + --hash=sha256:c90e930c9c192e5b3ee9fb8bcd920ea3926155e2e3ded39fc697323addecee17 \ + --hash=sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c \ + --hash=sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66 \ + --hash=sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb \ + --hash=sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c \ + --hash=sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25 \ + --hash=sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0 \ + --hash=sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927 \ + --hash=sha256:dc418cf4c873357964d6624445ed09472e50def990c65dd4e76fc3ba8cd9cef6 \ + --hash=sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c \ + --hash=sha256:e849e6e139b9671adeac505f72fc05f4af7fd1921faef40295e214fc3b361b59 \ + --hash=sha256:e883de250e299654b1f1680f72a1a9f9ba62c9bd1bce84099c90657349a8dfbb \ + --hash=sha256:e9a5e3406e3ed8125ae1a3b37c12f3434e2b1f0fa053197c5557895b4fb09606 \ + --hash=sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef \ + --hash=sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3 \ + --hash=sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da \ + --hash=sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132 \ + --hash=sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7 \ + --hash=sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f \ + --hash=sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2 \ + --hash=sha256:f88193799d43dbf8c8a806d6405c9c52fe2af40bf75072a606357b33cc336c7f \ + --hash=sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667 + # via sqlalchemy +h11==0.16.0 \ + --hash=sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1 \ + --hash=sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86 + # via + # httpcore + # uvicorn +httpcore==1.0.9 \ + --hash=sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55 \ + --hash=sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8 + # via httpx +httpx==0.28.1 \ + --hash=sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc \ + --hash=sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad + # via + # -r requirements-runtime.in + # anthropic + # google-genai +idna==3.18 \ + --hash=sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2 \ + --hash=sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848 + # via + # anyio + # httpx + # requests +iniconfig==2.3.0 \ + --hash=sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730 \ + --hash=sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12 + # via pytest +jiter==0.16.0 \ + --hash=sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536 \ + --hash=sha256:0758ab7747a984797cf048e8eedea1d8ef39d7994b25611daf5b48fc903e8873 \ + --hash=sha256:0df91907609837f33341b8e6fe73b95991fdaa57caf1a0fbd343dffe826f386f \ + --hash=sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3 \ + --hash=sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce \ + --hash=sha256:10438939205546132189c8e74a2d536a707841f3a25cd7c74ee91fe503407a26 \ + --hash=sha256:12288303c9844e61e1651d02a9a6f6633e47d39f897d6991d1427161ce6b746e \ + --hash=sha256:17ca7fae79f6d99cd9a042b75f917eaada7b895cfc7dd2ee3a16089dcaec7a85 \ + --hash=sha256:1878349266f8ee36ecb1375cc5ba2f115f35fd9f0a1a4119e725e379126647f7 \ + --hash=sha256:1a0f05e229edb29e68cdd0ccb83cea13b64263416120cf943767a6fd72e6787f \ + --hash=sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207 \ + --hash=sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5 \ + --hash=sha256:26d122613ada2b708eb714695446f40fce5bdf2edb4b02116dec62faa62dfab3 \ + --hash=sha256:2a0d46ef67cc58d906a6132dd3040ca70ae4f0b0d7c9c052fe432c658a69b3f6 \ + --hash=sha256:2a8e9e39cf083016137aa5cadafe3188adc2ba6ba1fbf1e5d18889ad3e9ad056 \ + --hash=sha256:2c842cbf374a8daf50b2c04212995bee34ca2ac2cdc29a901b4cdb072c9c4131 \ + --hash=sha256:2ed5738ae4af18271a51a528b8811b0cbfa4a1858de9d83359e4169855d6a331 \ + --hash=sha256:3126a5dbad56401989ac769aca0cb56005bfb3e2366eea0ca99d1a91c3c1ee03 \ + --hash=sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93 \ + --hash=sha256:3a23f0e4f957e1be65752d2dfac9a5a06b1917af8dc85deb639c3b9d02e31290 \ + --hash=sha256:3b21b412b899fd8bd51a3046934b59a3bb068b79f70a5c6010053ac77cc53f0c \ + --hash=sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a \ + --hash=sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284 \ + --hash=sha256:3fb7bc819187b56dc48aa5c833aaf92257da8e07efdb9306156667bd2eeb491c \ + --hash=sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195 \ + --hash=sha256:44d019fa8cdaf89bf29c71b39e3712143fdd0ac76725c6ef954f9957a5ea8730 \ + --hash=sha256:46add52f4ad47a08bfb1219f3e673da972191489a33016edefdb5ea55bfa8c48 \ + --hash=sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69 \ + --hash=sha256:491e7d072a253b156fff46b78bceac4652a697aa8d7082c9c18c03d7b7917d24 \ + --hash=sha256:4a1d68ff7ca1d3b5dee20a97a3decda7d5f15003823bf6d140c81f8561d3bc5c \ + --hash=sha256:51d7b836acb0108d7c77df1742332cac2a1fa04a74d6dacec46e7091f0e91274 \ + --hash=sha256:560b2cf3fb03240cd34f27409a238547488708f05b7c3924f571a60422251ec7 \ + --hash=sha256:561926e0573ffe4a32498420a76d64b16c513e1ab413b9d28158a8764ac701e5 \ + --hash=sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106 \ + --hash=sha256:5a71d0d2014c3275043e1170bf3d4e771493cb0dcf07be54c567155f4d8ee64b \ + --hash=sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00 \ + --hash=sha256:5af7780e4a26bd7d0d989592bf9ef12ebf806b74ab709223ecca37c749872ea9 \ + --hash=sha256:5cf109d010b4b05a105afb3d43be36a21322d345ad3111e13d15f680afef0e5b \ + --hash=sha256:5ed466aee31294d7cdcd4d37dfe5c42c97bc29d9a5f00eacf24504358309cb9b \ + --hash=sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818 \ + --hash=sha256:62c1b7fe1f77925acf5af68b6140b8810fa87dfd4dc0a9c8568ec2fa2a10429c \ + --hash=sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2 \ + --hash=sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077 \ + --hash=sha256:64c0203212098470032aabcde9356fc168f377aade3e43def61dfe17e92f2037 \ + --hash=sha256:64d613743df53199b1aa256a7d328340da6d7078aac7705a7db9d7a791e9cfd2 \ + --hash=sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84 \ + --hash=sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a \ + --hash=sha256:67c3bc1760f8c99d805dcab4e644027142a53b1d5d861f18780ebdbd5d40b72a \ + --hash=sha256:67fddeda1688f0cce2d2ae83ccf8a80f79936f2d2997d6cc2261f82fdb54a4d3 \ + --hash=sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb \ + --hash=sha256:70a490b55634dc0d2606ce8a8e01b1d62459011beb368d15d76e1eaf62460e3d \ + --hash=sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe \ + --hash=sha256:741eed508c233a76313a1c7b001f8f21b82f14327e9196ae8bd29a2cc164ae84 \ + --hash=sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c \ + --hash=sha256:7c9610fd25ebccb43fca584136f5c2fbb26802447eccd430dfdbab95a0fd5126 \ + --hash=sha256:850ccb1d7eedb4200f4014b1c0e8a577de114fc3cd88faad646dcc9bc4bb12ad \ + --hash=sha256:8597d23c87f59294f83bcb6229b9ed1fccee13dbba967b46930d2f1759466fee \ + --hash=sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053 \ + --hash=sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929 \ + --hash=sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450 \ + --hash=sha256:8fc4d94713c4697347e38faf7d6ef91547c142219bdcfc7220c4870879974244 \ + --hash=sha256:96e38eea538c8ddf853a35727c7be0741c76c13f04148ac5c116222f50ece3b3 \ + --hash=sha256:9acf1b2faec82d998811ecce7ae84d9005e53410773e9d37d61cdc424ba4581b \ + --hash=sha256:9c8a956fd72c2cf1e730d01ea080341f13aa0a97a4a33b51abebe725b7ae9ca9 \ + --hash=sha256:9ec553a99b0987efd7a3645a1a825cf29c224e494db267a83369fcc8da9aeda5 \ + --hash=sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734 \ + --hash=sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585 \ + --hash=sha256:a5c54ef4ff776d9675837ef535b3308d6e31c208d43ebc44a0f7ab8a208c68f7 \ + --hash=sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29 \ + --hash=sha256:adff21bc78edfe086c15eb495b900306076de378dc2337c132401fc39bd79c91 \ + --hash=sha256:b1e7923093a376d93c6eb507c77045ae258d689ba577392846a1b3f10d0b09a9 \ + --hash=sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee \ + --hash=sha256:b42e9ff5376819c053da25809a8d4b6fa6e473b4856ebe42e298ac958be3d7f9 \ + --hash=sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e \ + --hash=sha256:b7dc85ea77d4abbae8bad0d3538678aedee75bceec4e2f6c8dfb1c74772e5aa5 \ + --hash=sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db \ + --hash=sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd \ + --hash=sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620 \ + --hash=sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567 \ + --hash=sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898 \ + --hash=sha256:c22a488f7b9218e245a0025a9ba6b100e2e54700831cf4cf16833a27fba3ad01 \ + --hash=sha256:c4b4717bdb35ae456f831a6b08d01880fff399887a6bbc526a583a406e484eea \ + --hash=sha256:c5fc4f8def331036a7b8e981b4347ebe409981edbc8308a5ea842b8c3614fa6c \ + --hash=sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026 \ + --hash=sha256:c90c0f63df322be920eda6ce622e3083d8906ba267f8220fe7873213b8b4430e \ + --hash=sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e \ + --hash=sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8 \ + --hash=sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883 \ + --hash=sha256:d284fb8d94d5855d60c44fefcab4bf966f1da6fada73992b01f6f0c9bc0c6702 \ + --hash=sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09 \ + --hash=sha256:d5bf78d0e05e45cfdd66558893938d59afe3d1b1a824a202039b20e607d25a72 \ + --hash=sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8 \ + --hash=sha256:d8f80521644426d451e70f00c7974240cab8f6ee088aedaa9af2697153ab7805 \ + --hash=sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af \ + --hash=sha256:dab907db06fc593645e73109acf4581ba5b548897d28b9348dc41ddc8343b2d3 \ + --hash=sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf \ + --hash=sha256:e03a5f21a5ce96a9441b8cb32719a8b88ed5388f53e0f339c5bcf54f1317f9d0 \ + --hash=sha256:e34e97bda77eb63242a410243c071e28ac7e0d8c0948c5ee658498690a4b2f2f \ + --hash=sha256:e431cfc9caf44c1d5459ff77d4e64cbf85fddb6a35dad836a15c6a9ec23087c1 \ + --hash=sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e \ + --hash=sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e \ + --hash=sha256:f17d61a28b4b3e0e3e2ba98490c70501403b4d196f78732439160e7fd3678127 \ + --hash=sha256:f3bd327cdfa118bc1ce69c214c2678571d5bd39b8ccd0ebf43a54db00541ba9a \ + --hash=sha256:f4444a83f946605990c98f625cdd3d2725bfb818158760c5748c653170a20e0e \ + --hash=sha256:fb08c276dd02dac3a284acdd02cacc630d2e3cd6572a4b85519f35cbd133c3de + # via anthropic +librt==0.13.0 \ + --hash=sha256:05d96b80b95d3a2721b619f8982b8558848b04875bb4772fd54842b59f61dd97 \ + --hash=sha256:0763ca2ab66058174f9dee426dc64f5e0a89c24a7df8d3fe3f1836c04e25de4b \ + --hash=sha256:091b60a4d2174fc1ec5c34cdc0b72efb6224753d76b7da61ebeab7a191aec8bd \ + --hash=sha256:0b795f5fc70fbbb787ceaf79bb3a0d627bcc33c53de51741755263ec406b775a \ + --hash=sha256:0d38604854e8d22faadf683ec6c02bb0f886e2ba56ef981a1c36ee275f21ea22 \ + --hash=sha256:109b84a9edf69ad89dc1f66358659e14a031baca95e3e5b0060bd903ede8efd6 \ + --hash=sha256:1304368a3e7ffc3e9db986796cc5326fdb5943a3567ecc137cff318e4240c0e7 \ + --hash=sha256:17221a7569f8f292aa0014226e48aa25b8c2b08da18088cd230953d0ea0f9cd1 \ + --hash=sha256:1b5a7bbff495baedbd9b916c367d66854008f8f3b575908ded477c499dc60082 \ + --hash=sha256:1ce61b3746545029d4f5c17d6bd74b676254ad98433086c846ffb5e8fa73f007 \ + --hash=sha256:1d2a610c14ac0d0750ee0a3ab8548e83155258387891caaca04def4bf7289781 \ + --hash=sha256:21b7ac084f701a9cdff6139745a6620579d65a9379ac2d9d50a86368b109e63c \ + --hash=sha256:22034924f5b42d5a56371cf271771bfeaabf235a7a8b6264bef2d20013f786c6 \ + --hash=sha256:25218d94b1d2cbc0ba1d8a3f9dc9af578d9646e5ed16443a70cde1dfdcce6d71 \ + --hash=sha256:2608d3b39f9e0b4a66a130d9150c615cba40a5090d25eeeaa225e0e46de8c0ac \ + --hash=sha256:260c33e92263fa629b4f6d3c51967a1c2158fe6c33237aaa3ebeac586b085259 \ + --hash=sha256:2e56ea4ee4df77585a6b5c138f6538680886024fa559f5b55bd14b12e98e67b2 \ + --hash=sha256:2f281549a4c52ac7bb97997f14353f8bd0e53a34ca0dad1c905cfd0b4a58ae99 \ + --hash=sha256:301067672387902c55f94b51d5022304b36c966ea9fe1f21caab99a9bef487c9 \ + --hash=sha256:30536798f4504c0fad0885b1d371b0539abb081e4570c9d7c641cb51141b49f0 \ + --hash=sha256:32c26893cd085c1efe83219e78d866da23fb20a066101b8f68210004361d224c \ + --hash=sha256:34bc7938b9fdf14fe32a406c19c71faf894c5cee7e7474bd0be2f17200b82d14 \ + --hash=sha256:34e47058fcc69a313293d6dee94216a4f30c929ae6f2476e58c5ba635aa639d5 \ + --hash=sha256:3657346f867469e962549435aa05fd15330b1d6a92829f8e27988e194382d005 \ + --hash=sha256:36b306a623aaad96fe4b378692b54f9c0789fccd833b9851753d5fbf6138cfde \ + --hash=sha256:371f7ce73026815dafd51c50ce38416e91428b28c4b2ec97cd39271164b0045c \ + --hash=sha256:375f5af8f99cbaa99dd293af986e3d57caabc9ba81a5d3f021603764854197a1 \ + --hash=sha256:387e2f1d27e89bffe0d3f520f0da0662c973fd607ca16c1808f8a5085419485e \ + --hash=sha256:3aaedf52171bee90860704c560bc798fe83b76247df47568e0197e9b13c735a0 \ + --hash=sha256:3dbb2a31882456cadc7053378e81ad7ed7693db4ac9f98ab5f81ef034aa8ec9f \ + --hash=sha256:4000d961ff9598ac6ea603c6c836a5ed49bc205ade5fc378b998dfe1e2c36628 \ + --hash=sha256:40ccd13c252d3fe473ffc8a57be7565abc8b64cf1b108344c859d5164f7f3e0c \ + --hash=sha256:46c330e82565962c761dbce7941be2cff7db674ee807455a8d0cadc5f9b759b0 \ + --hash=sha256:4f6db193d2e5e0ed60359b9a5a682cd67205d0d3b1e459a867dd4b5c4e7eaa7a \ + --hash=sha256:531b2df3e9fe96b1fcf73a6d165921e4656be5f58d631d384ebce344298368db \ + --hash=sha256:54dab44a847d5ad1acd05c8a83fe518ae685516ecf4d3f7cc6e3df2a66767650 \ + --hash=sha256:5929da1981a46bcf4b28b1b9499905f0ff58e2419da402a048234e9783acbc4b \ + --hash=sha256:5f31b0aa13c9b04370d4da6be1ab7779776b3a075cceb6747a39a4be85fe1e40 \ + --hash=sha256:5fdcf34f86de8fb66d7dc7589f96ba91c4aa46671200d400e6fd6f109a483f18 \ + --hash=sha256:66c0e7e6b02a155576df2c77ec933a70b72da726e248c494abf690923e624348 \ + --hash=sha256:66cb1138f384a191a6d75f986064841fcfdc0cea98f7bd9c9ab9b38049917588 \ + --hash=sha256:68a5faee4bba381cb93b5961f684a514cf0053cb92308ff9c792c2fea0b174c6 \ + --hash=sha256:6bf6a559ffe4a93bbea6cf31ddf01a7fd9ba342ef51f27beb178e318b74acd61 \ + --hash=sha256:70d9c62a4cffd9f23396cd5ef93fc5d11b31596b9b7d6306074abe3d5fcf09bd \ + --hash=sha256:791aa18a373b90da8ac3c44fc77544f33fdf53ae403acdce9b39f1c26b4a3b94 \ + --hash=sha256:79e44cff71750d299d61a678e49995b0d5935a9cda238c2574daeca3ba536927 \ + --hash=sha256:7db9a3ff32ef5f7d1703d93831a3316cdf0b537de6a1cc03cc8fdd09b9194e89 \ + --hash=sha256:860bd1d8ba48456ce08feaf8d343a8aaeb2fa086f2bcaa2a923fa3f7a3ff9aa3 \ + --hash=sha256:9320d34c3376ae204b2cd176e8d4883a013934e0aef822f1aed9c536490c275d \ + --hash=sha256:93d24ebb82aa4420b1409c389e7857bc35bd0b668007ac8172427d5c73cc8cc5 \ + --hash=sha256:94b85d664d777bab6c0d709416cb42938251fda9e221b79e3a2215d85df5f4f9 \ + --hash=sha256:96bad8725a4f196a798366c25ce075d1f7543a4ec045ffc13e6a7ec095cdab04 \ + --hash=sha256:9af313c66157a69dc69ea0059a66961692250e0dc95af9c385a48ffb770a0d16 \ + --hash=sha256:9c5d02b89de5acd0379a51ec44a89476fb03df6145442e1c8ecd6bee2f91b176 \ + --hash=sha256:9e786428f291dd2d2f1cbfc0e0caa45a2e395fab0ad3e2c9314daa8873414390 \ + --hash=sha256:9f836c37478f167a81200d8c8b2c920a22224564bed2c23d7aeec760965c367a \ + --hash=sha256:9fd35e95ab5e45c3901d37110263c7db85a961110f5460588fe37f8c131f88a7 \ + --hash=sha256:a001519c315d5db40710f2665d32c4791f1d4779fc96a9423fd18d92c8b9ac7b \ + --hash=sha256:a3762e75fcac8c9e4dacaaf438bffd9003e2ca2c531b756f3c0035deefa674c8 \ + --hash=sha256:a38fb81d8376dfa2f8963b265fec07637802b0d01e2a127c19c66cb070fb24f5 \ + --hash=sha256:a3dfe4edf10e8ed7e55b026a8bfc2c2a8704218b659cd4bffdf604fab966dc39 \ + --hash=sha256:a4517d47b2b8af26975a406fba7d314de9696d864252e0257c6ea90238cfe27f \ + --hash=sha256:a468951af16155824e88bdd8326ebe5bdb371f3ec0ac04642994b98201d914f3 \ + --hash=sha256:a6e556d6aba31c93dd97ce661d66614d2429c0a3923f9dc8f0af7e8df10223a4 \ + --hash=sha256:ac04bcd3328eb91d99dfedf6a60d9c1f15d3434e6f6daf922f0420f7d90b85c7 \ + --hash=sha256:ae01d8512cc17079e53425635327dbf3f7ff57a42c00dec348bf79791c56444c \ + --hash=sha256:b15e26cc0fe622d0c67e98bee6ef6bc8f792e20ee3006aa12627a00463d9399f \ + --hash=sha256:b222493da6e7b6199db9bd79502436cf5a27da3c1f7fa83c7e285444fc93fd03 \ + --hash=sha256:c3cd253cf32fe4f4662960d6bf7d55cb8be0c31a5d644a4d48aeafebaff3409a \ + --hash=sha256:c6014e3c80f9c1fe268ef8b0e0ef113bac672cc032f2f93866e7ddad4f3e663d \ + --hash=sha256:c718e99a0992127af84385378460db624103b559ab260435abcfe77a4e4ed1c1 \ + --hash=sha256:c7897db4e95e22468bdda33d8e012ceacd0182abf001e6389d763f0def6286b9 \ + --hash=sha256:cb8a1adce42d8b75485a5d56a9623a50bcab995b6079f1dac59fc44034dd93d9 \ + --hash=sha256:cc99dfb62b23c9207c33d0be8a2e2af7a42e21e6ea388b380a0c948c7b88953b \ + --hash=sha256:d4c8d9bd5abce34b2e75edb3bf37ab0f34e49b1f915a40ae8468eb7c85bc5b46 \ + --hash=sha256:d4cb6fbfdf874340ab5e51450753c0f817b6958a3621125ee695bbc3de866566 \ + --hash=sha256:d63bae12a8aeb51380be3438e4dc4bd27354d0f8e19166b2f44e3e94d6f552dc \ + --hash=sha256:d6fb0eaa108814581c4d3bfbd068c3fb6757812a81415008d1bae08267cca360 \ + --hash=sha256:d9188caac26e47671b52836a5e2a49873a7fc11c673b0c122d22515f98bc14e1 \ + --hash=sha256:db327e7271e653c32040b85ae6188059c924b57d7e1e29f935523fa017cd4e82 \ + --hash=sha256:dbdd5b6509d0c2a8fe72cf494c299a61dbd58142a90a4190664ae159e4a7b547 \ + --hash=sha256:e4f9b472e7d308d94b62c801982065661158c6ed02790d6c7ddb4337cea0f9c1 \ + --hash=sha256:e54a315caf843c8d77e388cadc56ea9ded569935ee2d2347d7ea94992e5aa6fa \ + --hash=sha256:f125f5d46b20f89dc5587a55cc416b4ba2a5b2ffda36d048ee120e17598a653a \ + --hash=sha256:f19e181de5b3a1148bb3420b8c4b0b0ea0fce6950099724ad151d6cea5acc180 \ + --hash=sha256:f1f9cc4d09a46d9cb3c2063ae100629d3f52a6517c3c08c2f4c9828261883929 \ + --hash=sha256:f26629539d4893c2957a16c41bb058e1e135c1f150f6a2e25ed047f64cf3f5c6 \ + --hash=sha256:f2a7253458e34f33543551394ae4fe104b497ec2a65ac266074de64c1df82e37 \ + --hash=sha256:f40e56b61b41be5f7dec938cfeffd660668cf4b5e72c78e7bd671d66b7bc2c79 \ + --hash=sha256:f442e3954b1addc759faae22a7c9a3f1e16d7d1db3f484279dc27d62e06968fa \ + --hash=sha256:fadc63331f4388c3dc90090448f682a7e9feafc11481391c1e94f2f907a3976e \ + --hash=sha256:fc67741da44c6eaa90e01eafb586bbba9b51eb5b6ed381ee6f5ae72eb3316d21 + # via mypy +mako==1.3.12 \ + --hash=sha256:8f61569480282dbf557145ce441e4ba888be453c30989f879f0d652e39f53ea9 \ + --hash=sha256:9f778e93289bd410bb35daadeb4fc66d95a746f0b75777b942088b7fd7af550a + # via alembic +markupsafe==3.0.3 \ + --hash=sha256:0303439a41979d9e74d18ff5e2dd8c43ed6c6001fd40e5bf2e43f7bd9bbc523f \ + --hash=sha256:068f375c472b3e7acbe2d5318dea141359e6900156b5b2ba06a30b169086b91a \ + --hash=sha256:0bf2a864d67e76e5c9a34dc26ec616a66b9888e25e7b9460e1c76d3293bd9dbf \ + --hash=sha256:0db14f5dafddbb6d9208827849fad01f1a2609380add406671a26386cdf15a19 \ + --hash=sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf \ + --hash=sha256:0f4b68347f8c5eab4a13419215bdfd7f8c9b19f2b25520968adfad23eb0ce60c \ + --hash=sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175 \ + --hash=sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219 \ + --hash=sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb \ + --hash=sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6 \ + --hash=sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab \ + --hash=sha256:15d939a21d546304880945ca1ecb8a039db6b4dc49b2c5a400387cdae6a62e26 \ + --hash=sha256:177b5253b2834fe3678cb4a5f0059808258584c559193998be2601324fdeafb1 \ + --hash=sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce \ + --hash=sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218 \ + --hash=sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634 \ + --hash=sha256:1ba88449deb3de88bd40044603fafffb7bc2b055d626a330323a9ed736661695 \ + --hash=sha256:1cc7ea17a6824959616c525620e387f6dd30fec8cb44f649e31712db02123dad \ + --hash=sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73 \ + --hash=sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c \ + --hash=sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe \ + --hash=sha256:2a15a08b17dd94c53a1da0438822d70ebcd13f8c3a95abe3a9ef9f11a94830aa \ + --hash=sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559 \ + --hash=sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa \ + --hash=sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37 \ + --hash=sha256:3537e01efc9d4dccdf77221fb1cb3b8e1a38d5428920e0657ce299b20324d758 \ + --hash=sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f \ + --hash=sha256:38664109c14ffc9e7437e86b4dceb442b0096dfe3541d7864d9cbe1da4cf36c8 \ + --hash=sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d \ + --hash=sha256:3b562dd9e9ea93f13d53989d23a7e775fdfd1066c33494ff43f5418bc8c58a5c \ + --hash=sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97 \ + --hash=sha256:4bd4cd07944443f5a265608cc6aab442e4f74dff8088b0dfc8238647b8f6ae9a \ + --hash=sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19 \ + --hash=sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9 \ + --hash=sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9 \ + --hash=sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc \ + --hash=sha256:591ae9f2a647529ca990bc681daebdd52c8791ff06c2bfa05b65163e28102ef2 \ + --hash=sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4 \ + --hash=sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354 \ + --hash=sha256:6b5420a1d9450023228968e7e6a9ce57f65d148ab56d2313fcd589eee96a7a50 \ + --hash=sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698 \ + --hash=sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9 \ + --hash=sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b \ + --hash=sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc \ + --hash=sha256:7be7b61bb172e1ed687f1754f8e7484f1c8019780f6f6b0786e76bb01c2ae115 \ + --hash=sha256:7c3fb7d25180895632e5d3148dbdc29ea38ccb7fd210aa27acbd1201a1902c6e \ + --hash=sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485 \ + --hash=sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f \ + --hash=sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12 \ + --hash=sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025 \ + --hash=sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009 \ + --hash=sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d \ + --hash=sha256:949b8d66bc381ee8b007cd945914c721d9aba8e27f71959d750a46f7c282b20b \ + --hash=sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a \ + --hash=sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5 \ + --hash=sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f \ + --hash=sha256:a320721ab5a1aba0a233739394eb907f8c8da5c98c9181d1161e77a0c8e36f2d \ + --hash=sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1 \ + --hash=sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287 \ + --hash=sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6 \ + --hash=sha256:bc51efed119bc9cfdf792cdeaa4d67e8f6fcccab66ed4bfdd6bde3e59bfcbb2f \ + --hash=sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581 \ + --hash=sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed \ + --hash=sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b \ + --hash=sha256:c0c0b3ade1c0b13b936d7970b1d37a57acde9199dc2aecc4c336773e1d86049c \ + --hash=sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026 \ + --hash=sha256:c4ffb7ebf07cfe8931028e3e4c85f0357459a3f9f9490886198848f4fa002ec8 \ + --hash=sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676 \ + --hash=sha256:d2ee202e79d8ed691ceebae8e0486bd9a2cd4794cec4824e1c99b6f5009502f6 \ + --hash=sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e \ + --hash=sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d \ + --hash=sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d \ + --hash=sha256:de8a88e63464af587c950061a5e6a67d3632e36df62b986892331d4620a35c01 \ + --hash=sha256:df2449253ef108a379b8b5d6b43f4b1a8e81a061d6537becd5582fba5f9196d7 \ + --hash=sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419 \ + --hash=sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795 \ + --hash=sha256:e2103a929dfa2fcaf9bb4e7c091983a49c9ac3b19c9061b6d5427dd7d14d81a1 \ + --hash=sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5 \ + --hash=sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d \ + --hash=sha256:e8fc20152abba6b83724d7ff268c249fa196d8259ff481f3b1476383f8f24e42 \ + --hash=sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe \ + --hash=sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda \ + --hash=sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e \ + --hash=sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737 \ + --hash=sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523 \ + --hash=sha256:f42d0984e947b8adf7dd6dde396e720934d12c506ce84eea8476409563607591 \ + --hash=sha256:f71a396b3bf33ecaa1626c255855702aca4d3d9fea5e051b41ac59a9c1c41edc \ + --hash=sha256:f9e130248f4462aaa8e2552d547f36ddadbeaa573879158d721bbd33dfe4743a \ + --hash=sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50 + # via mako +mccabe==0.7.0 \ + --hash=sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325 \ + --hash=sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e + # via flake8 +mypy==2.3.0 \ + --hash=sha256:04e617030eca5221909c8b7d8d7fd1c637948199aa2100b2ad9813feb07e1491 \ + --hash=sha256:094af99f92638aa92852326188b85a89e50f4a472f44827c03362228482f0762 \ + --hash=sha256:09abd66d8685e73f8f7d17b847c3e104d9a7b164a8706ea87d6c96a3d45816d5 \ + --hash=sha256:0b025a93cffb9781d231f232be07a17912f35f10a313c24f301c81e842870654 \ + --hash=sha256:13b1b16e2fa39f3b2e33fb1c468abc7a69369fa2e886b4b87b5afc81472325cd \ + --hash=sha256:1c6c6bf687b17f90dbfcad95b960d32eaa0154c00da45f03ab50bf8952e047fe \ + --hash=sha256:1fa8d916ac3b705af733c4c1e6c9ebe38fd0d52beb15b105c3e8355b55e6ecdc \ + --hash=sha256:28e1e2af8cd8fff551fd30f2fe4b03fb76764ac8b1ba6c6a1bd00ad32b412db3 \ + --hash=sha256:2d53fc67b9d28a43c6199077f49fea0f05839e36cf6158500331c9549225e5a5 \ + --hash=sha256:3419d00717afbc5265b50dd14b1278f29ea4884dd398ab67873489ac093fd329 \ + --hash=sha256:37fa4de896a84e2dc9200d91e614c22563b43d1a266789d4bbac7b22ebe6192b \ + --hash=sha256:3961a4a34b05f7c74b0f05aa51fbfe99a2d1e126038df40318d15c8f558b7ef3 \ + --hash=sha256:3dd0bed92c4bdec57c42505b96416fb9e6a5aa7be84d2809bcd5f2ecec2860d7 \ + --hash=sha256:3e77244df3843048c3f927182916730e40c124cbaa43905c1fb86cb382aa0805 \ + --hash=sha256:4359424140d985192c778c1ce2c114a10c1ca58a381ed79cfa70d37df94b299f \ + --hash=sha256:465965d41cd9a2726694e983e8ce7113259327bec798115d1e1dfa2a52fb666e \ + --hash=sha256:56c184d2c20ca6b6378d58d1960270a767f41f5e44acbbd27f05effef4f4e1d7 \ + --hash=sha256:5e91adad1ca81742ac7ef9893959911df867752206b37135185e88dfb3c89494 \ + --hash=sha256:691fdc37132b1ae628d834f672e74de83462d9fb4aff621835767fb43a8dd373 \ + --hash=sha256:6b1cdb579446b60432432b2b2403a6201b4b475a004d7f488511c9ba177c9e88 \ + --hash=sha256:6f99ec626e3c3a2f7c0b22c5b90ddb5dabb1c18729c971e9bdaca1f1766d2cee \ + --hash=sha256:7247eb2824f996722a949530183394921ca71deb9680052a338cf53cff7925c2 \ + --hash=sha256:75b0984bb3cbd76bb5c9291a8671f7ae66ca3b51c7584c358fc2e923259f0757 \ + --hash=sha256:75cbb4b9ef04a0c84a957f07abc4504fbf64b8dcc145675101f2d3a78a4b1d6a \ + --hash=sha256:7da939dd335cfd2ad788bdfd081c9f4e47634ab995e5a45eb15fd1e5bc052f8b \ + --hash=sha256:85c5385b93012ffa3b31479ab579aef5415f4f3a32c6cf1ae07a984d2a0ff461 \ + --hash=sha256:91ad22a52ae2c7e621c2f67c94d5a17f66b3209a4cff5cf8a573579835c69e97 \ + --hash=sha256:944c665d984157cb96a679dfb7a4a81dd1d36b24b9c284b699514e6e626b82d4 \ + --hash=sha256:9559ab18a9c9957dfa3004ab57cd4bac5f26a724329a9584e583367f0c2e1117 \ + --hash=sha256:982e3d53dd23d0a4cef67dd66791fdbede0cf38f9eb617bf47663554c51e1e36 \ + --hash=sha256:99ac767cc5d3b64c8d0ae226ead10c96694f94e4e7da1668642225dcd4e75aac \ + --hash=sha256:adebc76aab4f3495a88b41d48aa4aff0c03f2822501da76625afcca5975f19e5 \ + --hash=sha256:aec15d465d477558fd842757b487849007311cf3897849cdda0e3162ac0ac556 \ + --hash=sha256:b1942b9314d4c784b8ea1dbab4972603290e5dd5630f06675f13aec97526bc4c \ + --hash=sha256:b352b7e49f5e6576009e8df730e1ff4f915cb565b851b396d2ffe2f5a6f5da88 \ + --hash=sha256:b5cd2f027a972a4a5f2278a11fac9747f5f81a53a30b714d74950b6807e55568 \ + --hash=sha256:be51653d7669d7d7955d613b8d0bb57d5b652eaf71a873ddf65ac87254dd2595 \ + --hash=sha256:cfca8ee88544090f86b6dcce05ec55d66eb48a762412ac2507810ba4bd793b6f \ + --hash=sha256:d78fcf900b59cb7e82cb7e3a235e31b462d9333d92285bd1e4952d355b8ffba1 \ + --hash=sha256:de121747278144fc9ae7caa2e978cf5df12aebc82933182f5b3b86081a30baef \ + --hash=sha256:de6d2c484742a4d7b0ed6d07b143375624d3b899c5749c7b3c947f56261f48a6 \ + --hash=sha256:ea317b060ce83e26050f8f9e4d7d6bf44ed7597c8ff9990bccffbb9d1d8522db \ + --hash=sha256:f1b3a98dfd21058bc759bb3337d5d1f61d0fdf9f3cf9c00f4291790fb5427bff \ + --hash=sha256:f4ed18f111bfe2d599bca7468e7f9251042c1c2118f762c8de2766a56d773c60 \ + --hash=sha256:fbc00cee7bdbb9291979ddc9d08034a29dfcda4932628c9bbc28c1edd589df0c + # via -r requirements.in +mypy-extensions==1.1.0 \ + --hash=sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505 \ + --hash=sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558 + # via mypy +packaging==26.2 \ + --hash=sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e \ + --hash=sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661 + # via pytest +pathspec==1.1.1 \ + --hash=sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a \ + --hash=sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189 + # via mypy +pluggy==1.6.0 \ + --hash=sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3 \ + --hash=sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746 + # via + # pytest + # pytest-cov +psycopg==3.3.4 \ + --hash=sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a \ + --hash=sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc + # via -r requirements-runtime.in +psycopg-binary==3.3.4 \ + --hash=sha256:018fbed325936da502feb546642c982dcc4b9ffdea32dfef78dbf3b7f7ad4070 \ + --hash=sha256:0579252a1202cd73e4da137a1426e2dae993ae44e757605344282af3a082848c \ + --hash=sha256:136f199a407b5348b9b857c504aff60c77622a28482e7195839ce1b51238c4cc \ + --hash=sha256:13a7f380824c35896dcac7fe0f61440f7ca49d6dc73f3c13a9a4471e6a3b302e \ + --hash=sha256:17a21953a9e5ff3a16dab692625a3676e2f101db5e40072f39dbee2250194d68 \ + --hash=sha256:1dc1f79fd16bb1f3f4421417a514607539f17804d95c7ed617265369d1981cae \ + --hash=sha256:1fbaa292a3c8bb61b45df1ad3da1908ccee7cb889db9425e3557d9e34e2a4829 \ + --hash=sha256:22cdbf5f91ef7bb91fe0c5757e1962d3127a8010256eefd9c61fcaf441802097 \ + --hash=sha256:26df2717e59c0473e4465a97dfb1b7afebaa479277870fd5784d1436470db47c \ + --hash=sha256:276904e3452d6a23d474ef9a21eee19f20eed3d53ddd2576af033827e0ba0992 \ + --hash=sha256:28b7398fdd19db3232c884fb24550bdfe951221f510e195e233299e4c9b78f97 \ + --hash=sha256:2c09aad7051326e7603c14e50636db9c01f78272dc54b3accff03d46370461e6 \ + --hash=sha256:32a6fbf8481e3a370d0d72b860d35948a693cb01281da217f7b2f307636e591a \ + --hash=sha256:41f2ec0fea529832982bcb6c9415de3c86264ebe562b77a467c0fbcd7efbba8d \ + --hash=sha256:46893c26858be12cc49ca4226ed6a60b4bfccadd946b3bebb783a60b38788228 \ + --hash=sha256:47c656a8a7ba6eb0cff1801a4caaa9c8bdc12d03080e273aff1c8ac39971a77e \ + --hash=sha256:494ca54901be8cf9eb7e02c25b731f2317c378efa44f43e8f9bd0e1184ae7be4 \ + --hash=sha256:514404ed543efd620c85602b747df2a23cf1241b4067199e1a66f2d2757aaa41 \ + --hash=sha256:574ea21a9651958f1535c5a1c649c7409e9168bcbffa29a3f2f961f58b322949 \ + --hash=sha256:580ae30a5f95ccd90008ec697d3ed6a4a2047a516407ad904283fa42086936e9 \ + --hash=sha256:5ab28a2a7649df3b72e6b674b4c190e448e8e77cf496a65bd846472048de2089 \ + --hash=sha256:5c4ab71be17bdca30cb34c34c4e1496e2f5d6f20c199c12bad226070b22ef9bf \ + --hash=sha256:612a627d733f695b1de1f9b4bd511c15f999a5d8b915d444bbd7dd71cf3370da \ + --hash=sha256:6402a9d8146cf4b3974ded3fd28a971e83dc6a0333eb7822524a3aa20b546578 \ + --hash=sha256:6b9016b1714da4dd5ecaaa75b82098aa5a0b87854ce9b092e21c27c4ae23e014 \ + --hash=sha256:71e55ccbdfae79a2ed9c6369c3008a3025817ff9d7e27b32a2d84e2a4267e66e \ + --hash=sha256:7465bfe6087d2d5b42d4c53b9b11ca9f218e477317a4a162a10e3c19e984ba8e \ + --hash=sha256:75a9067e236f9b9ae3535b66fe99bddb33d39c0de10112e49b9ab11eee53dc31 \ + --hash=sha256:773d573e11f437ce0bdb95b7c18dc58390494f96d43f8b45b9760436114f7652 \ + --hash=sha256:77df19583501ea288eaf15ac0fe7ad01e6d8091a91d5c41df5c718f307d8e31b \ + --hash=sha256:7f7668f30b9dd5163197e5cbf4e0efd54e00f0a859cc566ce56cfc31f4054839 \ + --hash=sha256:8c0056529e68dbe9184cd4019a1f3d8f3a4ead2f6fc7a5afcf27d3314edd1277 \ + --hash=sha256:94596f9e7633ee3f6440711d43bb70aa31cc0a46a900ab8b4201a366ace5c9e7 \ + --hash=sha256:ab8cca8ef8fb1ccf5b048ae5bd78ba55b9e4b5d472e3ce5ca39ff4d2a9c249e4 \ + --hash=sha256:ad3bc94054876155549fdaedf4a46d1ec69d39a5bcee377148afe498e84c4b8e \ + --hash=sha256:b56b603ebcea8aa10b46228b8410ba7f13e7c2ee54389d4d9be0927fd8ce2a70 \ + --hash=sha256:b6f5a29e9c775b9f12a1a717aa7a2c80f9e1db6f27ba44a5b59c80ac61d2ffcf \ + --hash=sha256:b7bfff1ca23732b488cbca3076fc11bc98d520ee122514fdb17a8e20d3338f5a \ + --hash=sha256:bdef84570ebbce1d42b4e7ea952d21c414c5f118ad02fee00c5625f35e134429 \ + --hash=sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8 \ + --hash=sha256:c677c4ad433cb7150c8cd304a0769ae3bcfbe5ea0676eb53faa7b1443b16d0d3 \ + --hash=sha256:cf7f73a4a792bc5db58a4b385d8a1467e8d468f7548702fb0ed1e9b7501b1c13 \ + --hash=sha256:cffc3408d77a27973f33e5d909b624cce683db5fc25964b02fe0aae7886c1007 \ + --hash=sha256:d7b4d40c153fa352ab3cca530f3a0baedf7621b2ebcbd7f084009522c21788fc \ + --hash=sha256:dbfdb9b6cc79f31104a7b162a2b921b765fcc62af6c00540a167a8de47e4ed38 \ + --hash=sha256:df1d567fc430f6df15c9fcf67d87685fc49bdb325adc0db5af1adfb2f44eb5c9 \ + --hash=sha256:e2631da29253a98bd496e6c4813b24e09a4fe3fb2a9e88513305d6f8747cce95 \ + --hash=sha256:e7510c37550f91a187e3660a8cc50d4b760f8c3b8b2f89ebc5698cd2c7f2c85d \ + --hash=sha256:eb05ee1c2b817d27c537333224c9e83c7afb86fe7296ba970990068baf819b16 \ + --hash=sha256:eb4eed2079c01a4850bf467deacfab56d356d4225040170af03dc9958321242d \ + --hash=sha256:ee17a2cf4943cde261adfad1bbc5bf38d6b3776d7afff74c7cabcbeaeb08c260 \ + --hash=sha256:f80e3f2b5331dbbf0901bcb658056c03eeb2c1ef31d774afb0d61598b242e744 \ + --hash=sha256:f9b1c2533af01cd7648378599f82b0b8ae32f293296e6eec5753a625bc97ef28 \ + --hash=sha256:fa1cbc10768a796c96d3243656016bf4e337c81c71097270bb7b0ad6210d9765 \ + --hash=sha256:fbd1d4ed566895ad2d3bf4ddfd8bae90026930ddf29df3b9d91d32c8c47866a7 + # via psycopg +pyasn1==0.6.4 \ + --hash=sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81 \ + --hash=sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b + # via + # pyasn1-modules + # python-jose + # rsa +pyasn1-modules==0.4.2 \ + --hash=sha256:29253a9207ce32b64c3ac6600edc75368f98473906e8fd1043bd6b5b1de2c14a \ + --hash=sha256:677091de870a80aae844b1ca6134f54652fa2c8c5a52aa396440ac3106e941e6 + # via google-auth +pycodestyle==2.14.0 \ + --hash=sha256:c4b5b517d278089ff9d0abdec919cd97262a3367449ea1c8b49b91529167b783 \ + --hash=sha256:dd6bf7cb4ee77f8e016f9c8e74a35ddd9f67e1d5fd4184d86c3b98e07099f42d + # via flake8 +pycountry==26.2.16 \ + --hash=sha256:115c4baf7cceaa30f59a4694d79483c9167dbce7a9de4d3d571c5f3ea77c305a \ + --hash=sha256:5b6027d453fcd6060112b951dd010f01f168b51b4bf8a1f1fc8c95c8d94a0801 + # via -r requirements-runtime.in +pycparser==3.0 \ + --hash=sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29 \ + --hash=sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992 + # via cffi +pydantic==2.13.4 \ + --hash=sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba \ + --hash=sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6 + # via + # -r requirements-runtime.in + # anthropic + # fastapi + # google-genai +pydantic-core==2.46.4 \ + --hash=sha256:00c603d540afdd6b80eb39f078f33ebd46211f02f33e34a32d9f053bba711de0 \ + --hash=sha256:0186750b482eefa11d7f435892b09c5c606193ef3375bcf94aa00ae6bfb66262 \ + --hash=sha256:041bde0a48fd37cf71cab1c9d56d3e8625a3793fef1f7dd232b3ff37e978ecda \ + --hash=sha256:0c563b08bca408dc7f65f700633d8442fffb2421fc47b8101377e9fd65051ff0 \ + --hash=sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e \ + --hash=sha256:0ce40cd7b21210e99342afafbd4d0f76d784eb5b1d60f3bdc566be4983c6c73b \ + --hash=sha256:0e96592440881c74a213e5ad528e2b24d3d4f940de2766bed9010ab1d9e51594 \ + --hash=sha256:10e17cbb10a330363733efc4d7c4d0dd827ac0909b8f6a6542298fed1ea62f29 \ + --hash=sha256:133878133d271ade3d41d1bfb2a45ec38dbdbda40bc065921c6b04e4630127e2 \ + --hash=sha256:14d4edf427bdcf950a8a02d7cb44a08614388dd6e1bdcbf4f67504fa7887da9c \ + --hash=sha256:14f4c5d6db102bd796a627bbb3a17b4cf4574b9ae861d8b7c9a9661c6dd3362d \ + --hash=sha256:17299feefe090f2caa5b8e37222bb5f663e4935a8bfa6931d4102e5df1a9f398 \ + --hash=sha256:184c081504d17f1c1066e430e117142b2c77d9448a97f7b65c6ac9fd9aee238d \ + --hash=sha256:18e5ceec2ab67e6d5f1a9085e5a24c9c4e2ac4545730bfe668680bca05e555f3 \ + --hash=sha256:19e51f073cd3df251856a8a4189fbdf1de4012c3ebacfb1884f94f1eb406079f \ + --hash=sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb \ + --hash=sha256:1d8ba486450b14f3b1d63bc521d410ec7565e52f887b9fb671791886436a42f7 \ + --hash=sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5 \ + --hash=sha256:228ee9bae8bef5b1e97ec58302f80357c37199e0d0a99174e138d28e6957b9d9 \ + --hash=sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462 \ + --hash=sha256:2412e734dcb48da14d4e4006b82b46b74f2518b8a26ee7e58c6844a6cd6d03c4 \ + --hash=sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b \ + --hash=sha256:2f84c03c8607173d16b5a854ec68a2f9079ae03237a54fb506d13af47e1d018d \ + --hash=sha256:3009f12e4e90b7f88b4f9adb1b0c4a3d58fe7820f3238c190047209d148026df \ + --hash=sha256:3245406455a5d98187ec35530fd772b1d799b26667980872c8d4614991e2c4a2 \ + --hash=sha256:3447661d99f75a3683a4cf5c87da72f2161964611864dbbeac7fbb118bb4bfc0 \ + --hash=sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519 \ + --hash=sha256:395aebd9183f9d112f569aeb5b2214d1a10a33bec8456447f7fbdfa51d38d4cd \ + --hash=sha256:3a233125ac121aa3ffba9a2b59edfc4a985a76092dc8279586ab4b71390875e7 \ + --hash=sha256:3be77f45df024d789a672ae34f8b06fb346c4f9f46ea714956660ea4862e89ac \ + --hash=sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6 \ + --hash=sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565 \ + --hash=sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898 \ + --hash=sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb \ + --hash=sha256:432c179df7874eeb73307aad2df0755e1ae0efa61ff0ea89b93e194411ae3928 \ + --hash=sha256:4a05d69cba51d852c5c3e92758653245a50c0b646ced0cf05bd793ed592839d6 \ + --hash=sha256:4c63ebc82684aa89d9a3bcbd13d515b3be44250dc68dd3bd81526c1cb31286c3 \ + --hash=sha256:4fc73cb559bdb54b1134a706a2802a4cddd27a0633f5abb7e53056268751ac6a \ + --hash=sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596 \ + --hash=sha256:56cb4851bcaf3d117eddcef4fe66afd750a50274b0da8e22be256d10e5611987 \ + --hash=sha256:5855698a4856556d86e8e6cd8434bc3ac0314ee8e12089ae0e143f64c6256e4e \ + --hash=sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d \ + --hash=sha256:5b712b53160b79a5850310b912a5ef8e57e56947c8ad690c227f5c9d7e561712 \ + --hash=sha256:5d5902252db0d3cedf8d4a1bc68f70eeb430f7e4c7104c8c476753519b423008 \ + --hash=sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd \ + --hash=sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1 \ + --hash=sha256:633147d34cf4550417f12e2b1a0383973bdf5cdfde212cb09e9a581cf10820be \ + --hash=sha256:66ce7632c22d837c95301830e111ad0128a32b8207533b60896a96c4915192ea \ + --hash=sha256:6b3ace8194b0e5204818c92802dcdca7fc6d88aabbb799d7c795540d9cd6d292 \ + --hash=sha256:6f2eeda33a839975441c86a4119e1383c50b47faf0cbb5176985565c6bb02c33 \ + --hash=sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3 \ + --hash=sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4 \ + --hash=sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b \ + --hash=sha256:7bfb192b3f4b9e8a89b6277b6ce787564f62cfd272055f6e685726b111dc7826 \ + --hash=sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac \ + --hash=sha256:8233f2947cf85404441fd7e0085f53b10c93e0ee78611099b5c7237e36aacbf7 \ + --hash=sha256:82cf5301172168103724d49a1444d3378cb20cdee30b116a1bd6031236298a5d \ + --hash=sha256:8358a950c8909158e3df31538a7e4edc2d7265a7c54b47f0864d9e5bae9dcebf \ + --hash=sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4 \ + --hash=sha256:86e1a4418c6cd97d60c95c71164158eaf7324fae7b0923264016baa993eba6fc \ + --hash=sha256:8b9bab013d1c7a79d3501ff86d0bc9c31bf587db4551677b96bec07df78c6b15 \ + --hash=sha256:8c5dac79fa1614d1e06ca695109c6105923bd9c7d1d6c918d4e637b7e6b32fd3 \ + --hash=sha256:8d0820e8192167f80d88d64038e609c31452eeca865b4e1d9950a27a4609b00b \ + --hash=sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914 \ + --hash=sha256:9037063db01f09b09e237c282b6792bd4da634b5402c4e7f0c61effed7701a04 \ + --hash=sha256:905a0ed8ea6f2d61c1738835f99b699348d7857379083e5fc497fa0c967a407c \ + --hash=sha256:90884113d8b48f760e9587002789ddd741e76ab9f89518cd1e43b1f1a52ec44b \ + --hash=sha256:91a06d2e259ecfbd8c901d70c3c507900458498142b3026a296b7de4d1322cc9 \ + --hash=sha256:926c9541b14b12b1681dca8a0b75feb510b06c6341b70a8e500c2fdcff837cce \ + --hash=sha256:9401557acd873c3a7f3eb9383edef8ac4968f9510e340f4808d427e75667e7b4 \ + --hash=sha256:9551187363ffc0de2a00b2e47c25aeaeb1020b69b668762966df15fc5659dd5a \ + --hash=sha256:962ccbab7b642487b1d8b7df90ef677e03134cf1fd8880bf698649b22a69371f \ + --hash=sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424 \ + --hash=sha256:9aa768456404a8bf48a4406685ac2bec8e72b62c69313734fa3b73cf33b3a894 \ + --hash=sha256:9bc519fbf2b7578398853d815009ae5e4d4603d12f4e3f91da8c06852d3da3e9 \ + --hash=sha256:9d56801be94b86a9da183e5f3766e6310752b99ff647e38b09a9500d88e46e76 \ + --hash=sha256:9f444c499b3eefd3a92e348059471ea0c3a6e303d9c1cec09fa748fd9f895201 \ + --hash=sha256:9fa8ae11da9e2b3126c6426f147e0fba88d96d65921799bb30c6abd1cb2c97fb \ + --hash=sha256:a0f62d0a58f4e7da165457e995725421e0064f2255d8eccebc49f41bbc23b109 \ + --hash=sha256:a396dcc17e5a0b164dbe026896245a4fa9ff402edca1dff0be3d53a517f74de4 \ + --hash=sha256:aaa2a54443eff1950ba5ddc6b6ccda0d9c84a364276a62f969bdf2a390650848 \ + --hash=sha256:ad785e92e6dc634c21555edc8bd6b64957ab844541bcb96a1366c202951ae526 \ + --hash=sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0 \ + --hash=sha256:b078afbc25f3a1436c7a1d2cd3e322497ee99615ba97c563566fdf46aff1ee01 \ + --hash=sha256:b2f69dec1725e79a012d920df1707de5caf7ed5e08f3be4435e25803efc47458 \ + --hash=sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e \ + --hash=sha256:bb63e0198ca18aad131c089b9204c23079c3afa95487e561f4c522d519e55aba \ + --hash=sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a \ + --hash=sha256:c1747f85cee84c26985853c6f3d9bd3e75da5212912443fa111c113b9c246f39 \ + --hash=sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c \ + --hash=sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000 \ + --hash=sha256:c68fcd102d71ea85c5b2dfac3f4f8476eff42a9e078fd5faefff6d145063536b \ + --hash=sha256:c7a7bd4e39e8e4c12c39cd480356842b6a8a06e41b23a55a5e3e191718838ddf \ + --hash=sha256:c94f0688e7b8d0a67abf40e57a7eaaecd17cc9586706a31b76c031f63df052b4 \ + --hash=sha256:cbaf13819775b7f769bf4a1f066cb6df7a28d4480081a589828ef190226881cd \ + --hash=sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28 \ + --hash=sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9 \ + --hash=sha256:d396ec2b979760aaf3218e76c24e65bd0aca24983298653b3a9d7a45f9e47b30 \ + --hash=sha256:d51026d73fcfd93610abc7b27789c26b313920fcfb20e27462d74a7f8b06e983 \ + --hash=sha256:d80ee3d731373b24cebbc10d689ca4ee1875caf0d5703a245db18efd4dd37fc1 \ + --hash=sha256:d995260fdf4e1db774581b4900e0f832abe3c7c84996726bbc161b19c8f29e76 \ + --hash=sha256:da4b951fe36dc7c3a1ccb4e3cd1747c3542b8c9ceede8fc86cae054e764485f5 \ + --hash=sha256:daa27d92c36f24388fe3ad306b174781c747627f134452e4f128ea00ce1fe8c4 \ + --hash=sha256:db06ffe51636ffe9ca531fe9023dd64bdd794be8754cb5df57c5498ae5b518a7 \ + --hash=sha256:e0d65b8c354be7fb5f720c3caa8bc940bc2d20ce749c8e06135f07f8ed95dd7c \ + --hash=sha256:e68b7a074f65a2fd746c52a7ce6142ab7006074ac269ace0c25cd8ba171f8066 \ + --hash=sha256:e739fee756ba1010f8bcccb534252e85a35fe45ae92c295a06059ce58b74ccd3 \ + --hash=sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02 \ + --hash=sha256:e9c26f834c65f5752f3f06cb08cb86a913ceb7274d0db6e267808a708b46bc89 \ + --hash=sha256:ea793e075b70290d89d8142074262885d3f7da19634845135751bd6344f73b50 \ + --hash=sha256:f027324c56cd5406ca49c124b0db10e56c69064fec039acc571c29020cc87c76 \ + --hash=sha256:f13a646d65d09fbf1bc6b3a9635d30095c8e7e5cc419ff35ecc563c5fd04cd49 \ + --hash=sha256:f47286a97f0bc9b8859519809077b91b2cefe4ae47fcbf5e466a009c1c5d742b \ + --hash=sha256:f747929cf940cddb5b3668a390056ddd5ba2e5010615ea2dcf4f9c4f3ab8791d \ + --hash=sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7 \ + --hash=sha256:f9fa868638bf362d3d138ea55829cefb3d5f4b0d7f142234382a15e2485dbec4 \ + --hash=sha256:fbdb89b3e1c94a30cc5edfce477c6e6a5dc4d8f84665b455c27582f211a1c72c \ + --hash=sha256:fc010ab034c8c7452522748bf937df58020d256ccae0874463d1f4d01758af8e \ + --hash=sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff \ + --hash=sha256:fd8b3d9fd264be37976686c7f65cd52a83f5e84f4bfd2adf9c1d469676bbb6ae + # via pydantic +pyflakes==3.4.0 \ + --hash=sha256:b24f96fafb7d2ab0ec5075b7350b3d2d2218eab42003821c06344973d3ea2f58 \ + --hash=sha256:f742a7dbd0d9cb9ea41e9a24a918996e8170c799fa528688d40dd582c8265f4f + # via flake8 +pygments==2.20.0 \ + --hash=sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f \ + --hash=sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176 + # via pytest +pysocks==1.7.1 \ + --hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \ + --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \ + --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0 + # via googlenewsdecoder +pytest==9.1.1 \ + --hash=sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313 \ + --hash=sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c + # via + # -r requirements.in + # pytest-cov +pytest-cov==7.1.0 \ + --hash=sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2 \ + --hash=sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678 + # via -r requirements.in +python-jose==3.5.0 \ + --hash=sha256:abd1202f23d34dfad2c3d28cb8617b90acf34132c7afd60abd0b0b7d3cb55771 \ + --hash=sha256:fb4eaa44dbeb1c26dcc69e4bd7ec54a1cb8dd64d3b4d81ef08d90ff453f2b01b + # via -r requirements-runtime.in +python-multipart==0.0.32 \ + --hash=sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e \ + --hash=sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23 + # via -r requirements-runtime.in +requests==2.34.2 \ + --hash=sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0 \ + --hash=sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed + # via + # -r requirements-runtime.in + # google-auth + # google-genai + # googlenewsdecoder +rsa==4.9.1 \ + --hash=sha256:68635866661c6836b8d39430f97a996acbd61bfa49406748ea243539fe239762 \ + --hash=sha256:e7bdbfdb5497da4c07dfd35530e1a902659db6ff241e39d9953cad06ebd0ae75 + # via python-jose +selectolax==0.4.11 \ + --hash=sha256:086ca6f7e4c475bfff871ec1448ae5d342d43d6a2ca2cea65160d01b3a6a75ec \ + --hash=sha256:168b34466dc4f998d7ceb04f218693546d141543e7c5d327f9e006c0799cd62c \ + --hash=sha256:1a6deb4464198ac67f32e56c4463aedf3e1d834b458eaac5b5b5b1ef02dcf15e \ + --hash=sha256:1da54e42ab99b9191269306e13c0fd67ada1c6654e8dc8d74fac615931dd3c62 \ + --hash=sha256:21359cf4d1261b314895c07321e76aa2334b1967956efba04cc0d223d7817f48 \ + --hash=sha256:224682039ca13eb822be626e49a03592ee2b8557bcdc6381e49417a995170c94 \ + --hash=sha256:25d633cddca0fc769e7d890e9e838908fb4a7326eec5e3b23ae42c27f457541d \ + --hash=sha256:28915b8fa90c1c3cb585858a6d24d433a3f38ea514aea59013bdb0930d9f6025 \ + --hash=sha256:2b565ddabce6c9a7b73fa28a39acf8f411a084fa2f169234ec2470f552d4421d \ + --hash=sha256:2b842c829f916fecb51f0f55882eca3e2ad49e85388178f14ae6fe0912be0a57 \ + --hash=sha256:2e8d01d9688e9670b7ed3f042dc7ba379f844fb7fcb8e85191885ed75d6178fb \ + --hash=sha256:3066be35f630e4c4a00cf2c829548574fcba5963735411e6a9ac78bd4ca830ce \ + --hash=sha256:36183968e5c122da414db0dd7d22d4a6ca808e9b9e4837532dfbe55df6f26bff \ + --hash=sha256:3637d21f7fe60fbd6ca3dbc67a1747f6a55a9389114d72f06b5d69ba2beddf01 \ + --hash=sha256:386494779e5464e587ed4dc076e1c48c24ebaf2da1e3a249690551d1f97fe8ed \ + --hash=sha256:3abf5aecfd0c314b62c10ce6e9baee2094cb385e2ec154bd1743febd2fcc9bdb \ + --hash=sha256:41f388c26304c1d840f5ee5e07c06bb9388ec834d10fec60dc148f22f98efd38 \ + --hash=sha256:466daca0599408c9d2cad7658a68490facc5c9b8d0f41ac5d17948914f57306f \ + --hash=sha256:47be0e591125484de14ff0c9aaaa814dd4a4019de35eabe360e88169a263a2b5 \ + --hash=sha256:48545b0351b6f92c4ad2fff835e832768d0cb37834766a82f40242e51fc9901d \ + --hash=sha256:4c3d4fd42deabfd03cc5c4fce9517e7226054f9de3984759476e6c4a7b7fe187 \ + --hash=sha256:5153157ed60d968ed303acbfd2c8762fa0c0462e2663bd04466471c565deb88a \ + --hash=sha256:57e71c021d9950113e7d49892eba56014c072c9d86b71c0b3b7cb846ee913e20 \ + --hash=sha256:597b8e065978be200c598ae6d682496d96fbce14d34b5d519e93cf5b6be5fb60 \ + --hash=sha256:5c7a91fbe1a94849d85228897c416ab9b4518bea6b04dce8ef8acd825ec80e9d \ + --hash=sha256:5ff73122f5fee3350528235349f1b9e5626a1a0ef1887fef82d7fddac18f6605 \ + --hash=sha256:634db1b7ae1b1f10ee0b0adbef0eee1aec65a3d01f09b75132bc5b043b7623d1 \ + --hash=sha256:663ff792f92ed749cfcf452ac19aff5da74b05521e7daacb3b74388deb14d117 \ + --hash=sha256:75c13f519ecb85185e149049bd4c4937aa0b953bd21e1748eb711c6cdcee571e \ + --hash=sha256:75eeddbd3db7e82293a325c210cf4d218d25961870dbb468383c7516e3f577aa \ + --hash=sha256:80af1c7345701934769679a83fcc86526c56eef590423cc2d55cc1eb81adfa5f \ + --hash=sha256:8166bb8cad8f2eabed664f689b708851078b74dd50dd93e426b291095badc2cb \ + --hash=sha256:81f21972788df01b83a5940d9b7d62e6e222f80872c2cfaf67927276e8ef4975 \ + --hash=sha256:83515cc0fe9b8c4a1ca8bfcd83cf48408bcdd3f2fccd088f04635619e93695c9 \ + --hash=sha256:8da39a07a589fd181b5e8d25f695d7d40d3a1d89e47c2e00c08ece7fff5ddd3d \ + --hash=sha256:8f0223a83051ece2591a1d0cf487f1d0719cb76660d7c5de4c0a201b761c5c23 \ + --hash=sha256:8ff49b24d4e5c81bcc96b25e2cf23d9af2d06e4d473865c812487be6665416c0 \ + --hash=sha256:903286e6d9752b63b3b37f9ee98a8ff7db403b23416b469b065cc992b2556b80 \ + --hash=sha256:9077fa36e99ef4bb801194ff8f492f67279c0562e7cdfa9b4d06f5c010131969 \ + --hash=sha256:96de39e2f28c359cf930547193b2cacf0ab6b9187309e2515461fff9d5a5aa3c \ + --hash=sha256:9862c1d0435e89e419e0c595084eaa8f6f71c3fb7a968bdff51db595cec07161 \ + --hash=sha256:991866e2f45ce85e3ce7a31714e522f620d83b1eca15fe530055108a94e6087e \ + --hash=sha256:a532f1993f08f627300891751982dd9641ce1bd29618b535232aec9fa023a1c8 \ + --hash=sha256:ad4bbf08af4e2c89f78dd12a7b8265f0924d0434705bd955b5a262297a924452 \ + --hash=sha256:b0f56c49161b18621ac452e42e02b0c5c61ba4c21095cfff3990e040bd9a043c \ + --hash=sha256:b530a2c4fad7400af27b2b7e0333c1318ecb5f5dc38e8a141dbe3bd81b398fdf \ + --hash=sha256:bd843540a197a33049a08fd80e59bfeafbaa688e632d53a05a9b65af5e88296f \ + --hash=sha256:bde121202b33e6041e9d2db1d62e7466b5883fd1c441eb96ff68d3ea015cfcbd \ + --hash=sha256:c3f6fac361b5f415c409dfd1a07dd0e9a5899d10daba8d88ce16bd552b0e06f2 \ + --hash=sha256:d00753a5f832625f69b585dda7a7ddc1a7d2803c31aa8710c5d209fca064d774 \ + --hash=sha256:d33e2ed75cc33e7af3fd50521c33e7d8634fae23bc197a6cee6a5015e056eef6 \ + --hash=sha256:d5ce592a92fceeca2694b369a83ad72891a9c356f668718fe7e1c83eea407bb4 \ + --hash=sha256:daf7a841d1baa795f940200bec45019c4a31020def16e3a4e35485e82ed64167 \ + --hash=sha256:dbee8998d426c1b6b78aab7a051009f9de38d77b93d073025ae19f7d71b3ead3 \ + --hash=sha256:dc8d449adfdef675bcbfc67d5548513271c7ea869e9b71a2e898138bf34a976a \ + --hash=sha256:dcd24bfc4899e4df2bdcbabc973384367686f391e5a2d8ebb229c043f43be82a \ + --hash=sha256:dff45ee85409e6126e4900e0db6950a59a7b58b7ff9d667a6a275b0405b87692 \ + --hash=sha256:e2d7e1a2b67a5e8d251733b2c3f978ed26a56d04552472e0a2fb5ea41fa4e663 \ + --hash=sha256:e5929cbe3eedfaf51a09ec89642ab5355b703486d43bcf3c8f0c27d6043a488d \ + --hash=sha256:f4894d4cb2b9cd4828dc9d9cbed683db051748fde66f5937aa823f2afff16b36 \ + --hash=sha256:f69c377298a09f571c89af5916ac85f69a9aaedd7312b0bb803a46d2ca2f9048 \ + --hash=sha256:f8121f4cbfe870d9ad24ae418a735d918d55844e905c3270077e97f4e579770f \ + --hash=sha256:f8f014f328f6f79a364293bca54d43af1cec776dc10a5302054a54b5fb2d8675 \ + --hash=sha256:fad5b1065f73eeaa07ea343cbc548aaa9f9a5c359c3bdd8d98f5d80b61550d1c + # via googlenewsdecoder +sgmllib3k==1.0.0 \ + --hash=sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9 + # via feedparser +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via ecdsa +sniffio==1.3.1 \ + --hash=sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2 \ + --hash=sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc + # via + # anthropic + # google-genai +soupsieve==2.9.1 \ + --hash=sha256:4f4477399246b7a0c720a88ca2454b11cd6bb9ae4c9d170140786e916776c14c \ + --hash=sha256:c33e6605bbc71dd628b00c632d58ae607c22bade247e52553928f83bbb75b4ba + # via beautifulsoup4 +sqlalchemy==2.0.51 \ + --hash=sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23 \ + --hash=sha256:0592bdadf86ddcabfd72d9ab66ea8a5d8d2cc6be1cc51fa7e66c03868ac5eac1 \ + --hash=sha256:08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8 \ + --hash=sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72 \ + --hash=sha256:0e8203d2fbd5c6254692ef0a72c740d75b2f3c7ca345404f4c1a4604813c77c0 \ + --hash=sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5 \ + --hash=sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e \ + --hash=sha256:111604e637da87031255ddc26c7d7bc22bc6af6f5d459ccff3af1b4660233a85 \ + --hash=sha256:1181256e0f16479691b5616d36375dc2620ad8332b25978763c3d206ad3f3f1d \ + --hash=sha256:159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2 \ + --hash=sha256:1aa10c0daee6705294d181daadaa793221e1a59ed55000a3fab1d42b088ce4ba \ + --hash=sha256:1af05726b3d0cdba1c55284bf408fd3b792e690fe2399bfb8304565551cda652 \ + --hash=sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f \ + --hash=sha256:1d21ce524ab86c23046e992a5b81cb54c21079c6df6e78b8fc77d77cac70a6b9 \ + --hash=sha256:1e47b1199c2e832e325eacabc8d32d2487f58c9358f97e9a00f5eb93c5680d84 \ + --hash=sha256:247acaa29ccef6250dfd6a3eedf8f94ddf23564180a39fe362e32ae9dbdbde46 \ + --hash=sha256:2a97eaad21c84b4ef8010b11eeba9fe6153eb0b3df3ff8b6abc309df1b978ef7 \ + --hash=sha256:2cf39aabdf48e87c1c2c2ed6d20d33ffa0733b3071ce9c5f66357947dd009080 \ + --hash=sha256:2e54ff2dd657f2e3e0fbf2b097db1182f7bfea263eca4353f00065bae2a67c3d \ + --hash=sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d \ + --hash=sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54 \ + --hash=sha256:436728ce18a80f6951a1e11cc6112c2ede9faf20766f1a26195a7c441ca12dbd \ + --hash=sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195 \ + --hash=sha256:4a011ea4510683319ce4ed274b56ee05194b39b6da9d09ca7a39388f0fa84dcc \ + --hash=sha256:581921d849d6e6f994d560389192955e80e2950e18fcdfe2ccea863e01158e6e \ + --hash=sha256:59cab3686b1bc039dd9cded2f8d0c08a246e84e76bd4ab5b4f18c7cdae293825 \ + --hash=sha256:6b588fd681ddf0c196b8df1ea49a8913514894b2b8f945a9511b4b48871f99c8 \ + --hash=sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522 \ + --hash=sha256:6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491 \ + --hash=sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400 \ + --hash=sha256:740cf6f35351b1ac3d82369152acf1d51d37e3dcf85d4dc0a22ca01410eabe2a \ + --hash=sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07 \ + --hash=sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7 \ + --hash=sha256:7d78702b26ba1c18b2d0fb2ea940ba7f17a9581b42e8361ff93920ebbee1235a \ + --hash=sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9 \ + --hash=sha256:9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7 \ + --hash=sha256:96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499 \ + --hash=sha256:9f380393be5abeb6815f68fd39271b95127173511b6706b0a630a9995d53f8f5 \ + --hash=sha256:a42ad6afcbaaa777241e347aa2e29155993045a0d6b7db74da61053ffe875fe0 \ + --hash=sha256:a5b2ed6d828f1f09bd812861f4f59ca3bc3803f9df871f4555187f0faf018604 \ + --hash=sha256:a6d26094615306d116dd5e4a51b0304c99dd2356fc569eed6922a80a6bd3b265 \ + --hash=sha256:aa18ae738b5170e253ad0bb6c4b0f07585081e8a6e50893e4d911d47b39a0904 \ + --hash=sha256:ad30ae663711786303fbcd46a47516302d201ee49a877cb3fac61f672895110a \ + --hash=sha256:b21f0e7efc7a5c509e953784e9d1575ebb8b4318960e7e7d7a93bb803626cf64 \ + --hash=sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d \ + --hash=sha256:b7f08588854bbb724041d9ae9d980d40040c922382e1d9a2ecb390edc4fd5032 \ + --hash=sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b \ + --hash=sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5 \ + --hash=sha256:bb1f5062f98b0b3290e72b707747fdd7e0f22d6956b236ba7ca7f5c9971d2da2 \ + --hash=sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d \ + --hash=sha256:c5d98a2709840027f5a347c3af0a7c3d5f6c1ff93af2ca1c54494e23cba8f389 \ + --hash=sha256:c68568f3facf8f66fa76c60e0ced69b67666ffa9941d1d0a3756fda196049080 \ + --hash=sha256:c95ef01f53233a305a874a44a63fbfb1d81cd79b49de0f8529b3548cde437e37 \ + --hash=sha256:ca216e8af5c05e326efc7e28716ac2381a7cf9791749f5ee1849dccdc99c9b00 \ + --hash=sha256:ca8435d13829b92f4a97362d91975154a4015db3a2634154e1754e9a915e6b86 \ + --hash=sha256:dc261707bf5739aea8a541593f3cc1d463c2701fb05fbcbba0ce031b69a21260 \ + --hash=sha256:e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de \ + --hash=sha256:fa268106c8987639a17a18514cfe0cd9bf17420ab887e1e1bf486da8836135b1 + # via + # -r requirements-runtime.in + # alembic + # sqlalchemy +starlette==1.3.1 \ + --hash=sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0 \ + --hash=sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6 + # via fastapi +tenacity==9.1.4 \ + --hash=sha256:6095a360c919085f28c6527de529e76a06ad89b23659fa881ae0649b867a9d55 \ + --hash=sha256:adb31d4c263f2bd041081ab33b498309a57c77f9acf2db65aadf0898179cf93a + # via google-genai +typing-extensions==4.16.0 \ + --hash=sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8 \ + --hash=sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5 + # via + # alembic + # anthropic + # anyio + # beautifulsoup4 + # fastapi + # google-genai + # mypy + # psycopg + # pydantic + # pydantic-core + # sqlalchemy + # starlette + # typing-inspection +typing-inspection==0.4.2 \ + --hash=sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7 \ + --hash=sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464 + # via + # fastapi + # pydantic +urllib3==2.7.0 \ + --hash=sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c \ + --hash=sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897 + # via requests +uvicorn==0.51.0 \ + --hash=sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b \ + --hash=sha256:f6f4b69b657c312f516dd2d268ab9ae6f254b11e4bac504f37b2ab58b24dd0b0 + # via -r requirements-runtime.in +websockets==16.1.1 \ + --hash=sha256:01fbdcbac298efe19360b94bc0039c8f746f0220ba570f327577bfee81059175 \ + --hash=sha256:024193f8551a2b0eafbdd160911012c4e6c228c28430c84433253299a9e42d6a \ + --hash=sha256:04fd29a0e2fe9414a95b00e92c67ae51bf900c50c0f8a4b2dafdad621f49ea1d \ + --hash=sha256:056ae37939ed7e9974f364f5864e76e49182622d8f9751ac1903c0d09b013985 \ + --hash=sha256:0f62863e8a00a6d33c3d6566ec0b89f23787b747ffe0c3bc71ec0e76b82c94b1 \ + --hash=sha256:0ffd3031ea8bda8d61762e84220186105ba3b748b3c8da2ae4f7816fac03e573 \ + --hash=sha256:1214e673c404684b9bf7154f5cf43b45025b1a6160fac3a9e438e9c1a97e22cb \ + --hash=sha256:125f22dbefaf1554fea66fc83851490edb284ce4f501d37ffed2752f418332d9 \ + --hash=sha256:130937b167a52af203c8d58e78d67705874e82759862e3b9671a452fec4abc87 \ + --hash=sha256:1427fb4cf0d72f66333e2cacc3ff5f575bf2d7008166ce991a4a470b21d51a22 \ + --hash=sha256:195c978b065fa40910582464f99d6b15c8b314c68e0546549a55ed83f4735328 \ + --hash=sha256:1d27fa8462ad6a1cb36206a3d0640b2333340def181fae11ed7f9adeaa5c0747 \ + --hash=sha256:1db4de4a0e95673f7545d393c49eeb0c2f18ac1ef93073218c79d5cdb2ee75ab \ + --hash=sha256:1f79c89b5eb034d1722938a891916582f8f7f503f58ca22518a63c3f2cd18499 \ + --hash=sha256:23253dd5bcae3f9aaee0a1d30967a8dbd52e5d3cff93a2e5b84df57b77d4750d \ + --hash=sha256:249116b4a76063d930a46391ad56e135c286e4562a18309029fc2c73f4ed4c62 \ + --hash=sha256:29dfa8114c4a620c69591c5973860f768eac29d3fd6904f37f34266cb219c512 \ + --hash=sha256:2a606d9c24035242a3e256e9d5b77ed9cd6bccfcb7cf993e5ca3c0f6f68fb6a7 \ + --hash=sha256:2a636ff1e7a5c4edf71ef0e79adae7f25dba93b4fcbe3dc958733477ffeb0eaf \ + --hash=sha256:2bb5d041a8307d2e18782e7ce777f6fdb1e8c2f5d09291484b18c294b789d9aa \ + --hash=sha256:2e28e602bb13da44fbe518c1781a88e3b9d4c3d48d02c9bad83e546164336f57 \ + --hash=sha256:30bbe120437b5648a77d3519b7024ea09530e0b5b18d3698c5a0ae536fe0cc2e \ + --hash=sha256:34420aaa64440ebd51ac72ca8a45ef4626429438c9b02e633ae412ed43f925d3 \ + --hash=sha256:38565aca3e01ea8734e578fb2118dade0ecb0250533f29e22b8d1a7a196cf4d0 \ + --hash=sha256:387e8e4aa5df2f90b198fa3cad3478822a89cf905b6a6d6c97dc3664689640cc \ + --hash=sha256:39f2a024af5c345ffe8fcf1ee18c049c024c94df393bb09b044a6917c77bde43 \ + --hash=sha256:3df13f73af9b3b38ab1195eb299ecb67a4330c911c97ae04043ff74085728abe \ + --hash=sha256:414e596c75f74e0994084694189d7dc9229fb278e33064d6784b73ffbba3ca31 \ + --hash=sha256:41c8e77f17294c0ac18008a7309b99b34ee72247ef10b6dff4c3f8b5ac29896b \ + --hash=sha256:42290eb6db4ccaca7012656738214f8514082fb6fa40cdeb61bb9a471b52e383 \ + --hash=sha256:42f599f4d48c7e1a3338fdaac3acd075be3b3cf02d4b274f3bf2767aedd3d217 \ + --hash=sha256:43e3a9fdd7cbf7ba6040c31fae0faf84ca1474fef777c4e37912f1540f854499 \ + --hash=sha256:443aefe96b7fdb132e2a70806cca1f2af49bb3f28e47abcd7c2e9dcf4d8fa1b8 \ + --hash=sha256:46dcaa042cd1de6c59e7d9269fa63ff7572b6df40510600b678f0826b3c7af51 \ + --hash=sha256:496af849a472b531f758dbd4d61338f5000538cb1a7b3d20d9d32a264517f509 \ + --hash=sha256:49ae99bdfcae803a885c926bf14f886196e84925395bb3f568fef5c0f0979d7d \ + --hash=sha256:4b57693728576d84ede0a77987ab16881b783d2cd9f1dc180a8fbbc3f79c4428 \ + --hash=sha256:4e3b680b1e0a27457e727a0d572fd81dffa87b6dbf8b228ab57da64f7d85aead \ + --hash=sha256:4e8d01cc3bcae7bbf8167f944aeafefed590fae5693552bba9794a9df68371cc \ + --hash=sha256:5283810d2646741a0d8da2aa733d6aefa0545809afccb2a5d105a26bc45125f1 \ + --hash=sha256:53260c8930da5771cec89439bff99c20c8cb03ddb9588b980697355a83cd4bd3 \ + --hash=sha256:536676848fc5961aca9d20389951f59169508f765637a172403dc5434d722fa0 \ + --hash=sha256:54509b8e92fee4453e152b7558ddef37ce9705a044922f2095a6105e3f80c96f \ + --hash=sha256:56cd5fc4f10a9ea8aa0804bddb7b42506cf9e136046f3b4c27de8fec9e2ecba5 \ + --hash=sha256:5bfd1ac19b1b9986a9c95a82d5e23a391ebb09e12c34d7be6094b86efcc35731 \ + --hash=sha256:5c31aa7e39ee3e8a358573257f1c0bb5c52430d1b637030dd9c8cc2c282926be \ + --hash=sha256:5e3b7d601f6f84156b08cc4a5e541c2b50ad7b36cfc302b657a12477c904a5df \ + --hash=sha256:61922544a0587a13fd3f53e4c0e5e606510c7b0d9d22c8444e5fae22a06b38cb \ + --hash=sha256:6456ff333092d509127d75a638cb411afae8ff17f092635015d1902efec8a293 \ + --hash=sha256:69159730a823dde3ea8d08783e8d47ef135a6d7e8d44eb127e32b321c9db8e3e \ + --hash=sha256:69e52d175a0a7d1e13b4b67ad41c560b7d98e8c6f6126eb0bda496c784faf8c7 \ + --hash=sha256:6aaface73b9c71974c6497366d8b9628357f6c9749e09c4ea3610176c63f2ae3 \ + --hash=sha256:6abbd3e82c731c8e531714466acd5d87b5e88ac3243465337ba71d68e23ae7e3 \ + --hash=sha256:6ff9417c0ada4d0f7d212f928303e5579bdf3ace4c802fa4afabb30995da58c3 \ + --hash=sha256:7421fad442de870a8cbf2287d1cad7e706ece0dbfeba5e911df132cbdc1cb56a \ + --hash=sha256:7883388947767080f094950b342b30d35a2a06b849cd967c422fa0db72b40ea9 \ + --hash=sha256:79eace538c6a97e96d0d03d4f9d314f9677f5ed85a8a984992ffd90b13cb8a56 \ + --hash=sha256:7b1b19636af86a3c7995d4d028dbe376f39b4bf31541146f9c123582a6c94562 \ + --hash=sha256:7dfcad78ea1492ee3a9ec765cb7f51bbc17d477107aaf6b22abf7b2558d1c5a0 \ + --hash=sha256:8087e82f842609734c9b5a1330464f8e94e346ba0e18c832c08bafa4b0d63c15 \ + --hash=sha256:820fb8450edddae3812fd58cbc08e2bf22812cb248ecb5f06dbb82119a56e869 \ + --hash=sha256:8483c2096363120eea8b07c06ae7304d520f686665fffd4811fad423930a65d7 \ + --hash=sha256:84a2cef8deffbd9ab8ee0ea546a2a6a7030c28f44e6cdd4547dbfeb489eb8999 \ + --hash=sha256:86d7f0f8bdb25d2c632b72527325e4776430fd5bc61b9118de4e2b8ddb5f5b01 \ + --hash=sha256:8fe0b50da2d84535fb4f7b4bfa951280f97ce3d558a0443b541166d609e67b57 \ + --hash=sha256:90001d893bc368e302ef168d82130b4e4fdd27b85fa094682df9b667c2d48838 \ + --hash=sha256:9246a0d063cfcbcc85f2359dd6876d681213f4790832272aa16641b4ed5d64d4 \ + --hash=sha256:92b820d345f7a3fc7b8163949ee92df910f290c3fc517b3d5301c78065adafe1 \ + --hash=sha256:952303a7318d4cbe1011400839bb2051c9f84fa0a35923267f5daba34b15d458 \ + --hash=sha256:97fd3a0e8b53efa41970ac1dff3d8cf0d2884cadeb4caaf95db7ad1526926ee3 \ + --hash=sha256:9c1c5705e314449e3308872fe084b8571ce078ee4fc55a98a769bdefe5917392 \ + --hash=sha256:9c9f23004a3d40e89c01a7955d186a6cc83418d93b749701944ce2de3e95a1f3 \ + --hash=sha256:9f63bcef7f4b02b06b35fc01c93b96c43b5e88e1e8868676caacf493d5a31f3a \ + --hash=sha256:a0eadbbf2c30f01efa58e1f110eb6fa293261f6b0b1aa38f7f48707107690af9 \ + --hash=sha256:a28fcbc9b6baf54a2e23f8655f308e4ccc6afdd7266f8fe7954f320dcda0f785 \ + --hash=sha256:a6a61aff018180c9c50b7b0da33bfd29d378af3497429c95006c589a23a11648 \ + --hash=sha256:aabe464bfd13bd25f4821faf111da6fefdc389f870265a53105580e45b0a2e49 \ + --hash=sha256:ab59169ace05dcb49a1d4118f0bde139557adf45091bd85747e36bf5de984dd1 \ + --hash=sha256:b436f6ec4fc3a6b4237c84d3f83170ed2b40bb584222f0ac47a0c8a5921980c7 \ + --hash=sha256:b6b9dadbef0cccd9f4c4ee96b08898afa73e26803bbe0f6aeb5bb12b0074206d \ + --hash=sha256:b852788aa51764e2d8e4cf5493d559326bcae5e38d16ba25ffa322b034df272a \ + --hash=sha256:bae954c382e013d5ea5b190d2830526bfa45ad121c326da0049b8c769f185db6 \ + --hash=sha256:bcce07e23e5769375158f5efdcdafa8d5cd014b93c6683865b840ed65b96f231 \ + --hash=sha256:cc97814dfb786a83b6e2dc2e79351e1b83e6d715647d6887fcabd83026417a00 \ + --hash=sha256:cd2ca96a082a36964aca83e992f72abeb61b7306c1a6cba4c7d06a7b93750cac \ + --hash=sha256:cfb70b4eb56cac4da0a83588f3ad50d46beb0690391082f3d4e2d488c70b68ea \ + --hash=sha256:d0fcf657e9f13ff4b177960ab2200237b12994232dfb6df16f1cfe1d4339f93c \ + --hash=sha256:d14bfb217eb4701e850f1525c9d29d79c44794cdf1c299ead25f39f8c78dea81 \ + --hash=sha256:d57685547e0060cc6fd90ee6a28405d6bd395e525545f13c8d7cd99c78afd79f \ + --hash=sha256:d6bec75c290fe484a8ba4cacdf838501e17c06ecfbbf31eede81a9e431bd7751 \ + --hash=sha256:d9531d9cbeac99af6f038fb1bc351403531f7d634a2c2e10e2f7c854c6ed5b68 \ + --hash=sha256:da4ca1a9d72f9030b3146b8d7022719a9f3d478f61efe6f7dd51d243f61c51b2 \ + --hash=sha256:dab9eb87869da2d6ed3af3f3adf28414baae6ec9d4df355ffc18889132f3436c \ + --hash=sha256:db234eda965dcce15df96bb9709f587cd87d4d52aaf0e80e2f34ec04c7670c57 \ + --hash=sha256:dc0fad4933f427acd5b1cec210f3ea6dce7089e1724e4b9ec6ef47c6c04d1b3b \ + --hash=sha256:dc385593a42e31cd6fb60c19f0ecb015b386603818fc2c6c274fb42bd2bb4165 \ + --hash=sha256:dcc04fedf83effaeb9cce98abc9469bb1b42ef85f03e01c8c1f4438ef7555737 \ + --hash=sha256:e047dc87ef7ca50f4d309bf775ad4a71711c58556d75d7bd0604b2317f43e94b \ + --hash=sha256:e09f753a169951eb4f28c2c774f71069304f66e7277e0f5a2892423599cfa854 \ + --hash=sha256:ed5bb271084b46530ee2ddc0410537a9961152c5ccba2fc98c5276d992ccba87 \ + --hash=sha256:f0aa4aad3b1b69ad3fd85a0fd0952ec64331c762bd77ec51cc814170873890b2 \ + --hash=sha256:f17dbe07eb3ea7f99e4df9b7e0efefe80fbf30d37a8cc4d561a0aed310bc8847 \ + --hash=sha256:f2769a0344a09e9ccf5b3cce538bc75a51b53eff3275d3896310c8552049195d \ + --hash=sha256:f55f0b01956a094c8587146d9558c91937e78789c333860ffaf35931a6e5dbc4 \ + --hash=sha256:f5d497865f05bb222cab7016c6034542e84e5f29f49c6fd3f4939cda7197b5b8 \ + --hash=sha256:f70541f3104339f59f830522d94ebadb1bf47426287381623443d8bb1cdbf33d \ + --hash=sha256:fb9a0a6dc3d1b3986cb88091b6899f0396651e0f74e2c9766ab8d6ffc3842e29 \ + --hash=sha256:fce6c48559c86d1ac3632ccb1bebc7d5442fbe79bd9bb0e40379ee54be2a4051 \ + --hash=sha256:fd46fff7eb62c24804d234f0051c7a8ea81285ad63e0337d3dcf33ca82aee58a + # via google-genai diff --git a/scripts/benchmark_postgres.py b/scripts/benchmark_postgres.py new file mode 100644 index 0000000..01f91e9 --- /dev/null +++ b/scripts/benchmark_postgres.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Bounded local PostgreSQL journey benchmark with percentile evidence.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +from time import perf_counter +from typing import Awaitable, Callable + +from sqlalchemy import text + +from bff.database import DatabaseManager, DatabaseSettings +from bff.postgres.analytics_repository import AnalyticsRepository +from bff.postgres.base import ANALYTICS_CACHE +from bff.postgres.funder_repository import SourceFunderRepository +from bff.postgres.organization_repository import OrganizationRepository +from bff.postgres.registry_repository import RegistryRepository + + +Journey = Callable[[], Awaitable[object]] + + +def _percentile(values: list[float], percentile: float) -> float: + ordered = sorted(values) + rank = max(1, math.ceil(percentile * len(ordered))) + return ordered[rank - 1] + + +async def _measure(journey: Journey, samples: int) -> dict[str, float | int]: + durations: list[float] = [] + errors = 0 + started = perf_counter() + for _ in range(samples): + sample_started = perf_counter() + try: + await journey() + except Exception: + errors += 1 + durations.append((perf_counter() - sample_started) * 1000) + elapsed = perf_counter() - started + return { + "samples": samples, + "p50_ms": round(_percentile(durations, 0.50), 2), + "p95_ms": round(_percentile(durations, 0.95), 2), + "p99_ms": round(_percentile(durations, 0.99), 2), + "throughput_rps": round(samples / elapsed, 3) if elapsed else 0.0, + "error_rate": round(errors / samples, 4), + } + + +async def benchmark(samples: int, concurrency: int) -> dict[str, object]: + database = DatabaseManager(DatabaseSettings.from_env()) + sessions = database.sessions() + analytics = AnalyticsRepository(sessions) + organizations = OrganizationRepository(sessions) + registry = RegistryRepository(sessions) + funders = SourceFunderRepository(sessions) + try: + async with sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + exact_name = await session.scalar( + text( + """ + SELECT registered_name FROM charity_registry_organizations + WHERE dataset_version=:dataset_version AND is_current_source_record + ORDER BY registry_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + text_name = str(exact_name).split()[0] + funder_country = await session.scalar( + text( + """ + SELECT country_code FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version AND country_code IS NOT NULL + GROUP BY country_code ORDER BY COUNT(*) DESC, country_code LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + + async def health(): + async with sessions() as session: + return await session.scalar(text("SELECT 1")) + + async def primary_dashboard(): + return await asyncio.gather( + organizations.stats(), + analytics.map(), + analytics.trends(months=24), + analytics.themes(), + analytics.summary(), + ) + + journeys: dict[str, Journey] = { + "health": health, + "organization_list": lambda: organizations.list(limit=20), + "default_map": lambda: analytics.map(), + "map_connections": lambda: analytics.map_connections(limit=250), + "monthly_trends": lambda: analytics.trends(months=24), + "yearly_trends": lambda: analytics.trends( + months=120, granularity="yearly" + ), + "programme_themes": lambda: analytics.themes(), + "network_summary": analytics.summary, + "primary_dashboard": primary_dashboard, + "registry_exact": lambda: registry.page(query=str(exact_name), limit=20), + "registry_text": lambda: registry.page(query=text_name, limit=20), + "funder_ranking": lambda: funders.list( + beneficiary_country=str(funder_country), page_size=25 + ), + } + results: dict[str, object] = {} + for name, journey in journeys.items(): + cold_samples = 20 if name == "primary_dashboard" else 1 + cold_durations = [] + for _ in range(cold_samples): + await ANALYTICS_CACHE.clear() + cold_started = perf_counter() + await journey() + cold_durations.append((perf_counter() - cold_started) * 1000) + measured = await _measure(journey, samples) + measured["cold_ms"] = round(cold_durations[0], 2) + if cold_samples > 1: + measured.update( + { + "cold_samples": cold_samples, + "cold_p50_ms": round(_percentile(cold_durations, 0.50), 2), + "cold_p95_ms": round(_percentile(cold_durations, 0.95), 2), + "cold_p99_ms": round(_percentile(cold_durations, 0.99), 2), + } + ) + results[name] = measured + + semaphore = asyncio.Semaphore(concurrency) + + async def bounded_primary(): + async with semaphore: + await primary_dashboard() + + concurrent_started = perf_counter() + concurrent_results = await asyncio.gather( + *(bounded_primary() for _ in range(concurrency)), + return_exceptions=True, + ) + concurrent_elapsed = perf_counter() - concurrent_started + results["primary_dashboard_concurrent"] = { + "concurrency": concurrency, + "elapsed_ms": round(concurrent_elapsed * 1000, 2), + "throughput_rps": round(concurrency / concurrent_elapsed, 3), + "errors": sum(isinstance(item, BaseException) for item in concurrent_results), + } + pool = database.engine().sync_engine.pool + results["database_pool"] = { + "configured_size": database.settings.pool_size, + "configured_max_overflow": database.settings.max_overflow, + "checked_out_after_run": pool.checkedout(), + "checked_in_after_run": pool.checkedin(), + } + results["analytics_cache"] = { + "hits": ANALYTICS_CACHE.hits, + "misses": ANALYTICS_CACHE.misses, + "hit_ratio": round(ANALYTICS_CACHE.hit_ratio, 4), + } + return { + "dataset_version": dataset_version, + "samples_per_journey": samples, + "results": results, + } + finally: + await database.close() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--samples", type=int, default=10) + parser.add_argument("--concurrency", type=int, default=5) + arguments = parser.parse_args() + if not 1 <= arguments.samples <= 100: + parser.error("--samples must be between 1 and 100") + if not 1 <= arguments.concurrency <= 20: + parser.error("--concurrency must be between 1 and 20") + print( + json.dumps( + asyncio.run(benchmark(arguments.samples, arguments.concurrency)), + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/check_licenses.py b/scripts/check_licenses.py new file mode 100644 index 0000000..1a8b8b5 --- /dev/null +++ b/scripts/check_licenses.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Fail on strong-copyleft runtime/development packages using local metadata.""" + +from __future__ import annotations + +import argparse +from importlib import metadata +import json +from pathlib import Path +import re +from typing import Iterable + + +ROOT = Path(__file__).resolve().parents[1] +FORBIDDEN = re.compile(r"(?:^|[^A-Z])(?:AGPL|SSPL|(? Iterable[tuple[str, str, str]]: + for distribution in metadata.distributions(): + name = distribution.metadata.get("Name", "unknown") + version = distribution.version + declared = distribution.metadata.get("License-Expression") or distribution.metadata.get( + "License", "" + ) + classifiers = [ + item.removeprefix("License :: ") + for item in distribution.metadata.get_all("Classifier", []) + if item.startswith("License :: ") + ] + license_text = str(declared or " OR ".join(classifiers) or "UNKNOWN") + yield str(name), str(version), license_text + + +def _node_licenses(node_modules: Path) -> Iterable[tuple[str, str, str]]: + for path in sorted(node_modules.rglob("package.json")): + relative_parts = path.relative_to(node_modules).parts + is_top_level_package = ( + len(relative_parts) == 2 + or ( + len(relative_parts) == 3 + and relative_parts[0].startswith("@") + ) + ) + if not is_top_level_package or ".bin" in path.parts: + continue + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + continue + name = str(payload.get("name", path.parent.name)) + version = str(payload.get("version", "unknown")) + declared = payload.get("license", "UNKNOWN") + if isinstance(declared, dict): + declared = declared.get("type", "UNKNOWN") + yield name, version, str(declared) + + +def scan(node_modules: Path) -> dict[str, object]: + records = list(_python_licenses()) + list(_node_licenses(node_modules)) + forbidden = sorted( + f"{name}@{version}: {license_name}" + for name, version, license_name in records + if FORBIDDEN.search(license_name) + ) + unknown = sum(1 for _, _, license_name in records if license_name == "UNKNOWN") + return { + "status": "failed" if forbidden else "passed", + "components_scanned": len(records), + "unknown_license_metadata": unknown, + "forbidden": forbidden, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--node-modules", + type=Path, + default=ROOT / "frontend" / "node_modules", + ) + args = parser.parse_args() + result = scan(args.node_modules) + print(json.dumps(result, sort_keys=True)) + return 1 if result["status"] == "failed" else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_sbom.py b/scripts/generate_sbom.py new file mode 100644 index 0000000..ab9eb21 --- /dev/null +++ b/scripts/generate_sbom.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +"""Generate deterministic CycloneDX JSON SBOMs from committed lockfiles.""" + +from __future__ import annotations + +import argparse +import base64 +import json +from pathlib import Path +import re +from typing import Any +from urllib.parse import quote + + +ROOT = Path(__file__).resolve().parents[1] +PYTHON_LOCK = ROOT / "requirements.txt" +NPM_LOCK = ROOT / "frontend" / "package-lock.json" + + +def _component(name: str, version: str, ecosystem: str) -> dict[str, Any]: + normalized_name = name.replace("_", "-").lower() if ecosystem == "pypi" else name + escaped_name = quote(normalized_name, safe="@/") + purl = f"pkg:{ecosystem}/{escaped_name}@{quote(version, safe='')}" + return { + "type": "library", + "bom-ref": purl, + "name": normalized_name, + "version": version, + "purl": purl, + } + + +def python_components(path: Path = PYTHON_LOCK) -> list[dict[str, Any]]: + components: list[dict[str, Any]] = [] + current: dict[str, Any] | None = None + for line in path.read_text(encoding="utf-8").splitlines(): + requirement = re.match(r"^([A-Za-z0-9_.-]+)==([^\s\\]+)", line) + if requirement: + current = _component(requirement.group(1), requirement.group(2), "pypi") + current["hashes"] = [] + components.append(current) + continue + digest = re.search(r"--hash=sha256:([0-9a-f]{64})", line) + if digest and current is not None: + current["hashes"].append({"alg": "SHA-256", "content": digest.group(1)}) + for component in components: + if not component["hashes"]: + raise ValueError(f"Python component lacks a locked hash: {component['name']}") + return sorted(components, key=lambda item: (item["name"], item["version"])) + + +def _npm_name(package_path: str, record: dict[str, Any]) -> str: + if record.get("name"): + return str(record["name"]) + return package_path.rsplit("node_modules/", 1)[-1] + + +def _integrity_hash(value: str) -> dict[str, str] | None: + if "-" not in value: + return None + algorithm, encoded = value.split("-", 1) + supported = {"sha256": "SHA-256", "sha384": "SHA-384", "sha512": "SHA-512"} + if algorithm not in supported: + return None + try: + content = base64.b64decode(encoded, validate=True).hex() + except ValueError as exc: + raise ValueError("Invalid npm lockfile integrity value") from exc + return {"alg": supported[algorithm], "content": content} + + +def npm_components(path: Path = NPM_LOCK) -> list[dict[str, Any]]: + payload = json.loads(path.read_text(encoding="utf-8")) + if int(payload.get("lockfileVersion", 0)) < 3: + raise ValueError("npm lockfile version 3 or newer is required") + unique: dict[tuple[str, str], dict[str, Any]] = {} + for package_path, record in payload.get("packages", {}).items(): + if not package_path or "node_modules/" not in package_path: + continue + version = str(record.get("version", "")).strip() + if not version: + raise ValueError(f"npm package lacks a version: {package_path}") + name = _npm_name(str(package_path), record) + key = (name, version) + component = unique.setdefault(key, _component(name, version, "npm")) + integrity = _integrity_hash(str(record.get("integrity", ""))) + if integrity: + component.setdefault("hashes", []) + if integrity not in component["hashes"]: + component["hashes"].append(integrity) + if record.get("dev") or record.get("optional"): + component["scope"] = "optional" + return sorted(unique.values(), key=lambda item: (item["name"], item["version"])) + + +def document(application: str, components: list[dict[str, Any]]) -> dict[str, Any]: + return { + "bomFormat": "CycloneDX", + "specVersion": "1.5", + "version": 1, + "metadata": { + "component": { + "type": "application", + "name": application, + "version": "1.0.0", + }, + "properties": [ + {"name": "generation.source", "value": "committed-lockfile"}, + {"name": "generation.reproducible", "value": "true"}, + ], + }, + "components": components, + } + + +def write_document(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output-directory", + type=Path, + default=ROOT / "docs" / "remediation" / "evidence" / "sbom", + ) + args = parser.parse_args() + backend = document("foundation-intelligence-backend", python_components()) + frontend = document("foundation-intelligence-frontend", npm_components()) + write_document(args.output_directory / "backend.cdx.json", backend) + write_document(args.output_directory / "frontend.cdx.json", frontend) + print( + json.dumps( + { + "status": "passed", + "backend_components": len(backend["components"]), + "frontend_components": len(frontend["components"]), + "output_directory": str(args.output_directory), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/http_load_smoke.py b/scripts/http_load_smoke.py new file mode 100644 index 0000000..2548a52 --- /dev/null +++ b/scripts/http_load_smoke.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Small authenticated HTTP smoke/load gate for an approved staging URL.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import math +from time import perf_counter + +import httpx + + +DEFAULT_PATHS = ( + "/health/ready", + "/api/charities/stats", + "/api/charities/grants/overview", + "/api/charities/grants/map", +) + + +def percentile(values: list[float], quantile: float) -> float: + if not values: + raise ValueError("At least one timing is required") + ordered = sorted(values) + return ordered[max(1, math.ceil(len(ordered) * quantile)) - 1] + + +async def run(base_url: str, token: str, samples: int, concurrency: int) -> dict[str, object]: + semaphore = asyncio.Semaphore(concurrency) + timings: dict[str, list[float]] = {path: [] for path in DEFAULT_PATHS} + failures: list[dict[str, object]] = [] + headers = {"Authorization": f"Bearer {token}"} + timeout = httpx.Timeout(15) + + async with httpx.AsyncClient( + base_url=base_url.rstrip("/"), + headers=headers, + timeout=timeout, + follow_redirects=False, + ) as client: + async def request(path: str) -> None: + async with semaphore: + started = perf_counter() + try: + response = await client.get(path) + except httpx.HTTPError as exc: + failures.append({"path": path, "error_class": exc.__class__.__name__}) + return + timings[path].append((perf_counter() - started) * 1000) + if response.status_code != 200: + failures.append({"path": path, "status": response.status_code}) + + await asyncio.gather( + *(request(path) for path in DEFAULT_PATHS for _ in range(samples)) + ) + + results = { + path: { + "samples": len(values), + "p50_ms": round(percentile(values, 0.5), 2) if values else None, + "p95_ms": round(percentile(values, 0.95), 2) if values else None, + } + for path, values in timings.items() + } + passed = not failures and all( + values and percentile(values, 0.95) < 3000 for values in timings.values() + ) + return {"status": "passed" if passed else "failed", "results": results, "failures": failures} + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base-url", required=True) + parser.add_argument("--token", required=True) + parser.add_argument("--samples", type=int, default=10) + parser.add_argument("--concurrency", type=int, default=5) + args = parser.parse_args() + if not 2 <= args.samples <= 100: + parser.error("--samples must be between 2 and 100") + if not 1 <= args.concurrency <= 20: + parser.error("--concurrency must be between 1 and 20") + result = asyncio.run( + run(args.base_url, args.token, args.samples, args.concurrency) + ) + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 if result["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/load_test_api.py b/scripts/load_test_api.py new file mode 100644 index 0000000..63028d8 --- /dev/null +++ b/scripts/load_test_api.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +"""Local in-process production API load test without external calls.""" + +from __future__ import annotations + +import argparse +import asyncio +from datetime import datetime, timedelta, timezone +import json +import math +import os +from time import perf_counter + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import httpx +from jose import jwt +from jose.utils import base64url_encode +from sqlalchemy import text + + +def _configure_production_auth() -> str: + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + numbers = private_key.public_key().public_numbers() + + def encoded(value: int) -> str: + payload = value.to_bytes((value.bit_length() + 7) // 8, "big") + return base64url_encode(payload).decode("ascii") + + os.environ.update( + { + "APP_ENV": "production", + "AUTH_MODE": "oidc", + "OIDC_ISSUER": "https://load-test.identity.invalid/", + "OIDC_AUDIENCE": "foundation-intelligence-api", + "OIDC_JWKS_JSON": json.dumps( + { + "keys": [ + { + "kty": "RSA", + "kid": "local-load-test", + "use": "sig", + "alg": "RS256", + "n": encoded(numbers.n), + "e": encoded(numbers.e), + } + ] + } + ), + "CORS_ORIGINS": "https://load-test.app.invalid", + "DEV_AUTH_ENABLED": "false", + "SESSION_COOKIE_SECURE": "true", + "CORE_PROXY_ENABLED": "false", + "RATE_LIMIT_REQUESTS": "10000", + } + ) + now = datetime.now(timezone.utc) + private_pem = private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + return jwt.encode( + { + "sub": "local-load-test", + "roles": ["viewer"], + "iss": os.environ["OIDC_ISSUER"], + "aud": os.environ["OIDC_AUDIENCE"], + "iat": now, + "exp": now + timedelta(minutes=10), + }, + private_pem, + algorithm="RS256", + headers={"kid": "local-load-test"}, + ) + + +def _percentile(values: list[float], percentile: float) -> float: + ordered = sorted(values) + return ordered[max(1, math.ceil(len(ordered) * percentile)) - 1] + + +async def run(samples: int, concurrency: int) -> dict[str, object]: + token = _configure_production_auth() + from bff.audit import MemoryAuditSink + from bff.main import app + from bff.postgres.base import ANALYTICS_CACHE + from bff.utils.logging import logger + + logger.setLevel("WARNING") + + headers = {"Authorization": f"Bearer {token}"} + async with app.router.lifespan_context(app): + app.state.audit_sink = MemoryAuditSink() + async with app.state.database.sessions()() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + registered_name = await session.scalar( + text( + """ + SELECT registered_name FROM charity_registry_organizations + WHERE dataset_version=:dataset_version AND is_current_source_record + ORDER BY registry_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + country = await session.scalar( + text( + """ + SELECT country_code FROM analytics_country_funder_rankings + WHERE dataset_version=:dataset_version + GROUP BY country_code ORDER BY COUNT(*) DESC LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient( + transport=transport, + base_url="https://load-test.app.invalid", + headers=headers, + timeout=30, + ) as client: + endpoints = { + "health": "/health/ready", + "organization_list": "/api/charities?limit=20", + "default_map": "/api/charities/grants/map", + "map_connections": ( + "/api/charities/grants/map/connections?limit=250" + ), + "overview": "/api/charities/grants/overview", + "yearly_trends": ( + "/api/charities/grants/overview/trends?" + "granularity=yearly" + ), + "registry_exact": ( + "/api/charities/directory/organizations?" + + str(httpx.QueryParams({"query": str(registered_name)})) + ), + "registry_text": ( + "/api/charities/directory/organizations?" + + str( + httpx.QueryParams( + {"query": str(registered_name).split()[0]} + ) + ) + ), + "funder_ranking": ( + "/api/charities/grants/funders?beneficiary_country=" + + str(country) + ), + } + results: dict[str, object] = {} + for name, path in endpoints.items(): + await ANALYTICS_CACHE.clear() + cold_started = perf_counter() + cold_response = await client.get(path) + cold_ms = (perf_counter() - cold_started) * 1000 + durations = [] + statuses = [] + for _ in range(samples): + started = perf_counter() + response = await client.get(path) + durations.append((perf_counter() - started) * 1000) + statuses.append(response.status_code) + results[name] = { + "samples": samples, + "cold_ms": round(cold_ms, 2), + "cold_status": cold_response.status_code, + "p50_ms": round(_percentile(durations, 0.50), 2), + "p95_ms": round(_percentile(durations, 0.95), 2), + "p99_ms": round(_percentile(durations, 0.99), 2), + "throughput_rps": round( + samples / (sum(durations) / 1000), 3 + ), + "error_rate": round( + sum(status >= 400 for status in statuses) / samples, 4 + ), + } + + panel_paths = ( + "/api/charities/stats", + "/api/charities/grants/map", + "/api/charities/grants/trends?months=24", + "/api/charities/grants/themes", + "/api/charities/grants/summary", + ) + + async def dashboard(): + responses = await asyncio.gather( + *(client.get(path) for path in panel_paths) + ) + if any(response.status_code >= 400 for response in responses): + raise RuntimeError("Dashboard request failed") + + await ANALYTICS_CACHE.clear() + cold_started = perf_counter() + await dashboard() + results["primary_dashboard_cold"] = { + "elapsed_ms": round((perf_counter() - cold_started) * 1000, 2) + } + started = perf_counter() + concurrent_results = await asyncio.gather( + *(dashboard() for _ in range(concurrency)), + return_exceptions=True, + ) + elapsed = perf_counter() - started + results["primary_dashboard_concurrent"] = { + "concurrency": concurrency, + "elapsed_ms": round(elapsed * 1000, 2), + "throughput_rps": round(concurrency / elapsed, 3), + "errors": sum( + isinstance(result, BaseException) for result in concurrent_results + ), + } + results["analytics_cache"] = { + "hits": ANALYTICS_CACHE.hits, + "misses": ANALYTICS_CACHE.misses, + "hit_ratio": round(ANALYTICS_CACHE.hit_ratio, 4), + } + pool = app.state.database.engine().sync_engine.pool + results["database_pool"] = { + "size": pool.size(), + "checked_out": pool.checkedout(), + "overflow": pool.overflow(), + } + return {"dataset_version": dataset_version, "results": results} + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--samples", type=int, default=20) + parser.add_argument("--concurrency", type=int, default=5) + arguments = parser.parse_args() + if not 2 <= arguments.samples <= 100: + parser.error("--samples must be between 2 and 100") + if not 1 <= arguments.concurrency <= 20: + parser.error("--concurrency must be between 1 and 20") + print( + json.dumps( + asyncio.run(run(arguments.samples, arguments.concurrency)), + indent=2, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_ci_workflows.py b/scripts/validate_ci_workflows.py new file mode 100644 index 0000000..54546c5 --- /dev/null +++ b/scripts/validate_ci_workflows.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Offline completeness/safety checks for GitHub workflow definitions.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW_ROOT = ROOT / ".github" / "workflows" +CI_PATH = WORKFLOW_ROOT / "ci.yml" +STAGING_PATH = WORKFLOW_ROOT / "deploy-staging.yml" +PRODUCTION_PATH = WORKFLOW_ROOT / "deploy-production.yml" + +REQUIRED_CI_MARKERS = frozenset( + { + "compileall", + "flake8", + "mypy --config-file", + "pytest -q --cov", + "npm run lint", + "npm test", + "npm run build", + "npm run test:e2e", + "gitleaks/gitleaks-action", + "gh-action-pip-audit", + "npm audit", + "check_licenses.py", + "github/codeql-action", + "generate_sbom.py", + "anchore/sbom-action", + "docker build", + "verify_container_image.sh", + "aquasecurity/trivy-action", + "terraform fmt -check", + "terraform -chdir=infra/terraform/environments/staging validate", + "alembic upgrade head", + "test_sqlite_to_postgres_migration.py", + "test_api_golden.py", + "test_ci_performance_smoke.py", + } +) + + +def validate() -> dict[str, object]: + workflows = [CI_PATH, STAGING_PATH, PRODUCTION_PATH] + for path in workflows: + if not path.is_file(): + raise ValueError(f"Missing workflow: {path.name}") + ci = CI_PATH.read_text(encoding="utf-8") + staging = STAGING_PATH.read_text(encoding="utf-8") + production = PRODUCTION_PATH.read_text(encoding="utf-8") + combined = "\n".join((ci, staging, production)) + + missing = sorted(marker for marker in REQUIRED_CI_MARKERS if marker not in ci) + if missing: + raise ValueError("Missing CI gates: " + ", ".join(missing)) + if "pull_request_target:" in combined: + raise ValueError("pull_request_target is forbidden for untrusted code") + if re.search(r"aws-(?:access-key-id|secret-access-key)", combined, re.IGNORECASE): + raise ValueError("Long-lived AWS key workflow input is forbidden") + if "terraform apply" in ci or "terraform destroy" in combined: + raise ValueError("PR CI must not apply and no workflow may destroy infrastructure") + if "--require-hashes" not in ci or "npm ci --ignore-scripts" not in ci: + raise ValueError("Deterministic dependency installation is required") + if re.search(r"(?m)^\s*(?:run:\s*)?pytest\b", ci): + raise ValueError( + "Invoke pytest through python -m so the repository root remains importable" + ) + + for action, reference in re.findall(r"uses:\s*([^@\s]+)@([^\s]+)", combined): + if reference in {"main", "master", "latest"}: + raise ValueError(f"Floating action reference is forbidden: {action}@{reference}") + + staging_markers = ( + "workflow_dispatch:", + "I_APPROVE_STAGING", + "environment: staging-publish", + "environment: staging", + "id-token: write", + "aws-actions/configure-aws-credentials", + "imageDigest", + "api_image=${api_repository}@${api_digest}", + "reviewed.tfplan", + "terraform -chdir=${TF_ROOT} apply", + "alembic", + "update-service", + "aws s3 sync", + "http_load_smoke.py", + "PostgreSQL reconciliation release gate", + "npm run test:e2e", + "previous-task-definitions.json", + ) + missing_staging = [marker for marker in staging_markers if marker not in staging] + if missing_staging: + raise ValueError("Missing staging gates: " + ", ".join(missing_staging)) + if "if: ${{ false }}" not in production or "environment: production" not in production: + raise ValueError("Production workflow must remain explicitly disabled and protected") + + return { + "status": "passed", + "workflow_files": len(workflows), + "ci_gate_markers": len(REQUIRED_CI_MARKERS), + "staging_requires_oidc": True, + "staging_requires_protected_environments": ["staging-publish", "staging"], + "production_enabled": False, + "workflow_execution": "not_tested", + } + + +def main() -> int: + try: + result = validate() + except ValueError as exc: + print(json.dumps({"status": "failed", "error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_terraform_static.py b/scripts/validate_terraform_static.py new file mode 100644 index 0000000..2a60d47 --- /dev/null +++ b/scripts/validate_terraform_static.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Offline Terraform structure/security checks when the Terraform CLI is absent.""" + +from __future__ import annotations + +import json +from pathlib import Path +import re +import sys + + +ROOT = Path(__file__).resolve().parents[1] +TERRAFORM_ROOT = ROOT / "infra" / "terraform" +MODULE_ROOT = TERRAFORM_ROOT / "modules" / "platform" +REQUIRED_RESOURCE_TYPES = frozenset( + { + "aws_acm_certificate", + "aws_appautoscaling_policy", + "aws_budgets_budget", + "aws_cloudfront_distribution", + "aws_cloudwatch_dashboard", + "aws_cloudwatch_log_group", + "aws_cloudwatch_metric_alarm", + "aws_db_instance", + "aws_db_parameter_group", + "aws_db_subnet_group", + "aws_ecr_repository", + "aws_ecs_cluster", + "aws_ecs_service", + "aws_ecs_task_definition", + "aws_iam_openid_connect_provider", + "aws_iam_role", + "aws_internet_gateway", + "aws_kms_key", + "aws_lb", + "aws_lb_target_group", + "aws_nat_gateway", + "aws_route53_record", + "aws_s3_bucket", + "aws_scheduler_schedule", + "aws_security_group", + "aws_sfn_state_machine", + "aws_sns_topic", + "aws_sqs_queue", + "aws_subnet", + "aws_vpc", + "aws_vpc_endpoint", + "aws_wafv2_web_acl", + } +) +ALLOWED_WILDCARD_RESOURCE_CONTEXTS = frozenset( + { + "AccountAdministration", + "CloudWatchLogsEncryption", + "AuthenticateToECR", + "ManageSynchronousTaskEvents", + "WriteExecutionLogs", + "PullImages", + "CloudFrontReadFrontendObjects", + "OperationalNotificationEncryption", + "ManageSynchronousWorkerTask", + "RegisterTaggedTaskDefinitions", + "ObserveReleaseGate", + } +) + + +def _structure_only(text: str) -> str: + """Remove comments and quoted content, preserving only structural tokens.""" + output: list[str] = [] + index = 0 + in_string = False + in_line_comment = False + in_block_comment = False + escaped = False + while index < len(text): + current = text[index] + following = text[index + 1] if index + 1 < len(text) else "" + if in_line_comment: + if current == "\n": + in_line_comment = False + output.append(current) + elif in_block_comment: + if current == "*" and following == "/": + in_block_comment = False + index += 1 + elif in_string: + if escaped: + escaped = False + elif current == "\\": + escaped = True + elif current == '"': + in_string = False + elif current == '"': + in_string = True + elif current == "#": + in_line_comment = True + elif current == "/" and following == "/": + in_line_comment = True + index += 1 + elif current == "/" and following == "*": + in_block_comment = True + index += 1 + else: + output.append(current) + index += 1 + if in_string or in_block_comment: + raise ValueError("Unterminated Terraform string or block comment") + return "".join(output) + + +def _balanced(path: Path, text: str) -> None: + pairs = {"}": "{", ")": "(", "]": "["} + stack: list[tuple[str, int]] = [] + for offset, token in enumerate(_structure_only(text)): + if token in "{([": + stack.append((token, offset)) + elif token in "})]": + if not stack or stack[-1][0] != pairs[token]: + raise ValueError(f"{path}: unmatched {token} at structural offset {offset}") + stack.pop() + if stack: + raise ValueError(f"{path}: unmatched {stack[-1][0]} at structural offset {stack[-1][1]}") + + +def validate() -> dict[str, object]: + files = sorted(TERRAFORM_ROOT.rglob("*.tf")) + if not files: + raise ValueError("No Terraform files found") + contents = {path: path.read_text(encoding="utf-8") for path in files} + for path, text in contents.items(): + _balanced(path, text) + if "TODO" in text or "FIXME" in text: + raise ValueError(f"{path}: placeholder marker is forbidden") + if re.search(r"(?:AKIA|ASIA)[0-9A-Z]{16}", text): + raise ValueError(f"{path}: AWS credential-shaped value detected") + + module_text = "\n".join( + text for path, text in contents.items() if MODULE_ROOT in path.parents + ) + resource_types = set(re.findall(r'\bresource\s+"([a-z0-9_]+)"\s+"', module_text)) + missing = sorted(REQUIRED_RESOURCE_TYPES - resource_types) + if missing: + raise ValueError(f"Missing required resource types: {', '.join(missing)}") + + required_contracts = { + "private RDS": r"publicly_accessible\s*=\s*false", + "RDS deletion protection": r"deletion_protection\s*=\s*true", + "Terraform destroy guard": r"prevent_destroy\s*=\s*true", + "S3 public ACL block": r"block_public_acls\s*=\s*true", + "S3 public policy block": r"block_public_policy\s*=\s*true", + "bucket owner enforcement": r'object_ownership\s*=\s*"BucketOwnerEnforced"', + "KMS bucket encryption": r'kms_master_key_id\s*=\s*aws_kms_key\.platform\.arn', + "non-destructive buckets": r"force_destroy\s*=\s*false", + "immutable ECR tags": r'image_tag_mutability\s*=\s*"IMMUTABLE"', + "immutable image validation": r'@sha256:\[0-9a-f\]\{64\}\$', + "non-root API": r'user\s*=\s*"10001:10001"', + "read-only API filesystem": r"readonlyRootFilesystem\s*=\s*true", + "private ECS tasks": r"assign_public_ip\s*=\s*false", + "OIDC audience pin": r'token\.actions\.githubusercontent\.com:aud', + "OIDC subject pin": r'token\.actions\.githubusercontent\.com:sub', + } + for name, pattern in required_contracts.items(): + if not re.search(pattern, module_text): + raise ValueError(f"Missing Terraform security contract: {name}") + + storage_text = (MODULE_ROOT / "storage.tf").read_text(encoding="utf-8") + if re.search(r"\bexpiration\s*\{", storage_text): + raise ValueError("Destructive S3 lifecycle expiration must remain disabled") + if re.search(r'Action\s*=\s*"\*"', module_text): + raise ValueError("Unbounded wildcard IAM action detected") + + wildcard_contexts: set[str] = set() + current_sid = "" + for line in module_text.splitlines(): + sid = re.search(r'Sid\s*=\s*"([^"]+)"', line) + if sid: + current_sid = sid.group(1) + if re.search(r'Resource\s*=\s*"\*"', line): + wildcard_contexts.add(current_sid) + unknown_wildcards = sorted(wildcard_contexts - ALLOWED_WILDCARD_RESOURCE_CONTEXTS) + if unknown_wildcards: + raise ValueError( + "Unjustified wildcard IAM resource contexts: " + ", ".join(unknown_wildcards) + ) + + for environment in ("dev", "staging"): + environment_root = TERRAFORM_ROOT / "environments" / environment + environment_text = "\n".join( + path.read_text(encoding="utf-8") for path in sorted(environment_root.glob("*.tf")) + ) + if f'environment = "{environment}"' not in environment_text: + raise ValueError(f"{environment}: module environment is missing") + if not re.search(r"enable_schedules\s*=\s*false", environment_text): + raise ValueError(f"{environment}: schedules must fail closed") + if not re.search(r'backend\s+"s3"\s*\{\s*encrypt\s*=\s*true\s*\}', environment_text): + raise ValueError(f"{environment}: encrypted backend configuration point is missing") + if re.search(r'\b(?:bucket|key|region)\s*=\s*"[^\"]+"', environment_text): + raise ValueError(f"{environment}: live backend coordinates must not be committed") + + return { + "status": "passed", + "terraform_files": len(files), + "resource_blocks": len(re.findall(r'\bresource\s+"', module_text)), + "resource_types": len(resource_types), + "environments": ["dev", "staging"], + "provider_dependent_validation": "not_tested", + "aws_actions_performed": False, + } + + +def main() -> int: + try: + result = validate() + except ValueError as exc: + print(json.dumps({"status": "failed", "error": str(exc)}, sort_keys=True)) + return 1 + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify_container_image.sh b/scripts/verify_container_image.sh new file mode 100755 index 0000000..dbf55e3 --- /dev/null +++ b/scripts/verify_container_image.sh @@ -0,0 +1,68 @@ +#!/bin/sh +set -eu + +image_name=${1:-foundation-intelligence-backend:local} +maximum_bytes=${MAXIMUM_IMAGE_BYTES:-524288000} + +image_user=$(docker image inspect --format '{{.Config.User}}' "$image_name") +case "$image_user" in + ""|0|0:0|root) + echo "FAIL: image user is privileged: ${image_user:-unset}" >&2 + exit 1 + ;; +esac + +healthcheck=$(docker image inspect --format '{{json .Config.Healthcheck.Test}}' "$image_name") +if [ "$healthcheck" = "null" ] || [ "$healthcheck" = "[]" ]; then + echo "FAIL: image has no healthcheck" >&2 + exit 1 +fi + +image_size=$(docker image inspect --format '{{.Size}}' "$image_name") +if [ "$image_size" -gt "$maximum_bytes" ]; then + echo "FAIL: image size ${image_size} exceeds ${maximum_bytes} bytes" >&2 + exit 1 +fi + +temporary_directory=$(mktemp -d "${TMPDIR:-/tmp}/fip-image-check.XXXXXX") +container_id="" +cleanup() { + if [ -n "$container_id" ]; then + docker container rm "$container_id" >/dev/null 2>&1 || true + fi + rm -rf "$temporary_directory" +} +trap cleanup EXIT INT TERM + +container_id=$(docker container create "$image_name") +docker container export "$container_id" >"$temporary_directory/rootfs.tar" +docker container rm "$container_id" >/dev/null +container_id="" + +tar -tf "$temporary_directory/rootfs.tar" >"$temporary_directory/files.txt" +if grep -E '(^|/)(\.env($|\.)|[^/]+\.(db|sqlite|sqlite3)$)' "$temporary_directory/files.txt"; then + echo "FAIL: image contains an environment or SQLite file" >&2 + exit 1 +fi +if grep -E '^(app|home|root)/(.*/)?(\.aws|\.netrc|\.npmrc|\.pypirc|credentials|id_(rsa|ecdsa|ed25519)|[^/]+\.(pem|key|p12|pfx))(/|$)' "$temporary_directory/files.txt"; then + echo "FAIL: image contains an application credential or private-key file" >&2 + exit 1 +fi +if grep -E '^app/src/data/(raw|processed|preprocessed)/' "$temporary_directory/files.txt"; then + echo "FAIL: image contains a domain-data payload" >&2 + exit 1 +fi +if grep -E '(^|/)(gcc|g\+\+|cc|make)$' "$temporary_directory/files.txt"; then + echo "FAIL: runtime image contains a compiler toolchain" >&2 + exit 1 +fi + +if [ -f docker-bake.hcl ]; then + grep -F '"linux/amd64"' docker-bake.hcl >/dev/null + grep -F '"linux/arm64"' docker-bake.hcl >/dev/null +else + echo "FAIL: docker-bake.hcl is missing" >&2 + exit 1 +fi + +echo "PASS: user=$image_user size_bytes=$image_size healthcheck=present data_files=absent multiarch=declared" diff --git a/scripts/verify_local_restore.sh b/scripts/verify_local_restore.sh new file mode 100755 index 0000000..2ee6496 --- /dev/null +++ b/scripts/verify_local_restore.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Full logical backup/restore proof for the local Docker PostgreSQL service. +# The temporary database and archive are always removed; the source is read-only. + +source_database="foundation_intelligence" +restore_database="fip_phase13_restore_check_20260729" +archive="/private/tmp/fip-phase13-restore-20260729.dump" + +if [[ "${restore_database}" != fip_phase13_restore_check_* ]]; then + echo "Refusing an unexpected restore database name" >&2 + exit 2 +fi +if [[ "${archive}" != /private/tmp/fip-phase13-restore-*.dump ]]; then + echo "Refusing an unexpected archive path" >&2 + exit 2 +fi + +compose=(docker-compose) +postgres=("${compose[@]}" exec -T postgres) + +cleanup() { + "${postgres[@]}" dropdb --if-exists --force -U foundation_app "${restore_database}" >/dev/null + rm -f -- "${archive}" +} +trap cleanup EXIT + +existing=$("${postgres[@]}" psql -U foundation_app -d postgres -Atc \ + "SELECT COUNT(*) FROM pg_database WHERE datname='${restore_database}'") +if [[ "${existing}" != "0" ]]; then + echo "Temporary restore database already exists" >&2 + exit 3 +fi + +"${postgres[@]}" pg_dump \ + -U foundation_app \ + -d "${source_database}" \ + --format=custom \ + --compress=zstd:3 \ + --no-owner \ + --no-privileges > "${archive}" + +archive_sha256=$(shasum -a 256 "${archive}" | awk '{print $1}') +archive_bytes=$(stat -f '%z' "${archive}") + +"${postgres[@]}" createdb -U foundation_app "${restore_database}" +"${postgres[@]}" pg_restore \ + -U foundation_app \ + -d "${restore_database}" \ + --exit-on-error \ + --no-owner \ + --no-privileges < "${archive}" + +control_sql=" +SELECT json_build_object( + 'schema_version', (SELECT version_num FROM alembic_version), + 'active_dataset', (SELECT dataset_version FROM dataset_versions WHERE is_active), + 'charities', (SELECT COUNT(*) FROM charities WHERE dataset_version=(SELECT dataset_version FROM dataset_versions WHERE is_active)), + 'registry', (SELECT COUNT(*) FROM charity_registry_organizations WHERE dataset_version=(SELECT dataset_version FROM dataset_versions WHERE is_active)), + 'grants', (SELECT COUNT(*) FROM grants WHERE dataset_version=(SELECT dataset_version FROM dataset_versions WHERE is_active)), + 'overview_total_eur_minor', (SELECT COALESCE(SUM(eur_amount_minor),0) FROM grant_overview_facts WHERE dataset_version=(SELECT dataset_version FROM dataset_versions WHERE is_active) AND eur_amount_status NOT IN ('missing','invalid','negative')), + 'materialization', (SELECT COUNT(*) FROM materialization_versions WHERE dataset_version=(SELECT dataset_version FROM dataset_versions WHERE is_active) AND is_active AND status='active') +)::text; +" +source_controls=$("${postgres[@]}" psql -U foundation_app -d "${source_database}" -Atc "${control_sql}") +restore_controls=$("${postgres[@]}" psql -U foundation_app -d "${restore_database}" -Atc "${control_sql}") + +if [[ "${source_controls}" != "${restore_controls}" ]]; then + echo "Restored database controls differ from source" >&2 + exit 4 +fi + +python3 -c 'import json,sys; print(json.dumps({"status":"passed","restore_type":"full_logical_pg_dump_pg_restore","archive_sha256":sys.argv[1],"archive_bytes":int(sys.argv[2]),"controls":json.loads(sys.argv[3]),"temporary_database":sys.argv[4],"temporary_database_removed":True,"archive_removed":True,"aws_actions_performed":False},sort_keys=True))' \ + "${archive_sha256}" "${archive_bytes}" "${restore_controls}" "${restore_database}" diff --git a/scripts/verify_local_rollback.py b/scripts/verify_local_rollback.py new file mode 100644 index 0000000..e4dbf98 --- /dev/null +++ b/scripts/verify_local_rollback.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Demonstrate reversible local dataset activation and restore the original target.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from bff.database import DatabaseSettings +from migration.sqlite_to_postgres import rollback_dataset + + +TABLES = ( + "charities", + "charity_registry_organizations", + "grants", + "grant_beneficiary_countries", + "grant_programme_categories", + "grant_overview_facts", + "grant_source_funder_facts", + "organization_registry_links", +) + + +async def _snapshot(connection: Any) -> dict[str, Any]: + active = str( + ( + await connection.execute( + text( + "SELECT dataset_version FROM dataset_versions " + "WHERE is_active AND status='active'" + ) + ) + ).scalar_one() + ) + counts = {} + for table in TABLES: + counts[table] = int( + ( + await connection.execute( + text(f"SELECT COUNT(*) FROM {table} WHERE dataset_version=:version"), + {"version": active}, + ) + ).scalar_one() + ) + materialization = bool( + ( + await connection.execute( + text( + """ + SELECT EXISTS ( + SELECT 1 FROM materialization_versions + WHERE dataset_version=:version + AND materialization_name='dashboard_analytics' + AND is_active AND status='active' + ) + """ + ), + {"version": active}, + ) + ).scalar_one() + ) + return {"dataset_version": active, "counts": counts, "materialization": materialization} + + +async def main() -> int: + engine = create_async_engine(DatabaseSettings.from_env().sqlalchemy_url(), pool_pre_ping=True) + original: str | None = None + prior: str | None = None + result: dict[str, Any] = { + "status": "failed", + "aws_actions_performed": False, + "bidirectional_replay_performed": False, + } + try: + async with engine.connect() as connection: + active_jobs = int( + ( + await connection.execute( + text( + "SELECT COUNT(*) FROM job_runs " + "WHERE status IN ('queued','running','retrying')" + ) + ) + ).scalar_one() + ) + if active_jobs: + raise RuntimeError("Local rollback proof requires all writers/jobs to be frozen") + before = await _snapshot(connection) + original = str(before["dataset_version"]) + prior_row = ( + await connection.execute( + text( + """ + SELECT dataset_version FROM dataset_versions + WHERE dataset_version<>:current + AND status IN ('approved','rolled_back') + ORDER BY activated_at DESC NULLS LAST, created_at DESC + LIMIT 1 + """ + ), + {"current": original}, + ) + ).scalar_one_or_none() + if prior_row is None: + raise RuntimeError("No approved prior dataset exists for rollback proof") + prior = str(prior_row) + + to_prior = await rollback_dataset(prior) + async with engine.connect() as connection: + prior_snapshot = await _snapshot(connection) + if prior_snapshot["dataset_version"] != prior: + raise RuntimeError("Rollback did not activate the approved prior dataset") + if not prior_snapshot["materialization"]: + raise RuntimeError("Rollback target lacks an active dashboard materialization") + if prior_snapshot["counts"] != before["counts"]: + raise RuntimeError("Prior dataset count controls differ from the original dataset") + + to_original = await rollback_dataset(original) + async with engine.connect() as connection: + after = await _snapshot(connection) + if after != before: + raise RuntimeError("Original local dataset was not restored exactly") + result.update( + { + "status": "passed", + "writers_frozen": True, + "original": original, + "prior": prior, + "rollback": to_prior, + "restore": to_original, + "active_after": after["dataset_version"], + "counts": after["counts"], + "materialization_active": after["materialization"], + } + ) + print(json.dumps(result, sort_keys=True)) + return 0 + finally: + if original is not None: + try: + async with engine.connect() as connection: + active = ( + await connection.execute( + text( + "SELECT dataset_version FROM dataset_versions " + "WHERE is_active AND status='active'" + ) + ) + ).scalar_one_or_none() + if str(active or "") != original: + await rollback_dataset(original) + finally: + await engine.dispose() + else: + await engine.dispose() + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/scripts/verify_transition.py b/scripts/verify_transition.py new file mode 100644 index 0000000..62dae12 --- /dev/null +++ b/scripts/verify_transition.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +"""Compare SQLite source facts with the active PostgreSQL dataset and write goldens.""" + +from __future__ import annotations + +import argparse +import asyncio +from dataclasses import dataclass +from datetime import date +from decimal import Decimal +import hashlib +import json +from pathlib import Path +import sqlite3 +from typing import Any, Iterable, Mapping, Optional + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from bff.database import DatabaseSettings +from scoring.engine import load_score_configuration, score_relevance +from transition.shadow import ComparisonPolicy, compare_payloads + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_SOURCE = PROJECT_ROOT / "src/data/charities.db" +DEFAULT_GOLDEN = PROJECT_ROOT / "config/golden/transition-domain.json" + + +@dataclass(frozen=True) +class QueryPair: + name: str + sqlite: str + postgresql: str + parameters: Mapping[str, Any] + + +QUERIES = ( + QueryPair( + "overview_totals", + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor, + SUM(CASE WHEN country_count>0 THEN 1 ELSE 0 END) AS mapped_grants, + SUM(CASE WHEN programme_category_count>0 THEN 1 ELSE 0 END) + AS classified_grants + FROM grant_overview_facts + """, + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor, + COUNT(*) FILTER (WHERE country_count>0) AS mapped_grants, + COUNT(*) FILTER (WHERE programme_category_count>0) AS classified_grants + FROM grant_overview_facts WHERE dataset_version=:dataset_version + """, + {}, + ), + QueryPair( + "country_totals", + """ + SELECT country.country_code, MIN(country.country_name) AS country_name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.country_count=1 + AND fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS single_country_eur_minor + FROM grant_beneficiary_countries AS country + JOIN grant_overview_facts AS fact ON fact.grant_id=country.grant_id + GROUP BY country.country_code ORDER BY grant_count DESC, country.country_code + """, + """ + SELECT country.country_code, MIN(country.country_name) AS country_name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.country_count=1 + AND fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS single_country_eur_minor + FROM grant_beneficiary_countries AS country + JOIN grant_overview_facts AS fact + ON fact.dataset_version=country.dataset_version AND fact.grant_id=country.grant_id + WHERE fact.dataset_version=:dataset_version + GROUP BY country.country_code ORDER BY grant_count DESC, country.country_code + """, + {}, + ), + QueryPair( + "programme_totals", + """ + SELECT category.programme_area, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS represented_eur_minor + FROM grant_programme_categories AS category + JOIN grant_overview_facts AS fact ON fact.grant_id=category.grant_id + GROUP BY category.programme_area ORDER BY grant_count DESC, category.programme_area + """, + """ + SELECT category.programme_area, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS represented_eur_minor + FROM grant_programme_categories AS category + JOIN grant_overview_facts AS fact + ON fact.dataset_version=category.dataset_version AND fact.grant_id=category.grant_id + WHERE fact.dataset_version=:dataset_version + GROUP BY category.programme_area ORDER BY grant_count DESC, category.programme_area + """, + {}, + ), + QueryPair( + "donor_rankings", + """ + SELECT COALESCE(CAST(grant_row.funding_charity_id AS TEXT), + 'name:' || fact.funding_name_normalized) AS identity, + MIN(fact.funding_name) AS name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grants AS grant_row ON grant_row.grant_id=fact.grant_id + GROUP BY identity ORDER BY total_eur_minor DESC, identity LIMIT 25 + """, + """ + SELECT COALESCE(CAST(grant_row.funding_charity_id AS TEXT), + 'name:' || fact.funding_name_normalized) AS identity, + MIN(fact.funding_name) AS name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grants AS grant_row + ON grant_row.dataset_version=fact.dataset_version AND grant_row.grant_id=fact.grant_id + WHERE fact.dataset_version=:dataset_version + GROUP BY identity ORDER BY total_eur_minor DESC, identity LIMIT 25 + """, + {}, + ), + QueryPair( + "recipient_rankings", + """ + SELECT COALESCE(CAST(grant_row.recipient_charity_id AS TEXT), + 'name:' || fact.recipient_name_normalized) AS identity, + MIN(fact.recipient_name) AS name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grants AS grant_row ON grant_row.grant_id=fact.grant_id + GROUP BY identity ORDER BY total_eur_minor DESC, identity LIMIT 25 + """, + """ + SELECT COALESCE(CAST(grant_row.recipient_charity_id AS TEXT), + 'name:' || fact.recipient_name_normalized) AS identity, + MIN(fact.recipient_name) AS name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grants AS grant_row + ON grant_row.dataset_version=fact.dataset_version AND grant_row.grant_id=fact.grant_id + WHERE fact.dataset_version=:dataset_version + GROUP BY identity ORDER BY total_eur_minor DESC, identity LIMIT 25 + """, + {}, + ), + QueryPair( + "date_filter", + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts WHERE award_date BETWEEN :date_from AND :date_to + """, + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + AND award_date BETWEEN CAST(:date_from AS date) AND CAST(:date_to AS date) + """, + {"date_from": "2020-01-01", "date_to": "2024-12-31"}, + ), + QueryPair( + "country_filter", + """ + SELECT COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.country_count=1 + AND fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grant_beneficiary_countries AS country ON country.grant_id=fact.grant_id + WHERE country.country_code=:country + """, + """ + SELECT COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.country_count=1 + AND fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=fact.dataset_version AND country.grant_id=fact.grant_id + WHERE fact.dataset_version=:dataset_version AND country.country_code=:country + """, + {"country": "GB"}, + ), + QueryPair( + "programme_filter", + """ + SELECT COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grant_programme_categories AS category ON category.grant_id=fact.grant_id + WHERE category.programme_area=:programme + """, + """ + SELECT COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM(CASE WHEN fact.eur_amount_status NOT IN ('missing','invalid','negative') + THEN fact.eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts AS fact + JOIN grant_programme_categories AS category + ON category.dataset_version=fact.dataset_version AND category.grant_id=fact.grant_id + WHERE fact.dataset_version=:dataset_version AND category.programme_area=:programme + """, + {"programme": "tech-enablement"}, + ), + QueryPair( + "donor_filter", + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts WHERE funding_name_normalized LIKE '%' || :term || '%' + """, + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + AND funding_name_normalized LIKE '%' || :term || '%' + """, + {"term": "trust"}, + ), + QueryPair( + "recipient_filter", + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts WHERE recipient_name_normalized LIKE '%' || :term || '%' + """, + """ + SELECT COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + AND recipient_name_normalized LIKE '%' || :term || '%' + """, + {"term": "university"}, + ), + QueryPair( + "monthly_trends", + """ + SELECT SUBSTR(award_date, 1, 7) AS period, COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts WHERE award_date IS NOT NULL + GROUP BY period ORDER BY period + """, + """ + SELECT TO_CHAR(award_date, 'YYYY-MM') AS period, COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND award_date IS NOT NULL + GROUP BY period ORDER BY period + """, + {}, + ), + QueryPair( + "yearly_trends", + """ + SELECT SUBSTR(award_date, 1, 4) AS period, COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts WHERE award_date IS NOT NULL + GROUP BY period ORDER BY period + """, + """ + SELECT TO_CHAR(award_date, 'YYYY') AS period, COUNT(*) AS grant_count, + SUM(CASE WHEN eur_amount_status NOT IN ('missing','invalid','negative') + THEN eur_amount_minor ELSE 0 END) AS total_eur_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND award_date IS NOT NULL + GROUP BY period ORDER BY period + """, + {}, + ), + QueryPair( + "registry_search", + """ + SELECT registry_id, charity_number, registered_name, registration_status, + country_code FROM charity_registry_organizations + WHERE normalized_name LIKE '%' || :term || '%' + ORDER BY normalized_name, registry_id LIMIT 10 + """, + """ + SELECT registry_id, charity_number, registered_name, registration_status, + country_code FROM charity_registry_organizations + WHERE dataset_version=:dataset_version + AND normalized_name LIKE '%' || :term || '%' + ORDER BY normalized_name, registry_id LIMIT 10 + """, + {"term": "trust"}, + ), + QueryPair( + "organisation_details", + """ + SELECT charity_id, name, organization_type, primary_source, + headquarters_country, transaction_coverage + FROM charities ORDER BY charity_id LIMIT 10 + """, + """ + SELECT charity_id, name, organization_type, primary_source, + headquarters_country, transaction_coverage + FROM charities WHERE dataset_version=:dataset_version + ORDER BY charity_id LIMIT 10 + """, + {}, + ), + QueryPair( + "grant_list", + """ + SELECT grant_id, funding_name, recipient_name, currency, + CAST(ROUND(amount * 100) AS INTEGER) AS amount_minor + FROM grants ORDER BY grant_id LIMIT 20 + """, + """ + SELECT grant_id, funding_name, recipient_name, currency, + CAST(ROUND(amount * 100) AS BIGINT) AS amount_minor + FROM grants WHERE dataset_version=:dataset_version + ORDER BY grant_id LIMIT 20 + """, + {}, + ), + QueryPair( + "sankey", + """ + SELECT funding_name, recipient_name, currency, COUNT(*) AS grant_count, + CAST(ROUND(SUM(amount) * 100) AS INTEGER) AS amount_minor + FROM grants WHERE funding_name IS NOT NULL + GROUP BY funding_name, recipient_name, currency + ORDER BY grant_count DESC, funding_name, recipient_name LIMIT 25 + """, + """ + SELECT funding_name, recipient_name, currency, COUNT(*) AS grant_count, + CAST(ROUND(SUM(amount) * 100) AS BIGINT) AS amount_minor + FROM grants WHERE dataset_version=:dataset_version AND funding_name IS NOT NULL + GROUP BY funding_name, recipient_name, currency + ORDER BY grant_count DESC, funding_name, recipient_name LIMIT 25 + """, + {}, + ), + QueryPair( + "map_relationships", + """ + SELECT fact.origin_country_code, + MIN(fact.origin_country_name) AS origin_country_name, + country.country_code AS destination_country_code, + MIN(country.country_name) AS destination_country_name, + COUNT(DISTINCT fact.grant_id) AS grant_count + FROM grant_overview_facts AS fact + JOIN grant_beneficiary_countries AS country ON country.grant_id=fact.grant_id + WHERE fact.origin_country_code IS NOT NULL + AND fact.origin_country_code<>country.country_code + GROUP BY fact.origin_country_code, country.country_code + ORDER BY grant_count DESC, fact.origin_country_code, country.country_code LIMIT 100 + """, + """ + SELECT fact.origin_country_code, + MIN(fact.origin_country_name) AS origin_country_name, + country.country_code AS destination_country_code, + MIN(country.country_name) AS destination_country_name, + COUNT(DISTINCT fact.grant_id) AS grant_count + FROM grant_overview_facts AS fact + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=fact.dataset_version AND country.grant_id=fact.grant_id + WHERE fact.dataset_version=:dataset_version + AND fact.origin_country_code IS NOT NULL + AND fact.origin_country_code<>country.country_code + GROUP BY fact.origin_country_code, country.country_code + ORDER BY grant_count DESC, fact.origin_country_code, country.country_code LIMIT 100 + """, + {}, + ), + QueryPair( + "currency_statuses", + """ + SELECT COALESCE(currency, 'UNKNOWN') AS currency, + COALESCE(conversion_status, 'missing') AS conversion_status, + COUNT(*) AS grant_count + FROM grants GROUP BY currency, conversion_status + ORDER BY currency, conversion_status + """, + """ + SELECT COALESCE(currency, 'UNKNOWN') AS currency, + COALESCE(conversion_status, 'missing') AS conversion_status, + COUNT(*) AS grant_count + FROM grants WHERE dataset_version=:dataset_version + GROUP BY currency, conversion_status ORDER BY currency, conversion_status + """, + {}, + ), +) + + +def _normalize(value: Any) -> Any: + if isinstance(value, Decimal): + if value == value.to_integral_value(): + return int(value) + return format(value, "f") + if hasattr(value, "isoformat"): + return value.isoformat() + if isinstance(value, Mapping): + return {str(key): _normalize(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [_normalize(item) for item in value] + return value + + +def _sqlite_rows(connection: sqlite3.Connection, query: QueryPair) -> list[dict[str, Any]]: + cursor = connection.execute(query.sqlite, dict(query.parameters)) + columns = [str(item[0]) for item in cursor.description] + return [ + {column: _normalize(value) for column, value in zip(columns, row)} + for row in cursor.fetchall() + ] + + +async def _postgres_rows(connection: Any, query: QueryPair, dataset_version: str) -> list[dict[str, Any]]: + parameters = {**query.parameters, "dataset_version": dataset_version} + for key in ("date_from", "date_to"): + if isinstance(parameters.get(key), str): + parameters[key] = date.fromisoformat(parameters[key]) + result = await connection.execute(text(query.postgresql), parameters) + return [_normalize(dict(row)) for row in result.mappings()] + + +def _source_checksum(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +async def verify(source_path: Path) -> tuple[dict[str, Any], dict[str, Any], str]: + source_uri = f"file:{source_path.resolve()}?mode=ro&immutable=1" + sqlite_connection = sqlite3.connect(source_uri, uri=True) + engine = create_async_engine(DatabaseSettings.from_env().sqlalchemy_url(), pool_pre_ping=True) + source: dict[str, Any] = {} + target: dict[str, Any] = {} + try: + async with engine.connect() as postgres_connection: + dataset_version = str( + ( + await postgres_connection.execute( + text( + "SELECT dataset_version FROM dataset_versions " + "WHERE is_active AND status='active'" + ) + ) + ).scalar_one() + ) + for query in QUERIES: + source[query.name] = _sqlite_rows(sqlite_connection, query) + target[query.name] = await _postgres_rows( + postgres_connection, query, dataset_version + ) + score_configuration = load_score_configuration() + score = score_relevance( + { + "programme_areas_source": ["Education", "Health"], + "programme_areas_inferred": [], + "geographic_focus_source": ["United Kingdom", "Ghana"], + "geographic_focus_inferred": [], + "annual_expenditure": 2_000_000, + "organization_type": "foundation", + }, + score_configuration.example_target_profile, + {"average_amount": 60_000, "currency": "EUR", "grant_count": 4}, + score_configuration, + ) + score_fixture = { + "score": score["score"], + "confidence": score["confidence"], + "data_completeness": score["data_completeness"], + "score_version": score["score_version"], + "configuration_status": score["configuration_status"], + "components": { + name: { + key: component[key] + for key in ( + "score", "weight", "weighted_score", "confidence", "available" + ) + } + for name, component in sorted(score["components"].items()) + }, + } + source["score_components"] = score_fixture + target["score_components"] = score_fixture + finally: + sqlite_connection.close() + await engine.dispose() + return source, target, dataset_version + + +def _build_golden(source: dict[str, Any], source_path: Path) -> dict[str, Any]: + return { + "fixture_version": "1", + "source_sha256": _source_checksum(source_path), + "semantic_contract": source, + "non_database_journeys": { + "news": "live_network_calls_disabled_in_local_acceptance", + "pipeline_status": "durable_postgresql_job_status_contract", + "manual_refresh_permissions": "operator_role_and_idempotency_required", + }, + } + + +async def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--source", type=Path, default=DEFAULT_SOURCE) + parser.add_argument("--golden", type=Path, default=DEFAULT_GOLDEN) + parser.add_argument("--write-golden", action="store_true") + arguments = parser.parse_args() + source, target, dataset_version = await verify(arguments.source) + result = compare_payloads( + source, + target, + journey="local_sqlite_postgresql_semantic_projection", + request_id="phase13-local", + policy=ComparisonPolicy(maximum_differences=100), + ) + golden = _build_golden(source, arguments.source) + if arguments.write_golden: + arguments.golden.parent.mkdir(parents=True, exist_ok=True) + arguments.golden.write_text( + json.dumps(golden, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + elif arguments.golden.exists(): + expected = json.loads(arguments.golden.read_text(encoding="utf-8")) + golden_result = compare_payloads( + expected, + golden, + journey="transition_golden_fixture", + request_id="phase13-local", + ) + if golden_result.status != "match": + print(json.dumps({"status": "failed", "golden": _normalize(golden_result.__dict__)})) + return 1 + report = { + "status": "passed" if result.status == "match" else "failed", + "dataset_version": dataset_version, + "source_sha256": golden["source_sha256"], + "semantic_projections": len(QUERIES), + "difference_count": result.difference_count, + "differences": [_normalize(item.__dict__) for item in result.differences], + "golden_written": arguments.write_golden, + "aws_actions_performed": False, + } + print(json.dumps(report, sort_keys=True)) + return 0 if result.status == "match" else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/src/bff/admin.py b/src/bff/admin.py index 7db3108..0d92f6e 100644 --- a/src/bff/admin.py +++ b/src/bff/admin.py @@ -6,9 +6,9 @@ import time from fastapi import APIRouter, Depends, HTTPException, status, BackgroundTasks from typing import Optional, List -from bff.auth import get_current_user_token +from bff.security import Role, require_roles from bff.schemas import PipelineStatus, PipelineTrigger -from bff.utils.logging import logger +from bff.utils.logging import logger, redact_text # Set up directories relative to project root BFF_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -23,7 +23,7 @@ router = APIRouter( prefix="/api/admin", tags=["Administrative & Monitoring Data"], - dependencies=[Depends(get_current_user_token)] + dependencies=[Depends(require_roles(Role.OPERATOR, action="administration.access"))], ) def read_status(): @@ -261,7 +261,11 @@ async def get_pipeline_status(): return read_status() -@router.post("/pipeline/trigger", response_model=PipelineStatus) +@router.post( + "/pipeline/trigger", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="pipeline.trigger", idempotent=True))], +) async def trigger_pipeline( payload: PipelineTrigger, background_tasks: BackgroundTasks @@ -304,7 +308,10 @@ async def trigger_pipeline( -@router.get("/pipeline/logs") +@router.get( + "/pipeline/logs", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="pipeline.logs.read"))], +) async def get_pipeline_logs(): """Returns the tail output (last 100 lines) of the execution log file.""" if not os.path.exists(LOG_FILE): @@ -314,9 +321,9 @@ async def get_pipeline_logs(): with open(LOG_FILE, "r", encoding="utf-8") as f: lines = f.readlines() tail_lines = lines[-100:] - return {"logs": "".join(tail_lines)} - except Exception as e: + return {"logs": redact_text("".join(tail_lines))} + except Exception: raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Failed to read execution logs: {e}" + detail="Failed to read execution logs." ) diff --git a/src/bff/audit.py b/src/bff/audit.py new file mode 100644 index 0000000..eed6b41 --- /dev/null +++ b/src/bff/audit.py @@ -0,0 +1,66 @@ +"""Structured security audit events with a replaceable durable sink.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +import json +from typing import List, Optional, Protocol + +from bff.utils.logging import logger + + +@dataclass(frozen=True) +class AuditEvent: + actor_id: str + actor_role: str + action: str + target: str + reason: str + timestamp: str + request_id: str + result: str + http_status: int + error_class: Optional[str] + dataset_version: Optional[str] + + +class AuditSink(Protocol): + def record(self, event: AuditEvent) -> object: ... + + +class StructuredLogAuditSink: + def record(self, event: AuditEvent) -> None: + logger.info("security_audit %s", json.dumps(asdict(event), sort_keys=True, separators=(",", ":"))) + + +class MemoryAuditSink: + """Deterministic test sink; never selected by the application runtime.""" + + def __init__(self) -> None: + self.events: List[AuditEvent] = [] + + def record(self, event: AuditEvent) -> None: + self.events.append(event) + + def clear(self) -> None: + self.events.clear() + + +def event_from_request(request, status_code: int, error_class: Optional[str] = None) -> AuditEvent: + principal = getattr(request.state, "principal", None) + result = "success" if status_code < 400 else "denied" if status_code in {401, 403, 429} else "failed" + reason = request.headers.get("x-action-reason", "not_provided").strip()[:500] or "not_provided" + return AuditEvent( + actor_id=principal.actor_id if principal else "anonymous", + actor_role=principal.primary_role if principal else "none", + action=getattr(request.state, "audit_action", "unclassified"), + target=getattr(request.state, "audit_target", request.url.path), + reason=reason, + timestamp=datetime.now(timezone.utc).isoformat(), + request_id=getattr(request.state, "request_id", "unavailable"), + result=result, + http_status=status_code, + error_class=error_class, + dataset_version=getattr(request.state, "dataset_version", None), + ) diff --git a/src/bff/auth.py b/src/bff/auth.py index 40d2c12..eb017e2 100644 --- a/src/bff/auth.py +++ b/src/bff/auth.py @@ -1,105 +1,77 @@ -from datetime import datetime, timedelta, timezone -from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, Request, Response, status -from fastapi.security import APIKeyCookie -from jose import JWTError, jwt -from bff.config import ( - JWT_SECRET_KEY, - JWT_ALGORITHM, - ACCESS_TOKEN_EXPIRE_MINUTES, - BFF_ADMIN_USER, - BFF_ADMIN_PASSWORD -) -from bff.schemas import UserLogin, Token -from bff.utils.logging import logger +"""Authentication endpoints. -router = APIRouter(prefix="/api/auth", tags=["Authentication"]) +Production identities are validated from OIDC bearer tokens. The password endpoint +exists only when the local/test development mode is explicitly enabled. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Request, Response + +from bff.config import SECURITY_SETTINGS +from bff.schemas import UserLogin +from bff.security import ( + Principal, + Role, + create_development_access_token, + enforce_login_rate_limit, + require_roles, + validate_development_credentials, +) -# Setup cookie extractor for OpenAPI documentation auto-discovery -cookie_sec = APIKeyCookie(name="session_id", auto_error=False) -def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str: - """Helper to generate signed JWT token.""" - to_encode = data.copy() - if expires_delta: - expire = datetime.now(timezone.utc) + expires_delta - else: - expire = datetime.now(timezone.utc) + timedelta(minutes=15) - to_encode.update({"exp": expire}) - encoded_jwt = jwt.encode(to_encode, JWT_SECRET_KEY, algorithm=JWT_ALGORITHM) - return encoded_jwt +router = APIRouter(prefix="/api/auth", tags=["Authentication"]) -def get_current_user_token(request: Request) -> str: - """ - Dependency that extracts the session cookie from the frontend, - decodes it, and returns the token string if valid. - """ - token = request.cookies.get("session_id") - if not token: - # Fallback to Authorization header if testing in Swagger / direct API requests - auth_header = request.headers.get("Authorization") - if auth_header and auth_header.startswith("Bearer "): - token = auth_header.split(" ")[1] - - if not token: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Not authenticated. Missing session cookie.", - ) - - try: - # Verify and decode JWT - payload = jwt.decode(token, JWT_SECRET_KEY, algorithms=[JWT_ALGORITHM]) - username: str = payload.get("sub") - if username is None: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Invalid session token content." - ) - return token - except JWTError as e: - logger.warning(f"Failed JWT verification: {e}") - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Session expired or invalid token." - ) @router.post("/login") -async def login(response: Response, credentials: UserLogin): - """ - Verifies user credentials, generates a JWT, and sets it - as a secure HTTP-Only cookie 'session_id'. - """ - if credentials.username != BFF_ADMIN_USER or credentials.password != BFF_ADMIN_PASSWORD: - raise HTTPException( - status_code=status.HTTP_401_UNAUTHORIZED, - detail="Incorrect username or password" - ) - - # Generate token - access_token_expires = timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES) - access_token = create_access_token( - data={"sub": credentials.username}, - expires_delta=access_token_expires +async def login( + request: Request, + response: Response, + credentials: UserLogin, + _: None = Depends(enforce_login_rate_limit), +): + """Issue a local-only development session; absent from the production auth path.""" + request.state.audit_action = "auth.development_login" + request.state.audit_target = "/api/auth/login" + settings = validate_development_credentials( + request, + credentials.username, + credentials.password, + ) + token = create_development_access_token(credentials.username, settings=settings) + request.state.principal = Principal( + actor_id=credentials.username, + roles=frozenset({Role.ADMINISTRATOR}), + claims={"sub": credentials.username, "roles": [Role.ADMINISTRATOR.value]}, ) - - # Set secure HttpOnly cookie response.set_cookie( - key="session_id", - value=access_token, + key=settings.session_cookie_name, + value=token, httponly=True, - max_age=ACCESS_TOKEN_EXPIRE_MINUTES * 60, - expires=ACCESS_TOKEN_EXPIRE_MINUTES * 60, - samesite="lax", - secure=False # Set to True in production with HTTPS + max_age=settings.access_token_expire_minutes * 60, + samesite="strict", + secure=settings.session_cookie_secure, + path="/api", ) - - logger.info(f"User '{credentials.username}' logged in successfully.") return {"message": "Successfully logged in"} + @router.post("/logout") -async def logout(response: Response, token: str = Depends(get_current_user_token)): - """Clears the session cookie, logging the user out.""" - response.delete_cookie(key="session_id", samesite="lax") - logger.info("User logged out successfully.") +async def logout( + request: Request, + response: Response, + principal: Principal = Depends( + require_roles(Role.VIEWER, action="auth.logout") + ), +): + """Clear the development session cookie; OIDC logout is owned by the IdP.""" + del principal + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + response.delete_cookie( + key=settings.session_cookie_name, + samesite="strict", + secure=settings.session_cookie_secure, + path="/api", + ) return {"message": "Successfully logged out"} +__all__ = ["router"] diff --git a/src/bff/charity.py b/src/bff/charity.py index e5fe035..66ae0ee 100644 --- a/src/bff/charity.py +++ b/src/bff/charity.py @@ -7,7 +7,7 @@ from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Query, status from starlette.concurrency import run_in_threadpool from typing import Any, Dict, List, Optional -from bff.auth import get_current_user_token +from bff.security import Role, require_roles from bff.config import DB_PATH from bff.schemas import ( CharityBase, @@ -36,7 +36,7 @@ router = APIRouter( prefix="/api/charities", tags=["Organization and Grant Data"], - dependencies=[Depends(get_current_user_token)] + dependencies=[Depends(require_roles(Role.VIEWER, action="charity.read"))], ) @router.get("", response_model=List[CharityBase]) @@ -681,7 +681,11 @@ async def _start_confirmed_profile_enrichment( } -@router.post("/grants/funders/enrich", response_model=PipelineStatus) +@router.post( + "/grants/funders/enrich", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="funder.enrich", idempotent=True))], +) async def enrich_source_funders( payload: SourceFunderEnrichmentRequest, ): @@ -692,7 +696,11 @@ async def enrich_source_funders( ) -@router.post("/directory/organizations/enrich", response_model=PipelineStatus) +@router.post( + "/directory/organizations/enrich", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="registry.enrich", idempotent=True))], +) async def enrich_registry_organization( payload: SourceFunderEnrichmentRequest, ): @@ -708,7 +716,10 @@ async def enrich_registry_organization( ) -@router.post("/grants/funders/{source_funder_key}/reset-to-observed") +@router.post( + "/grants/funders/{source_funder_key}/reset-to-observed", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="funder.reset", idempotent=True))], +) async def reset_source_funder_to_observed( source_funder_key: str, repo: CharityRepository = Depends(get_charity_repository), @@ -728,7 +739,10 @@ async def reset_source_funder_to_observed( return {"status": "observed_only", **reset} -@router.post("/grants/funders/{source_funder_key}/relink") +@router.post( + "/grants/funders/{source_funder_key}/relink", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="funder.relink", idempotent=True))], +) async def relink_source_funder_profile( source_funder_key: str, payload: SourceFunderRelinkRequest, @@ -749,7 +763,10 @@ async def relink_source_funder_profile( return {"status": "linked", **relinked} -@router.post("/grants/funders/{source_funder_key}/profile-cache") +@router.post( + "/grants/funders/{source_funder_key}/profile-cache", + dependencies=[Depends(require_roles(Role.OPERATOR, action="profile_cache.refresh", idempotent=True))], +) async def queue_source_funder_profile_cache( source_funder_key: str, background_tasks: BackgroundTasks, @@ -968,7 +985,25 @@ async def get_charity_sankey( return await repo.get_sankey_data(reg_charity_number, currency=currency, limit=limit) -@router.post("/{reg_charity_number}/score", response_model=ScoreResponse) +@router.get("/{reg_charity_number}/score", response_model=ScoreResponse) +async def get_default_charity_relevance_score( + reg_charity_number: int, + repo: CharityRepository = Depends(get_charity_repository), +): + charity = await repo.get_by_id(reg_charity_number) + if not charity: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Organization {reg_charity_number} not found.", + ) + return await repo.get_score(reg_charity_number, target_profile=None) + + +@router.post( + "/{reg_charity_number}/score", + response_model=ScoreResponse, + dependencies=[Depends(require_roles(Role.ANALYST, action="score.calculate"))], +) async def score_charity_relevance( reg_charity_number: int, request: ScoreRequest, diff --git a/src/bff/config.py b/src/bff/config.py index 1690d43..ace280d 100644 --- a/src/bff/config.py +++ b/src/bff/config.py @@ -1,44 +1,267 @@ +"""Application configuration with fail-closed production security defaults.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json import os +from pathlib import Path +from typing import Mapping, Optional, Tuple +from urllib.parse import urlparse + + +BFF_DIR = Path(__file__).resolve().parent +SRC_DIR = BFF_DIR.parent +PROJECT_ROOT = SRC_DIR.parent -# Load .env file manually if it exists in the project root to ensure credentials are populated -BFF_DIR = os.path.dirname(os.path.abspath(__file__)) -SRC_DIR = os.path.dirname(BFF_DIR) -PROJECT_ROOT = os.path.dirname(SRC_DIR) -env_path = os.path.join(PROJECT_ROOT, ".env") -if os.path.exists(env_path): - with open(env_path, "r", encoding="utf-8") as f: - for line in f: - line = line.strip() - if not line or line.startswith("#"): + +def _load_local_env() -> None: + """Load a developer .env without ever using it in staging/production.""" + app_environment = os.environ.get("APP_ENV", "development").strip().lower() + if app_environment not in {"development", "test"}: + return + env_path = PROJECT_ROOT / ".env" + if not env_path.exists(): + return + with env_path.open("r", encoding="utf-8") as handle: + for raw_line in handle: + line = raw_line.strip() + if not line or line.startswith("#") or "=" not in line: continue - if "=" in line: - key, val = line.split("=", 1) - key = key.strip() - val = val.strip().strip("'\"") - if key and not os.environ.get(key): - os.environ[key] = val - -# JWT Configuration -JWT_SECRET_KEY = os.environ.get("JWT_SECRET_KEY", "bff_super_secret_signing_key_for_session_tokens_2026") -JWT_ALGORITHM = os.environ.get("JWT_ALGORITHM", "HS256") -ACCESS_TOKEN_EXPIRE_MINUTES = int(os.environ.get("ACCESS_TOKEN_EXPIRE_MINUTES", "30")) - -# Downstream Service Configuration -CORE_API_URL = os.environ.get("CORE_API_URL", "http://localhost:8080") - -# Data File Configuration -# By default, look for the results relative to the project root -BFF_DIR = os.path.dirname(os.path.abspath(__file__)) -SRC_DIR = os.path.dirname(BFF_DIR) -DEFAULT_DATA_PATH = os.path.join(SRC_DIR, "data", "raw", "register_of_charities_results.json") + key, value = line.split("=", 1) + key = key.strip() + if key and key not in os.environ: + os.environ[key] = value.strip().strip("'\"") + + +_load_local_env() + + +def _as_bool(value: Optional[str], default: bool = False) -> bool: + if value is None: + return default + normalized = value.strip().lower() + if normalized in {"1", "true", "yes", "on"}: + return True + if normalized in {"0", "false", "no", "off"}: + return False + raise ValueError(f"Expected a boolean value, received {value!r}") + + +def _as_int(value: Optional[str], default: int, *, minimum: int = 1) -> int: + parsed = int(value) if value is not None else default + if parsed < minimum: + raise ValueError(f"Expected an integer greater than or equal to {minimum}") + return parsed + + +def _as_csv(value: Optional[str], default: str = "") -> Tuple[str, ...]: + raw = value if value is not None else default + return tuple(item.strip() for item in raw.split(",") if item.strip()) + + +class SecurityConfigurationError(RuntimeError): + """Raised when a runtime would start with an unsafe security posture.""" + + +@dataclass(frozen=True) +class SecuritySettings: + app_env: str + data_runtime_mode: str + auth_mode: str + oidc_issuer: Optional[str] + oidc_audience: Optional[str] + oidc_jwks_url: Optional[str] + oidc_jwks_json: Optional[str] + oidc_algorithms: Tuple[str, ...] + oidc_role_claim: str + oidc_jwks_cache_seconds: int + dev_auth_enabled: bool + dev_auth_username: Optional[str] + dev_auth_password: Optional[str] + dev_auth_secret: Optional[str] + dev_auth_allowed_hosts: Tuple[str, ...] + access_token_expire_minutes: int + session_cookie_name: str + session_cookie_secure: bool + cors_origins: Tuple[str, ...] + rate_limit_requests: int + rate_limit_window_seconds: int + max_request_body_bytes: int + request_timeout_seconds: int + core_proxy_enabled: bool + core_api_url: str + core_api_allowed_hosts: Tuple[str, ...] + core_proxy_allowed_paths: Tuple[str, ...] + core_proxy_allowed_methods: Tuple[str, ...] + core_proxy_forward_headers: Tuple[str, ...] + core_proxy_response_headers: Tuple[str, ...] + core_api_bearer_token: Optional[str] + + @classmethod + def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "SecuritySettings": + env = os.environ if environ is None else environ + app_env = env.get("APP_ENV", "development").strip().lower() + return cls( + app_env=app_env, + data_runtime_mode=env.get( + "DATA_RUNTIME_MODE", "sqlite_migration_source" + ).strip().lower(), + auth_mode=env.get("AUTH_MODE", "disabled").strip().lower(), + oidc_issuer=env.get("OIDC_ISSUER"), + oidc_audience=env.get("OIDC_AUDIENCE"), + oidc_jwks_url=env.get("OIDC_JWKS_URL"), + oidc_jwks_json=env.get("OIDC_JWKS_JSON"), + oidc_algorithms=_as_csv(env.get("OIDC_ALGORITHMS"), "RS256"), + oidc_role_claim=env.get("OIDC_ROLE_CLAIM", "roles").strip(), + oidc_jwks_cache_seconds=_as_int(env.get("OIDC_JWKS_CACHE_SECONDS"), 300), + dev_auth_enabled=_as_bool(env.get("DEV_AUTH_ENABLED")), + dev_auth_username=env.get("DEV_AUTH_USERNAME"), + dev_auth_password=env.get("DEV_AUTH_PASSWORD"), + dev_auth_secret=env.get("DEV_AUTH_SECRET"), + dev_auth_allowed_hosts=_as_csv( + env.get("DEV_AUTH_ALLOWED_HOSTS"), "127.0.0.1,::1,localhost" + ), + access_token_expire_minutes=_as_int(env.get("ACCESS_TOKEN_EXPIRE_MINUTES"), 30), + session_cookie_name=env.get("SESSION_COOKIE_NAME", "session_id").strip(), + session_cookie_secure=_as_bool( + env.get("SESSION_COOKIE_SECURE"), app_env in {"staging", "production"} + ), + cors_origins=_as_csv( + env.get("CORS_ORIGINS"), + "http://localhost:5173,http://127.0.0.1:5173", + ), + rate_limit_requests=_as_int(env.get("RATE_LIMIT_REQUESTS"), 120), + rate_limit_window_seconds=_as_int(env.get("RATE_LIMIT_WINDOW_SECONDS"), 60), + max_request_body_bytes=_as_int(env.get("MAX_REQUEST_BODY_BYTES"), 1_048_576), + request_timeout_seconds=_as_int(env.get("REQUEST_TIMEOUT_SECONDS"), 30), + core_proxy_enabled=_as_bool(env.get("CORE_PROXY_ENABLED")), + core_api_url=env.get("CORE_API_URL", "http://127.0.0.1:8080").strip(), + core_api_allowed_hosts=tuple( + host.lower() for host in _as_csv(env.get("CORE_API_ALLOWED_HOSTS")) + ), + core_proxy_allowed_paths=_as_csv(env.get("CORE_PROXY_ALLOWED_PATHS")), + core_proxy_allowed_methods=tuple( + method.upper() + for method in _as_csv(env.get("CORE_PROXY_ALLOWED_METHODS"), "GET") + ), + core_proxy_forward_headers=tuple( + header.lower() + for header in _as_csv( + env.get("CORE_PROXY_FORWARD_HEADERS"), + "accept,content-type,x-request-id", + ) + ), + core_proxy_response_headers=tuple( + header.lower() + for header in _as_csv( + env.get("CORE_PROXY_RESPONSE_HEADERS"), + "content-type,cache-control,etag,last-modified", + ) + ), + core_api_bearer_token=env.get("CORE_API_BEARER_TOKEN"), + ) + + +def validate_security_settings(settings: SecuritySettings) -> None: + errors = [] + production = settings.app_env in {"staging", "production"} + + if settings.auth_mode not in {"disabled", "development", "oidc", "public_readonly"}: + errors.append( + "AUTH_MODE must be disabled, development, oidc or public_readonly" + ) + if production and settings.auth_mode != "oidc": + errors.append("staging/production requires AUTH_MODE=oidc") + if production and settings.dev_auth_enabled: + errors.append("development authentication must be disabled outside local/test environments") + + if settings.auth_mode == "development": + if settings.app_env not in {"development", "test"}: + errors.append("development authentication is local/test only") + if not settings.dev_auth_enabled: + errors.append("AUTH_MODE=development requires DEV_AUTH_ENABLED=true") + if not all( + (settings.dev_auth_username, settings.dev_auth_password, settings.dev_auth_secret) + ): + errors.append("development authentication credentials must be configured explicitly") + if settings.dev_auth_secret and len(settings.dev_auth_secret) < 32: + errors.append("DEV_AUTH_SECRET must contain at least 32 characters") + + if settings.auth_mode == "public_readonly": + if settings.app_env != "demo": + errors.append("public_readonly authentication requires APP_ENV=demo") + if settings.data_runtime_mode != "postgresql": + errors.append( + "public_readonly authentication requires DATA_RUNTIME_MODE=postgresql" + ) + if settings.dev_auth_enabled: + errors.append("development authentication must be disabled in public_readonly mode") + if settings.core_proxy_enabled: + errors.append("the core proxy must be disabled in public_readonly mode") + + if settings.auth_mode == "oidc": + if not settings.oidc_issuer or not settings.oidc_audience: + errors.append("OIDC_ISSUER and OIDC_AUDIENCE are required") + if not settings.oidc_jwks_url and not settings.oidc_jwks_json: + errors.append("OIDC_JWKS_URL or OIDC_JWKS_JSON is required") + if not settings.oidc_algorithms or any( + algorithm not in {"RS256", "RS384", "RS512", "ES256", "ES384", "ES512"} + for algorithm in settings.oidc_algorithms + ): + errors.append("OIDC algorithms must be an explicit asymmetric allowlist") + if production: + for name, value in ( + ("OIDC_ISSUER", settings.oidc_issuer), + ("OIDC_JWKS_URL", settings.oidc_jwks_url), + ): + if value and urlparse(value).scheme != "https": + errors.append(f"{name} must use HTTPS outside development") + + if production and not settings.session_cookie_secure: + errors.append("SESSION_COOKIE_SECURE must be true outside development") + if production and not settings.cors_origins: + errors.append("CORS_ORIGINS must be explicit outside development") + if any(origin == "*" for origin in settings.cors_origins): + errors.append("wildcard CORS origins are not permitted") + if production and any(urlparse(origin).scheme != "https" for origin in settings.cors_origins): + errors.append("production CORS origins must use HTTPS") + + if settings.core_proxy_enabled: + parsed_core_url = urlparse(settings.core_api_url) + if parsed_core_url.scheme not in {"http", "https"} or not parsed_core_url.hostname: + errors.append("CORE_API_URL must be an absolute HTTP(S) URL") + elif parsed_core_url.hostname.lower() not in settings.core_api_allowed_hosts: + errors.append("CORE_API_URL host must be present in CORE_API_ALLOWED_HOSTS") + if production and parsed_core_url.scheme != "https": + errors.append("CORE_API_URL must use HTTPS outside development") + if not settings.core_proxy_allowed_paths: + errors.append("enabled proxy requires CORE_PROXY_ALLOWED_PATHS") + if not settings.core_proxy_allowed_methods: + errors.append("enabled proxy requires CORE_PROXY_ALLOWED_METHODS") + if "authorization" in settings.core_proxy_forward_headers or "cookie" in settings.core_proxy_forward_headers: + errors.append("browser Authorization and Cookie headers cannot be forwarded") + + if settings.oidc_jwks_json: + try: + parsed_jwks = json.loads(settings.oidc_jwks_json) + if not isinstance(parsed_jwks, dict) or not isinstance(parsed_jwks.get("keys"), list): + errors.append("OIDC_JWKS_JSON must contain a keys array") + except json.JSONDecodeError: + errors.append("OIDC_JWKS_JSON must be valid JSON") + + if errors: + raise SecurityConfigurationError("; ".join(errors)) + + +SECURITY_SETTINGS = SecuritySettings.from_env() + +# Data configuration remains local until the PostgreSQL conversion phases. +DEFAULT_DATA_PATH = str(SRC_DIR / "data" / "raw" / "register_of_charities_results.json") DATA_PATH = os.environ.get("DATA_PATH", DEFAULT_DATA_PATH) -DEFAULT_DB_PATH = os.path.join(SRC_DIR, "data", "charities.db") +DEFAULT_DB_PATH = str(SRC_DIR / "data" / "charities.db") DB_PATH = os.environ.get("DB_PATH", DEFAULT_DB_PATH) -# Basic Mock Authentication credentials -BFF_ADMIN_USER = os.environ.get("BFF_ADMIN_USER", "admin") -BFF_ADMIN_PASSWORD = os.environ.get("BFF_ADMIN_PASSWORD", "password") - -# Claude API Configuration (used by the Foundation News summary endpoint) +# Optional news summarisation. Live use remains approval-gated by the remediation contract. ANTHROPIC_AUTH_TOKEN = os.environ.get("ANTHROPIC_AUTH_TOKEN") ANTHROPIC_BASE_URL = os.environ.get("ANTHROPIC_BASE_URL") diff --git a/src/bff/database.py b/src/bff/database.py new file mode 100644 index 0000000..03788e1 --- /dev/null +++ b/src/bff/database.py @@ -0,0 +1,343 @@ +"""Bounded async PostgreSQL engine and readiness contract.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +import os +from pathlib import Path +from typing import Mapping, Optional + +from sqlalchemy import URL, text +from sqlalchemy.engine import make_url +from sqlalchemy.ext.asyncio import AsyncEngine, create_async_engine +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker +from sqlalchemy.pool import NullPool + +from bff.utils.logging import logger + + +class DatabaseConfigurationError(RuntimeError): + """Raised when PostgreSQL configuration is incomplete or unsafe.""" + + +def _positive_int(value: Optional[str], default: int) -> int: + parsed = int(value) if value is not None else default + if parsed <= 0: + raise DatabaseConfigurationError("Database numeric settings must be positive") + return parsed + + +def _ssl_mode(environment: Mapping[str, str]) -> str: + """Resolve one unambiguous asyncpg SSL mode from env or DATABASE_URL.""" + explicit = environment.get("DATABASE_SSL_MODE") + url = environment.get("DATABASE_URL") + query_modes: set[str] = set() + if url: + try: + query = make_url(url).query + except Exception: + query = {} + for key in ("ssl", "sslmode"): + value = query.get(key) + if value is None: + continue + values = value if isinstance(value, tuple) else (value,) + query_modes.update(str(item).strip().lower() for item in values) + if len(query_modes) > 1: + raise DatabaseConfigurationError( + "DATABASE_URL contains conflicting PostgreSQL SSL modes" + ) + query_mode = next(iter(query_modes), None) + if explicit is not None: + mode = explicit.strip().lower() + if query_mode is not None and query_mode != mode: + raise DatabaseConfigurationError( + "DATABASE_SSL_MODE conflicts with DATABASE_URL SSL configuration" + ) + else: + mode = query_mode or "disable" + if mode not in {"disable", "require"}: + raise DatabaseConfigurationError( + "DATABASE_SSL_MODE must be disable or require" + ) + return mode + + +@dataclass(frozen=True) +class DatabaseSettings: + url: Optional[str] + host: Optional[str] + port: int + name: Optional[str] + user: Optional[str] + password: Optional[str] + password_file: Optional[str] + ssl_mode: str + pool_size: int + max_overflow: int + pool_timeout_seconds: int + connect_timeout_seconds: int + statement_timeout_ms: int + + @classmethod + def from_env(cls, environ: Optional[Mapping[str, str]] = None) -> "DatabaseSettings": + env = os.environ if environ is None else environ + return cls( + url=env.get("DATABASE_URL"), + host=env.get("DATABASE_HOST"), + port=_positive_int(env.get("DATABASE_PORT"), 5432), + name=env.get("DATABASE_NAME"), + user=env.get("DATABASE_USER"), + password=env.get("DATABASE_PASSWORD"), + password_file=env.get("DATABASE_PASSWORD_FILE"), + ssl_mode=_ssl_mode(env), + pool_size=_positive_int(env.get("DATABASE_POOL_SIZE"), 5), + max_overflow=_positive_int(env.get("DATABASE_MAX_OVERFLOW"), 5), + pool_timeout_seconds=_positive_int(env.get("DATABASE_POOL_TIMEOUT_SECONDS"), 5), + connect_timeout_seconds=_positive_int(env.get("DATABASE_CONNECT_TIMEOUT_SECONDS"), 5), + statement_timeout_ms=_positive_int(env.get("DATABASE_STATEMENT_TIMEOUT_MS"), 30_000), + ) + + @property + def configured(self) -> bool: + return bool( + self.url + or ( + all((self.host, self.name, self.user)) + and (self.password is not None or self.password_file) + ) + ) + + def sqlalchemy_url(self) -> URL: + if self.url: + parsed = make_url(self.url) + if parsed.get_backend_name() != "postgresql": + raise DatabaseConfigurationError("DATABASE_URL must use PostgreSQL") + if parsed.drivername in {"postgres", "postgresql"}: + parsed = parsed.set(drivername="postgresql+asyncpg") + if parsed.drivername != "postgresql+asyncpg": + raise DatabaseConfigurationError("DATABASE_URL must use the asyncpg driver") + parsed = parsed.difference_update_query(["ssl", "sslmode"]) + if self.ssl_mode == "require": + parsed = parsed.update_query_dict({"ssl": "require"}) + return parsed + + if not self.configured: + raise DatabaseConfigurationError("PostgreSQL connection settings are incomplete") + if self.password is not None: + password = self.password.strip() + else: + password_path = Path(str(self.password_file)) + try: + if password_path.stat().st_size > 4096: + raise DatabaseConfigurationError( + "Database password file exceeds the safety bound" + ) + password = password_path.read_text(encoding="utf-8").strip() + except OSError as exc: + raise DatabaseConfigurationError( + "Database password file is unavailable" + ) from exc + if not password: + raise DatabaseConfigurationError("Database password file is empty") + url = URL.create( + "postgresql+asyncpg", + username=self.user, + password=password, + host=self.host, + port=self.port, + database=self.name, + ) + if self.ssl_mode == "require": + url = url.update_query_dict({"ssl": "require"}) + return url + + def raw_sqlalchemy_url(self) -> str: + """Render a normal URL for direct SQLAlchemy/asyncpg consumption. + + The returned value is deliberately not escaped for ConfigParser. Callers + crossing a real ConfigParser boundary must perform that escaping there. + """ + return self.sqlalchemy_url().render_as_string(hide_password=False) + + +class DatabaseManager: + def __init__(self, settings: DatabaseSettings): + self.settings = settings + self._engine: Optional[AsyncEngine] = None + self._health_engine: Optional[AsyncEngine] = None + + @property + def configured(self) -> bool: + return self.settings.configured + + def engine(self) -> AsyncEngine: + if self._engine is None: + url = self.settings.sqlalchemy_url() + connect_args = { + "command_timeout": self.settings.statement_timeout_ms / 1000, + "timeout": self.settings.connect_timeout_seconds, + "server_settings": { + "application_name": "foundation-intelligence-api", + "statement_timeout": str(self.settings.statement_timeout_ms), + }, + } + self._engine = create_async_engine( + url, + pool_size=self.settings.pool_size, + max_overflow=self.settings.max_overflow, + pool_timeout=self.settings.pool_timeout_seconds, + pool_recycle=1800, + pool_pre_ping=True, + connect_args=connect_args, + ) + return self._engine + + def health_engine(self) -> AsyncEngine: + """Use independent connections so analytical pool pressure cannot block health.""" + if self._health_engine is None: + connect_args = { + "command_timeout": min(self.settings.statement_timeout_ms / 1000, 5), + "timeout": self.settings.connect_timeout_seconds, + "server_settings": { + "application_name": "foundation-intelligence-health", + "statement_timeout": "5000", + }, + } + self._health_engine = create_async_engine( + self.settings.sqlalchemy_url(), + poolclass=NullPool, + connect_args=connect_args, + ) + return self._health_engine + + def sessions(self) -> async_sessionmaker[AsyncSession]: + """Create sessions with explicit transactions and no implicit expiry.""" + return async_sessionmaker( + self.engine(), + class_=AsyncSession, + expire_on_commit=False, + autoflush=False, + ) + + async def check(self) -> bool: + if not self.configured: + return False + try: + async with asyncio.timeout(self.settings.connect_timeout_seconds + 1): + async with self.health_engine().connect() as connection: + result = await connection.execute(text("SELECT 1")) + return result.scalar_one() == 1 + except Exception as exc: + logger.warning("PostgreSQL readiness check failed; class=%s", exc.__class__.__name__) + return False + + async def readiness( + self, + *, + expected_schema_version: str, + require_critical_configuration: bool = True, + ) -> dict[str, object]: + unavailable = { + "ready": False, + "checks": { + "postgresql": "unavailable", + "schema_version": "unknown", + "active_dataset": "unknown", + "critical_configuration": "unknown", + "queue": "unknown", + }, + } + if not self.configured: + return unavailable + try: + async with asyncio.timeout(self.settings.connect_timeout_seconds + 1): + async with self.health_engine().connect() as connection: + row = ( + await connection.execute( + text( + """ + SELECT + (SELECT version_num FROM alembic_version) AS schema_version, + (SELECT dataset_version FROM dataset_versions + WHERE is_active AND status='active') AS active_dataset, + (SELECT COUNT(*) FROM source_configurations) AS source_count, + (SELECT COUNT(*) FROM retention_policies) AS policy_count, + to_regclass('job_dispatch_outbox') IS NOT NULL AS queue_available, + COALESCE(( + SELECT EXTRACT(EPOCH FROM ( + CURRENT_TIMESTAMP - MIN(created_at) + )) FROM job_dispatch_outbox + WHERE status IN ('pending', 'failed') + ), 0) AS queue_age_seconds, + (SELECT COUNT(*) FROM job_runs + WHERE status='dead_lettered') AS dead_letter_count + """ + ) + ) + ).mappings().one() + schema_healthy = str(row["schema_version"]) == expected_schema_version + dataset_healthy = bool(row["active_dataset"]) + configuration_healthy = int(row["source_count"]) > 0 and int(row["policy_count"]) > 0 + queue_healthy = bool(row["queue_available"]) + ready = ( + schema_healthy + and dataset_healthy + and queue_healthy + and (configuration_healthy or not require_critical_configuration) + ) + return { + "ready": ready, + "checks": { + "postgresql": "healthy", + "schema_version": "healthy" if schema_healthy else "mismatch", + "active_dataset": "healthy" if dataset_healthy else "missing", + "critical_configuration": ( + "healthy" + if configuration_healthy + else ( + "not_required" + if not require_critical_configuration + else "missing" + ) + ), + "queue": "healthy" if queue_healthy else "unavailable", + }, + "metadata": { + "schema_version": str(row["schema_version"]), + "active_dataset": str(row["active_dataset"] or ""), + "source_count": int(row["source_count"]), + "policy_count": int(row["policy_count"]), + "queue_age_seconds": float(row["queue_age_seconds"]), + "dead_letter_count": int(row["dead_letter_count"]), + }, + } + except Exception as exc: + logger.warning( + "PostgreSQL readiness contract failed", + extra={"operation": "readiness", "error_class": exc.__class__.__name__}, + ) + return unavailable + + def pool_status(self) -> dict[str, float]: + if self._engine is None: + return {"checked_out": 0.0, "capacity": 0.0, "utilization_ratio": 0.0} + pool = self._engine.sync_engine.pool + checked_out = float(pool.checkedout()) if hasattr(pool, "checkedout") else 0.0 + size = float(pool.size()) if hasattr(pool, "size") else 0.0 + overflow = float(pool.overflow()) if hasattr(pool, "overflow") else 0.0 + capacity = max(size + max(overflow, 0.0), size, 1.0) + return { + "checked_out": checked_out, + "capacity": capacity, + "utilization_ratio": checked_out / capacity, + } + + async def close(self) -> None: + if self._engine is not None: + await self._engine.dispose() + self._engine = None + if self._health_engine is not None: + await self._health_engine.dispose() + self._health_engine = None diff --git a/src/bff/main.py b/src/bff/main.py index 0b0e8a4..3fcbf0a 100644 --- a/src/bff/main.py +++ b/src/bff/main.py @@ -1,14 +1,127 @@ +import asyncio +from contextlib import asynccontextmanager +import inspect +import re import time +import uuid from fastapi import FastAPI, Request, Response from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse, RedirectResponse from bff.auth import router as auth_router -from bff.charity import router as charity_router from bff.proxy import router as proxy_router -from bff.admin import router as admin_router from bff.news import router as news_router -from bff.repositories import get_charity_repository -from bff.utils.logging import logger +from bff.audit import StructuredLogAuditSink, event_from_request +from bff.config import SECURITY_SETTINGS, validate_security_settings +from bff.database import DatabaseManager, DatabaseSettings +from bff.security import IdempotencyStore, SlidingWindowRateLimiter +from bff.utils.logging import logger, pseudonymous_actor_id +from observability.metrics import MetricsRegistry, load_observability_configuration +from transition.runtime import load_transition_settings +from transition.shadow import ( + ComparisonPolicy, + ShadowComparisonCoordinator, + ShadowComparisonMiddleware, + StructuredLogEvidenceSink, +) + + +TRANSITION_SETTINGS = load_transition_settings() +POSTGRESQL_ONLY_RUNTIME = TRANSITION_SETTINGS.postgresql_authoritative +if POSTGRESQL_ONLY_RUNTIME: + from bff.postgres.admin_routes import router as admin_router + from bff.postgres.audit_repository import PostgresAuditSink + from bff.postgres.base import ANALYTICS_CACHE + from bff.postgres.governance_repository import GovernanceRepository + from bff.postgres.governance_routes import router as governance_router + from bff.postgres.idempotency_repository import PostgresIdempotencyStore + from bff.postgres.observability_routes import router as observability_router + from bff.postgres.pipeline_repository import PipelineRepository + from bff.postgres.routes import router as charity_router + from governance.retention import load_governance_configuration + from pipelines.durable import load_source_configurations +else: + # The legacy SQLite repository is restricted to development/test while the + # remaining domain journeys are ported in Phase 5. + from bff.charity import router as charity_router + from bff.admin import router as admin_router + from bff.repositories import get_charity_repository + + +def _shadow_log(payload): + logger.info("shadow_comparison", extra={"shadow": payload}) + + +if TRANSITION_SETTINGS.shadow_enabled: + from transition.sqlite_source import SQLiteShadowReader, resolve_shadow_journey + + SHADOW_COORDINATOR = ShadowComparisonCoordinator( + SQLiteShadowReader(str(TRANSITION_SETTINGS.shadow_sqlite_path)), + StructuredLogEvidenceSink(_shadow_log), + policy=ComparisonPolicy( + unordered_paths=frozenset(TRANSITION_SETTINGS.approved_unordered_paths), + ignored_paths=frozenset(TRANSITION_SETTINGS.ignored_operational_paths), + maximum_differences=TRANSITION_SETTINGS.maximum_recorded_differences, + ), + timeout_seconds=TRANSITION_SETTINGS.shadow_timeout_seconds, + maximum_pending=TRANSITION_SETTINGS.maximum_pending_comparisons, + ) +else: + SHADOW_COORDINATOR = None + + def resolve_shadow_journey(method, path, query_string): + return None + + +@asynccontextmanager +async def lifespan(application: FastAPI): + """Validate security before accepting traffic, then initialize repository state.""" + validate_security_settings(application.state.security_settings) + application.state.database = DatabaseManager(DatabaseSettings.from_env()) + if POSTGRESQL_ONLY_RUNTIME: + # Constructing the pool remains lazy; readiness owns the first bounded + # connection and all production repositories use the same manager. + sessions = application.state.database.sessions() + public_readonly = ( + application.state.security_settings.auth_mode == "public_readonly" + ) + if public_readonly: + # The public web task has a database role with SELECT only. It must + # never synchronize configuration or persist audit/idempotency rows + # as a side effect of startup or anonymous reads. + application.state.audit_sink = StructuredLogAuditSink() + application.state.idempotency_store = IdempotencyStore() + logger.info("Public read-only PostgreSQL runtime initialized without writes.") + else: + application.state.audit_sink = PostgresAuditSink(sessions) + application.state.idempotency_store = PostgresIdempotencyStore(sessions) + synchronized_sources = await PipelineRepository(sessions).synchronize_sources( + load_source_configurations() + ) + synchronized_policies = await GovernanceRepository(sessions).synchronize_policies( + load_governance_configuration() + ) + logger.info( + "Synchronized %s governance-gated source configurations.", + synchronized_sources, + ) + logger.info( + "Synchronized %s non-destructive retention policies.", + synchronized_policies, + ) + logger.info("PostgreSQL repository runtime initialized.") + else: + get_charity_repository() + logger.info( + "Development/test legacy repository initialized; expensive Overview " + "aggregation is request-driven." + ) + try: + yield + finally: + if SHADOW_COORDINATOR is not None: + await SHADOW_COORDINATOR.drain() + await application.state.database.close() + app = FastAPI( title="Foundation Intelligence Platform BFF API", @@ -16,72 +129,213 @@ "Backend for the Foundation Intelligence Platform. It serves normalized organization, " "grant, provenance, enrichment, and experimental relevance-score data to the dashboard." ), - version="1.0.0" + version="1.0.0", + lifespan=lifespan, ) -# CORS configuration -# Note: when allow_credentials=True, allow_origins cannot be ["*"]. -# We explicitly list common development and localhost URLs. -origins = [ - "http://localhost:3000", # React / Next.js default - "http://localhost:5173", # Vite default (React/Vue/Svelte) - "http://localhost:8000", # FastAPI docs / local - "http://127.0.0.1:3000", - "http://127.0.0.1:5173", - # Vite selects the next port when a previous local dev session owns 5173. - "http://127.0.0.1:5174", -] - -# Vite moves to the next available port when another local development session -# already owns its default port. Keep credentialed browser requests local-only, -# while allowing those fallback Vite ports without having to restart the BFF for -# each one. -local_development_origin = r"http://(localhost|127\.0\.0\.1):\d+" +app.state.security_settings = SECURITY_SETTINGS +app.state.rate_limiter = SlidingWindowRateLimiter( + SECURITY_SETTINGS.rate_limit_requests, + SECURITY_SETTINGS.rate_limit_window_seconds, +) +app.state.idempotency_store = IdempotencyStore() +app.state.audit_sink = StructuredLogAuditSink() +app.state.database = DatabaseManager(DatabaseSettings.from_env()) +app.state.observability_configuration = load_observability_configuration() +app.state.metrics_registry = MetricsRegistry(app.state.observability_configuration) + +app.add_middleware( + ShadowComparisonMiddleware, + coordinator=SHADOW_COORDINATOR, + maximum_response_bytes=TRANSITION_SETTINGS.maximum_response_bytes, + journey_resolver=resolve_shadow_journey, +) app.add_middleware( CORSMiddleware, - allow_origins=origins, - allow_origin_regex=local_development_origin, + allow_origins=list(SECURITY_SETTINGS.cors_origins), allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], + allow_methods=["GET", "POST", "PUT", "OPTIONS"], + allow_headers=[ + "Accept", + "Authorization", + "Content-Type", + "Idempotency-Key", + "X-Action-Reason", + "X-Request-ID", + "X-Trace-ID", + ], ) -@app.on_event("startup") -async def warm_repository() -> None: - """Initialize the repository without blocking readiness on a full scan. +# Request safety, audit and request-response logging middleware. +_REQUEST_ID_PATTERN = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$") - Persisted derived indexes and cached Overview payloads are reused by the - first relevant request. A synchronous full-Overview warmup previously kept - the health endpoint unavailable for tens of seconds on the audited 1.3 GB - SQLite file, making a healthy local process look crashed. - """ - get_charity_repository() - logger.info( - "Repository initialized; expensive Overview aggregation is request-driven." - ) -# Request-Response logging middleware @app.middleware("http") async def log_requests(request: Request, call_next): start_time = time.time() - - # Process request - response = await call_next(request) - - duration = time.time() - start_time + supplied_request_id = request.headers.get("x-request-id", "") + request.state.request_id = ( + supplied_request_id + if _REQUEST_ID_PATTERN.fullmatch(supplied_request_id) + else str(uuid.uuid4()) + ) + supplied_trace_id = request.headers.get("x-trace-id", "") + request.state.trace_id = ( + supplied_trace_id + if _REQUEST_ID_PATTERN.fullmatch(supplied_trace_id) + else request.state.request_id + ) + settings = app.state.security_settings + + if ( + settings.auth_mode == "public_readonly" + and request.url.path.startswith("/api/") + and request.method not in {"GET", "HEAD"} + ): + # Fail before routing so an anonymous mutation-shaped request never + # becomes a 405/validation oracle or reaches a current/future handler. + response = JSONResponse( + status_code=401, + content={"detail": "Public demo access is read-only."}, + headers={"WWW-Authenticate": "Bearer"}, + ) + elif request.method in {"POST", "PUT", "PATCH", "DELETE"}: + content_length = request.headers.get("content-length") + try: + declared_size = int(content_length) if content_length else 0 + except ValueError: + declared_size = settings.max_request_body_bytes + 1 + if declared_size > settings.max_request_body_bytes: + response = JSONResponse( + status_code=413, + content={"detail": "Request body exceeds the configured limit."}, + ) + else: + body = await request.body() + if len(body) > settings.max_request_body_bytes: + response = JSONResponse( + status_code=413, + content={"detail": "Request body exceeds the configured limit."}, + ) + else: + try: + response = await asyncio.wait_for( + call_next(request), + timeout=settings.request_timeout_seconds, + ) + except asyncio.TimeoutError: + response = JSONResponse( + status_code=504, + content={"detail": "Request processing timed out."}, + ) + else: + try: + response = await asyncio.wait_for( + call_next(request), + timeout=settings.request_timeout_seconds, + ) + except asyncio.TimeoutError: + response = JSONResponse( + status_code=504, + content={"detail": "Request processing timed out."}, + ) + + response.headers["X-Request-ID"] = request.state.request_id + response.headers["X-Trace-ID"] = request.state.trace_id + record_key = getattr(request.state, "idempotency_record_key", None) + if record_key: + if response.status_code < 400 or response.status_code >= 500: + completion = app.state.idempotency_store.complete(record_key) + else: + completion = app.state.idempotency_store.release(record_key) + if inspect.isawaitable(completion): + await completion + if hasattr(request.state, "audit_action"): + error_class = None if response.status_code < 400 else f"http_{response.status_code}" + audit_result = app.state.audit_sink.record( + event_from_request(request, response.status_code, error_class) + ) + if inspect.isawaitable(audit_result): + await audit_result + + duration_ms = (time.time() - start_time) * 1000 + route = request.scope.get("route") + route_path = getattr(route, "path", request.url.path) + operation = f"{request.method} {route_path}" + principal = getattr(request.state, "principal", None) + actor_id = pseudonymous_actor_id( + getattr(principal, "actor_id", "anonymous") + ) + role = getattr(principal, "primary_role", "anonymous") + error_class = None if response.status_code < 400 else f"http_{response.status_code}" + registry = app.state.metrics_registry + metric_dimensions = { + "service": app.state.observability_configuration.service, + "environment": settings.app_env, + "operation": operation, + "status": str(response.status_code), + } + registry.observe("api_request_duration_ms", duration_ms, **metric_dimensions) + if response.status_code >= 500: + registry.increment( + "api_errors_total", + service=app.state.observability_configuration.service, + environment=settings.app_env, + operation=operation, + error_class=error_class, + ) + pool_status = app.state.database.pool_status() + registry.set_gauge( + "db_pool_checked_out", + pool_status["checked_out"], + service=app.state.observability_configuration.service, + environment=settings.app_env, + ) + registry.set_gauge( + "db_pool_utilization_ratio", + pool_status["utilization_ratio"], + service=app.state.observability_configuration.service, + environment=settings.app_env, + ) + if POSTGRESQL_ONLY_RUNTIME: + registry.set_gauge( + "cache_hit_ratio", + ANALYTICS_CACHE.hit_ratio, + service=app.state.observability_configuration.service, + environment=settings.app_env, + cache="analytics", + ) logger.info( - f"Request: {request.method} {request.url.path} " - f"| Status: {response.status_code} " - f"| Duration: {duration:.4f}s" + "request_completed", + extra={ + "request_id": request.state.request_id, + "trace_id": request.state.trace_id, + "actor_id": actor_id, + "role": role, + "operation": operation, + "duration_ms": round(duration_ms, 3), + "status": response.status_code, + "error_class": error_class, + }, ) return response # Custom centralized exception handlers @app.exception_handler(Exception) async def global_exception_handler(request: Request, exc: Exception): - logger.error(f"Unhandled exception occurred on path {request.url.path}: {exc}", exc_info=True) + logger.error( + "request_unhandled_exception", + extra={ + "request_id": getattr(request.state, "request_id", "unknown"), + "trace_id": getattr(request.state, "trace_id", "unknown"), + "operation": f"{request.method} {request.url.path}", + "status": 500, + "error_class": exc.__class__.__name__, + }, + exc_info=True, + ) return JSONResponse( status_code=500, content={"detail": "An internal server error occurred. Please try again later."} @@ -92,6 +346,9 @@ async def global_exception_handler(request: Request, exc: Exception): app.include_router(charity_router) app.include_router(proxy_router) app.include_router(admin_router) +if POSTGRESQL_ONLY_RUNTIME: + app.include_router(governance_router) + app.include_router(observability_router) app.include_router(news_router) @app.get("/", include_in_schema=False) @@ -102,5 +359,50 @@ async def root_redirect(): @app.get("/health", tags=["Health Check"]) async def health_check(): - """Simple endpoint to verify that the BFF is running and healthy.""" + """Backward-compatible liveness endpoint.""" return {"status": "healthy", "service": "bff"} + + +@app.get("/health/live", tags=["Health Check"]) +async def liveness_check(): + """Report process liveness without checking external dependencies.""" + return {"status": "healthy", "service": "bff"} + + +@app.get("/health/ready", tags=["Health Check"]) +async def readiness_check(): + """Check schema, dataset, critical controls and durable queue independently.""" + settings = app.state.security_settings + result = await app.state.database.readiness( + expected_schema_version=app.state.observability_configuration.expected_schema_version, + require_critical_configuration=settings.auth_mode != "public_readonly", + ) + metadata = result.get("metadata", {}) + registry = app.state.metrics_registry + registry.set_gauge( + "readiness_success", + 1.0 if result["ready"] else 0.0, + service=app.state.observability_configuration.service, + environment=settings.app_env, + ) + if metadata: + registry.set_gauge( + "queue_oldest_message_age_seconds", + float(metadata["queue_age_seconds"]), + service=app.state.observability_configuration.service, + environment=settings.app_env, + queue="pipeline", + ) + registry.set_gauge( + "dlq_depth", + float(metadata["dead_letter_count"]), + service=app.state.observability_configuration.service, + environment=settings.app_env, + queue="pipeline", + ) + if result["ready"]: + return {"status": "ready", "checks": result["checks"]} + return JSONResponse( + status_code=503, + content={"status": "not_ready", "checks": result["checks"]}, + ) diff --git a/src/bff/news.py b/src/bff/news.py index 5d4ab00..026e461 100644 --- a/src/bff/news.py +++ b/src/bff/news.py @@ -27,7 +27,7 @@ from fastapi.responses import StreamingResponse from bff import config -from bff.auth import get_current_user_token +from bff.security import Role, require_roles from bff.schemas import NewsSource, NewsSummary from bff.utils.logging import logger @@ -895,7 +895,7 @@ def _parse_news_aliases(aliases: Optional[str]) -> list[str]: router = APIRouter( prefix="/api/news", tags=["Foundation News"], - dependencies=[Depends(get_current_user_token)], + dependencies=[Depends(require_roles(Role.ANALYST, action="news.read"))], ) diff --git a/src/bff/postgres/__init__.py b/src/bff/postgres/__init__.py new file mode 100644 index 0000000..172fb35 --- /dev/null +++ b/src/bff/postgres/__init__.py @@ -0,0 +1,5 @@ +"""Asynchronous PostgreSQL repositories for the normal application runtime.""" + +from bff.postgres.registry_repository import RegistrySearchRepository, SearchCursor + +__all__ = ["RegistrySearchRepository", "SearchCursor"] diff --git a/src/bff/postgres/admin_routes.py b/src/bff/postgres/admin_routes.py new file mode 100644 index 0000000..648cb88 --- /dev/null +++ b/src/bff/postgres/admin_routes.py @@ -0,0 +1,156 @@ +"""PostgreSQL-backed pipeline administration without local subprocess state.""" + +from __future__ import annotations + +import json + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status + +from bff.postgres.job_repository import PIPELINE_JOB_TYPES, PostgresJobRepository +from bff.postgres.pipeline_repository import PipelineRepository +from bff.schemas import PipelineStatus, PipelineTrigger, SourceScheduleUpdate +from bff.security import Role, require_roles +from bff.utils.logging import redact_text +from governance.exposure import redact_for_logs, serialize_exposed_fields +from governance.retention import load_governance_configuration +from pipelines.durable import redacted_configuration + + +router = APIRouter( + prefix="/api/admin", + tags=["Administrative & Monitoring Data"], + dependencies=[Depends(require_roles(Role.OPERATOR, action="administration.access"))], +) +GOVERNANCE_CONFIGURATION = load_governance_configuration() + + +def _jobs(request: Request) -> PostgresJobRepository: + return PostgresJobRepository(request.app.state.database.sessions()) + + +def _actor(request: Request) -> str: + principal = getattr(request.state, "principal", None) + return principal.actor_id if principal else "unknown" + + +def _pipelines(request: Request) -> PipelineRepository: + return PipelineRepository(request.app.state.database.sessions()) + + +@router.get("/pipeline/status", response_model=PipelineStatus) +async def get_pipeline_status( + repository: PostgresJobRepository = Depends(_jobs), +): + return await repository.latest_status() + + +@router.post( + "/pipeline/trigger", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="pipeline.trigger", idempotent=True))], +) +async def trigger_pipeline( + payload: PipelineTrigger, + request: Request, + repository: PostgresJobRepository = Depends(_jobs), +): + if payload.source not in PIPELINE_JOB_TYPES[:4]: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=f"Unsupported execution mode: {payload.source}", + ) + job = await repository.enqueue( + payload.source, + payload.model_dump(exclude={"source"}), + actor_id=_actor(request), + idempotency_key=str(request.headers.get("idempotency-key") or "").strip(), + ) + return { + "status": job["status"], + "started_at": job["requested_at"], + "finished_at": None, + "last_run_source": payload.source, + "error": None, + "job_id": job["job_id"], + } + + +@router.get("/pipeline/jobs") +async def get_pipeline_jobs( + limit: int = Query(default=50, ge=1, le=100), + repository: PostgresJobRepository = Depends(_jobs), +): + jobs = await repository.history(limit=limit) + return { + "jobs": [ + redact_for_logs( + serialize_exposed_fields( + job, + policy_name="job_history", + configuration=GOVERNANCE_CONFIGURATION, + ), + GOVERNANCE_CONFIGURATION, + ) + for job in jobs + ] + } + + +@router.get( + "/pipeline/logs", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="pipeline.logs.read"))], +) +async def get_pipeline_logs( + limit: int = Query(default=100, ge=1, le=100), + repository: PostgresJobRepository = Depends(_jobs), +): + events = [ + serialize_exposed_fields( + event, + policy_name="pipeline_event", + configuration=GOVERNANCE_CONFIGURATION, + ) + for event in await repository.events(limit=limit) + ] + redacted_events = redact_for_logs(events, GOVERNANCE_CONFIGURATION) + encoded = redact_text(json.dumps(redacted_events, sort_keys=True, default=str)) + return {"logs": encoded, "events": redacted_events} + + +@router.get("/pipeline/sources") +async def get_pipeline_sources( + repository: PipelineRepository = Depends(_pipelines), +): + return { + "sources": [ + serialize_exposed_fields( + redacted_configuration(source), + policy_name="source_configuration", + configuration=GOVERNANCE_CONFIGURATION, + ) + for source in await repository.sources() + ] + } + + +@router.put( + "/pipeline/sources/{source_name}/schedule", + dependencies=[ + Depends( + require_roles( + Role.ADMINISTRATOR, + action="pipeline.schedule.update", + idempotent=True, + ) + ) + ], +) +async def update_pipeline_source_schedule( + source_name: str, + payload: SourceScheduleUpdate, + repository: PipelineRepository = Depends(_pipelines), +): + try: + return await repository.set_source_enabled(source_name, enabled=payload.enabled) + except ValueError as exc: + raise HTTPException(status_code=409, detail=str(exc)) from exc diff --git a/src/bff/postgres/analytics_repository.py b/src/bff/postgres/analytics_repository.py new file mode 100644 index 0000000..1ad90dc --- /dev/null +++ b/src/bff/postgres/analytics_repository.py @@ -0,0 +1,1486 @@ +"""Async PostgreSQL grant aggregates, map facts, trends and drill-downs.""" + +from __future__ import annotations + +import asyncio +from datetime import date +from decimal import Decimal +from typing import Any, Optional, Sequence + +from sqlalchemy import text + +from bff.postgres.base import ( + ANALYTICS_CACHE, + MaterializationUnavailable, + PostgresRepository, + iso_value, + number_value, + utc_now, +) + + +VALID_ORIGINAL = "fact.original_amount_status NOT IN ('negative','invalid','missing')" +VALID_EUR = "fact.eur_amount_status NOT IN ('negative','missing','invalid')" + + +def _amount_policy(maximum: Optional[float] = None) -> dict[str, Any]: + return { + "monetary_precision": "minor_units_2_decimal_places", + "rounding": "ROUND_HALF_UP", + "zero_amounts": "included_when_source_value_is_numeric_zero", + "negative_amounts": "excluded_and_reported", + "upper_bound": "no_unapproved_implausibility_threshold_applied", + "maximum_observed_amount": maximum, + } + + +def _scope() -> dict[str, str]: + return { + "coverage_note": "Available PostgreSQL active-dataset grant records only.", + "market_scope": "available cached 360Giving records", + } + + +class AnalyticsRepository(PostgresRepository): + _MATERIALIZED_FILTERS = { + "date_from", + "date_to", + "beneficiary_geographies", + "programme_areas", + "donor", + "recipient", + "sources", + "foundation_regions", + "min_annual_giving", + "min_avg_grant_size", + } + + @classmethod + def _can_use_materializations(cls, filters: dict[str, Any]) -> bool: + return not any(filters.get(name) for name in cls._MATERIALIZED_FILTERS) + + @staticmethod + def _basis(currency: Optional[str]) -> tuple[str, str]: + selected = str(currency or "").upper() + return ("original", selected) if selected else ("eur_converted", "EUR") + + @staticmethod + async def _require_materialization(session, dataset_version: str) -> None: + available = await session.scalar( + text( + """ + SELECT EXISTS ( + SELECT 1 FROM materialization_versions + WHERE dataset_version=:dataset_version + AND materialization_name='dashboard_analytics' + AND is_active AND status='active' + ) + """ + ), + {"dataset_version": dataset_version}, + ) + if not available: + raise MaterializationUnavailable( + f"Dataset {dataset_version!r} has no active dashboard materialization" + ) + + @staticmethod + def _filtered_scope(filters: dict[str, Any]) -> tuple[str, dict[str, Any], str, str]: + conditions = ["fact.dataset_version=:dataset_version"] + parameters: dict[str, Any] = { + "currency": str(filters.get("currency") or "").upper() or None, + "date_from": filters.get("date_from"), + "date_to": filters.get("date_to"), + "beneficiary_geographies": list(filters.get("beneficiary_geographies") or []), + "programme_areas": list(filters.get("programme_areas") or []), + "donor": f"%{str(filters.get('donor') or '').strip()}%", + "recipient": f"%{str(filters.get('recipient') or '').strip()}%", + "sources": list(filters.get("sources") or []), + } + if parameters["currency"]: + conditions.append("fact.currency=:currency") + amount_minor = "fact.original_amount_minor" + amount_valid = VALID_ORIGINAL + selected_currency = str(parameters["currency"]) + else: + amount_minor = "fact.eur_amount_minor" + amount_valid = VALID_EUR + selected_currency = "EUR" + if parameters["date_from"]: + conditions.append("fact.award_date>=CAST(:date_from AS date)") + if parameters["date_to"]: + conditions.append("fact.award_date<=CAST(:date_to AS date)") + if parameters["beneficiary_geographies"]: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM grant_beneficiary_countries AS scope_country + WHERE scope_country.dataset_version=fact.dataset_version + AND scope_country.grant_id=fact.grant_id + AND scope_country.country_name=ANY( + CAST(:beneficiary_geographies AS text[]) + ) + ) + """ + ) + if parameters["programme_areas"]: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM grant_programme_categories AS scope_category + WHERE scope_category.dataset_version=fact.dataset_version + AND scope_category.grant_id=fact.grant_id + AND scope_category.programme_area=ANY( + CAST(:programme_areas AS text[]) + ) + ) + """ + ) + if filters.get("donor"): + conditions.append("fact.funding_name ILIKE :donor") + if filters.get("recipient"): + conditions.append("fact.recipient_name ILIKE :recipient") + if parameters["sources"]: + conditions.append("fact.source_namespace=ANY(CAST(:sources AS text[]))") + return " AND ".join(conditions), parameters, amount_minor, selected_currency + + async def beneficiary_geographies(self) -> list[str]: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await self._require_materialization(session, dataset_version) + rows = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version + AND dimension='beneficiary_country' + ORDER BY value + """ + ), + {"dataset_version": dataset_version}, + ) + return [str(row[0]) for row in rows if row[0]] + + async def map(self, **filters: Any) -> dict[str, Any]: + if self._can_use_materializations(filters): + return await self._materialized_map(**filters) + conditions, parameters, amount_minor, selected_currency = self._filtered_scope(filters) + valid = VALID_ORIGINAL if parameters["currency"] else VALID_EUR + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT country.country_code, country.country_name, + COUNT(DISTINCT fact.grant_id) AS grant_count, + SUM( + CASE WHEN {valid} + THEN {amount_minor}::numeric + / NULLIF(fact.country_count, 0) + ELSE 0 END + ) / 100 AS total_amount, + COUNT(DISTINCT fact.funding_name) AS distinct_funders, + COUNT(DISTINCT fact.recipient_name) AS distinct_recipients, + COUNT(*) AS association_count, + COUNT(*) FILTER (WHERE fact.country_count>1) + AS multi_country_count + FROM grant_overview_facts AS fact + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=fact.dataset_version + AND country.grant_id=fact.grant_id + WHERE {conditions} + GROUP BY country.country_code, country.country_name + ORDER BY total_amount DESC NULLS LAST, + country.country_name, country.country_code + LIMIT 500 + """ + ), + parameters, + ) + ).mappings() + ] + totals = ( + await session.execute( + text( + f""" + SELECT COUNT(*) AS total_grants, + COUNT(*) FILTER ( + WHERE EXISTS ( + SELECT 1 FROM grant_beneficiary_countries AS country + WHERE country.dataset_version=fact.dataset_version + AND country.grant_id=fact.grant_id + ) + ) AS known_grants, + COUNT(*) FILTER (WHERE fact.country_count>1) AS multi_country, + COUNT(*) FILTER (WHERE NOT ({valid})) AS invalid_amount + FROM grant_overview_facts AS fact WHERE {conditions} + """ + ), + parameters, + ) + ).mappings().one() + currencies = await session.execute( + text( + """ + SELECT DISTINCT currency FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND currency IS NOT NULL + ORDER BY currency + """ + ), + {"dataset_version": dataset_version}, + ) + sources = await session.execute( + text( + """ + SELECT DISTINCT source_namespace FROM grant_overview_facts + WHERE dataset_version=:dataset_version ORDER BY source_namespace + """ + ), + {"dataset_version": dataset_version}, + ) + total_grants = int(totals["total_grants"] or 0) + known_grants = int(totals["known_grants"] or 0) + items = [ + { + "region_or_country_code": row.get("country_code"), + "region_or_country_name": str(row.get("country_name") or row.get("country_code")), + "grant_count": int(row["grant_count"]), + "total_amount": number_value(row.get("total_amount")), + "currency": selected_currency, + "distinct_funders": int(row["distinct_funders"] or 0), + "distinct_recipients": int(row["distinct_recipients"] or 0), + "top_programme_areas": [], + "top_funders": [], + "top_recipients": [], + "original_geographies": [str(row.get("country_name") or "")], + "funding_grant_count": int(row["grant_count"]), + "excluded_multi_country_grant_count": int(row["multi_country_count"] or 0), + "excluded_invalid_amount_grant_count": int(totals["invalid_amount"] or 0), + } + for row in rows + ] + return { + "status": "available" if items else "no_transactions_found", + "geographic_dimension": "beneficiary_country", + "items": items, + "known_geography_count": known_grants, + "unknown_geography_count": max(0, total_grants - known_grants), + "coverage_percentage": round(known_grants / total_grants * 100, 2) if total_grants else 0.0, + "currencies": [str(row[0]) for row in currencies], + "selected_currency": selected_currency, + "funding_status": "available", + "funding_mode_available": True, + "grant_country_association_count": sum(int(row["association_count"]) for row in rows), + "multi_country_grant_count": int(totals["multi_country"] or 0), + "funding_excluded_multi_country_count": 0, + "funding_excluded_multi_country_amount": 0.0, + "funding_excluded_currency_count": 0, + "funding_excluded_invalid_amount_count": int(totals["invalid_amount"] or 0), + "connections": [], + "connection_grant_count": 0, + "connection_excluded_no_headquarters_count": 0, + "connection_same_country_count": 0, + "minimum_coverage_threshold": float(filters.get("min_coverage") or 0.30), + "metadata": { + "data_mode": "postgresql_materialized_grant_facts", + "source": [str(row[0]) for row in sources], + "generated_at": utc_now(), + "record_count": total_grants, + "derivation": "versioned grant_overview_facts and grant_beneficiary_countries", + "coverage": known_grants / total_grants if total_grants else 0.0, + "limitations": ["Connection-heavy relationships are loaded separately."], + }, + } + + async def _materialized_map(self, **filters: Any) -> dict[str, Any]: + basis, selected_currency = self._basis(filters.get("currency")) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await ANALYTICS_CACHE.retain_dataset(dataset_version) + key = ( + dataset_version, + "map", + basis, + selected_currency, + float(filters.get("min_coverage") or 0.30), + ) + + async def load() -> dict[str, Any]: + async with self.sessions() as session: + await self._require_materialization(session, dataset_version) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + """ + SELECT country_code, country_name, grant_count, + total_amount_minor / 100.0 AS total_amount, + distinct_funders, distinct_recipients, + association_count, multi_country_count, + invalid_amount_count + FROM analytics_country_aggregates + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + ORDER BY total_amount_minor DESC, country_name, + country_code LIMIT 500 + """ + ), + { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + }, + ) + ).mappings() + ] + totals = ( + await session.execute( + text( + """ + SELECT * FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + """ + ), + { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + }, + ) + ).mappings().first() + currencies = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='currency' + ORDER BY value + """ + ), + {"dataset_version": dataset_version}, + ) + sources = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='source' + ORDER BY value + """ + ), + {"dataset_version": dataset_version}, + ) + if totals is None: + raise MaterializationUnavailable( + f"Materialized currency scope {basis}/{selected_currency} is unavailable" + ) + total_grants = int(totals["total_grants"]) + known_grants = int(totals["known_geography_grants"]) + items = [ + { + "region_or_country_code": row["country_code"], + "region_or_country_name": str(row["country_name"]), + "grant_count": int(row["grant_count"]), + "total_amount": number_value(row["total_amount"]), + "currency": selected_currency, + "distinct_funders": int(row["distinct_funders"]), + "distinct_recipients": int(row["distinct_recipients"]), + "top_programme_areas": [], + "top_funders": [], + "top_recipients": [], + "original_geographies": [str(row["country_name"])], + "funding_grant_count": int(row["grant_count"]), + "excluded_multi_country_grant_count": int(row["multi_country_count"]), + "excluded_invalid_amount_grant_count": int(row["invalid_amount_count"]), + } + for row in rows + ] + return { + "status": "available" if items else "no_transactions_found", + "geographic_dimension": "beneficiary_country", + "items": items, + "known_geography_count": known_grants, + "unknown_geography_count": max(0, total_grants - known_grants), + "coverage_percentage": ( + round(known_grants / total_grants * 100, 2) if total_grants else 0.0 + ), + "currencies": [str(row[0]) for row in currencies], + "selected_currency": selected_currency, + "funding_status": "available", + "funding_mode_available": True, + "grant_country_association_count": sum( + int(row["association_count"]) for row in rows + ), + "multi_country_grant_count": int(totals["multi_country_grants"]), + "funding_excluded_multi_country_count": 0, + "funding_excluded_multi_country_amount": 0.0, + "funding_excluded_currency_count": 0, + "funding_excluded_invalid_amount_count": int( + totals["invalid_amount_grants"] + ), + "connections": [], + "connection_grant_count": 0, + "connection_excluded_no_headquarters_count": 0, + "connection_same_country_count": 0, + "minimum_coverage_threshold": float( + filters.get("min_coverage") or 0.30 + ), + "metadata": { + "data_mode": "postgresql_versioned_country_aggregates", + "source": [str(row[0]) for row in sources], + "generated_at": iso_value(totals["refreshed_at"]), + "record_count": total_grants, + "derivation": "versioned analytics_country_aggregates", + "coverage": known_grants / total_grants if total_grants else 0.0, + "limitations": [ + "Country connections are a bounded lazy secondary journey." + ], + }, + } + + return await ANALYTICS_CACHE.get_or_create(key, load) + + async def map_connections( + self, *, currency: Optional[str] = None, limit: int = 100 + ) -> dict[str, Any]: + basis, selected_currency = self._basis(currency) + limit = min(max(int(limit), 1), 250) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await self._require_materialization(session, dataset_version) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + """ + SELECT origin_country_code, origin_country_name, + destination_country_code, + destination_country_name, grant_count, + origin_sources + FROM analytics_country_connections + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + ORDER BY grant_count DESC, origin_country_code, + destination_country_code LIMIT :limit + """ + ), + { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + "limit": limit, + }, + ) + ).mappings() + ] + return { + "status": "available" if rows else "no_transactions_found", + "connections": [ + { + "origin_country_code": row["origin_country_code"], + "origin_country_name": row["origin_country_name"], + "destination_country_code": row["destination_country_code"], + "destination_country_name": row["destination_country_name"], + "grant_count": int(row["grant_count"]), + "top_funders": [], + "origin_sources": list(row["origin_sources"] or []), + } + for row in rows + ], + "connection_grant_count": sum(int(row["grant_count"]) for row in rows), + "selected_currency": selected_currency, + "limit": limit, + "metadata": { + "data_mode": "postgresql_versioned_country_connections", + "loading_mode": "lazy_bounded_secondary_request", + "dataset_version": dataset_version, + }, + } + + async def suggestions( + self, + *, + sources: Optional[Sequence[str]], + limit: int, + ) -> dict[str, Any]: + limit = min(max(int(limit), 1), 5000) + if not sources: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await self._require_materialization(session, dataset_version) + parameters = { + "dataset_version": dataset_version, + "limit": limit, + } + donors = await session.execute( + text( + """ + SELECT organization_name, grant_count + FROM analytics_entity_rankings + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + AND entity_role='funder' + ORDER BY grant_count DESC, organization_name, entity_key + LIMIT :limit + """ + ), + parameters, + ) + recipients = await session.execute( + text( + """ + SELECT organization_name, grant_count + FROM analytics_entity_rankings + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + AND entity_role='recipient' + ORDER BY grant_count DESC, organization_name, entity_key + LIMIT :limit + """ + ), + parameters, + ) + return { + "status": "available", + "donors": [ + {"name": str(row[0]), "grant_count": int(row[1])} + for row in donors + ], + "recipients": [ + {"name": str(row[0]), "grant_count": int(row[1])} + for row in recipients + ], + "metadata": { + "data_mode": "postgresql_versioned_entity_rankings", + "bounded_limit": limit, + }, + } + source_condition = ( + "AND source_namespace=ANY(CAST(:sources AS text[]))" if sources else "" + ) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters = { + "dataset_version": dataset_version, + "sources": list(sources or []), + "limit": limit, + } + donors = await session.execute( + text( + f""" + SELECT funding_name, COUNT(*) AS count + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + AND funding_name IS NOT NULL {source_condition} + GROUP BY funding_name ORDER BY count DESC, funding_name LIMIT :limit + """ + ), + parameters, + ) + recipients = await session.execute( + text( + f""" + SELECT recipient_name, COUNT(*) AS count + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + AND recipient_name IS NOT NULL {source_condition} + GROUP BY recipient_name ORDER BY count DESC, recipient_name LIMIT :limit + """ + ), + parameters, + ) + return { + "status": "available", + "donors": [{"name": str(row[0]), "grant_count": int(row[1])} for row in donors], + "recipients": [{"name": str(row[0]), "grant_count": int(row[1])} for row in recipients], + "metadata": {"data_mode": "postgresql_active_dataset", "bounded_limit": limit}, + } + + async def trends(self, **filters: Any) -> dict[str, Any]: + if self._can_use_materializations(filters): + return await self._materialized_trends(**filters) + months = min(max(int(filters.get("months") or 120), 1), 120) + conditions, parameters, amount_minor, selected_currency = self._filtered_scope(filters) + valid = VALID_ORIGINAL if parameters["currency"] else VALID_EUR + if not any(filters.get(key) for key in ("date_from", "date_to")) and "months" in filters: + conditions += ( + " AND fact.award_date >= CURRENT_DATE " + "- CAST(:months AS integer) * INTERVAL '1 month'" + ) + parameters["months"] = months + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT to_char(date_trunc('month', fact.award_date), 'YYYY-MM') AS month, + COUNT(*) AS source_record_count, + COUNT(*) FILTER (WHERE {valid}) AS grant_count, + SUM(CASE WHEN {valid} THEN {amount_minor} ELSE 0 END) / 100.0 + AS total_amount, + COUNT(*) FILTER (WHERE fact.country_count>0) AS mapped, + COUNT(*) FILTER (WHERE fact.country_count=0) AS unmapped + FROM grant_overview_facts AS fact + WHERE {conditions} AND fact.award_date IS NOT NULL + GROUP BY date_trunc('month', fact.award_date) + ORDER BY date_trunc('month', fact.award_date) + """ + ), + parameters, + ) + ).mappings() + ] + summary = ( + await session.execute( + text( + f""" + SELECT MIN(fact.award_date) AS first_date, + MAX(fact.award_date) AS latest_date, + COUNT(*) FILTER (WHERE fact.award_date IS NULL) AS missing_date, + COUNT(*) FILTER (WHERE fact.original_amount_status='negative') AS negative, + COUNT(*) FILTER (WHERE fact.original_amount_status='zero') AS zero, + COUNT(*) FILTER (WHERE NOT ({valid})) AS invalid, + MAX(CASE WHEN {valid} THEN {amount_minor} END) / 100.0 AS maximum + FROM grant_overview_facts AS fact WHERE {conditions} + """ + ), + parameters, + ) + ).mappings().one() + currencies = await session.execute( + text( + """ + SELECT DISTINCT currency FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND currency IS NOT NULL + ORDER BY currency + """ + ), + {"dataset_version": dataset_version}, + ) + sources = await session.execute( + text( + """ + SELECT DISTINCT source_namespace FROM grant_overview_facts + WHERE dataset_version=:dataset_version ORDER BY source_namespace + """ + ), + {"dataset_version": dataset_version}, + ) + first_date = iso_value(summary["first_date"]) + latest_date = iso_value(summary["latest_date"]) + return { + "status": "available" if rows else "no_transactions_found", + "currency": selected_currency, + "available_currencies": [str(row[0]) for row in currencies], + "date_basis": "award_date", + "granularity": "monthly", + "period": { + "from": str(first_date)[:7] if first_date else "unknown", + "to": str(latest_date)[:7] if latest_date else "unknown", + "months": len(rows), + "anchor": "observed_award_dates", + } if rows else None, + "items": [ + { + "month": row["month"], + "grant_count": int(row["grant_count"] or 0), + "source_record_count": int(row["source_record_count"] or 0), + "total_amount": number_value(row["total_amount"]), + "coverage_status": "observed", + "mapped_grant_count": int(row["mapped"] or 0), + "unmapped_grant_count": int(row["unmapped"] or 0), + } + for row in rows + ], + "excluded": { + "missing_date": int(summary["missing_date"] or 0), + "invalid_date": 0, + "missing_amount": int(summary["invalid"] or 0), + "invalid_amount": int(summary["invalid"] or 0), + "negative_amount": int(summary["negative"] or 0), + "unsupported_currency": 0, + "currency_filtered": 0, + "unsupported_source": 0, + "outside_period": 0, + }, + "zero_amount_count": int(summary["zero"] or 0), + "latest_award_date": latest_date, + "last_refreshed_at": utc_now(), + "source": [str(row[0]) for row in sources], + "data_mode": "postgresql_materialized_grant_facts", + "amount_policy": _amount_policy(number_value(summary["maximum"])), + "scope": _scope(), + } + + async def _materialized_trends(self, **filters: Any) -> dict[str, Any]: + basis, selected_currency = self._basis(filters.get("currency")) + granularity = "yearly" if filters.get("granularity") == "yearly" else "monthly" + months = ( + min(max(int(filters.get("months") or 120), 1), 120) + if "months" in filters else None + ) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await ANALYTICS_CACHE.retain_dataset(dataset_version) + key = ( + dataset_version, + "trends", + basis, + selected_currency, + granularity, + months, + ) + + async def load() -> dict[str, Any]: + month_condition = ( + "AND period_start >= CURRENT_DATE " + "- CAST(:months AS integer) * INTERVAL '1 month'" + if months is not None else "" + ) + parameters = { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + "granularity": granularity, + "months": months, + } + async with self.sessions() as session: + await self._require_materialization(session, dataset_version) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT period_start, source_record_count, grant_count, + total_amount_minor / 100.0 AS total_amount, + mapped_grant_count, unmapped_grant_count + FROM analytics_period_aggregates + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + AND granularity=:granularity {month_condition} + ORDER BY period_start + """ + ), + parameters, + ) + ).mappings() + ] + totals = ( + await session.execute( + text( + """ + SELECT * FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + """ + ), + parameters, + ) + ).mappings().first() + currencies = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='currency' + ORDER BY value + """ + ), + parameters, + ) + sources = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='source' + ORDER BY value + """ + ), + parameters, + ) + if totals is None: + raise MaterializationUnavailable( + f"Materialized currency scope {basis}/{selected_currency} is unavailable" + ) + first_date = iso_value(totals["first_award_date"]) + latest_date = iso_value(totals["latest_award_date"]) + format_period = ( + (lambda value: value.strftime("%Y")) + if granularity == "yearly" + else (lambda value: value.strftime("%Y-%m")) + ) + return { + "status": "available" if rows else "no_transactions_found", + "currency": selected_currency, + "available_currencies": [str(row[0]) for row in currencies], + "date_basis": "award_date", + "granularity": granularity, + "period": { + "from": format_period(rows[0]["period_start"]), + "to": format_period(rows[-1]["period_start"]), + "months": len(rows), + "anchor": "observed_award_dates", + } if rows else None, + "items": [ + { + "month": format_period(row["period_start"]), + "grant_count": int(row["grant_count"]), + "source_record_count": int(row["source_record_count"]), + "total_amount": number_value(row["total_amount"]), + "coverage_status": "observed", + "mapped_grant_count": int(row["mapped_grant_count"]), + "unmapped_grant_count": int(row["unmapped_grant_count"]), + } + for row in rows + ], + "excluded": { + "missing_date": int(totals["missing_date_grants"]), + "invalid_date": 0, + "missing_amount": int(totals["invalid_amount_grants"]), + "invalid_amount": int(totals["invalid_amount_grants"]), + "negative_amount": int(totals["negative_amount_grants"]), + "unsupported_currency": 0, + "currency_filtered": 0, + "unsupported_source": 0, + "outside_period": 0, + }, + "zero_amount_count": int(totals["zero_amount_grants"]), + "latest_award_date": latest_date, + "last_refreshed_at": iso_value(totals["refreshed_at"]), + "source": [str(row[0]) for row in sources], + "data_mode": "postgresql_versioned_period_aggregates", + "amount_policy": _amount_policy( + number_value(totals["maximum_amount_minor"]) / 100 + if totals["maximum_amount_minor"] is not None else None + ), + "scope": _scope(), + } + + return await ANALYTICS_CACHE.get_or_create(key, load) + + async def themes(self, *, currency: Optional[str] = None, **filters: Any) -> dict[str, Any]: + if self._can_use_materializations(filters): + return await self._materialized_themes(currency=currency) + filters = {**filters, "currency": currency} + conditions, parameters, amount_minor, selected_currency = self._filtered_scope(filters) + valid = VALID_ORIGINAL if parameters["currency"] else VALID_EUR + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT category.programme_area, + COUNT(DISTINCT fact.grant_id) AS distinct_grants, + SUM(1.0 / NULLIF(fact.programme_category_count, 0)) + AS weighted_grants, + SUM(CASE WHEN {valid} + THEN {amount_minor}::numeric + / NULLIF(fact.programme_category_count, 0) + ELSE 0 END) / 100 AS allocated_amount, + COUNT(DISTINCT fact.grant_id) FILTER ( + WHERE fact.programme_provenance='source' + ) AS source_count, + COUNT(DISTINCT fact.grant_id) FILTER ( + WHERE fact.programme_provenance='inferred' + ) AS inferred_count + FROM grant_overview_facts AS fact + JOIN grant_programme_categories AS category + ON category.dataset_version=fact.dataset_version + AND category.grant_id=fact.grant_id + WHERE {conditions} + GROUP BY category.programme_area + ORDER BY allocated_amount DESC, category.programme_area + """ + ), + parameters, + ) + ).mappings() + ] + coverage = ( + await session.execute( + text( + f""" + SELECT COUNT(*) FILTER (WHERE {valid}) AS qualifying, + COUNT(*) FILTER ( + WHERE {valid} AND fact.programme_category_count>0 + ) AS classified, + COUNT(*) FILTER ( + WHERE {valid} AND fact.programme_category_count=0 + ) AS unclassified, + COUNT(*) FILTER (WHERE fact.programme_provenance='source') + AS source_count, + COUNT(*) FILTER (WHERE fact.programme_provenance='inferred') + AS inferred_count, + COUNT(*) FILTER (WHERE fact.programme_category_count>1) + AS multiple_count, + COUNT(*) FILTER (WHERE fact.invalid_source_label) AS invalid_label, + COUNT(*) FILTER (WHERE fact.low_confidence_inference) AS low_confidence, + SUM(CASE WHEN {valid} THEN {amount_minor} ELSE 0 END) / 100.0 + AS qualifying_amount, + COUNT(*) FILTER (WHERE fact.original_amount_status='negative') AS negative, + COUNT(*) FILTER (WHERE fact.original_amount_status='zero') AS zero, + MAX(CASE WHEN {valid} THEN {amount_minor} END) / 100.0 AS maximum + FROM grant_overview_facts AS fact WHERE {conditions} + """ + ), + parameters, + ) + ).mappings().one() + currencies = await session.execute( + text( + """ + SELECT DISTINCT currency FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND currency IS NOT NULL + ORDER BY currency + """ + ), + {"dataset_version": dataset_version}, + ) + sources = await session.execute( + text( + """ + SELECT DISTINCT source_namespace FROM grant_overview_facts + WHERE dataset_version=:dataset_version ORDER BY source_namespace + """ + ), + {"dataset_version": dataset_version}, + ) + qualifying = int(coverage["qualifying"] or 0) + classified = int(coverage["classified"] or 0) + source_count = int(coverage["source_count"] or 0) + inferred_count = int(coverage["inferred_count"] or 0) + items = [ + { + "programme_area": row["programme_area"], + "distinct_grant_count": int(row["distinct_grants"]), + "weighted_grant_count": round(float(row["weighted_grants"] or 0), 4), + "allocated_amount": round(float(row["allocated_amount"] or 0), 2), + "source_classified_grant_count": int(row["source_count"] or 0), + "inferred_classified_grant_count": int(row["inferred_count"] or 0), + "unclassified_grant_count": 0, + } + for row in rows + ] + allocated = round(sum(item["allocated_amount"] for item in items), 2) + return { + "status": "available" if qualifying else "no_transactions_found", + "currency": selected_currency, + "available_currencies": [str(row[0]) for row in currencies], + "allocation_method": "equal_split_across_available_categories", + "classification_precedence": ["source", "inferred", "unclassified"], + "inference_confidence_threshold": 0.65, + "items": items, + "classification_coverage": { + "qualifying_grant_count": qualifying, + "classified_grant_count": classified, + "unclassified_grant_count": int(coverage["unclassified"] or 0), + "classified_percentage": round(classified / qualifying * 100, 2) if qualifying else 0.0, + "source_classified_grant_count": source_count, + "inferred_classified_grant_count": inferred_count, + "source_percentage": round(source_count / qualifying * 100, 2) if qualifying else 0.0, + "inferred_percentage": round(inferred_count / qualifying * 100, 2) if qualifying else 0.0, + "multiple_programme_area_grant_count": int(coverage["multiple_count"] or 0), + "invalid_source_label_count": int(coverage["invalid_label"] or 0), + "low_confidence_inference_count": int(coverage["low_confidence"] or 0), + }, + "qualifying_amount": number_value(coverage["qualifying_amount"]) or 0.0, + "allocated_amount": allocated, + "excluded": { + "missing_date": 0, + "invalid_date": 0, + "missing_amount": 0, + "invalid_amount": 0, + "negative_amount": int(coverage["negative"] or 0), + "unsupported_currency": 0, + "currency_filtered": 0, + "unsupported_source": 0, + "outside_period": 0, + }, + "zero_amount_count": int(coverage["zero"] or 0), + "last_refreshed_at": utc_now(), + "source": [str(row[0]) for row in sources], + "data_mode": "postgresql_materialized_grant_facts", + "amount_policy": _amount_policy(number_value(coverage["maximum"])), + "scope": _scope(), + } + + async def _materialized_themes( + self, *, currency: Optional[str] + ) -> dict[str, Any]: + basis, selected_currency = self._basis(currency) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await ANALYTICS_CACHE.retain_dataset(dataset_version) + key = (dataset_version, "themes", basis, selected_currency) + + async def load() -> dict[str, Any]: + parameters = { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + } + async with self.sessions() as session: + await self._require_materialization(session, dataset_version) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + """ + SELECT programme_area, distinct_grant_count, + weighted_grant_count, + allocated_amount_minor / 100.0 AS allocated_amount, + source_classified_count, + inferred_classified_count + FROM analytics_programme_aggregates + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + ORDER BY allocated_amount_minor DESC, programme_area + """ + ), + parameters, + ) + ).mappings() + ] + totals = ( + await session.execute( + text( + """ + SELECT * FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + """ + ), + parameters, + ) + ).mappings().first() + currencies = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='currency' + ORDER BY value + """ + ), + parameters, + ) + sources = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='source' + ORDER BY value + """ + ), + parameters, + ) + if totals is None: + raise MaterializationUnavailable( + f"Materialized currency scope {basis}/{selected_currency} is unavailable" + ) + qualifying = int(totals["total_grants"] - totals["invalid_amount_grants"]) + classified = int(totals["classified_grants"]) + source_count = int(totals["source_classified_grants"]) + inferred_count = int(totals["inferred_classified_grants"]) + items = [ + { + "programme_area": str(row["programme_area"]), + "distinct_grant_count": int(row["distinct_grant_count"]), + "weighted_grant_count": round( + float(row["weighted_grant_count"]), 4 + ), + "allocated_amount": round(float(row["allocated_amount"]), 2), + "source_classified_grant_count": int( + row["source_classified_count"] + ), + "inferred_classified_grant_count": int( + row["inferred_classified_count"] + ), + "unclassified_grant_count": 0, + } + for row in rows + ] + allocated = round(sum(item["allocated_amount"] for item in items), 2) + return { + "status": "available" if qualifying else "no_transactions_found", + "currency": selected_currency, + "available_currencies": [str(row[0]) for row in currencies], + "allocation_method": "equal_split_across_available_categories", + "classification_precedence": ["source", "inferred", "unclassified"], + "inference_confidence_threshold": 0.65, + "items": items, + "classification_coverage": { + "qualifying_grant_count": qualifying, + "classified_grant_count": classified, + "unclassified_grant_count": int(totals["unclassified_grants"]), + "classified_percentage": ( + round(classified / qualifying * 100, 2) if qualifying else 0.0 + ), + "source_classified_grant_count": source_count, + "inferred_classified_grant_count": inferred_count, + "source_percentage": ( + round(source_count / qualifying * 100, 2) if qualifying else 0.0 + ), + "inferred_percentage": ( + round(inferred_count / qualifying * 100, 2) if qualifying else 0.0 + ), + "multiple_programme_area_grant_count": int( + totals["multiple_programme_grants"] + ), + "invalid_source_label_count": int( + totals["invalid_source_label_grants"] + ), + "low_confidence_inference_count": int( + totals["low_confidence_grants"] + ), + }, + "qualifying_amount": round( + float(totals["total_amount_minor"]) / 100, 2 + ), + "allocated_amount": allocated, + "excluded": { + "missing_date": 0, + "invalid_date": 0, + "missing_amount": int(totals["invalid_amount_grants"]), + "invalid_amount": int(totals["invalid_amount_grants"]), + "negative_amount": int(totals["negative_amount_grants"]), + "unsupported_currency": 0, + "currency_filtered": 0, + "unsupported_source": 0, + "outside_period": 0, + }, + "zero_amount_count": int(totals["zero_amount_grants"]), + "last_refreshed_at": iso_value(totals["refreshed_at"]), + "source": [str(row[0]) for row in sources], + "data_mode": "postgresql_versioned_programme_aggregates", + "amount_policy": _amount_policy( + number_value(totals["maximum_amount_minor"]) / 100 + if totals["maximum_amount_minor"] is not None else None + ), + "scope": _scope(), + } + + return await ANALYTICS_CACHE.get_or_create(key, load) + + async def summary(self) -> dict[str, Any]: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await ANALYTICS_CACHE.retain_dataset(dataset_version) + key = (dataset_version, "network_summary") + + async def load() -> dict[str, Any]: + parameters = {"dataset_version": dataset_version} + async with self.sessions() as session: + await self._require_materialization(session, dataset_version) + total = await session.scalar( + text( + """ + SELECT total_grants FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + """ + ), + parameters, + ) + currencies = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='currency' + ORDER BY value + """ + ), + parameters, + ) + donor_rows = await session.execute( + text( + """ + SELECT organization_id, organization_name, + total_amount_minor / 100.0, grant_count + FROM analytics_entity_rankings + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + AND entity_role='funder' + ORDER BY total_amount_minor DESC, entity_key LIMIT 10 + """ + ), + parameters, + ) + recipient_rows = await session.execute( + text( + """ + SELECT organization_id, organization_name, + total_amount_minor / 100.0, grant_count + FROM analytics_entity_rankings + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + AND entity_role='recipient' + ORDER BY total_amount_minor DESC, entity_key LIMIT 10 + """ + ), + parameters, + ) + ranking = lambda rows: [ + { + "organization_id": row[0], + "organization_name": str(row[1] or "Unknown"), + "total_amount": float(row[2] or 0), + "currency": "EUR", + "grant_count": int(row[3]), + } + for row in rows + ] + return { + "status": "available" if total else "no_transactions_found", + "total_grant_count": int(total or 0), + "currencies": [str(row[0]) for row in currencies], + "largest_donors": ranking(donor_rows), + "largest_recipients": ranking(recipient_rows), + "metadata": { + "data_mode": "postgresql_versioned_entity_rankings", + "source": ["360Giving"], + "generated_at": utc_now(), + "record_count": int(total or 0), + "derivation": "versioned analytics_entity_rankings", + "limitations": ["Default rankings use converted EUR facts."], + }, + } + + return await ANALYTICS_CACHE.get_or_create(key, load) + + async def overview(self, **filters: Any) -> dict[str, Any]: + if self._can_use_materializations(filters): + basis, selected_currency = self._basis(filters.get("currency")) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + await self._require_materialization(session, dataset_version) + parameters = { + "dataset_version": dataset_version, + "basis": basis, + "currency": selected_currency, + } + totals = ( + await session.execute( + text( + """ + SELECT total_grants, total_amount_minor, + first_award_date, latest_award_date + FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + """ + ), + parameters, + ) + ).mappings().first() + entity_counts = dict( + ( + await session.execute( + text( + """ + SELECT entity_role, COUNT(*) + FROM analytics_entity_rankings + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + GROUP BY entity_role + """ + ), + parameters, + ) + ).all() + ) + if totals is None: + raise MaterializationUnavailable( + f"Materialized currency scope {basis}/{selected_currency} is unavailable" + ) + map_payload, trend_payload, theme_payload = await asyncio.gather( + self.map(**filters), + self.trends(**filters), + self.themes( + currency=filters.get("currency"), + **{key: value for key, value in filters.items() if key != "currency"}, + ), + ) + grant_count = int(totals["total_grants"]) + awarded_funding = float(totals["total_amount_minor"]) / 100 + classification = theme_payload["classification_coverage"] + return { + "status": "available" if totals["total_grants"] else "no_transactions_found", + "kpis": { + "awarded_funding": awarded_funding, + "grants_monitored": grant_count, + "country_coverage_percentage": map_payload["coverage_percentage"], + "mapped_grant_count": map_payload["known_geography_count"], + "unmapped_grant_count": map_payload["unknown_geography_count"], + "programme_coverage_percentage": classification["classified_percentage"], + "classified_grant_count": classification["classified_grant_count"], + "qualifying_programme_grant_count": classification["qualifying_grant_count"], + "grant_count": grant_count, + "funder_count": int(entity_counts.get("funder", 0)), + "recipient_count": int(entity_counts.get("recipient", 0)), + "total_amount": awarded_funding, + "currency": selected_currency, + }, + "map": map_payload, + "trends": trend_payload, + "themes": theme_payload, + "available_date_range": { + "from": iso_value(totals["first_award_date"]), + "to": iso_value(totals["latest_award_date"]), + }, + "applied_filters": filters, + "metadata": { + "data_mode": "postgresql_versioned_dashboard_aggregates" + }, + } + conditions, parameters, amount_minor, selected_currency = self._filtered_scope(filters) + valid = VALID_ORIGINAL if parameters["currency"] else VALID_EUR + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + kpis = ( + await session.execute( + text( + f""" + SELECT COUNT(*) AS grant_count, + COUNT(DISTINCT fact.funding_name) AS funder_count, + COUNT(DISTINCT fact.recipient_name) AS recipient_count, + SUM(CASE WHEN {valid} THEN {amount_minor} ELSE 0 END) / 100.0 + AS total_amount, + MIN(fact.award_date) AS date_from, + MAX(fact.award_date) AS date_to + FROM grant_overview_facts AS fact WHERE {conditions} + """ + ), + parameters, + ) + ).mappings().one() + map_payload = await self.map(**filters) + trend_payload = await self.trends(**filters) + theme_payload = await self.themes(currency=filters.get("currency"), **{ + key: value for key, value in filters.items() if key != "currency" + }) + grant_count = int(kpis["grant_count"] or 0) + awarded_funding = number_value(kpis["total_amount"]) or 0.0 + classification = theme_payload["classification_coverage"] + return { + "status": "available" if grant_count else "no_transactions_found", + "kpis": { + "awarded_funding": awarded_funding, + "grants_monitored": grant_count, + "country_coverage_percentage": map_payload["coverage_percentage"], + "mapped_grant_count": map_payload["known_geography_count"], + "unmapped_grant_count": map_payload["unknown_geography_count"], + "programme_coverage_percentage": classification["classified_percentage"], + "classified_grant_count": classification["classified_grant_count"], + "qualifying_programme_grant_count": classification["qualifying_grant_count"], + "grant_count": grant_count, + "funder_count": int(kpis["funder_count"] or 0), + "recipient_count": int(kpis["recipient_count"] or 0), + "total_amount": awarded_funding, + "currency": selected_currency, + }, + "map": map_payload, + "trends": trend_payload, + "themes": theme_payload, + "available_date_range": { + "from": iso_value(kpis["date_from"]), + "to": iso_value(kpis["date_to"]), + }, + "applied_filters": filters, + "metadata": {"data_mode": "postgresql_materialized_grant_facts"}, + } + + async def drilldown( + self, + *, + selection_type: str, + selection_value: str, + **filters: Any, + ) -> dict[str, Any]: + conditions, parameters, amount_minor, selected_currency = self._filtered_scope(filters) + valid = VALID_ORIGINAL if parameters["currency"] else VALID_EUR + parameters["selection_value"] = selection_value + if selection_type == "period": + conditions += " AND to_char(fact.award_date, 'YYYY-MM')=:selection_value" + elif selection_type == "programme_area": + conditions += ( + " AND EXISTS (SELECT 1 FROM grant_programme_categories AS selection_category " + "WHERE selection_category.dataset_version=fact.dataset_version " + "AND selection_category.grant_id=fact.grant_id " + "AND selection_category.programme_area=:selection_value)" + ) + else: + raise ValueError("Unsupported drill-down selection type") + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT fact.grant_id, fact.funding_name, fact.recipient_name, + fact.award_date, fact.currency, + CASE WHEN {valid} THEN {amount_minor} / 100.0 END AS amount + FROM grant_overview_facts AS fact WHERE {conditions} + ORDER BY amount DESC NULLS LAST, fact.grant_id LIMIT 250 + """ + ), + parameters, + ) + ).mappings() + ] + funders: dict[str, dict[str, Any]] = {} + recipients: dict[str, dict[str, Any]] = {} + for row in rows: + amount = number_value(row.get("amount")) or 0.0 + for target, name in ((funders, row.get("funding_name")), (recipients, row.get("recipient_name"))): + label = str(name or "Unknown") + item = target.setdefault(label, {"name": label, "grant_count": 0, "total_amount": 0.0}) + item["grant_count"] += 1 + item["total_amount"] += amount + ranked = lambda values: sorted( + values.values(), key=lambda item: (-item["total_amount"], item["name"]) + )[:25] + return { + "status": "available" if rows else "no_transactions_found", + "selection": {"type": selection_type, "value": selection_value, "label": selection_value}, + "summary": { + "grant_count": len(rows), + "total_amount": round(sum(number_value(row.get("amount")) or 0 for row in rows), 2), + "currency": selected_currency, + }, + "funders": ranked(funders), + "recipients": ranked(recipients), + "countries": [], + "grants": [ + { + "grant_id": row["grant_id"], + "funding_name": row.get("funding_name"), + "recipient_name": row.get("recipient_name"), + "award_date": iso_value(row.get("award_date")), + "amount": number_value(row.get("amount")), + "currency": selected_currency, + } + for row in rows + ], + "metadata": { + "data_mode": "postgresql_materialized_grant_facts", + "bounded_grant_limit": 250, + }, + } diff --git a/src/bff/postgres/audit_repository.py b/src/bff/postgres/audit_repository.py new file mode 100644 index 0000000..a1cdb55 --- /dev/null +++ b/src/bff/postgres/audit_repository.py @@ -0,0 +1,67 @@ +"""Append-only PostgreSQL security audit sink.""" + +from __future__ import annotations + +from datetime import datetime +import json +import uuid + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from bff.audit import AuditEvent +from bff.utils.logging import logger + + +class PostgresAuditSink: + def __init__(self, sessions: async_sessionmaker[AsyncSession]): + self._sessions = sessions + + async def record(self, event: AuditEvent) -> None: + outcome = { + "success": "succeeded", + "denied": "denied", + "failed": "failed", + }.get(event.result, "failed") + actor_role = event.actor_role if event.actor_role != "none" else "anonymous" + async with self._sessions() as session, session.begin(): + dataset_version = event.dataset_version or await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + await session.execute( + text( + """ + INSERT INTO audit_events ( + audit_event_id, request_id, actor_id, actor_role, + action, target, reason, outcome, http_status, + dataset_version, error_class, occurred_at, details + ) VALUES ( + :event_id, :request_id, :actor_id, :actor_role, + :action, :target, :reason, :outcome, :http_status, + :dataset_version, :error_class, :occurred_at, + CAST(:details AS jsonb) + ) + """ + ), + { + "event_id": uuid.uuid4(), + "request_id": event.request_id, + "actor_id": event.actor_id, + "actor_role": actor_role, + "action": event.action, + "target": event.target, + "reason": event.reason, + "outcome": outcome, + "http_status": event.http_status, + "dataset_version": dataset_version, + "error_class": event.error_class, + "occurred_at": datetime.fromisoformat(event.timestamp), + "details": json.dumps({"runtime": "postgresql"}, sort_keys=True), + }, + ) + logger.info( + "security_audit_persisted request_id=%s action=%s outcome=%s", + event.request_id, + event.action, + outcome, + ) diff --git a/src/bff/postgres/base.py b/src/bff/postgres/base.py new file mode 100644 index 0000000..65625b5 --- /dev/null +++ b/src/bff/postgres/base.py @@ -0,0 +1,140 @@ +"""Shared PostgreSQL repository primitives without domain query ownership.""" + +from __future__ import annotations + +import asyncio +from collections import OrderedDict +from copy import deepcopy +from datetime import date, datetime, timezone +from decimal import Decimal +from time import monotonic +from typing import Any, Awaitable, Callable, Mapping + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + + +class ActiveDatasetUnavailable(RuntimeError): + """Raised when a serving query has no approved active dataset.""" + + +class MaterializationUnavailable(RuntimeError): + """Raised when an active dataset lacks its validated serving aggregates.""" + + +class VersionedTTLCache: + """Small async single-flight cache whose keys start with dataset version.""" + + def __init__(self, *, ttl_seconds: float = 30.0, max_entries: int = 256): + self._ttl_seconds = ttl_seconds + self._max_entries = max_entries + self._entries: OrderedDict[tuple[Any, ...], tuple[float, Any]] = OrderedDict() + self._inflight: dict[tuple[Any, ...], asyncio.Task[Any]] = {} + self._lock = asyncio.Lock() + self.hits = 0 + self.misses = 0 + + async def get_or_create( + self, + key: tuple[Any, ...], + loader: Callable[[], Awaitable[Any]], + ) -> Any: + now = monotonic() + async with self._lock: + entry = self._entries.get(key) + if entry and entry[0] > now: + self.hits += 1 + self._entries.move_to_end(key) + return deepcopy(entry[1]) + if entry: + self._entries.pop(key, None) + task = self._inflight.get(key) + if task is None: + self.misses += 1 + task = asyncio.create_task(loader()) + self._inflight[key] = task + else: + self.hits += 1 + try: + value = await asyncio.shield(task) + finally: + async with self._lock: + if task.done() and self._inflight.get(key) is task: + self._inflight.pop(key, None) + if not task.cancelled() and task.exception() is None: + async with self._lock: + self._entries[key] = (monotonic() + self._ttl_seconds, deepcopy(value)) + self._entries.move_to_end(key) + while len(self._entries) > self._max_entries: + self._entries.popitem(last=False) + return deepcopy(value) + + async def retain_dataset(self, dataset_version: str) -> None: + async with self._lock: + stale = [key for key in self._entries if not key or key[0] != dataset_version] + for key in stale: + self._entries.pop(key, None) + + async def clear(self) -> None: + async with self._lock: + self._entries.clear() + self._inflight.clear() + self.hits = 0 + self.misses = 0 + + @property + def hit_ratio(self) -> float: + total = self.hits + self.misses + return self.hits / total if total else 0.0 + + +ANALYTICS_CACHE = VersionedTTLCache() + + +class PostgresRepository: + def __init__(self, sessions: async_sessionmaker[AsyncSession]): + self.sessions = sessions + + @staticmethod + async def active_dataset(session: AsyncSession) -> str: + dataset_version = await session.scalar( + text( + """ + SELECT dataset_version + FROM dataset_versions + WHERE is_active AND status='active' + """ + ) + ) + if not dataset_version: + raise ActiveDatasetUnavailable("No approved PostgreSQL dataset is active") + return str(dataset_version) + + +def json_value(value: Any, fallback: Any) -> Any: + return value if isinstance(value, type(fallback)) else fallback + + +def iso_value(value: Any) -> Any: + if isinstance(value, (date, datetime)): + return value.isoformat() + return value + + +def number_value(value: Any) -> float | None: + if value is None: + return None + if isinstance(value, (int, float, Decimal)): + return float(value) + return float(str(value)) + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def row_dict(row: Mapping[str, Any]) -> dict[str, Any]: + return { + key: iso_value(number_value(value) if isinstance(value, Decimal) else value) + for key, value in row.items() + } diff --git a/src/bff/postgres/funder_repository.py b/src/bff/postgres/funder_repository.py new file mode 100644 index 0000000..72d42cd --- /dev/null +++ b/src/bff/postgres/funder_repository.py @@ -0,0 +1,1137 @@ +"""Source-funder reads, explicit link overrides and profile-cache jobs.""" + +from __future__ import annotations + +import json +import math +from typing import Any, Optional, Sequence +import uuid + +from sqlalchemy import text + +from bff.postgres.base import PostgresRepository, iso_value, number_value, utc_now + + +class SourceFunderRepository(PostgresRepository): + @staticmethod + def _can_use_materialization(filters: dict[str, Any]) -> bool: + return not any( + filters.get(name) + for name in ( + "date_from", + "date_to", + "beneficiary_geographies", + "programme_areas", + "donor", + "recipient", + "sources", + ) + ) + + @staticmethod + def _item(row: dict[str, Any], rank: Optional[int] = None) -> dict[str, Any]: + linked_profile = row.get("effective_profile_id") + amount = number_value(row.get("selected_amount")) + selected_currency = row.get("selected_currency") + grant_count = int(row.get("grant_count") or 0) + recipient_count = int(row.get("recipient_count") or 0) + latest_award_date = iso_value(row.get("latest_award_date")) + included_count = int(row.get("included_count") or 0) + conversion_excluded = int(row.get("conversion_excluded") or 0) + multi_country_count = int(row.get("multi_country_count") or 0) + identity_method = str(row.get("identity_method") or "unknown") + source_funder_key = str(row["source_funder_key"]) + profile_name = str(row.get("profile_name") or row.get("display_name") or "Organization") + profile_link = ( + { + "status": "single", + "profile_id": int(linked_profile), + "profile_name": profile_name, + "method": row.get("override_mode") or "source_identifier", + } + if linked_profile is not None + else {"status": "none"} + ) + return { + "rank": rank, + "kind": "source_funder", + "identity": { + "key": source_funder_key, + "namespace": row.get("source_namespace"), + "method": identity_method, + "source_organization_id": row.get("source_organization_id"), + "normalized_name_fallback": row.get("normalized_name_fallback"), + }, + "source_funder_key": source_funder_key, + "display_name": str(row.get("display_name") or "Unknown source funder"), + "identity_method": identity_method, + "source_ids": list(row.get("source_ids") or []), + "sources": list(row.get("sources") or []), + "source_only": linked_profile is None, + "linked_directory_profile": ( + { + "charity_id": int(linked_profile), + "name": row.get("profile_name"), + } + if linked_profile is not None else None + ), + "profile_link": profile_link, + "evidence_sources": list(row.get("sources") or []), + "activity": { + "grant_count": grant_count, + "distinct_recipient_count": recipient_count, + "first_award_date": iso_value(row.get("first_award_date")), + "latest_award_date": latest_award_date, + }, + "observed_activity": { + "grant_count": grant_count, + "recipient_count": recipient_count, + "latest_grant_date": latest_award_date, + "observed_funding": amount, + "displayed_currency": selected_currency, + "programme_areas": list(row.get("leading_programme_areas") or []), + }, + "observed_funding": { + "amount": amount, + "currency": selected_currency, + "included_grant_count": included_count, + "excluded_multi_country_grant_count": multi_country_count, + "excluded_multi_country_amount": 0.0, + "excluded_conversion_grant_count": conversion_excluded, + "excluded_missing_amount_grant_count": int(row.get("missing_count") or 0), + "excluded_invalid_amount_grant_count": int(row.get("invalid_count") or 0), + "excluded_negative_amount_grant_count": int(row.get("negative_count") or 0), + "fallback_original_amount": number_value(row.get("fallback_amount")), + "fallback_original_currency": row.get("fallback_currency"), + "fallback_original_grant_count": int(row.get("fallback_count") or 0), + }, + "amount_policy": { + "mode": "automatic_eur" if selected_currency == "EUR" else "original_currency", + "converted_grant_count": included_count, + "unconverted_grant_count": conversion_excluded, + "multi_country_amount_excluded": 0.0, + "multi_country": "counted_once_per_selected_country_scope", + "negative": "excluded_and_reported", + }, + "leading_programme_areas": list(row.get("leading_programme_areas") or []), + "representative_source_url": row.get("publisher_source_url"), + } + + @staticmethod + def _summary(row: dict[str, Any], selected_currency: str) -> dict[str, Any]: + total_items = int(row.get("total_items") or 0) + linked_count = int(row.get("linked_count") or 0) + observed_only_count = max(0, total_items - linked_count) + return { + "matching_funder_count": total_items, + "matching_grant_count": int(row.get("matching_grant_count") or 0), + "unattributed_funder_grant_count": 0, + "distinct_recipient_count": int(row.get("recipient_count") or 0), + "source_only_funder_count": observed_only_count, + "linked_directory_funder_count": linked_count, + "status_counts": { + "all": total_items, + "linked": linked_count, + "observed_only": observed_only_count, + }, + "monetary": { + "currency_mode": ( + "auto_converted_eur" if selected_currency == "EUR" + else "source_currency" + ), + "display_currency": selected_currency, + "included_funding_total": number_value( + row.get("included_funding_total") + ), + "included_grant_count": int(row.get("included_count") or 0), + "excluded_multi_country_grant_count": int( + row.get("multi_country_count") or 0 + ), + "excluded_multi_country_amount": 0.0, + "excluded_conversion_grant_count": int( + row.get("conversion_excluded") or 0 + ), + "excluded_missing_amount_grant_count": int( + row.get("missing_count") or 0 + ), + "excluded_invalid_amount_grant_count": int( + row.get("invalid_count") or 0 + ), + "excluded_negative_amount_grant_count": int( + row.get("negative_count") or 0 + ), + }, + } + + @staticmethod + def _conditions(filters: dict[str, Any]) -> tuple[str, dict[str, Any], str, str]: + currency = str(filters.get("currency") or "").upper() or None + conditions = [ + "fact.dataset_version=:dataset_version", + "fact.country_code=:beneficiary_country", + ] + parameters: dict[str, Any] = { + "beneficiary_country": str(filters["beneficiary_country"]).upper(), + "currency": currency, + "date_from": filters.get("date_from"), + "date_to": filters.get("date_to"), + "beneficiary_geographies": list(filters.get("beneficiary_geographies") or []), + "programme_areas": list(filters.get("programme_areas") or []), + "donor": f"%{str(filters.get('donor') or '').strip()}%", + "recipient": f"%{str(filters.get('recipient') or '').strip()}%", + "sources": list(filters.get("sources") or []), + "search": f"%{str(filters.get('search') or '').strip()}%", + } + if currency: + conditions.append("fact.currency=:currency") + amount = "fact.original_amount_minor" + valid = "fact.original_amount_status NOT IN ('negative','invalid','missing')" + selected_currency = currency + else: + amount = "fact.eur_amount_minor" + valid = "fact.eur_amount_status NOT IN ('negative','missing','invalid')" + selected_currency = "EUR" + if parameters["date_from"]: + conditions.append("fact.award_date>=CAST(:date_from AS date)") + if parameters["date_to"]: + conditions.append("fact.award_date<=CAST(:date_to AS date)") + if parameters["beneficiary_geographies"]: + conditions.append( + "fact.country_name=ANY(CAST(:beneficiary_geographies AS text[]))" + ) + if parameters["programme_areas"]: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM grant_programme_categories AS category + WHERE category.dataset_version=fact.dataset_version + AND category.grant_id=fact.grant_id + AND category.programme_area=ANY(CAST(:programme_areas AS text[])) + ) + """ + ) + if filters.get("donor"): + conditions.append("fact.display_name ILIKE :donor") + if filters.get("recipient"): + conditions.append("fact.recipient_name ILIKE :recipient") + if parameters["sources"]: + conditions.append("fact.source_namespace=ANY(CAST(:sources AS text[]))") + if filters.get("search"): + conditions.append( + "(fact.display_name ILIKE :search OR fact.source_funder_key ILIKE :search)" + ) + return " AND ".join(conditions), parameters, amount, selected_currency + + async def list( + self, + *, + beneficiary_country: str, + currency: Optional[str] = None, + date_from: Optional[str] = None, + date_to: Optional[str] = None, + beneficiary_geographies: Optional[Sequence[str]] = None, + programme_areas: Optional[Sequence[str]] = None, + donor: Optional[str] = None, + recipient: Optional[str] = None, + sources: Optional[Sequence[str]] = None, + search: Optional[str] = None, + profile_status: str = "all", + sort: str = "largest_observed_funding", + page: int = 1, + page_size: int = 25, + ) -> dict[str, Any]: + if self._can_use_materialization(locals()): + return await self._materialized_list( + beneficiary_country=beneficiary_country, + currency=currency, + search=search, + profile_status=profile_status, + sort=sort, + page=page, + page_size=page_size, + ) + page = max(int(page), 1) + page_size = min(max(int(page_size), 1), 100) + filters = locals().copy() + conditions, parameters, amount, selected_currency = self._conditions(filters) + valid = ( + "fact.original_amount_status NOT IN ('negative','invalid','missing')" + if currency else "fact.eur_amount_status NOT IN ('negative','missing','invalid')" + ) + profile_condition = { + "all": "TRUE", + "linked": "effective_profile_id IS NOT NULL", + "observed_only": "effective_profile_id IS NULL", + }.get(profile_status) + if not profile_condition: + raise ValueError("Unsupported profile status") + order = { + "largest_observed_funding": "selected_amount DESC NULLS LAST, source_funder_key", + "most_grants": "grant_count DESC, source_funder_key", + "most_recently_active": "latest_award_date DESC NULLS LAST, source_funder_key", + "most_active": "grant_count DESC, latest_award_date DESC NULLS LAST, source_funder_key", + "most_recent": "latest_award_date DESC NULLS LAST, source_funder_key", + }.get(sort) + if not order: + raise ValueError("Unsupported source-funder sort") + parameters.update( + { + "limit": page_size, + "offset": (page - 1) * page_size, + "selected_currency": selected_currency, + } + ) + cte = f""" + WITH scoped AS ( + SELECT fact.*, + COALESCE( + NULLIF(fact.source_organization_id, ''), + 'source-funder-key:' || fact.source_funder_key + ) AS override_identifier + FROM grant_source_funder_facts AS fact + WHERE {conditions} + ), aggregated AS ( + SELECT source_funder_key, + MIN(source_namespace) AS source_namespace, + MIN(source_organization_id) AS source_organization_id, + MIN(display_name) AS display_name, + MIN(identity_method) AS identity_method, + array_agg(DISTINCT source_organization_id) + FILTER (WHERE source_organization_id IS NOT NULL) AS source_ids, + array_agg(DISTINCT source_namespace) AS sources, + MIN(country_code) AS country_code, + COUNT(DISTINCT grant_id) AS grant_count, + COUNT(DISTINCT recipient_key) AS recipient_count, + MIN(award_date) AS first_award_date, + MAX(award_date) AS latest_award_date, + SUM(CASE WHEN {valid} THEN {amount} ELSE 0 END) / 100.0 + AS selected_amount, + COUNT(*) FILTER (WHERE {valid}) AS included_count, + COUNT(*) FILTER (WHERE country_count>1) AS multi_country_count, + COUNT(*) FILTER ( + WHERE conversion_status IS NULL OR conversion_status='missing' + ) AS conversion_excluded, + COUNT(*) FILTER (WHERE original_amount_status='missing') AS missing_count, + COUNT(*) FILTER (WHERE original_amount_status='invalid') AS invalid_count, + COUNT(*) FILTER (WHERE original_amount_status='negative') AS negative_count, + SUM(original_amount_minor) FILTER ( + WHERE original_amount_status NOT IN ('negative','invalid','missing') + ) / 100.0 AS fallback_amount, + MIN(currency) AS fallback_currency, + COUNT(*) FILTER ( + WHERE original_amount_status NOT IN ('negative','invalid','missing') + ) AS fallback_count, + MAX(linked_profile_id) AS observed_profile_id, + MIN(publisher_source_url) AS publisher_source_url, + MIN(override_identifier) AS override_identifier + FROM scoped AS fact GROUP BY source_funder_key + ), effective AS ( + SELECT aggregated.*, + override.link_mode AS override_mode, + override.revision AS override_revision, + CASE + WHEN override.link_mode IN ('observed_only','unlink','blocked') THEN NULL + WHEN override.link_mode='link_profile' THEN override.target_profile_id + ELSE aggregated.observed_profile_id + END AS effective_profile_id, + :selected_currency AS selected_currency + FROM aggregated + LEFT JOIN source_funder_link_overrides AS override + ON override.source_namespace=aggregated.source_namespace + AND override.source_organization_id=aggregated.override_identifier + ) + """ + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + summary_row = dict( + ( + await session.execute( + text( + f""" + {cte} + SELECT COUNT(*) AS total_items, + COALESCE(SUM(grant_count), 0) AS matching_grant_count, + COALESCE(SUM(recipient_count), 0) AS recipient_count, + COUNT(*) FILTER ( + WHERE effective_profile_id IS NOT NULL + ) AS linked_count, + COALESCE(SUM(selected_amount), 0) + AS included_funding_total, + COALESCE(SUM(included_count), 0) AS included_count, + COALESCE(SUM(multi_country_count), 0) + AS multi_country_count, + COALESCE(SUM(conversion_excluded), 0) + AS conversion_excluded, + COALESCE(SUM(missing_count), 0) AS missing_count, + COALESCE(SUM(invalid_count), 0) AS invalid_count, + COALESCE(SUM(negative_count), 0) AS negative_count + FROM effective WHERE {profile_condition} + """ + ), + parameters, + ) + ).mappings().one() + ) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + {cte} + SELECT effective.*, profile.name AS profile_name + FROM effective + LEFT JOIN charities AS profile + ON profile.dataset_version=:dataset_version + AND profile.charity_id=effective.effective_profile_id + WHERE {profile_condition} + ORDER BY {order} LIMIT :limit OFFSET :offset + """ + ), + parameters, + ) + ).mappings() + ] + range_row = ( + await session.execute( + text( + f""" + SELECT MIN(fact.award_date), MAX(fact.award_date) + FROM grant_source_funder_facts AS fact WHERE {conditions} + """ + ), + parameters, + ) + ).one() + currency_rows = await session.execute( + text( + """ + SELECT DISTINCT currency FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version AND currency IS NOT NULL + ORDER BY currency + """ + ), + {"dataset_version": dataset_version}, + ) + country_name = await session.scalar( + text( + """ + SELECT MIN(country_name) FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version AND country_code=:country + """ + ), + {"dataset_version": dataset_version, "country": beneficiary_country.upper()}, + ) + items = [self._item(row, parameters["offset"] + index + 1) for index, row in enumerate(rows)] + summary = self._summary(summary_row, selected_currency) + total_items = summary["matching_funder_count"] + return { + "status": "available" if total_items else "no_transactions_found", + "country": { + "code": beneficiary_country.upper(), + "name": str(country_name or beneficiary_country.upper()), + }, + "summary": summary, + "items": items, + "pagination": { + "page": page, + "page_size": page_size, + "total_items": total_items, + "total_pages": math.ceil(total_items / page_size) if total_items else 0, + }, + "available_date_range": { + "from": iso_value(range_row[0]), + "to": iso_value(range_row[1]), + }, + "available_currencies": [str(row[0]) for row in currency_rows], + "available_sort_modes": [ + "largest_observed_funding", "most_grants", "most_recently_active", + "most_active", "most_recent", + ], + "applied_filters": { + "beneficiary_country": beneficiary_country.upper(), + "currency": currency, + "search": search, + "profile_status": profile_status, + "sort": sort, + }, + "metadata": { + "data_mode": "postgresql_source_funder_facts", + "identity_boundary": "source funders remain distinct from directory profiles", + }, + } + + async def _materialized_list( + self, + *, + beneficiary_country: str, + currency: Optional[str], + search: Optional[str], + profile_status: str, + sort: str, + page: int, + page_size: int, + ) -> dict[str, Any]: + page = max(int(page), 1) + page_size = min(max(int(page_size), 1), 100) + basis = "original" if currency else "eur_converted" + selected_currency = str(currency or "EUR").upper() + profile_condition = { + "all": "TRUE", + "linked": "effective_profile_id IS NOT NULL", + "observed_only": "effective_profile_id IS NULL", + }.get(profile_status) + if not profile_condition: + raise ValueError("Unsupported profile status") + order = { + "largest_observed_funding": ( + "selected_amount_minor DESC, source_funder_key" + ), + "most_grants": "grant_count DESC, source_funder_key", + "most_recently_active": ( + "latest_award_date DESC NULLS LAST, source_funder_key" + ), + "most_active": ( + "grant_count DESC, latest_award_date DESC NULLS LAST, " + "source_funder_key" + ), + "most_recent": ( + "latest_award_date DESC NULLS LAST, source_funder_key" + ), + }.get(sort) + if not order: + raise ValueError("Unsupported source-funder sort") + search_condition = ( + "AND (ranking.display_name ILIKE :search " + "OR ranking.source_funder_key ILIKE :search)" + if search else "" + ) + parameters = { + "beneficiary_country": beneficiary_country.upper(), + "basis": basis, + "currency": selected_currency, + "search": f"%{str(search or '').strip()}%", + "limit": page_size, + "offset": (page - 1) * page_size, + } + cte = f""" + WITH effective AS ( + SELECT ranking.*, + override.link_mode AS override_mode, + override.revision AS override_revision, + CASE + WHEN override.link_mode IN ('observed_only','unlink','blocked') + THEN NULL + WHEN override.link_mode='link_profile' + THEN override.target_profile_id + ELSE ranking.observed_profile_id + END AS effective_profile_id + FROM analytics_country_funder_rankings AS ranking + LEFT JOIN source_funder_link_overrides AS override + ON override.source_namespace=ranking.source_namespace + AND override.source_organization_id=ranking.override_identifier + WHERE ranking.dataset_version=:dataset_version + AND ranking.amount_basis=:basis AND ranking.currency=:currency + AND ranking.country_code=:beneficiary_country + {search_condition} + ) + """ + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + materialized = await session.scalar( + text( + """ + SELECT EXISTS ( + SELECT 1 FROM materialization_versions + WHERE dataset_version=:dataset_version + AND materialization_name='dashboard_analytics' + AND is_active AND status='active' + ) + """ + ), + parameters, + ) + if not materialized: + raise RuntimeError("Active dashboard materialization is unavailable") + summary_row = dict( + ( + await session.execute( + text( + f""" + {cte} + SELECT COUNT(*) AS total_items, + COALESCE(SUM(grant_count), 0) AS matching_grant_count, + COALESCE(SUM(recipient_count), 0) AS recipient_count, + COUNT(*) FILTER ( + WHERE effective_profile_id IS NOT NULL + ) AS linked_count, + COALESCE(SUM(selected_amount_minor), 0) / 100.0 + AS included_funding_total, + COALESCE(SUM(included_count), 0) AS included_count, + COALESCE(SUM(multi_country_count), 0) + AS multi_country_count, + COALESCE(SUM(conversion_excluded), 0) + AS conversion_excluded, + COALESCE(SUM(missing_count), 0) AS missing_count, + COALESCE(SUM(invalid_count), 0) AS invalid_count, + COALESCE(SUM(negative_count), 0) AS negative_count + FROM effective WHERE {profile_condition} + """ + ), + parameters, + ) + ).mappings().one() + ) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + {cte} + SELECT effective.*, + effective.selected_amount_minor / 100.0 + AS selected_amount, + effective.fallback_amount_minor / 100.0 + AS fallback_amount, + effective.currency AS selected_currency, + profile.name AS profile_name + FROM effective + LEFT JOIN charities AS profile + ON profile.dataset_version=:dataset_version + AND profile.charity_id=effective.effective_profile_id + WHERE {profile_condition} + ORDER BY {order} LIMIT :limit OFFSET :offset + """ + ), + parameters, + ) + ).mappings() + ] + range_row = ( + await session.execute( + text( + f""" + {cte} + SELECT MIN(first_award_date), MAX(latest_award_date) + FROM effective + """ + ), + parameters, + ) + ).one() + currency_rows = await session.execute( + text( + """ + SELECT value FROM analytics_filter_values + WHERE dataset_version=:dataset_version AND dimension='currency' + ORDER BY value + """ + ), + parameters, + ) + country_name = await session.scalar( + text( + """ + SELECT MIN(country_name) FROM analytics_country_funder_rankings + WHERE dataset_version=:dataset_version + AND country_code=:beneficiary_country + """ + ), + parameters, + ) + items = [ + self._item(row, parameters["offset"] + index + 1) + for index, row in enumerate(rows) + ] + summary = self._summary(summary_row, selected_currency) + total_items = summary["matching_funder_count"] + return { + "status": "available" if total_items else "no_transactions_found", + "country": { + "code": beneficiary_country.upper(), + "name": str(country_name or beneficiary_country.upper()), + }, + "summary": summary, + "items": items, + "pagination": { + "page": page, + "page_size": page_size, + "total_items": total_items, + "total_pages": math.ceil(total_items / page_size) if total_items else 0, + }, + "available_date_range": { + "from": iso_value(range_row[0]), + "to": iso_value(range_row[1]), + }, + "available_currencies": [str(row[0]) for row in currency_rows], + "available_sort_modes": [ + "largest_observed_funding", + "most_grants", + "most_recently_active", + "most_active", + "most_recent", + ], + "applied_filters": { + "beneficiary_country": beneficiary_country.upper(), + "currency": currency, + "search": search, + "profile_status": profile_status, + "sort": sort, + }, + "metadata": { + "data_mode": "postgresql_versioned_country_funder_rankings", + "identity_boundary": ( + "source funders remain distinct from directory profiles" + ), + }, + } + + async def detail( + self, + source_funder_key: str, + **filters: Any, + ) -> Optional[dict[str, Any]]: + key = str(source_funder_key or "").strip() + if not key or len(key) > 500: + raise ValueError("source_funder_key must be a non-empty canonical key") + response = await self.list( + beneficiary_country=filters["beneficiary_country"], + currency=filters.get("currency"), + date_from=filters.get("date_from"), + date_to=filters.get("date_to"), + beneficiary_geographies=filters.get("beneficiary_geographies"), + programme_areas=filters.get("programme_areas"), + donor=filters.get("donor"), + recipient=filters.get("recipient"), + sources=filters.get("sources"), + search=key, + profile_status="all", + page=1, + page_size=100, + ) + item = next((candidate for candidate in response["items"] if candidate["source_funder_key"] == key), None) + if not item: + return None + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + basis = "original" if filters.get("currency") else "eur_converted" + selected_currency = str(filters.get("currency") or "EUR").upper() + parameters = { + "dataset_version": dataset_version, + "key": key, + "country": str(filters["beneficiary_country"]).upper(), + "basis": basis, + "currency": selected_currency, + } + recipient_rows = await session.execute( + text( + """ + SELECT recipient_key, recipient_name, grant_count, + total_amount_minor / 100.0 AS amount + FROM analytics_funder_relationships + WHERE dataset_version=:dataset_version + AND amount_basis=:basis AND currency=:currency + AND source_funder_key=:key AND country_code=:country + ORDER BY rank_within_funder LIMIT 25 + """ + ), + parameters, + ) + grant_rows = await session.execute( + text( + """ + SELECT fact.grant_id, fact.recipient_name, fact.award_date, + fact.eur_amount_minor / 100.0 AS amount, 'EUR' AS currency, + fact.original_amount_minor / 100.0 AS original_amount, + fact.currency AS original_currency, fact.publisher_source_url, + grant_row.description + FROM grant_source_funder_facts AS fact + JOIN grants AS grant_row USING (dataset_version, grant_id) + WHERE fact.dataset_version=:dataset_version + AND fact.source_funder_key=:key AND fact.country_code=:country + ORDER BY fact.award_date DESC NULLS LAST, fact.grant_id LIMIT 50 + """ + ), + parameters, + ) + recipients = [ + { + "recipient_key": str(row[0]), + "name": str(row[1]), + "grant_count": int(row[2]), + "observed_funding": number_value(row[3]), + "amount": number_value(row[3]), + "currency": selected_currency, + "latest_award_date": None, + } + for row in recipient_rows + ] + funder_node = f"funder:{key}" + nodes = [{"id": funder_node, "label": item["display_name"], "role": "funder"}] + links = [] + for index, recipient in enumerate(recipients): + node_id = f"recipient:{index}" + nodes.append({"id": node_id, "label": recipient["name"], "role": "recipient"}) + links.append( + { + "source": funder_node, + "target": node_id, + "value": float(recipient["amount"] or 0), + "currency": selected_currency, + "grant_count": recipient["grant_count"], + } + ) + sample = [ + { + "grant_id": str(row[0]), + "recipient_name": str(row[1]), + "award_date": iso_value(row[2]), + "amount": number_value(row[3]), + "currency": row[4], + "original_amount": number_value(row[5]), + "original_currency": row[6], + "source_url": row[7], + "description": row[8], + "evidence_links": ( + [{ + "kind": "grant_source", + "label": "Publisher source record", + "role": "funder", + "organization_name": item["display_name"], + "link_type": "website", + "url": row[7], + "origin": "publisher_source_url", + }] if row[7] else [] + ), + } + for row in grant_rows + ] + return { + "status": "available", + "country": response["country"], + "funder": item, + "top_recipients": recipients, + "relationships": { + "status": "available" if links else "no_transactions_found", + "nodes": nodes, + "links": links, + "metadata": {"data_mode": "postgresql_source_funder_facts"}, + }, + "grant_sample": sample, + "source_evidence": [], + "relationship_summary": { + "recipient_count": len(recipients), + "sample_grant_count": len(sample), + }, + "metadata": {"detail_level": filters.get("detail_level", "full")}, + } + + async def _identity(self, session, key: str) -> Optional[dict[str, Any]]: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT source_namespace, + COALESCE(NULLIF(source_organization_id, ''), + 'source-funder-key:' || source_funder_key) + AS override_identifier, + MAX(linked_profile_id) OVER () AS linked_profile_id, + dataset_version + FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version AND source_funder_key=:key + ORDER BY grant_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version, "key": key}, + ) + ).mappings().first() + return dict(row) if row else None + + async def reset( + self, source_funder_key: str, *, actor_id: str + ) -> Optional[dict[str, Any]]: + key = str(source_funder_key or "").strip() + async with self.sessions() as session, session.begin(): + identity = await self._identity(session, key) + if not identity: + return None + revision = await session.scalar( + text( + """ + SELECT revision FROM source_funder_link_overrides + WHERE source_namespace=:namespace AND source_organization_id=:identifier + FOR UPDATE + """ + ), + {"namespace": identity["source_namespace"], "identifier": identity["override_identifier"]}, + ) + new_revision = int(revision) + 1 if revision is not None else 0 + await session.execute( + text( + """ + INSERT INTO source_funder_link_overrides ( + source_namespace, source_organization_id, link_mode, + target_profile_id, target_dataset_version, reason, + updated_by, updated_at, revision + ) VALUES ( + :namespace, :identifier, 'observed_only', NULL, NULL, + 'explicit administrator reset', :actor_id, + CURRENT_TIMESTAMP, :revision + ) + ON CONFLICT (source_namespace, source_organization_id) + DO UPDATE SET link_mode='observed_only', target_profile_id=NULL, + target_dataset_version=NULL, + reason='explicit administrator reset', updated_by=:actor_id, + updated_at=CURRENT_TIMESTAMP, revision=:revision + """ + ), + { + "namespace": identity["source_namespace"], + "identifier": identity["override_identifier"], + "actor_id": actor_id, + "revision": new_revision, + }, + ) + await session.execute( + text( + """ + DELETE FROM source_funder_profile_cache + WHERE dataset_version=:dataset_version AND source_funder_key=:key + """ + ), + {"dataset_version": identity["dataset_version"], "key": key}, + ) + return {"source_funder_key": key, "revision": new_revision} + + async def relink( + self, + source_funder_key: str, + profile_id: int, + *, + actor_id: str, + ) -> Optional[dict[str, Any]]: + key = str(source_funder_key or "").strip() + async with self.sessions() as session, session.begin(): + identity = await self._identity(session, key) + if not identity: + return None + profile_name = await session.scalar( + text( + """ + SELECT name FROM charities + WHERE dataset_version=:dataset_version AND charity_id=:profile_id + """ + ), + {"dataset_version": identity["dataset_version"], "profile_id": profile_id}, + ) + if profile_name is None: + raise ValueError("The requested active-dataset profile does not exist") + revision = await session.scalar( + text( + """ + SELECT revision FROM source_funder_link_overrides + WHERE source_namespace=:namespace AND source_organization_id=:identifier + FOR UPDATE + """ + ), + {"namespace": identity["source_namespace"], "identifier": identity["override_identifier"]}, + ) + new_revision = int(revision) + 1 if revision is not None else 0 + await session.execute( + text( + """ + INSERT INTO source_funder_link_overrides ( + source_namespace, source_organization_id, link_mode, + target_profile_id, target_dataset_version, reason, + updated_by, updated_at, revision + ) VALUES ( + :namespace, :identifier, 'link_profile', :profile_id, + :dataset_version, 'explicit administrator relink', + :actor_id, CURRENT_TIMESTAMP, :revision + ) + ON CONFLICT (source_namespace, source_organization_id) + DO UPDATE SET link_mode='link_profile', target_profile_id=:profile_id, + target_dataset_version=:dataset_version, + reason='explicit administrator relink', updated_by=:actor_id, + updated_at=CURRENT_TIMESTAMP, revision=:revision + """ + ), + { + "namespace": identity["source_namespace"], + "identifier": identity["override_identifier"], + "profile_id": profile_id, + "dataset_version": identity["dataset_version"], + "actor_id": actor_id, + "revision": new_revision, + }, + ) + await session.execute( + text( + """ + DELETE FROM source_funder_profile_cache + WHERE dataset_version=:dataset_version AND source_funder_key=:key + """ + ), + {"dataset_version": identity["dataset_version"], "key": key}, + ) + return { + "source_funder_key": key, + "profile_id": profile_id, + "profile_name": str(profile_name), + "revision": new_revision, + } + + async def _effective_profile(self, session, key: str) -> Optional[dict[str, Any]]: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + WITH identity AS ( + SELECT source_namespace, + COALESCE(NULLIF(source_organization_id, ''), + 'source-funder-key:' || source_funder_key) + AS override_identifier, + MAX(linked_profile_id) OVER () AS observed_profile_id + FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version AND source_funder_key=:key + ORDER BY grant_id LIMIT 1 + ) + SELECT CASE + WHEN override.link_mode IN ('observed_only','unlink','blocked') + THEN NULL + WHEN override.link_mode='link_profile' + THEN override.target_profile_id + ELSE identity.observed_profile_id + END AS profile_id, + COALESCE(override.revision, 0) AS link_revision + FROM identity + LEFT JOIN source_funder_link_overrides AS override + ON override.source_namespace=identity.source_namespace + AND override.source_organization_id=identity.override_identifier + """ + ), + {"dataset_version": dataset_version, "key": key}, + ) + ).mappings().first() + if not row or row["profile_id"] is None: + return None + return { + "dataset_version": dataset_version, + "profile_id": int(row["profile_id"]), + "link_revision": int(row["link_revision"]), + } + + async def queue_profile_cache( + self, + source_funder_key: str, + *, + actor_id: str, + idempotency_key: str, + ) -> Optional[dict[str, Any]]: + key = str(source_funder_key or "").strip() + async with self.sessions() as session, session.begin(): + linked = await self._effective_profile(session, key) + if not linked: + return None + job_id = uuid.uuid4() + existing = ( + await session.execute( + text( + """ + SELECT job_run_id, status FROM job_runs + WHERE job_type='source_funder_profile_hydration' + AND idempotency_key=:idempotency_key + """ + ), + {"idempotency_key": idempotency_key}, + ) + ).mappings().first() + if existing: + job_id = existing["job_run_id"] + else: + await session.execute( + text( + """ + INSERT INTO job_runs ( + job_run_id, job_type, status, dataset_version, + idempotency_key, requested_by, input + ) VALUES ( + :job_id, 'source_funder_profile_hydration', 'queued', + :dataset_version, :idempotency_key, :actor_id, + CAST(:input AS jsonb) + ) + """ + ), + { + "job_id": job_id, + "dataset_version": linked["dataset_version"], + "idempotency_key": idempotency_key, + "actor_id": actor_id, + "input": json.dumps({"source_funder_key": key}, sort_keys=True), + }, + ) + await session.execute( + text( + """ + INSERT INTO source_funder_profile_cache ( + dataset_version, source_funder_key, profile_id, status, + payload, error, updated_at, job_token, link_revision + ) VALUES ( + :dataset_version, :key, :profile_id, 'pending', NULL, NULL, + CURRENT_TIMESTAMP, :job_id, :link_revision + ) + ON CONFLICT (dataset_version, source_funder_key) + DO UPDATE SET profile_id=:profile_id, status='pending', payload=NULL, + error=NULL, updated_at=CURRENT_TIMESTAMP, job_token=:job_id, + link_revision=:link_revision + """ + ), + { + **linked, + "key": key, + "job_id": job_id, + }, + ) + return { + "source_funder_key": key, + "profile_id": linked["profile_id"], + "status": "pending", + "job_id": str(job_id), + "updated_at": utc_now(), + "link_revision": linked["link_revision"], + } + + async def profile_cache(self, source_funder_key: str) -> Optional[dict[str, Any]]: + key = str(source_funder_key or "").strip() + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT profile_id, status, payload, error, updated_at, + job_token, link_revision + FROM source_funder_profile_cache + WHERE dataset_version=:dataset_version AND source_funder_key=:key + """ + ), + {"dataset_version": dataset_version, "key": key}, + ) + ).mappings().first() + if not row: + return None + return { + "source_funder_key": key, + "profile_id": int(row["profile_id"]), + "status": str(row["status"]), + "payload": row["payload"] if row["status"] == "ready" else None, + "error": row["error"], + "updated_at": iso_value(row["updated_at"]), + "job_id": str(row["job_token"]) if row["job_token"] else None, + "link_revision": int(row["link_revision"]), + } diff --git a/src/bff/postgres/governance_repository.py b/src/bff/postgres/governance_repository.py new file mode 100644 index 0000000..d4fccf9 --- /dev/null +++ b/src/bff/postgres/governance_repository.py @@ -0,0 +1,380 @@ +"""PostgreSQL governance, holds and non-destructive retention evidence.""" + +from __future__ import annotations + +from datetime import datetime +import json +from typing import Any, Iterable, Mapping +import uuid + +from sqlalchemy import text + +from bff.postgres.base import PostgresRepository, iso_value +from governance.retention import ( + ClassificationPolicy, + DataHold, + GovernanceConfiguration, + RetentionPlanEntry, +) + + +class GovernanceRepository(PostgresRepository): + async def synchronize_policies(self, configuration: GovernanceConfiguration) -> int: + configuration.validate() + records = [ + policy.database_record(policy_status=configuration.policy_status) + for policy in configuration.policies + ] + async with self.sessions() as session, session.begin(): + for record in records: + await session.execute( + text( + """ + INSERT INTO retention_policies ( + retention_class, classification, policy_status, + archive_after_days, delete_after_days, + export_expire_after_days, + destructive_deletion_enabled, production_approved, + configuration_checksum + ) VALUES ( + :retention_class, :classification, :policy_status, + :archive_after_days, :delete_after_days, + :export_expire_after_days, + :destructive_deletion_enabled, :production_approved, + :configuration_checksum + ) + ON CONFLICT (retention_class) DO UPDATE SET + classification=EXCLUDED.classification, + policy_status=EXCLUDED.policy_status, + archive_after_days=EXCLUDED.archive_after_days, + delete_after_days=EXCLUDED.delete_after_days, + export_expire_after_days=EXCLUDED.export_expire_after_days, + destructive_deletion_enabled=EXCLUDED.destructive_deletion_enabled, + production_approved=EXCLUDED.production_approved, + configuration_checksum=EXCLUDED.configuration_checksum, + updated_at=CURRENT_TIMESTAMP + """ + ), + record, + ) + return len(records) + + async def policies(self) -> list[dict[str, Any]]: + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT retention_class, classification, policy_status, + archive_after_days, delete_after_days, + export_expire_after_days, + destructive_deletion_enabled, production_approved, + approval_reference, configuration_checksum, updated_at + FROM retention_policies ORDER BY retention_class + """ + ) + ) + ).mappings() + return [ + {**dict(row), "updated_at": iso_value(row["updated_at"])} + for row in rows + ] + + async def create_hold( + self, + *, + hold_type: str, + scope_type: str, + scope_id: str, + reason: str, + created_by: str, + expires_at: datetime | None = None, + ) -> dict[str, Any]: + if hold_type not in {"legal", "incident"}: + raise ValueError("Hold type must be legal or incident") + if any(not str(value).strip() for value in (scope_type, scope_id, reason, created_by)): + raise ValueError("Hold scope, reason and actor are required") + hold_id = uuid.uuid4() + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + INSERT INTO data_holds ( + data_hold_id, hold_type, scope_type, scope_id, + reason, created_by, expires_at + ) VALUES ( + :hold_id, :hold_type, :scope_type, :scope_id, + :reason, :created_by, :expires_at + ) + RETURNING data_hold_id, hold_type, scope_type, scope_id, + reason, status, created_by, created_at, expires_at + """ + ), + { + "hold_id": hold_id, + "hold_type": hold_type, + "scope_type": scope_type, + "scope_id": scope_id, + "reason": reason, + "created_by": created_by, + "expires_at": expires_at, + }, + ) + ).mappings().one() + return self._hold_row(row) + + async def release_hold( + self, + hold_id: str, + *, + released_by: str, + release_reason: str, + ) -> dict[str, Any]: + if not released_by.strip() or not release_reason.strip(): + raise ValueError("Hold release actor and reason are required") + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + UPDATE data_holds + SET status='released', released_by=:released_by, + released_at=CURRENT_TIMESTAMP, + release_reason=:release_reason + WHERE data_hold_id=:hold_id AND status='active' + RETURNING data_hold_id, hold_type, scope_type, scope_id, + reason, status, created_by, created_at, expires_at, + released_by, released_at, release_reason + """ + ), + { + "hold_id": uuid.UUID(str(hold_id)), + "released_by": released_by, + "release_reason": release_reason, + }, + ) + ).mappings().first() + if not row: + raise ValueError("Active hold does not exist") + return self._hold_row(row) + + async def active_holds(self) -> tuple[DataHold, ...]: + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT data_hold_id, hold_type, scope_type, scope_id, + status, expires_at + FROM data_holds + WHERE status='active' + AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP) + ORDER BY created_at, data_hold_id + """ + ) + ) + ).mappings() + return tuple( + DataHold( + hold_id=str(row["data_hold_id"]), + hold_type=str(row["hold_type"]), + scope_type=str(row["scope_type"]), + scope_id=str(row["scope_id"]), + status=str(row["status"]), + expires_at=row["expires_at"], + ) + for row in rows + ) + + async def record_retention_plan( + self, + entries: Iterable[RetentionPlanEntry], + *, + requested_by: str, + ) -> list[str]: + action_ids: list[str] = [] + async with self.sessions() as session, session.begin(): + for entry in entries: + action_id = uuid.uuid4() + manifest_id = uuid.uuid4() + hold_ids = [uuid.UUID(value) for value in entry.hold_ids] + manifest = entry.manifest + await session.execute( + text( + """ + INSERT INTO retention_actions ( + retention_action_id, target_type, target_id, status, + dry_run, legal_hold, reason, requested_by, manifest, + retention_class, action_type, hold_ids, + manifest_checksum, production_approved + ) VALUES ( + :action_id, :target_type, :target_id, :status, + TRUE, :has_hold, :reason, :requested_by, + CAST(:manifest AS jsonb), :retention_class, + :action_type, :hold_ids, :manifest_checksum, FALSE + ) + """ + ), + { + "action_id": action_id, + "target_type": entry.target_type, + "target_id": entry.target_id, + "status": entry.status, + "has_hold": bool(hold_ids), + "reason": entry.reason, + "requested_by": requested_by, + "manifest": json.dumps(manifest, sort_keys=True, default=str), + "retention_class": entry.retention_class, + "action_type": entry.action_type, + "hold_ids": hold_ids, + "manifest_checksum": entry.manifest_checksum, + }, + ) + await session.execute( + text( + """ + INSERT INTO deletion_manifests ( + deletion_manifest_id, retention_action_id, + target_type, target_id, retention_class, + action_type, dry_run, object_count, record_count, + total_bytes, target_checksums, hold_ids, + manifest_checksum, manifest + ) VALUES ( + :manifest_id, :action_id, :target_type, :target_id, + :retention_class, :action_type, TRUE, :object_count, + :record_count, :total_bytes, + CAST(:target_checksums AS jsonb), :hold_ids, + :manifest_checksum, CAST(:manifest AS jsonb) + ) + """ + ), + { + "manifest_id": manifest_id, + "action_id": action_id, + "target_type": entry.target_type, + "target_id": entry.target_id, + "retention_class": entry.retention_class, + "action_type": entry.action_type, + "object_count": entry.object_count, + "record_count": entry.record_count, + "total_bytes": entry.total_bytes, + "target_checksums": json.dumps(entry.target_checksums), + "hold_ids": hold_ids, + "manifest_checksum": entry.manifest_checksum, + "manifest": json.dumps(manifest, sort_keys=True, default=str), + }, + ) + action_ids.append(str(action_id)) + return action_ids + + async def record_restore_verification( + self, + *, + target_type: str, + target_id: str, + backup_reference: str, + backup_checksum: str, + verification_status: str, + verified_by: str, + evidence: Mapping[str, Any], + ) -> str: + if verification_status not in {"passed", "failed"}: + raise ValueError("Restore verification status must be passed or failed") + verification_id = uuid.uuid4() + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + INSERT INTO restore_verifications ( + restore_verification_id, target_type, target_id, + backup_reference, backup_checksum, verification_status, + verified_by, evidence + ) VALUES ( + :verification_id, :target_type, :target_id, + :backup_reference, :backup_checksum, :verification_status, + :verified_by, CAST(:evidence AS jsonb) + ) + """ + ), + { + "verification_id": verification_id, + "target_type": target_type, + "target_id": target_id, + "backup_reference": backup_reference, + "backup_checksum": backup_checksum, + "verification_status": verification_status, + "verified_by": verified_by, + "evidence": json.dumps(evidence, sort_keys=True, default=str), + }, + ) + return str(verification_id) + + async def expired_exports(self) -> list[dict[str, Any]]: + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT export_job_id, dataset_version, export_type, status, + requested_at, completed_at, object_key, checksum, + row_count, expires_at, hold_until, retention_class + FROM export_jobs + WHERE expires_at <= CURRENT_TIMESTAMP + AND (hold_until IS NULL OR hold_until <= CURRENT_TIMESTAMP) + AND status='succeeded' + ORDER BY expires_at, export_job_id + """ + ) + ) + ).mappings() + return [ + { + **dict(row), + "export_job_id": str(row["export_job_id"]), + "requested_at": iso_value(row["requested_at"]), + "completed_at": iso_value(row["completed_at"]), + "expires_at": iso_value(row["expires_at"]), + "hold_until": iso_value(row["hold_until"]), + } + for row in rows + ] + + async def create_data_subject_request( + self, + *, + request_type: str, + subject_reference_hash: str, + due_at: datetime | None, + ) -> str: + request_id = uuid.uuid4() + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + INSERT INTO data_subject_requests ( + data_subject_request_id, request_type, + subject_reference_hash, status, due_at + ) VALUES ( + :request_id, :request_type, :subject_reference_hash, + 'identity_pending', :due_at + ) + """ + ), + { + "request_id": request_id, + "request_type": request_type, + "subject_reference_hash": subject_reference_hash, + "due_at": due_at, + }, + ) + return str(request_id) + + @staticmethod + def _hold_row(row: Mapping[str, Any]) -> dict[str, Any]: + result = dict(row) + result["data_hold_id"] = str(result["data_hold_id"]) + for field in ("created_at", "expires_at", "released_at"): + if field in result: + result[field] = iso_value(result[field]) + return result diff --git a/src/bff/postgres/governance_routes.py b/src/bff/postgres/governance_routes.py new file mode 100644 index 0000000..b56fa19 --- /dev/null +++ b/src/bff/postgres/governance_routes.py @@ -0,0 +1,202 @@ +"""Administrator-only governance and non-destructive retention routes.""" + +from __future__ import annotations + +from dataclasses import asdict + +from fastapi import APIRouter, Depends, HTTPException, Request + +from bff.postgres.governance_repository import GovernanceRepository +from bff.schemas import ( + DataHoldRelease, + DataHoldRequest, + DataSubjectRequestCreate, + RetentionDryRunRequest, +) +from bff.security import Role, require_roles +from governance.retention import ( + RetentionCandidate, + RetentionPlanner, + load_governance_configuration, +) + + +router = APIRouter( + prefix="/api/admin/governance", + tags=["Governance & Retention"], + dependencies=[ + Depends(require_roles(Role.ADMINISTRATOR, action="governance.access")) + ], +) +GOVERNANCE_CONFIGURATION = load_governance_configuration() +RETENTION_PLANNER = RetentionPlanner(GOVERNANCE_CONFIGURATION) + + +def _repository(request: Request) -> GovernanceRepository: + return GovernanceRepository(request.app.state.database.sessions()) + + +def _actor(request: Request) -> str: + principal = getattr(request.state, "principal", None) + return principal.actor_id if principal else "unknown" + + +@router.get("/retention/policies") +async def retention_policies( + repository: GovernanceRepository = Depends(_repository), +): + return { + "destructive_deletion_enabled": False, + "production_activation_approved": False, + "policies": await repository.policies(), + } + + +@router.post( + "/retention/dry-run", + dependencies=[ + Depends( + require_roles( + Role.ADMINISTRATOR, + action="retention.dry_run", + idempotent=True, + ) + ) + ], +) +async def retention_dry_run( + payload: RetentionDryRunRequest, + request: Request, + repository: GovernanceRepository = Depends(_repository), +): + candidate = RetentionCandidate( + target_type=payload.target_type, + target_id=payload.target_id, + retention_class=payload.retention_class, + last_modified_at=payload.last_modified_at, + object_count=payload.object_count, + record_count=payload.record_count, + total_bytes=payload.total_bytes, + target_checksums=tuple(payload.target_checksums), + ) + try: + entries = RETENTION_PLANNER.plan( + [candidate], await repository.active_holds() + ) + action_ids = await repository.record_retention_plan( + entries, requested_by=_actor(request) + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return { + "dry_run": True, + "destructive_deletion_enabled": False, + "actions": [ + { + **entry.manifest, + "manifest_checksum": entry.manifest_checksum, + "retention_action_id": action_id, + } + for entry, action_id in zip(entries, action_ids) + ], + } + + +@router.get("/holds") +async def active_holds( + repository: GovernanceRepository = Depends(_repository), +): + return {"holds": [asdict(hold) for hold in await repository.active_holds()]} + + +@router.post( + "/holds", + dependencies=[ + Depends( + require_roles( + Role.ADMINISTRATOR, + action="governance.hold.create", + idempotent=True, + ) + ) + ], +) +async def create_hold( + payload: DataHoldRequest, + request: Request, + repository: GovernanceRepository = Depends(_repository), +): + try: + return await repository.create_hold( + hold_type=payload.hold_type, + scope_type=payload.scope_type, + scope_id=payload.scope_id, + reason=payload.reason, + created_by=_actor(request), + expires_at=payload.expires_at, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.post( + "/holds/{hold_id}/release", + dependencies=[ + Depends( + require_roles( + Role.ADMINISTRATOR, + action="governance.hold.release", + idempotent=True, + ) + ) + ], +) +async def release_hold( + hold_id: str, + payload: DataHoldRelease, + request: Request, + repository: GovernanceRepository = Depends(_repository), +): + try: + return await repository.release_hold( + hold_id, + released_by=_actor(request), + release_reason=payload.reason, + ) + except (ValueError, TypeError) as exc: + raise HTTPException(status_code=404, detail="Active hold does not exist") from exc + + +@router.get("/exports/expiration-report") +async def export_expiration_report( + repository: GovernanceRepository = Depends(_repository), +): + return { + "dry_run": True, + "destructive_deletion_enabled": False, + "exports": await repository.expired_exports(), + } + + +@router.post( + "/data-subject-requests", + dependencies=[ + Depends( + require_roles( + Role.ADMINISTRATOR, + action="governance.data_subject_request.create", + idempotent=True, + ) + ) + ], +) +async def create_data_subject_request( + payload: DataSubjectRequestCreate, + repository: GovernanceRepository = Depends(_repository), +): + request_id = await repository.create_data_subject_request( + request_type=payload.request_type, + subject_reference_hash=payload.subject_reference_hash, + due_at=payload.due_at, + ) + return {"data_subject_request_id": request_id, "status": "identity_pending"} diff --git a/src/bff/postgres/idempotency_repository.py b/src/bff/postgres/idempotency_repository.py new file mode 100644 index 0000000..0a2f9d8 --- /dev/null +++ b/src/bff/postgres/idempotency_repository.py @@ -0,0 +1,120 @@ +"""PostgreSQL-backed request idempotency for horizontally scaled runtimes.""" + +from __future__ import annotations + +from datetime import timedelta +from typing import Tuple + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from bff.security import IdempotencyConflict + + +RecordKey = Tuple[str, str, str] + + +class PostgresIdempotencyStore: + def __init__( + self, + sessions: async_sessionmaker[AsyncSession], + ttl_seconds: int = 86_400, + ) -> None: + self.sessions = sessions + self.ttl_seconds = int(timedelta(seconds=ttl_seconds).total_seconds()) + + async def reserve( + self, + actor_id: str, + action: str, + key: str, + fingerprint: str, + ) -> RecordKey: + record_key = (actor_id, action, key) + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + DELETE FROM idempotency_records + WHERE actor_id=:actor_id AND action=:action + AND idempotency_key=:idempotency_key + AND expires_at <= CURRENT_TIMESTAMP + """ + ), + { + "actor_id": actor_id, + "action": action, + "idempotency_key": key, + }, + ) + inserted = await session.scalar( + text( + """ + INSERT INTO idempotency_records ( + actor_id, action, idempotency_key, request_hash, + status, expires_at + ) VALUES ( + :actor_id, :action, :idempotency_key, :request_hash, + 'reserved', CURRENT_TIMESTAMP + + CAST(:ttl_seconds AS integer) * INTERVAL '1 second' + ) + ON CONFLICT (actor_id, action, idempotency_key) DO NOTHING + RETURNING request_hash + """ + ), + { + "actor_id": actor_id, + "action": action, + "idempotency_key": key, + "request_hash": fingerprint, + "ttl_seconds": self.ttl_seconds, + }, + ) + if inserted is None: + existing = await session.scalar( + text( + """ + SELECT request_hash FROM idempotency_records + WHERE actor_id=:actor_id AND action=:action + AND idempotency_key=:idempotency_key + """ + ), + { + "actor_id": actor_id, + "action": action, + "idempotency_key": key, + }, + ) + raise IdempotencyConflict(different_request=existing != fingerprint) + return record_key + + async def complete(self, record_key: RecordKey) -> None: + actor_id, action, key = record_key + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + UPDATE idempotency_records + SET status='completed', completed_at=CURRENT_TIMESTAMP + WHERE actor_id=:actor_id AND action=:action + AND idempotency_key=:idempotency_key + AND status='reserved' + """ + ), + {"actor_id": actor_id, "action": action, "idempotency_key": key}, + ) + + async def release(self, record_key: RecordKey) -> None: + actor_id, action, key = record_key + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + DELETE FROM idempotency_records + WHERE actor_id=:actor_id AND action=:action + AND idempotency_key=:idempotency_key + AND status='reserved' + """ + ), + {"actor_id": actor_id, "action": action, "idempotency_key": key}, + ) diff --git a/src/bff/postgres/interfaces.py b/src/bff/postgres/interfaces.py new file mode 100644 index 0000000..c7fe076 --- /dev/null +++ b/src/bff/postgres/interfaces.py @@ -0,0 +1,66 @@ +"""Domain-sized repository interfaces for the PostgreSQL application runtime.""" + +from __future__ import annotations + +from typing import Any, Mapping, Optional, Protocol, Sequence + + +class OrganizationReader(Protocol): + async def list(self, **filters: Any) -> list[dict[str, Any]]: ... + async def detail(self, organization_id: int) -> Optional[dict[str, Any]]: ... + async def stats(self) -> dict[str, Any]: ... + async def grants(self, organization_id: int, role: str) -> dict[str, Any]: ... + async def sankey( + self, organization_id: int, *, currency: Optional[str], limit: int + ) -> dict[str, Any]: ... + async def score( + self, organization_id: int, target_profile: Optional[Mapping[str, Any]] + ) -> dict[str, Any]: ... + + +class RegistryReader(Protocol): + async def page(self, **filters: Any) -> dict[str, Any]: ... + async def detail(self, registry_id: str) -> Optional[dict[str, Any]]: ... + + +class GrantAnalyticsReader(Protocol): + async def beneficiary_geographies(self) -> list[str]: ... + async def map(self, **filters: Any) -> dict[str, Any]: ... + async def map_connections( + self, *, currency: Optional[str], limit: int + ) -> dict[str, Any]: ... + async def overview(self, **filters: Any) -> dict[str, Any]: ... + async def suggestions( + self, *, sources: Optional[Sequence[str]], limit: int + ) -> dict[str, Any]: ... + async def trends(self, **filters: Any) -> dict[str, Any]: ... + async def drilldown(self, **filters: Any) -> dict[str, Any]: ... + async def summary(self) -> dict[str, Any]: ... + async def themes(self, *, currency: Optional[str]) -> dict[str, Any]: ... + + +class FunderRepository(Protocol): + async def list(self, **filters: Any) -> dict[str, Any]: ... + async def detail(self, source_funder_key: str, **filters: Any) -> Optional[dict[str, Any]]: ... + async def reset(self, source_funder_key: str, *, actor_id: str) -> Optional[dict[str, Any]]: ... + async def relink( + self, source_funder_key: str, profile_id: int, *, actor_id: str + ) -> Optional[dict[str, Any]]: ... + async def queue_profile_cache( + self, source_funder_key: str, *, actor_id: str, idempotency_key: str + ) -> Optional[dict[str, Any]]: ... + async def profile_cache(self, source_funder_key: str) -> Optional[dict[str, Any]]: ... + + +class JobRepository(Protocol): + async def enqueue( + self, + job_type: str, + payload: Mapping[str, Any], + *, + actor_id: str, + idempotency_key: str, + ) -> dict[str, Any]: ... + async def latest_status(self) -> dict[str, Any]: ... + async def history(self, *, limit: int) -> list[dict[str, Any]]: ... + async def events(self, *, limit: int) -> list[dict[str, Any]]: ... diff --git a/src/bff/postgres/job_repository.py b/src/bff/postgres/job_repository.py new file mode 100644 index 0000000..ad40a2e --- /dev/null +++ b/src/bff/postgres/job_repository.py @@ -0,0 +1,651 @@ +"""Durable PostgreSQL job enqueue, status, history and event reads.""" + +from __future__ import annotations + +import json +from typing import Any, Mapping +import uuid + +from sqlalchemy import text + +from bff.postgres.base import PostgresRepository, iso_value +from pipelines.durable import QueueEnvelope + + +PIPELINE_JOB_TYPES = ( + "quick_consolidate", + "refresh_charities", + "refresh_grants", + "full_run", + "registry_enrichment", + "source_funder_enrichment", +) + + +class PostgresJobRepository(PostgresRepository): + async def enqueue( + self, + job_type: str, + payload: Mapping[str, Any], + *, + actor_id: str, + idempotency_key: str, + queue_name: str = "pipeline", + max_attempts: int = 3, + timeout_seconds: int = 3600, + ) -> dict[str, Any]: + job_type = str(job_type).strip() + if not job_type or len(job_type) > 120: + raise ValueError("Invalid job type") + if not idempotency_key or len(idempotency_key) > 200: + raise ValueError("A bounded idempotency key is required") + queue_name = str(queue_name).strip() + if not queue_name or len(queue_name) > 120: + raise ValueError("A bounded queue name is required") + if not 1 <= int(max_attempts) <= 20: + raise ValueError("max_attempts must be between 1 and 20") + if not 1 <= int(timeout_seconds) <= 86_400: + raise ValueError("timeout_seconds must be between 1 and 86400") + async with self.sessions() as session, session.begin(): + dataset_version = await self.active_dataset(session) + existing = ( + await session.execute( + text( + """ + SELECT job_run_id, status, requested_at + FROM job_runs + WHERE job_type=:job_type AND idempotency_key=:idempotency_key + """ + ), + {"job_type": job_type, "idempotency_key": idempotency_key}, + ) + ).mappings().first() + if existing: + return { + "job_id": str(existing["job_run_id"]), + "status": str(existing["status"]), + "requested_at": iso_value(existing["requested_at"]), + "idempotent_noop": True, + } + job_id = uuid.uuid4() + requested_at = await session.scalar( + text( + """ + INSERT INTO job_runs ( + job_run_id, job_type, status, dataset_version, + idempotency_key, requested_by, input, queue_name, + max_attempts, timeout_seconds, last_good_dataset_version + ) VALUES ( + :job_id, :job_type, 'queued', :dataset_version, + :idempotency_key, :actor_id, CAST(:payload AS jsonb), + :queue_name, :max_attempts, :timeout_seconds, :dataset_version + ) + RETURNING requested_at + """ + ), + { + "job_id": job_id, + "job_type": job_type, + "dataset_version": dataset_version, + "idempotency_key": idempotency_key, + "actor_id": actor_id, + "payload": json.dumps(payload, sort_keys=True, default=str), + "queue_name": queue_name, + "max_attempts": int(max_attempts), + "timeout_seconds": int(timeout_seconds), + }, + ) + envelope = QueueEnvelope( + schema_version="job-envelope-v1", + job_id=job_id, + job_type=job_type, + queue_name=queue_name, + requested_at=str(iso_value(requested_at)), + attempt=1, + max_attempts=int(max_attempts), + ) + await session.execute( + text( + """ + INSERT INTO job_dispatch_outbox ( + job_dispatch_outbox_id, job_run_id, queue_name, message_body + ) VALUES ( + :outbox_id, :job_id, :queue_name, CAST(:message_body AS jsonb) + ) + """ + ), + { + "outbox_id": uuid.uuid4(), + "job_id": job_id, + "queue_name": queue_name, + "message_body": json.dumps(envelope.payload, sort_keys=True), + }, + ) + await session.execute( + text( + """ + INSERT INTO job_events ( + job_event_id, job_run_id, sequence, event_type, + actor_id, details + ) VALUES ( + :event_id, :job_id, 1, 'queued', :actor_id, + CAST(:details AS jsonb) + ) + """ + ), + { + "event_id": uuid.uuid4(), + "job_id": job_id, + "actor_id": actor_id, + "details": json.dumps({"job_type": job_type}, sort_keys=True), + }, + ) + return { + "job_id": str(job_id), + "status": "queued", + "requested_at": iso_value(requested_at), + "idempotent_noop": False, + } + + async def claim( + self, + *, + worker_id: str, + queue_name: str = "pipeline", + lease_seconds: int = 60, + ) -> dict[str, Any] | None: + worker_id = str(worker_id).strip() + queue_name = str(queue_name).strip() + if not worker_id or len(worker_id) > 200: + raise ValueError("A bounded worker ID is required") + if not queue_name or len(queue_name) > 120: + raise ValueError("A bounded queue name is required") + if not 5 <= int(lease_seconds) <= 3600: + raise ValueError("lease_seconds must be between 5 and 3600") + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + WITH candidate AS ( + SELECT job_run_id + FROM job_runs + WHERE status='queued' AND queue_name=:queue_name + ORDER BY requested_at, job_run_id + FOR UPDATE SKIP LOCKED + LIMIT 1 + ) + UPDATE job_runs AS run + SET status='running', + started_at=COALESCE(run.started_at, CURRENT_TIMESTAMP), + heartbeat_at=CURRENT_TIMESTAMP, + lease_expires_at=CURRENT_TIMESTAMP + + CAST(:lease_seconds AS integer) * INTERVAL '1 second' + FROM candidate + WHERE run.job_run_id=candidate.job_run_id + RETURNING run.job_run_id, run.job_type, run.input, + run.dataset_version, run.requested_at, + run.started_at, run.attempt, run.max_attempts, + run.timeout_seconds, run.last_good_dataset_version + """ + ), + {"queue_name": queue_name, "lease_seconds": int(lease_seconds)}, + ) + ).mappings().first() + await session.execute( + text( + """ + INSERT INTO worker_heartbeats ( + worker_id, queue_name, job_run_id, status + ) VALUES ( + :worker_id, :queue_name, :job_run_id, :status + ) + ON CONFLICT (worker_id) DO UPDATE + SET queue_name=EXCLUDED.queue_name, + job_run_id=EXCLUDED.job_run_id, + status=EXCLUDED.status, + heartbeat_at=CURRENT_TIMESTAMP + """ + ), + { + "worker_id": worker_id, + "queue_name": queue_name, + "job_run_id": row["job_run_id"] if row else None, + "status": "running" if row else "idle", + }, + ) + if not row: + return None + await self._event( + session, + row["job_run_id"], + "claimed", + worker_id, + {"worker_id": worker_id, "attempt": row["attempt"]}, + ) + return self._job_row(row) + + async def heartbeat( + self, + job_id: str, + *, + worker_id: str, + lease_seconds: int = 60, + ) -> bool: + async with self.sessions() as session, session.begin(): + updated = await session.scalar( + text( + """ + UPDATE job_runs + SET heartbeat_at=CURRENT_TIMESTAMP, + lease_expires_at=CURRENT_TIMESTAMP + + CAST(:lease_seconds AS integer) * INTERVAL '1 second' + WHERE job_run_id=:job_id AND status='running' + RETURNING job_run_id + """ + ), + {"job_id": uuid.UUID(str(job_id)), "lease_seconds": int(lease_seconds)}, + ) + if updated: + await session.execute( + text( + """ + UPDATE worker_heartbeats + SET heartbeat_at=CURRENT_TIMESTAMP, status='running', + job_run_id=:job_id + WHERE worker_id=:worker_id + """ + ), + {"job_id": updated, "worker_id": worker_id}, + ) + return updated is not None + + async def succeed( + self, + job_id: str, + *, + worker_id: str, + result: Mapping[str, Any], + ) -> bool: + async with self.sessions() as session, session.begin(): + updated = await session.scalar( + text( + """ + UPDATE job_runs + SET status='succeeded', completed_at=CURRENT_TIMESTAMP, + heartbeat_at=CURRENT_TIMESTAMP, lease_expires_at=NULL, + result=CAST(:result AS jsonb), error_class=NULL, + error_message=NULL, failure_reason=NULL + WHERE job_run_id=:job_id AND status='running' + RETURNING job_run_id + """ + ), + { + "job_id": uuid.UUID(str(job_id)), + "result": json.dumps(result, sort_keys=True, default=str), + }, + ) + if updated: + await self._event(session, updated, "succeeded", worker_id, dict(result)) + await self._idle_worker(session, worker_id) + return updated is not None + + async def fail( + self, + job_id: str, + *, + worker_id: str, + error_class: str, + failure_reason: str, + retryable: bool, + ) -> str: + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + SELECT job_run_id, status, attempt, max_attempts + FROM job_runs WHERE job_run_id=:job_id + FOR UPDATE + """ + ), + {"job_id": uuid.UUID(str(job_id))}, + ) + ).mappings().first() + if not row or row["status"] != "running": + raise ValueError("Job is not running") + will_retry = retryable and row["attempt"] < row["max_attempts"] + status_value = "queued" if will_retry else ( + "dead_lettered" if retryable else "failed" + ) + await session.execute( + text( + """ + UPDATE job_runs + SET status=:status, + attempt=attempt + CAST(:attempt_increment AS integer), + completed_at=CASE WHEN :will_retry THEN NULL ELSE CURRENT_TIMESTAMP END, + heartbeat_at=NULL, lease_expires_at=NULL, + error_class=:error_class, error_message=:failure_reason, + failure_reason=:failure_reason + WHERE job_run_id=:job_id + """ + ), + { + "job_id": row["job_run_id"], + "status": status_value, + "attempt_increment": 1 if will_retry else 0, + "will_retry": will_retry, + "error_class": str(error_class)[:200], + "failure_reason": str(failure_reason)[:2000], + }, + ) + await self._event( + session, + row["job_run_id"], + "retry_queued" if will_retry else status_value, + worker_id, + {"error_class": str(error_class)[:200], "retryable": retryable}, + ) + await self._idle_worker(session, worker_id) + return status_value + + async def requeue_expired(self, *, queue_name: str = "pipeline") -> dict[str, int]: + async with self.sessions() as session, session.begin(): + retry_rows = ( + await session.execute( + text( + """ + UPDATE job_runs + SET status='queued', attempt=attempt + 1, + heartbeat_at=NULL, lease_expires_at=NULL, + error_class='WorkerLeaseExpired', + error_message='Worker heartbeat lease expired', + failure_reason='worker_lease_expired' + WHERE queue_name=:queue_name AND status='running' + AND lease_expires_at < CURRENT_TIMESTAMP + AND attempt < max_attempts + RETURNING job_run_id + """ + ), + {"queue_name": queue_name}, + ) + ).scalars().all() + dead_rows = ( + await session.execute( + text( + """ + UPDATE job_runs + SET status='dead_lettered', completed_at=CURRENT_TIMESTAMP, + heartbeat_at=NULL, lease_expires_at=NULL, + error_class='WorkerLeaseExpired', + error_message='Worker heartbeat lease expired after final attempt', + failure_reason='worker_lease_expired' + WHERE queue_name=:queue_name AND status='running' + AND lease_expires_at < CURRENT_TIMESTAMP + AND attempt >= max_attempts + RETURNING job_run_id + """ + ), + {"queue_name": queue_name}, + ) + ).scalars().all() + for job_id in retry_rows: + await self._event( + session, job_id, "retry_queued", "lease-reaper", {"reason": "lease_expired"} + ) + for job_id in dead_rows: + await self._event( + session, job_id, "dead_lettered", "lease-reaper", {"reason": "lease_expired"} + ) + return {"requeued": len(retry_rows), "dead_lettered": len(dead_rows)} + + async def due_dispatches(self, *, limit: int = 100) -> list[dict[str, Any]]: + bounded_limit = min(max(int(limit), 1), 100) + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT job_dispatch_outbox_id, job_run_id, queue_name, + message_body, publish_attempts, created_at + FROM job_dispatch_outbox + WHERE status IN ('pending', 'failed') + AND next_attempt_at <= CURRENT_TIMESTAMP + ORDER BY created_at, job_dispatch_outbox_id + LIMIT :limit + """ + ), + {"limit": bounded_limit}, + ) + ).mappings() + return [self._dispatch_row(row) for row in rows] + + async def mark_dispatch_published( + self, outbox_id: str, *, queue_message_id: str + ) -> bool: + async with self.sessions() as session, session.begin(): + updated = await session.scalar( + text( + """ + UPDATE job_dispatch_outbox + SET status='published', publish_attempts=publish_attempts + 1, + queue_message_id=:queue_message_id, + published_at=CURRENT_TIMESTAMP, last_error_class=NULL + WHERE job_dispatch_outbox_id=:outbox_id + AND status IN ('pending', 'failed') + RETURNING job_dispatch_outbox_id + """ + ), + { + "outbox_id": uuid.UUID(str(outbox_id)), + "queue_message_id": str(queue_message_id)[:200], + }, + ) + return updated is not None + + async def mark_dispatch_failed( + self, + outbox_id: str, + *, + error_class: str, + retry_delay_seconds: int = 30, + maximum_attempts: int = 5, + ) -> str: + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + SELECT publish_attempts FROM job_dispatch_outbox + WHERE job_dispatch_outbox_id=:outbox_id + FOR UPDATE + """ + ), + {"outbox_id": uuid.UUID(str(outbox_id))}, + ) + ).mappings().first() + if not row: + raise ValueError("Dispatch outbox record does not exist") + attempts = int(row["publish_attempts"]) + 1 + status_value = "dead_lettered" if attempts >= maximum_attempts else "failed" + await session.execute( + text( + """ + UPDATE job_dispatch_outbox + SET status=:status, publish_attempts=:attempts, + next_attempt_at=CURRENT_TIMESTAMP + + CAST(:retry_delay_seconds AS integer) * INTERVAL '1 second', + last_error_class=:error_class + WHERE job_dispatch_outbox_id=:outbox_id + """ + ), + { + "outbox_id": uuid.UUID(str(outbox_id)), + "status": status_value, + "attempts": attempts, + "retry_delay_seconds": max(1, int(retry_delay_seconds)), + "error_class": str(error_class)[:200], + }, + ) + return status_value + + @staticmethod + async def _event(session, job_id, event_type, actor_id, details) -> None: + await session.execute( + text( + """ + INSERT INTO job_events ( + job_event_id, job_run_id, sequence, event_type, actor_id, details + ) VALUES ( + :event_id, :job_id, + (SELECT COALESCE(MAX(sequence), 0) + 1 + FROM job_events WHERE job_run_id=:job_id), + :event_type, :actor_id, CAST(:details AS jsonb) + ) + """ + ), + { + "event_id": uuid.uuid4(), + "job_id": job_id, + "event_type": event_type, + "actor_id": actor_id, + "details": json.dumps(details, sort_keys=True, default=str), + }, + ) + + @staticmethod + async def _idle_worker(session, worker_id: str) -> None: + await session.execute( + text( + """ + UPDATE worker_heartbeats + SET status='idle', job_run_id=NULL, heartbeat_at=CURRENT_TIMESTAMP + WHERE worker_id=:worker_id + """ + ), + {"worker_id": worker_id}, + ) + + @staticmethod + def _job_row(row: Mapping[str, Any]) -> dict[str, Any]: + result = dict(row) + result["job_id"] = str(result.pop("job_run_id")) + for field in ("requested_at", "started_at", "completed_at"): + if field in result: + result[field] = iso_value(result[field]) + return result + + @staticmethod + def _dispatch_row(row: Mapping[str, Any]) -> dict[str, Any]: + result = dict(row) + result["outbox_id"] = str(result.pop("job_dispatch_outbox_id")) + result["job_id"] = str(result.pop("job_run_id")) + result["created_at"] = iso_value(result["created_at"]) + return result + + async def latest_status(self) -> dict[str, Any]: + async with self.sessions() as session: + row = ( + await session.execute( + text( + """ + SELECT job_run_id, job_type, status, requested_at, + started_at, completed_at, error_message + FROM job_runs + WHERE job_type=ANY(CAST(:types AS text[])) + ORDER BY requested_at DESC, job_run_id DESC LIMIT 1 + """ + ), + {"types": list(PIPELINE_JOB_TYPES)}, + ) + ).mappings().first() + if not row: + return { + "status": "idle", + "started_at": None, + "finished_at": None, + "last_run_source": None, + "error": None, + "job_id": None, + } + mapped_status = { + "created": "running", + "queued": "running", + "running": "running", + "succeeded": "success", + "failed": "failed", + "cancelled": "failed", + "timed_out": "failed", + "dead_lettered": "failed", + }.get(str(row["status"]), str(row["status"])) + return { + "status": mapped_status, + "started_at": iso_value(row["started_at"] or row["requested_at"]), + "finished_at": iso_value(row["completed_at"]), + "last_run_source": row["job_type"], + "error": row["error_message"], + "job_id": str(row["job_run_id"]), + } + + async def history(self, *, limit: int = 50) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 100) + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT job_run_id, job_type, status, dataset_version, + requested_by, requested_at, started_at, completed_at, + attempt, max_attempts, input, result, + error_class, error_message, queue_name, + heartbeat_at, lease_expires_at, timeout_seconds, + failure_reason, last_good_dataset_version + FROM job_runs + ORDER BY requested_at DESC, job_run_id DESC LIMIT :limit + """ + ), + {"limit": limit}, + ) + ).mappings() + return [ + { + **dict(row), + "job_run_id": str(row["job_run_id"]), + "requested_at": iso_value(row["requested_at"]), + "started_at": iso_value(row["started_at"]), + "completed_at": iso_value(row["completed_at"]), + "heartbeat_at": iso_value(row["heartbeat_at"]), + "lease_expires_at": iso_value(row["lease_expires_at"]), + } + for row in rows + ] + + async def events(self, *, limit: int = 100) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 100) + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT event.job_event_id, event.job_run_id, event.sequence, + event.event_type, event.occurred_at, event.actor_id, + event.details, run.job_type + FROM job_events AS event + JOIN job_runs AS run USING (job_run_id) + ORDER BY event.occurred_at DESC, event.job_event_id DESC + LIMIT :limit + """ + ), + {"limit": limit}, + ) + ).mappings() + return [ + { + **dict(row), + "job_event_id": str(row["job_event_id"]), + "job_run_id": str(row["job_run_id"]), + "occurred_at": iso_value(row["occurred_at"]), + } + for row in rows + ] diff --git a/src/bff/postgres/observability_routes.py b/src/bff/postgres/observability_routes.py new file mode 100644 index 0000000..a945584 --- /dev/null +++ b/src/bff/postgres/observability_routes.py @@ -0,0 +1,31 @@ +"""Administrator-only local telemetry evidence and definitions.""" + +from __future__ import annotations + +from dataclasses import asdict + +from fastapi import APIRouter, Depends, Request + +from bff.security import Role, require_roles + + +router = APIRouter( + prefix="/api/admin/observability", + tags=["Observability"], + dependencies=[ + Depends(require_roles(Role.ADMINISTRATOR, action="observability.read")) + ], +) + + +@router.get("/metrics") +async def metrics(request: Request): + configuration = request.app.state.observability_configuration + registry = request.app.state.metrics_registry + return { + "service": configuration.service, + "metrics": [asdict(definition) for definition in configuration.metrics], + "alarms": [asdict(definition) for definition in configuration.alarms], + "local_snapshot": registry.snapshot(), + "cloudwatch_execution": "not_tested", + } diff --git a/src/bff/postgres/organization_repository.py b/src/bff/postgres/organization_repository.py new file mode 100644 index 0000000..06e7d66 --- /dev/null +++ b/src/bff/postgres/organization_repository.py @@ -0,0 +1,679 @@ +"""Async PostgreSQL organization, grant-history, Sankey and score queries.""" + +from __future__ import annotations + +from collections import defaultdict +from hashlib import sha256 +from typing import Any, Mapping, Optional, Sequence + +from sqlalchemy import text + +from bff.postgres.base import PostgresRepository, iso_value, json_value, number_value, row_dict, utc_now +from scoring.engine import load_score_configuration, score_relevance + + +def _registered_number(row: Mapping[str, Any], raw: Mapping[str, Any]) -> int: + value = raw.get("registered_charity_number", row["charity_id"]) + try: + return int(value) + except (TypeError, ValueError): + return int(row["charity_id"]) + + +def _base_item(row: Mapping[str, Any]) -> dict[str, Any]: + raw = json_value(row.get("raw_source_data"), {}) + details = json_value(raw.get("all_details"), {}) + return { + "registered_charity_number": _registered_number(row, raw), + "suffix": int(raw.get("suffix") or 0), + "link": raw.get("link") or row.get("source_url"), + "charity_name": str(row["name"]), + "reg_status": str(details.get("reg_status") or "Unknown"), + "reporting_status": details.get("reporting_status"), + "removal_reason": details.get("removal_reason"), + "latest_income": number_value( + details.get("latest_income", row.get("annual_income")) + ), + "latest_expenditure": number_value( + details.get("latest_expenditure", row.get("annual_expenditure")) + ), + "programme_areas_source": json_value(row.get("programme_areas_source"), []), + "programme_areas_inferred": json_value(row.get("programme_areas_inferred"), []), + "geographic_focus_source": json_value(row.get("geographic_focus_source"), []), + "geographic_focus_inferred": json_value(row.get("geographic_focus_inferred"), []), + "headquarters_country": row.get("headquarters_country"), + "headquarters_region": row.get("headquarters_region"), + "programme_area_review_required": bool(row.get("programme_area_review_required")), + "geography_review_required": bool(row.get("geography_review_required")), + "enrichment_rule_version": row.get("enrichment_rule_version"), + "organization_type": str(row.get("organization_type") or "unknown"), + "primary_source": row.get("primary_source"), + "source_names": json_value(row.get("source_names"), []), + "source_record_id": row.get("source_record_id"), + "source_url": row.get("source_url"), + "transaction_coverage": str(row.get("transaction_coverage") or "unknown"), + } + + +class OrganizationRepository(PostgresRepository): + async def list( + self, + *, + search: Optional[str] = None, + reg_status: Optional[str] = None, + tag: Optional[str] = None, + region: Optional[str] = None, + size: Optional[str] = None, + tags: Optional[Sequence[str]] = None, + foundation_regions: Optional[Sequence[str]] = None, + funding_regions: Optional[Sequence[str]] = None, + sources: Optional[Sequence[str]] = None, + min_annual_giving: Optional[float] = None, + max_annual_giving: Optional[float] = None, + min_avg_grant_size: Optional[float] = None, + max_avg_grant_size: Optional[float] = None, + include_score: bool = False, + sort: str = "name_asc", + skip: int = 0, + limit: int = 20, + ) -> list[dict[str, Any]]: + limit = min(max(int(limit), 1), 100) + skip = min(max(int(skip), 0), 1_000_000) + conditions = ["charity.dataset_version=:dataset_version"] + parameters: dict[str, Any] = { + "search": f"%{str(search or '').strip()}%", + "reg_status": reg_status, + "tags": list(dict.fromkeys([*(tags or []), *([tag] if tag else [])])), + "foundation_regions": list(foundation_regions or []), + "funding_regions": list(funding_regions or []), + "sources": list(sources or []), + "min_annual_giving": min_annual_giving, + "max_annual_giving": max_annual_giving, + "min_avg_grant_size": min_avg_grant_size, + "max_avg_grant_size": max_avg_grant_size, + "limit": 500 if sort == "score_desc" else limit, + "offset": 0 if sort == "score_desc" else skip, + } + if search and search.strip(): + conditions.append("(charity.name ILIKE :search OR charity.normalized_name ILIKE :search)") + if reg_status: + conditions.append( + "COALESCE(charity.raw_source_data #>> '{all_details,reg_status}', 'Unknown')=:reg_status" + ) + if parameters["tags"]: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM jsonb_array_elements_text( + COALESCE(charity.programme_areas_source, '[]'::jsonb) + || COALESCE(charity.programme_areas_inferred, '[]'::jsonb) + ) AS category(value) + WHERE category.value=ANY(CAST(:tags AS text[])) + ) + """ + ) + if region: + parameters["region"] = region + conditions.append( + "(charity.headquarters_region=:region OR charity.headquarters_country=:region)" + ) + if parameters["foundation_regions"]: + conditions.append( + "charity.headquarters_region=ANY(CAST(:foundation_regions AS text[]))" + ) + if parameters["sources"]: + conditions.append("charity.primary_source=ANY(CAST(:sources AS text[]))") + if size: + ranges = { + "small": (0, 100_000), + "medium": (100_000, 1_000_000), + "large": (1_000_000, None), + } + if size in ranges: + low, high = ranges[size] + parameters.update({"size_low": low, "size_high": high}) + conditions.append("COALESCE(charity.annual_income, 0)>=:size_low") + if high is not None: + conditions.append("COALESCE(charity.annual_income, 0)<:size_high") + giving_expression = "COALESCE(grant_stats.annual_giving, 0)" + average_expression = "grant_stats.average_grant" + if min_annual_giving is not None: + conditions.append(f"{giving_expression}>=:min_annual_giving") + if max_annual_giving is not None: + conditions.append(f"{giving_expression}<=:max_annual_giving") + if min_avg_grant_size is not None: + conditions.append(f"{average_expression}>=:min_avg_grant_size") + if max_avg_grant_size is not None: + conditions.append(f"{average_expression}<=:max_avg_grant_size") + if parameters["funding_regions"]: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM grants AS funding_grant + JOIN grant_beneficiary_countries AS funding_country + ON funding_country.dataset_version=funding_grant.dataset_version + AND funding_country.grant_id=funding_grant.grant_id + WHERE funding_grant.dataset_version=charity.dataset_version + AND funding_grant.funding_charity_id=charity.charity_id + AND funding_country.country_name=ANY(CAST(:funding_regions AS text[])) + ) + """ + ) + order = { + "name_asc": "charity.name ASC, charity.charity_id ASC", + "income_desc": "charity.annual_income DESC NULLS LAST, charity.charity_id ASC", + "score_desc": "charity.name ASC, charity.charity_id ASC", + }.get(sort, "charity.name ASC, charity.charity_id ASC") + sql = text( + f""" + SELECT charity.*, grant_stats.annual_giving, grant_stats.average_grant, + grant_stats.grant_count + FROM charities AS charity + LEFT JOIN LATERAL ( + SELECT SUM(CASE WHEN grant_row.amount_eur>0 THEN grant_row.amount_eur ELSE 0 END) + AS annual_giving, + AVG(grant_row.amount_eur) FILTER (WHERE grant_row.amount_eur>0) + AS average_grant, + COUNT(*) AS grant_count + FROM grants AS grant_row + WHERE grant_row.dataset_version=charity.dataset_version + AND grant_row.funding_charity_id=charity.charity_id + ) AS grant_stats ON TRUE + WHERE {' AND '.join(conditions)} + ORDER BY {order} + LIMIT :limit OFFSET :offset + """ + ) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [dict(row) for row in (await session.execute(sql, parameters)).mappings()] + items = [_base_item(row) for row in rows] + if include_score or sort == "score_desc": + config = load_score_configuration() + for item, row in zip(items, rows): + score = score_relevance( + { + **item, + "annual_expenditure": item["latest_expenditure"], + }, + config.example_target_profile, + grant_statistics={ + "currency": "EUR", + "average_amount": number_value(row.get("average_grant")), + "grant_count": int(row.get("grant_count") or 0), + }, + configuration=config, + ) + item.update( + { + "relevance_score": score["score"], + "score_confidence": score["confidence"], + "score_completeness": score["data_completeness"], + "score_target": score["score_target"], + "score_version": score["score_version"], + "score_configuration_status": score["configuration_status"], + } + ) + if sort == "score_desc": + items.sort( + key=lambda item: ( + -(item.get("relevance_score") or -1), + item["charity_name"].casefold(), + item["registered_charity_number"], + ) + ) + items = items[skip : skip + limit] + return items + + async def detail(self, organization_id: int) -> Optional[dict[str, Any]]: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT * FROM charities + WHERE dataset_version=:dataset_version + AND ( + charity_id=:organization_id + OR raw_source_data->>'registered_charity_number'=:registered + ) + ORDER BY CASE WHEN charity_id=:organization_id THEN 0 ELSE 1 END + LIMIT 1 + """ + ), + { + "dataset_version": dataset_version, + "organization_id": organization_id, + "registered": str(organization_id), + }, + ) + ).mappings().first() + if not row: + return None + raw_row = dict(row) + raw = json_value(raw_row.get("raw_source_data"), {}) + base = _base_item(raw_row) + all_details = json_value(raw.get("all_details"), {}) + all_details = { + "organisation_number": int(all_details.get("organisation_number") or raw_row["charity_id"]), + "reg_charity_number": int( + all_details.get("reg_charity_number") or base["registered_charity_number"] + ), + "group_subsid_suffix": int(all_details.get("group_subsid_suffix") or base["suffix"]), + "charity_name": str(all_details.get("charity_name") or raw_row["name"]), + "reg_status": str(all_details.get("reg_status") or base["reg_status"]), + **all_details, + } + return { + "registered_charity_number": base["registered_charity_number"], + "suffix": base["suffix"], + "link": base["link"], + "all_details": all_details, + "assets_liabilities": json_value(raw.get("assets_liabilities"), []), + "primary_grants": raw.get("primary_grants"), + "who_what_how": json_value(raw.get("who_what_how"), []), + "financial_history": json_value(raw.get("financial_history"), []), + "programme_areas_source": base["programme_areas_source"], + "programme_areas_inferred": base["programme_areas_inferred"], + "programme_area_scores": json_value(raw_row.get("programme_area_scores"), {}), + "programme_area_method": raw_row.get("programme_area_method"), + "programme_area_evidence": json_value(raw_row.get("programme_area_evidence"), []), + "programme_area_review_required": base["programme_area_review_required"], + "geographic_focus_source": base["geographic_focus_source"], + "geographic_focus_inferred": base["geographic_focus_inferred"], + "headquarters_country": base["headquarters_country"], + "headquarters_region": base["headquarters_region"], + "geography_method": raw_row.get("geography_method"), + "geography_confidence": number_value(raw_row.get("geography_confidence")), + "geography_evidence": json_value(raw_row.get("geography_evidence"), []), + "geography_review_required": base["geography_review_required"], + "enrichment_rule_version": base["enrichment_rule_version"], + "organization_type": base["organization_type"], + "primary_source": base["primary_source"], + "source_names": base["source_names"], + "source_record_id": base["source_record_id"], + "source_url": base["source_url"], + "source_records": json_value(raw_row.get("source_records"), []), + "ingestion_timestamp": iso_value(raw_row.get("ingestion_timestamp")), + "transaction_coverage": base["transaction_coverage"], + "deduplication_status": raw_row.get("deduplication_status"), + "deduplication_candidates": json_value(raw_row.get("deduplication_candidates"), []), + } + + async def stats(self) -> dict[str, Any]: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT COUNT(*) AS total_charities, + COUNT(*) FILTER ( + WHERE COALESCE(raw_source_data #>> '{all_details,reg_status}', '') + NOT ILIKE '%removed%' + ) AS active_charities, + COUNT(*) FILTER ( + WHERE COALESCE(raw_source_data #>> '{all_details,reg_status}', '') + ILIKE '%removed%' + ) AS removed_charities, + COALESCE(AVG(annual_income), 0) AS average_income, + COALESCE(AVG(annual_expenditure), 0) AS average_expenditure + FROM charities WHERE dataset_version=:dataset_version + """ + ), + {"dataset_version": dataset_version}, + ) + ).mappings().one() + grant_count = await session.scalar( + text("SELECT COUNT(*) FROM grants WHERE dataset_version=:dataset_version"), + {"dataset_version": dataset_version}, + ) + sources = ( + await session.execute( + text( + """ + SELECT COALESCE(primary_source, 'Unknown') AS source, COUNT(*) AS count + FROM charities WHERE dataset_version=:dataset_version + GROUP BY COALESCE(primary_source, 'Unknown') ORDER BY source + """ + ), + {"dataset_version": dataset_version}, + ) + ).all() + grant_sources = ( + await session.execute( + text( + """ + SELECT DISTINCT source + FROM grants + WHERE dataset_version=:dataset_version + AND source IS NOT NULL + AND source<>'' + ORDER BY source + """ + ), + {"dataset_version": dataset_version}, + ) + ).scalars().all() + types = ( + await session.execute( + text( + """ + SELECT COALESCE(organization_type, 'unknown') AS type, COUNT(*) AS count + FROM charities WHERE dataset_version=:dataset_version + GROUP BY COALESCE(organization_type, 'unknown') ORDER BY type + """ + ), + {"dataset_version": dataset_version}, + ) + ).all() + return { + **row_dict(row), + "total_grants": int(grant_count or 0), + "data_mode": "postgresql_active_dataset", + "source": sorted( + {str(item[0]) for item in sources if str(item[0]) != "Unknown"} + | {str(source) for source in grant_sources} + ), + "source_counts": {str(item[0]): int(item[1]) for item in sources}, + "organization_type_counts": {str(item[0]): int(item[1]) for item in types}, + } + + @staticmethod + def _grant_item(row: Mapping[str, Any]) -> dict[str, Any]: + return { + "grant_id": str(row["grant_id"]), + "funding_charity_id": row.get("funding_charity_id"), + "funding_name": row.get("funding_name"), + "funding_org_source_id": row.get("funding_org_source_id"), + "recipient_name": str(row.get("recipient_name") or "Unknown recipient"), + "recipient_charity_id": row.get("recipient_charity_id"), + "recipient_org_source_id": row.get("recipient_org_source_id"), + "amount": number_value(row.get("amount")), + "amount_eur": number_value(row.get("amount_eur")), + "exchange_rate": number_value(row.get("exchange_rate")), + "exchange_rate_date": row.get("exchange_rate_date"), + "exchange_rate_source": row.get("exchange_rate_source"), + "conversion_status": row.get("conversion_status"), + "currency": str(row.get("currency") or "UNK"), + "description": str(row.get("description") or ""), + "date": str(row.get("award_date") or ""), + "recipient_region": row.get("recipient_region"), + "beneficiary_geography": [row["beneficiary_geography"]] + if row.get("beneficiary_geography") else [], + "tags": json_value(row.get("tags"), []), + "source": row.get("source"), + "source_record_id": row.get("source_record_id"), + "source_url": row.get("source_url"), + "programme_area_source": [row["programme_area_source"]] + if row.get("programme_area_source") else [], + "programme_area_inferred": [row["programme_area_inferred"]] + if row.get("programme_area_inferred") else [], + "programme_area_scores": json_value(row.get("programme_area_scores"), {}), + "programme_area_method": row.get("programme_area_method"), + "programme_area_evidence": json_value(row.get("programme_area_evidence"), []), + "programme_area_review_required": bool(row.get("programme_area_review_required")), + "beneficiary_geography_normalized": json_value(row.get("countries"), []), + "geographic_focus_inferred": [row["geographic_focus_inferred"]] + if row.get("geographic_focus_inferred") else [], + "geography_method": row.get("geography_method"), + "geography_confidence": number_value(row.get("geography_confidence")), + "geography_evidence": json_value(row.get("geography_evidence"), []), + "geography_review_required": bool(row.get("geography_review_required")), + "enrichment_rule_version": row.get("enrichment_rule_version"), + } + + async def grants(self, organization_id: int, role: str) -> dict[str, Any]: + conditions = { + "all": "(grant_row.funding_charity_id=:organization_id OR grant_row.recipient_charity_id=:organization_id)", + "funder": "grant_row.funding_charity_id=:organization_id", + "recipient": "grant_row.recipient_charity_id=:organization_id", + } + if role not in conditions: + raise ValueError("role must be all, funder, or recipient") + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + result = await session.execute( + text( + f""" + SELECT grant_row.*, + COALESCE( + jsonb_agg(DISTINCT jsonb_build_object( + 'code', country.country_code, + 'name', country.country_name + )) FILTER (WHERE country.country_code IS NOT NULL), + '[]'::jsonb + ) AS countries + FROM grants AS grant_row + LEFT JOIN grant_beneficiary_countries AS country + ON country.dataset_version=grant_row.dataset_version + AND country.grant_id=grant_row.grant_id + WHERE grant_row.dataset_version=:dataset_version + AND {conditions[role]} + GROUP BY grant_row.dataset_version, grant_row.grant_id + ORDER BY grant_row.award_date DESC, grant_row.grant_id + LIMIT 5000 + """ + ), + {"dataset_version": dataset_version, "organization_id": organization_id}, + ) + rows = [dict(row) for row in result.mappings()] + currencies = sorted({str(row["currency"]) for row in rows if row.get("currency")}) + return { + "status": "available" if rows else "no_transactions_found", + "organization_id": organization_id, + "role": role, + "transaction_coverage": "observed_transactions_only", + "grant_count": len(rows), + "currencies": currencies, + "grants": [self._grant_item(row) for row in rows], + "metadata": { + "data_mode": "postgresql_active_dataset", + "source": sorted({str(row["source"]) for row in rows if row.get("source")}), + "generated_at": utc_now(), + "record_count": len(rows), + "derivation": "stored grant transactions", + "limitations": ["Response is bounded to 5000 transactions." ] + if len(rows) == 5000 else [], + }, + } + + @staticmethod + def _entity_id(role: str, charity_id: Any, source_id: Any, name: Any) -> str: + if charity_id is not None: + return f"organization:{int(charity_id)}" + material = f"{role}|{source_id or ''}|{str(name or '').casefold()}" + return f"source:{role}:{sha256(material.encode()).hexdigest()[:20]}" + + async def sankey( + self, + organization_id: int, + *, + currency: Optional[str] = None, + limit: int = 30, + ) -> dict[str, Any]: + selected_currency = str(currency or "EUR").upper() + amount_column = "amount_eur" if currency is None or selected_currency == "EUR" else "amount" + currency_condition = "" if amount_column == "amount_eur" else "AND currency=:currency" + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + rows = [ + dict(row) + for row in ( + await session.execute( + text( + f""" + SELECT grant_id, funding_charity_id, funding_name, + funding_org_source_id, recipient_charity_id, + recipient_name, recipient_org_source_id, + {amount_column} AS selected_amount, + currency, source + FROM grants + WHERE dataset_version=:dataset_version + AND (funding_charity_id=:organization_id + OR recipient_charity_id=:organization_id) + AND {amount_column}>0 {currency_condition} + ORDER BY {amount_column} DESC, grant_id + LIMIT 10000 + """ + ), + { + "dataset_version": dataset_version, + "organization_id": organization_id, + "currency": selected_currency, + }, + ) + ).mappings() + ] + aggregated: dict[tuple[str, str], dict[str, Any]] = defaultdict( + lambda: {"value": 0.0, "grant_count": 0} + ) + nodes: dict[str, dict[str, Any]] = {} + for row in rows: + source = self._entity_id( + "donor", row.get("funding_charity_id"), + row.get("funding_org_source_id"), row.get("funding_name"), + ) + target = self._entity_id( + "recipient", row.get("recipient_charity_id"), + row.get("recipient_org_source_id"), row.get("recipient_name"), + ) + if source == target: + continue + nodes.setdefault(source, {"id": source, "label": row.get("funding_name") or "Unnamed donor", "role": "donor"}) + nodes.setdefault(target, {"id": target, "label": row.get("recipient_name") or "Unnamed recipient", "role": "recipient"}) + aggregate = aggregated[(source, target)] + aggregate["value"] += float(row["selected_amount"]) + aggregate["grant_count"] += 1 + links = [ + { + "source": source, + "target": target, + "value": round(values["value"], 2), + "currency": selected_currency, + "grant_count": values["grant_count"], + } + for (source, target), values in aggregated.items() + ] + links.sort(key=lambda item: (-item["value"], item["source"], item["target"])) + truncated = len(links) > limit + links = links[:limit] + node_ids = {item["source"] for item in links} | {item["target"] for item in links} + included = sum(item["grant_count"] for item in links) + return { + "status": "available" if links else "no_transactions_found", + "nodes": [nodes[node_id] for node_id in sorted(node_ids)], + "links": links, + "metadata": { + "source": sorted({str(row["source"]) for row in rows if row.get("source")}), + "generated_at": utc_now(), + "grant_count": len(rows), + "included_grant_count": included, + "excluded_grant_count": max(0, len(rows) - included), + "excluded_reasons": {"truncated": max(0, len(rows) - included)} if truncated else {}, + "included_value": round(sum(item["value"] for item in links), 2), + "currencies": [selected_currency], + "selected_currency": selected_currency, + "conversion_method": "ecb_historic_reference_rate" if amount_column == "amount_eur" else "none", + "filters_applied": {"organization_id": organization_id, "limit": limit}, + "truncation_applied": truncated, + }, + } + + async def score( + self, + organization_id: int, + target_profile: Optional[Mapping[str, Any]], + ) -> dict[str, Any]: + organization = await self.detail(organization_id) + if not organization: + raise KeyError(organization_id) + config = load_score_configuration() + profile = dict(target_profile or config.example_target_profile) + requested_currency = str(profile.get("currency") or "").upper() + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + if requested_currency == "EUR": + grant_row = ( + await session.execute( + text( + """ + SELECT 'EUR' AS currency, AVG(amount_eur) AS average_amount, + COUNT(*) AS grant_count + FROM grants WHERE dataset_version=:dataset_version + AND funding_charity_id=:organization_id AND amount_eur>0 + """ + ), + {"dataset_version": dataset_version, "organization_id": organization_id}, + ) + ).mappings().one() + else: + grant_row = ( + await session.execute( + text( + """ + SELECT currency, AVG(amount) AS average_amount, COUNT(*) AS grant_count + FROM grants WHERE dataset_version=:dataset_version + AND funding_charity_id=:organization_id AND amount>0 + AND (:currency='' OR currency=:currency) + GROUP BY currency ORDER BY COUNT(*) DESC, currency LIMIT 1 + """ + ), + { + "dataset_version": dataset_version, + "organization_id": organization_id, + "currency": requested_currency, + }, + ) + ).mappings().first() + programme_rows = await session.execute( + text( + """ + SELECT DISTINCT category.programme_area + FROM grants AS grant_row + JOIN grant_programme_categories AS category + ON category.dataset_version=grant_row.dataset_version + AND category.grant_id=grant_row.grant_id + WHERE grant_row.dataset_version=:dataset_version + AND grant_row.funding_charity_id=:organization_id + """ + ), + {"dataset_version": dataset_version, "organization_id": organization_id}, + ) + geography_rows = await session.execute( + text( + """ + SELECT DISTINCT country.country_name + FROM grants AS grant_row + JOIN grant_beneficiary_countries AS country + ON country.dataset_version=grant_row.dataset_version + AND country.grant_id=grant_row.grant_id + WHERE grant_row.dataset_version=:dataset_version + AND grant_row.funding_charity_id=:organization_id + """ + ), + {"dataset_version": dataset_version, "organization_id": organization_id}, + ) + score_input = { + **organization, + "annual_expenditure": organization["all_details"].get("latest_expenditure"), + "observed_grant_programme_areas": [row[0] for row in programme_rows], + "observed_beneficiary_geographies": [row[0] for row in geography_rows], + } + grant_statistics = ( + { + "currency": grant_row["currency"], + "average_amount": number_value(grant_row["average_amount"]), + "grant_count": int(grant_row["grant_count"] or 0), + } + if grant_row and grant_row["average_amount"] is not None + else {} + ) + return score_relevance( + score_input, + profile, + grant_statistics=grant_statistics, + configuration=config, + ) diff --git a/src/bff/postgres/pipeline_repository.py b/src/bff/postgres/pipeline_repository.py new file mode 100644 index 0000000..fbb646e --- /dev/null +++ b/src/bff/postgres/pipeline_repository.py @@ -0,0 +1,288 @@ +"""PostgreSQL persistence for source controls and immutable ingestion evidence.""" + +from __future__ import annotations + +import json +from typing import Any, Iterable, Mapping +import uuid + +from sqlalchemy import text + +from bff.postgres.base import PostgresRepository, iso_value +from pipelines.durable import IngestionManifest, SourceConfiguration, StorageObject + + +class PipelineRepository(PostgresRepository): + async def synchronize_sources( + self, configurations: Iterable[SourceConfiguration] + ) -> int: + records = [configuration.database_record() for configuration in configurations] + async with self.sessions() as session, session.begin(): + for record in records: + await session.execute( + text( + """ + INSERT INTO source_configurations ( + source_name, source_owner, technical_owner, legal_status, + licence_status, terms_url, rate_limit_per_minute, + user_agent, freshness_sla_hours, schedule_expression, + enabled, governance_blocked, last_success_at, watermark, + classification, retention_class, schema_version, + credentials_reference, retry_limit, timeout_seconds, + maximum_pages, maximum_records, configuration_checksum + ) VALUES ( + :source_name, :source_owner, :technical_owner, :legal_status, + :licence_status, :terms_url, :rate_limit_per_minute, + :user_agent, :freshness_sla_hours, :schedule_expression, + :enabled, :governance_blocked, CAST(:last_success AS timestamptz), + :watermark, :classification, :retention_class, :schema_version, + :credentials_reference, :retry_limit, :timeout_seconds, + :maximum_pages, :maximum_records, :configuration_checksum + ) + ON CONFLICT (source_name) DO UPDATE SET + source_owner=EXCLUDED.source_owner, + technical_owner=EXCLUDED.technical_owner, + legal_status=EXCLUDED.legal_status, + licence_status=EXCLUDED.licence_status, + terms_url=EXCLUDED.terms_url, + rate_limit_per_minute=EXCLUDED.rate_limit_per_minute, + user_agent=EXCLUDED.user_agent, + freshness_sla_hours=EXCLUDED.freshness_sla_hours, + schedule_expression=EXCLUDED.schedule_expression, + enabled=EXCLUDED.enabled, + governance_blocked=EXCLUDED.governance_blocked, + classification=EXCLUDED.classification, + retention_class=EXCLUDED.retention_class, + schema_version=EXCLUDED.schema_version, + credentials_reference=EXCLUDED.credentials_reference, + retry_limit=EXCLUDED.retry_limit, + timeout_seconds=EXCLUDED.timeout_seconds, + maximum_pages=EXCLUDED.maximum_pages, + maximum_records=EXCLUDED.maximum_records, + configuration_checksum=EXCLUDED.configuration_checksum, + updated_at=CURRENT_TIMESTAMP + """ + ), + record, + ) + return len(records) + + async def sources(self) -> list[dict[str, Any]]: + async with self.sessions() as session: + rows = ( + await session.execute( + text( + """ + SELECT source_name, source_owner, technical_owner, + legal_status, licence_status, terms_url, + rate_limit_per_minute, user_agent, + freshness_sla_hours, schedule_expression, + enabled, governance_blocked, last_success_at, + watermark, classification, retention_class, + schema_version, credentials_reference, retry_limit, + timeout_seconds, maximum_pages, maximum_records, + configuration_checksum, updated_at + FROM source_configurations ORDER BY source_name + """ + ) + ) + ).mappings() + return [ + { + **dict(row), + "last_success_at": iso_value(row["last_success_at"]), + "updated_at": iso_value(row["updated_at"]), + } + for row in rows + ] + + async def set_source_enabled(self, source_name: str, *, enabled: bool) -> dict[str, Any]: + async with self.sessions() as session, session.begin(): + row = ( + await session.execute( + text( + """ + SELECT legal_status, licence_status, governance_blocked + FROM source_configurations + WHERE source_name=:source_name + FOR UPDATE + """ + ), + {"source_name": source_name}, + ) + ).mappings().first() + if not row: + raise ValueError("Unknown source configuration") + if enabled and ( + row["legal_status"] != "approved" + or row["licence_status"] != "approved" + or row["governance_blocked"] + ): + raise ValueError("Source schedule is blocked by unresolved governance") + updated = ( + await session.execute( + text( + """ + UPDATE source_configurations + SET enabled=:enabled, updated_at=CURRENT_TIMESTAMP + WHERE source_name=:source_name + RETURNING source_name, enabled, governance_blocked, + legal_status, licence_status, schedule_expression, + updated_at + """ + ), + {"source_name": source_name, "enabled": enabled}, + ) + ).mappings().one() + return {**dict(updated), "updated_at": iso_value(updated["updated_at"])} + + async def start_ingestion( + self, + *, + source_name: str, + dataset_version: str, + job_id: str | None, + source_version: str | None, + source_uri: str | None, + watermark_before: str | None, + ) -> str: + run_id = uuid.uuid4() + async with self.sessions() as session, session.begin(): + active = await self.active_dataset(session) + await session.execute( + text( + """ + INSERT INTO source_ingestion_runs ( + source_ingestion_run_id, source_namespace, dataset_version, + job_run_id, status, source_version, source_uri, + watermark_before, last_good_dataset_version + ) VALUES ( + :run_id, :source_name, :dataset_version, :job_id, + 'created', :source_version, :source_uri, + :watermark_before, :last_good_dataset_version + ) + """ + ), + { + "run_id": run_id, + "source_name": source_name, + "dataset_version": dataset_version, + "job_id": uuid.UUID(job_id) if job_id else None, + "source_version": source_version, + "source_uri": source_uri, + "watermark_before": watermark_before, + "last_good_dataset_version": active, + }, + ) + return str(run_id) + + async def record_object( + self, descriptor: StorageObject, *, metadata: Mapping[str, Any] | None = None + ) -> None: + descriptor.validate() + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + INSERT INTO storage_objects ( + storage_object_id, zone, bucket_alias, object_key, + object_version, checksum, content_length, content_type, + source_name, source_ingestion_run_id, immutable, metadata + ) VALUES ( + :storage_object_id, :zone, :bucket_alias, :object_key, + :object_version, :checksum, :content_length, :content_type, + :source_name, :source_ingestion_run_id, :immutable, + CAST(:metadata AS jsonb) + ) + ON CONFLICT (zone, bucket_alias, object_key, object_version) + DO NOTHING + """ + ), + { + **descriptor.__dict__, + "metadata": json.dumps(metadata or {}, sort_keys=True, default=str), + }, + ) + + async def complete_ingestion( + self, + manifest: IngestionManifest, + *, + status: str = "loaded", + ) -> str: + manifest.validate() + manifest_id = uuid.uuid4() + payload = manifest.payload + async with self.sessions() as session, session.begin(): + await session.execute( + text( + """ + INSERT INTO ingestion_run_manifests ( + ingestion_run_manifest_id, source_ingestion_run_id, + schema_version, source_name, source_version, dataset_version, + raw_object_id, validated_object_id, curated_object_id, + watermark_before, watermark_after, record_count, + accepted_count, rejected_count, quarantined_count, + retry_count, manifest_checksum, manifest + ) VALUES ( + :manifest_id, :run_id, :schema_version, :source_name, + :source_version, :dataset_version, :raw_object_id, + :validated_object_id, :curated_object_id, :watermark_before, + :watermark_after, :record_count, :accepted_count, + :rejected_count, :quarantined_count, :retry_count, + :manifest_checksum, CAST(:manifest AS jsonb) + ) + """ + ), + { + "manifest_id": manifest_id, + "run_id": manifest.source_ingestion_run_id, + **payload, + "raw_object_id": manifest.raw_object_id, + "validated_object_id": manifest.validated_object_id, + "curated_object_id": manifest.curated_object_id, + "manifest_checksum": manifest.checksum, + "manifest": json.dumps(payload, sort_keys=True), + }, + ) + updated = await session.scalar( + text( + """ + UPDATE source_ingestion_runs + SET status=:status, completed_at=CURRENT_TIMESTAMP, + object_checksum=:manifest_checksum, + record_count=:record_count, + accepted_count=:accepted_count, + rejected_count=:rejected_count, + quarantined_count=:quarantined_count, + retry_count=:retry_count, + watermark_after=:watermark_after, + metrics=CAST(:metrics AS jsonb) + WHERE source_ingestion_run_id=:run_id + AND status IN ('created', 'fetching', 'validating') + RETURNING source_ingestion_run_id + """ + ), + { + "status": status, + "manifest_checksum": manifest.checksum, + "record_count": manifest.record_count, + "accepted_count": manifest.accepted_count, + "rejected_count": manifest.rejected_count, + "quarantined_count": manifest.quarantined_count, + "retry_count": manifest.retry_count, + "watermark_after": manifest.watermark_after, + "metrics": json.dumps( + { + "accepted": manifest.accepted_count, + "rejected": manifest.rejected_count, + "quarantined": manifest.quarantined_count, + }, + sort_keys=True, + ), + "run_id": manifest.source_ingestion_run_id, + }, + ) + if updated is None: + raise ValueError("Ingestion run cannot transition to completed") + return str(manifest_id) diff --git a/src/bff/postgres/registry_repository.py b/src/bff/postgres/registry_repository.py new file mode 100644 index 0000000..44a16a6 --- /dev/null +++ b/src/bff/postgres/registry_repository.py @@ -0,0 +1,631 @@ +"""Deterministic PostgreSQL full-text and trigram registry search.""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from decimal import Decimal, InvalidOperation +import hashlib +import json +from typing import Any, Optional, Sequence + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker + +from bff.postgres.base import ( + ANALYTICS_CACHE, + PostgresRepository, + iso_value, + number_value, +) + + +_SEARCH_SQL = text( + """ + WITH active_dataset AS ( + SELECT dataset_version + FROM dataset_versions + WHERE is_active + ), + ranked AS ( + SELECT + registry.registry_id, + registry.charity_number, + registry.linked_charity_number, + registry.registered_name, + registry.registration_status, + registry.income, + registry.expenditure, + registry.city, + registry.administrative_region, + registry.country_code, + round(( + ts_rank_cd( + registry.search_vector, + websearch_to_tsquery('simple'::regconfig, :query) + ) + + similarity(registry.normalized_name, :normalized_query) + )::numeric, 8) AS search_rank + FROM charity_registry_organizations AS registry + JOIN active_dataset + ON active_dataset.dataset_version = registry.dataset_version + WHERE ( + registry.search_vector + @@ websearch_to_tsquery('simple'::regconfig, :query) + OR registry.normalized_name % :normalized_query + ) + AND registry.is_current_source_record + AND (CAST(:registration_status AS text) IS NULL + OR registry.registration_status = CAST(:registration_status AS text)) + ) + SELECT * + FROM ranked + WHERE ( + CAST(:cursor_rank AS numeric) IS NULL + OR search_rank < CAST(:cursor_rank AS numeric) + OR (search_rank = CAST(:cursor_rank AS numeric) + AND registry_id > :cursor_registry_id) + ) + ORDER BY search_rank DESC, registry_id ASC + LIMIT :limit + """ +) + + +@dataclass(frozen=True) +class SearchCursor: + """Opaque stable cursor ordered by rank descending then registry ID.""" + + rank: Decimal + registry_id: str + + def encode(self) -> str: + payload = json.dumps( + {"rank": format(self.rank, "f"), "registry_id": self.registry_id}, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + @classmethod + def decode(cls, value: str) -> "SearchCursor": + if not value or len(value) > 2048: + raise ValueError("Invalid search cursor") + try: + padding = "=" * (-len(value) % 4) + payload = json.loads(base64.urlsafe_b64decode(value + padding)) + rank = Decimal(str(payload["rank"])) + registry_id = str(payload["registry_id"]) + except (ValueError, KeyError, TypeError, InvalidOperation, json.JSONDecodeError) as exc: + raise ValueError("Invalid search cursor") from exc + if not rank.is_finite() or not registry_id or len(registry_id) > 500: + raise ValueError("Invalid search cursor") + return cls(rank=rank, registry_id=registry_id) + + +class RegistrySearchRepository: + """Execute bounded, index-backed registry searches with stable pagination.""" + + def __init__(self, sessions: async_sessionmaker[AsyncSession]): + self._sessions = sessions + + async def search( + self, + query: str, + *, + registration_status: Optional[str] = None, + cursor: Optional[str] = None, + limit: int = 50, + ) -> dict[str, Any]: + normalized_query = " ".join(str(query or "").casefold().split()) + if len(normalized_query) < 2 or len(normalized_query) > 200: + raise ValueError("Search query must contain between 2 and 200 characters") + if limit < 1 or limit > 100: + raise ValueError("Search limit must be between 1 and 100") + decoded = SearchCursor.decode(cursor) if cursor else None + parameters = { + "query": str(query).strip(), + "normalized_query": normalized_query, + "registration_status": registration_status, + "cursor_rank": format(decoded.rank, "f") if decoded else None, + "cursor_registry_id": decoded.registry_id if decoded else "", + "limit": limit + 1, + } + async with self._sessions() as session: + result = await session.execute(_SEARCH_SQL, parameters) + rows = [dict(row) for row in result.mappings().all()] + has_more = len(rows) > limit + page = rows[:limit] + next_cursor = None + if has_more and page: + final_row = page[-1] + next_cursor = SearchCursor( + rank=Decimal(str(final_row["search_rank"])), + registry_id=str(final_row["registry_id"]), + ).encode() + return {"items": page, "next_cursor": next_cursor} + + +@dataclass(frozen=True) +class RegistryPageCursor: + offset: int + signature: str + + def encode(self) -> str: + payload = json.dumps( + {"offset": self.offset, "signature": self.signature}, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return base64.urlsafe_b64encode(payload).decode("ascii").rstrip("=") + + @classmethod + def decode(cls, value: str, signature: str) -> "RegistryPageCursor": + if not value or len(value) > 500: + raise ValueError("Invalid registry cursor") + try: + padding = "=" * (-len(value) % 4) + payload = json.loads(base64.urlsafe_b64decode(value + padding)) + cursor = cls(offset=int(payload["offset"]), signature=str(payload["signature"])) + except (ValueError, KeyError, TypeError, json.JSONDecodeError) as exc: + raise ValueError("Invalid registry cursor") from exc + if cursor.offset < 0 or cursor.offset > 10_000_000 or cursor.signature != signature: + raise ValueError("Registry cursor does not match the requested filters") + return cursor + + +class RegistryRepository(PostgresRepository): + """Bounded directory filters and lazy record detail for the active dataset.""" + + @staticmethod + def _filter_signature(filters: dict[str, Any]) -> str: + payload = json.dumps(filters, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(payload.encode()).hexdigest()[:20] + + async def _registry_count(self, dataset_version: str) -> int: + async def load() -> int: + async with self.sessions() as session: + count = await session.scalar( + text( + """ + SELECT COUNT(*) FROM charity_registry_organizations + WHERE dataset_version=:dataset_version + AND is_current_source_record + """ + ), + {"dataset_version": dataset_version}, + ) + return int(count or 0) + + return await ANALYTICS_CACHE.get_or_create( + (dataset_version, "registry_count"), load + ) + + async def page( + self, + *, + query: Optional[str] = None, + charity_number: Optional[str] = None, + status: Optional[str] = None, + income_min: Optional[float] = None, + income_max: Optional[float] = None, + expenditure_min: Optional[float] = None, + expenditure_max: Optional[float] = None, + country: Optional[str] = None, + region: Optional[str] = None, + beneficiary_geography: Optional[str] = None, + has_enriched_profile: Optional[bool] = None, + has_grant_data: Optional[bool] = None, + cursor: Optional[str] = None, + limit: int = 50, + sort: str = "name", + ) -> dict[str, Any]: + limit = min(max(int(limit), 1), 100) + normalized_query = " ".join(str(query or "").casefold().split()) + if normalized_query and len(normalized_query) < 2: + raise ValueError("Search query must contain at least two characters") + filters = { + "query": normalized_query or None, + "charity_number": charity_number, + "status": status, + "income_min": income_min, + "income_max": income_max, + "expenditure_min": expenditure_min, + "expenditure_max": expenditure_max, + "country": country.upper() if country else None, + "region": region, + "beneficiary_geography": beneficiary_geography, + "has_enriched_profile": has_enriched_profile, + "has_grant_data": has_grant_data, + "sort": sort, + } + advanced_filters = any( + value is not None + for value in ( + charity_number, + income_min, + income_max, + expenditure_min, + expenditure_max, + country, + region, + beneficiary_geography, + has_enriched_profile, + has_grant_data, + ) + ) + if normalized_query and not advanced_filters and sort == "name": + return await self._search_page( + query=str(query), + normalized_query=normalized_query, + status=status, + cursor=cursor, + limit=limit, + filters=filters, + ) + signature = self._filter_signature(filters) + decoded = RegistryPageCursor.decode(cursor, signature) if cursor else None + offset = decoded.offset if decoded else 0 + conditions = ["registry.dataset_version=:dataset_version", "registry.is_current_source_record"] + parameters: dict[str, Any] = { + **filters, + "query_text": str(query or "").strip(), + "normalized_query": normalized_query, + "limit": limit + 1, + "offset": offset, + } + if normalized_query: + conditions.append( + """ + (registry.search_vector @@ websearch_to_tsquery( + 'simple'::regconfig, :query_text + ) OR registry.normalized_name % :normalized_query) + """ + ) + if charity_number: + conditions.append("registry.charity_number=:charity_number") + if status: + conditions.append("registry.registration_status=:status") + if income_min is not None: + conditions.append("registry.income>=:income_min") + if income_max is not None: + conditions.append("registry.income<=:income_max") + if expenditure_min is not None: + conditions.append("registry.expenditure>=:expenditure_min") + if expenditure_max is not None: + conditions.append("registry.expenditure<=:expenditure_max") + if country: + conditions.append("registry.country_code=:country") + if region: + conditions.append("registry.administrative_region=:region") + if beneficiary_geography: + conditions.append( + """ + EXISTS ( + SELECT 1 FROM organization_registry_links AS link + JOIN grants AS grant_row + ON grant_row.dataset_version=link.dataset_version + AND grant_row.funding_charity_id=link.enriched_organization_id + JOIN grant_beneficiary_countries AS geography + ON geography.dataset_version=grant_row.dataset_version + AND geography.grant_id=grant_row.grant_id + WHERE link.dataset_version=registry.dataset_version + AND link.registry_id=registry.registry_id + AND link.match_status='accepted' + AND geography.country_name=:beneficiary_geography + ) + """ + ) + enriched_expression = ( + "EXISTS (SELECT 1 FROM organization_registry_links AS link " + "WHERE link.dataset_version=registry.dataset_version " + "AND link.registry_id=registry.registry_id AND link.match_status='accepted')" + ) + grant_expression = ( + "EXISTS (SELECT 1 FROM grants AS grant_row WHERE " + "grant_row.dataset_version=registry.dataset_version AND (" + "grant_row.funding_org_source_id='GB-CHC-' || registry.charity_number OR " + "grant_row.recipient_org_source_id='GB-CHC-' || registry.charity_number))" + ) + if has_enriched_profile is not None: + conditions.append(f"{enriched_expression}=:has_enriched_profile") + if has_grant_data is not None: + conditions.append(f"{grant_expression}=:has_grant_data") + order = { + "name": "registry.registered_name ASC, registry.registry_id ASC", + "income_desc": "registry.income DESC NULLS LAST, registry.registry_id ASC", + "expenditure_desc": "registry.expenditure DESC NULLS LAST, registry.registry_id ASC", + }.get(sort) + if not order: + raise ValueError("Unsupported registry sort") + sql = text( + f""" + SELECT registry.registry_id, registry.charity_number, + registry.registered_name, registry.registration_status, + registry.income, registry.expenditure, registry.city, + registry.administrative_region, registry.country_code, + registry.source_record_updated_at, + {enriched_expression} AS has_enriched_profile, + {grant_expression} AS has_grant_data, + EXISTS ( + SELECT 1 FROM organization_registry_links AS philea_link + JOIN charities AS profile + ON profile.dataset_version=philea_link.dataset_version + AND profile.charity_id=philea_link.enriched_organization_id + WHERE philea_link.dataset_version=registry.dataset_version + AND philea_link.registry_id=registry.registry_id + AND philea_link.match_status='accepted' + AND profile.primary_source='Philea' + ) AS has_philea_data + FROM charity_registry_organizations AS registry + WHERE {' AND '.join(conditions)} + ORDER BY {order} + LIMIT :limit OFFSET :offset + """ + ) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + parameters["dataset_version"] = dataset_version + rows = [ + dict(row) + for row in (await session.execute(sql, parameters)).mappings() + ] + registry_count = await self._registry_count(dataset_version) + has_more = len(rows) > limit + page_rows = rows[:limit] + results = [ + { + **row, + "income": number_value(row.get("income")), + "expenditure": number_value(row.get("expenditure")), + "source_record_updated_at": iso_value(row.get("source_record_updated_at")), + } + for row in page_rows + ] + next_cursor = ( + RegistryPageCursor(offset=offset + limit, signature=signature).encode() + if has_more else None + ) + return { + "results": results, + "next_cursor": next_cursor, + "has_more": has_more, + "applied_filters": filters, + "page_size": len(results), + "registry_count": int(registry_count or 0), + "search_strategy": ( + "postgresql_tsvector_trigram" if normalized_query else "postgresql_indexed_filters" + ), + } + + async def _search_page( + self, + *, + query: str, + normalized_query: str, + status: Optional[str], + cursor: Optional[str], + limit: int, + filters: dict[str, Any], + ) -> dict[str, Any]: + limit = min(max(int(limit), 1), 100) + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + exact_rows = [ + dict(row) + for row in ( + await session.execute( + text( + """ + SELECT registry_id, charity_number, linked_charity_number, + registered_name, registration_status, income, + expenditure, city, administrative_region, + country_code, 1.0::numeric AS search_rank + FROM charity_registry_organizations + WHERE dataset_version=:dataset_version + AND is_current_source_record + AND normalized_name=:normalized_query + AND (CAST(:status AS text) IS NULL + OR registration_status=CAST(:status AS text)) + ORDER BY registry_id LIMIT :limit + """ + ), + { + "dataset_version": dataset_version, + "normalized_query": normalized_query, + "status": status, + "limit": limit + 1, + }, + ) + ).mappings() + ] + if exact_rows and cursor: + raise ValueError("Exact registry results do not accept a continuation cursor") + if exact_rows: + candidates = exact_rows[:limit] + next_cursor = None + has_more = len(exact_rows) > limit + strategy = "postgresql_exact_normalized_name" + else: + search_result = await RegistrySearchRepository(self.sessions).search( + query, + registration_status=status, + cursor=cursor, + limit=limit, + ) + candidates = list(search_result["items"]) + next_cursor = search_result["next_cursor"] + has_more = next_cursor is not None + strategy = "postgresql_tsvector_trigram_ranked" + candidate_ids = [str(row["registry_id"]) for row in candidates] + registry_count = await self._registry_count(dataset_version) + if not candidate_ids: + return { + "results": [], + "next_cursor": None, + "has_more": False, + "applied_filters": filters, + "page_size": 0, + "registry_count": registry_count, + "search_strategy": strategy, + } + async with self.sessions() as session: + rows = [ + dict(row) + for row in ( + await session.execute( + text( + """ + SELECT registry.registry_id, registry.charity_number, + registry.registered_name, + registry.registration_status, registry.income, + registry.expenditure, registry.city, + registry.administrative_region, + registry.country_code, + registry.source_record_updated_at, + EXISTS ( + SELECT 1 FROM organization_registry_links AS link + WHERE link.dataset_version=registry.dataset_version + AND link.registry_id=registry.registry_id + AND link.match_status='accepted' + ) AS has_enriched_profile, + EXISTS ( + SELECT 1 FROM grants AS grant_row + WHERE grant_row.dataset_version=registry.dataset_version + AND ( + grant_row.funding_org_source_id= + 'GB-CHC-' || registry.charity_number + OR grant_row.recipient_org_source_id= + 'GB-CHC-' || registry.charity_number + ) + ) AS has_grant_data, + EXISTS ( + SELECT 1 FROM organization_registry_links AS link + JOIN charities AS profile + ON profile.dataset_version=link.dataset_version + AND profile.charity_id=link.enriched_organization_id + WHERE link.dataset_version=registry.dataset_version + AND link.registry_id=registry.registry_id + AND link.match_status='accepted' + AND profile.primary_source='Philea' + ) AS has_philea_data + FROM charity_registry_organizations AS registry + WHERE registry.dataset_version=:dataset_version + AND registry.registry_id=ANY(CAST(:ids AS text[])) + """ + ), + {"dataset_version": dataset_version, "ids": candidate_ids}, + ) + ).mappings() + ] + by_id = {str(row["registry_id"]): row for row in rows} + results = [] + for registry_id in candidate_ids: + row = by_id[registry_id] + results.append( + { + **row, + "income": number_value(row.get("income")), + "expenditure": number_value(row.get("expenditure")), + "source_record_updated_at": iso_value( + row.get("source_record_updated_at") + ), + } + ) + return { + "results": results, + "next_cursor": next_cursor, + "has_more": has_more, + "applied_filters": filters, + "page_size": len(results), + "registry_count": int(registry_count or 0), + "search_strategy": strategy, + } + + async def detail(self, registry_id: str) -> Optional[dict[str, Any]]: + async with self.sessions() as session: + dataset_version = await self.active_dataset(session) + row = ( + await session.execute( + text( + """ + SELECT registry.*, + link.enriched_organization_id, + profile.name AS organization_name, + link.match_status, link.match_method, + link.match_confidence, link.match_reason, + EXISTS ( + SELECT 1 FROM grants AS grant_row + WHERE grant_row.dataset_version=registry.dataset_version + AND (grant_row.funding_charity_id=link.enriched_organization_id + OR grant_row.recipient_charity_id=link.enriched_organization_id) + ) AS linked_has_grant_data, + (profile.primary_source='Philea') AS linked_has_philea_data + FROM charity_registry_organizations AS registry + LEFT JOIN organization_registry_links AS link + ON link.dataset_version=registry.dataset_version + AND link.registry_id=registry.registry_id + AND link.match_status='accepted' + LEFT JOIN charities AS profile + ON profile.dataset_version=link.dataset_version + AND profile.charity_id=link.enriched_organization_id + WHERE registry.dataset_version=:dataset_version + AND registry.registry_id=:registry_id + ORDER BY link.updated_at DESC NULLS LAST + LIMIT 1 + """ + ), + {"dataset_version": dataset_version, "registry_id": registry_id}, + ) + ).mappings().first() + if not row: + return None + item = dict(row) + address_lines = [ + str(item[name]).strip() + for name in ( + "address_line_one", "address_line_two", "address_line_three", + "address_line_four", "address_line_five", + ) + if item.get(name) and str(item[name]).strip() + ] + enriched = None + if item.get("enriched_organization_id") is not None: + enriched = { + "enriched_organization_id": int(item["enriched_organization_id"]), + "organization_name": str(item.get("organization_name") or "Unknown"), + "match_status": str(item.get("match_status") or "accepted"), + "match_method": str(item.get("match_method") or "unknown"), + "match_confidence": number_value(item.get("match_confidence")), + "match_reason": item.get("match_reason"), + "has_grant_data": bool(item.get("linked_has_grant_data")), + "has_philea_data": bool(item.get("linked_has_philea_data")), + } + return { + "registry_id": str(item["registry_id"]), + "charity_number": str(item["charity_number"]), + "linked_charity_number": item.get("linked_charity_number"), + "registered_name": str(item["registered_name"]), + "registration_status": item.get("registration_status"), + "registration_date": iso_value(item.get("registration_date")), + "removal_date": iso_value(item.get("removal_date")), + "income": number_value(item.get("income")), + "expenditure": number_value(item.get("expenditure")), + "financial_period_end_date": iso_value(item.get("financial_period_end_date")), + "address_lines": address_lines, + "postcode": item.get("postcode"), + "city": item.get("city"), + "administrative_region": item.get("administrative_region"), + "country_code": item.get("country_code"), + "activity_text": item.get("activity_text"), + "source_name": str(item["source_name"]), + "source_record_updated_at": iso_value(item.get("source_record_updated_at")), + "imported_at": iso_value(item["imported_at"]), + "is_current_source_record": bool(item["is_current_source_record"]), + "observed_grant_data_message": ( + "Observed grant data is available for the accepted enriched profile." + if enriched and enriched["has_grant_data"] + else "Registry presence does not imply observed grant data." + ), + "enriched_profile": enriched, + } diff --git a/src/bff/postgres/routes.py b/src/bff/postgres/routes.py new file mode 100644 index 0000000..a63c795 --- /dev/null +++ b/src/bff/postgres/routes.py @@ -0,0 +1,642 @@ +"""Complete PostgreSQL-only organization and grant API surface.""" + +from __future__ import annotations + +from datetime import date +from typing import Any, Optional + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, status + +from bff.postgres.analytics_repository import AnalyticsRepository +from bff.postgres.funder_repository import SourceFunderRepository +from bff.postgres.job_repository import PostgresJobRepository +from bff.postgres.organization_repository import OrganizationRepository +from bff.postgres.registry_repository import RegistryRepository +from bff.schemas import ( + CharityBase, + CharityDetail, + CharityStats, + GrantListResponse, + GrantMapResponse, + GrantNetworkSummary, + GrantThemesResponse, + GrantTrendsResponse, + PipelineStatus, + RegistryDirectoryPage, + RegistryOrganizationDetail, + SankeyData, + ScoreRequest, + ScoreResponse, + SourceFunderDetailResponse, + SourceFunderEnrichmentRequest, + SourceFunderListResponse, + SourceFunderRelinkRequest, +) +from bff.security import Role, require_roles + + +router = APIRouter( + prefix="/api/charities", + tags=["Organization and Grant Data"], + dependencies=[Depends(require_roles(Role.VIEWER, action="charity.read"))], +) + + +def _sessions(request: Request): + return request.app.state.database.sessions() + + +def _organizations(request: Request) -> OrganizationRepository: + return OrganizationRepository(_sessions(request)) + + +def _registry(request: Request) -> RegistryRepository: + return RegistryRepository(_sessions(request)) + + +def _analytics(request: Request) -> AnalyticsRepository: + return AnalyticsRepository(_sessions(request)) + + +def _funders(request: Request) -> SourceFunderRepository: + return SourceFunderRepository(_sessions(request)) + + +def _jobs(request: Request) -> PostgresJobRepository: + return PostgresJobRepository(_sessions(request)) + + +def _split(value: Optional[str]) -> list[str]: + return [item.strip() for item in (value or "").split(",") if item.strip()] + + +def _iso_date(value: Optional[str], field: str) -> Optional[str]: + if value is None or not value.strip(): + return None + try: + return date.fromisoformat(value).isoformat() + except ValueError as exc: + raise HTTPException( + status_code=400, + detail=f"{field} must be an ISO date (YYYY-MM-DD)", + ) from exc + + +def _date_range(date_from: Optional[str], date_to: Optional[str]) -> tuple[Optional[str], Optional[str]]: + parsed_from = _iso_date(date_from, "date_from") + parsed_to = _iso_date(date_to, "date_to") + if parsed_from and parsed_to and parsed_from > parsed_to: + raise HTTPException(status_code=400, detail="date_from cannot be after date_to") + return parsed_from, parsed_to + + +def _actor(request: Request) -> str: + principal = getattr(request.state, "principal", None) + return principal.actor_id if principal else "unknown" + + +def _idempotency_key(request: Request) -> str: + return str(request.headers.get("idempotency-key") or "").strip() + + +def _grant_filters( + *, + currency: Optional[str], + date_from: Optional[str], + date_to: Optional[str], + beneficiary_geographies: Optional[str], + programme_areas: Optional[str], + donor: Optional[str], + recipient: Optional[str], + sources: Optional[str], +) -> dict[str, Any]: + parsed_from, parsed_to = _date_range(date_from, date_to) + return { + "currency": currency, + "date_from": parsed_from, + "date_to": parsed_to, + "beneficiary_geographies": _split(beneficiary_geographies), + "programme_areas": _split(programme_areas), + "donor": donor, + "recipient": recipient, + "sources": _split(sources) if sources is not None else None, + } + + +@router.get("", response_model=list[CharityBase]) +async def list_charities( + search: Optional[str] = None, + reg_status: Optional[str] = None, + tag: Optional[str] = None, + region: Optional[str] = None, + size: Optional[str] = None, + tags: Optional[str] = None, + foundation_regions: Optional[str] = None, + funding_regions: Optional[str] = None, + sources: Optional[str] = Query(default=None, max_length=500), + min_annual_giving: Optional[float] = None, + max_annual_giving: Optional[float] = Query(default=None, ge=0), + min_avg_grant_size: Optional[float] = None, + max_avg_grant_size: Optional[float] = Query(default=None, ge=0), + include_score: bool = Query(default=False), + sort: str = Query(default="name_asc", pattern="^(score_desc|income_desc|name_asc)$"), + skip: int = Query(default=0, ge=0), + limit: int = Query(default=20, ge=1, le=100), + repository: OrganizationRepository = Depends(_organizations), +): + return await repository.list( + search=search, + reg_status=reg_status, + tag=tag, + region=region, + size=size, + tags=_split(tags), + foundation_regions=_split(foundation_regions), + funding_regions=_split(funding_regions), + sources=_split(sources) if sources is not None else None, + min_annual_giving=min_annual_giving, + max_annual_giving=max_annual_giving, + min_avg_grant_size=min_avg_grant_size, + max_avg_grant_size=max_avg_grant_size, + include_score=include_score, + sort=sort, + skip=skip, + limit=limit, + ) + + +@router.get("/stats", response_model=CharityStats) +async def charity_stats(repository: OrganizationRepository = Depends(_organizations)): + return await repository.stats() + + +@router.get("/directory/organizations", response_model=RegistryDirectoryPage) +async def registry_page( + query: Optional[str] = Query(default=None, max_length=160), + charity_number: Optional[str] = Query(default=None, max_length=64), + status_filter: Optional[str] = Query(default=None, alias="status", max_length=80), + income_min: Optional[float] = Query(default=None, ge=0), + income_max: Optional[float] = Query(default=None, ge=0), + expenditure_min: Optional[float] = Query(default=None, ge=0), + expenditure_max: Optional[float] = Query(default=None, ge=0), + country: Optional[str] = Query(default=None, max_length=8), + region: Optional[str] = Query(default=None, max_length=120), + beneficiary_geography: Optional[str] = Query(default=None, max_length=120), + has_enriched_profile: Optional[bool] = None, + has_grant_data: Optional[bool] = None, + cursor: Optional[str] = Query(default=None, max_length=500), + limit: int = Query(default=50, ge=1, le=100), + sort: str = Query(default="name", pattern="^(name|income_desc|expenditure_desc)$"), + repository: RegistryRepository = Depends(_registry), +): + if income_min is not None and income_max is not None and income_min > income_max: + raise HTTPException(status_code=400, detail="income_min cannot exceed income_max") + if expenditure_min is not None and expenditure_max is not None and expenditure_min > expenditure_max: + raise HTTPException(status_code=400, detail="expenditure_min cannot exceed expenditure_max") + try: + return await repository.page( + query=query, + charity_number=charity_number, + status=status_filter, + income_min=income_min, + income_max=income_max, + expenditure_min=expenditure_min, + expenditure_max=expenditure_max, + country=country, + region=region, + beneficiary_geography=beneficiary_geography, + has_enriched_profile=has_enriched_profile, + has_grant_data=has_grant_data, + cursor=cursor, + limit=limit, + sort=sort, + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + +@router.get( + "/directory/organizations/{registry_id}", + response_model=RegistryOrganizationDetail, +) +async def registry_detail( + registry_id: str, + repository: RegistryRepository = Depends(_registry), +): + result = await repository.detail(registry_id) + if not result: + raise HTTPException(status_code=404, detail="Registry organization not found.") + return result + + +@router.post( + "/directory/organizations/enrich", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="registry.enrich", idempotent=True))], +) +async def enrich_registry( + payload: SourceFunderEnrichmentRequest, + request: Request, + jobs: PostgresJobRepository = Depends(_jobs), +): + numbers = sorted({int(value) for value in payload.reg_numbers}) + if len(numbers) != 1: + raise HTTPException(status_code=400, detail="Exactly one registry organization is required") + job = await jobs.enqueue( + "registry_enrichment", + {"reg_numbers": numbers, "skip_contact_crawler": payload.skip_contact_crawler}, + actor_id=_actor(request), + idempotency_key=_idempotency_key(request), + ) + return { + "status": job["status"], + "started_at": job["requested_at"], + "finished_at": None, + "last_run_source": "registry_enrichment", + "error": None, + "job_id": job["job_id"], + } + + +@router.get("/grants/beneficiary-geographies", response_model=list[str]) +async def beneficiary_geographies( + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.beneficiary_geographies() + + +@router.get("/grants/map", response_model=GrantMapResponse) +async def grant_map( + currency: Optional[str] = None, + min_coverage: float = Query(default=0.30, ge=0, le=1), + search: Optional[str] = None, + tags: Optional[str] = None, + foundation_regions: Optional[str] = None, + funding_regions: Optional[str] = None, + min_annual_giving: Optional[float] = None, + min_avg_grant_size: Optional[float] = None, + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.map( + currency=currency, + min_coverage=min_coverage, + donor=search, + programme_areas=_split(tags), + beneficiary_geographies=_split(funding_regions), + foundation_regions=_split(foundation_regions), + min_annual_giving=min_annual_giving, + min_avg_grant_size=min_avg_grant_size, + ) + + +@router.get("/grants/map/connections", response_model=dict[str, Any]) +async def grant_map_connections( + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + limit: int = Query(default=100, ge=1, le=250), + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.map_connections(currency=currency, limit=limit) + + +@router.get("/grants/overview", response_model=dict[str, Any]) +async def grant_overview( + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + date_from: Optional[str] = Query(default=None, max_length=10), + date_to: Optional[str] = Query(default=None, max_length=10), + beneficiary_geographies: Optional[str] = Query(default=None, max_length=500), + programme_areas: Optional[str] = Query(default=None, max_length=1000), + donor: Optional[str] = Query(default=None, max_length=160), + recipient: Optional[str] = Query(default=None, max_length=160), + sources: Optional[str] = Query(default=None, max_length=500), + granularity: str = Query(default="auto", pattern="^(auto|monthly|yearly)$"), + include_connections: bool = False, + repository: AnalyticsRepository = Depends(_analytics), +): + filters = _grant_filters( + currency=currency, date_from=date_from, date_to=date_to, + beneficiary_geographies=beneficiary_geographies, + programme_areas=programme_areas, donor=donor, recipient=recipient, + sources=sources, + ) + return await repository.overview( + **filters, + granularity=granularity, + include_connections=include_connections, + ) + + +@router.get("/grants/overview/entity-suggestions", response_model=dict[str, Any]) +async def grant_suggestions( + sources: Optional[str] = Query(default=None, max_length=500), + limit: int = Query(default=2500, ge=1, le=5000), + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.suggestions( + sources=_split(sources) if sources is not None else None, + limit=limit, + ) + + +@router.get("/grants/overview/trends", response_model=GrantTrendsResponse) +async def overview_trends( + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + date_from: Optional[str] = Query(default=None, max_length=10), + date_to: Optional[str] = Query(default=None, max_length=10), + beneficiary_geographies: Optional[str] = Query(default=None, max_length=500), + programme_areas: Optional[str] = Query(default=None, max_length=1000), + donor: Optional[str] = Query(default=None, max_length=160), + recipient: Optional[str] = Query(default=None, max_length=160), + sources: Optional[str] = Query(default=None, max_length=500), + granularity: str = Query(default="auto", pattern="^(auto|monthly|yearly)$"), + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.trends( + **_grant_filters( + currency=currency, date_from=date_from, date_to=date_to, + beneficiary_geographies=beneficiary_geographies, + programme_areas=programme_areas, donor=donor, recipient=recipient, + sources=sources, + ), + granularity=granularity, + ) + + +@router.get("/grants/overview/drilldown", response_model=dict[str, Any]) +async def overview_drilldown( + selection_type: str = Query(..., pattern="^(period|programme_area)$"), + selection_value: str = Query(..., min_length=1, max_length=160), + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + date_from: Optional[str] = Query(default=None, max_length=10), + date_to: Optional[str] = Query(default=None, max_length=10), + beneficiary_geographies: Optional[str] = Query(default=None, max_length=500), + programme_areas: Optional[str] = Query(default=None, max_length=1000), + donor: Optional[str] = Query(default=None, max_length=160), + recipient: Optional[str] = Query(default=None, max_length=160), + sources: Optional[str] = Query(default=None, max_length=500), + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.drilldown( + selection_type=selection_type, + selection_value=selection_value, + **_grant_filters( + currency=currency, date_from=date_from, date_to=date_to, + beneficiary_geographies=beneficiary_geographies, + programme_areas=programme_areas, donor=donor, recipient=recipient, + sources=sources, + ), + ) + + +@router.post( + "/grants/funders/enrich", + response_model=PipelineStatus, + dependencies=[Depends(require_roles(Role.OPERATOR, action="funder.enrich", idempotent=True))], +) +async def enrich_funders( + payload: SourceFunderEnrichmentRequest, + request: Request, + jobs: PostgresJobRepository = Depends(_jobs), +): + job = await jobs.enqueue( + "source_funder_enrichment", + payload.model_dump(), + actor_id=_actor(request), + idempotency_key=_idempotency_key(request), + ) + return { + "status": job["status"], + "started_at": job["requested_at"], + "finished_at": None, + "last_run_source": "source_funder_enrichment", + "error": None, + "job_id": job["job_id"], + } + + +@router.post( + "/grants/funders/{source_funder_key}/reset-to-observed", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="funder.reset", idempotent=True))], +) +async def reset_funder( + source_funder_key: str, + request: Request, + repository: SourceFunderRepository = Depends(_funders), +): + result = await repository.reset(source_funder_key, actor_id=_actor(request)) + if not result: + raise HTTPException(status_code=404, detail="Source-funder entry not found.") + return {"status": "observed_only", **result} + + +@router.post( + "/grants/funders/{source_funder_key}/relink", + dependencies=[Depends(require_roles(Role.ADMINISTRATOR, action="funder.relink", idempotent=True))], +) +async def relink_funder( + source_funder_key: str, + payload: SourceFunderRelinkRequest, + request: Request, + repository: SourceFunderRepository = Depends(_funders), +): + try: + result = await repository.relink( + source_funder_key, payload.profile_id, actor_id=_actor(request) + ) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + if not result: + raise HTTPException(status_code=404, detail="Source-funder entry not found.") + return {"status": "linked", **result} + + +@router.post( + "/grants/funders/{source_funder_key}/profile-cache", + dependencies=[Depends(require_roles(Role.OPERATOR, action="profile_cache.refresh", idempotent=True))], +) +async def queue_profile_cache( + source_funder_key: str, + request: Request, + repository: SourceFunderRepository = Depends(_funders), +): + result = await repository.queue_profile_cache( + source_funder_key, + actor_id=_actor(request), + idempotency_key=_idempotency_key(request), + ) + if not result: + raise HTTPException( + status_code=409, + detail="This source funder has no active organization profile to hydrate.", + ) + return result + + +@router.get("/grants/funders/{source_funder_key}/profile-cache") +async def profile_cache( + source_funder_key: str, + repository: SourceFunderRepository = Depends(_funders), +): + result = await repository.profile_cache(source_funder_key) + if not result: + raise HTTPException(status_code=404, detail="No source-profile cache is available.") + return result + + +@router.get("/grants/funders", response_model=SourceFunderListResponse) +async def source_funders( + beneficiary_country: str = Query(..., min_length=2, max_length=2, pattern="^[A-Za-z]{2}$"), + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + date_from: Optional[str] = Query(default=None, max_length=10), + date_to: Optional[str] = Query(default=None, max_length=10), + beneficiary_geographies: Optional[str] = Query(default=None, max_length=500), + programme_areas: Optional[str] = Query(default=None, max_length=1000), + donor: Optional[str] = Query(default=None, max_length=160), + recipient: Optional[str] = Query(default=None, max_length=160), + sources: Optional[str] = Query(default=None, max_length=500), + search: Optional[str] = Query(default=None, max_length=160), + profile_status: str = Query(default="all", pattern="^(all|linked|observed_only)$"), + sort: str = Query(default="largest_observed_funding", pattern="^(largest_observed_funding|most_grants|most_recently_active|most_active|most_recent)$"), + page: int = Query(default=1, ge=1), + page_size: int = Query(default=25, ge=1, le=100), + repository: SourceFunderRepository = Depends(_funders), +): + filters = _grant_filters( + currency=currency, date_from=date_from, date_to=date_to, + beneficiary_geographies=beneficiary_geographies, + programme_areas=programme_areas, donor=donor, recipient=recipient, + sources=sources, + ) + return await repository.list( + beneficiary_country=beneficiary_country.upper(), + **filters, + search=search, + profile_status=profile_status, + sort=sort, + page=page, + page_size=page_size, + ) + + +@router.get( + "/grants/funders/{source_funder_key}", + response_model=SourceFunderDetailResponse, +) +async def source_funder_detail( + source_funder_key: str, + beneficiary_country: str = Query(..., min_length=2, max_length=2, pattern="^[A-Za-z]{2}$"), + currency: Optional[str] = Query(default=None, min_length=3, max_length=4), + date_from: Optional[str] = Query(default=None, max_length=10), + date_to: Optional[str] = Query(default=None, max_length=10), + beneficiary_geographies: Optional[str] = Query(default=None, max_length=500), + programme_areas: Optional[str] = Query(default=None, max_length=1000), + donor: Optional[str] = Query(default=None, max_length=160), + recipient: Optional[str] = Query(default=None, max_length=160), + sources: Optional[str] = Query(default=None, max_length=500), + detail_level: str = Query(default="full", pattern="^(summary|full)$"), + repository: SourceFunderRepository = Depends(_funders), +): + result = await repository.detail( + source_funder_key, + beneficiary_country=beneficiary_country.upper(), + **_grant_filters( + currency=currency, date_from=date_from, date_to=date_to, + beneficiary_geographies=beneficiary_geographies, + programme_areas=programme_areas, donor=donor, recipient=recipient, + sources=sources, + ), + detail_level=detail_level, + ) + if not result: + raise HTTPException(status_code=404, detail="Source funder not found in this scope.") + return result + + +@router.get("/grants/summary", response_model=GrantNetworkSummary) +async def grant_summary(repository: AnalyticsRepository = Depends(_analytics)): + return await repository.summary() + + +@router.get("/grants/trends", response_model=GrantTrendsResponse) +async def grant_trends( + currency: Optional[str] = None, + months: int = Query(default=24, ge=1, le=120), + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.trends(currency=currency, months=months) + + +@router.get("/grants/themes", response_model=GrantThemesResponse) +async def grant_themes( + currency: Optional[str] = None, + repository: AnalyticsRepository = Depends(_analytics), +): + return await repository.themes(currency=currency) + + +@router.get("/{reg_charity_number}", response_model=CharityDetail) +async def charity_detail( + reg_charity_number: int, + repository: OrganizationRepository = Depends(_organizations), +): + result = await repository.detail(reg_charity_number) + if not result: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Charity registration number {reg_charity_number} not found.", + ) + return result + + +@router.get("/{reg_charity_number}/grants", response_model=GrantListResponse) +async def charity_grants( + reg_charity_number: int, + role: str = Query(default="all", pattern="^(all|funder|recipient)$"), + repository: OrganizationRepository = Depends(_organizations), +): + if not await repository.detail(reg_charity_number): + raise HTTPException(status_code=404, detail="Organization not found.") + return await repository.grants(reg_charity_number, role) + + +@router.get("/{reg_charity_number}/sankey", response_model=SankeyData) +async def charity_sankey( + reg_charity_number: int, + currency: Optional[str] = None, + limit: int = Query(default=30, ge=1, le=100), + repository: OrganizationRepository = Depends(_organizations), +): + if not await repository.detail(reg_charity_number): + raise HTTPException(status_code=404, detail="Organization not found.") + return await repository.sankey( + reg_charity_number, + currency=currency, + limit=limit, + ) + + +@router.get("/{reg_charity_number}/score", response_model=ScoreResponse) +async def default_charity_score( + reg_charity_number: int, + repository: OrganizationRepository = Depends(_organizations), +): + try: + return await repository.score(reg_charity_number, None) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Organization not found.") from exc + + +@router.post( + "/{reg_charity_number}/score", + response_model=ScoreResponse, + dependencies=[Depends(require_roles(Role.ANALYST, action="score.calculate"))], +) +async def charity_score( + reg_charity_number: int, + payload: ScoreRequest, + repository: OrganizationRepository = Depends(_organizations), +): + profile = payload.target_profile.model_dump(exclude_none=True) if payload.target_profile else None + try: + return await repository.score(reg_charity_number, profile) + except KeyError as exc: + raise HTTPException(status_code=404, detail="Organization not found.") from exc diff --git a/src/bff/proxy.py b/src/bff/proxy.py index 2dfb44f..2e5fb6a 100644 --- a/src/bff/proxy.py +++ b/src/bff/proxy.py @@ -1,73 +1,116 @@ +"""Narrow, opt-in downstream proxy with fixed destination and allowlists.""" + +from __future__ import annotations + +from pathlib import PurePosixPath +from urllib.parse import quote, urlparse + import httpx -from fastapi import APIRouter, Depends, Request, Response, HTTPException, status -from bff.config import CORE_API_URL -from bff.auth import get_current_user_token +from fastapi import APIRouter, Depends, HTTPException, Request, Response, status + +from bff.config import SECURITY_SETTINGS +from bff.security import Principal, Role, require_roles, reserve_idempotency from bff.utils.logging import logger + router = APIRouter( prefix="/api/core", tags=["Downstream Core API Proxy"], - dependencies=[Depends(get_current_user_token)] ) -@router.api_route("/{path:path}", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"]) + +def _normalized_path(path: str) -> str: + if not path or "\\" in path or "//" in path: + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Proxy path is not allowed.") + parts = PurePosixPath(path).parts + if any(part in {"", ".", ".."} for part in parts): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Proxy path is not allowed.") + return "/".join(parts) + + +def _path_allowed(path: str, allowlist) -> bool: + for rule in allowlist: + normalized_rule = rule.strip().strip("/") + if normalized_rule.endswith("/*"): + prefix = normalized_rule[:-2] + if path == prefix or path.startswith(f"{prefix}/"): + return True + elif path == normalized_rule: + return True + return False + + +@router.api_route( + "/{path:path}", + methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS", "HEAD"], + include_in_schema=False, +) async def proxy_to_core( - path: str, - request: Request, - token: str = Depends(get_current_user_token) + path: str, + request: Request, + principal: Principal = Depends(require_roles(Role.ADMINISTRATOR, action="proxy.request")), ): - """ - Reverse proxy that forwards requests to downstream core APIs. - Translates browser cookie session into an Authorization Bearer JWT header. - """ - # Build downstream URL preserving query string - query_params = request.url.query - url = f"{CORE_API_URL.rstrip('/')}/{path}" - if query_params: - url = f"{url}?{query_params}" - - # Extract method and body - method = request.method - body = await request.body() + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + if not settings.core_proxy_enabled: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + method = request.method.upper() + if method not in settings.core_proxy_allowed_methods: + raise HTTPException(status_code=status.HTTP_405_METHOD_NOT_ALLOWED, detail="Proxy method is not allowed.") + if method not in {"GET", "HEAD", "OPTIONS"}: + await reserve_idempotency(request, principal, "proxy.request") - # Construct request headers - # Copy request headers, ignoring Host and Cookie (so session cookie doesn't leak downstream) - headers = {} - for key, value in request.headers.items(): - if key.lower() not in ["host", "cookie", "content-length"]: - headers[key] = value + safe_path = _normalized_path(path) + if not _path_allowed(safe_path, settings.core_proxy_allowed_paths): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Proxy path is not allowed.") - # Translate Cookie into Bearer Token for internal services - headers["Authorization"] = f"Bearer {token}" + parsed_base = urlparse(settings.core_api_url) + if not parsed_base.hostname or parsed_base.hostname.lower() not in settings.core_api_allowed_hosts: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Downstream proxy destination is not configured safely.", + ) + url = f"{settings.core_api_url.rstrip('/')}/{quote(safe_path, safe='/-._~')}" + if request.url.query: + url = f"{url}?{request.url.query}" - logger.info(f"BFF Proxying {method} request to downstream target: {url}") + headers = { + key.lower(): value + for key, value in request.headers.items() + if key.lower() in settings.core_proxy_forward_headers + and key.lower() not in {"authorization", "cookie", "host", "content-length"} + } + headers["x-request-id"] = request.state.request_id + if settings.core_api_bearer_token: + headers["authorization"] = f"Bearer {settings.core_api_bearer_token}" + body = await request.body() + logger.info("Proxy request: method=%s path=%s", method, safe_path) try: - async with httpx.AsyncClient(timeout=10.0) as client: + async with httpx.AsyncClient( + timeout=float(settings.request_timeout_seconds), + follow_redirects=False, + ) as client: downstream_response = await client.request( method=method, url=url, headers=headers, - content=body if method not in ["GET", "HEAD"] else None, - follow_redirects=True + content=body if method not in {"GET", "HEAD"} else None, ) - - # Return downstream response to frontend, including status code and headers - # Exclude transfer-encoding headers that might interfere with HTTP protocols - response_headers = { - k: v for k, v in downstream_response.headers.items() - if k.lower() not in ["content-length", "content-encoding", "transfer-encoding"] - } - - return Response( - content=downstream_response.content, - status_code=downstream_response.status_code, - headers=response_headers, - media_type=downstream_response.headers.get("content-type") - ) - except httpx.RequestError as e: - logger.error(f"Downstream service proxy request failed for {url}: {e}") + except httpx.RequestError as exc: + logger.error("Downstream proxy request failed; class=%s", exc.__class__.__name__) raise HTTPException( status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Downstream core service is currently unavailable or timed out." - ) + detail="Downstream core service is currently unavailable or timed out.", + ) from exc + + response_headers = { + key: value + for key, value in downstream_response.headers.items() + if key.lower() in settings.core_proxy_response_headers + } + return Response( + content=downstream_response.content, + status_code=downstream_response.status_code, + headers=response_headers, + media_type=downstream_response.headers.get("content-type"), + ) diff --git a/src/bff/schemas.py b/src/bff/schemas.py index 0900707..77ece69 100644 --- a/src/bff/schemas.py +++ b/src/bff/schemas.py @@ -1,5 +1,6 @@ from pydantic import BaseModel, Field from typing import List, Optional, Dict, Any +from datetime import datetime # Authentication Schemas class UserLogin(BaseModel): @@ -622,11 +623,12 @@ class ScoreResponse(BaseModel): not_a_prediction: bool = True class PipelineStatus(BaseModel): - status: str = Field(..., description="idle, running, success, failed") + status: str = Field(..., description="idle, queued, running, success, failed") started_at: Optional[str] = None finished_at: Optional[str] = None last_run_source: Optional[str] = None error: Optional[str] = None + job_id: Optional[str] = None class PipelineTrigger(BaseModel): source: str = Field(..., description="quick_consolidate, refresh_charities, refresh_grants, full_run") @@ -637,6 +639,41 @@ class PipelineTrigger(BaseModel): skip_contact_crawler: Optional[bool] = False +class SourceScheduleUpdate(BaseModel): + enabled: bool + + +class RetentionDryRunRequest(BaseModel): + target_type: str = Field(..., min_length=1, max_length=120) + target_id: str = Field(..., min_length=1, max_length=500) + retention_class: str = Field(..., min_length=1, max_length=120) + last_modified_at: datetime + object_count: int = Field(default=0, ge=0) + record_count: int = Field(default=0, ge=0) + total_bytes: int = Field(default=0, ge=0) + target_checksums: List[str] = Field(default=[], max_length=100) + + +class DataHoldRequest(BaseModel): + hold_type: str = Field(..., pattern="^(legal|incident)$") + scope_type: str = Field(..., min_length=1, max_length=120) + scope_id: str = Field(..., min_length=1, max_length=500) + reason: str = Field(..., min_length=1, max_length=1000) + expires_at: Optional[datetime] = None + + +class DataHoldRelease(BaseModel): + reason: str = Field(..., min_length=1, max_length=1000) + + +class DataSubjectRequestCreate(BaseModel): + request_type: str = Field( + ..., pattern="^(access|correction|deletion|restriction|objection)$" + ) + subject_reference_hash: str = Field(..., pattern="^[a-f0-9]{64}$") + due_at: Optional[datetime] = None + + class SourceFunderEnrichmentRequest(BaseModel): """A deliberately bounded request to enrich observed funders. diff --git a/src/bff/security.py b/src/bff/security.py new file mode 100644 index 0000000..cc56c75 --- /dev/null +++ b/src/bff/security.py @@ -0,0 +1,512 @@ +"""OIDC authentication, role authorization and request abuse controls.""" + +from __future__ import annotations + +from collections import defaultdict, deque +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone +from enum import Enum +import hashlib +import hmac +import inspect +import json +import threading +import time +from typing import Any, Deque, Dict, FrozenSet, Iterable, Mapping, Optional, Tuple + +import httpx +from fastapi import HTTPException, Request, status +from jose import JWTError, jwt + +from bff.config import SECURITY_SETTINGS, SecuritySettings +from bff.utils.logging import logger + + +class Role(str, Enum): + VIEWER = "viewer" + ANALYST = "analyst" + OPERATOR = "operator" + ADMINISTRATOR = "administrator" + + +_ROLE_RANK = { + Role.VIEWER: 10, + Role.ANALYST: 20, + Role.OPERATOR: 30, + Role.ADMINISTRATOR: 40, +} + + +# This allowlist deliberately contains only the read-only route templates used +# by the normal demo UI. Operational, authentication, live-research, cache and +# mutation surfaces stay private even when they use GET internally. +# Route templates, rather than path prefixes or a blanket GET rule, make newly +# added viewer endpoints private until they receive an explicit data review. +PUBLIC_READONLY_ROUTE_ALLOWLIST: FrozenSet[str] = frozenset( + { + "/api/charities", + "/api/charities/stats", + "/api/charities/{reg_charity_number}", + "/api/charities/{reg_charity_number}/grants", + "/api/charities/{reg_charity_number}/sankey", + "/api/charities/{reg_charity_number}/score", + "/api/charities/directory/organizations", + "/api/charities/directory/organizations/{registry_id}", + "/api/charities/grants/beneficiary-geographies", + "/api/charities/grants/funders", + "/api/charities/grants/funders/{source_funder_key}", + "/api/charities/grants/map", + "/api/charities/grants/map/connections", + "/api/charities/grants/overview", + "/api/charities/grants/overview/drilldown", + "/api/charities/grants/overview/entity-suggestions", + "/api/charities/grants/overview/trends", + "/api/charities/grants/summary", + "/api/charities/grants/themes", + "/api/charities/grants/trends", + } +) +PUBLIC_READONLY_METHOD_ALLOWLIST: FrozenSet[str] = frozenset({"GET", "HEAD"}) + + +@dataclass(frozen=True) +class Principal: + actor_id: str + roles: FrozenSet[Role] + claims: Mapping[str, Any] + + @property + def primary_role(self) -> str: + if not self.roles: + return "unassigned" + return max(self.roles, key=lambda role: _ROLE_RANK[role]).value + + def permits_any(self, required_roles: Iterable[Role]) -> bool: + required = tuple(required_roles) + if not required: + return True + highest_grant = max((_ROLE_RANK[role] for role in self.roles), default=0) + return any(highest_grant >= _ROLE_RANK[role] for role in required) + + +class SlidingWindowRateLimiter: + """Process-local protection; the AWS edge layer supplies distributed limiting.""" + + def __init__(self, limit: int, window_seconds: int): + self.limit = limit + self.window_seconds = window_seconds + self._events: Dict[str, Deque[float]] = defaultdict(deque) + self._lock = threading.Lock() + + def check(self, key: str, now: Optional[float] = None) -> Optional[int]: + timestamp = time.monotonic() if now is None else now + oldest_allowed = timestamp - self.window_seconds + with self._lock: + events = self._events[key] + while events and events[0] <= oldest_allowed: + events.popleft() + if len(events) >= self.limit: + retry_after = max(1, int(self.window_seconds - (timestamp - events[0]))) + return retry_after + events.append(timestamp) + return None + + def clear(self) -> None: + with self._lock: + self._events.clear() + + +@dataclass +class _IdempotencyRecord: + fingerprint: str + state: str + created_at: float + + +class IdempotencyStore: + """Thread-safe local/test at-most-once guard.""" + + def __init__(self, ttl_seconds: int = 86_400): + self.ttl_seconds = ttl_seconds + self._records: Dict[Tuple[str, str, str], _IdempotencyRecord] = {} + self._lock = threading.Lock() + + def reserve(self, actor_id: str, action: str, key: str, fingerprint: str) -> Tuple[str, str, str]: + record_key = (actor_id, action, key) + now = time.monotonic() + with self._lock: + expired = [ + candidate + for candidate, record in self._records.items() + if now - record.created_at > self.ttl_seconds + ] + for candidate in expired: + del self._records[candidate] + existing = self._records.get(record_key) + if existing: + raise IdempotencyConflict( + different_request=existing.fingerprint != fingerprint + ) + self._records[record_key] = _IdempotencyRecord(fingerprint, "reserved", now) + return record_key + + def complete(self, record_key: Tuple[str, str, str]) -> None: + with self._lock: + record = self._records.get(record_key) + if record: + record.state = "completed" + + def release(self, record_key: Tuple[str, str, str]) -> None: + with self._lock: + record = self._records.get(record_key) + if record and record.state == "reserved": + del self._records[record_key] + + def clear(self) -> None: + with self._lock: + self._records.clear() + + +class IdempotencyConflict(RuntimeError): + """A durable or local key already exists for the same actor/action.""" + + def __init__(self, *, different_request: bool): + self.different_request = different_request + super().__init__( + "Idempotency-Key was already used with a different request." + if different_request + else "Idempotency-Key has already been processed." + ) + + +class OIDCVerifier: + def __init__(self): + self._jwks: Optional[Mapping[str, Any]] = None + self._loaded_at = 0.0 + self._cache_source: Optional[str] = None + self._lock = threading.Lock() + + async def _load_jwks(self, settings: SecuritySettings) -> Mapping[str, Any]: + now = time.monotonic() + cache_source = settings.oidc_jwks_json or str(settings.oidc_jwks_url) + with self._lock: + if ( + self._jwks + and self._cache_source == cache_source + and now - self._loaded_at < settings.oidc_jwks_cache_seconds + ): + return self._jwks + + if settings.oidc_jwks_json: + payload = json.loads(settings.oidc_jwks_json) + else: + try: + async with httpx.AsyncClient(timeout=5.0, follow_redirects=False) as client: + response = await client.get(str(settings.oidc_jwks_url)) + response.raise_for_status() + if len(response.content) > 256_000: + raise ValueError("OIDC JWKS response exceeds the configured safety bound") + payload = response.json() + except (httpx.HTTPError, ValueError, json.JSONDecodeError) as exc: + logger.warning("OIDC key retrieval failed: %s", exc.__class__.__name__) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Identity provider keys are temporarily unavailable.", + ) from exc + + if not isinstance(payload, dict) or not isinstance(payload.get("keys"), list): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Identity provider key set is invalid.", + ) + with self._lock: + self._jwks = payload + self._loaded_at = now + self._cache_source = cache_source + return payload + + async def decode(self, token: str, settings: SecuritySettings) -> Mapping[str, Any]: + try: + header = jwt.get_unverified_header(token) + except JWTError as exc: + raise _invalid_token() from exc + algorithm = header.get("alg") + key_id = header.get("kid") + if algorithm not in settings.oidc_algorithms or not key_id: + raise _invalid_token() + jwks = await self._load_jwks(settings) + matching_keys = [key for key in jwks["keys"] if key.get("kid") == key_id] + if len(matching_keys) != 1: + raise _invalid_token() + try: + return jwt.decode( + token, + matching_keys[0], + algorithms=list(settings.oidc_algorithms), + audience=settings.oidc_audience, + issuer=settings.oidc_issuer, + options={"require_exp": True, "require_iat": True, "require_sub": True}, + ) + except JWTError as exc: + logger.warning("OIDC token validation failed: %s", exc.__class__.__name__) + raise _invalid_token() from exc + + +_oidc_verifier = OIDCVerifier() + + +def _invalid_token() -> HTTPException: + return HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Access token is invalid or expired.", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def _request_host(request: Request) -> str: + return request.client.host.lower() if request.client and request.client.host else "unknown" + + +def _extract_token(request: Request, settings: SecuritySettings) -> Optional[str]: + authorization = request.headers.get("authorization", "") + if authorization: + scheme, _, credentials = authorization.partition(" ") + if scheme.lower() != "bearer" or not credentials.strip(): + raise _invalid_token() + return credentials.strip() + if settings.auth_mode == "development": + return request.cookies.get(settings.session_cookie_name) + return None + + +def _public_readonly_principal( + request: Request, + settings: SecuritySettings, +) -> Optional[Principal]: + """Return the demo viewer only for one explicitly reviewed API route.""" + if settings.auth_mode != "public_readonly" or settings.app_env != "demo": + return None + if request.method.upper() not in PUBLIC_READONLY_METHOD_ALLOWLIST: + return None + route = request.scope.get("route") + route_template = getattr(route, "path", None) + if route_template not in PUBLIC_READONLY_ROUTE_ALLOWLIST: + return None + return Principal( + actor_id="public-readonly-demo", + roles=frozenset({Role.VIEWER}), + claims={"sub": "public-readonly-demo", "auth_mode": "public_readonly"}, + ) + + +def _roles_from_claims(claims: Mapping[str, Any], claim_name: str) -> FrozenSet[Role]: + raw_roles = claims.get(claim_name, []) + if isinstance(raw_roles, str): + candidates = raw_roles.replace(",", " ").split() + elif isinstance(raw_roles, (list, tuple, set)): + candidates = [str(role) for role in raw_roles] + else: + candidates = [] + normalized = set() + aliases = {"admin": Role.ADMINISTRATOR, "administrator": Role.ADMINISTRATOR} + for candidate in candidates: + value = candidate.strip().lower() + try: + normalized.add(Role(value)) + except ValueError: + if value in aliases: + normalized.add(aliases[value]) + return frozenset(normalized) + + +def create_development_access_token( + subject: str, + roles: Iterable[Role] = (Role.ADMINISTRATOR,), + *, + settings: SecuritySettings = SECURITY_SETTINGS, + expires_delta: Optional[timedelta] = None, +) -> str: + if settings.auth_mode != "development" or not settings.dev_auth_enabled or not settings.dev_auth_secret: + raise RuntimeError("Development authentication is not explicitly enabled") + now = datetime.now(timezone.utc) + claims = { + "sub": subject, + "roles": [role.value for role in roles], + "iss": "foundation-intelligence-development", + "aud": "foundation-intelligence-api", + "iat": now, + "exp": now + (expires_delta or timedelta(minutes=settings.access_token_expire_minutes)), + "token_use": "development", + } + return jwt.encode(claims, settings.dev_auth_secret, algorithm="HS256") + + +async def _decode_token(token: str, request: Request, settings: SecuritySettings) -> Mapping[str, Any]: + if settings.auth_mode == "development": + if _request_host(request) not in {host.lower() for host in settings.dev_auth_allowed_hosts}: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Development authentication is restricted to explicitly allowed local hosts.", + ) + try: + return jwt.decode( + token, + str(settings.dev_auth_secret), + algorithms=["HS256"], + issuer="foundation-intelligence-development", + audience="foundation-intelligence-api", + options={"require_exp": True, "require_iat": True, "require_sub": True}, + ) + except JWTError as exc: + raise _invalid_token() from exc + if settings.auth_mode == "oidc": + return await _oidc_verifier.decode(token, settings) + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Authentication is not configured.", + headers={"WWW-Authenticate": "Bearer"}, + ) + + +def _limiter_for(request: Request, settings: SecuritySettings) -> SlidingWindowRateLimiter: + limiter = getattr(request.app.state, "rate_limiter", None) + if limiter is None: + limiter = SlidingWindowRateLimiter( + settings.rate_limit_requests, + settings.rate_limit_window_seconds, + ) + request.app.state.rate_limiter = limiter + return limiter + + +def _idempotency_store_for(request: Request) -> IdempotencyStore: + store = getattr(request.app.state, "idempotency_store", None) + if store is None: + store = IdempotencyStore() + request.app.state.idempotency_store = store + return store + + +async def authenticate_request(request: Request) -> Principal: + cached = getattr(request.state, "principal", None) + if cached is not None: + return cached + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + token = _extract_token(request, settings) + if not token: + public_principal = _public_readonly_principal(request, settings) + if public_principal is not None: + retry_after = _limiter_for(request, settings).check( + f"public-demo:{_request_host(request)}" + ) + if retry_after is not None: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Request rate limit exceeded.", + headers={"Retry-After": str(retry_after)}, + ) + request.state.principal = public_principal + return public_principal + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="A bearer token or secure session is required.", + headers={"WWW-Authenticate": "Bearer"}, + ) + claims = await _decode_token(token, request, settings) + subject = claims.get("sub") + if not isinstance(subject, str) or not subject.strip(): + raise _invalid_token() + principal = Principal( + actor_id=subject.strip(), + roles=_roles_from_claims(claims, settings.oidc_role_claim), + claims=claims, + ) + retry_after = _limiter_for(request, settings).check(f"actor:{principal.actor_id}") + if retry_after is not None: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Request rate limit exceeded.", + headers={"Retry-After": str(retry_after)}, + ) + request.state.principal = principal + return principal + + +async def reserve_idempotency(request: Request, principal: Principal, action: str) -> None: + key = request.headers.get("idempotency-key", "").strip() + if not key or len(key) > 128: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="A valid Idempotency-Key header is required for this action.", + ) + body = await request.body() + fingerprint = hashlib.sha256( + request.method.encode("ascii") + b"\0" + request.url.path.encode("utf-8") + b"\0" + body + ).hexdigest() + try: + record_key = _idempotency_store_for(request).reserve( + principal.actor_id, + action, + key, + fingerprint, + ) + if inspect.isawaitable(record_key): + record_key = await record_key + except IdempotencyConflict as exc: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=str(exc), + ) from exc + request.state.idempotency_record_key = record_key + + +def require_roles(*roles: Role, action: str = "authenticated.read", idempotent: bool = False): + """Create a dependency that classifies and authorizes one API action.""" + + async def dependency(request: Request) -> Principal: + request.state.audit_action = action + request.state.audit_target = request.url.path + principal = await authenticate_request(request) + if not principal.permits_any(roles): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="The authenticated role is not permitted to perform this action.", + ) + if idempotent: + await reserve_idempotency(request, principal, action) + return principal + + return dependency + + +async def enforce_login_rate_limit(request: Request) -> None: + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + retry_after = _limiter_for(request, settings).check(f"login:{_request_host(request)}") + if retry_after is not None: + raise HTTPException( + status_code=status.HTTP_429_TOO_MANY_REQUESTS, + detail="Authentication rate limit exceeded.", + headers={"Retry-After": str(retry_after)}, + ) + + +def validate_development_credentials( + request: Request, + username: str, + password: str, +) -> SecuritySettings: + settings = getattr(request.app.state, "security_settings", SECURITY_SETTINGS) + allowed_hosts = {host.lower() for host in settings.dev_auth_allowed_hosts} + if ( + settings.auth_mode != "development" + or not settings.dev_auth_enabled + or _request_host(request) not in allowed_hosts + ): + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + valid = hmac.compare_digest(username, settings.dev_auth_username or "") and hmac.compare_digest( + password, + settings.dev_auth_password or "", + ) + if not valid: + raise HTTPException(status_code=status.HTTP_401_UNAUTHORIZED, detail="Incorrect credentials") + return settings diff --git a/src/bff/utils/logging.py b/src/bff/utils/logging.py index 66392f1..6f68225 100644 --- a/src/bff/utils/logging.py +++ b/src/bff/utils/logging.py @@ -1,22 +1,97 @@ +from datetime import datetime, timezone +import hashlib +import json import logging +import os import sys +from governance.exposure import redact_for_logs, redact_text as governance_redact_text +from governance.retention import load_governance_configuration + + +GOVERNANCE_CONFIGURATION = load_governance_configuration() +SERVICE_NAME = "foundation-intelligence-api" +STRUCTURED_FIELDS = ( + "request_id", + "trace_id", + "job_id", + "migration_run_id", + "dataset_version", + "schema_version", + "source", + "actor_id", + "role", + "operation", + "duration_ms", + "status", + "error_class", + "record_count", + "accepted_count", + "rejected_count", + "quarantined_count", + "retry_count", +) + +def redact_text(value: object) -> str: + """Remove credential-shaped values before logs or admin output expose them.""" + return governance_redact_text(value) + + +class RedactingFilter(logging.Filter): + def filter(self, record: logging.LogRecord) -> bool: + record.msg = redact_text(record.getMessage()) + record.args = () + return True + + +class RedactingFormatter(logging.Formatter): + """Compatibility formatter for callers that still need plain-text output.""" + + def formatException(self, exception_info) -> str: + return redact_text(super().formatException(exception_info)) + + +def pseudonymous_actor_id(value: object) -> str: + normalized = str(value or "").strip() + if not normalized: + return "unknown" + return "sha256:" + hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:16] + + +class JsonFormatter(logging.Formatter): + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, object] = { + "timestamp": datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(), + "level": record.levelname.lower(), + "service": SERVICE_NAME, + "environment": os.environ.get("APP_ENV", "development").strip().lower(), + "message": record.getMessage(), + } + for field in STRUCTURED_FIELDS: + value = getattr(record, field, None) + if value is not None: + payload[field] = value + if record.exc_info and "error_class" not in payload: + payload["error_class"] = record.exc_info[0].__name__ + redacted = redact_for_logs(payload, GOVERNANCE_CONFIGURATION) + return json.dumps(redacted, sort_keys=True, separators=(",", ":"), default=str) + + def setup_logging(): """ Sets up a standardized logging configuration for the BFF service. """ logger = logging.getLogger("bff") logger.setLevel(logging.INFO) + logger.propagate = False # Avoid duplicate handlers if already configured if not logger.handlers: - formatter = logging.Formatter( - fmt="%(asctime)s [%(levelname)s] [%(name)s] %(message)s", - datefmt="%Y-%m-%d %H:%M:%S" - ) + formatter = JsonFormatter() handler = logging.StreamHandler(sys.stdout) handler.setFormatter(formatter) + handler.addFilter(RedactingFilter()) logger.addHandler(handler) return logger diff --git a/src/governance/__init__.py b/src/governance/__init__.py new file mode 100644 index 0000000..ebdf138 --- /dev/null +++ b/src/governance/__init__.py @@ -0,0 +1 @@ +"""Data governance, exposure and non-destructive retention controls.""" diff --git a/src/governance/exposure.py b/src/governance/exposure.py new file mode 100644 index 0000000..556aaa0 --- /dev/null +++ b/src/governance/exposure.py @@ -0,0 +1,86 @@ +"""Explicit serializer allowlists and recursive sensitive-data redaction.""" + +from __future__ import annotations + +import re +from typing import Any, Mapping + +from governance.retention import GovernanceConfiguration + + +_CREDENTIAL_PATTERNS = ( + re.compile(r'''(?i)(["']?authorization["']?\s*[:=]\s*["']?(?:bearer\s+)?)[^"'\s,;]+'''), + re.compile( + r'''(?i)(["']?(?:password|passwd|token|secret|api[_-]?key)["']?\s*[:=]\s*["']?)[^"'\s,;]+''' + ), + re.compile(r"(?i)(postgres(?:ql)?://[^:/\s]+:)[^@\s]+(@)"), + re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), + re.compile(r"\bsk-[A-Za-z0-9_-]{12,}\b"), +) +_EMAIL_PATTERN = re.compile(r"(? str: + redacted = str(value) + for pattern in _CREDENTIAL_PATTERNS: + if pattern.groups >= 2: + redacted = pattern.sub(rf"\1{replacement}\2", redacted) + elif pattern.groups == 1: + redacted = pattern.sub(rf"\1{replacement}", redacted) + else: + redacted = pattern.sub(replacement, redacted) + return _EMAIL_PATTERN.sub(replacement, redacted) + + +def redact_data( + value: Any, + *, + sensitive_keys: set[str], + replacement: str = "[REDACTED]", +) -> Any: + if isinstance(value, Mapping): + return { + str(key): ( + replacement + if str(key).casefold() in sensitive_keys + else redact_data( + item, + sensitive_keys=sensitive_keys, + replacement=replacement, + ) + ) + for key, item in value.items() + } + if isinstance(value, (list, tuple, set)): + return [ + redact_data(item, sensitive_keys=sensitive_keys, replacement=replacement) + for item in value + ] + if isinstance(value, str): + return redact_text(value, replacement) + return value + + +def redact_for_logs(value: Any, configuration: GovernanceConfiguration) -> Any: + keys = { + str(key).casefold() + for key in configuration.log_redaction.get("sensitive_keys", ()) + } + replacement = str(configuration.log_redaction.get("replacement", "[REDACTED]")) + return redact_data(value, sensitive_keys=keys, replacement=replacement) + + +def serialize_exposed_fields( + record: Mapping[str, Any], + *, + policy_name: str, + configuration: GovernanceConfiguration, +) -> dict[str, Any]: + fields = configuration.field_exposure_policies.get(policy_name) + if fields is None: + raise ValueError(f"Unknown field exposure policy: {policy_name}") + return { + field: record[field] + for field in fields + if field in record + } diff --git a/src/governance/retention.py b/src/governance/retention.py new file mode 100644 index 0000000..365816e --- /dev/null +++ b/src/governance/retention.py @@ -0,0 +1,336 @@ +"""Configurable, hold-aware and non-destructive retention planning.""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime, timedelta, timezone +import hashlib +import json +from pathlib import Path +from typing import Any, Iterable, Mapping + + +GOVERNANCE_CONFIGURATION_PATH = ( + Path(__file__).resolve().parents[2] / "config" / "data-governance.json" +) +REQUIRED_CLASSIFICATIONS = frozenset( + { + "public_organisation_data", + "contact_data", + "personal_email_addresses", + "postal_addresses", + "raw_source_evidence", + "article_metadata", + "article_content", + "pipeline_logs", + "exports", + "audit_events", + "derived_classifications", + "enriched_profiles", + "credentials", + "user_identities", + } +) + + +def canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def checksum(value: object) -> str: + return hashlib.sha256(canonical_json(value).encode("utf-8")).hexdigest() + + +@dataclass(frozen=True) +class ClassificationPolicy: + classification: str + retention_class: str + exposure: str + archive_after_days: int | None + delete_after_days: int | None + export_expire_after_days: int | None + + def validate(self) -> None: + if not self.classification.strip() or not self.retention_class.strip(): + raise ValueError("Classification and retention class are required") + if self.exposure not in {"authenticated", "restricted", "internal", "never"}: + raise ValueError(f"Invalid exposure for {self.classification}") + for value in ( + self.archive_after_days, + self.delete_after_days, + self.export_expire_after_days, + ): + if value is not None and value < 1: + raise ValueError(f"Retention windows must be positive for {self.classification}") + + @property + def configuration_checksum(self) -> str: + return checksum(asdict(self)) + + def database_record(self, *, policy_status: str) -> dict[str, Any]: + self.validate() + return { + **asdict(self), + "policy_status": policy_status, + "destructive_deletion_enabled": False, + "production_approved": False, + "configuration_checksum": self.configuration_checksum, + } + + +@dataclass(frozen=True) +class GovernanceConfiguration: + policy_status: str + destructive_deletion_enabled: bool + production_activation_approved: bool + restore_before_delete_required: bool + policies: tuple[ClassificationPolicy, ...] + data_owners: Mapping[str, str] + field_exposure_policies: Mapping[str, tuple[str, ...]] + log_redaction: Mapping[str, Any] + privacy_checklist: Mapping[str, Any] + backup_policy: Mapping[str, Any] + pitr_policy: Mapping[str, Any] + service_recovery: Mapping[str, Any] + data_subject_workflow: tuple[str, ...] + + def validate(self) -> None: + if self.policy_status not in {"proposed", "approved", "retired"}: + raise ValueError("Invalid governance policy status") + if self.destructive_deletion_enabled: + raise ValueError("Initial governance configuration must disable destructive deletion") + if self.production_activation_approved: + raise ValueError("Production retention activation has not been approved") + classifications = {policy.classification for policy in self.policies} + missing = sorted(REQUIRED_CLASSIFICATIONS - classifications) + if missing: + raise ValueError(f"Missing classifications: {', '.join(missing)}") + retention_classes = [policy.retention_class for policy in self.policies] + if len(retention_classes) != len(set(retention_classes)): + raise ValueError("Retention classes must be unique") + for policy in self.policies: + policy.validate() + if policy.delete_after_days is not None: + raise ValueError("Initial destructive deletion windows must remain unset") + required_owners = { + "business_owner", + "data_owner", + "privacy_owner", + "legal_owner", + "security_owner", + "technical_owner", + } + if required_owners - set(self.data_owners): + raise ValueError("Governance owner register is incomplete") + if not self.restore_before_delete_required: + raise ValueError("Restore verification must precede any future deletion") + if not self.field_exposure_policies: + raise ValueError("Explicit field exposure policies are required") + if not self.data_subject_workflow: + raise ValueError("Data-subject workflow is required") + + @property + def by_retention_class(self) -> dict[str, ClassificationPolicy]: + return {policy.retention_class: policy for policy in self.policies} + + +def load_governance_configuration( + path: Path = GOVERNANCE_CONFIGURATION_PATH, +) -> GovernanceConfiguration: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("configuration_version") != "1": + raise ValueError("Unsupported governance configuration version") + configuration = GovernanceConfiguration( + policy_status=str(payload["policy_status"]), + destructive_deletion_enabled=bool(payload["destructive_deletion_enabled"]), + production_activation_approved=bool(payload["production_activation_approved"]), + restore_before_delete_required=bool(payload["restore_before_delete_required"]), + policies=tuple( + ClassificationPolicy(**entry) for entry in payload.get("classifications", []) + ), + data_owners=dict(payload.get("data_owners", {})), + field_exposure_policies={ + name: tuple(fields) + for name, fields in payload.get("field_exposure_policies", {}).items() + }, + log_redaction=dict(payload.get("log_redaction", {})), + privacy_checklist=dict(payload.get("privacy_checklist", {})), + backup_policy=dict(payload.get("backup_policy", {})), + pitr_policy=dict(payload.get("pitr_policy", {})), + service_recovery=dict(payload.get("service_recovery", {})), + data_subject_workflow=tuple(payload.get("data_subject_workflow", [])), + ) + configuration.validate() + return configuration + + +@dataclass(frozen=True) +class RetentionCandidate: + target_type: str + target_id: str + retention_class: str + last_modified_at: datetime + object_count: int = 0 + record_count: int = 0 + total_bytes: int = 0 + target_checksums: tuple[str, ...] = () + + def validate(self) -> None: + if not self.target_type.strip() or not self.target_id.strip(): + raise ValueError("Retention target identity is required") + if self.last_modified_at.tzinfo is None: + raise ValueError("Retention timestamps must be timezone-aware") + if min(self.object_count, self.record_count, self.total_bytes) < 0: + raise ValueError("Retention counts cannot be negative") + for value in self.target_checksums: + if len(value) != 64 or any(character not in "0123456789abcdef" for character in value): + raise ValueError("Target checksums must be lowercase SHA-256") + + +@dataclass(frozen=True) +class DataHold: + hold_id: str + hold_type: str + scope_type: str + scope_id: str + status: str + expires_at: datetime | None = None + + def applies(self, candidate: RetentionCandidate, now: datetime) -> bool: + active = self.status == "active" and ( + self.expires_at is None or self.expires_at > now + ) + exact = self.scope_type == candidate.target_type and self.scope_id == candidate.target_id + global_scope = self.scope_type == "all" and self.scope_id == "*" + class_scope = ( + self.scope_type == "retention_class" + and self.scope_id == candidate.retention_class + ) + return active and (exact or global_scope or class_scope) + + +@dataclass(frozen=True) +class RetentionPlanEntry: + target_type: str + target_id: str + retention_class: str + action_type: str + status: str + dry_run: bool + reason: str + hold_ids: tuple[str, ...] + object_count: int + record_count: int + total_bytes: int + target_checksums: tuple[str, ...] + generated_at: str + + @property + def manifest(self) -> dict[str, Any]: + return asdict(self) + + @property + def manifest_checksum(self) -> str: + return checksum(self.manifest) + + +class RetentionPlanner: + def __init__(self, configuration: GovernanceConfiguration): + configuration.validate() + self.configuration = configuration + + def plan( + self, + candidates: Iterable[RetentionCandidate], + holds: Iterable[DataHold], + *, + now: datetime | None = None, + ) -> tuple[RetentionPlanEntry, ...]: + current = now or datetime.now(timezone.utc) + if current.tzinfo is None: + raise ValueError("Planner timestamp must be timezone-aware") + available_holds = tuple(holds) + entries: list[RetentionPlanEntry] = [] + policies = self.configuration.by_retention_class + for candidate in candidates: + candidate.validate() + policy = policies.get(candidate.retention_class) + if policy is None: + raise ValueError(f"Unknown retention class: {candidate.retention_class}") + active_holds = tuple( + hold.hold_id + for hold in available_holds + if hold.applies(candidate, current) + ) + archive_due = ( + policy.archive_after_days is not None + and candidate.last_modified_at + <= current - timedelta(days=policy.archive_after_days) + ) + if active_holds: + action_type = "report" + status = "held" + reason = "active_legal_or_incident_hold" + elif archive_due: + action_type = "archive" + status = "reported" + reason = "archive_dry_run_only" + else: + action_type = "report" + status = "reported" + reason = "retention_window_not_reached" + entries.append( + RetentionPlanEntry( + target_type=candidate.target_type, + target_id=candidate.target_id, + retention_class=candidate.retention_class, + action_type=action_type, + status=status, + dry_run=True, + reason=reason, + hold_ids=active_holds, + object_count=candidate.object_count, + record_count=candidate.record_count, + total_bytes=candidate.total_bytes, + target_checksums=candidate.target_checksums, + generated_at=current.isoformat(), + ) + ) + return tuple(entries) + + def assert_deletion_authorized( + self, + *, + retention_class: str, + actor_role: str, + restore_verified: bool, + active_hold_ids: Iterable[str], + ) -> None: + policy = self.configuration.by_retention_class.get(retention_class) + if policy is None: + raise PermissionError("Unknown retention class") + if not self.configuration.destructive_deletion_enabled: + raise PermissionError("Destructive deletion is globally disabled") + if not self.configuration.production_activation_approved: + raise PermissionError("Production retention activation is not approved") + if actor_role != "administrator": + raise PermissionError("Administrator role is required") + if self.configuration.restore_before_delete_required and not restore_verified: + raise PermissionError("Successful restore verification is required") + if tuple(active_hold_ids): + raise PermissionError("Active holds override deletion") + if policy.delete_after_days is None: + raise PermissionError("No destructive retention window is configured") + + +def export_lifecycle_status( + *, expires_at: datetime | None, hold_until: datetime | None, now: datetime | None = None +) -> str: + current = now or datetime.now(timezone.utc) + if current.tzinfo is None: + raise ValueError("Lifecycle timestamp must be timezone-aware") + if hold_until is not None and hold_until > current: + return "held" + if expires_at is not None and expires_at <= current: + return "expiration_report_due" + return "active" diff --git a/src/migration/__init__.py b/src/migration/__init__.py new file mode 100644 index 0000000..0a689fa --- /dev/null +++ b/src/migration/__init__.py @@ -0,0 +1 @@ +"""Read-only legacy migration and reconciliation tools.""" diff --git a/src/migration/aws_entrypoint.py b/src/migration/aws_entrypoint.py new file mode 100644 index 0000000..02e3934 --- /dev/null +++ b/src/migration/aws_entrypoint.py @@ -0,0 +1,317 @@ +"""Fail-closed bootstrap for the approved one-off AWS migration task. + +The source file is downloaded by a separate S3-only init container. This +process holds a PostgreSQL advisory lock, applies Alembic as the RDS master, +delegates loading, reconciliation and transactional activation to +``sqlite_to_postgres.migrate``, then creates/rotates a SELECT-only application +role. +Passwords are read only from ECS secret environment variables and are never +printed or included in subprocess arguments. +""" + +from __future__ import annotations + +import asyncio +import json +import os +from pathlib import Path +import re +import subprocess +from typing import Mapping + +import asyncpg + +from bff.database import DatabaseManager, DatabaseSettings +from migration.release_gate import release_state +from migration.sqlite_to_postgres import migrate, source_preflight + + +MIGRATION_LOCK_ID = 2_083_370_803_870 +IDENTIFIER_PATTERN = re.compile(r"^[a-z_][a-z0-9_]{0,62}$") + + +class AwsMigrationConfigurationError(RuntimeError): + """Raised before any database mutation when task input is incomplete.""" + + +def _required(environment: Mapping[str, str], name: str) -> str: + value = environment.get(name, "").strip() + if not value: + raise AwsMigrationConfigurationError(f"{name} is required") + return value + + +def _identifier(value: str, name: str) -> str: + if not IDENTIFIER_PATTERN.fullmatch(value): + raise AwsMigrationConfigurationError(f"{name} is not a safe PostgreSQL identifier") + return value + + +def _quoted_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _quoted_literal(value: str) -> str: + return "'" + value.replace("'", "''") + "'" + + +async def _connect_admin(environment: Mapping[str, str]) -> asyncpg.Connection: + admin_environment = dict(environment) + admin_environment["DATABASE_USER"] = _required(environment, "DATABASE_ADMIN_USER") + admin_environment["DATABASE_PASSWORD"] = _required( + environment, "DATABASE_ADMIN_PASSWORD" + ) + settings = DatabaseSettings.from_env(admin_environment) + url = settings.sqlalchemy_url() + return await asyncpg.connect( + host=url.host, + port=url.port or 5432, + user=url.username, + password=url.password, + database=url.database, + ssl=settings.ssl_mode, + command_timeout=None, + ) + + +def _run_alembic(environment: Mapping[str, str]) -> None: + admin_environment = dict(environment) + admin_environment["DATABASE_USER"] = _required(environment, "DATABASE_ADMIN_USER") + admin_environment["DATABASE_PASSWORD"] = _required( + environment, "DATABASE_ADMIN_PASSWORD" + ) + subprocess.run( + ["alembic", "upgrade", "head"], + env=admin_environment, + check=True, + timeout=900, + ) + + +async def _configure_application_role( + connection: asyncpg.Connection, + environment: Mapping[str, str], +) -> None: + application_user = _identifier( + _required(environment, "DATABASE_APP_USER"), "DATABASE_APP_USER" + ) + admin_user = _identifier( + _required(environment, "DATABASE_ADMIN_USER"), "DATABASE_ADMIN_USER" + ) + database_name = _identifier( + _required(environment, "DATABASE_NAME"), "DATABASE_NAME" + ) + application_password = _required(environment, "DATABASE_APP_PASSWORD") + role = _quoted_identifier(application_user) + password = _quoted_literal(application_password) + exists = await connection.fetchval( + "SELECT EXISTS (SELECT 1 FROM pg_roles WHERE rolname=$1)", + application_user, + ) + if exists: + await connection.execute( + f"ALTER ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " + f"NOINHERIT NOREPLICATION PASSWORD {password}" + ) + else: + await connection.execute( + f"CREATE ROLE {role} LOGIN NOSUPERUSER NOCREATEDB NOCREATEROLE " + f"NOINHERIT NOREPLICATION PASSWORD {password}" + ) + await connection.execute(f"REVOKE ALL ON DATABASE {_quoted_identifier(database_name)} FROM PUBLIC") + await connection.execute("REVOKE CREATE ON SCHEMA public FROM PUBLIC") + await connection.execute( + f"GRANT CONNECT ON DATABASE {_quoted_identifier(database_name)} TO {role}" + ) + await connection.execute(f"GRANT USAGE ON SCHEMA public TO {role}") + await connection.execute( + f"REVOKE ALL ON ALL TABLES IN SCHEMA public FROM {role}" + ) + await connection.execute(f"GRANT SELECT ON ALL TABLES IN SCHEMA public TO {role}") + await connection.execute( + f"REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM {role}" + ) + await connection.execute("REVOKE EXECUTE ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC") + await connection.execute(f"REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM {role}") + owner = _quoted_identifier(admin_user) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON TABLES FROM {role}" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"GRANT SELECT ON TABLES TO {role}" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON SEQUENCES FROM {role}" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + "REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC" + ) + await connection.execute( + f"ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA public " + f"REVOKE ALL ON FUNCTIONS FROM {role}" + ) + await connection.execute(f"ALTER ROLE {role} SET default_transaction_read_only = on") + + +async def _verify_application_role( + environment: Mapping[str, str], +) -> dict[str, bool]: + application_environment = dict(environment) + application_environment["DATABASE_USER"] = _required( + environment, "DATABASE_APP_USER" + ) + application_environment["DATABASE_PASSWORD"] = _required( + environment, "DATABASE_APP_PASSWORD" + ) + settings = DatabaseSettings.from_env(application_environment) + url = settings.sqlalchemy_url() + connection = await asyncpg.connect( + host=url.host, + port=url.port or 5432, + user=url.username, + password=url.password, + database=url.database, + ssl=settings.ssl_mode, + command_timeout=None, + ) + try: + select_succeeded = await connection.fetchval("SELECT 1") == 1 + tls_in_use = bool( + await connection.fetchval( + "SELECT ssl FROM pg_stat_ssl WHERE pid=pg_backend_pid()" + ) + ) + default_read_only = ( + await connection.fetchval("SHOW default_transaction_read_only") == "on" + ) + update_denied = False + try: + await connection.execute( + "UPDATE dataset_versions SET status=status WHERE FALSE" + ) + except ( + asyncpg.InsufficientPrivilegeError, + asyncpg.ReadOnlySQLTransactionError, + ): + update_denied = True + result = { + "select_succeeded": select_succeeded, + "tls_in_use": tls_in_use, + "default_read_only": default_read_only, + "update_denied": update_denied, + } + if not all(result.values()): + raise RuntimeError("application database role verification failed") + return result + finally: + await connection.close() + + +async def run(environment: Mapping[str, str] | None = None) -> dict[str, object]: + task_environment = dict(os.environ if environment is None else environment) + source = Path(_required(task_environment, "MIGRATION_SOURCE_PATH")) + output_directory = Path(_required(task_environment, "MIGRATION_OUTPUT_DIRECTORY")) + expected_checksum = _required(task_environment, "MIGRATION_EXPECTED_CHECKSUM") + expected_schema_version = _required( + task_environment, "MIGRATION_EXPECTED_SCHEMA_VERSION" + ) + dataset_version = _required(task_environment, "MIGRATION_DATASET_VERSION") + code_revision = _required(task_environment, "MIGRATION_CODE_REVISION") + if not re.fullmatch(r"[a-f0-9]{40}", code_revision): + raise AwsMigrationConfigurationError( + "MIGRATION_CODE_REVISION must be a full lowercase Git SHA" + ) + if not re.fullmatch(r"[A-Za-z0-9._-]{1,200}", dataset_version): + raise AwsMigrationConfigurationError("MIGRATION_DATASET_VERSION is unsafe") + # Validate the immutable source before Alembic or any other database write. + # migrate() deliberately repeats the check immediately before loading. + source_preflight( + source, + expected_checksum, + expected_schema_version, + remote_postgres=True, + ) + admin = await _connect_admin(task_environment) + try: + acquired = await admin.fetchval("SELECT pg_try_advisory_lock($1)", MIGRATION_LOCK_ID) + if not acquired: + raise AwsMigrationConfigurationError( + "another migration or refresh task holds the database lock" + ) + await asyncio.to_thread(_run_alembic, task_environment) + previous_user = os.environ.get("DATABASE_USER") + previous_password = os.environ.get("DATABASE_PASSWORD") + os.environ.update( + { + "DATABASE_USER": _required(task_environment, "DATABASE_ADMIN_USER"), + "DATABASE_PASSWORD": _required( + task_environment, "DATABASE_ADMIN_PASSWORD" + ), + } + ) + try: + report = await migrate( + source, + expected_checksum, + expected_schema_version, + dataset_version, + code_revision, + "ecs-migration-task", + "service", + output_directory, + remote_postgres=True, + ) + await _configure_application_role(admin, task_environment) + application_role = await _verify_application_role(task_environment) + os.environ.update( + { + "DATABASE_USER": _required(task_environment, "DATABASE_APP_USER"), + "DATABASE_PASSWORD": _required( + task_environment, "DATABASE_APP_PASSWORD" + ), + } + ) + gate_database = DatabaseManager(DatabaseSettings.from_env()) + try: + gate = await release_state(gate_database) + finally: + await gate_database.close() + finally: + if previous_user is None: + os.environ.pop("DATABASE_USER", None) + else: + os.environ["DATABASE_USER"] = previous_user + if previous_password is None: + os.environ.pop("DATABASE_PASSWORD", None) + else: + os.environ["DATABASE_PASSWORD"] = previous_password + if not gate["ready"]: + raise RuntimeError("release gate rejected the migrated dataset") + result = { + "dataset_version": report["dataset_version"], + "activation_status": report["activation_status"], + "application_role": application_role, + "release_gate_ready": True, + } + output_directory.mkdir(parents=True, exist_ok=True, mode=0o700) + (output_directory / "task-result.json").write_text( + json.dumps(result, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + return result + finally: + await admin.close() + + +def main() -> int: + result = asyncio.run(run()) + print(json.dumps(result, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/migration/release_gate.py b/src/migration/release_gate.py new file mode 100644 index 0000000..6ea4637 --- /dev/null +++ b/src/migration/release_gate.py @@ -0,0 +1,103 @@ +"""Fail-closed PostgreSQL release/reconciliation gate for one-off ECS tasks.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any, Mapping + +from sqlalchemy import text + +from bff.database import DatabaseManager, DatabaseSettings +from observability.metrics import load_observability_configuration + + +EXPECTED_SCHEMA_VERSION = load_observability_configuration().expected_schema_version + + +def evaluate_release_state(row: Mapping[str, Any]) -> dict[str, Any]: + reconciliation = dict(row.get("reconciliation_results") or {}) + reconciliation_failures = sorted( + name + for name, result in reconciliation.items() + if not isinstance(result, Mapping) or result.get("status") != "pass" + ) + checks = { + "schema": str(row.get("schema_version")) == EXPECTED_SCHEMA_VERSION, + "single_active_dataset": int(row.get("active_dataset_count") or 0) == 1, + "migration_active": row.get("migration_status") == "active", + "reconciliation": bool(reconciliation) and not reconciliation_failures, + "materialization": bool(row.get("materialization_active")), + "active_quality_blocks": int(row.get("active_quality_blocks") or 0) == 0, + "dead_letter_queue": int(row.get("dead_letter_count") or 0) == 0, + } + return { + "ready": all(checks.values()), + "dataset_version": str(row.get("dataset_version") or ""), + "checks": checks, + "reconciliation_failures": reconciliation_failures, + } + + +async def release_state(database: DatabaseManager) -> dict[str, Any]: + sessions = database.sessions() + async with sessions() as session: + row = ( + await session.execute( + text( + """ + WITH active AS ( + SELECT dataset_version + FROM dataset_versions + WHERE is_active AND status='active' + ) + SELECT + (SELECT version_num FROM alembic_version) AS schema_version, + (SELECT COUNT(*) FROM active) AS active_dataset_count, + (SELECT dataset_version FROM active) AS dataset_version, + migration.status AS migration_status, + migration.reconciliation_results, + EXISTS ( + SELECT 1 FROM materialization_versions materialization + WHERE materialization.dataset_version=active.dataset_version + AND materialization.materialization_name='dashboard_analytics' + AND materialization.status='active' + AND materialization.is_active + ) AS materialization_active, + ( + SELECT COUNT(*) FROM data_quality_issues quality + WHERE quality.dataset_version=active.dataset_version + AND quality.status IN ('open', 'quarantined') + ) AS active_quality_blocks, + ( + SELECT COUNT(*) FROM job_runs + WHERE status='dead_lettered' + ) AS dead_letter_count + FROM active + LEFT JOIN migration_runs migration + ON migration.target_dataset_version=active.dataset_version + """ + ) + ) + ).mappings().one_or_none() + if row is None: + return evaluate_release_state({}) + return evaluate_release_state(row) + + +async def _main() -> int: + database = DatabaseManager(DatabaseSettings.from_env()) + try: + result = await release_state(database) + finally: + await database.close() + print(json.dumps(result, sort_keys=True)) + return 0 if result["ready"] else 1 + + +def main() -> int: + return asyncio.run(_main()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/migration/sqlite_to_postgres.py b/src/migration/sqlite_to_postgres.py new file mode 100644 index 0000000..2bafaa9 --- /dev/null +++ b/src/migration/sqlite_to_postgres.py @@ -0,0 +1,1345 @@ +"""Deterministic read-only SQLite to versioned PostgreSQL migration command.""" + +from __future__ import annotations + +import argparse +import asyncio +from dataclasses import dataclass +from datetime import date, datetime, timezone +from decimal import Decimal, InvalidOperation +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import sqlite3 +import sys +from typing import Any, Callable, Iterable, Optional, Sequence +from urllib.parse import quote +import uuid + +import asyncpg + +from bff.database import DatabaseSettings +from observability.metrics import load_observability_configuration + + +GIB = 1024**3 +MIGRATION_SCHEMA_VERSION = load_observability_configuration().expected_schema_version +EXPECTED_COUNTS = { + "charities": 373, + "charity_registry_organizations": 397_469, + "grants": 302_546, + "organization_registry_links": 345, +} +EXPECTED_CONTROLS = { + "distinct_mapped_grants": 104_191, + "classified_grants": 134_554, + "overview_total_eur": "22435986707.70", + "duplicate_source_identity_groups": 0, + "missing_eur_conversions": 432, + "negative_grants": 2, + "zero_grants": 2_101, + "future_dated_grants": 1, + "business_key_duplicate_groups": 4_271, + "duplicate_charity_number_groups": 9_073, + "foreign_key_violations": 0, +} +VERSIONED_TABLES = { + "charities", + "charity_registry_organizations", + "grants", + "grant_beneficiary_countries", + "grant_beneficiary_terms", + "grant_programme_categories", + "grant_overview_facts", + "grant_source_funder_facts", + "organization_registry_links", + "source_funder_profile_cache", +} +LOAD_ORDER = ( + "charities", + "charity_registry_organizations", + "grants", + "grant_beneficiary_countries", + "grant_beneficiary_terms", + "grant_programme_categories", + "grant_overview_facts", + "grant_source_funder_facts", + "organization_registry_links", + "source_funder_link_overrides", + "source_funder_profile_cache", + "exchange_rates", +) +GLOBAL_KEYS = { + "source_funder_link_overrides": ("source_namespace", "source_organization_id"), + "exchange_rates": ("currency", "rate_date"), +} +RENAMES = { + ("charities", "raw_cc_data"): "raw_source_data", + ("grants", "date"): "award_date", +} +JSON_COLUMNS = { + ("charities", name) + for name in ( + "raw_cc_data", + "programme_areas_source", + "programme_areas_inferred", + "programme_area_scores", + "programme_area_evidence", + "geographic_focus_source", + "geographic_focus_inferred", + "geography_evidence", + "enrichment_review_reasons", + "source_names", + "source_records", + "deduplication_candidates", + ) +} | { + ("grants", name) + for name in ( + "tags", + "raw_grant_data", + "programme_area_scores", + "programme_area_evidence", + "geography_evidence", + "enrichment_review_reasons", + ) +} | {("source_funder_profile_cache", "payload")} +BOOLEAN_COLUMNS = { + ("charities", "programme_area_review_required"), + ("charities", "geography_review_required"), + ("charities", "insufficient_source_text"), + ("charity_registry_organizations", "is_current_source_record"), + ("grants", "programme_area_review_required"), + ("grants", "geography_review_required"), + ("grants", "insufficient_source_text"), + ("grant_overview_facts", "invalid_source_label"), + ("grant_overview_facts", "low_confidence_inference"), +} +DATE_COLUMNS = { + ("charity_registry_organizations", name) + for name in ("registration_date", "removal_date", "financial_period_end_date") +} | {("grant_overview_facts", "award_date")} | { + ("grant_source_funder_facts", "award_date") +} | {("exchange_rates", "rate_date")} +TIMESTAMP_COLUMNS = { + ("charities", "ingestion_timestamp"), + ("charity_registry_organizations", "source_record_updated_at"), + ("charity_registry_organizations", "imported_at"), + ("grants", "ingestion_timestamp"), + ("organization_registry_links", "reviewed_at"), + ("organization_registry_links", "created_at"), + ("organization_registry_links", "updated_at"), + ("source_funder_link_overrides", "updated_at"), + ("source_funder_profile_cache", "updated_at"), + ("exchange_rates", "retrieved_at"), +} +NUMERIC_COLUMNS = { + ("charities", name) + for name in ( + "latitude", + "longitude", + "annual_income", + "annual_expenditure", + "geography_confidence", + ) +} | { + ("charity_registry_organizations", name) + for name in ( + "income", + "expenditure", + "registered_latitude", + "registered_longitude", + ) +} | { + ("grants", name) + for name in ( + "amount", + "amount_eur", + "recipient_latitude", + "recipient_longitude", + "geography_confidence", + "exchange_rate", + ) +} | {("organization_registry_links", "match_confidence")} | { + ("exchange_rates", "eur_reference_rate") +} +UUID_COLUMNS = {("source_funder_profile_cache", "job_token")} +MONTH_COLUMNS = {("grants", "exchange_rate_date")} +RAW_DATE_COLUMNS = {("grants", "date")} +CLASSIFICATION_METHODS = { + "deterministic_regex", + "source_normalization", + "source_normalization+deterministic_regex", + "unavailable", +} +SOURCE_ID_COLUMNS = { + "charities": ("charity_id",), + "charity_registry_organizations": ("registry_id",), + "grants": ("grant_id",), + "grant_beneficiary_countries": ("grant_id", "country_code"), + "grant_beneficiary_terms": ("grant_id", "term"), + "grant_programme_categories": ("grant_id", "programme_area"), + "grant_overview_facts": ("grant_id",), + "grant_source_funder_facts": ("grant_id", "country_code"), + "organization_registry_links": ("registry_id", "enriched_organization_id"), + "source_funder_link_overrides": ("source_namespace", "source_organization_id"), + "source_funder_profile_cache": ("source_funder_key",), + "exchange_rates": ("currency", "rate_date"), +} + + +class MigrationError(RuntimeError): + """Base class for a migration failure that must never activate data.""" + + +class PreflightError(MigrationError): + """Raised before PostgreSQL mutation when source or capacity is unsafe.""" + + +class ValueConversionError(MigrationError): + def __init__(self, column: str, value: Any, message: str): + super().__init__(message) + self.column = column + self.value = value + + +def _quote_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +@dataclass(frozen=True) +class CapacityEstimate: + source_bytes: int + estimated_postgres_and_indexes_bytes: int + estimated_wal_and_temp_bytes: int + safety_margin_bytes: int + minimum_free_bytes: int + available_bytes: int + + @property + def sufficient(self) -> bool: + return self.available_bytes >= self.minimum_free_bytes + + def as_dict(self) -> dict[str, Any]: + return { + **self.__dict__, + "sufficient": self.sufficient, + } + + +@dataclass(frozen=True) +class SourcePreflight: + checksum: str + schema_version: str + metadata: dict[str, str] + counts: dict[str, int] + capacity: CapacityEstimate + + +def sha256_file(path: Path, chunk_size: int = 8 * 1024 * 1024) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + while chunk := source.read(chunk_size): + digest.update(chunk) + return digest.hexdigest() + + +def capacity_estimate( + source_path: Path, + *, + remote_postgres: bool = False, +) -> CapacityEstimate: + source_bytes = source_path.stat().st_size + if remote_postgres: + # An ECS migration task streams the read-only SQLite source to RDS. Its + # local filesystem never stores PostgreSQL data or WAL, so retain a + # bounded 4 GiB margin for the image, reports and SQLite temporary work. + postgres_and_indexes = 0 + wal_and_temp = 0 + safety_margin = 4 * GIB + else: + postgres_and_indexes = source_bytes * 3 + wal_and_temp = source_bytes * 2 + safety_margin = 10 * GIB + return CapacityEstimate( + source_bytes=source_bytes, + estimated_postgres_and_indexes_bytes=postgres_and_indexes, + estimated_wal_and_temp_bytes=wal_and_temp, + safety_margin_bytes=safety_margin, + minimum_free_bytes=source_bytes + postgres_and_indexes + wal_and_temp + safety_margin, + available_bytes=shutil.disk_usage(source_path.parent).free, + ) + + +def open_sqlite_read_only(path: Path) -> sqlite3.Connection: + resolved = path.resolve() + uri = f"file:{quote(str(resolved))}?mode=ro&immutable=1" + connection = sqlite3.connect(uri, uri=True) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + return connection + + +def source_preflight( + source_path: Path, + expected_checksum: str, + expected_schema_version: str, + *, + enforce_capacity: bool = True, + remote_postgres: bool = False, +) -> SourcePreflight: + if not source_path.is_file(): + raise PreflightError("SQLite source does not exist") + if not re.fullmatch(r"[a-f0-9]{64}", expected_checksum): + raise PreflightError("Expected checksum must be a lowercase SHA-256 value") + actual_checksum = sha256_file(source_path) + if actual_checksum != expected_checksum: + raise PreflightError("SQLite source checksum does not match the approved value") + capacity = capacity_estimate(source_path, remote_postgres=remote_postgres) + if enforce_capacity and not capacity.sufficient: + raise PreflightError( + f"Insufficient disk capacity: requires {capacity.minimum_free_bytes} bytes, " + f"has {capacity.available_bytes} bytes" + ) + with open_sqlite_read_only(source_path) as source: + integrity_rows = [row[0] for row in source.execute("PRAGMA integrity_check")] + if integrity_rows != ["ok"]: + raise PreflightError("SQLite integrity_check did not return ok") + metadata = { + str(row[0]): str(row[1]) + for row in source.execute("SELECT key, value FROM metadata ORDER BY key") + } + schema_version = metadata.get("schema_version", "") + if schema_version != expected_schema_version: + raise PreflightError( + f"SQLite schema version {schema_version!r} does not match " + f"{expected_schema_version!r}" + ) + source_tables = { + row[0] + for row in source.execute( + "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name" + ) + } + missing = sorted(set(LOAD_ORDER) - source_tables) + if missing: + raise PreflightError(f"SQLite source is missing tables: {', '.join(missing)}") + counts = { + table: int(source.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + for table in LOAD_ORDER + } + return SourcePreflight( + checksum=actual_checksum, + schema_version=schema_version, + metadata=metadata, + counts=counts, + capacity=capacity, + ) + + +def _json_value(table: str, column: str, value: Any) -> Optional[str]: + if value is None or value == "": + return None + if isinstance(value, (dict, list, int, float, bool)): + parsed = value + else: + try: + parsed = json.loads(str(value)) + except (TypeError, ValueError, json.JSONDecodeError) as exc: + raise ValueConversionError(column, value, "invalid JSON source value") from exc + return json.dumps(parsed, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + + +def _date_value(column: str, value: Any) -> Optional[date]: + if value is None or str(value).strip() == "": + return None + candidate = str(value).strip()[:10] + try: + return date.fromisoformat(candidate) + except ValueError as exc: + raise ValueConversionError(column, value, "invalid ISO date source value") from exc + + +def _timestamp_value(column: str, value: Any) -> Optional[datetime]: + if value is None or str(value).strip() == "": + return None + candidate = str(value).strip().replace("Z", "+00:00") + try: + parsed = datetime.fromisoformat(candidate) + except ValueError as exc: + raise ValueConversionError(column, value, "invalid ISO timestamp source value") from exc + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def _decimal_value(column: str, value: Any) -> Optional[Decimal]: + if value is None or str(value).strip() == "": + return None + try: + parsed = Decimal(str(value)) + except InvalidOperation as exc: + raise ValueConversionError(column, value, "invalid numeric source value") from exc + if not parsed.is_finite(): + raise ValueConversionError(column, value, "non-finite numeric source value") + return parsed + + +def _uuid_value(column: str, value: Any) -> Optional[uuid.UUID]: + if value is None or str(value).strip() == "": + return None + try: + return uuid.UUID(str(value)) + except ValueError as exc: + raise ValueConversionError(column, value, "invalid UUID source value") from exc + + +def _month_value(column: str, value: Any) -> Optional[str]: + if value is None or str(value).strip() == "": + return None + candidate = str(value).strip() + try: + date.fromisoformat(f"{candidate}-01") + except ValueError as exc: + raise ValueConversionError(column, value, "invalid ISO month source value") from exc + if not re.fullmatch(r"[0-9]{4}-(0[1-9]|1[0-2])", candidate): + raise ValueConversionError(column, value, "invalid ISO month source value") + return candidate + + +def _raw_date_value(column: str, value: Any) -> Optional[str]: + if value is None or str(value).strip() == "": + return None + candidate = str(value) + try: + date.fromisoformat(candidate[:10]) + except ValueError as exc: + raise ValueConversionError(column, value, "invalid ISO date source value") from exc + return candidate + + +def convert_value(table: str, column: str, value: Any) -> Any: + identity = (table, column) + converted: Any + if identity in JSON_COLUMNS: + converted = _json_value(table, column, value) + elif identity in BOOLEAN_COLUMNS: + if value in (None, ""): + converted = False + else: + if value not in (0, 1, False, True): + raise ValueConversionError(column, value, "invalid boolean source value") + converted = bool(value) + elif identity in DATE_COLUMNS: + converted = _date_value(column, value) + elif identity in TIMESTAMP_COLUMNS: + converted = _timestamp_value(column, value) + elif identity in NUMERIC_COLUMNS: + converted = _decimal_value(column, value) + elif identity in UUID_COLUMNS: + converted = _uuid_value(column, value) + elif identity in MONTH_COLUMNS: + converted = _month_value(column, value) + elif identity in RAW_DATE_COLUMNS: + converted = _raw_date_value(column, value) + else: + converted = value + + if converted is not None and column == "currency" and not re.fullmatch( + r"[A-Z]{3}", str(converted) + ): + raise ValueConversionError(column, value, "invalid ISO-style currency code") + if converted is not None and column in {"country_code", "origin_country_code"}: + if not re.fullmatch(r"[A-Z]{2}", str(converted)): + raise ValueConversionError(column, value, "invalid ISO-style country code") + if column in {"programme_area_method", "geography_method"}: + if converted is not None and converted not in CLASSIFICATION_METHODS: + raise ValueConversionError(column, value, "unknown classification method") + if column in {"geography_confidence", "match_confidence"} and converted is not None: + if Decimal(converted) < 0 or Decimal(converted) > 1: + raise ValueConversionError(column, value, "confidence outside [0,1]") + return converted + + +def _source_identity(table: str, row: sqlite3.Row) -> str: + return "|".join(str(row[column]) for column in SOURCE_ID_COLUMNS[table]) + + +def _target_columns(table: str, source_columns: Sequence[str]) -> tuple[str, ...]: + columns = tuple(RENAMES.get((table, column), column) for column in source_columns) + return (("dataset_version",) + columns) if table in VERSIONED_TABLES else columns + + +def _converted_record( + table: str, + source_columns: Sequence[str], + row: sqlite3.Row, + dataset_version: str, +) -> tuple[Any, ...]: + values = tuple(convert_value(table, column, row[column]) for column in source_columns) + return ((dataset_version,) + values) if table in VERSIONED_TABLES else values + + +async def _connect_postgres() -> asyncpg.Connection: + settings = DatabaseSettings.from_env() + url = settings.sqlalchemy_url() + connection = await asyncpg.connect( + host=url.host, + port=url.port or 5432, + user=url.username, + password=url.password, + database=url.database, + ssl=settings.ssl_mode, + command_timeout=None, + ) + await connection.execute("SET statement_timeout = 0") + await connection.execute("SET lock_timeout = '10s'") + return connection + + +async def _ensure_schema(connection: asyncpg.Connection) -> None: + revision = await connection.fetchval("SELECT version_num FROM alembic_version") + if revision != MIGRATION_SCHEMA_VERSION: + raise PreflightError( + f"PostgreSQL schema revision {revision!r} is not {MIGRATION_SCHEMA_VERSION!r}" + ) + + +async def _delete_candidate_payload( + connection: asyncpg.Connection, + dataset_version: str, +) -> None: + for table in reversed(LOAD_ORDER): + if table in VERSIONED_TABLES: + await connection.execute( + f'DELETE FROM "{table}" WHERE dataset_version=$1', + dataset_version, + ) + + +async def _prepare_candidate( + connection: asyncpg.Connection, + preflight: SourcePreflight, + dataset_version: str, + migration_run_id: uuid.UUID, + code_revision: str, + actor_id: str, + actor_type: str, +) -> str: + existing = await connection.fetchrow( + "SELECT status, is_active FROM dataset_versions WHERE dataset_version=$1", + dataset_version, + ) + if existing and existing["is_active"] and existing["status"] == "active": + return "already_active" + async with connection.transaction(): + if existing: + await connection.execute( + "DELETE FROM data_quality_issues WHERE dataset_version=$1", + dataset_version, + ) + await _delete_candidate_payload(connection, dataset_version) + await connection.execute( + "DELETE FROM migration_runs WHERE target_dataset_version=$1", + dataset_version, + ) + await connection.execute( + "DELETE FROM dataset_versions WHERE dataset_version=$1", + dataset_version, + ) + previous = await connection.fetchval( + "SELECT dataset_version FROM dataset_versions WHERE is_active" + ) + await connection.execute( + """ + INSERT INTO dataset_versions ( + dataset_version, status, is_active, source_checksum, + source_schema_version, code_revision, previous_dataset_version, + metadata + ) VALUES ($1, 'loading', FALSE, $2, $3, $4, $5, $6::jsonb) + """, + dataset_version, + preflight.checksum, + preflight.schema_version, + code_revision, + previous, + json.dumps({"source_metadata": preflight.metadata}, sort_keys=True), + ) + await connection.execute( + """ + INSERT INTO migration_runs ( + migration_run_id, target_dataset_version, + source_database_checksum, source_schema_version, + source_fact_version, target_schema_version, code_revision, + status, source_counts, actor_id, actor_type + ) VALUES ( + $1, $2, $3, $4, $5, $6, $7, 'loading', $8::jsonb, $9, $10 + ) + """, + migration_run_id, + dataset_version, + preflight.checksum, + preflight.schema_version, + preflight.metadata.get("grant_overview_schema_version", "unknown"), + MIGRATION_SCHEMA_VERSION, + code_revision, + json.dumps(preflight.counts, sort_keys=True), + actor_id, + actor_type, + ) + return "prepared" + + +async def _record_quality_issue( + connection: asyncpg.Connection, + dataset_version: str, + migration_run_id: uuid.UUID, + table: str, + record_identity: str, + error: ValueConversionError, +) -> None: + await connection.execute( + """ + INSERT INTO data_quality_issues ( + data_quality_issue_id, dataset_version, migration_run_id, + source_table, source_record_id, field_name, original_value, + reason, rule, status + ) VALUES ($1, $2, $3, $4, $5, $6, $7::jsonb, $8, $9, 'quarantined') + """, + uuid.uuid4(), + dataset_version, + migration_run_id, + table, + record_identity, + error.column, + json.dumps(error.value, ensure_ascii=False), + str(error), + "target_type_and_constraint_validation", + ) + + +async def _upsert_global_records( + connection: asyncpg.Connection, + table: str, + columns: Sequence[str], + records: Sequence[tuple[Any, ...]], +) -> None: + if not records: + return + key_columns = GLOBAL_KEYS[table] + records_to_write = list(records) + if table == "source_funder_link_overrides": + column_list = ", ".join(f'"{column}"' for column in columns) + key_indexes = tuple(columns.index(column) for column in key_columns) + key_predicate = " AND ".join( + f'"{column}"=${index}' for index, column in enumerate(key_columns, 1) + ) + revision_index = columns.index("revision") + records_to_write = [] + for record in records: + existing = await connection.fetchrow( + f'SELECT {column_list} FROM "{table}" WHERE {key_predicate}', + *(record[index] for index in key_indexes), + ) + if existing is None: + records_to_write.append(record) + continue + existing_values = tuple(existing[column] for column in columns) + if existing_values == record: + continue + if record[revision_index] != existing_values[revision_index] + 1: + raise MigrationError( + "Conflicting source-funder override must increment revision " + "by exactly one" + ) + records_to_write.append(record) + if not records_to_write: + return + update_columns = tuple(column for column in columns if column not in key_columns) + placeholders = ", ".join(f"${index}" for index in range(1, len(columns) + 1)) + assignments = ", ".join(f'"{column}"=EXCLUDED."{column}"' for column in update_columns) + sql = ( + f'INSERT INTO "{table}" ({", ".join(f"{column}" for column in columns)}) ' + f"VALUES ({placeholders}) ON CONFLICT ({', '.join(key_columns)}) " + f"DO UPDATE SET {assignments}" + ) + await connection.executemany(sql, records_to_write) + + +async def load_table( + source: sqlite3.Connection, + target: asyncpg.Connection, + table: str, + dataset_version: str, + migration_run_id: uuid.UUID, + batch_size: int, + staged_global_records: dict[str, list[tuple[Any, ...]]], + staged_global_columns: dict[str, tuple[str, ...]], +) -> int: + table_info = source.execute(f'PRAGMA table_info("{table}")').fetchall() + source_columns = tuple(str(row[1]) for row in table_info) + primary_key_columns = tuple( + str(row[1]) for row in sorted(table_info, key=lambda item: int(item[5])) if row[5] + ) + order = ", ".join(f'"{column}"' for column in primary_key_columns) + query = f'SELECT * FROM "{table}" ORDER BY {order}' + cursor = source.execute(query) + target_columns = _target_columns(table, source_columns) + loaded = 0 + while rows := cursor.fetchmany(batch_size): + records: list[tuple[Any, ...]] = [] + for row in rows: + try: + records.append(_converted_record(table, source_columns, row, dataset_version)) + except ValueConversionError as exc: + await _record_quality_issue( + target, + dataset_version, + migration_run_id, + table, + _source_identity(table, row), + exc, + ) + raise MigrationError( + f"Quarantined invalid value in {table}.{exc.column}; activation stopped" + ) from exc + if table in GLOBAL_KEYS: + staged_global_columns.setdefault(table, target_columns) + staged_global_records.setdefault(table, []).extend(records) + else: + async with target.transaction(): + await target.copy_records_to_table( + table, + records=records, + columns=target_columns, + ) + loaded += len(records) + return loaded + + +async def _target_counts( + connection: asyncpg.Connection, + dataset_version: str, + loaded_counts: dict[str, int], +) -> dict[str, int]: + counts: dict[str, int] = {} + for table in LOAD_ORDER: + if table in VERSIONED_TABLES: + counts[table] = int( + await connection.fetchval( + f'SELECT COUNT(*) FROM "{table}" WHERE dataset_version=$1', + dataset_version, + ) + ) + else: + counts[table] = loaded_counts[table] + return counts + + +async def _control_values( + connection: asyncpg.Connection, + dataset_version: str, +) -> dict[str, Any]: + values: dict[str, Any] = {} + values["distinct_mapped_grants"] = int( + await connection.fetchval( + """ + SELECT COUNT(DISTINCT grant_id) + FROM grant_beneficiary_countries WHERE dataset_version=$1 + """, + dataset_version, + ) + ) + values["classified_grants"] = int( + await connection.fetchval( + """ + SELECT COUNT(*) FROM grant_overview_facts + WHERE dataset_version=$1 + AND programme_provenance <> 'unclassified' + AND original_amount_status NOT IN ('negative', 'invalid', 'missing') + AND eur_amount_status NOT IN ('missing', 'invalid') + """, + dataset_version, + ) + ) + overview_minor = await connection.fetchval( + """ + SELECT COALESCE(SUM(eur_amount_minor), 0) + FROM grant_overview_facts + WHERE dataset_version=$1 + AND original_amount_status NOT IN ('negative', 'invalid', 'missing') + AND eur_amount_status NOT IN ('missing', 'invalid') + """, + dataset_version, + ) + values["overview_total_eur"] = format(Decimal(overview_minor) / 100, ".2f") + values["duplicate_source_identity_groups"] = int( + await connection.fetchval( + """ + SELECT COUNT(*) FROM ( + SELECT source, source_record_id + FROM grants + WHERE dataset_version=$1 AND source IS NOT NULL + AND source_record_id IS NOT NULL + GROUP BY source, source_record_id HAVING COUNT(*) > 1 + ) duplicates + """, + dataset_version, + ) + ) + status_counts = await connection.fetchrow( + """ + SELECT + COUNT(*) FILTER (WHERE eur_amount_status='missing') AS missing, + COUNT(*) FILTER (WHERE original_amount_status='negative') AS negative, + COUNT(*) FILTER (WHERE original_amount_status='zero') AS zero, + COUNT(*) FILTER (WHERE award_date > CURRENT_DATE) AS future + FROM grant_overview_facts WHERE dataset_version=$1 + """, + dataset_version, + ) + values["missing_eur_conversions"] = int(status_counts["missing"]) + values["negative_grants"] = int(status_counts["negative"]) + values["zero_grants"] = int(status_counts["zero"]) + values["future_dated_grants"] = int(status_counts["future"]) + values["business_key_duplicate_groups"] = int( + await connection.fetchval( + """ + SELECT COUNT(*) FROM ( + SELECT funding_name, recipient_name, amount, currency, + award_date, description + FROM grants WHERE dataset_version=$1 + GROUP BY funding_name, recipient_name, amount, currency, + award_date, description + HAVING COUNT(*) > 1 + ) duplicates + """, + dataset_version, + ) + ) + values["duplicate_charity_number_groups"] = int( + await connection.fetchval( + """ + SELECT COUNT(*) FROM ( + SELECT charity_number + FROM charity_registry_organizations WHERE dataset_version=$1 + GROUP BY charity_number HAVING COUNT(*) > 1 + ) duplicates + """, + dataset_version, + ) + ) + constraints = await connection.fetch( + """ + SELECT + child_namespace.nspname AS child_schema, + child_table.relname AS child_table, + parent_namespace.nspname AS parent_schema, + parent_table.relname AS parent_table, + array_agg(child_column.attname ORDER BY child_key.ordinality) AS child_columns, + array_agg(parent_column.attname ORDER BY child_key.ordinality) AS parent_columns + FROM pg_constraint AS constraint_definition + JOIN pg_class AS child_table + ON child_table.oid=constraint_definition.conrelid + JOIN pg_namespace AS child_namespace + ON child_namespace.oid=child_table.relnamespace + JOIN pg_class AS parent_table + ON parent_table.oid=constraint_definition.confrelid + JOIN pg_namespace AS parent_namespace + ON parent_namespace.oid=parent_table.relnamespace + JOIN LATERAL unnest(constraint_definition.conkey) WITH ORDINALITY + AS child_key(attnum, ordinality) ON TRUE + JOIN LATERAL unnest(constraint_definition.confkey) WITH ORDINALITY + AS parent_key(attnum, ordinality) + ON parent_key.ordinality=child_key.ordinality + JOIN pg_attribute AS child_column + ON child_column.attrelid=child_table.oid + AND child_column.attnum=child_key.attnum + JOIN pg_attribute AS parent_column + ON parent_column.attrelid=parent_table.oid + AND parent_column.attnum=parent_key.attnum + WHERE constraint_definition.contype='f' + AND child_namespace.nspname=current_schema() + GROUP BY child_namespace.nspname, child_table.relname, + parent_namespace.nspname, parent_table.relname, + constraint_definition.conname + ORDER BY child_table.relname, constraint_definition.conname + """ + ) + foreign_key_violations = 0 + for constraint in constraints: + child_table = ".".join( + _quote_identifier(part) + for part in (constraint["child_schema"], constraint["child_table"]) + ) + parent_table = ".".join( + _quote_identifier(part) + for part in (constraint["parent_schema"], constraint["parent_table"]) + ) + child_columns = [ + _quote_identifier(column) + for column in constraint["child_columns"] + ] + parent_columns = [ + _quote_identifier(column) + for column in constraint["parent_columns"] + ] + present = " AND ".join(f"child.{column} IS NOT NULL" for column in child_columns) + matches = " AND ".join( + f"parent.{parent} = child.{child}" + for child, parent in zip(child_columns, parent_columns) + ) + foreign_key_violations += int( + await connection.fetchval( + f""" + SELECT COUNT(*) FROM {child_table} AS child + WHERE {present} + AND NOT EXISTS ( + SELECT 1 FROM {parent_table} AS parent WHERE {matches} + ) + """ + ) + ) + values["foreign_key_violations"] = foreign_key_violations + return values + + +def _reconciliation( + source_counts: dict[str, int], + target_counts: dict[str, int], + controls: dict[str, Any], + expected_counts: Optional[dict[str, int]], + expected_controls: Optional[dict[str, Any]], +) -> dict[str, dict[str, Any]]: + results: dict[str, dict[str, Any]] = {} + for table, source_count in source_counts.items(): + target_count = target_counts[table] + results[f"count.{table}"] = { + "status": "pass" if source_count == target_count else "fail", + "source": source_count, + "target": target_count, + "difference": target_count - source_count, + "reason": None, + "approval_reference": None, + } + for name, expected in (expected_counts or {}).items(): + actual = target_counts.get(name) + results[f"baseline_count.{name}"] = { + "status": "pass" if actual == expected else "fail", + "source": expected, + "target": actual, + "difference": (actual - expected) if actual is not None else None, + "reason": None, + "approval_reference": None, + } + for name, expected in (expected_controls or {}).items(): + actual = controls.get(name) + results[f"control.{name}"] = { + "status": "pass" if actual == expected else "fail", + "source": expected, + "target": actual, + "difference": None, + "reason": None, + "approval_reference": None, + } + return results + + +async def _activate( + connection: asyncpg.Connection, + dataset_version: str, + migration_run_id: uuid.UUID, + counts: dict[str, int], + reconciliation: dict[str, dict[str, Any]], + staged_global_records: dict[str, list[tuple[Any, ...]]], + staged_global_columns: dict[str, tuple[str, ...]], +) -> Optional[str]: + failures = [name for name, result in reconciliation.items() if result["status"] == "fail"] + if failures: + await connection.execute( + """ + UPDATE dataset_versions SET status='rejected', rejected_at=CURRENT_TIMESTAMP + WHERE dataset_version=$1 AND NOT is_active + """, + dataset_version, + ) + await connection.execute( + """ + UPDATE migration_runs + SET status='rejected', completed_at=CURRENT_TIMESTAMP, + target_counts=$2::jsonb, reconciliation_results=$3::jsonb + WHERE migration_run_id=$1 + """, + migration_run_id, + json.dumps(counts, sort_keys=True), + json.dumps(reconciliation, sort_keys=True), + ) + raise MigrationError(f"Reconciliation failed: {', '.join(failures)}") + async with connection.transaction(): + for table in LOAD_ORDER: + if table in GLOBAL_KEYS: + records = staged_global_records.get(table, []) + if records: + await _upsert_global_records( + connection, + table, + staged_global_columns[table], + records, + ) + await connection.fetchval( + "SELECT refresh_analytics_materializations($1)", dataset_version + ) + previous = await connection.fetchval( + "SELECT dataset_version FROM dataset_versions WHERE is_active FOR UPDATE" + ) + if previous: + await connection.execute( + """ + UPDATE dataset_versions + SET is_active=FALSE, status='rolled_back' + WHERE dataset_version=$1 + """, + previous, + ) + await connection.execute( + """ + UPDATE dataset_versions + SET is_active=TRUE, status='active', approved_at=CURRENT_TIMESTAMP, + activated_at=CURRENT_TIMESTAMP + WHERE dataset_version=$1 AND NOT is_active + """, + dataset_version, + ) + await connection.execute( + """ + UPDATE migration_runs + SET status='active', completed_at=CURRENT_TIMESTAMP, + target_counts=$2::jsonb, reconciliation_results=$3::jsonb + WHERE migration_run_id=$1 + """, + migration_run_id, + json.dumps(counts, sort_keys=True), + json.dumps(reconciliation, sort_keys=True), + ) + return previous + + +async def migrate( + source_path: Path, + expected_checksum: str, + expected_schema_version: str, + dataset_version: str, + code_revision: str, + actor_id: str, + actor_type: str, + output_directory: Path, + *, + batch_size: int = 10_000, + enforce_baseline: bool = True, + enforce_capacity: bool = True, + remote_postgres: bool = False, +) -> dict[str, Any]: + if not re.fullmatch(r"[a-f0-9]{40}", code_revision): + raise PreflightError("Code revision must be a full lowercase Git SHA") + if not dataset_version or len(dataset_version) > 200: + raise PreflightError("Dataset version must contain 1 to 200 characters") + preflight = source_preflight( + source_path, + expected_checksum, + expected_schema_version, + enforce_capacity=enforce_capacity, + remote_postgres=remote_postgres, + ) + migration_run_id = uuid.uuid5( + uuid.NAMESPACE_URL, + f"foundation-intelligence:{preflight.checksum}:{dataset_version}:{code_revision}", + ) + target = await _connect_postgres() + started_at = datetime.now(timezone.utc) + try: + await _ensure_schema(target) + preparation = await _prepare_candidate( + target, + preflight, + dataset_version, + migration_run_id, + code_revision, + actor_id, + actor_type, + ) + if preparation == "already_active": + return { + "migration_run_id": str(migration_run_id), + "dataset_version": dataset_version, + "activation_status": "active", + "idempotent_noop": True, + } + staged_global_records: dict[str, list[tuple[Any, ...]]] = {} + staged_global_columns: dict[str, tuple[str, ...]] = {} + with open_sqlite_read_only(source_path) as source: + loaded_counts = {} + for table in LOAD_ORDER: + loaded_counts[table] = await load_table( + source, + target, + table, + dataset_version, + migration_run_id, + batch_size, + staged_global_records, + staged_global_columns, + ) + target_counts = await _target_counts(target, dataset_version, loaded_counts) + controls = await _control_values(target, dataset_version) + reconciliation = _reconciliation( + preflight.counts, + target_counts, + controls, + EXPECTED_COUNTS if enforce_baseline else None, + EXPECTED_CONTROLS if enforce_baseline else None, + ) + previous = await _activate( + target, + dataset_version, + migration_run_id, + target_counts, + reconciliation, + staged_global_records, + staged_global_columns, + ) + completed_at = datetime.now(timezone.utc) + report: dict[str, Any] = { + "migration_run_id": str(migration_run_id), + "source_database_checksum": preflight.checksum, + "source_schema_version": preflight.schema_version, + "source_fact_version": preflight.metadata.get( + "grant_overview_schema_version", "unknown" + ), + "target_schema_version": MIGRATION_SCHEMA_VERSION, + "dataset_version": dataset_version, + "code_revision": code_revision, + "started_at": started_at.isoformat(), + "completed_at": completed_at.isoformat(), + "source_counts": preflight.counts, + "target_counts": target_counts, + "controls": controls, + "reconciliation_results": reconciliation, + "activation_status": "active", + "rollback_dataset_version": previous, + "errors": [], + "actor": {"id": actor_id, "type": actor_type}, + "capacity": preflight.capacity.as_dict(), + "idempotent_noop": False, + } + write_reports(output_directory, report) + return report + except Exception as exc: + try: + await target.execute( + """ + UPDATE migration_runs + SET status='failed', completed_at=CURRENT_TIMESTAMP, + errors=$2::jsonb + WHERE migration_run_id=$1 AND status NOT IN ('active', 'rejected') + """, + migration_run_id, + json.dumps( + [{"code": exc.__class__.__name__, "message": str(exc)[:1000]}], + sort_keys=True, + ), + ) + await target.execute( + """ + UPDATE dataset_versions SET status='failed' + WHERE dataset_version=$1 AND NOT is_active + AND status NOT IN ('rejected', 'active') + """, + dataset_version, + ) + except Exception: + pass + raise + finally: + await target.close() + + +async def rollback_dataset(target_dataset_version: str) -> dict[str, Optional[str]]: + connection = await _connect_postgres() + try: + await _ensure_schema(connection) + async with connection.transaction(): + current = await connection.fetchval( + "SELECT dataset_version FROM dataset_versions WHERE is_active FOR UPDATE" + ) + target = await connection.fetchrow( + "SELECT status FROM dataset_versions WHERE dataset_version=$1 FOR UPDATE", + target_dataset_version, + ) + if not target: + raise MigrationError("Rollback target dataset does not exist") + if current == target_dataset_version: + return {"from": current, "to": target_dataset_version} + if target["status"] not in ("approved", "rolled_back"): + raise MigrationError("Rollback target is not an approved prior dataset") + materialized = await connection.fetchval( + """ + SELECT EXISTS ( + SELECT 1 FROM materialization_versions + WHERE dataset_version=$1 + AND materialization_name='dashboard_analytics' + AND is_active + ) + """, + target_dataset_version, + ) + if not materialized: + await connection.fetchval( + "SELECT refresh_analytics_materializations($1)", + target_dataset_version, + ) + if current: + await connection.execute( + """ + UPDATE dataset_versions SET is_active=FALSE, status='rolled_back' + WHERE dataset_version=$1 + """, + current, + ) + await connection.execute( + """ + UPDATE dataset_versions + SET is_active=TRUE, status='active', activated_at=CURRENT_TIMESTAMP + WHERE dataset_version=$1 + """, + target_dataset_version, + ) + return {"from": current, "to": target_dataset_version} + finally: + await connection.close() + + +def write_reports(output_directory: Path, report: dict[str, Any]) -> None: + output_directory.mkdir(parents=True, exist_ok=True) + stem = f"migration-{report['dataset_version']}" + json_path = output_directory / f"{stem}.json" + markdown_path = output_directory / f"{stem}.md" + json_path.write_text( + json.dumps(report, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + failures = [ + name + for name, result in report["reconciliation_results"].items() + if result["status"] != "pass" + ] + lines = [ + f"# Migration report: {report['dataset_version']}", + "", + f"- Run: `{report['migration_run_id']}`", + f"- Source checksum: `{report['source_database_checksum']}`", + f"- Target schema: `{report['target_schema_version']}`", + f"- Activation: `{report['activation_status']}`", + f"- Rollback dataset: `{report['rollback_dataset_version']}`", + f"- Reconciliation failures: `{len(failures)}`", + "", + "## Counts", + "", + "| Table | Source | Target |", + "|---|---:|---:|", + ] + for table in LOAD_ORDER: + lines.append( + f"| `{table}` | {report['source_counts'][table]} | " + f"{report['target_counts'][table]} |" + ) + lines.extend(["", "## Controls", "", "| Control | Value |", "|---|---:|"]) + for name, value in sorted(report["controls"].items()): + lines.append(f"| `{name}` | {value} |") + markdown_path.write_text("\n".join(lines) + "\n", encoding="utf-8") + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + preflight_parser = subparsers.add_parser("preflight") + migrate_parser = subparsers.add_parser("migrate") + for command_parser in (preflight_parser, migrate_parser): + command_parser.add_argument("--source", type=Path, required=True) + command_parser.add_argument("--expected-checksum", required=True) + command_parser.add_argument("--expected-schema-version", required=True) + migrate_parser.add_argument("--dataset-version", required=True) + migrate_parser.add_argument("--code-revision", required=True) + migrate_parser.add_argument("--actor-id", required=True) + migrate_parser.add_argument( + "--actor-type", choices=("human", "service", "ci"), required=True + ) + migrate_parser.add_argument("--output-directory", type=Path, required=True) + migrate_parser.add_argument("--batch-size", type=int, default=10_000) + migrate_parser.add_argument("--fixture", action="store_true") + migrate_parser.add_argument( + "--remote-postgres-capacity", + action="store_true", + help=( + "Size local capacity for a read-only SQLite source streamed to remote " + "PostgreSQL; PostgreSQL data and WAL are not counted as local files." + ), + ) + rollback_parser = subparsers.add_parser("rollback") + rollback_parser.add_argument("--dataset-version", required=True) + return parser + + +def main(argv: Optional[Sequence[str]] = None) -> int: + arguments = _parser().parse_args(argv) + try: + if arguments.command == "preflight": + result = source_preflight( + arguments.source, + arguments.expected_checksum, + arguments.expected_schema_version, + ) + print(json.dumps({ + "checksum": result.checksum, + "schema_version": result.schema_version, + "counts": result.counts, + "capacity": result.capacity.as_dict(), + }, indent=2, sort_keys=True)) + return 0 + if arguments.command == "rollback": + print(json.dumps(asyncio.run(rollback_dataset(arguments.dataset_version)))) + return 0 + if arguments.batch_size < 1 or arguments.batch_size > 100_000: + raise PreflightError("Batch size must be between 1 and 100000") + report = asyncio.run( + migrate( + arguments.source, + arguments.expected_checksum, + arguments.expected_schema_version, + arguments.dataset_version, + arguments.code_revision, + arguments.actor_id, + arguments.actor_type, + arguments.output_directory, + batch_size=arguments.batch_size, + enforce_baseline=not arguments.fixture, + enforce_capacity=not arguments.fixture, + remote_postgres=arguments.remote_postgres_capacity, + ) + ) + print(json.dumps({ + "migration_run_id": report["migration_run_id"], + "dataset_version": report["dataset_version"], + "activation_status": report["activation_status"], + "idempotent_noop": report.get("idempotent_noop", False), + }, sort_keys=True)) + return 0 + except MigrationError as exc: + print(f"migration failed: {exc}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/observability/__init__.py b/src/observability/__init__.py new file mode 100644 index 0000000..71caac0 --- /dev/null +++ b/src/observability/__init__.py @@ -0,0 +1 @@ +"""Structured telemetry contracts and local metric instrumentation.""" diff --git a/src/observability/metrics.py b/src/observability/metrics.py new file mode 100644 index 0000000..2d13ab8 --- /dev/null +++ b/src/observability/metrics.py @@ -0,0 +1,234 @@ +"""Dependency-free metric/alarm definitions with a bounded local registry.""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from pathlib import Path +import threading +from typing import Any, Mapping + + +OBSERVABILITY_CONFIGURATION_PATH = ( + Path(__file__).resolve().parents[2] / "config" / "observability.json" +) +REQUIRED_METRICS = frozenset( + { + "api_request_duration_ms", + "readiness_success", + "api_errors_total", + "db_pool_checked_out", + "db_pool_utilization_ratio", + "query_duration_ms", + "cache_hit_ratio", + "dataset_age_seconds", + "source_freshness_seconds", + "pipeline_duration_ms", + "pipeline_failures_total", + "pipeline_record_count", + "reconciliation_mismatch_count", + "conversion_gap_count", + "programme_coverage_ratio", + "geography_coverage_ratio", + "queue_oldest_message_age_seconds", + "dlq_depth", + "worker_failures_total", + "task_restarts_total", + "estimated_cost_usd", + } +) +REQUIRED_ALARMS = frozenset( + { + "readiness-failure", + "api-5xx-budget", + "api-latency-budget", + "stale-data", + "pipeline-failure", + "reconciliation-mismatch", + "dlq-messages", + "queue-backlog", + "conversion-gap-increase", + "programme-coverage-decrease", + "geography-coverage-decrease", + "cost-threshold", + "rds-cpu", + "rds-connections", + "rds-storage", + } +) + + +@dataclass(frozen=True) +class MetricDefinition: + name: str + type: str + unit: str + dimensions: tuple[str, ...] + + def validate(self) -> None: + if self.type not in {"counter", "gauge", "histogram"}: + raise ValueError(f"Invalid metric type for {self.name}") + if not self.name.strip() or not self.unit.strip(): + raise ValueError("Metric name and unit are required") + + +@dataclass(frozen=True) +class AlarmDefinition: + name: str + metric: str + comparison: str + threshold: float + evaluation_periods: int + period_seconds: int + runbook: str + + def validate(self) -> None: + if self.comparison not in { + "GreaterThanThreshold", + "GreaterThanOrEqualToThreshold", + "LessThanThreshold", + "LessThanOrEqualToThreshold", + }: + raise ValueError(f"Invalid alarm comparison for {self.name}") + if self.evaluation_periods < 1 or self.period_seconds < 10: + raise ValueError(f"Invalid alarm evaluation window for {self.name}") + if not self.runbook.strip(): + raise ValueError(f"Alarm runbook is required for {self.name}") + + +@dataclass(frozen=True) +class ObservabilityConfiguration: + service: str + expected_schema_version: str + metrics: tuple[MetricDefinition, ...] + alarms: tuple[AlarmDefinition, ...] + + def validate(self) -> None: + if not self.service.strip() or not self.expected_schema_version.strip(): + raise ValueError("Service and expected schema version are required") + for definition in self.metrics: + definition.validate() + for alarm in self.alarms: + alarm.validate() + metric_names = {definition.name for definition in self.metrics} + alarm_names = {definition.name for definition in self.alarms} + missing_metrics = sorted(REQUIRED_METRICS - metric_names) + missing_alarms = sorted(REQUIRED_ALARMS - alarm_names) + if missing_metrics: + raise ValueError(f"Missing metrics: {', '.join(missing_metrics)}") + if missing_alarms: + raise ValueError(f"Missing alarms: {', '.join(missing_alarms)}") + unknown_alarm_metrics = sorted( + { + alarm.metric + for alarm in self.alarms + if alarm.metric not in metric_names and not alarm.metric.startswith("aws_") + } + ) + if unknown_alarm_metrics: + raise ValueError( + f"Alarms reference undefined metrics: {', '.join(unknown_alarm_metrics)}" + ) + if len(metric_names) != len(self.metrics) or len(alarm_names) != len(self.alarms): + raise ValueError("Metric and alarm names must be unique") + + +def load_observability_configuration( + path: Path = OBSERVABILITY_CONFIGURATION_PATH, +) -> ObservabilityConfiguration: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("configuration_version") != "1": + raise ValueError("Unsupported observability configuration version") + configuration = ObservabilityConfiguration( + service=str(payload["service"]), + expected_schema_version=str(payload["expected_schema_version"]), + metrics=tuple( + MetricDefinition( + name=str(entry["name"]), + type=str(entry["type"]), + unit=str(entry["unit"]), + dimensions=tuple(entry.get("dimensions", ())), + ) + for entry in payload.get("metrics", []) + ), + alarms=tuple( + AlarmDefinition( + name=str(entry["name"]), + metric=str(entry["metric"]), + comparison=str(entry["comparison"]), + threshold=float(entry["threshold"]), + evaluation_periods=int(entry["evaluation_periods"]), + period_seconds=int(entry["period_seconds"]), + runbook=str(entry["runbook"]), + ) + for entry in payload.get("alarms", []) + ), + ) + configuration.validate() + return configuration + + +class MetricsRegistry: + """Bounded process-local evidence; production publishing uses CloudWatch.""" + + def __init__(self, configuration: ObservabilityConfiguration): + configuration.validate() + self.configuration = configuration + self._definitions = {definition.name: definition for definition in configuration.metrics} + self._values: dict[tuple[str, tuple[tuple[str, str], ...]], dict[str, float]] = {} + self._lock = threading.Lock() + + @staticmethod + def _key(name: str, dimensions: Mapping[str, object]) -> tuple[str, tuple[tuple[str, str], ...]]: + return name, tuple(sorted((str(key), str(value)) for key, value in dimensions.items())) + + def _validate(self, name: str, dimensions: Mapping[str, object]) -> MetricDefinition: + definition = self._definitions.get(name) + if definition is None: + raise ValueError(f"Unknown metric: {name}") + allowed = set(definition.dimensions) + if set(dimensions) - allowed: + raise ValueError(f"Unexpected dimensions for metric {name}") + return definition + + def increment(self, name: str, value: float = 1.0, **dimensions: object) -> None: + definition = self._validate(name, dimensions) + if definition.type != "counter" or value < 0: + raise ValueError(f"Metric {name} is not a non-negative counter") + key = self._key(name, dimensions) + with self._lock: + record = self._values.setdefault(key, {"value": 0.0}) + record["value"] += float(value) + + def set_gauge(self, name: str, value: float, **dimensions: object) -> None: + definition = self._validate(name, dimensions) + if definition.type != "gauge": + raise ValueError(f"Metric {name} is not a gauge") + with self._lock: + self._values[self._key(name, dimensions)] = {"value": float(value)} + + def observe(self, name: str, value: float, **dimensions: object) -> None: + definition = self._validate(name, dimensions) + if definition.type != "histogram" or value < 0: + raise ValueError(f"Metric {name} is not a non-negative histogram") + key = self._key(name, dimensions) + with self._lock: + record = self._values.setdefault( + key, + {"count": 0.0, "sum": 0.0, "min": float(value), "max": float(value)}, + ) + record["count"] += 1.0 + record["sum"] += float(value) + record["min"] = min(record["min"], float(value)) + record["max"] = max(record["max"], float(value)) + + def snapshot(self) -> list[dict[str, Any]]: + with self._lock: + return [ + { + "name": name, + "dimensions": dict(dimensions), + **values, + } + for (name, dimensions), values in sorted(self._values.items()) + ] diff --git a/src/pipelines/durable.py b/src/pipelines/durable.py new file mode 100644 index 0000000..39f9e59 --- /dev/null +++ b/src/pipelines/durable.py @@ -0,0 +1,305 @@ +"""Pure durable-pipeline contracts shared by API, workers and local tests. + +This module deliberately contains no AWS SDK calls. Production adapters can +publish the queue envelope to SQS and write the immutable object descriptor to +S3, while the same contracts remain fully testable without network access. +""" + +from __future__ import annotations + +from dataclasses import asdict, dataclass +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import re +from typing import Any, Iterable, Mapping, Protocol +from uuid import UUID + + +SOURCE_CONFIGURATION_PATH = ( + Path(__file__).resolve().parents[2] / "config" / "source-pipelines.json" +) +LEGAL_STATES = frozenset({"approved", "unresolved", "restricted", "prohibited"}) +OBJECT_ZONES = frozenset({"raw", "validated", "curated", "export"}) +SOURCE_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9_-]{0,79}$") + + +def canonical_json(value: object) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +@dataclass(frozen=True) +class SourceConfiguration: + source_name: str + source_owner: str + technical_owner: str + legal_status: str + licence_status: str + terms_url: str | None + rate_limit_per_minute: int + user_agent: str + freshness_sla_hours: int + schedule_expression: str + enabled: bool + governance_blocked: bool + last_success: str | None + watermark: str | None + classification: str + retention_class: str + schema_version: str + credentials_reference: str | None + retry_limit: int + timeout_seconds: int + maximum_pages: int + maximum_records: int + + def validate(self) -> None: + if not SOURCE_NAME_PATTERN.fullmatch(self.source_name): + raise ValueError(f"Invalid source name: {self.source_name!r}") + if self.legal_status not in LEGAL_STATES or self.licence_status not in LEGAL_STATES: + raise ValueError(f"Invalid governance state for {self.source_name}") + required_text = ( + self.source_owner, + self.technical_owner, + self.user_agent, + self.schedule_expression, + self.classification, + self.retention_class, + self.schema_version, + ) + if any(not value.strip() for value in required_text): + raise ValueError(f"Incomplete source configuration for {self.source_name}") + if min( + self.rate_limit_per_minute, + self.freshness_sla_hours, + self.timeout_seconds, + self.maximum_pages, + self.maximum_records, + ) < 1: + raise ValueError(f"Source limits must be positive for {self.source_name}") + if not 0 <= self.retry_limit <= 20: + raise ValueError(f"Retry limit is outside the allowed range for {self.source_name}") + if self.enabled and ( + self.governance_blocked + or self.legal_status != "approved" + or self.licence_status != "approved" + ): + raise ValueError(f"Governance-blocked source cannot be scheduled: {self.source_name}") + + @property + def configuration_checksum(self) -> str: + return sha256_bytes(canonical_json(asdict(self)).encode("utf-8")) + + def database_record(self) -> dict[str, Any]: + self.validate() + result = asdict(self) + result["configuration_checksum"] = self.configuration_checksum + return result + + +def load_source_configurations( + path: Path = SOURCE_CONFIGURATION_PATH, +) -> tuple[SourceConfiguration, ...]: + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("configuration_version") != "1": + raise ValueError("Unsupported source configuration version") + sources = tuple(SourceConfiguration(**entry) for entry in payload.get("sources", [])) + if not sources: + raise ValueError("At least one source configuration is required") + names = [source.source_name for source in sources] + if len(names) != len(set(names)): + raise ValueError("Source names must be unique") + for source in sources: + source.validate() + return sources + + +@dataclass(frozen=True) +class StorageObject: + storage_object_id: UUID + zone: str + bucket_alias: str + object_key: str + object_version: str + checksum: str + content_length: int + content_type: str + source_name: str | None = None + source_ingestion_run_id: UUID | None = None + immutable: bool = True + + def validate(self) -> None: + if self.zone not in OBJECT_ZONES: + raise ValueError(f"Unsupported object zone: {self.zone}") + if self.zone == "raw" and not self.immutable: + raise ValueError("Raw source objects must be immutable") + if not re.fullmatch(r"[a-f0-9]{64}", self.checksum): + raise ValueError("Object checksum must be lowercase SHA-256") + if self.content_length < 0: + raise ValueError("Object length cannot be negative") + if not all( + value.strip() + for value in (self.bucket_alias, self.object_key, self.object_version, self.content_type) + ): + raise ValueError("Object location and content type are required") + + +def object_key( + *, source_name: str, run_id: UUID, zone: str, checksum: str, extension: str +) -> str: + if not SOURCE_NAME_PATTERN.fullmatch(source_name): + raise ValueError("Invalid source name") + if zone not in OBJECT_ZONES: + raise ValueError("Invalid object zone") + if not re.fullmatch(r"[a-f0-9]{64}", checksum): + raise ValueError("Invalid object checksum") + suffix = extension.strip().lower().lstrip(".") + if not re.fullmatch(r"[a-z0-9]{1,12}", suffix): + raise ValueError("Invalid object extension") + return f"{zone}/{source_name}/{run_id}/{checksum}.{suffix}" + + +@dataclass(frozen=True) +class IngestionManifest: + source_ingestion_run_id: UUID + schema_version: str + source_name: str + source_version: str | None + dataset_version: str + raw_object_id: UUID | None + validated_object_id: UUID | None + curated_object_id: UUID | None + watermark_before: str | None + watermark_after: str | None + record_count: int + accepted_count: int + rejected_count: int + quarantined_count: int + retry_count: int + generated_at: str + + @classmethod + def create(cls, **values: Any) -> "IngestionManifest": + values.setdefault("generated_at", datetime.now(timezone.utc).isoformat()) + manifest = cls(**values) + manifest.validate() + return manifest + + def validate(self) -> None: + if not SOURCE_NAME_PATTERN.fullmatch(self.source_name): + raise ValueError("Invalid manifest source") + counts = ( + self.record_count, + self.accepted_count, + self.rejected_count, + self.quarantined_count, + self.retry_count, + ) + if min(counts) < 0: + raise ValueError("Manifest counts cannot be negative") + classified = self.accepted_count + self.rejected_count + self.quarantined_count + if classified > self.record_count: + raise ValueError("Manifest result counts exceed the source record count") + if not self.schema_version.strip() or not self.dataset_version.strip(): + raise ValueError("Manifest schema and dataset versions are required") + + @property + def payload(self) -> dict[str, Any]: + self.validate() + result = asdict(self) + for key, value in tuple(result.items()): + if isinstance(value, UUID): + result[key] = str(value) + return result + + @property + def checksum(self) -> str: + return sha256_bytes(canonical_json(self.payload).encode("utf-8")) + + +@dataclass(frozen=True) +class QueueEnvelope: + schema_version: str + job_id: UUID + job_type: str + queue_name: str + requested_at: str + attempt: int + max_attempts: int + + def validate(self) -> None: + if self.schema_version != "job-envelope-v1": + raise ValueError("Unsupported queue envelope version") + if not self.job_type.strip() or not self.queue_name.strip(): + raise ValueError("Queue and job type are required") + if self.attempt < 1 or self.max_attempts < self.attempt: + raise ValueError("Invalid queue attempt") + + @property + def payload(self) -> dict[str, Any]: + self.validate() + result = asdict(self) + result["job_id"] = str(self.job_id) + return result + + +class ObjectStore(Protocol): + """Minimal immutable object-store boundary implemented by an S3 adapter.""" + + async def put_if_absent(self, descriptor: StorageObject, payload: bytes) -> None: ... + + async def get(self, descriptor: StorageObject) -> bytes: ... + + +class InMemoryObjectStore: + """Deterministic local substitute; never used as production coordination.""" + + def __init__(self) -> None: + self._objects: dict[tuple[str, str, str, str], tuple[StorageObject, bytes]] = {} + + @staticmethod + def _identity(descriptor: StorageObject) -> tuple[str, str, str, str]: + return ( + descriptor.zone, + descriptor.bucket_alias, + descriptor.object_key, + descriptor.object_version, + ) + + async def put_if_absent(self, descriptor: StorageObject, payload: bytes) -> None: + descriptor.validate() + if descriptor.content_length != len(payload): + raise ValueError("Object length does not match payload") + if descriptor.checksum != sha256_bytes(payload): + raise ValueError("Object checksum does not match payload") + identity = self._identity(descriptor) + existing = self._objects.get(identity) + if existing: + if existing[1] != payload: + raise ValueError("Immutable object identity already contains different bytes") + return + self._objects[identity] = (descriptor, bytes(payload)) + + async def get(self, descriptor: StorageObject) -> bytes: + return bytes(self._objects[self._identity(descriptor)][1]) + + +def require_sources( + configurations: Iterable[SourceConfiguration], required: Iterable[str] +) -> None: + names = {source.source_name for source in configurations} + missing = sorted(set(required) - names) + if missing: + raise ValueError(f"Missing source configurations: {', '.join(missing)}") + + +def redacted_configuration(configuration: Mapping[str, Any]) -> dict[str, Any]: + result = dict(configuration) + if result.get("credentials_reference"): + result["credentials_reference"] = "configured-reference" + return result diff --git a/src/pipelines/durable_worker.py b/src/pipelines/durable_worker.py new file mode 100644 index 0000000..dbe4ae8 --- /dev/null +++ b/src/pipelines/durable_worker.py @@ -0,0 +1,183 @@ +"""Queue/outbox worker lifecycle without API subprocesses or local lock files.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Any, Awaitable, Callable, Mapping, Protocol + +from bff.postgres.job_repository import PostgresJobRepository +from bff.utils.logging import logger + + +JobHandler = Callable[[Mapping[str, Any]], Awaitable[Mapping[str, Any]]] + + +class QueuePublisher(Protocol): + """SQS-compatible publishing boundary used by the transactional outbox.""" + + async def publish( + self, *, queue_name: str, message: Mapping[str, Any], deduplication_id: str + ) -> str: ... + + +@dataclass(frozen=True) +class WorkerResult: + status: str + job_id: str | None + + +class DurableWorker: + def __init__( + self, + repository: PostgresJobRepository, + handlers: Mapping[str, JobHandler], + *, + worker_id: str, + queue_name: str = "pipeline", + lease_seconds: int = 60, + ) -> None: + self.repository = repository + self.handlers = dict(handlers) + self.worker_id = worker_id + self.queue_name = queue_name + self.lease_seconds = lease_seconds + + async def run_once(self) -> WorkerResult: + job = await self.repository.claim( + worker_id=self.worker_id, + queue_name=self.queue_name, + lease_seconds=self.lease_seconds, + ) + if job is None: + return WorkerResult(status="idle", job_id=None) + job_id = str(job["job_id"]) + logger.info( + "job_started", + extra={ + "job_id": job_id, + "dataset_version": job.get("dataset_version"), + "operation": str(job["job_type"]), + "status": "running", + "retry_count": max(int(job.get("attempt", 1)) - 1, 0), + }, + ) + handler = self.handlers.get(str(job["job_type"])) + if handler is None: + await self.repository.fail( + job_id, + worker_id=self.worker_id, + error_class="UnsupportedJobType", + failure_reason="No worker handler is registered for this job type", + retryable=False, + ) + logger.error( + "job_failed", + extra={ + "job_id": job_id, + "operation": str(job["job_type"]), + "status": "failed", + "error_class": "UnsupportedJobType", + }, + ) + return WorkerResult(status="failed", job_id=job_id) + try: + async with asyncio.timeout(int(job["timeout_seconds"])): + result = await handler(job) + except asyncio.TimeoutError: + status = await self.repository.fail( + job_id, + worker_id=self.worker_id, + error_class="JobTimeout", + failure_reason="Worker execution exceeded the durable job timeout", + retryable=True, + ) + logger.error( + "job_timed_out", + extra={ + "job_id": job_id, + "operation": str(job["job_type"]), + "status": status, + "error_class": "JobTimeout", + "retry_count": int(job.get("attempt", 1)), + }, + ) + return WorkerResult(status=status, job_id=job_id) + except Exception as exc: + status = await self.repository.fail( + job_id, + worker_id=self.worker_id, + error_class=exc.__class__.__name__, + failure_reason="Worker handler failed; last-good data remains active", + retryable=True, + ) + logger.error( + "job_failed", + extra={ + "job_id": job_id, + "operation": str(job["job_type"]), + "status": status, + "error_class": exc.__class__.__name__, + "retry_count": int(job.get("attempt", 1)), + }, + ) + return WorkerResult(status=status, job_id=job_id) + succeeded = await self.repository.succeed( + job_id, + worker_id=self.worker_id, + result=result, + ) + logger.info( + "job_completed", + extra={ + "job_id": job_id, + "dataset_version": job.get("dataset_version"), + "operation": str(job["job_type"]), + "status": "succeeded" if succeeded else "lost_lease", + "record_count": result.get("record_count"), + "accepted_count": result.get("accepted_count"), + "rejected_count": result.get("rejected_count"), + "quarantined_count": result.get("quarantined_count"), + "retry_count": max(int(job.get("attempt", 1)) - 1, 0), + }, + ) + return WorkerResult(status="succeeded" if succeeded else "lost_lease", job_id=job_id) + + +class OutboxDispatcher: + def __init__( + self, + repository: PostgresJobRepository, + publisher: QueuePublisher, + ) -> None: + self.repository = repository + self.publisher = publisher + + async def run_once(self, *, limit: int = 100) -> dict[str, int]: + published = 0 + failed = 0 + dead_lettered = 0 + for dispatch in await self.repository.due_dispatches(limit=limit): + try: + message_id = await self.publisher.publish( + queue_name=str(dispatch["queue_name"]), + message=dict(dispatch["message_body"]), + deduplication_id=str(dispatch["job_id"]), + ) + await self.repository.mark_dispatch_published( + str(dispatch["outbox_id"]), queue_message_id=message_id + ) + published += 1 + except Exception as exc: + status = await self.repository.mark_dispatch_failed( + str(dispatch["outbox_id"]), error_class=exc.__class__.__name__ + ) + if status == "dead_lettered": + dead_lettered += 1 + else: + failed += 1 + return { + "published": published, + "failed": failed, + "dead_lettered": dead_lettered, + } diff --git a/src/tests/conftest.py b/src/tests/conftest.py new file mode 100644 index 0000000..301b215 --- /dev/null +++ b/src/tests/conftest.py @@ -0,0 +1,25 @@ +"""Explicit, local-only security configuration for the test process.""" + +import os + + +os.environ.update( + { + "APP_ENV": "test", + "DATA_RUNTIME_MODE": "sqlite_migration_source", + "AUTH_MODE": "development", + "DEV_AUTH_ENABLED": "true", + "DEV_AUTH_USERNAME": "admin", + "DEV_AUTH_PASSWORD": "password", + "DEV_AUTH_SECRET": "unit-test-signing-key-with-at-least-32-characters", + "DEV_AUTH_ALLOWED_HOSTS": "testclient,127.0.0.1,::1,localhost", + "SESSION_COOKIE_SECURE": "false", + "RATE_LIMIT_REQUESTS": "10000", + "CORE_PROXY_ENABLED": "true", + "CORE_API_URL": "http://127.0.0.1:8080", + "CORE_API_ALLOWED_HOSTS": "127.0.0.1", + "CORE_PROXY_ALLOWED_PATHS": "v1/data,users", + "CORE_PROXY_ALLOWED_METHODS": "GET", + "CORE_PROXY_FORWARD_HEADERS": "accept,content-type,x-request-id", + } +) diff --git a/src/tests/test_api_golden.py b/src/tests/test_api_golden.py new file mode 100644 index 0000000..73dd877 --- /dev/null +++ b/src/tests/test_api_golden.py @@ -0,0 +1,49 @@ +import json +from pathlib import Path +import unittest + +from fastapi.testclient import TestClient + +from bff.main import app + + +CONTRACT = json.loads( + (Path(__file__).resolve().parents[2] / "config/golden/api-contract.json").read_text( + encoding="utf-8" + ) +) + + +class _GoldenDatabase: + @staticmethod + def pool_status(): + return {"checked_out": 0.0, "capacity": 1.0, "utilization_ratio": 0.0} + + async def readiness( + self, *, expected_schema_version, require_critical_configuration=True + ): + return { + "ready": True, + "checks": CONTRACT["health"]["ready"]["checks"], + "metadata": {"queue_age_seconds": 0, "dead_letter_count": 0}, + } + + +class TestApiGoldenContract(unittest.TestCase): + def test_health_payloads_and_required_openapi_paths(self): + original = app.state.database + app.state.database = _GoldenDatabase() + try: + client = TestClient(app) + self.assertEqual(client.get("/health").json(), CONTRACT["health"]["legacy"]) + self.assertEqual(client.get("/health/live").json(), CONTRACT["health"]["live"]) + self.assertEqual(client.get("/health/ready").json(), CONTRACT["health"]["ready"]) + paths = client.get("/openapi.json").json()["paths"] + for required in CONTRACT["required_paths"]: + self.assertIn(required, paths) + finally: + app.state.database = original + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_aws_migration_entrypoint.py b/src/tests/test_aws_migration_entrypoint.py new file mode 100644 index 0000000..ce8bc7b --- /dev/null +++ b/src/tests/test_aws_migration_entrypoint.py @@ -0,0 +1,150 @@ +import unittest +from unittest.mock import AsyncMock, patch + +from migration.aws_entrypoint import ( + AwsMigrationConfigurationError, + _connect_admin, + _configure_application_role, + _identifier, + _run_alembic, + _verify_application_role, +) + + +class _RecordingConnection: + def __init__(self, role_exists: bool = False): + self.role_exists = role_exists + self.statements: list[str] = [] + + async def fetchval(self, statement: str, *_args): + self.statements.append(statement) + return self.role_exists + + async def execute(self, statement: str): + self.statements.append(statement) + + +class _VerifiedApplicationConnection: + async def fetchval(self, statement: str): + if statement == "SELECT 1": + return 1 + if statement.startswith("SELECT ssl FROM pg_stat_ssl"): + return True + if statement == "SHOW default_transaction_read_only": + return "on" + raise AssertionError(f"Unexpected query: {statement}") + + async def execute(self, statement: str): + if statement.startswith("UPDATE dataset_versions"): + import asyncpg + + raise asyncpg.ReadOnlySQLTransactionError("read-only transaction") + raise AssertionError(f"Unexpected statement: {statement}") + + async def close(self): + return None + + +class TestAwsMigrationRoleSafety(unittest.IsolatedAsyncioTestCase): + async def test_admin_connection_requires_tls_for_rds(self): + password = "A@b:%/+#?&xyz" + connection = AsyncMock() + with patch( + "migration.aws_entrypoint.asyncpg.connect", + new=AsyncMock(return_value=connection), + ) as connect: + result = await _connect_admin( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_ADMIN_USER": "foundation_admin", + "DATABASE_ADMIN_PASSWORD": password, + "DATABASE_SSL_MODE": "require", + } + ) + self.assertIs(result, connection) + self.assertEqual(connect.await_args.kwargs["ssl"], "require") + self.assertEqual(connect.await_args.kwargs["password"], password) + + def test_alembic_environment_receives_unmodified_reserved_password(self): + password = "A@b:%/+#?&xyz" + with patch("migration.aws_entrypoint.subprocess.run") as run: + _run_alembic( + { + "DATABASE_ADMIN_USER": "foundation_admin", + "DATABASE_ADMIN_PASSWORD": password, + "DATABASE_SSL_MODE": "require", + } + ) + self.assertEqual(run.call_args.kwargs["env"]["DATABASE_PASSWORD"], password) + + async def test_application_role_verification_requires_tls_and_denied_update(self): + connection = _VerifiedApplicationConnection() + with patch( + "migration.aws_entrypoint.asyncpg.connect", + new=AsyncMock(return_value=connection), + ) as connect: + result = await _verify_application_role( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_APP_USER": "foundation_app", + "DATABASE_APP_PASSWORD": "runtime-only-secret", + "DATABASE_SSL_MODE": "require", + } + ) + self.assertEqual(connect.await_args.kwargs["ssl"], "require") + self.assertEqual( + result, + { + "select_succeeded": True, + "tls_in_use": True, + "default_read_only": True, + "update_denied": True, + }, + ) + + async def test_application_role_is_select_only_and_default_read_only(self): + connection = _RecordingConnection() + await _configure_application_role( + connection, # type: ignore[arg-type] + { + "DATABASE_APP_USER": "foundation_app", + "DATABASE_APP_PASSWORD": "generatedsecret", + "DATABASE_ADMIN_USER": "foundation_admin", + "DATABASE_NAME": "foundation_intelligence", + }, + ) + sql = "\n".join(connection.statements) + self.assertIn("NOSUPERUSER NOCREATEDB NOCREATEROLE", sql) + self.assertIn("GRANT SELECT ON ALL TABLES", sql) + self.assertIn("REVOKE ALL ON ALL SEQUENCES", sql) + self.assertIn("REVOKE EXECUTE ON ALL FUNCTIONS", sql) + self.assertIn("default_transaction_read_only = on", sql) + self.assertNotIn("GRANT SELECT, INSERT", sql) + + async def test_existing_role_is_demoted_before_reuse(self): + connection = _RecordingConnection(role_exists=True) + await _configure_application_role( + connection, # type: ignore[arg-type] + { + "DATABASE_APP_USER": "foundation_app", + "DATABASE_APP_PASSWORD": "generatedsecret", + "DATABASE_ADMIN_USER": "foundation_admin", + "DATABASE_NAME": "foundation_intelligence", + }, + ) + alter_role = next( + statement + for statement in connection.statements + if statement.startswith("ALTER ROLE \"foundation_app\" LOGIN") + ) + self.assertIn("NOSUPERUSER NOCREATEDB NOCREATEROLE", alter_role) + + def test_unsafe_identifiers_fail_before_sql(self): + with self.assertRaises(AwsMigrationConfigurationError): + _identifier("foundation_app; DROP SCHEMA public", "DATABASE_APP_USER") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_bff.py b/src/tests/test_bff.py index 12f7b81..801371d 100644 --- a/src/tests/test_bff.py +++ b/src/tests/test_bff.py @@ -15,7 +15,6 @@ sys.path.insert(0, SRC_DIR) from bff.main import app -from bff.config import JWT_SECRET_KEY, JWT_ALGORITHM, DATA_PATH from bff.repositories import JSONCharityRepository, get_charity_repository from bff import admin as admin_module @@ -305,7 +304,7 @@ def test_proxy_unauthenticated(self): self.assertEqual(response.status_code, 401) @patch("httpx.AsyncClient.request") - def test_proxy_success_translates_cookie_to_bearer(self, mock_request): + def test_proxy_success_does_not_forward_browser_credentials(self, mock_request): # Create token and login login_resp = self.client.post( "/api/auth/login", @@ -337,15 +336,12 @@ def test_proxy_success_translates_cookie_to_bearer(self, mock_request): self.assertEqual(kwargs["method"], "GET") # Ensure query string was preserved and path appended to core url self.assertTrue(kwargs["url"].endswith("/v1/data?filter=active")) - # Ensure cookie was stripped and replaced with Bearer Authorization + # Browser credentials and unlisted headers must not cross the boundary. headers = kwargs["headers"] - self.assertNotIn("Cookie", headers) - self.assertIn("Authorization", headers) - self.assertTrue(headers["Authorization"].startswith("Bearer ")) - # Ensure other request headers were forwarded - # ASGI lowercases header names, so check case-insensitively - test_header_val = headers.get("x-test-header") or headers.get("X-Test-Header") - self.assertEqual(test_header_val, "yes") + self.assertNotIn("cookie", headers) + self.assertNotIn("authorization", headers) + self.assertNotIn("x-test-header", headers) + self.assertIn("x-request-id", headers) @patch("httpx.AsyncClient.request") @@ -526,6 +522,7 @@ def test_enrich_source_funders_is_bounded_and_uses_confirmed_numbers(self, mock_ "/api/charities/grants/funders/enrich", json={"reg_numbers": [1002, 1001, 1001]}, cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-enrich-source-funders-success"}, ) self.assertEqual(response.status_code, 200) @@ -543,6 +540,7 @@ def test_enrich_source_funders_rejects_more_than_five_organizations(self): "/api/charities/grants/funders/enrich", json={"reg_numbers": [1, 2, 3, 4, 5, 6]}, cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-enrich-source-funders-too-many"}, ) self.assertEqual(response.status_code, 422) @@ -559,6 +557,7 @@ def test_registry_detail_enrichment_is_single_and_uses_confirmed_number(self, mo "/api/charities/directory/organizations/enrich", json={"reg_numbers": [1165944]}, cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-registry-enrichment-success"}, ) self.assertEqual(response.status_code, 200) @@ -576,6 +575,7 @@ def test_registry_detail_enrichment_rejects_more_than_one_organization(self): "/api/charities/directory/organizations/enrich", json={"reg_numbers": [1001, 1002]}, cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-registry-enrichment-too-many"}, ) self.assertEqual(response.status_code, 400) @@ -591,7 +591,8 @@ def test_trigger_pipeline_success(self, mock_run_task): response = self.client.post( "/api/admin/pipeline/trigger", json={"source": "quick_consolidate"}, - cookies={"session_id": session_cookie} + cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-trigger-pipeline-success"}, ) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["status"], "running") @@ -609,7 +610,8 @@ def test_trigger_pipeline_full_run_success(self, mock_run_task): response = self.client.post( "/api/admin/pipeline/trigger", json={"source": "full_run", "limit": 10, "fresh": True}, - cookies={"session_id": session_cookie} + cookies={"session_id": session_cookie}, + headers={"Idempotency-Key": "test-trigger-pipeline-full-run"}, ) self.assertEqual(response.status_code, 200) self.assertEqual(response.json()["status"], "running") @@ -831,7 +833,8 @@ def test_trigger_pipeline_already_running(self): response = self.client.post( "/api/admin/pipeline/trigger", json={"source": "quick_consolidate"}, - cookies={"session_id": self.session_cookie} + cookies={"session_id": self.session_cookie}, + headers={"Idempotency-Key": "test-trigger-pipeline-running"}, ) self.assertEqual(response.status_code, 409) self.assertIn("already in progress", response.json()["detail"]) @@ -840,7 +843,8 @@ def test_trigger_pipeline_invalid_mode(self): response = self.client.post( "/api/admin/pipeline/trigger", json={"source": "invalid_mode"}, - cookies={"session_id": self.session_cookie} + cookies={"session_id": self.session_cookie}, + headers={"Idempotency-Key": "test-trigger-pipeline-invalid"}, ) self.assertEqual(response.status_code, 400) diff --git a/src/tests/test_ci_performance_smoke.py b/src/tests/test_ci_performance_smoke.py new file mode 100644 index 0000000..105647c --- /dev/null +++ b/src/tests/test_ci_performance_smoke.py @@ -0,0 +1,38 @@ +import asyncio +from dataclasses import replace +import os +from time import perf_counter +import unittest + +from bff.database import DatabaseManager, DatabaseSettings + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "requires an explicitly enabled PostgreSQL integration environment", +) +class TestCiPerformanceSmoke(unittest.IsolatedAsyncioTestCase): + async def test_health_pool_is_bounded_concurrent_and_isolated(self): + manager = DatabaseManager( + replace( + DatabaseSettings.from_env(), + pool_size=1, + max_overflow=0, + pool_timeout_seconds=1, + ) + ) + held = await manager.engine().connect() + try: + started = perf_counter() + results = await asyncio.gather(*(manager.check() for _ in range(5))) + elapsed_ms = (perf_counter() - started) * 1000 + self.assertEqual(results, [True] * 5) + self.assertLess(elapsed_ms, 3000) + self.assertEqual(manager.pool_status()["checked_out"], 1.0) + finally: + await held.close() + await manager.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_ci_workflows.py b/src/tests/test_ci_workflows.py new file mode 100644 index 0000000..504fddb --- /dev/null +++ b/src/tests/test_ci_workflows.py @@ -0,0 +1,20 @@ +import unittest + +from scripts.validate_ci_workflows import validate + + +class TestCiWorkflowContract(unittest.TestCase): + def test_pr_and_deployment_workflows_are_complete_and_guarded(self): + result = validate() + self.assertEqual(result["status"], "passed") + self.assertGreaterEqual(result["ci_gate_markers"], 20) + self.assertEqual( + result["staging_requires_protected_environments"], + ["staging-publish", "staging"], + ) + self.assertFalse(result["production_enabled"]) + self.assertEqual(result["workflow_execution"], "not_tested") + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_database.py b/src/tests/test_database.py new file mode 100644 index 0000000..f631936 --- /dev/null +++ b/src/tests/test_database.py @@ -0,0 +1,264 @@ +import os +import tempfile +import unittest +from dataclasses import replace + +from fastapi.testclient import TestClient +from sqlalchemy.engine import make_url + +from bff.database import DatabaseConfigurationError, DatabaseSettings +from bff.main import app + + +class _HealthyDatabase: + def __init__(self): + self.require_critical_configuration = None + + async def readiness( + self, *, expected_schema_version, require_critical_configuration=True + ): + self.require_critical_configuration = require_critical_configuration + return { + "ready": True, + "checks": { + "postgresql": "healthy", + "schema_version": "healthy", + "active_dataset": "healthy", + "critical_configuration": "healthy", + "queue": "healthy", + }, + "metadata": { + "queue_age_seconds": 0, + "dead_letter_count": 0, + }, + } + + @staticmethod + def pool_status(): + return {"checked_out": 0.0, "capacity": 1.0, "utilization_ratio": 0.0} + + +class _UnavailableDatabase: + async def readiness( + self, *, expected_schema_version, require_critical_configuration=True + ): + return { + "ready": False, + "checks": { + "postgresql": "unavailable", + "schema_version": "unknown", + "active_dataset": "unknown", + "critical_configuration": "unknown", + "queue": "unknown", + }, + } + + @staticmethod + def pool_status(): + return {"checked_out": 0.0, "capacity": 1.0, "utilization_ratio": 0.0} + + +class TestDatabaseFoundation(unittest.TestCase): + def test_secret_file_builds_encoded_async_postgresql_url(self): + descriptor, password_path = tempfile.mkstemp() + try: + os.write(descriptor, b"local:p@ssword\n") + os.close(descriptor) + settings = DatabaseSettings.from_env( + { + "DATABASE_HOST": "postgres", + "DATABASE_PORT": "5432", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD_FILE": password_path, + } + ) + url = settings.sqlalchemy_url() + self.assertEqual(url.drivername, "postgresql+asyncpg") + self.assertEqual(url.host, "postgres") + self.assertEqual(url.password, "local:p@ssword") + self.assertNotIn("local:p@ssword", url.render_as_string(hide_password=True)) + finally: + try: + os.close(descriptor) + except OSError: + pass + os.unlink(password_path) + + def test_non_postgresql_url_is_rejected(self): + settings = DatabaseSettings.from_env({"DATABASE_URL": "sqlite+aiosqlite:///tmp/test.db"}) + with self.assertRaises(DatabaseConfigurationError): + settings.sqlalchemy_url() + + def test_runtime_secret_environment_builds_postgresql_url(self): + settings = DatabaseSettings.from_env( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD": "runtime-only-secret", + } + ) + url = settings.sqlalchemy_url() + self.assertTrue(settings.configured) + self.assertEqual(url.password, "runtime-only-secret") + self.assertNotIn("runtime-only-secret", url.render_as_string(hide_password=True)) + + def test_tls_required_is_embedded_for_every_sqlalchemy_connection(self): + settings = DatabaseSettings.from_env( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_app", + "DATABASE_PASSWORD": "runtime-only-secret", + "DATABASE_SSL_MODE": "require", + } + ) + url = settings.sqlalchemy_url() + self.assertEqual(url.query["ssl"], "require") + self.assertNotIn("sslmode", url.query) + + def test_raw_database_url_round_trips_combined_reserved_password(self): + password = "A@b:%/+#?&xyz" + settings = DatabaseSettings.from_env( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_admin", + "DATABASE_PASSWORD": password, + "DATABASE_SSL_MODE": "require", + } + ) + raw_url = settings.raw_sqlalchemy_url() + self.assertNotIn("%%", raw_url) + self.assertEqual(make_url(raw_url).password, password) + + def test_raw_database_url_round_trips_each_reserved_password_character(self): + for character in ("%", "@", ":", "/", "+", "#", "?", "&"): + with self.subTest(character=character): + password = f"prefix{character}suffix" + settings = DatabaseSettings.from_env( + { + "DATABASE_HOST": "private-postgresql.internal", + "DATABASE_NAME": "foundation_intelligence", + "DATABASE_USER": "foundation_admin", + "DATABASE_PASSWORD": password, + "DATABASE_SSL_MODE": "require", + } + ) + raw_url = settings.raw_sqlalchemy_url() + self.assertNotIn("%%", raw_url) + self.assertEqual(make_url(raw_url).password, password) + + def test_database_url_sslmode_is_normalized_without_losing_query_parameters(self): + settings = DatabaseSettings.from_env( + { + "DATABASE_URL": ( + "postgresql+asyncpg://foundation_app:secret@postgres/database" + "?prepared_statement_cache_size=0&sslmode=require&sslmode=require" + ) + } + ) + url = settings.sqlalchemy_url() + self.assertEqual(url.query["ssl"], "require") + self.assertEqual(url.query["prepared_statement_cache_size"], "0") + self.assertNotIn("sslmode", url.query) + raw_url = settings.raw_sqlalchemy_url() + self.assertEqual(raw_url.count("ssl=require"), 1) + self.assertNotIn("sslmode=", raw_url) + + def test_database_url_asyncpg_ssl_remains_single_and_required(self): + settings = DatabaseSettings.from_env( + { + "DATABASE_URL": ( + "postgresql+asyncpg://foundation_app:secret@postgres/database" + "?prepared_statement_cache_size=0&ssl=require" + ) + } + ) + raw_url = settings.raw_sqlalchemy_url() + parsed = make_url(raw_url) + self.assertEqual(parsed.query["ssl"], "require") + self.assertEqual(parsed.query["prepared_statement_cache_size"], "0") + self.assertEqual(raw_url.count("ssl=require"), 1) + + def test_conflicting_database_ssl_configuration_is_rejected(self): + with self.assertRaises(DatabaseConfigurationError): + DatabaseSettings.from_env( + { + "DATABASE_URL": ( + "postgresql+asyncpg://foundation_app:secret@postgres/database" + "?sslmode=disable" + ), + "DATABASE_SSL_MODE": "require", + } + ) + + def test_conflicting_database_url_ssl_keys_are_rejected(self): + with self.assertRaises(DatabaseConfigurationError): + DatabaseSettings.from_env( + { + "DATABASE_URL": ( + "postgresql+asyncpg://foundation_app:secret@postgres/database" + "?ssl=require&sslmode=disable" + ) + } + ) + + def test_local_postgresql_keeps_tls_disabled_by_default(self): + settings = DatabaseSettings.from_env( + { + "DATABASE_URL": ( + "postgresql+asyncpg://foundation_app:secret@postgres/database" + "?prepared_statement_cache_size=0" + ) + } + ) + url = settings.sqlalchemy_url() + self.assertEqual(settings.ssl_mode, "disable") + self.assertNotIn("ssl", url.query) + self.assertEqual(url.query["prepared_statement_cache_size"], "0") + + def test_readiness_reflects_postgresql_state(self): + original = app.state.database + client = TestClient(app) + try: + healthy_database = _HealthyDatabase() + app.state.database = healthy_database + ready = client.get("/health/ready") + self.assertEqual(ready.status_code, 200) + self.assertEqual(ready.json()["checks"]["postgresql"], "healthy") + self.assertEqual(ready.json()["checks"]["schema_version"], "healthy") + self.assertTrue(healthy_database.require_critical_configuration) + + app.state.database = _UnavailableDatabase() + unavailable = client.get("/health/ready") + self.assertEqual(unavailable.status_code, 503) + self.assertEqual(unavailable.json()["checks"]["postgresql"], "unavailable") + finally: + app.state.database = original + + def test_public_readonly_readiness_does_not_require_pipeline_configuration(self): + original_database = app.state.database + original_security = app.state.security_settings + database = _HealthyDatabase() + try: + app.state.database = database + app.state.security_settings = replace( + original_security, + app_env="demo", + data_runtime_mode="postgresql", + auth_mode="public_readonly", + dev_auth_enabled=False, + core_proxy_enabled=False, + ) + response = TestClient(app).get("/health/ready") + self.assertEqual(response.status_code, 200) + self.assertFalse(database.require_critical_configuration) + finally: + app.state.database = original_database + app.state.security_settings = original_security + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_durable_pipeline.py b/src/tests/test_durable_pipeline.py new file mode 100644 index 0000000..3c584e6 --- /dev/null +++ b/src/tests/test_durable_pipeline.py @@ -0,0 +1,411 @@ +"""Phase-8 durable pipeline, queue and immutable-storage gates.""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +import os +from pathlib import Path +import unittest +import uuid + +from sqlalchemy import text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from bff.database import DatabaseSettings +from bff.postgres.idempotency_repository import PostgresIdempotencyStore +from bff.postgres.job_repository import PostgresJobRepository +from bff.postgres.pipeline_repository import PipelineRepository +from bff.security import IdempotencyConflict +from pipelines.durable import ( + InMemoryObjectStore, + IngestionManifest, + QueueEnvelope, + StorageObject, + load_source_configurations, + object_key, + require_sources, + sha256_bytes, +) +from pipelines.durable_worker import DurableWorker, OutboxDispatcher + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MIGRATION_PATH = REPOSITORY_ROOT / "alembic" / "versions" / "0005_durable_pipeline.py" +REQUIRED_SOURCES = { + "360giving", + "charity_commission", + "philea", + "hinchilla", + "ecb", + "google_news_rss", + "article_content", + "anthropic_news_summary", +} + + +class TestDurablePipelineContracts(unittest.IsolatedAsyncioTestCase): + def test_source_register_is_complete_and_fail_closed(self): + configurations = load_source_configurations() + require_sources(configurations, REQUIRED_SOURCES) + self.assertEqual({source.source_name for source in configurations}, REQUIRED_SOURCES) + for source in configurations: + self.assertFalse(source.enabled) + self.assertTrue(source.governance_blocked) + self.assertEqual(source.legal_status, "unresolved") + self.assertEqual(source.licence_status, "unresolved") + self.assertEqual(len(source.configuration_checksum), 64) + with self.assertRaisesRegex(ValueError, "Governance-blocked"): + replace(configurations[0], enabled=True).validate() + + def test_migration_contains_queue_storage_worker_and_immutability_contracts(self): + source = MIGRATION_PATH.read_text(encoding="utf-8") + for table_name in ( + "source_configurations", + "storage_objects", + "ingestion_run_manifests", + "job_dispatch_outbox", + "worker_heartbeats", + ): + self.assertIn(f"CREATE TABLE {table_name}", source) + self.assertIn("FOR EACH ROW EXECUTE FUNCTION protect_immutable_storage_object", source) + self.assertIn("NOT enabled OR", source) + self.assertIn("last_good_dataset_version", source) + + async def test_immutable_object_and_manifest_contracts(self): + payload = b'{"source":"fixture","records":[1,2]}' + checksum = sha256_bytes(payload) + run_id = uuid.UUID("11111111-1111-4111-8111-111111111111") + descriptor = StorageObject( + storage_object_id=uuid.UUID("22222222-2222-4222-8222-222222222222"), + zone="raw", + bucket_alias="raw", + object_key=object_key( + source_name="360giving", + run_id=run_id, + zone="raw", + checksum=checksum, + extension="json", + ), + object_version="fixture-v1", + checksum=checksum, + content_length=len(payload), + content_type="application/json", + source_name="360giving", + source_ingestion_run_id=run_id, + ) + store = InMemoryObjectStore() + await store.put_if_absent(descriptor, payload) + await store.put_if_absent(descriptor, payload) + self.assertEqual(await store.get(descriptor), payload) + with self.assertRaisesRegex(ValueError, "checksum"): + await store.put_if_absent(descriptor, b"x" * len(payload)) + + manifest = IngestionManifest.create( + source_ingestion_run_id=run_id, + schema_version="fixture-v1", + source_name="360giving", + source_version="2026-07-29", + dataset_version="fixture-dataset", + raw_object_id=descriptor.storage_object_id, + validated_object_id=None, + curated_object_id=None, + watermark_before="100", + watermark_after="200", + record_count=10, + accepted_count=7, + rejected_count=2, + quarantined_count=1, + retry_count=0, + generated_at="2026-07-29T00:00:00+00:00", + ) + self.assertEqual(len(manifest.checksum), 64) + self.assertEqual(manifest.checksum, manifest.checksum) + with self.assertRaisesRegex(ValueError, "exceed"): + replace(manifest, accepted_count=11).validate() + + def test_queue_envelope_is_bounded_and_contains_no_job_payload(self): + envelope = QueueEnvelope( + schema_version="job-envelope-v1", + job_id=uuid.UUID("33333333-3333-4333-8333-333333333333"), + job_type="quick_consolidate", + queue_name="pipeline", + requested_at="2026-07-29T00:00:00+00:00", + attempt=1, + max_attempts=3, + ) + self.assertEqual(envelope.payload["job_id"], str(envelope.job_id)) + self.assertNotIn("input", envelope.payload) + with self.assertRaisesRegex(ValueError, "attempt"): + replace(envelope, attempt=4).validate() + + async def test_worker_executes_handlers_outside_the_api_lifecycle(self): + class Repository: + def __init__(self): + self.failed = None + self.succeeded = None + + async def claim(self, **_kwargs): + return { + "job_id": "44444444-4444-4444-8444-444444444444", + "job_type": "fixture", + "input": {"bounded": True}, + "timeout_seconds": 5, + } + + async def succeed(self, job_id, **kwargs): + self.succeeded = (job_id, kwargs) + return True + + async def fail(self, job_id, **kwargs): + self.failed = (job_id, kwargs) + return "queued" + + repository = Repository() + + async def handler(job): + return {"accepted": 1, "job_type": job["job_type"]} + + result = await DurableWorker( + repository, + {"fixture": handler}, + worker_id="phase8-worker", + ).run_once() + self.assertEqual(result.status, "succeeded") + self.assertIsNotNone(repository.succeeded) + self.assertIsNone(repository.failed) + + async def test_outbox_dispatcher_uses_queue_contract_and_deduplication_id(self): + class Repository: + def __init__(self): + self.published = None + + async def due_dispatches(self, *, limit): + self.asserted_limit = limit + return [ + { + "outbox_id": "55555555-5555-4555-8555-555555555555", + "job_id": "66666666-6666-4666-8666-666666666666", + "queue_name": "pipeline", + "message_body": {"schema_version": "job-envelope-v1"}, + } + ] + + async def mark_dispatch_published(self, outbox_id, *, queue_message_id): + self.published = (outbox_id, queue_message_id) + + async def mark_dispatch_failed(self, *_args, **_kwargs): + raise AssertionError("success path must not mark failure") + + class Publisher: + async def publish(self, *, queue_name, message, deduplication_id): + self.call = (queue_name, message, deduplication_id) + return "local-message-id" + + repository = Repository() + publisher = Publisher() + result = await OutboxDispatcher(repository, publisher).run_once(limit=10) + self.assertEqual(result, {"published": 1, "failed": 0, "dead_lettered": 0}) + self.assertEqual(publisher.call[2], "66666666-6666-4666-8666-666666666666") + self.assertEqual(repository.published[1], "local-message-id") + + def test_production_pipeline_code_has_no_local_lock_or_subprocess_coordination(self): + paths = [ + REPOSITORY_ROOT / "src" / "bff" / "postgres" / "admin_routes.py", + REPOSITORY_ROOT / "src" / "bff" / "postgres" / "job_repository.py", + REPOSITORY_ROOT / "src" / "pipelines" / "durable_worker.py", + ] + combined = "\n".join(path.read_text(encoding="utf-8") for path in paths) + for forbidden in ( + "LOCK_FILE", + "import fcntl", + "import subprocess", + "subprocess.", + "Popen(", + "pipeline_run.lock", + ): + self.assertNotIn(forbidden, combined) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestDurablePipelinePostgresIntegration(unittest.TestCase): + def test_jobs_idempotency_ingestion_and_last_good_preservation(self): + asyncio.run(self._exercise()) + + @staticmethod + def _database_url(): + return os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + + async def _exercise(self): + engine = create_async_engine(self._database_url(), pool_pre_ping=True) + connection = await engine.connect() + outer_transaction = await connection.begin() + sessions = async_sessionmaker( + bind=connection, + expire_on_commit=False, + autoflush=False, + join_transaction_mode="create_savepoint", + ) + unique = uuid.uuid4().hex + queue_name = f"phase8-{unique}" + try: + dataset_version = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertIsNotNone(dataset_version) + jobs = PostgresJobRepository(sessions) + first = await jobs.enqueue( + "quick_consolidate", + {"fixture": True}, + actor_id="phase8-integration", + idempotency_key=f"phase8-job-{unique}", + queue_name=queue_name, + max_attempts=2, + timeout_seconds=60, + ) + duplicate = await jobs.enqueue( + "quick_consolidate", + {"fixture": True}, + actor_id="phase8-integration", + idempotency_key=f"phase8-job-{unique}", + ) + self.assertEqual(first["job_id"], duplicate["job_id"]) + self.assertTrue(duplicate["idempotent_noop"]) + dispatches = await jobs.due_dispatches() + self.assertIn(first["job_id"], {row["job_id"] for row in dispatches}) + + claimed = await jobs.claim( + worker_id=f"worker-{unique}", queue_name=queue_name + ) + self.assertEqual(claimed["job_id"], first["job_id"]) + self.assertTrue( + await jobs.heartbeat(first["job_id"], worker_id=f"worker-{unique}") + ) + self.assertEqual( + await jobs.fail( + first["job_id"], + worker_id=f"worker-{unique}", + error_class="FixtureFailure", + failure_reason="last-good preservation fixture", + retryable=True, + ), + "queued", + ) + retried = await jobs.claim( + worker_id=f"worker-{unique}", queue_name=queue_name + ) + self.assertEqual(retried["attempt"], 2) + self.assertTrue( + await jobs.succeed( + first["job_id"], + worker_id=f"worker-{unique}", + result={"accepted_count": 1}, + ) + ) + + durable_keys = PostgresIdempotencyStore(sessions, ttl_seconds=60) + key = await durable_keys.reserve( + "phase8-actor", "phase8.action", unique, "a" * 64 + ) + with self.assertRaises(IdempotencyConflict): + await durable_keys.reserve( + "phase8-actor", "phase8.action", unique, "a" * 64 + ) + await durable_keys.release(key) + await durable_keys.reserve( + "phase8-actor", "phase8.action", unique, "b" * 64 + ) + + pipelines = PipelineRepository(sessions) + configurations = load_source_configurations() + self.assertEqual( + await pipelines.synchronize_sources(configurations), len(configurations) + ) + with self.assertRaisesRegex(ValueError, "unresolved governance"): + await pipelines.set_source_enabled("360giving", enabled=True) + + ingestion_id = await pipelines.start_ingestion( + source_name="360giving", + dataset_version=str(dataset_version), + job_id=first["job_id"], + source_version=f"fixture-{unique}", + source_uri="s3://raw/fixture", + watermark_before="100", + ) + raw_payload = b'{"fixture":true}' + raw_checksum = sha256_bytes(raw_payload) + raw_id = uuid.uuid4() + raw = StorageObject( + storage_object_id=raw_id, + zone="raw", + bucket_alias="raw", + object_key=object_key( + source_name="360giving", + run_id=uuid.UUID(ingestion_id), + zone="raw", + checksum=raw_checksum, + extension="json", + ), + object_version=f"fixture-{unique}", + checksum=raw_checksum, + content_length=len(raw_payload), + content_type="application/json", + source_name="360giving", + source_ingestion_run_id=uuid.UUID(ingestion_id), + ) + await pipelines.record_object(raw) + manifest = IngestionManifest.create( + source_ingestion_run_id=uuid.UUID(ingestion_id), + schema_version="fixture-v1", + source_name="360giving", + source_version=f"fixture-{unique}", + dataset_version=str(dataset_version), + raw_object_id=raw_id, + validated_object_id=None, + curated_object_id=None, + watermark_before="100", + watermark_after="101", + record_count=1, + accepted_count=1, + rejected_count=0, + quarantined_count=0, + retry_count=1, + generated_at="2026-07-29T00:00:00+00:00", + ) + await pipelines.complete_ingestion(manifest) + + with self.assertRaises(DBAPIError): + async with sessions() as session, session.begin(): + await session.execute( + text( + "UPDATE storage_objects SET object_key='changed' " + "WHERE storage_object_id=:object_id" + ), + {"object_id": raw_id}, + ) + + active_after = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual(active_after, dataset_version) + last_good = await connection.scalar( + text( + "SELECT last_good_dataset_version FROM job_runs " + "WHERE job_run_id=:job_id" + ), + {"job_id": uuid.UUID(first["job_id"])}, + ) + self.assertEqual(last_good, dataset_version) + finally: + if outer_transaction.is_active: + await outer_transaction.rollback() + await connection.close() + await engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_governance_retention.py b/src/tests/test_governance_retention.py new file mode 100644 index 0000000..d40e617 --- /dev/null +++ b/src/tests/test_governance_retention.py @@ -0,0 +1,351 @@ +"""Phase-9 governance, exposure, retention and hold gates.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timedelta, timezone +import os +from pathlib import Path +import unittest +import uuid + +from sqlalchemy import text +from sqlalchemy.exc import DBAPIError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from bff.database import DatabaseSettings +from bff.postgres.governance_repository import GovernanceRepository +from bff.utils.logging import redact_text +from governance.exposure import redact_for_logs, serialize_exposed_fields +from governance.retention import ( + DataHold, + RetentionCandidate, + RetentionPlanner, + export_lifecycle_status, + load_governance_configuration, +) + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MIGRATION_PATH = ( + REPOSITORY_ROOT / "alembic" / "versions" / "0006_governance_retention.py" +) +NOW = datetime(2026, 7, 29, 12, 0, tzinfo=timezone.utc) + + +class TestGovernanceRetentionContracts(unittest.TestCase): + def setUp(self): + self.configuration = load_governance_configuration() + self.planner = RetentionPlanner(self.configuration) + + def test_configuration_is_complete_honest_and_non_destructive(self): + self.configuration.validate() + self.assertFalse(self.configuration.destructive_deletion_enabled) + self.assertFalse(self.configuration.production_activation_approved) + self.assertTrue(self.configuration.restore_before_delete_required) + self.assertTrue( + all(policy.delete_after_days is None for policy in self.configuration.policies) + ) + self.assertEqual(self.configuration.data_owners["data_owner"], "unassigned") + self.assertEqual(self.configuration.service_recovery["rto_hours"], None) + self.assertEqual(self.configuration.service_recovery["rpo_hours"], None) + + def test_field_exposure_is_allowlist_only(self): + source = { + "id": 7, + "name": "Example Foundation", + "reg_status": "registered", + "email": "person@example.org", + "postal_address": "1 Private Street", + "secret": "must-not-escape", + "database_column_added_later": "must-not-escape", + } + exposed = serialize_exposed_fields( + source, + policy_name="organization_summary", + configuration=self.configuration, + ) + self.assertEqual( + exposed, + {"id": 7, "name": "Example Foundation", "reg_status": "registered"}, + ) + with self.assertRaisesRegex(ValueError, "Unknown field exposure"): + serialize_exposed_fields( + source, + policy_name="automatic_database_columns", + configuration=self.configuration, + ) + + def test_log_redaction_covers_credentials_and_personal_fields(self): + payload = { + "authorization": "Bearer sensitive-token", + "email": "person@example.org", + "postal_address": "1 Private Street", + "nested": { + "database_url": "postgresql://user:password@example.invalid/db", + "message": "Contact person@example.org", + }, + "safe": "retained", + } + redacted = redact_for_logs(payload, self.configuration) + self.assertEqual(redacted["authorization"], "[REDACTED]") + self.assertEqual(redacted["email"], "[REDACTED]") + self.assertEqual(redacted["postal_address"], "[REDACTED]") + self.assertEqual(redacted["nested"]["database_url"], "[REDACTED]") + self.assertNotIn("person@example.org", redacted["nested"]["message"]) + self.assertEqual(redacted["safe"], "retained") + self.assertNotIn("person@example.org", redact_text("email=person@example.org")) + + def test_archive_plan_is_dry_run_and_has_immutable_checksum(self): + candidate = RetentionCandidate( + target_type="storage_object", + target_id="raw/example", + retention_class="raw_source_evidence", + last_modified_at=NOW - timedelta(days=31), + object_count=1, + record_count=10, + total_bytes=500, + target_checksums=("a" * 64,), + ) + entry = self.planner.plan([candidate], [], now=NOW)[0] + self.assertTrue(entry.dry_run) + self.assertEqual(entry.action_type, "archive") + self.assertEqual(entry.status, "reported") + self.assertEqual(len(entry.manifest_checksum), 64) + self.assertNotEqual(entry.action_type, "delete") + + def test_legal_and_incident_holds_override_retention(self): + candidate = RetentionCandidate( + target_type="dataset", + target_id="v1", + retention_class="raw_source_evidence", + last_modified_at=NOW - timedelta(days=365), + ) + for hold_type in ("legal", "incident"): + hold = DataHold( + hold_id=str(uuid.uuid4()), + hold_type=hold_type, + scope_type="dataset", + scope_id="v1", + status="active", + ) + entry = self.planner.plan([candidate], [hold], now=NOW)[0] + self.assertEqual(entry.status, "held") + self.assertEqual(entry.action_type, "report") + self.assertEqual(entry.hold_ids, (hold.hold_id,)) + + def test_deletion_authorisation_is_fail_closed(self): + for role, restored, holds in ( + ("viewer", False, []), + ("administrator", False, []), + ("administrator", True, ["legal-hold"]), + ("administrator", True, []), + ): + with self.assertRaisesRegex(PermissionError, "globally disabled"): + self.planner.assert_deletion_authorized( + retention_class="exports", + actor_role=role, + restore_verified=restored, + active_hold_ids=holds, + ) + + def test_export_expiration_reports_without_deleting(self): + self.assertEqual( + export_lifecycle_status( + expires_at=NOW - timedelta(seconds=1), + hold_until=None, + now=NOW, + ), + "expiration_report_due", + ) + self.assertEqual( + export_lifecycle_status( + expires_at=NOW - timedelta(days=1), + hold_until=NOW + timedelta(days=1), + now=NOW, + ), + "held", + ) + + def test_migration_and_routes_have_no_destructive_endpoint(self): + migration = MIGRATION_PATH.read_text(encoding="utf-8") + for table in ( + "retention_policies", + "data_holds", + "restore_verifications", + "deletion_manifests", + "data_subject_requests", + ): + self.assertIn(f"CREATE TABLE {table}", migration) + self.assertIn("dry_run AND action_type <> 'delete'", migration) + route_source = ( + REPOSITORY_ROOT / "src" / "bff" / "postgres" / "governance_routes.py" + ).read_text(encoding="utf-8") + self.assertIn("Role.ADMINISTRATOR", route_source) + self.assertNotIn('@router.delete(', route_source) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestGovernanceRetentionPostgresIntegration(unittest.TestCase): + def test_holds_manifests_restore_exports_and_subject_workflow(self): + asyncio.run(self._exercise()) + + @staticmethod + def _database_url(): + return os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + + async def _exercise(self): + engine = create_async_engine(self._database_url(), pool_pre_ping=True) + connection = await engine.connect() + outer_transaction = await connection.begin() + sessions = async_sessionmaker( + bind=connection, + expire_on_commit=False, + autoflush=False, + join_transaction_mode="create_savepoint", + ) + unique = uuid.uuid4().hex + configuration = load_governance_configuration() + planner = RetentionPlanner(configuration) + repository = GovernanceRepository(sessions) + try: + dataset_version = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual( + await repository.synchronize_policies(configuration), + len(configuration.policies), + ) + self.assertTrue( + all( + not policy["destructive_deletion_enabled"] + for policy in await repository.policies() + ) + ) + + hold = await repository.create_hold( + hold_type="incident", + scope_type="dataset", + scope_id=str(dataset_version), + reason="phase9 integration fixture", + created_by="phase9-integration", + ) + candidate = RetentionCandidate( + target_type="dataset", + target_id=str(dataset_version), + retention_class="raw_source_evidence", + last_modified_at=NOW - timedelta(days=365), + record_count=1, + target_checksums=("b" * 64,), + ) + held_entry = planner.plan( + [candidate], await repository.active_holds(), now=NOW + )[0] + self.assertEqual(held_entry.status, "held") + held_action = ( + await repository.record_retention_plan( + [held_entry], requested_by="phase9-integration" + ) + )[0] + self.assertIsNotNone(held_action) + + await repository.release_hold( + hold["data_hold_id"], + released_by="phase9-integration", + release_reason="fixture complete", + ) + archive_entry = planner.plan( + [candidate], await repository.active_holds(), now=NOW + )[0] + self.assertEqual(archive_entry.action_type, "archive") + archive_action = ( + await repository.record_retention_plan( + [archive_entry], requested_by="phase9-integration" + ) + )[0] + + manifest_id = await connection.scalar( + text( + "SELECT deletion_manifest_id FROM deletion_manifests " + "WHERE retention_action_id=:action_id" + ), + {"action_id": uuid.UUID(archive_action)}, + ) + with self.assertRaises(DBAPIError): + async with sessions() as session, session.begin(): + await session.execute( + text( + "UPDATE deletion_manifests SET target_id='changed' " + "WHERE deletion_manifest_id=:manifest_id" + ), + {"manifest_id": manifest_id}, + ) + + restore_id = await repository.record_restore_verification( + target_type="dataset", + target_id=str(dataset_version), + backup_reference="local://phase9-fixture", + backup_checksum="c" * 64, + verification_status="passed", + verified_by="phase9-integration", + evidence={"local_fixture": True}, + ) + with self.assertRaises(DBAPIError): + async with sessions() as session, session.begin(): + await session.execute( + text( + "UPDATE restore_verifications SET target_id='changed' " + "WHERE restore_verification_id=:restore_id" + ), + {"restore_id": uuid.UUID(restore_id)}, + ) + + await connection.execute( + text( + """ + INSERT INTO export_jobs ( + export_job_id, dataset_version, export_type, status, + requested_by, completed_at, object_key, checksum, + row_count, expires_at, retention_class + ) VALUES ( + :export_id, :dataset_version, 'phase9-fixture', 'succeeded', + 'phase9-integration', CURRENT_TIMESTAMP, + 'exports/phase9-fixture', :checksum, 1, + CURRENT_TIMESTAMP - INTERVAL '1 day', 'exports' + ) + """ + ), + { + "export_id": uuid.uuid4(), + "dataset_version": dataset_version, + "checksum": "d" * 64, + }, + ) + self.assertTrue( + any( + export["export_type"] == "phase9-fixture" + for export in await repository.expired_exports() + ) + ) + subject_request = await repository.create_data_subject_request( + request_type="access", + subject_reference_hash="e" * 64, + due_at=NOW + timedelta(days=30), + ) + self.assertIsNotNone(subject_request) + active_after = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual(active_after, dataset_version) + finally: + if outer_transaction.is_active: + await outer_transaction.rollback() + await connection.close() + await engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_http_load_smoke.py b/src/tests/test_http_load_smoke.py new file mode 100644 index 0000000..d982e4d --- /dev/null +++ b/src/tests/test_http_load_smoke.py @@ -0,0 +1,18 @@ +import unittest + +from scripts.http_load_smoke import percentile + + +class TestHttpLoadSmoke(unittest.TestCase): + def test_percentile_uses_nearest_rank(self): + values = [100, 20, 50, 10, 80] + self.assertEqual(percentile(values, 0.5), 50) + self.assertEqual(percentile(values, 0.95), 100) + + def test_percentile_rejects_empty_input(self): + with self.assertRaises(ValueError): + percentile([], 0.95) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_observability.py b/src/tests/test_observability.py new file mode 100644 index 0000000..f5319d3 --- /dev/null +++ b/src/tests/test_observability.py @@ -0,0 +1,183 @@ +import asyncio +from dataclasses import replace +import io +import json +import logging +import os +from pathlib import Path +import unittest + +from fastapi.testclient import TestClient +from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError + +from bff.database import DatabaseManager, DatabaseSettings +from bff.main import app +from bff.utils.logging import JsonFormatter, pseudonymous_actor_id +from observability.metrics import ( + REQUIRED_ALARMS, + REQUIRED_METRICS, + MetricsRegistry, + load_observability_configuration, +) + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] + + +class _ExplodingDatabase: + @staticmethod + def pool_status(): + return {"checked_out": 0.0, "capacity": 0.0, "utilization_ratio": 0.0} + + async def readiness( + self, *, expected_schema_version, require_critical_configuration=True + ): + raise AssertionError("liveness must not query PostgreSQL") + + +class TestObservabilityContracts(unittest.TestCase): + def test_configuration_covers_required_metrics_alarms_and_runbooks(self): + configuration = load_observability_configuration() + self.assertTrue(REQUIRED_METRICS.issubset({item.name for item in configuration.metrics})) + self.assertTrue(REQUIRED_ALARMS.issubset({item.name for item in configuration.alarms})) + + runbooks = (REPOSITORY_ROOT / "docs/remediation/observability-runbooks.md").read_text( + encoding="utf-8" + ) + for alarm in configuration.alarms: + self.assertIn(f"## {alarm.runbook}", runbooks) + + def test_local_registry_records_bounded_typed_values(self): + registry = MetricsRegistry(load_observability_configuration()) + dimensions = { + "service": "foundation-intelligence-api", + "environment": "test", + } + registry.increment( + "api_errors_total", + operation="GET /health/ready", + error_class="http_503", + **dimensions, + ) + registry.set_gauge("readiness_success", 1, **dimensions) + registry.observe( + "api_request_duration_ms", + 12.5, + operation="GET /health/ready", + status="200", + **dimensions, + ) + registry.observe( + "api_request_duration_ms", + 7.5, + operation="GET /health/ready", + status="200", + **dimensions, + ) + + snapshot = {item["name"]: item for item in registry.snapshot()} + self.assertEqual(snapshot["api_errors_total"]["value"], 1.0) + self.assertEqual(snapshot["readiness_success"]["value"], 1.0) + self.assertEqual(snapshot["api_request_duration_ms"]["count"], 2.0) + self.assertEqual(snapshot["api_request_duration_ms"]["sum"], 20.0) + + with self.assertRaises(ValueError): + registry.increment("unknown_metric") + with self.assertRaises(ValueError): + registry.set_gauge("readiness_success", 1, unbounded="value") + + def test_json_formatter_redacts_sensitive_fields(self): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + handler.setFormatter(JsonFormatter()) + logger = logging.getLogger("observability-contract-test") + logger.handlers = [handler] + logger.propagate = False + logger.setLevel(logging.INFO) + + logger.info( + "contact analyst@example.org token=secret-value", + extra={ + "request_id": "request-1", + "trace_id": "trace-1", + "actor_id": pseudonymous_actor_id("analyst@example.org"), + "operation": "GET /api/charities/{reg_charity_number}", + "status": 200, + }, + ) + payload = json.loads(stream.getvalue()) + serialized = json.dumps(payload) + self.assertEqual(payload["service"], "foundation-intelligence-api") + self.assertEqual(payload["request_id"], "request-1") + self.assertEqual(payload["trace_id"], "trace-1") + self.assertTrue(payload["actor_id"].startswith("sha256:")) + self.assertNotIn("analyst@example.org", serialized) + self.assertNotIn("secret-value", serialized) + self.assertIn("[REDACTED]", serialized) + + def test_liveness_is_independent_of_database_readiness(self): + original = app.state.database + app.state.database = _ExplodingDatabase() + try: + client = TestClient(app) + legacy = client.get("/health") + live = client.get("/health/live", headers={"X-Trace-ID": "trace-local"}) + self.assertEqual(legacy.status_code, 200) + self.assertEqual(live.status_code, 200) + self.assertEqual(live.headers["X-Trace-ID"], "trace-local") + finally: + app.state.database = original + + def test_observability_route_is_admin_only_and_marks_cloudwatch_unexecuted(self): + matching = [ + route + for route in app.routes + if getattr(route, "path", "") == "/api/admin/observability/metrics" + ] + if os.environ.get("APP_ENV", "development") not in {"staging", "production"}: + self.assertEqual(matching, []) + return + self.assertEqual(len(matching), 1) + source = (REPOSITORY_ROOT / "src/bff/postgres/observability_routes.py").read_text( + encoding="utf-8" + ) + self.assertIn("Role.ADMINISTRATOR", source) + self.assertIn('"cloudwatch_execution": "not_tested"', source) + + +@unittest.skipUnless( + os.environ.get("RUN_POSTGRES_INTEGRATION") == "1", + "requires the explicitly enabled local PostgreSQL integration environment", +) +class TestPostgresObservabilityIntegration(unittest.IsolatedAsyncioTestCase): + async def test_readiness_bypasses_an_exhausted_application_pool(self): + settings = replace( + DatabaseSettings.from_env(), + pool_size=1, + max_overflow=0, + pool_timeout_seconds=1, + ) + manager = DatabaseManager(settings) + held_connection = await manager.engine().connect() + try: + with self.assertRaises(SQLAlchemyTimeoutError): + async with asyncio.timeout(1.5): + async with manager.engine().connect(): + pass + + result = await manager.readiness( + expected_schema_version="0006_governance_retention" + ) + self.assertTrue(result["ready"]) + self.assertEqual(result["checks"]["postgresql"], "healthy") + self.assertEqual(result["checks"]["schema_version"], "healthy") + self.assertGreaterEqual(result["metadata"]["source_count"], 8) + self.assertGreaterEqual(result["metadata"]["policy_count"], 14) + self.assertIsNot(manager.engine(), manager.health_engine()) + finally: + await held_connection.close() + await manager.close() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_postgres_application.py b/src/tests/test_postgres_application.py new file mode 100644 index 0000000..3ac191a --- /dev/null +++ b/src/tests/test_postgres_application.py @@ -0,0 +1,411 @@ +"""Application-journey parity tests for the PostgreSQL runtime.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime, timezone +import os +from pathlib import Path +import subprocess +import sys +import unittest +import uuid + +from fastapi.routing import APIRoute +from sqlalchemy import text +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from bff.audit import AuditEvent +from bff.database import DatabaseSettings +from bff.postgres.analytics_repository import AnalyticsRepository +from bff.postgres.audit_repository import PostgresAuditSink +from bff.postgres.funder_repository import SourceFunderRepository +from bff.postgres.job_repository import PostgresJobRepository +from bff.postgres.organization_repository import OrganizationRepository +from bff.postgres.registry_repository import RegistryRepository +from bff.schemas import ( + CharityBase, + CharityDetail, + CharityStats, + GrantListResponse, + GrantMapResponse, + GrantNetworkSummary, + GrantThemesResponse, + GrantTrendsResponse, + RegistryDirectoryPage, + RegistryOrganizationDetail, + SankeyData, + ScoreResponse, + SourceFunderDetailResponse, + SourceFunderListResponse, +) + + +def _route_contract(router) -> set[tuple[str, tuple[str, ...]]]: + return { + (route.path, tuple(sorted(route.methods))) + for route in router.routes + if isinstance(route, APIRoute) + } + + +class TestPostgreSQLRouteParity(unittest.TestCase): + def test_postgresql_router_covers_every_legacy_application_route(self): + from bff.charity import router as legacy_router + from bff.postgres.routes import router as postgresql_router + + legacy = _route_contract(legacy_router) + postgresql = _route_contract(postgresql_router) + self.assertTrue(legacy.issubset(postgresql)) + self.assertEqual( + postgresql - legacy, + {("/api/charities/grants/map/connections", ("GET",))}, + ) + + def test_production_startup_fails_without_postgresql_configuration(self): + script = """ +from fastapi.testclient import TestClient +from bff.main import app +with TestClient(app): + pass +""" + environment = { + key: value + for key, value in os.environ.items() + if not key.startswith("DATABASE_") + } + environment.update( + { + "APP_ENV": "production", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "oidc", + "OIDC_ISSUER": "https://identity.example.invalid/", + "OIDC_AUDIENCE": "foundation-intelligence-api", + "OIDC_JWKS_JSON": '{"keys":[]}', + "CORS_ORIGINS": "https://app.example.invalid", + "DEV_AUTH_ENABLED": "false", + "SESSION_COOKIE_SECURE": "true", + "CORE_PROXY_ENABLED": "false", + "PYTHONPATH": str(Path(__file__).resolve().parents[2] / "src"), + } + ) + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + env=environment, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn("PostgreSQL connection settings are incomplete", result.stderr) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestPostgreSQLApplicationIntegration(unittest.TestCase): + def test_all_read_journeys_against_real_postgresql(self): + asyncio.run(self._exercise_reads()) + + def test_mutations_are_transactional_and_audited_in_real_postgresql(self): + asyncio.run(self._exercise_mutations()) + + def test_production_startup_and_route_selection_against_real_postgresql(self): + script = """ +import sys +from fastapi.routing import APIRoute +from fastapi.testclient import TestClient +from bff.main import app +with TestClient(app) as client: + assert client.get('/health/ready').status_code == 200 + def descendants(routes): + for route in routes: + nested = getattr(route, 'routes', None) + if nested is None and getattr(route, 'original_router', None) is not None: + nested = route.original_router.routes + if nested is not None: + yield from descendants(nested) + else: + yield route + routes = list(descendants(app.routes)) + endpoint_modules = { + route.endpoint.__module__ for route in routes if isinstance(route, APIRoute) + } + assert 'bff.postgres.routes' in endpoint_modules, endpoint_modules + assert 'bff.postgres.admin_routes' in endpoint_modules, endpoint_modules + assert 'bff.charity' not in endpoint_modules + assert 'bff.admin' not in endpoint_modules + assert 'bff.charity' not in sys.modules + assert 'bff.repositories' not in sys.modules + assert 'sqlite3' not in sys.modules +""" + environment = { + **os.environ, + "APP_ENV": "production", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "oidc", + "OIDC_ISSUER": "https://identity.example.invalid/", + "OIDC_AUDIENCE": "foundation-intelligence-api", + "OIDC_JWKS_JSON": '{"keys":[]}', + "CORS_ORIGINS": "https://app.example.invalid", + "DEV_AUTH_ENABLED": "false", + "SESSION_COOKIE_SECURE": "true", + "CORE_PROXY_ENABLED": "false", + "PYTHONPATH": str(Path(__file__).resolve().parents[2] / "src"), + } + result = subprocess.run( + [sys.executable, "-c", script], + cwd=Path(__file__).resolve().parents[2], + env=environment, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + @staticmethod + def _database_url(): + return os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + + async def _exercise_reads(self): + engine = create_async_engine(self._database_url(), pool_pre_ping=True) + sessions = async_sessionmaker(engine, expire_on_commit=False, autoflush=False) + organizations = OrganizationRepository(sessions) + registry = RegistryRepository(sessions) + analytics = AnalyticsRepository(sessions) + funders = SourceFunderRepository(sessions) + try: + async with sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + organization_id = await session.scalar( + text( + """ + SELECT charity_id FROM charities + WHERE dataset_version=:dataset_version + ORDER BY EXISTS ( + SELECT 1 FROM grants WHERE grants.dataset_version=charities.dataset_version + AND (grants.funding_charity_id=charities.charity_id + OR grants.recipient_charity_id=charities.charity_id) + ) DESC, charity_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + registry_id = await session.scalar( + text( + """ + SELECT registry_id FROM charity_registry_organizations + WHERE dataset_version=:dataset_version AND is_current_source_record + ORDER BY registry_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + funder_scope = ( + await session.execute( + text( + """ + SELECT country_code, source_funder_key + FROM grant_source_funder_facts + WHERE dataset_version=:dataset_version + AND country_code IS NOT NULL + GROUP BY country_code, source_funder_key + ORDER BY COUNT(*) DESC, country_code, source_funder_key LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + ).one() + period = await session.scalar( + text( + """ + SELECT to_char(award_date, 'YYYY-MM') FROM grant_overview_facts + WHERE dataset_version=:dataset_version AND award_date IS NOT NULL + ORDER BY award_date DESC LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + + listed = await organizations.list(limit=2) + self.assertTrue(listed) + for item in listed: + CharityBase.model_validate(item) + stats = await organizations.stats() + CharityStats.model_validate(stats) + async with sessions() as session: + grant_sources = set( + ( + await session.execute( + text( + """ + SELECT DISTINCT source FROM grants + WHERE dataset_version=:dataset_version + AND source IS NOT NULL AND source<>'' + """ + ), + {"dataset_version": dataset_version}, + ) + ).scalars() + ) + self.assertTrue(grant_sources.issubset(set(stats["source"]))) + CharityDetail.model_validate(await organizations.detail(int(organization_id))) + GrantListResponse.model_validate(await organizations.grants(int(organization_id), "all")) + SankeyData.model_validate(await organizations.sankey(int(organization_id))) + ScoreResponse.model_validate(await organizations.score(int(organization_id), None)) + + page = await registry.page(limit=2) + RegistryDirectoryPage.model_validate(page) + empty_page = await registry.page( + query="qzxjkvbpygfwmuqzxjkvbpygfwmu", limit=2 + ) + self.assertEqual(empty_page["results"], []) + self.assertGreater(empty_page["registry_count"], 0) + RegistryOrganizationDetail.model_validate(await registry.detail(str(registry_id))) + + self.assertIsInstance(await analytics.beneficiary_geographies(), list) + GrantMapResponse.model_validate(await analytics.map()) + overview = await analytics.overview() + self.assertIn("kpis", overview) + self.assertEqual( + overview["kpis"]["awarded_funding"], + overview["kpis"]["total_amount"], + ) + self.assertEqual( + overview["kpis"]["grants_monitored"], + overview["kpis"]["grant_count"], + ) + filtered_overview = await analytics.overview(sources=["360Giving"]) + self.assertIn("awarded_funding", filtered_overview["kpis"]) + self.assertIn("programme_coverage_percentage", filtered_overview["kpis"]) + self.assertIn("donors", await analytics.suggestions(sources=None, limit=5)) + GrantTrendsResponse.model_validate(await analytics.trends(months=24)) + GrantThemesResponse.model_validate(await analytics.themes()) + GrantNetworkSummary.model_validate(await analytics.summary()) + self.assertIn( + "grants", + await analytics.drilldown( + selection_type="period", selection_value=str(period) + ), + ) + + country, funder_key = str(funder_scope[0]), str(funder_scope[1]) + funder_list = await funders.list( + beneficiary_country=country, + sources=["360Giving"], + page_size=5, + ) + SourceFunderListResponse.model_validate(funder_list) + self.assertIn("status_counts", funder_list["summary"]) + self.assertIn("monetary", funder_list["summary"]) + if funder_list["items"]: + item = funder_list["items"][0] + self.assertIn(item["profile_link"]["status"], {"none", "single"}) + self.assertIn("grant_count", item["observed_activity"]) + self.assertIn("recipient_count", item["observed_activity"]) + self.assertIn("displayed_currency", item["observed_activity"]) + SourceFunderDetailResponse.model_validate( + await funders.detail(funder_key, beneficiary_country=country) + ) + finally: + await engine.dispose() + + async def _exercise_mutations(self): + engine = create_async_engine(self._database_url(), pool_pre_ping=True) + connection = await engine.connect() + outer_transaction = await connection.begin() + sessions = async_sessionmaker( + bind=connection, + expire_on_commit=False, + autoflush=False, + join_transaction_mode="create_savepoint", + ) + try: + async with sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + identity = ( + await session.execute( + text( + """ + SELECT source_funder_key, charity_id + FROM grant_source_funder_facts + CROSS JOIN LATERAL ( + SELECT charities.charity_id FROM charities + WHERE charities.dataset_version=:dataset_version + ORDER BY charities.charity_id LIMIT 1 + ) AS profile + WHERE grant_source_funder_facts.dataset_version=:dataset_version + ORDER BY source_funder_key LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + ).one() + + key, profile_id = str(identity[0]), int(identity[1]) + unique = uuid.uuid4().hex + funders = SourceFunderRepository(sessions) + relinked = await funders.relink(key, profile_id, actor_id="phase5-integration") + self.assertIsNotNone(relinked) + queued = await funders.queue_profile_cache( + key, + actor_id="phase5-integration", + idempotency_key=f"profile-{unique}", + ) + self.assertEqual(queued["status"], "pending") + self.assertEqual((await funders.profile_cache(key))["status"], "pending") + self.assertIsNotNone(await funders.reset(key, actor_id="phase5-integration")) + + jobs = PostgresJobRepository(sessions) + job = await jobs.enqueue( + "quick_consolidate", + {"integration_test": True}, + actor_id="phase5-integration", + idempotency_key=f"pipeline-{unique}", + ) + duplicate = await jobs.enqueue( + "quick_consolidate", + {"integration_test": True}, + actor_id="phase5-integration", + idempotency_key=f"pipeline-{unique}", + ) + self.assertEqual(job["job_id"], duplicate["job_id"]) + self.assertTrue(duplicate["idempotent_noop"]) + + audit = PostgresAuditSink(sessions) + await audit.record( + AuditEvent( + actor_id="phase5-integration", + actor_role="administrator", + action="phase5.integration", + target="postgresql", + reason="transaction rollback test", + timestamp=datetime.now(timezone.utc).isoformat(), + request_id=f"phase5-{unique}", + result="success", + http_status=200, + error_class=None, + dataset_version=str(dataset_version), + ) + ) + async with sessions() as session: + self.assertEqual( + await session.scalar( + text("SELECT COUNT(*) FROM audit_events WHERE request_id=:request_id"), + {"request_id": f"phase5-{unique}"}, + ), + 1, + ) + finally: + if outer_transaction.is_active: + await outer_transaction.rollback() + await connection.close() + await engine.dispose() diff --git a/src/tests/test_postgres_performance.py b/src/tests/test_postgres_performance.py new file mode 100644 index 0000000..5bbdb57 --- /dev/null +++ b/src/tests/test_postgres_performance.py @@ -0,0 +1,315 @@ +"""Performance, cache, plan and concurrency gates for PostgreSQL serving.""" + +from __future__ import annotations + +import asyncio +import os +from time import perf_counter +import unittest + +from sqlalchemy import text + +from bff.database import DatabaseManager, DatabaseSettings +from bff.postgres.analytics_repository import AnalyticsRepository +from bff.postgres.base import ANALYTICS_CACHE, VersionedTTLCache +from bff.postgres.organization_repository import OrganizationRepository +from bff.postgres.registry_repository import RegistryRepository, _SEARCH_SQL + + +def _plan_nodes(plan: dict) -> list[dict]: + nodes = [plan] + for child in plan.get("Plans", []): + nodes.extend(_plan_nodes(child)) + return nodes + + +class TestVersionedCache(unittest.IsolatedAsyncioTestCase): + async def test_cache_is_single_flight_copy_safe_and_dataset_scoped(self): + cache = VersionedTTLCache(ttl_seconds=60, max_entries=4) + calls = 0 + + async def loader(): + nonlocal calls + calls += 1 + await asyncio.sleep(0.01) + return {"items": [1]} + + first, second = await asyncio.gather( + cache.get_or_create(("dataset-a", "map"), loader), + cache.get_or_create(("dataset-a", "map"), loader), + ) + self.assertEqual(calls, 1) + first["items"].append(2) + self.assertEqual(second, {"items": [1]}) + self.assertEqual( + await cache.get_or_create(("dataset-a", "map"), loader), + {"items": [1]}, + ) + self.assertGreater(cache.hit_ratio, 0) + await cache.retain_dataset("dataset-b") + await cache.get_or_create(("dataset-b", "map"), loader) + self.assertEqual(calls, 2) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestPostgreSQLPerformanceIntegration(unittest.TestCase): + def test_materialization_controls_and_bounded_relationships(self): + asyncio.run(self._materialization_controls()) + + def test_query_plans_use_versioned_facts_and_search_indexes(self): + asyncio.run(self._query_plans()) + + def test_cold_warm_concurrency_and_heavy_query_isolation(self): + asyncio.run(self._concurrency_and_isolation()) + + def test_timeout_cancels_database_work_and_connection_recovers(self): + asyncio.run(self._timeout_and_cancellation()) + + @staticmethod + def _manager() -> DatabaseManager: + return DatabaseManager(DatabaseSettings.from_env()) + + async def _materialization_controls(self): + database = self._manager() + sessions = database.sessions() + analytics = AnalyticsRepository(sessions) + try: + async with sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + raw = ( + await session.execute( + text( + """ + SELECT COUNT(*) AS grant_count, + COALESCE(SUM(eur_amount_minor) FILTER ( + WHERE eur_amount_status NOT IN ( + 'missing','invalid','negative' + ) + ), 0) AS amount_minor + FROM grant_overview_facts + WHERE dataset_version=:dataset_version + """ + ), + {"dataset_version": dataset_version}, + ) + ).one() + aggregate = ( + await session.execute( + text( + """ + SELECT total_grants, total_amount_minor + FROM analytics_scope_totals + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + """ + ), + {"dataset_version": dataset_version}, + ) + ).one() + materialization = ( + await session.execute( + text( + """ + SELECT status, is_active, row_count + FROM materialization_versions + WHERE dataset_version=:dataset_version + AND materialization_name='dashboard_analytics' + """ + ), + {"dataset_version": dataset_version}, + ) + ).one() + granularities = set( + ( + await session.execute( + text( + """ + SELECT DISTINCT granularity + FROM analytics_period_aggregates + WHERE dataset_version=:dataset_version + """ + ), + {"dataset_version": dataset_version}, + ) + ).scalars() + ) + self.assertEqual(tuple(raw), tuple(aggregate)) + self.assertEqual(materialization[0:2], ("active", True)) + self.assertGreater(materialization[2], 0) + self.assertEqual(granularities, {"monthly", "yearly"}) + connections = await analytics.map_connections(limit=250) + self.assertLessEqual(len(connections["connections"]), 250) + self.assertEqual( + connections["metadata"]["loading_mode"], + "lazy_bounded_secondary_request", + ) + finally: + await database.close() + + async def _query_plans(self): + database = self._manager() + sessions = database.sessions() + try: + async with sessions() as session: + dataset_version = await session.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + exact_name = await session.scalar( + text( + """ + SELECT normalized_name FROM charity_registry_organizations + WHERE dataset_version=:dataset_version + AND is_current_source_record + ORDER BY registry_id LIMIT 1 + """ + ), + {"dataset_version": dataset_version}, + ) + text_query = str(exact_name).split()[0] + map_plan = await session.scalar( + text( + """ + EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT * FROM analytics_country_aggregates + WHERE dataset_version=:dataset_version + AND amount_basis='eur_converted' AND currency='EUR' + ORDER BY total_amount_minor DESC LIMIT 500 + """ + ), + {"dataset_version": dataset_version}, + ) + exact_plan = await session.scalar( + text( + """ + EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) + SELECT registry_id FROM charity_registry_organizations + WHERE dataset_version=:dataset_version + AND is_current_source_record + AND normalized_name=:normalized_name LIMIT 21 + """ + ), + { + "dataset_version": dataset_version, + "normalized_name": exact_name, + }, + ) + search_plan = await session.scalar( + text( + "EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON) " + + str(_SEARCH_SQL) + ), + { + "query": text_query, + "normalized_query": text_query.casefold(), + "registration_status": None, + "cursor_rank": None, + "cursor_registry_id": "", + "limit": 21, + }, + ) + map_document = map_plan[0] + map_relations = { + node.get("Relation Name") for node in _plan_nodes(map_document["Plan"]) + } + self.assertIn("analytics_country_aggregates", map_relations) + self.assertNotIn("grant_overview_facts", map_relations) + exact_indexes = { + node.get("Index Name") for node in _plan_nodes(exact_plan[0]["Plan"]) + } + self.assertIn("ix_registry_current_normalized_name", exact_indexes) + search_indexes = { + node.get("Index Name") for node in _plan_nodes(search_plan[0]["Plan"]) + } + self.assertTrue( + { + "ix_registry_search_vector", + "ix_registry_normalized_name_trgm", + } + & search_indexes + ) + self.assertLess(exact_plan[0]["Execution Time"], 300) + # EXPLAIN instrumentation is intentionally allowed more headroom than + # the separately measured warm API p95 service-level objective. + self.assertLess(search_plan[0]["Execution Time"], 2000) + finally: + await database.close() + + async def _concurrency_and_isolation(self): + database = self._manager() + sessions = database.sessions() + analytics = AnalyticsRepository(sessions) + organizations = OrganizationRepository(sessions) + + async def dashboard(): + return await asyncio.gather( + organizations.stats(), + analytics.map(), + analytics.trends(months=24), + analytics.themes(), + analytics.summary(), + ) + + async def health_duration() -> float: + started = perf_counter() + async with sessions() as session: + self.assertEqual(await session.scalar(text("SELECT 1")), 1) + return (perf_counter() - started) * 1000 + + try: + cold_durations = [] + for _ in range(20): + await ANALYTICS_CACHE.clear() + cold_started = perf_counter() + await dashboard() + cold_durations.append((perf_counter() - cold_started) * 1000) + cold_p95 = sorted(cold_durations)[18] + self.assertLess(cold_p95, 3000) + + warm_durations = [] + for _ in range(10): + started = perf_counter() + await dashboard() + warm_durations.append((perf_counter() - started) * 1000) + self.assertLess(sorted(warm_durations)[-1], 3000) + + concurrent_started = perf_counter() + results = await asyncio.gather(*(dashboard() for _ in range(5))) + self.assertEqual(len(results), 5) + self.assertLess((perf_counter() - concurrent_started) * 1000, 3000) + + async def heavy_query(): + async with sessions() as session: + await session.execute(text("SELECT pg_sleep(1)")) + + heavy = asyncio.create_task(heavy_query()) + await asyncio.sleep(0.05) + health_durations = await asyncio.gather( + *(health_duration() for _ in range(5)) + ) + await heavy + self.assertLess(sorted(health_durations)[-1], 100) + self.assertGreater(ANALYTICS_CACHE.hit_ratio, 0.5) + self.assertEqual(database.engine().sync_engine.pool.checkedout(), 0) + finally: + await database.close() + + async def _timeout_and_cancellation(self): + database = self._manager() + sessions = database.sessions() + try: + started = perf_counter() + async with sessions() as session: + with self.assertRaises(TimeoutError): + async with asyncio.timeout(0.05): + await session.execute(text("SELECT pg_sleep(2)")) + await session.rollback() + self.assertLess((perf_counter() - started) * 1000, 1000) + async with sessions() as session: + self.assertEqual(await session.scalar(text("SELECT 1")), 1) + finally: + await database.close() diff --git a/src/tests/test_postgres_schema.py b/src/tests/test_postgres_schema.py new file mode 100644 index 0000000..7ff8aae --- /dev/null +++ b/src/tests/test_postgres_schema.py @@ -0,0 +1,231 @@ +import ast +import asyncio +from decimal import Decimal +import os +from pathlib import Path +import subprocess +import sys +import unittest +import uuid + +from sqlalchemy import text +from sqlalchemy.exc import IntegrityError +from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + +from bff.database import DatabaseSettings +from bff.postgres.registry_repository import RegistrySearchRepository, SearchCursor + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MIGRATION_PATH = ( + REPOSITORY_ROOT / "alembic" / "versions" / "0001_postgresql_foundation.py" +) +REQUIRED_TABLES = { + "charities", + "charity_registry_organizations", + "grants", + "grant_beneficiary_countries", + "grant_beneficiary_terms", + "grant_programme_categories", + "grant_overview_facts", + "grant_source_funder_facts", + "organization_registry_links", + "source_funder_link_overrides", + "source_funder_profile_cache", + "exchange_rates", + "job_runs", + "job_events", + "audit_events", + "dataset_versions", + "source_ingestion_runs", + "data_quality_issues", + "materialization_versions", + "retention_actions", + "export_jobs", +} + + +class TestPostgreSQLSchemaContract(unittest.TestCase): + def test_authoritative_migration_contains_required_schema_and_search(self): + source = MIGRATION_PATH.read_text(encoding="utf-8") + for table_name in REQUIRED_TABLES: + self.assertIn(f"CREATE TABLE {table_name}", source) + self.assertIn("CREATE EXTENSION IF NOT EXISTS pg_trgm", source) + self.assertIn("TSVECTOR GENERATED ALWAYS", source) + self.assertIn("USING GIN (search_vector)", source) + self.assertIn("gin_trgm_ops", source) + self.assertIn("FOREIGN KEY", source) + self.assertIn("ON UPDATE CASCADE ON DELETE", source) + + def test_postgresql_runtime_foundation_does_not_import_sqlite(self): + runtime_paths = [REPOSITORY_ROOT / "src" / "bff" / "database.py"] + runtime_paths.extend( + (REPOSITORY_ROOT / "src" / "bff" / "postgres").glob("*.py") + ) + for path in runtime_paths: + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + imported = { + alias.name + for node in ast.walk(tree) + if isinstance(node, ast.Import) + for alias in node.names + } + imported.update( + node.module or "" + for node in ast.walk(tree) + if isinstance(node, ast.ImportFrom) + ) + self.assertNotIn("sqlite3", imported, str(path)) + + def test_production_application_import_does_not_load_sqlite(self): + guard = """ +import builtins +original_import = builtins.__import__ +def guarded_import(name, *args, **kwargs): + if name == 'sqlite3' or name.startswith('sqlite3.'): + raise RuntimeError('production imported sqlite3') + return original_import(name, *args, **kwargs) +builtins.__import__ = guarded_import +import bff.main +assert bff.main.POSTGRESQL_ONLY_RUNTIME +""" + environment = { + **os.environ, + "APP_ENV": "production", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "disabled", + "PYTHONPATH": str(REPOSITORY_ROOT / "src"), + } + result = subprocess.run( + [sys.executable, "-c", guard], + cwd=REPOSITORY_ROOT, + env=environment, + capture_output=True, + text=True, + timeout=15, + check=False, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_search_cursor_round_trip_is_exact_and_bounded(self): + cursor = SearchCursor(Decimal("0.12345678"), "registry:001") + self.assertEqual(SearchCursor.decode(cursor.encode()), cursor) + with self.assertRaises(ValueError): + SearchCursor.decode("not-json") + with self.assertRaises(ValueError): + SearchCursor.decode("x" * 2049) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestPostgreSQLSchemaIntegration(unittest.TestCase): + def test_real_postgresql_foreign_keys_and_search(self): + asyncio.run(self._exercise_schema()) + + async def _exercise_schema(self): + configured_url = os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + engine = create_async_engine(configured_url, pool_pre_ping=True) + dataset_version = f"schema-test-{uuid.uuid4()}" + prior_active = None + try: + async with engine.connect() as connection: + table_rows = await connection.execute( + text( + """ + SELECT tablename + FROM pg_catalog.pg_tables + WHERE schemaname = current_schema() + """ + ) + ) + self.assertTrue(REQUIRED_TABLES.issubset(set(table_rows.scalars()))) + extension = await connection.scalar( + text("SELECT EXISTS (SELECT 1 FROM pg_extension WHERE extname='pg_trgm')") + ) + self.assertTrue(extension) + prior_active = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + + async with engine.begin() as connection: + if prior_active: + await connection.execute( + text( + "UPDATE dataset_versions " + "SET is_active=FALSE, status='rolled_back' " + "WHERE dataset_version=:version" + ), + {"version": prior_active}, + ) + await connection.execute( + text( + """ + INSERT INTO dataset_versions ( + dataset_version, status, is_active, activated_at + ) VALUES (:version, 'active', TRUE, CURRENT_TIMESTAMP) + """ + ), + {"version": dataset_version}, + ) + await connection.execute( + text( + """ + INSERT INTO charity_registry_organizations ( + dataset_version, registry_id, charity_number, + registered_name, normalized_name, source_name, imported_at + ) VALUES + (:version, 'registry:001', '1', 'Alpha Aid', + 'alpha aid', 'fixture', CURRENT_TIMESTAMP), + (:version, 'registry:002', '2', 'Alpha Aid', + 'alpha aid', 'fixture', CURRENT_TIMESTAMP) + """ + ), + {"version": dataset_version}, + ) + + sessions = async_sessionmaker(engine, expire_on_commit=False) + page = await RegistrySearchRepository(sessions).search("Alpha", limit=1) + self.assertEqual(page["items"][0]["registry_id"], "registry:001") + self.assertIsNotNone(page["next_cursor"]) + second_page = await RegistrySearchRepository(sessions).search( + "Alpha", cursor=page["next_cursor"], limit=1 + ) + self.assertEqual(second_page["items"][0]["registry_id"], "registry:002") + + with self.assertRaises(IntegrityError): + async with engine.begin() as connection: + await connection.execute( + text( + """ + INSERT INTO grants ( + dataset_version, grant_id, funding_charity_id, + recipient_name + ) VALUES (:version, 'orphan', 999999, 'Recipient') + """ + ), + {"version": dataset_version}, + ) + finally: + try: + async with engine.begin() as connection: + await connection.execute( + text("DELETE FROM dataset_versions WHERE dataset_version=:version"), + {"version": dataset_version}, + ) + if prior_active: + await connection.execute( + text( + "UPDATE dataset_versions " + "SET is_active=TRUE, status='active' " + "WHERE dataset_version=:version" + ), + {"version": prior_active}, + ) + finally: + await engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_release_gate.py b/src/tests/test_release_gate.py new file mode 100644 index 0000000..79f7d57 --- /dev/null +++ b/src/tests/test_release_gate.py @@ -0,0 +1,46 @@ +import unittest + +from migration.release_gate import evaluate_release_state + + +class TestReleaseGate(unittest.TestCase): + def test_approved_active_dataset_passes(self): + result = evaluate_release_state( + { + "schema_version": "0006_governance_retention", + "active_dataset_count": 1, + "dataset_version": "dataset-approved", + "migration_status": "active", + "reconciliation_results": { + "grant_count": {"status": "pass"}, + "registry_count": {"status": "pass"}, + }, + "materialization_active": True, + "active_quality_blocks": 0, + "dead_letter_count": 0, + } + ) + self.assertTrue(result["ready"]) + self.assertTrue(all(result["checks"].values())) + + def test_any_mismatch_fails_closed_without_sensitive_output(self): + result = evaluate_release_state( + { + "schema_version": "unexpected", + "active_dataset_count": 2, + "dataset_version": "dataset-rejected", + "migration_status": "failed", + "reconciliation_results": {"grant_count": {"status": "fail"}}, + "materialization_active": False, + "active_quality_blocks": 1, + "dead_letter_count": 1, + } + ) + self.assertFalse(result["ready"]) + self.assertEqual(result["reconciliation_failures"], ["grant_count"]) + self.assertNotIn("password", result) + self.assertNotIn("connection", result) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_security.py b/src/tests/test_security.py new file mode 100644 index 0000000..07e60c8 --- /dev/null +++ b/src/tests/test_security.py @@ -0,0 +1,426 @@ +import dataclasses +from datetime import datetime, timedelta, timezone +import json +import logging +import os +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +from fastapi.testclient import TestClient +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from jose import jwt +from jose.utils import base64url_encode + +from bff import admin as admin_module +from bff.audit import MemoryAuditSink +from bff.config import ( + SECURITY_SETTINGS, + SecurityConfigurationError, + SecuritySettings, + validate_security_settings, +) +from bff.main import app +from bff.security import ( + IdempotencyStore, + PUBLIC_READONLY_METHOD_ALLOWLIST, + PUBLIC_READONLY_ROUTE_ALLOWLIST, + Role, + SlidingWindowRateLimiter, + create_development_access_token, +) +from bff.utils.logging import RedactingFormatter, redact_text + + +class TestSecurityGate(unittest.TestCase): + def setUp(self): + self.original_settings = app.state.security_settings + self.original_limiter = app.state.rate_limiter + self.original_idempotency_store = app.state.idempotency_store + self.original_audit_sink = app.state.audit_sink + app.state.security_settings = SECURITY_SETTINGS + app.state.rate_limiter = SlidingWindowRateLimiter(10_000, 60) + app.state.idempotency_store = IdempotencyStore() + self.audit_sink = MemoryAuditSink() + app.state.audit_sink = self.audit_sink + self.admin_temp_dir = tempfile.TemporaryDirectory() + self.admin_paths = patch.multiple( + admin_module, + STATUS_FILE=os.path.join(self.admin_temp_dir.name, "pipeline_status.json"), + LOCK_FILE=os.path.join(self.admin_temp_dir.name, "pipeline_run.lock"), + LOG_FILE=os.path.join(self.admin_temp_dir.name, "pipeline_run.log"), + ) + self.admin_paths.start() + self.client = TestClient(app) + + def tearDown(self): + self.admin_paths.stop() + self.admin_temp_dir.cleanup() + app.state.security_settings = self.original_settings + app.state.rate_limiter = self.original_limiter + app.state.idempotency_store = self.original_idempotency_store + app.state.audit_sink = self.original_audit_sink + + def token(self, role: Role, subject: str = "security-test-user") -> str: + return create_development_access_token(subject, (role,)) + + def authorization(self, role: Role, subject: str = "security-test-user") -> dict: + return {"Authorization": f"Bearer {self.token(role, subject)}"} + + def test_protected_and_admin_routes_reject_anonymous_requests(self): + self.assertEqual(self.client.get("/api/charities").status_code, 401) + self.assertEqual(self.client.get("/api/admin/pipeline/status").status_code, 401) + mutation_cases = ( + ("/api/admin/pipeline/trigger", {"source": "quick_consolidate"}), + ("/api/charities/directory/organizations/enrich", {"reg_numbers": [1]}), + ("/api/charities/grants/funders/enrich", {"reg_numbers": [1]}), + ("/api/charities/grants/funders/source/profile-cache", None), + ("/api/charities/grants/funders/source/reset-to-observed", None), + ("/api/charities/grants/funders/source/relink", {"profile_id": 1}), + ("/api/charities/1/score", {}), + ("/api/core/v1/data", {}), + ) + for index, (path, payload) in enumerate(mutation_cases): + with self.subTest(path=path): + response = self.client.post( + path, + json=payload, + headers={"Idempotency-Key": f"anonymous-{index}"}, + ) + self.assertEqual(response.status_code, 401) + + def test_viewer_is_forbidden_and_operator_is_authorized(self): + forbidden = self.client.get( + "/api/admin/pipeline/status", + headers=self.authorization(Role.VIEWER), + ) + allowed = self.client.get( + "/api/admin/pipeline/status", + headers=self.authorization(Role.OPERATOR), + ) + self.assertEqual(forbidden.status_code, 403) + self.assertEqual(allowed.status_code, 200) + + def test_oidc_signature_claims_and_role_are_validated(self): + private_key = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_numbers = private_key.public_key().public_numbers() + + def encoded_integer(value: int) -> str: + raw = value.to_bytes((value.bit_length() + 7) // 8, "big") + return base64url_encode(raw).decode("ascii") + + jwks = { + "keys": [ + { + "kty": "RSA", + "kid": "security-test-key", + "use": "sig", + "alg": "RS256", + "n": encoded_integer(public_numbers.n), + "e": encoded_integer(public_numbers.e), + } + ] + } + app.state.security_settings = dataclasses.replace( + SECURITY_SETTINGS, + auth_mode="oidc", + oidc_issuer="https://identity.example.invalid/", + oidc_audience="foundation-intelligence-api", + oidc_jwks_json=json.dumps(jwks), + oidc_algorithms=("RS256",), + ) + now = datetime.now(timezone.utc) + private_pem = private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + token = jwt.encode( + { + "sub": "oidc-operator", + "roles": ["operator"], + "iss": "https://identity.example.invalid/", + "aud": "foundation-intelligence-api", + "iat": now, + "exp": now + timedelta(minutes=5), + }, + private_pem, + algorithm="RS256", + headers={"kid": "security-test-key"}, + ) + response = self.client.get( + "/api/admin/pipeline/status", + headers={"Authorization": f"Bearer {token}"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(self.audit_sink.events[-1].actor_id, "oidc-operator") + + def test_request_id_and_complete_audit_event_are_created(self): + response = self.client.get( + "/api/admin/pipeline/status", + headers={**self.authorization(Role.OPERATOR, "operator-42"), "X-Request-ID": "request-42"}, + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.headers["X-Request-ID"], "request-42") + event = self.audit_sink.events[-1] + self.assertEqual(event.actor_id, "operator-42") + self.assertEqual(event.actor_role, "operator") + self.assertEqual(event.action, "administration.access") + self.assertEqual(event.target, "/api/admin/pipeline/status") + self.assertEqual(event.reason, "not_provided") + self.assertEqual(event.request_id, "request-42") + self.assertEqual(event.result, "success") + self.assertIsNone(event.error_class) + self.assertIsNone(event.dataset_version) + + @patch("httpx.AsyncClient.request") + def test_proxy_allowlist_and_forwarded_header_allowlist(self, mock_request): + downstream = MagicMock() + downstream.status_code = 200 + downstream.content = b'{"ok":true}' + downstream.headers = { + "Content-Type": "application/json", + "Set-Cookie": "downstream=forbidden", + } + mock_request.return_value = downstream + response = self.client.get( + "/api/core/v1/data", + headers={ + **self.authorization(Role.ADMINISTRATOR), + "X-Forbidden-Forward": "secret-shaped-value", + }, + ) + self.assertEqual(response.status_code, 200) + sent_headers = mock_request.call_args.kwargs["headers"] + self.assertNotIn("authorization", sent_headers) + self.assertNotIn("x-forbidden-forward", sent_headers) + self.assertNotIn("set-cookie", {key.lower() for key in response.headers}) + + forbidden_path = self.client.get( + "/api/core/private/admin", + headers=self.authorization(Role.ADMINISTRATOR), + ) + self.assertEqual(forbidden_path.status_code, 403) + + def test_rate_limit_returns_429(self): + app.state.rate_limiter = SlidingWindowRateLimiter(1, 60) + headers = self.authorization(Role.OPERATOR, "rate-limited-operator") + self.assertEqual(self.client.get("/api/admin/pipeline/status", headers=headers).status_code, 200) + limited = self.client.get("/api/admin/pipeline/status", headers=headers) + self.assertEqual(limited.status_code, 429) + self.assertIn("Retry-After", limited.headers) + + @patch("bff.admin.run_pipeline_task") + def test_mutation_requires_idempotency_and_rejects_replay(self, run_pipeline_task): + headers = self.authorization(Role.OPERATOR, "idempotent-operator") + missing = self.client.post( + "/api/admin/pipeline/trigger", + json={"source": "quick_consolidate"}, + headers=headers, + ) + self.assertEqual(missing.status_code, 400) + + headers = {**headers, "Idempotency-Key": "pipeline-once"} + first = self.client.post( + "/api/admin/pipeline/trigger", + json={"source": "quick_consolidate"}, + headers=headers, + ) + second = self.client.post( + "/api/admin/pipeline/trigger", + json={"source": "quick_consolidate"}, + headers=headers, + ) + self.assertEqual(first.status_code, 200) + self.assertEqual(second.status_code, 409) + run_pipeline_task.assert_called_once() + + def test_payload_limit_returns_413_before_handler(self): + app.state.security_settings = dataclasses.replace( + SECURITY_SETTINGS, + max_request_body_bytes=32, + ) + response = self.client.post( + "/api/auth/login", + content=b"x" * 33, + headers={"Content-Type": "application/json"}, + ) + self.assertEqual(response.status_code, 413) + + def test_sensitive_values_are_redacted(self): + value = redact_text( + "Authorization: Bearer example-token password=example-password " + "postgresql://app:database-password@db.internal/name " + "{\"token\":\"json-token\"}" + ) + self.assertNotIn("example-token", value) + self.assertNotIn("example-password", value) + self.assertNotIn("database-password", value) + self.assertNotIn("json-token", value) + self.assertGreaterEqual(value.count("[REDACTED]"), 4) + + try: + raise RuntimeError("Authorization: Bearer traceback-token") + except RuntimeError: + record = logging.LogRecord("test", logging.ERROR, __file__, 1, "failed", (), sys.exc_info()) + formatted = RedactingFormatter("%(message)s").format(record) + self.assertNotIn("traceback-token", formatted) + self.assertIn("[REDACTED]", formatted) + + def test_public_endpoint_is_read_only(self): + self.assertEqual(self.client.get("/health").status_code, 200) + self.assertEqual(self.client.post("/health").status_code, 405) + + def test_public_readonly_allows_only_reviewed_ui_read_routes(self): + app.state.security_settings = dataclasses.replace( + SECURITY_SETTINGS, + app_env="demo", + data_runtime_mode="postgresql", + auth_mode="public_readonly", + dev_auth_enabled=False, + core_proxy_enabled=False, + ) + + allowed = self.client.get("/api/charities/grants/overview") + self.assertEqual(allowed.status_code, 200) + self.assertIsNone(allowed.headers.get("set-cookie")) + self.assertEqual( + PUBLIC_READONLY_ROUTE_ALLOWLIST, + { + "/api/charities", + "/api/charities/stats", + "/api/charities/{reg_charity_number}", + "/api/charities/{reg_charity_number}/grants", + "/api/charities/{reg_charity_number}/sankey", + "/api/charities/{reg_charity_number}/score", + "/api/charities/directory/organizations", + "/api/charities/directory/organizations/{registry_id}", + "/api/charities/grants/beneficiary-geographies", + "/api/charities/grants/funders", + "/api/charities/grants/funders/{source_funder_key}", + "/api/charities/grants/map", + "/api/charities/grants/map/connections", + "/api/charities/grants/overview", + "/api/charities/grants/overview/drilldown", + "/api/charities/grants/overview/entity-suggestions", + "/api/charities/grants/overview/trends", + "/api/charities/grants/summary", + "/api/charities/grants/themes", + "/api/charities/grants/trends", + }, + ) + self.assertEqual(PUBLIC_READONLY_METHOD_ALLOWLIST, {"GET", "HEAD"}) + + for method in ("POST", "PUT", "PATCH", "DELETE"): + with self.subTest(method=method): + blocked = self.client.request( + method, + "/api/charities/grants/overview", + ) + self.assertEqual(blocked.status_code, 401) + self.assertIsNone(blocked.headers.get("set-cookie")) + + protected = ( + "/api/charities/grants/funders/example/profile-cache", + "/api/admin/pipeline/status", + "/api/news/example/summary", + ) + for path in protected: + with self.subTest(path=path): + self.assertIn(self.client.get(path).status_code, {401, 403}) + + mutation = self.client.post( + "/api/admin/pipeline/trigger", + json={"source": "quick_consolidate"}, + headers={"Idempotency-Key": "public-demo-write"}, + ) + self.assertIn(mutation.status_code, {401, 403}) + + def test_public_readonly_fails_closed_outside_demo(self): + for environment in ("development", "test", "staging", "production"): + with self.subTest(environment=environment): + settings = SecuritySettings.from_env( + { + "APP_ENV": environment, + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "public_readonly", + "SESSION_COOKIE_SECURE": "true", + "CORS_ORIGINS": "https://app.example.invalid", + } + ) + with self.assertRaisesRegex( + SecurityConfigurationError, + "public_readonly authentication requires APP_ENV=demo", + ): + validate_security_settings(settings) + + def test_public_readonly_rejects_development_auth_and_proxy_bypasses(self): + settings = SecuritySettings.from_env( + { + "APP_ENV": "demo", + "DATA_RUNTIME_MODE": "postgresql", + "AUTH_MODE": "public_readonly", + "DEV_AUTH_ENABLED": "true", + "CORE_PROXY_ENABLED": "true", + "CORE_API_URL": "https://core.example.invalid", + "CORE_API_ALLOWED_HOSTS": "core.example.invalid", + "CORE_PROXY_ALLOWED_PATHS": "v1/data", + "CORS_ORIGINS": "", + } + ) + with self.assertRaises(SecurityConfigurationError): + validate_security_settings(settings) + + def test_public_readonly_rejects_legacy_data_runtime(self): + settings = SecuritySettings.from_env( + { + "APP_ENV": "demo", + "DATA_RUNTIME_MODE": "sqlite_migration_source", + "AUTH_MODE": "public_readonly", + "DEV_AUTH_ENABLED": "false", + "CORE_PROXY_ENABLED": "false", + "CORS_ORIGINS": "", + } + ) + with self.assertRaisesRegex( + SecurityConfigurationError, + "public_readonly authentication requires DATA_RUNTIME_MODE=postgresql", + ): + validate_security_settings(settings) + + def test_production_configuration_fails_closed(self): + settings = SecuritySettings.from_env( + { + "APP_ENV": "production", + "AUTH_MODE": "disabled", + "SESSION_COOKIE_SECURE": "false", + } + ) + with self.assertRaises(SecurityConfigurationError): + validate_security_settings(settings) + app.state.security_settings = settings + with self.assertRaises(SecurityConfigurationError): + with TestClient(app): + pass + app.state.security_settings = SECURITY_SETTINGS + + unsafe_bypass = SecuritySettings.from_env( + { + "APP_ENV": "production", + "AUTH_MODE": "development", + "DEV_AUTH_ENABLED": "true", + "DEV_AUTH_USERNAME": "local-user", + "DEV_AUTH_PASSWORD": "local-password", + "DEV_AUTH_SECRET": "local-signing-key-with-at-least-32-characters", + "SESSION_COOKIE_SECURE": "true", + "CORS_ORIGINS": "https://app.example.invalid", + } + ) + with self.assertRaises(SecurityConfigurationError): + validate_security_settings(unsafe_bypass) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_shadow_transition.py b/src/tests/test_shadow_transition.py new file mode 100644 index 0000000..caa4d00 --- /dev/null +++ b/src/tests/test_shadow_transition.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +import tempfile +import time +import unittest + +from transition.runtime import ( + RuntimeMode, + TransitionConfigurationError, + load_transition_settings, +) +from transition.shadow import ( + ComparisonPolicy, + ShadowComparisonCoordinator, + ShadowRequest, + compare_payloads, +) +from transition.sqlite_source import resolve_shadow_journey + + +ROOT = Path(__file__).resolve().parents[2] + + +class _Sink: + def __init__(self): + self.results = [] + + def record(self, result): + self.results.append(result) + + +class _Reader: + def __init__(self, payload, delay=0.0): + self.payload = payload + self.delay = delay + + async def read(self, request): + await asyncio.sleep(self.delay) + return self.payload + + +class TestTransitionConfiguration(unittest.TestCase): + def test_operational_environments_default_to_postgresql(self): + settings = load_transition_settings({"APP_ENV": "production"}) + self.assertEqual(settings.mode, RuntimeMode.POSTGRESQL) + self.assertTrue(settings.postgresql_authoritative) + self.assertFalse(settings.shadow_enabled) + + def test_production_cannot_select_sqlite_migration_source(self): + with self.assertRaises(TransitionConfigurationError): + load_transition_settings( + {"APP_ENV": "production", "DATA_RUNTIME_MODE": "sqlite_migration_source"} + ) + + def test_shadow_requires_a_separate_snapshot(self): + with self.assertRaises(TransitionConfigurationError): + load_transition_settings( + {"APP_ENV": "development", "DATA_RUNTIME_MODE": "shadow_compare"} + ) + with tempfile.NamedTemporaryFile(suffix=".db") as snapshot: + settings = load_transition_settings( + { + "APP_ENV": "development", + "DATA_RUNTIME_MODE": "shadow_compare", + "SHADOW_SQLITE_PATH": snapshot.name, + "DB_PATH": str(ROOT / "src/data/charities.db"), + } + ) + self.assertTrue(settings.shadow_enabled) + self.assertTrue(settings.postgresql_authoritative) + + def test_shadow_cannot_alias_a_custom_migration_source(self): + with tempfile.NamedTemporaryFile(suffix=".db") as source: + with self.assertRaises(TransitionConfigurationError): + load_transition_settings( + { + "APP_ENV": "development", + "DATA_RUNTIME_MODE": "shadow_compare", + "SHADOW_SQLITE_PATH": source.name, + "DB_PATH": source.name, + } + ) + + +class TestShadowComparison(unittest.TestCase): + def test_only_approved_list_paths_ignore_order(self): + policy = ComparisonPolicy(unordered_paths=frozenset({"$.currencies"})) + match = compare_payloads( + {"currencies": ["GBP", "EUR"]}, + {"currencies": ["EUR", "GBP"]}, + journey="dashboard", + policy=policy, + ) + difference = compare_payloads( + {"rankings": ["A", "B"]}, + {"rankings": ["B", "A"]}, + journey="donor_ranking", + policy=policy, + ) + self.assertEqual(match.status, "match") + self.assertEqual(difference.status, "difference") + + def test_difference_evidence_contains_fingerprints_not_values(self): + result = compare_payloads( + {"profile": {"email": "sensitive@example.invalid"}}, + {"profile": {"email": "different@example.invalid"}}, + journey="profile_detail", + ) + serialized = json.dumps([item.__dict__ for item in result.differences]) + self.assertEqual(result.difference_count, 1) + self.assertNotIn("sensitive@example.invalid", serialized) + self.assertIn("primary_fingerprint", serialized) + + def test_submission_never_waits_for_the_shadow_read(self): + async def exercise(): + sink = _Sink() + coordinator = ShadowComparisonCoordinator( + _Reader({"value": 1}, delay=0.05), + sink, + policy=ComparisonPolicy(), + timeout_seconds=1, + maximum_pending=2, + ) + request = ShadowRequest( + journey="dashboard", + method="GET", + path="/api/charities/grants/overview", + query_string="", + primary_payload={"value": 1}, + ) + started = time.perf_counter() + accepted = coordinator.submit(request) + elapsed = time.perf_counter() - started + self.assertTrue(accepted) + self.assertLess(elapsed, 0.02) + self.assertEqual(sink.results, []) + await coordinator.drain() + self.assertEqual(sink.results[0].status, "match") + + asyncio.run(exercise()) + + def test_queue_bound_drops_work_without_changing_primary_authority(self): + async def exercise(): + sink = _Sink() + coordinator = ShadowComparisonCoordinator( + _Reader({}, delay=0.05), + sink, + policy=ComparisonPolicy(), + timeout_seconds=1, + maximum_pending=1, + ) + request = ShadowRequest("map", "GET", "/map", "", {}) + self.assertTrue(coordinator.submit(request)) + self.assertFalse(coordinator.submit(request)) + self.assertEqual(sink.results[0].status, "dropped_queue_full") + self.assertTrue(sink.results[0].primary_authoritative) + await coordinator.drain() + + asyncio.run(exercise()) + + def test_all_required_journeys_are_versioned_and_routed(self): + config = json.loads((ROOT / "config/runtime-transition.json").read_text()) + self.assertEqual(len(config["journeys"]), 21) + cases = { + "dashboard": ("/api/charities/grants/overview", ""), + "date_filters": ("/api/charities/grants/overview", "date_from=2024-01-01"), + "country_filters": ("/api/charities/grants/overview", "beneficiary_geographies=Germany"), + "programme_filters": ("/api/charities/grants/overview", "programme_areas=Education"), + "donor_filters": ("/api/charities/grants/overview", "donor=Trust"), + "recipient_filters": ("/api/charities/grants/overview", "recipient=School"), + "map": ("/api/charities/grants/map", ""), + "map_relationships": ("/api/charities/grants/map/connections", ""), + "monthly_trends": ("/api/charities/grants/overview/trends", "granularity=monthly"), + "yearly_trends": ("/api/charities/grants/overview/trends", "granularity=yearly"), + "donor_ranking": ("/api/charities/grants/funders", "beneficiary_country=GB"), + "registry_search": ("/api/charities/directory/organizations", "query=trust"), + "profile_detail": ("/api/charities/1", ""), + "grant_list": ("/api/charities/1/grants", ""), + "drill_down": ("/api/charities/grants/overview/drilldown", "selection_type=period&selection_value=2024"), + "sankey": ("/api/charities/1/sankey", ""), + } + for expected, (path, query) in cases.items(): + self.assertEqual(resolve_shadow_journey("GET", path, query), expected) + for contract_only in { + "recipient_ranking", "score", "news", "pipeline_status", + "manual_refresh_permissions", + }: + self.assertIn(contract_only, config["journeys"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_source_funders.py b/src/tests/test_source_funders.py index cac89c2..699e799 100644 --- a/src/tests/test_source_funders.py +++ b/src/tests/test_source_funders.py @@ -442,6 +442,7 @@ async def test_endpoint_validates_country_and_returns_paginated_source_funders(s reset = client.post( f"/api/charities/grants/funders/{payload['items'][0]['source_funder_key']}/reset-to-observed", cookies=login.cookies, + headers={"Idempotency-Key": "test-source-funder-reset"}, ) self.assertEqual(reset.status_code, 200) self.assertEqual(reset.json()["status"], "observed_only") diff --git a/src/tests/test_sqlite_to_postgres_migration.py b/src/tests/test_sqlite_to_postgres_migration.py new file mode 100644 index 0000000..9648e8e --- /dev/null +++ b/src/tests/test_sqlite_to_postgres_migration.py @@ -0,0 +1,452 @@ +import asyncio +import hashlib +import os +from pathlib import Path +import sqlite3 +import tempfile +import unittest +import uuid + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from bff.database import DatabaseSettings +from migration.sqlite_to_postgres import ( + MigrationError, + PreflightError, + convert_value, + migrate, + open_sqlite_read_only, + rollback_dataset, + source_preflight, +) + + +CODE_REVISION = "a" * 40 + + +def _fixture_database(path: Path, *, invalid_currency: bool = False) -> str: + connection = sqlite3.connect(path) + connection.executescript( + """ + CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); + CREATE TABLE charities ( + charity_id INTEGER PRIMARY KEY, name TEXT NOT NULL, + primary_source TEXT, source_record_id TEXT + ); + CREATE TABLE charity_registry_organizations ( + registry_id TEXT PRIMARY KEY, charity_number TEXT NOT NULL, + registered_name TEXT NOT NULL, normalized_name TEXT NOT NULL, + country_code TEXT, source_name TEXT NOT NULL, + imported_at TEXT NOT NULL, is_current_source_record INTEGER NOT NULL + ); + CREATE TABLE grants ( + grant_id TEXT PRIMARY KEY, funding_charity_id INTEGER, + funding_name TEXT, recipient_name TEXT NOT NULL, + recipient_charity_id INTEGER, amount REAL, amount_eur REAL, + currency TEXT, date TEXT, source TEXT, source_record_id TEXT, + programme_area_method TEXT, geography_method TEXT, + conversion_status TEXT + ); + CREATE TABLE grant_beneficiary_countries ( + grant_id TEXT NOT NULL, country_code TEXT NOT NULL, + country_name TEXT NOT NULL, PRIMARY KEY (grant_id, country_code) + ); + CREATE TABLE grant_beneficiary_terms ( + grant_id TEXT NOT NULL, term TEXT NOT NULL, + PRIMARY KEY (grant_id, term) + ); + CREATE TABLE grant_programme_categories ( + grant_id TEXT NOT NULL, programme_area TEXT NOT NULL, + PRIMARY KEY (grant_id, programme_area) + ); + CREATE TABLE grant_overview_facts ( + grant_id TEXT PRIMARY KEY, source_namespace TEXT NOT NULL, + award_date TEXT, award_date_status TEXT NOT NULL, + currency TEXT, original_amount_minor INTEGER, + original_amount_status TEXT NOT NULL, eur_amount_minor INTEGER, + eur_amount_status TEXT NOT NULL, conversion_status TEXT, + funding_name TEXT NOT NULL, funding_name_normalized TEXT NOT NULL, + recipient_name TEXT NOT NULL, recipient_name_normalized TEXT NOT NULL, + country_count INTEGER NOT NULL, programme_category_count INTEGER NOT NULL, + programme_provenance TEXT NOT NULL, invalid_source_label INTEGER NOT NULL, + low_confidence_inference INTEGER NOT NULL, origin_country_code TEXT, + origin_country_name TEXT, origin_source TEXT, data_revision TEXT NOT NULL + ); + CREATE TABLE grant_source_funder_facts ( + grant_id TEXT NOT NULL, country_code TEXT NOT NULL, + country_name TEXT NOT NULL, source_namespace TEXT NOT NULL, + source_funder_key TEXT NOT NULL, identity_method TEXT NOT NULL, + source_organization_id TEXT, display_name TEXT NOT NULL, + recipient_key TEXT NOT NULL, recipient_name TEXT NOT NULL, + award_date TEXT, currency TEXT, original_amount_minor INTEGER, + original_amount_status TEXT NOT NULL, eur_amount_minor INTEGER, + eur_amount_status TEXT NOT NULL, conversion_status TEXT, + country_count INTEGER NOT NULL, linked_profile_id INTEGER, + data_revision TEXT NOT NULL, PRIMARY KEY (grant_id, country_code) + ); + CREATE TABLE organization_registry_links ( + registry_id TEXT NOT NULL, enriched_organization_id INTEGER NOT NULL, + match_status TEXT NOT NULL, match_method TEXT NOT NULL, + match_confidence REAL, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, + PRIMARY KEY (registry_id, enriched_organization_id) + ); + CREATE TABLE source_funder_link_overrides ( + source_namespace TEXT NOT NULL, source_organization_id TEXT NOT NULL, + link_mode TEXT NOT NULL, updated_at TEXT NOT NULL, + revision INTEGER NOT NULL, PRIMARY KEY (source_namespace, source_organization_id) + ); + CREATE TABLE source_funder_profile_cache ( + source_funder_key TEXT PRIMARY KEY, profile_id INTEGER NOT NULL, + status TEXT NOT NULL, payload TEXT, updated_at TEXT NOT NULL, + link_revision INTEGER NOT NULL + ); + CREATE TABLE exchange_rates ( + currency TEXT NOT NULL, rate_date TEXT NOT NULL, + eur_reference_rate REAL NOT NULL, source_series TEXT NOT NULL, + source_url TEXT NOT NULL, retrieved_at TEXT NOT NULL, + PRIMARY KEY (currency, rate_date) + ); + """ + ) + currency = "BADX" if invalid_currency else "GBP" + connection.executemany( + "INSERT INTO metadata(key, value) VALUES (?, ?)", + [ + ("schema_version", "7"), + ("grant_overview_schema_version", "fixture-v1"), + ("registry_schema_version", "1"), + ], + ) + connection.execute( + "INSERT INTO charities VALUES (1, 'Fixture Foundation', 'fixture', 'charity-1')" + ) + connection.execute( + """ + INSERT INTO charity_registry_organizations VALUES ( + 'registry-1', '100', 'Fixture Foundation', 'fixture foundation', + 'GB', 'fixture', '2026-01-01T00:00:00Z', 1 + ) + """ + ) + connection.execute( + """ + INSERT INTO grants VALUES ( + 'grant-1', 1, 'Fixture Foundation', 'Fixture Recipient', 1, + 100.0, 100.0, ?, '2025-01-01', 'fixture', 'source-grant-1', + 'source_normalization', 'source_normalization', 'ecb_monthly_average' + ) + """, + (currency,), + ) + connection.execute( + "INSERT INTO grant_beneficiary_countries VALUES ('grant-1', 'GB', 'United Kingdom')" + ) + connection.execute( + "INSERT INTO grant_beneficiary_terms VALUES ('grant-1', 'education')" + ) + connection.execute( + "INSERT INTO grant_programme_categories VALUES ('grant-1', 'Education')" + ) + connection.execute( + """ + INSERT INTO grant_overview_facts VALUES ( + 'grant-1', 'fixture', '2025-01-01', 'valid', 'GBP', 10000, + 'valid', 10000, 'valid', 'ecb_monthly_average', + 'Fixture Foundation', 'fixture foundation', 'Fixture Recipient', + 'fixture recipient', 1, 1, 'source', 0, 0, 'GB', + 'United Kingdom', 'fixture', 'fixture-revision' + ) + """ + ) + connection.execute( + """ + INSERT INTO grant_source_funder_facts VALUES ( + 'grant-1', 'GB', 'United Kingdom', 'fixture', 'fixture:foundation', + 'source_id', 'foundation-1', 'Fixture Foundation', + 'fixture:recipient', 'Fixture Recipient', '2025-01-01', 'GBP', + 10000, 'valid', 10000, 'valid', 'ecb_monthly_average', 1, 1, + 'fixture-revision' + ) + """ + ) + connection.execute( + """ + INSERT INTO organization_registry_links VALUES ( + 'registry-1', 1, 'accepted', 'exact_identifier', 1.0, + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z' + ) + """ + ) + connection.execute( + """ + INSERT INTO source_funder_link_overrides VALUES ( + 'fixture', 'foundation-1', 'observed_only', + '2026-01-01T00:00:00Z', 0 + ) + """ + ) + connection.execute( + """ + INSERT INTO source_funder_profile_cache VALUES ( + 'fixture:foundation', 1, 'ready', '{}', + '2026-01-01T00:00:00Z', 0 + ) + """ + ) + connection.execute( + """ + INSERT INTO exchange_rates VALUES ( + 'GBP', '2025-01-01', 0.8, 'fixture', 'https://example.invalid/rate', + '2026-01-01T00:00:00Z' + ) + """ + ) + connection.commit() + connection.close() + return hashlib.sha256(path.read_bytes()).hexdigest() + + +class TestMigrationSourceSafety(unittest.TestCase): + def test_exchange_rate_month_is_preserved(self): + self.assertEqual( + convert_value("grants", "exchange_rate_date", "2025-07"), + "2025-07", + ) + + def test_grant_award_timestamp_is_preserved(self): + value = "2016-03-11T16:57:17.743000+00:00" + self.assertEqual(convert_value("grants", "date", value), value) + + def test_source_is_checksum_verified_and_opened_read_only(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "fixture.db" + checksum = _fixture_database(path) + result = source_preflight( + path, checksum, "7", enforce_capacity=False + ) + self.assertEqual(result.checksum, checksum) + self.assertEqual(result.counts["grants"], 1) + with open_sqlite_read_only(path) as connection: + with self.assertRaises(sqlite3.OperationalError): + connection.execute("CREATE TABLE forbidden (id INTEGER)") + + def test_checksum_mismatch_fails_before_open(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "fixture.db" + _fixture_database(path) + with self.assertRaises(PreflightError): + source_preflight(path, "0" * 64, "7", enforce_capacity=False) + + def test_remote_postgres_capacity_does_not_count_rds_storage_as_local(self): + with tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "fixture.db" + checksum = _fixture_database(path) + result = source_preflight( + path, + checksum, + "7", + enforce_capacity=False, + remote_postgres=True, + ) + self.assertEqual(result.capacity.estimated_postgres_and_indexes_bytes, 0) + self.assertEqual(result.capacity.estimated_wal_and_temp_bytes, 0) + self.assertEqual(result.capacity.safety_margin_bytes, 4 * 1024**3) + + +@unittest.skipUnless( + os.getenv("RUN_POSTGRES_INTEGRATION") == "1" or os.getenv("TEST_DATABASE_URL"), + "RUN_POSTGRES_INTEGRATION=1 or TEST_DATABASE_URL is required", +) +class TestMigrationPostgreSQLIntegration(unittest.TestCase): + def test_fixture_activation_idempotence_failure_isolation_and_rollback(self): + asyncio.run(self._exercise()) + + async def _exercise(self): + configured_url = os.getenv("TEST_DATABASE_URL") or DatabaseSettings.from_env().sqlalchemy_url() + engine = create_async_engine(configured_url) + async with engine.connect() as connection: + prior_active = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + suffix = uuid.uuid4().hex + first = f"fixture-first-{suffix}" + second = f"fixture-second-{suffix}" + invalid = f"fixture-invalid-{suffix}" + try: + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + source = directory / "valid.db" + invalid_source = directory / "invalid.db" + checksum = _fixture_database(source) + invalid_checksum = _fixture_database( + invalid_source, invalid_currency=True + ) + with sqlite3.connect(invalid_source) as invalid_connection: + invalid_connection.execute( + "UPDATE exchange_rates SET source_series='must-not-activate'" + ) + invalid_checksum = hashlib.sha256(invalid_source.read_bytes()).hexdigest() + report = await migrate( + source, checksum, "7", first, CODE_REVISION, + "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, enforce_capacity=False, + ) + self.assertEqual(report["activation_status"], "active") + self.assertTrue(all( + item["status"] == "pass" + for item in report["reconciliation_results"].values() + )) + self.assertTrue((directory / f"migration-{first}.json").is_file()) + + noop = await migrate( + source, checksum, "7", first, CODE_REVISION, + "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, enforce_capacity=False, + ) + self.assertTrue(noop["idempotent_noop"]) + + await migrate( + source, checksum, "7", second, CODE_REVISION, + "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, enforce_capacity=False, + ) + rolled_back = await rollback_dataset(first) + self.assertEqual(rolled_back, {"from": second, "to": first}) + + with self.assertRaises(MigrationError): + await migrate( + invalid_source, invalid_checksum, "7", invalid, + CODE_REVISION, "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, + enforce_capacity=False, + ) + async with engine.connect() as connection: + active = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual(active, first) + quality_count = await connection.scalar( + text( + """ + SELECT COUNT(*) FROM data_quality_issues + WHERE dataset_version=:version AND status='quarantined' + """ + ), + {"version": invalid}, + ) + self.assertEqual(quality_count, 1) + exchange_rate_source = await connection.scalar( + text( + """ + SELECT source_series FROM exchange_rates + WHERE currency='GBP' AND rate_date=DATE '2025-01-01' + """ + ) + ) + self.assertEqual(exchange_rate_source, "fixture") + with sqlite3.connect(invalid_source) as invalid_connection: + invalid_connection.execute( + "UPDATE grants SET currency='GBP'" + ) + invalid_connection.execute( + "UPDATE source_funder_link_overrides SET link_mode='blocked'" + ) + conflicting_checksum = hashlib.sha256( + invalid_source.read_bytes() + ).hexdigest() + with self.assertRaises(MigrationError): + await migrate( + invalid_source, conflicting_checksum, "7", invalid, + CODE_REVISION, "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, + enforce_capacity=False, + ) + async with engine.connect() as connection: + active = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual(active, first) + override_mode = await connection.scalar( + text( + """ + SELECT link_mode FROM source_funder_link_overrides + WHERE source_namespace='fixture' + AND source_organization_id='foundation-1' + """ + ) + ) + self.assertEqual(override_mode, "observed_only") + recovered = await migrate( + source, checksum, "7", invalid, CODE_REVISION, + "integration-test", "ci", directory, + batch_size=2, enforce_baseline=False, enforce_capacity=False, + ) + self.assertEqual(recovered["activation_status"], "active") + recovered_rollback = await rollback_dataset(first) + self.assertEqual(recovered_rollback, {"from": invalid, "to": first}) + finally: + if prior_active: + await rollback_dataset(prior_active) + async with engine.begin() as connection: + for table in ( + "source_funder_profile_cache", + "organization_registry_links", + "grant_source_funder_facts", + "grant_overview_facts", + "grant_programme_categories", + "grant_beneficiary_terms", + "grant_beneficiary_countries", + "grants", + "charity_registry_organizations", + "charities", + ): + await connection.execute( + text( + f'DELETE FROM "{table}" ' + "WHERE dataset_version IN (:first, :second, :invalid)" + ), + {"first": first, "second": second, "invalid": invalid}, + ) + await connection.execute( + text( + """ + DELETE FROM data_quality_issues + WHERE dataset_version IN (:first, :second, :invalid) + """ + ), + {"first": first, "second": second, "invalid": invalid}, + ) + await connection.execute( + text( + """ + DELETE FROM migration_runs + WHERE target_dataset_version IN (:first, :second, :invalid) + """ + ), + {"first": first, "second": second, "invalid": invalid}, + ) + await connection.execute( + text( + """ + DELETE FROM dataset_versions + WHERE dataset_version IN (:first, :second, :invalid) + """ + ), + {"first": first, "second": second, "invalid": invalid}, + ) + await connection.execute( + text("DELETE FROM source_funder_link_overrides WHERE source_namespace='fixture'") + ) + await connection.execute( + text("DELETE FROM exchange_rates WHERE source_series='fixture'") + ) + active_after_cleanup = await connection.scalar( + text("SELECT dataset_version FROM dataset_versions WHERE is_active") + ) + self.assertEqual(active_after_cleanup, prior_active) + await engine.dispose() + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_supply_chain.py b/src/tests/test_supply_chain.py new file mode 100644 index 0000000..af9951a --- /dev/null +++ b/src/tests/test_supply_chain.py @@ -0,0 +1,36 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from scripts.check_licenses import ROOT, scan +from scripts.generate_sbom import document, npm_components, python_components, write_document + + +class TestSupplyChainContracts(unittest.TestCase): + def test_lockfiles_generate_deterministic_cyclonedx_components(self): + backend = document("backend", python_components()) + frontend = document("frontend", npm_components()) + self.assertEqual(backend["bomFormat"], "CycloneDX") + self.assertEqual(backend["specVersion"], "1.5") + self.assertGreater(len(backend["components"]), 50) + self.assertGreater(len(frontend["components"]), 50) + self.assertTrue(all(component.get("version") for component in backend["components"])) + self.assertTrue(all(component.get("purl") for component in frontend["components"])) + + with tempfile.TemporaryDirectory() as temporary: + first = Path(temporary) / "first.json" + second = Path(temporary) / "second.json" + write_document(first, backend) + write_document(second, backend) + self.assertEqual(first.read_bytes(), second.read_bytes()) + json.loads(first.read_text(encoding="utf-8")) + + def test_installed_dependencies_have_no_forbidden_declared_license(self): + result = scan(ROOT / "frontend" / "node_modules") + self.assertEqual(result["status"], "passed", result["forbidden"]) + self.assertGreater(result["components_scanned"], 100) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_terraform_contract.py b/src/tests/test_terraform_contract.py new file mode 100644 index 0000000..98e5cad --- /dev/null +++ b/src/tests/test_terraform_contract.py @@ -0,0 +1,36 @@ +from pathlib import Path +import unittest + +from scripts.validate_terraform_static import ROOT, validate + + +class TestTerraformContract(unittest.TestCase): + def test_static_structure_and_security_contract(self): + result = validate() + self.assertEqual(result["status"], "passed") + self.assertEqual(result["environments"], ["dev", "staging"]) + self.assertGreaterEqual(result["resource_types"], 30) + self.assertGreaterEqual(result["resource_blocks"], 70) + self.assertEqual(result["provider_dependent_validation"], "not_tested") + self.assertFalse(result["aws_actions_performed"]) + + def test_environment_examples_contain_no_secret_values(self): + for environment in ("dev", "staging"): + path = ( + ROOT + / "infra" + / "terraform" + / "environments" + / environment + / "terraform.tfvars.example" + ) + text = path.read_text(encoding="utf-8") + self.assertNotIn("aws_access_key_id", text.casefold()) + self.assertNotIn("aws_secret_access_key", text.casefold()) + self.assertNotIn("password", text.casefold()) + self.assertIn("@sha256:", text) + self.assertIn("manage_dns = false", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/tests/test_transition_golden.py b/src/tests/test_transition_golden.py new file mode 100644 index 0000000..47b805e --- /dev/null +++ b/src/tests/test_transition_golden.py @@ -0,0 +1,31 @@ +import json +import os +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +GOLDEN_PATH = ROOT / "config/golden/transition-domain.json" + + +class TestTransitionGoldenFixture(unittest.TestCase): + def test_fixture_covers_every_required_domain_projection(self): + if not GOLDEN_PATH.exists(): + self.skipTest("Phase-13 golden fixture is generated by the local PostgreSQL gate") + payload = json.loads(GOLDEN_PATH.read_text(encoding="utf-8")) + self.assertEqual(payload["fixture_version"], "1") + self.assertEqual(len(payload["source_sha256"]), 64) + required = { + "overview_totals", "country_totals", "programme_totals", + "donor_rankings", "recipient_rankings", "organisation_details", + "registry_search", "sankey", "currency_statuses", "score_components", + } + self.assertTrue(required.issubset(payload["semantic_contract"])) + self.assertEqual( + set(payload["non_database_journeys"]), + {"news", "pipeline_status", "manual_refresh_permissions"}, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/transition/__init__.py b/src/transition/__init__.py new file mode 100644 index 0000000..26652f5 --- /dev/null +++ b/src/transition/__init__.py @@ -0,0 +1,21 @@ +"""Temporary PostgreSQL cutover and shadow-comparison controls.""" + +from transition.runtime import RuntimeMode, TransitionSettings, load_transition_settings +from transition.shadow import ( + ComparisonPolicy, + ComparisonResult, + ShadowComparisonCoordinator, + ShadowRequest, + compare_payloads, +) + +__all__ = [ + "ComparisonPolicy", + "ComparisonResult", + "RuntimeMode", + "ShadowComparisonCoordinator", + "ShadowRequest", + "TransitionSettings", + "compare_payloads", + "load_transition_settings", +] diff --git a/src/transition/runtime.py b/src/transition/runtime.py new file mode 100644 index 0000000..164053c --- /dev/null +++ b/src/transition/runtime.py @@ -0,0 +1,109 @@ +"""Fail-closed configuration for the temporary storage transition.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +import json +import os +from pathlib import Path +from typing import Mapping, Optional + + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +DEFAULT_CONFIG_PATH = PROJECT_ROOT / "config" / "runtime-transition.json" + + +class RuntimeMode(str, Enum): + POSTGRESQL = "postgresql" + SQLITE_MIGRATION_SOURCE = "sqlite_migration_source" + SHADOW_COMPARE = "shadow_compare" + + +class TransitionConfigurationError(RuntimeError): + """Raised when a transition mode could serve from an unsafe source.""" + + +@dataclass(frozen=True) +class TransitionSettings: + mode: RuntimeMode + app_environment: str + migration_source_path: Path + shadow_sqlite_path: Optional[Path] + maximum_pending_comparisons: int + maximum_response_bytes: int + shadow_timeout_seconds: float + maximum_recorded_differences: int + approved_unordered_paths: tuple[str, ...] + ignored_operational_paths: tuple[str, ...] + journeys: tuple[str, ...] + + @property + def postgresql_authoritative(self) -> bool: + return self.mode in {RuntimeMode.POSTGRESQL, RuntimeMode.SHADOW_COMPARE} + + @property + def shadow_enabled(self) -> bool: + return self.mode is RuntimeMode.SHADOW_COMPARE + + def validate(self) -> None: + if self.app_environment in {"staging", "production"} and ( + self.mode is RuntimeMode.SQLITE_MIGRATION_SOURCE + ): + raise TransitionConfigurationError( + "SQLite migration-source mode is forbidden in staging/production" + ) + if self.shadow_enabled: + if self.shadow_sqlite_path is None: + raise TransitionConfigurationError( + "SHADOW_SQLITE_PATH is required for shadow comparison" + ) + if not self.shadow_sqlite_path.is_file(): + raise TransitionConfigurationError("Shadow SQLite snapshot is unavailable") + if self.shadow_sqlite_path.resolve() == self.migration_source_path.resolve(): + raise TransitionConfigurationError( + "Shadow comparison requires a separate coherent SQLite snapshot" + ) + if self.maximum_pending_comparisons < 1: + raise TransitionConfigurationError("Shadow queue bound must be positive") + if self.maximum_response_bytes < 1024: + raise TransitionConfigurationError("Shadow response bound is too small") + if self.shadow_timeout_seconds <= 0: + raise TransitionConfigurationError("Shadow timeout must be positive") + if self.maximum_recorded_differences < 1: + raise TransitionConfigurationError("Difference bound must be positive") + + +def load_transition_settings( + environ: Optional[Mapping[str, str]] = None, + path: Path = DEFAULT_CONFIG_PATH, +) -> TransitionSettings: + env = os.environ if environ is None else environ + payload = json.loads(path.read_text(encoding="utf-8")) + if payload.get("configuration_version") != "1": + raise TransitionConfigurationError("Unsupported transition configuration version") + app_environment = env.get("APP_ENV", "development").strip().lower() + implicit_mode = str(payload["default_operational_mode"]) + try: + mode = RuntimeMode(env.get("DATA_RUNTIME_MODE", implicit_mode).strip().lower()) + except ValueError as exc: + raise TransitionConfigurationError("Unsupported DATA_RUNTIME_MODE") from exc + shadow_path_value = env.get("SHADOW_SQLITE_PATH") + shadow = payload["shadow"] + settings = TransitionSettings( + mode=mode, + app_environment=app_environment, + migration_source_path=Path( + env.get("DB_PATH", str(PROJECT_ROOT / "src/data/charities.db")) + ), + shadow_sqlite_path=Path(shadow_path_value) if shadow_path_value else None, + maximum_pending_comparisons=int(shadow["maximum_pending_comparisons"]), + maximum_response_bytes=int(shadow["maximum_response_bytes"]), + shadow_timeout_seconds=float(shadow["timeout_seconds"]), + maximum_recorded_differences=int(shadow["maximum_recorded_differences"]), + approved_unordered_paths=tuple(shadow["approved_unordered_paths"]), + ignored_operational_paths=tuple(shadow["ignored_operational_paths"]), + journeys=tuple(payload["journeys"]), + ) + settings.validate() + return settings diff --git a/src/transition/shadow.py b/src/transition/shadow.py new file mode 100644 index 0000000..dae96c5 --- /dev/null +++ b/src/transition/shadow.py @@ -0,0 +1,358 @@ +"""Bounded asynchronous shadow reads with privacy-safe difference evidence.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import asdict, dataclass, is_dataclass +from datetime import date, datetime +from decimal import Decimal +import hashlib +import json +import time +from typing import Any, Optional, Protocol + + +@dataclass(frozen=True) +class ShadowRequest: + journey: str + method: str + path: str + query_string: str + primary_payload: Any + request_id: str = "unknown" + + +@dataclass(frozen=True) +class Difference: + path: str + kind: str + primary_fingerprint: str + shadow_fingerprint: str + + +@dataclass(frozen=True) +class ComparisonResult: + journey: str + request_id: str + status: str + difference_count: int + differences: tuple[Difference, ...] + shadow_duration_ms: float + primary_authoritative: bool = True + + +@dataclass(frozen=True) +class ComparisonPolicy: + unordered_paths: frozenset[str] = frozenset() + ignored_paths: frozenset[str] = frozenset() + maximum_differences: int = 100 + + +class ShadowReader(Protocol): + async def read(self, request: ShadowRequest) -> Any: ... + + +class EvidenceSink(Protocol): + def record(self, result: ComparisonResult) -> None: ... + + +def _json_value(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return asdict(value) + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, Mapping): + return {str(key): _json_value(item) for key, item in value.items()} + if isinstance(value, Sequence) and not isinstance(value, (str, bytes, bytearray)): + return [_json_value(item) for item in value] + if isinstance(value, Decimal): + return format(value, "f") + if isinstance(value, (date, datetime)): + return value.isoformat() + return value + + +def _fingerprint(value: Any) -> str: + encoded = json.dumps( + _json_value(value), sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] + + +def _canonical(value: Any, path: str, policy: ComparisonPolicy) -> Any: + value = _json_value(value) + if isinstance(value, dict): + return { + key: _canonical(item, f"{path}.{key}", policy) + for key, item in sorted(value.items()) + if f"{path}.{key}" not in policy.ignored_paths + } + if isinstance(value, list): + canonical = [ + _canonical(item, f"{path}[{index}]", policy) + for index, item in enumerate(value) + ] + if path in policy.unordered_paths: + canonical.sort(key=lambda item: json.dumps(item, sort_keys=True, default=str)) + return canonical + if isinstance(value, float): + return format(Decimal(str(value)), "f") + return value + + +def compare_payloads( + primary: Any, + shadow: Any, + *, + journey: str, + request_id: str = "unknown", + policy: ComparisonPolicy = ComparisonPolicy(), + shadow_duration_ms: float = 0.0, +) -> ComparisonResult: + left = _canonical(primary, "$", policy) + right = _canonical(shadow, "$", policy) + differences: list[Difference] = [] + + def walk(primary_value: Any, shadow_value: Any, path: str) -> None: + if len(differences) >= policy.maximum_differences: + return + if type(primary_value) is not type(shadow_value): + differences.append( + Difference(path, "type", _fingerprint(primary_value), _fingerprint(shadow_value)) + ) + return + if isinstance(primary_value, dict): + all_keys = sorted(set(primary_value) | set(shadow_value)) + for key in all_keys: + child_path = f"{path}.{key}" + if key not in primary_value: + differences.append( + Difference(child_path, "missing_primary", "absent", _fingerprint(shadow_value[key])) + ) + elif key not in shadow_value: + differences.append( + Difference(child_path, "missing_shadow", _fingerprint(primary_value[key]), "absent") + ) + else: + walk(primary_value[key], shadow_value[key], child_path) + if len(differences) >= policy.maximum_differences: + return + return + if isinstance(primary_value, list): + if len(primary_value) != len(shadow_value): + differences.append( + Difference(path, "length", _fingerprint(len(primary_value)), _fingerprint(len(shadow_value))) + ) + for index, (primary_item, shadow_item) in enumerate( + zip(primary_value, shadow_value) + ): + walk(primary_item, shadow_item, f"{path}[{index}]") + if len(differences) >= policy.maximum_differences: + return + return + if primary_value != shadow_value: + differences.append( + Difference(path, "value", _fingerprint(primary_value), _fingerprint(shadow_value)) + ) + + walk(left, right, "$") + return ComparisonResult( + journey=journey, + request_id=request_id, + status="match" if not differences else "difference", + difference_count=len(differences), + differences=tuple(differences), + shadow_duration_ms=round(shadow_duration_ms, 3), + ) + + +class StructuredLogEvidenceSink: + """Record paths and fingerprints without serializing potentially sensitive values.""" + + def __init__(self, emit: Callable[[dict[str, Any]], None]): + self._emit = emit + + def record(self, result: ComparisonResult) -> None: + self._emit( + { + "event": "shadow_comparison", + "journey": result.journey, + "request_id": result.request_id, + "status": result.status, + "difference_count": result.difference_count, + "shadow_duration_ms": result.shadow_duration_ms, + "primary_authoritative": True, + "differences": [asdict(item) for item in result.differences], + } + ) + + +class ShadowComparisonCoordinator: + """Schedule shadow work without awaiting it on the primary response path.""" + + def __init__( + self, + reader: ShadowReader, + sink: EvidenceSink, + *, + policy: ComparisonPolicy, + timeout_seconds: float, + maximum_pending: int, + ): + self.reader = reader + self.sink = sink + self.policy = policy + self.timeout_seconds = timeout_seconds + self.maximum_pending = maximum_pending + self._tasks: set[asyncio.Task[None]] = set() + + @property + def pending(self) -> int: + return len(self._tasks) + + def submit(self, request: ShadowRequest) -> bool: + if len(self._tasks) >= self.maximum_pending: + self.sink.record( + ComparisonResult( + journey=request.journey, + request_id=request.request_id, + status="dropped_queue_full", + difference_count=0, + differences=(), + shadow_duration_ms=0.0, + ) + ) + return False + task = asyncio.create_task(self._execute(request)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return True + + async def _execute(self, request: ShadowRequest) -> None: + started = time.perf_counter() + try: + shadow = await asyncio.wait_for( + self.reader.read(request), timeout=self.timeout_seconds + ) + duration = (time.perf_counter() - started) * 1000 + result = compare_payloads( + request.primary_payload, + shadow, + journey=request.journey, + request_id=request.request_id, + policy=self.policy, + shadow_duration_ms=duration, + ) + except asyncio.TimeoutError: + result = ComparisonResult( + request.journey, + request.request_id, + "shadow_timeout", + 0, + (), + (time.perf_counter() - started) * 1000, + ) + except Exception as exc: + result = ComparisonResult( + request.journey, + request.request_id, + f"shadow_error_{exc.__class__.__name__}", + 0, + (), + (time.perf_counter() - started) * 1000, + ) + self.sink.record(result) + + async def drain(self) -> None: + tasks = tuple(self._tasks) + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + + +class ShadowComparisonMiddleware: + """Pass response bytes through immediately and compare only after the final body.""" + + def __init__( + self, + app: Any, + *, + coordinator: Optional[ShadowComparisonCoordinator], + maximum_response_bytes: int, + journey_resolver: Callable[[str, str, str], Optional[str]], + ): + self.app = app + self.coordinator = coordinator + self.maximum_response_bytes = maximum_response_bytes + self.journey_resolver = journey_resolver + + async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None: + if scope.get("type") != "http" or self.coordinator is None: + await self.app(scope, receive, send) + return + method = str(scope.get("method", "GET")).upper() + path = str(scope.get("path", "")) + query_string = scope.get("query_string", b"").decode("utf-8") + journey = self.journey_resolver(method, path, query_string) + if journey is None: + await self.app(scope, receive, send) + return + coordinator = self.coordinator + if coordinator is None: + await self.app(scope, receive, send) + return + response_status = 0 + content_type = "" + response_request_id = "unknown" + body = bytearray() + oversized = False + + async def send_wrapper(message: dict[str, Any]) -> None: + nonlocal response_status, content_type, response_request_id, oversized + if message["type"] == "http.response.start": + response_status = int(message["status"]) + headers = { + key.decode("latin-1").lower(): value.decode("latin-1") + for key, value in message.get("headers", []) + } + content_type = headers.get("content-type", "") + response_request_id = headers.get("x-request-id", "unknown") + elif message["type"] == "http.response.body" and not oversized: + chunk = message.get("body", b"") + if len(body) + len(chunk) <= self.maximum_response_bytes: + body.extend(chunk) + else: + oversized = True + body.clear() + await send(message) + if ( + message["type"] == "http.response.body" + and not message.get("more_body", False) + and response_status == 200 + and not oversized + and "application/json" in content_type + ): + try: + payload = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError): + return + headers = { + key.decode("latin-1").lower(): value.decode("latin-1") + for key, value in scope.get("headers", []) + } + coordinator.submit( + ShadowRequest( + journey=journey, + method=method, + path=path, + query_string=query_string, + primary_payload=payload, + request_id=( + response_request_id + if response_request_id != "unknown" + else headers.get("x-request-id", "unknown") + ), + ) + ) + + await self.app(scope, receive, send_wrapper) diff --git a/src/transition/sqlite_source.py b/src/transition/sqlite_source.py new file mode 100644 index 0000000..cfddf4d --- /dev/null +++ b/src/transition/sqlite_source.py @@ -0,0 +1,293 @@ +"""SQLite shadow adapter used only with an explicit, separate snapshot copy.""" + +from __future__ import annotations + +import asyncio +import re +from typing import Any, Optional +from urllib.parse import parse_qs, unquote + +from transition.shadow import ShadowRequest + + +def _first(query: dict[str, list[str]], key: str, default: Optional[str] = None) -> Optional[str]: + values = query.get(key) + return values[-1] if values else default + + +def _integer(query: dict[str, list[str]], key: str, default: int) -> int: + return int(_first(query, key, str(default)) or default) + + +def _number(query: dict[str, list[str]], key: str) -> Optional[float]: + value = _first(query, key) + return float(value) if value not in {None, ""} else None + + +def _boolean(query: dict[str, list[str]], key: str) -> Optional[bool]: + value = _first(query, key) + if value is None: + return None + return value.strip().lower() in {"1", "true", "yes", "on"} + + +def _split(query: dict[str, list[str]], key: str) -> Optional[list[str]]: + value = _first(query, key) + if value is None: + return None + return [item.strip() for item in value.split(",") if item.strip()] + + +class SQLiteShadowReader: + """Execute legacy reads off the event loop; never return them to the user.""" + + def __init__(self, snapshot_path: str): + # Lazy import preserves the PostgreSQL-only runtime import boundary. + from bff.repositories import SQLiteCharityRepository + + self.repository = SQLiteCharityRepository(snapshot_path) + + async def _call(self, method: Any, *args: Any, **kwargs: Any) -> Any: + return await asyncio.to_thread(lambda: asyncio.run(method(*args, **kwargs))) + + async def read(self, request: ShadowRequest) -> Any: + if request.method != "GET": + raise ValueError("Only read-only GET journeys can use the SQLite shadow adapter") + query = parse_qs(request.query_string, keep_blank_values=True) + path = request.path + repository = self.repository + + if path == "/api/charities": + return await self._call( + repository.get_all, + search=_first(query, "search"), + reg_status=_first(query, "reg_status"), + tag=_first(query, "tag"), + region=_first(query, "region"), + size=_first(query, "size"), + tags=_split(query, "tags"), + foundation_regions=_split(query, "foundation_regions"), + funding_regions=_split(query, "funding_regions"), + sources=_split(query, "sources"), + min_annual_giving=_number(query, "min_annual_giving"), + max_annual_giving=_number(query, "max_annual_giving"), + min_avg_grant_size=_number(query, "min_avg_grant_size"), + max_avg_grant_size=_number(query, "max_avg_grant_size"), + include_score=bool(_boolean(query, "include_score")), + sort=_first(query, "sort", "name_asc") or "name_asc", + skip=_integer(query, "skip", 0), + limit=_integer(query, "limit", 20), + ) + if path == "/api/charities/stats": + return await self._call(repository.get_stats) + if path == "/api/charities/grants/beneficiary-geographies": + return await self._call( + repository.get_beneficiary_geography_options, + sources=_split(query, "sources") or ["360Giving"], + ) + if path == "/api/charities/directory/organizations": + return await self._call( + repository.get_registry_page, + query=_first(query, "query"), + charity_number=_first(query, "charity_number"), + status=_first(query, "status"), + income_min=_number(query, "income_min"), + income_max=_number(query, "income_max"), + expenditure_min=_number(query, "expenditure_min"), + expenditure_max=_number(query, "expenditure_max"), + country=_first(query, "country"), + region=_first(query, "region"), + beneficiary_geography=_first(query, "beneficiary_geography"), + has_enriched_profile=_boolean(query, "has_enriched_profile"), + has_grant_data=_boolean(query, "has_grant_data"), + cursor=_first(query, "cursor"), + limit=_integer(query, "limit", 50), + sort=_first(query, "sort", "name") or "name", + ) + registry_match = re.fullmatch(r"/api/charities/directory/organizations/([^/]+)", path) + if registry_match: + return await self._call( + repository.get_registry_detail, unquote(registry_match.group(1)) + ) + if path == "/api/charities/grants/map": + return await self._call( + repository.get_grants_map, + currency=_first(query, "currency"), + min_coverage=_number(query, "min_coverage") or 0.30, + search=_first(query, "search"), + tags=_split(query, "tags"), + foundation_regions=_split(query, "foundation_regions"), + funding_regions=_split(query, "funding_regions"), + min_annual_giving=_number(query, "min_annual_giving"), + min_avg_grant_size=_number(query, "min_avg_grant_size"), + ) + if path == "/api/charities/grants/map/connections": + result = await self._call( + repository.get_grants_map, + currency=_first(query, "currency"), + min_coverage=0.0, + ) + limit = min(_integer(query, "limit", 100), 250) + connections = result.get("connections", [])[:limit] + return { + "status": "available" if connections else "no_transactions_found", + "connections": connections, + "connection_grant_count": sum( + int(item.get("grant_count", 0)) for item in connections + ), + "selected_currency": result.get("selected_currency") or "EUR", + "limit": limit, + "metadata": { + "data_mode": "sqlite_shadow_snapshot", + "loading_mode": "lazy_bounded_secondary_request", + }, + } + if path in { + "/api/charities/grants/overview", + "/api/charities/grants/overview/trends", + }: + filters = { + "currency": _first(query, "currency"), + "date_from": _first(query, "date_from"), + "date_to": _first(query, "date_to"), + "beneficiary_geographies": _split(query, "beneficiary_geographies"), + "programme_areas": _split(query, "programme_areas"), + "donor": _first(query, "donor"), + "recipient": _first(query, "recipient"), + "sources": _split(query, "sources"), + "granularity": _first(query, "granularity", "auto") or "auto", + } + method = ( + repository.get_grant_overview + if path.endswith("/overview") + else repository.get_grant_overview_trends + ) + if path.endswith("/overview"): + filters["include_connections"] = bool( + _boolean(query, "include_connections") + ) + return await self._call(method, **filters) + if path == "/api/charities/grants/overview/entity-suggestions": + return await self._call( + repository.get_grant_entity_suggestions, + sources=_split(query, "sources"), + limit=_integer(query, "limit", 2500), + ) + if path == "/api/charities/grants/overview/drilldown": + return await self._call( + repository.get_grant_overview_drilldown, + selection_type=_first(query, "selection_type") or "", + selection_value=_first(query, "selection_value") or "", + currency=_first(query, "currency"), + date_from=_first(query, "date_from"), + date_to=_first(query, "date_to"), + beneficiary_geographies=_split(query, "beneficiary_geographies"), + programme_areas=_split(query, "programme_areas"), + donor=_first(query, "donor"), + recipient=_first(query, "recipient"), + sources=_split(query, "sources"), + ) + if path == "/api/charities/grants/funders": + return await self._call( + repository.get_source_funders, + beneficiary_country=(_first(query, "beneficiary_country") or "").upper(), + currency=_first(query, "currency"), + date_from=_first(query, "date_from"), + date_to=_first(query, "date_to"), + beneficiary_geographies=_split(query, "beneficiary_geographies"), + programme_areas=_split(query, "programme_areas"), + donor=_first(query, "donor"), + recipient=_first(query, "recipient"), + sources=_split(query, "sources"), + search=_first(query, "search"), + profile_status=_first(query, "profile_status", "all") or "all", + sort=_first(query, "sort", "largest_observed_funding") or "largest_observed_funding", + page=_integer(query, "page", 1), + page_size=_integer(query, "page_size", 25), + ) + funder_match = re.fullmatch(r"/api/charities/grants/funders/([^/]+)", path) + if funder_match: + return await self._call( + repository.get_source_funder_detail, + unquote(funder_match.group(1)), + beneficiary_country=(_first(query, "beneficiary_country") or "").upper(), + currency=_first(query, "currency"), + date_from=_first(query, "date_from"), + date_to=_first(query, "date_to"), + beneficiary_geographies=_split(query, "beneficiary_geographies"), + programme_areas=_split(query, "programme_areas"), + donor=_first(query, "donor"), + recipient=_first(query, "recipient"), + sources=_split(query, "sources"), + detail_level=_first(query, "detail_level", "full") or "full", + ) + if path == "/api/charities/grants/summary": + return await self._call(repository.get_grant_summary) + if path == "/api/charities/grants/trends": + return await self._call( + repository.get_grant_trends, + currency=_first(query, "currency"), + months=_integer(query, "months", 24), + ) + if path == "/api/charities/grants/themes": + return await self._call( + repository.get_grant_themes, currency=_first(query, "currency") + ) + profile_match = re.fullmatch(r"/api/charities/(-?\d+)", path) + if profile_match: + return await self._call(repository.get_by_id, int(profile_match.group(1))) + grants_match = re.fullmatch(r"/api/charities/(-?\d+)/grants", path) + if grants_match: + return await self._call( + repository.get_grants_for_charity, + int(grants_match.group(1)), + role=_first(query, "role", "all") or "all", + ) + sankey_match = re.fullmatch(r"/api/charities/(-?\d+)/sankey", path) + if sankey_match: + return await self._call( + repository.get_sankey_data, + int(sankey_match.group(1)), + currency=_first(query, "currency"), + limit=_integer(query, "limit", 30), + ) + raise ValueError(f"No SQLite shadow adapter for {path}") + + +def resolve_shadow_journey(method: str, path: str, query_string: str = "") -> Optional[str]: + if method != "GET" or not path.startswith("/api/charities"): + return None + query = parse_qs(query_string, keep_blank_values=True) + if path == "/api/charities/grants/overview": + if _first(query, "date_from") or _first(query, "date_to"): + return "date_filters" + if _first(query, "beneficiary_geographies"): + return "country_filters" + if _first(query, "programme_areas"): + return "programme_filters" + if _first(query, "donor"): + return "donor_filters" + if _first(query, "recipient"): + return "recipient_filters" + return "dashboard" + if path == "/api/charities/grants/map": + return "map" + if path == "/api/charities/grants/map/connections": + return "map_relationships" + if path == "/api/charities/grants/overview/trends": + return "monthly_trends" if _first(query, "granularity") == "monthly" else "yearly_trends" + if path == "/api/charities/grants/overview/drilldown": + return "drill_down" + if path == "/api/charities/grants/funders": + return "donor_ranking" + if path == "/api/charities/directory/organizations": + return "registry_search" + if re.fullmatch(r"/api/charities/(-?\d+)/grants", path): + return "grant_list" + if re.fullmatch(r"/api/charities/(-?\d+)/sankey", path): + return "sankey" + if re.fullmatch(r"/api/charities/(-?\d+)", path) or path.startswith( + "/api/charities/directory/organizations/" + ): + return "profile_detail" + return "dashboard"