From f3652bde4ae4e979e7a04986f3f427196b600718 Mon Sep 17 00:00:00 2001 From: GrabowMar <44323258+GrabowMar@users.noreply.github.com> Date: Mon, 26 Oct 2026 17:52:14 +0000 Subject: [PATCH 1/4] feat: analyzer runs rework, dynamic scanners, model insights and settings/experiments UI - analysis: run batches, heartbeats and per-tool timing; tool modes, technologies and file patterns; dynamic scanners (zap, nuclei, wapiti, trivy, osv-scanner, retire, djlint) with parsers; coverage service; tool-reported severities - llm_models: enrichment and insights services with periodic tasks - users: activity tracking; settings and profile sections rebuilt - frontend: analysis run/finding explorer, experiment study tabs, model detail tabs, composer section nav, user guide docs Co-Authored-By: Claude Opus 5.5 --- .envs/.local/.django.example | 10 +- .envs/.production/.django.example | 38 +- .envs/.production/.frontend.example | 3 +- .github/copilot-instructions.md | 23 +- README.md | 128 +-- backend/analysis/api/schema.py | 190 ++++- backend/analysis/api/views/runs.py | 691 ++++++++++++++-- backend/analysis/api/views/tools.py | 19 + backend/analysis/apps.py | 6 +- backend/analysis/data/tools/bandit.yaml | 14 +- backend/analysis/data/tools/catalog.yaml | 63 +- backend/analysis/data/tools/codespell.yaml | 4 +- backend/analysis/data/tools/depcheck.yaml | 3 + backend/analysis/data/tools/deptry.yaml | 3 + .../analysis/data/tools/detect-secrets.yaml | 2 + backend/analysis/data/tools/djlint.yaml | 47 ++ backend/analysis/data/tools/eslint-react.yaml | 3 + .../analysis/data/tools/eslint-security.yaml | 3 + backend/analysis/data/tools/eslint.yaml | 18 +- backend/analysis/data/tools/flake8.yaml | 3 + backend/analysis/data/tools/gitleaks.yaml | 14 +- backend/analysis/data/tools/hadolint.yaml | 11 +- backend/analysis/data/tools/htmlhint.yaml | 3 + backend/analysis/data/tools/jscpd.yaml | 7 +- backend/analysis/data/tools/lizard.yaml | 3 + backend/analysis/data/tools/llm_review.yaml | 17 +- backend/analysis/data/tools/mypy.yaml | 14 +- backend/analysis/data/tools/npm-audit.yaml | 22 +- backend/analysis/data/tools/nuclei.yaml | 44 + backend/analysis/data/tools/osv-scanner.yaml | 38 + backend/analysis/data/tools/perflint.yaml | 3 + backend/analysis/data/tools/pip-audit.yaml | 13 +- backend/analysis/data/tools/pylint.yaml | 21 +- backend/analysis/data/tools/radon.yaml | 3 + backend/analysis/data/tools/retire.yaml | 28 + backend/analysis/data/tools/ruff.yaml | 9 +- backend/analysis/data/tools/semgrep.yaml | 3 + backend/analysis/data/tools/shellcheck.yaml | 3 + backend/analysis/data/tools/stylelint.yaml | 3 + backend/analysis/data/tools/trivy.yaml | 46 ++ backend/analysis/data/tools/vulture.yaml | 8 +- backend/analysis/data/tools/wapiti.yaml | 51 ++ backend/analysis/data/tools/zap.yaml | 65 ++ .../0006_tool_mode_and_technologies.py | 28 + .../migrations/0007_tool_file_patterns.py | 18 + .../0008_analysisrun_heartbeat_at.py | 17 + ...009_run_batch_tool_timing_tool_severity.py | 58 ++ backend/analysis/models.py | 91 ++- backend/analysis/seeding.py | 18 +- backend/analysis/services/ai_runner.py | 70 +- backend/analysis/services/base.py | 12 + backend/analysis/services/coverage.py | 122 +++ backend/analysis/services/extra_parsers.py | 209 +++-- backend/analysis/services/parsers.py | 407 +++++++++- backend/analysis/services/runner.py | 641 +++++++++++++-- backend/analysis/services/scanner_parsers.py | 612 ++++++++++++++ backend/analysis/services/serialize.py | 24 + backend/analysis/services/tool_tester.py | 17 +- .../analysis/services/workspace_service.py | 2 + backend/analysis/tests/test_ai_runner.py | 33 + backend/analysis/tests/test_api.py | 63 ++ backend/analysis/tests/test_coverage.py | 112 +++ backend/analysis/tests/test_dynamic_tools.py | 174 ++++ backend/analysis/tests/test_extra_parsers.py | 289 ++++++- backend/analysis/tests/test_parsers.py | 322 +++++++- backend/analysis/tests/test_run_lifecycle.py | 175 ++++ backend/analysis/tests/test_runner.py | 278 ++++++- backend/analysis/tests/test_runs_api.py | 273 +++++++ .../analysis/tests/test_scanner_parsers.py | 627 +++++++++++++++ backend/analysis/tests/test_seed.py | 47 ++ backend/automation/tasks.py | 15 +- backend/credentials/services/resolver.py | 2 +- backend/docs/services.py | 4 + backend/export/services/review_sample.py | 2 + backend/generation/api/views/templates.py | 11 +- .../migrations/0021_job_help_texts.py | 36 + backend/generation/models.py | 7 +- backend/generation/services/agent/runner.py | 2 +- backend/generation/services/data_lint.py | 17 + backend/generation/services/experiments.py | 53 +- backend/generation/services/job_cloning.py | 5 +- .../generation/services/openrouter_client.py | 2 +- backend/generation/services/orchestrator.py | 24 +- .../generation/services/packages/importers.py | 20 +- .../generation/services/profile_resolver.py | 53 +- .../tests/test_experiment_service.py | 23 + .../tests/test_experiments_v2_api.py | 19 + .../tests/test_multifile_and_robustness.py | 21 + backend/generation/tests/test_post_run.py | 12 + backend/generation/tests/test_profile_api.py | 11 +- .../tests/test_template_packages.py | 88 +- backend/integrations/mcp/tools.py | 2 +- backend/integrations/mcp/views.py | 2 +- backend/integrations/services/github.py | 2 +- backend/llm_models/api/views.py | 54 +- backend/llm_models/management/__init__.py | 0 .../management/commands/__init__.py | 0 .../management/commands/enrich_models.py | 23 + backend/llm_models/services/enrichment.py | 327 ++++++++ backend/llm_models/services/insights.py | 458 +++++++++++ backend/llm_models/services/sync.py | 23 +- backend/llm_models/tasks.py | 13 + backend/llm_models/tests/test_api.py | 19 +- backend/llm_models/tests/test_enrichment.py | 206 +++++ backend/rankings/services/aggregator.py | 55 +- backend/rankings/tests/test_services.py | 18 + backend/realtime/authz.py | 18 +- backend/realtime/events.py | 2 +- backend/realtime/tests/test_events.py | 4 +- backend/realtime/tests/test_sse_async.py | 14 +- backend/reports/services/generators.py | 3 +- backend/reports/services/loc.py | 72 +- backend/reports/services/renderers.py | 15 +- backend/reports/services/study_report.py | 133 +++- backend/reports/tests/test_services.py | 16 + backend/reports/tests/test_study_report.py | 61 ++ backend/runtime/api/views.py | 67 +- backend/runtime/proxy.py | 47 +- backend/runtime/services/container_service.py | 17 + backend/runtime/services/docker_manager.py | 29 +- backend/runtime/services/smoke.py | 12 +- backend/runtime/tests/test_api.py | 268 ++++++- backend/runtime/tests/test_smoke.py | 13 + backend/statistics/api/views.py | 7 +- .../commands/export_study_tables.py | 17 +- backend/statistics/services/aggregations.py | 3 +- backend/statistics/services/dedup.py | 63 +- backend/statistics/services/study.py | 125 ++- backend/statistics/tests/test_dedup.py | 112 +++ backend/statistics/tests/test_services.py | 21 +- backend/statistics/tests/test_study.py | 57 +- backend/system/api/views.py | 6 +- backend/system/services/maintenance.py | 6 +- backend/system/services/metrics.py | 12 +- backend/system/tests/test_system.py | 38 +- backend/users/activity.py | 87 ++ backend/users/api/schema.py | 26 + backend/users/api/views.py | 8 + backend/users/tests/api/test_views.py | 38 + .../traefik/dynamic/static.yml.template | 5 +- compose/production/traefik/entrypoint.sh | 11 +- config/settings/local.py | 4 + config/settings/production.py | 86 +- config/urls.py | 3 + docker-compose.local.yml | 3 +- docker-compose.production.yml | 6 + docs/ANALYSIS_PIPELINE.md | 146 +++- docs/ANALYZER_GUIDE.md | 201 ++++- docs/ARCHITECTURE.md | 153 ++-- docs/AUTOMATION_GUIDE.md | 19 +- docs/BACKGROUND_SERVICES.md | 106 ++- docs/EXECUTION.md | 280 +++---- docs/EXPERIMENTS.md | 126 ++- docs/GENERATION_PROCESS.md | 381 +++++---- docs/INTEGRATIONS.md | 127 ++- docs/LOCAL_MODELS.md | 255 +++--- docs/MODELS_REFERENCE.md | 63 +- docs/QUICKSTART.md | 51 +- docs/README.md | 41 +- docs/STUDY_TABLES.md | 174 ++-- docs/TEMPLATE_PACKAGES.md | 21 +- docs/TROUBLESHOOTING.md | 118 ++- docs/USER_GUIDE.md | 449 +++++++++++ docs/api-reference.md | 93 ++- docs/deployment-guide.md | 116 ++- docs/development-guide.md | 168 +++- frontend/DESIGN_SYSTEM.md | 174 +++- frontend/README.md | 76 +- frontend/src/app.css | 214 ++++- frontend/src/lib/api/analysis.ts | 332 -------- frontend/src/lib/api/analyzers.ts | 42 + frontend/src/lib/api/client.ts | 1 - frontend/src/lib/api/experiments.ts | 7 +- frontend/src/lib/api/models.ts | 206 ++++- frontend/src/lib/api/runs.ts | 381 +++++++-- frontend/src/lib/api/runtime.ts | 3 + frontend/src/lib/api/sse.ts | 2 +- frontend/src/lib/api/users.ts | 30 + frontend/src/lib/components/AppFooter.svelte | 20 +- frontend/src/lib/components/AppHeader.svelte | 53 +- frontend/src/lib/components/EmptyState.svelte | 2 +- frontend/src/lib/components/FilterBar.svelte | 24 +- frontend/src/lib/components/PageHeader.svelte | 76 +- .../src/lib/components/SectionHeading.svelte | 47 ++ .../analysis/AnalysisTargetForm.svelte | 183 ----- .../lib/components/analysis/AppPicker.svelte | 280 +++++++ .../analysis/CodeFilesEditor.svelte | 89 +++ .../components/analysis/FindingDetail.svelte | 251 ++++++ .../analysis/FindingsExplorer.svelte | 527 ++++++++++++ .../lib/components/analysis/LevelBadge.svelte | 35 + .../components/analysis/MetricsPanel.svelte | 132 +++ .../analysis/RunFindingsCell.svelte | 32 + .../components/analysis/RunLiveStrip.svelte | 79 ++ .../components/analysis/RunOverview.svelte | 260 ++++++ .../components/analysis/RunStatusCell.svelte | 45 ++ .../analysis/ToolOutputDialog.svelte | 143 ++++ .../lib/components/analysis/ToolPicker.svelte | 175 ++++ .../components/analysis/ToolPipeline.svelte | 134 ++++ .../lib/components/analysis/analysis.test.ts | 274 +++++++ .../lib/components/analysis/finding-data.ts | 306 +++++++ .../src/lib/components/analysis/format.ts | 273 +++++++ .../lib/components/analysis/live.svelte.ts | 156 ++++ frontend/src/lib/components/analysis/tools.ts | 62 ++ .../components/analyzers/CoveragePanel.svelte | 122 +++ .../analyzers/ToolConfigDialog.svelte | 7 +- .../lib/components/analyzers/ToolIcon.svelte | 16 +- .../applications/ApplicationAnalyses.svelte | 159 ++-- .../applications/ApplicationHeader.svelte | 55 +- .../components/applications/JobCards.svelte | 5 +- .../applications/JobProvenance.svelte | 5 +- .../components/applications/JobsTable.svelte | 312 ++++---- .../components/dashboard/ActivityFeed.svelte | 51 +- .../dashboard/AnalyzerHealthGrid.svelte | 45 +- .../lib/components/dashboard/HeroPanel.svelte | 187 +++-- .../components/dashboard/QuickActions.svelte | 90 --- .../components/dashboard/SeverityPanel.svelte | 12 +- .../components/dashboard/TrendPanel.svelte | 23 +- .../src/lib/components/docs/DocsNav.svelte | 73 ++ .../experiments/CapacityStrip.svelte | 29 +- .../experiments/ComparabilityTable.svelte | 15 +- .../experiments/ComposerSectionNav.svelte | 131 +++ .../experiments/ExperimentComposer.svelte | 37 + .../ExperimentDesignSummary.svelte | 100 --- .../experiments/ExperimentRow.svelte | 14 +- .../components/experiments/Leaderboard.svelte | 218 ++--- .../experiments/PairedDeltas.svelte | 230 ++++-- .../components/experiments/PlanRail.svelte | 13 +- .../components/experiments/PostRunCard.svelte | 7 +- .../experiments/PostRunEditor.svelte | 27 +- .../experiments/ScheduleEditor.svelte | 8 +- .../experiments/SchedulesCard.svelte | 8 +- .../src/lib/components/experiments/format.ts | 7 + .../src/lib/components/experiments/index.ts | 1 - .../experiments/monitor/ActivityFeed.svelte | 4 +- .../experiments/monitor/ConditionGrid.svelte | 27 +- .../experiments/monitor/EndpointLane.svelte | 28 +- .../experiments/monitor/FailureSummary.svelte | 120 +++ .../experiments/monitor/LiveStrip.svelte | 66 ++ .../experiments/study/PassRatePlot.svelte | 168 ++++ .../experiments/study/StudyFigure.svelte | 52 ++ .../experiments/study/StudyMasthead.svelte | 159 ++++ .../experiments/study/StudyRecord.svelte | 135 ++++ .../experiments/study/StudySection.svelte | 32 + .../experiments/study/findings.test.ts | 254 ++++++ .../components/experiments/study/findings.ts | 305 +++++++ .../experiments/study/results.svelte.ts | 69 ++ .../lib/components/experiments/study/tabs.ts | 36 + .../experiments/tabs/AutomationTab.svelte | 92 --- .../experiments/tabs/DataTab.svelte | 354 ++++++++ .../experiments/tabs/DesignTab.svelte | 291 ------- .../experiments/tabs/MethodTab.svelte | 335 ++++++++ .../experiments/tabs/OperationsTab.svelte | 327 ++++++++ .../experiments/tabs/OverviewTab.svelte | 219 ----- .../experiments/tabs/ResultsTab.svelte | 240 +++--- .../experiments/tabs/RunsTab.svelte | 300 ------- .../experiments/tabs/SummaryTab.svelte | 288 +++++++ .../components/generator/StackSelect.svelte | 2 +- .../generator/batches/BatchJobsTable.svelte | 11 +- .../generator/batches/ResultsMatrix.svelte | 4 +- .../src/lib/components/generator/catalog.ts | 19 +- .../generator/composer/AgentComposer.svelte | 96 ++- .../composer/AgentPreviewPanel.svelte | 46 +- .../generator/composer/ComposerSection.svelte | 100 ++- .../generator/composer/RunFormula.svelte | 80 ++ .../composer/SamplingSettings.svelte | 5 +- .../composer/ScaffoldingComposer.svelte | 60 +- .../composer/SimpleSummaryPanel.svelte | 23 +- .../composer/SinglePromptComposer.svelte | 53 +- .../composer/SummaryChecklist.svelte | 2 +- .../generator/composer/section-summary.ts | 18 + .../components/inference/CapacityCard.svelte | 229 +++--- .../components/inference/EndpointForm.svelte | 11 +- .../inference/tests/catalog.test.ts | 19 + .../integrations/GitHubExportDialog.svelte | 4 +- .../integrations/GitHubSection.svelte | 173 ++-- .../components/integrations/McpSection.svelte | 312 ++++---- .../integrations/ProvidersSection.svelte | 169 ++-- .../integrations/WebhooksSection.svelte | 207 ++--- .../src/lib/components/integrations/index.ts | 2 +- .../models/ModelBenchmarksTab.svelte | 173 ++++ .../models/ModelCapabilitiesTab.svelte | 217 +++++ .../models/ModelComparisonChart.svelte | 120 ++- .../lib/components/models/ModelDataTab.svelte | 78 ++ .../lib/components/models/ModelHeader.svelte | 82 +- .../components/models/ModelOverviewTab.svelte | 281 +++++++ .../components/models/ModelPricingTab.svelte | 263 ++++++ .../components/models/ModelResultsTab.svelte | 342 ++++++++ .../components/models/ModelStatsTabs.svelte | 609 -------------- .../models/ModelUsageSection.svelte | 48 +- .../components/models/PercentileMeter.svelte | 40 + frontend/src/lib/components/models/helpers.ts | 272 ++++--- frontend/src/lib/components/table/index.ts | 10 +- .../components/templates/BlocksSection.svelte | 1 - .../components/templates/LibraryNav.svelte | 7 +- .../components/templates/LibraryPane.svelte | 66 +- .../components/templates/PresetList.svelte | 11 +- .../templates/ProfilesSection.svelte | 1 - .../components/templates/SpecEditor.svelte | 24 +- .../lib/components/templates/SpecList.svelte | 184 +++-- .../components/templates/SpecsSection.svelte | 8 +- .../lib/components/ui/switch/switch.svelte | 3 +- .../users-profile/ProfileHero.svelte | 73 +- .../users-profile/ProfileStats.svelte | 39 - .../lib/components/users-profile/initials.ts | 10 + .../users-settings/ApiKeysSection.svelte | 64 +- .../users-settings/AppearanceSection.svelte | 181 ++--- .../CookieSettingsSection.svelte | 82 -- .../users-settings/CopyField.svelte | 38 + .../users-settings/CredentialCard.svelte | 226 ------ .../users-settings/CredentialRow.svelte | 234 ++++++ .../DataManagementSection.svelte | 122 --- .../users-settings/PrivacyDataSection.svelte | 146 ++++ .../users-settings/ProfileSection.svelte | 168 ++-- .../users-settings/SecuritySection.svelte | 160 ++-- .../users-settings/SettingsBlock.svelte | 45 ++ .../users-settings/SettingsRow.svelte | 54 ++ .../users-settings/TokenSection.svelte | 533 ++++++------- .../users-settings/WorkspaceSection.svelte | 111 +-- .../users-settings/section-status.svelte.ts | 30 + frontend/src/lib/constants/analysis.ts | 18 - frontend/src/lib/docs/categories.ts | 2 +- frontend/src/lib/nav.ts | 8 +- frontend/src/lib/settings-sections.ts | 160 ++++ frontend/src/lib/stores/auth.svelte.ts | 6 + frontend/src/lib/stores/page-title.svelte.ts | 3 + frontend/src/lib/stores/preferences.svelte.ts | 265 ++---- frontend/src/lib/utils/dashboard.ts | 8 - frontend/src/lib/utils/scroll-hint.ts | 64 ++ frontend/src/routes/(app)/+page.svelte | 101 +-- frontend/src/routes/(app)/about/+page.svelte | 95 +-- .../src/routes/(app)/analysis/+page.svelte | 753 ++++++++++-------- .../routes/(app)/analysis/[id]/+page.svelte | 462 +++++++++++ .../(app)/analysis/[taskId]/+page.svelte | 401 ---------- .../routes/(app)/analysis/create/+page.svelte | 724 ++++++++--------- .../src/routes/(app)/analyzers/+page.svelte | 453 ++++++++--- .../routes/(app)/applications/+page.svelte | 397 +++++---- .../(app)/applications/[id]/+page.svelte | 90 ++- .../applications/[id]/failure/+page.svelte | 4 +- frontend/src/routes/(app)/docs/+layout.svelte | 187 ++--- frontend/src/routes/(app)/docs/+page.svelte | 183 ++--- .../routes/(app)/docs/[...slug]/+page.svelte | 20 +- .../src/routes/(app)/endpoints/+page.svelte | 3 +- .../routes/(app)/endpoints/[id]/+page.svelte | 500 ++++++------ .../routes/(app)/endpoints/new/+page.svelte | 44 +- .../src/routes/(app)/experiments/+page.svelte | 143 ++-- .../(app)/experiments/[id]/+page.svelte | 301 ++++--- .../routes/(app)/experiments/[id]/+page.ts | 31 +- .../routes/(app)/experiments/new/+page.svelte | 11 +- .../routes/(app)/integrations/+page.svelte | 51 -- .../src/routes/(app)/integrations/+page.ts | 14 + frontend/src/routes/(app)/models/+page.svelte | 36 +- .../routes/(app)/models/[slug]/+page.svelte | 194 +++-- .../routes/(app)/models/compare/+page.svelte | 85 +- .../routes/(app)/models/import/+page.svelte | 15 +- .../src/routes/(app)/privacy/+page.svelte | 375 +++------ .../src/routes/(app)/rankings/+page.svelte | 162 ++-- .../src/routes/(app)/reports/+page.svelte | 109 +-- .../routes/(app)/reports/[id]/+page.svelte | 31 +- .../routes/(app)/reports/create/+page.svelte | 6 +- .../src/routes/(app)/runtime/+page.svelte | 160 ++-- .../routes/(app)/runtime/[id]/+page.svelte | 182 ++--- .../(app)/sample-generator/+page.svelte | 38 +- .../sample-generator/batches/+page.svelte | 257 +++--- .../batches/[id]/+page.svelte | 153 ++-- .../src/routes/(app)/statistics/+page.svelte | 84 +- frontend/src/routes/(app)/system/+page.svelte | 87 +- .../src/routes/(app)/templates/+page.svelte | 2 +- .../src/routes/(app)/users/[pk]/+page.svelte | 253 ++++-- .../routes/(app)/users/settings/+page.svelte | 538 +++++++------ frontend/src/routes/(auth)/+layout.svelte | 2 +- frontend/vite.config.ts | 63 +- relay/README.md | 56 +- 372 files changed, 27814 insertions(+), 11102 deletions(-) create mode 100644 backend/analysis/data/tools/djlint.yaml create mode 100644 backend/analysis/data/tools/nuclei.yaml create mode 100644 backend/analysis/data/tools/osv-scanner.yaml create mode 100644 backend/analysis/data/tools/retire.yaml create mode 100644 backend/analysis/data/tools/trivy.yaml create mode 100644 backend/analysis/data/tools/wapiti.yaml create mode 100644 backend/analysis/data/tools/zap.yaml create mode 100644 backend/analysis/migrations/0006_tool_mode_and_technologies.py create mode 100644 backend/analysis/migrations/0007_tool_file_patterns.py create mode 100644 backend/analysis/migrations/0008_analysisrun_heartbeat_at.py create mode 100644 backend/analysis/migrations/0009_run_batch_tool_timing_tool_severity.py create mode 100644 backend/analysis/services/coverage.py create mode 100644 backend/analysis/services/scanner_parsers.py create mode 100644 backend/analysis/services/serialize.py create mode 100644 backend/analysis/tests/test_coverage.py create mode 100644 backend/analysis/tests/test_dynamic_tools.py create mode 100644 backend/analysis/tests/test_run_lifecycle.py create mode 100644 backend/analysis/tests/test_runs_api.py create mode 100644 backend/analysis/tests/test_scanner_parsers.py create mode 100644 backend/generation/migrations/0021_job_help_texts.py create mode 100644 backend/llm_models/management/__init__.py create mode 100644 backend/llm_models/management/commands/__init__.py create mode 100644 backend/llm_models/management/commands/enrich_models.py create mode 100644 backend/llm_models/services/enrichment.py create mode 100644 backend/llm_models/services/insights.py create mode 100644 backend/llm_models/tasks.py create mode 100644 backend/llm_models/tests/test_enrichment.py create mode 100644 backend/users/activity.py create mode 100644 docs/USER_GUIDE.md delete mode 100644 frontend/src/lib/api/analysis.ts create mode 100644 frontend/src/lib/components/SectionHeading.svelte delete mode 100644 frontend/src/lib/components/analysis/AnalysisTargetForm.svelte create mode 100644 frontend/src/lib/components/analysis/AppPicker.svelte create mode 100644 frontend/src/lib/components/analysis/CodeFilesEditor.svelte create mode 100644 frontend/src/lib/components/analysis/FindingDetail.svelte create mode 100644 frontend/src/lib/components/analysis/FindingsExplorer.svelte create mode 100644 frontend/src/lib/components/analysis/LevelBadge.svelte create mode 100644 frontend/src/lib/components/analysis/MetricsPanel.svelte create mode 100644 frontend/src/lib/components/analysis/RunFindingsCell.svelte create mode 100644 frontend/src/lib/components/analysis/RunLiveStrip.svelte create mode 100644 frontend/src/lib/components/analysis/RunOverview.svelte create mode 100644 frontend/src/lib/components/analysis/RunStatusCell.svelte create mode 100644 frontend/src/lib/components/analysis/ToolOutputDialog.svelte create mode 100644 frontend/src/lib/components/analysis/ToolPicker.svelte create mode 100644 frontend/src/lib/components/analysis/ToolPipeline.svelte create mode 100644 frontend/src/lib/components/analysis/analysis.test.ts create mode 100644 frontend/src/lib/components/analysis/finding-data.ts create mode 100644 frontend/src/lib/components/analysis/format.ts create mode 100644 frontend/src/lib/components/analysis/live.svelte.ts create mode 100644 frontend/src/lib/components/analysis/tools.ts create mode 100644 frontend/src/lib/components/analyzers/CoveragePanel.svelte delete mode 100644 frontend/src/lib/components/dashboard/QuickActions.svelte create mode 100644 frontend/src/lib/components/docs/DocsNav.svelte create mode 100644 frontend/src/lib/components/experiments/ComposerSectionNav.svelte delete mode 100644 frontend/src/lib/components/experiments/ExperimentDesignSummary.svelte create mode 100644 frontend/src/lib/components/experiments/monitor/FailureSummary.svelte create mode 100644 frontend/src/lib/components/experiments/monitor/LiveStrip.svelte create mode 100644 frontend/src/lib/components/experiments/study/PassRatePlot.svelte create mode 100644 frontend/src/lib/components/experiments/study/StudyFigure.svelte create mode 100644 frontend/src/lib/components/experiments/study/StudyMasthead.svelte create mode 100644 frontend/src/lib/components/experiments/study/StudyRecord.svelte create mode 100644 frontend/src/lib/components/experiments/study/StudySection.svelte create mode 100644 frontend/src/lib/components/experiments/study/findings.test.ts create mode 100644 frontend/src/lib/components/experiments/study/findings.ts create mode 100644 frontend/src/lib/components/experiments/study/results.svelte.ts create mode 100644 frontend/src/lib/components/experiments/study/tabs.ts delete mode 100644 frontend/src/lib/components/experiments/tabs/AutomationTab.svelte create mode 100644 frontend/src/lib/components/experiments/tabs/DataTab.svelte delete mode 100644 frontend/src/lib/components/experiments/tabs/DesignTab.svelte create mode 100644 frontend/src/lib/components/experiments/tabs/MethodTab.svelte create mode 100644 frontend/src/lib/components/experiments/tabs/OperationsTab.svelte delete mode 100644 frontend/src/lib/components/experiments/tabs/OverviewTab.svelte delete mode 100644 frontend/src/lib/components/experiments/tabs/RunsTab.svelte create mode 100644 frontend/src/lib/components/experiments/tabs/SummaryTab.svelte create mode 100644 frontend/src/lib/components/generator/composer/RunFormula.svelte create mode 100644 frontend/src/lib/components/generator/composer/section-summary.ts create mode 100644 frontend/src/lib/components/models/ModelBenchmarksTab.svelte create mode 100644 frontend/src/lib/components/models/ModelCapabilitiesTab.svelte create mode 100644 frontend/src/lib/components/models/ModelDataTab.svelte create mode 100644 frontend/src/lib/components/models/ModelOverviewTab.svelte create mode 100644 frontend/src/lib/components/models/ModelPricingTab.svelte create mode 100644 frontend/src/lib/components/models/ModelResultsTab.svelte delete mode 100644 frontend/src/lib/components/models/ModelStatsTabs.svelte create mode 100644 frontend/src/lib/components/models/PercentileMeter.svelte delete mode 100644 frontend/src/lib/components/users-profile/ProfileStats.svelte create mode 100644 frontend/src/lib/components/users-profile/initials.ts delete mode 100644 frontend/src/lib/components/users-settings/CookieSettingsSection.svelte create mode 100644 frontend/src/lib/components/users-settings/CopyField.svelte delete mode 100644 frontend/src/lib/components/users-settings/CredentialCard.svelte create mode 100644 frontend/src/lib/components/users-settings/CredentialRow.svelte delete mode 100644 frontend/src/lib/components/users-settings/DataManagementSection.svelte create mode 100644 frontend/src/lib/components/users-settings/PrivacyDataSection.svelte create mode 100644 frontend/src/lib/components/users-settings/SettingsBlock.svelte create mode 100644 frontend/src/lib/components/users-settings/SettingsRow.svelte create mode 100644 frontend/src/lib/components/users-settings/section-status.svelte.ts delete mode 100644 frontend/src/lib/constants/analysis.ts create mode 100644 frontend/src/lib/settings-sections.ts create mode 100644 frontend/src/lib/stores/page-title.svelte.ts create mode 100644 frontend/src/lib/utils/scroll-hint.ts create mode 100644 frontend/src/routes/(app)/analysis/[id]/+page.svelte delete mode 100644 frontend/src/routes/(app)/analysis/[taskId]/+page.svelte delete mode 100644 frontend/src/routes/(app)/integrations/+page.svelte diff --git a/.envs/.local/.django.example b/.envs/.local/.django.example index 0dc5b74a..b020e15c 100644 --- a/.envs/.local/.django.example +++ b/.envs/.local/.django.example @@ -20,7 +20,7 @@ CELERY_FLOWER_PASSWORD=changeme # OpenRouter — get a key at https://openrouter.ai/keys # ------------------------------------------------------------------------------ # Optional global fallback key. Each user can also configure their own key via -# Settings → API Access in the web UI (stored encrypted in the DB, recommended). +# Settings → API keys in the web UI (stored encrypted in the DB, recommended). # When a job runs, the user's personal key is used if set; otherwise we fall # back to this global key when OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK is True. OPENROUTER_API_KEY= @@ -36,10 +36,12 @@ FRONTEND_PUBLIC_ORIGIN=http://localhost:8000 # Sample-app subdomain routing (see /docs/deployment-guide). # Serve each generated app at https://llm-./ through a # wildcard edge route (*. → Django) instead of raw host ports. -# Needs wildcard DNS + a wildcard TLS cert at the edge, and . -# added to DJANGO_ALLOWED_HOSTS below. +# Needs wildcard DNS (*.) and edge TLS for it: a wildcard cert, or +# Caddy on_demand_tls with ask → /api/runtime/app-tls-check/. Django adds +# . to ALLOWED_HOSTS itself; the vite dev server needs the same +# APPS_DOMAIN in its environment to pass app hosts through to Django. # ------------------------------------------------------------------------------ -# APPS_DOMAIN=example.com +# APPS_DOMAIN=apps.example.com # APPS_SUBDOMAIN_PROXY=True # Without it, apps are served at /apps// by Django's path diff --git a/.envs/.production/.django.example b/.envs/.production/.django.example index 9374779c..198e8b47 100644 --- a/.envs/.production/.django.example +++ b/.envs/.production/.django.example @@ -10,24 +10,48 @@ DJANGO_DOMAIN=example.com # Secrets # ------------------------------------------------------------------------------ DJANGO_SECRET_KEY= -DJANGO_ADMIN_URL= +# Path of the Django admin, with a trailing slash. Any value works — Traefik's +# django router follows it — so pick something non-obvious. Empty = admin/. +DJANGO_ADMIN_URL=admin/ DJANGO_ALLOWED_HOSTS=example.com,www.example.com +# DJANGO_SETTINGS_MODULE is set to config.settings.production by +# docker-compose.production.yml — do not override it here. + +# CORS / CSRF — comma-separated full origins (scheme + host). The SvelteKit app +# is served from the same origin as the API, so this is normally just the site. +DJANGO_CORS_ALLOWED_ORIGINS=https://example.com,https://www.example.com +DJANGO_CSRF_TRUSTED_ORIGINS=https://example.com,https://www.example.com + # Security # ------------------------------------------------------------------------------ DJANGO_SECURE_SSL_REDIRECT=False # Email # ------------------------------------------------------------------------------ +# From address; defaults to "LLM Eval Lab ". +# DJANGO_DEFAULT_FROM_EMAIL=LLM Eval Lab +# DJANGO_SERVER_EMAIL= +# Option A — Mailgun (django-anymail). Used whenever MAILGUN_API_KEY is set. MAILGUN_API_KEY= -DJANGO_SERVER_EMAIL= +# Sending domain; defaults to DJANGO_DOMAIN. MAILGUN_DOMAIN= +# EU-region accounts: https://api.eu.mailgun.net/v3 +# MAILGUN_API_URL=https://api.mailgun.net/v3 +# Option B — SMTP (used when MAILGUN_API_KEY is empty). Unset = localhost:25, +# no auth, no TLS. Uncomment and fill in for a real relay: +# EMAIL_HOST=smtp.example.com +# EMAIL_PORT=587 +# EMAIL_HOST_USER= +# EMAIL_HOST_PASSWORD= +# EMAIL_USE_TLS=True +# EMAIL_USE_SSL=False -# AWS +# Sentry (optional) — error reporting is enabled only when SENTRY_DSN is set. # ------------------------------------------------------------------------------ -DJANGO_AWS_ACCESS_KEY_ID= -DJANGO_AWS_SECRET_ACCESS_KEY= -DJANGO_AWS_STORAGE_BUCKET_NAME= +SENTRY_DSN= +# SENTRY_ENVIRONMENT=production +# SENTRY_TRACES_SAMPLE_RATE=0.0 # django-allauth # ------------------------------------------------------------------------------ @@ -75,7 +99,7 @@ CELERY_FLOWER_PASSWORD= # OpenRouter — get a key at https://openrouter.ai/keys # ------------------------------------------------------------------------------ # Optional global fallback key. In production we recommend leaving this empty -# and requiring every user to add their own key via Settings → API Access +# and requiring every user to add their own key via Settings → API keys # (stored encrypted in the DB). Set OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=False # to enforce per-user keys. OPENROUTER_API_KEY= diff --git a/.envs/.production/.frontend.example b/.envs/.production/.frontend.example index 1c380392..36f1b9e4 100644 --- a/.envs/.production/.frontend.example +++ b/.envs/.production/.frontend.example @@ -1,7 +1,8 @@ # Production frontend (SvelteKit) environment. # ------------------------------------------------------------------------------ NODE_ENV=production -PORT=8080 +# Must stay 3000: the image EXPOSEs it and Traefik routes to frontend:3000. +PORT=3000 HOST=0.0.0.0 API_TARGET=http://django:8000 diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index bb73338a..4c685ed6 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -34,7 +34,8 @@ uv run python manage.py ```sh cd frontend npm install # install deps -npm run dev # dev server (port 8080) +npm run dev # dev server (in compose: container port 8080, published as http://localhost:8000) +npm test # vitest npm run check # svelte-check (type-check) npm run build # production build ``` @@ -56,7 +57,7 @@ just frontend-dev # start frontend dev server in container - `config/` — Django project config: settings (`base.py`, `local.py`, `production.py`, `test.py`), root URLconf, ASGI/WSGI, Celery app - `config/api.py` — Django Ninja API root. Registers routers from Django apps. -- `backend/` — Django apps directory (`APPS_DIR`). Apps: `users`, `llm_models`, `credentials`, `generation`, `runtime`, `analysis`, `reports`, `rankings`, `statistics`, `automation`, `realtime`, `tokens`, `export`, `docs`, `system`, `common`, `contrib` +- `backend/` — Django apps directory (`APPS_DIR`). Apps: `users`, `tokens`, `credentials`, `llm_models`, `generation`, `execution`, `inference`, `runtime`, `analysis`, `statistics`, `rankings`, `reports`, `export`, `integrations`, `realtime`, `docs`, `system`, `automation` (retired, router unmounted), plus `common` and `contrib`. See `docs/development-guide.md` - `tests/` — Project-level tests Settings are environment-specific (`config/settings/{base,local,production,test}.py`) and use `django-environ` for env var loading. @@ -66,7 +67,7 @@ Settings are environment-specific (`config/settings/{base,local,production,test} - `frontend/src/routes/` — SvelteKit file-based routing - `/(auth)/` — Login, signup, password reset, email verification, 2FA - `/(app)/` — Authenticated app pages (dashboard, models, analysis, etc.) -- Vite proxies `/api`, `/_allauth`, `/admin`, `/media` to the Django backend +- Vite (dev) and `src/hooks.server.ts` (dev and prod) proxy `/api`, `/_allauth`, `/admin`, `/media`, `/apps` to the Django backend ### API conventions (Django Ninja) @@ -84,7 +85,7 @@ def list_items(request): Schemas use `ninja.ModelSchema` (Pydantic-based). Custom computed fields use `@staticmethod resolve_()`. Routers are registered in `config/api.py`. -Authentication is `SessionAuth` globally. API docs are restricted to staff. +Authentication is Bearer API tokens or the session cookie on every router (a few exceptions are listed in `docs/api-reference.md`). API docs are restricted to staff. ### Testing conventions @@ -98,17 +99,16 @@ Authentication is `SessionAuth` globally. API docs are restricted to staff. The `realtime` app exposes `GET /api/realtime/stream?channels=` as a Server-Sent Events endpoint. Backend services publish events via `backend.realtime.events.publish(channel, event_dict)` which writes to Redis pub/sub. -Channel naming convention: -- `generation:` -- `analysis:` -- `runtime:` -- `dashboard` (broadcast) +Channels (authorized per user in `backend/realtime/authz.py`; full table in `docs/api-reference.md`): +- `generation:`, `experiment:`, `endpoint:` +- `analysis:`, `runtime:` +- `capacity` (aggregate executor capacity) Frontend subscription via `$lib/api/sse.ts`: ```ts import { subscribe } from '$lib/api/sse'; -const cleanup = subscribe(['generation:42', 'dashboard'], (e) => { +const cleanup = subscribe(['generation:42', 'capacity'], (e) => { console.log(e.type, e.data); }); // on component destroy: @@ -134,8 +134,7 @@ Custom `User` model with email as the login field (no username). Single `name` f ### Design system -- Colors: background `#0F172A`, primary `#1E293B`, CTA `#22C55E`, text `#F8FAFC` -- Fonts: headings Fira Code, body Fira Sans +- Colors, fonts and chart rules are tokens in `src/app.css` (light "paper" / dark "observatory"); never hard-code colors - Icons: **SVG only** (Heroicons/Lucide) — no emoji as icons - All clickable elements must have `cursor-pointer` and visible hover/focus states (150–300ms transition) - Breakpoints: 375 px, 768 px, 1024 px, 1440 px; respect `prefers-reduced-motion` diff --git a/README.md b/README.md index d9118ca3..7ca071b3 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@
-# LLM Eval Lab +# LLM Lab **Which model writes the best code for a given task?** -Generate one application per LLM, run each in a Docker sandbox, -and score the code with 14 analysis tools. +Have each LLM build the same application, run every app in a Docker sandbox, +grade it with a hidden test suite, and score the code with 33 analysis tools. [![CI](https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework/actions/workflows/ci.yml/badge.svg)](https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework/actions/workflows/ci.yml) [![CodeQL](https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework/actions/workflows/codeql.yml/badge.svg)](https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework/actions/workflows/codeql.yml) @@ -25,51 +25,68 @@ and score the code with 14 analysis tools. ## 💡 The idea Benchmarks tell you how a model scores on puzzles; they say little about -the quality of a full application it writes. Here the experiment is -end-to-end: you pick a requirement template and a set of models from -[OpenRouter][openrouter], and for each model the platform - -1. **generates** a complete application from the template - ([how generation works](docs/GENERATION_PROCESS.md)), -2. **runs it** in an isolated container with its own subdomain, so you can - click through the live app, -3. **analyzes** the code with the tools you select — ruff, bandit, semgrep, - eslint, mypy, pylint, gitleaks, detect-secrets, hadolint, codespell, - vulture, radon, jscpd, and an LLM code reviewer, -4. **aggregates** the findings and metrics into reports and per-model - rankings. - -The comparison rests on measured findings, not gut feeling. Everything runs -from the web UI, or unattended as [automation pipelines](docs/AUTOMATION_GUIDE.md) -composed in a node-based editor — schedule them or fan them out over a -parameter matrix. Multi-user support (email login, optional MFA, per-user -OpenRouter keys, API tokens) makes a shared instance practical. - -The stack: Django 6 + django-ninja + Celery + PostgreSQL + Redis on the -backend, SvelteKit (Svelte 5) on the frontend, everything in Docker. - -This began as a master's-thesis project and is maintained by one person — -expect sharp edges, and feel free to file issues. +the quality of a full application it builds. Here the experiment is +end-to-end: you pick an app spec and a set of models (from +[OpenRouter][openrouter], or your own Ollama / OpenAI-compatible server), +and for each model the platform + +1. **generates** the app. In the primary *agent* mode a coding-agent + harness (OpenCode, mini-swe-agent or Aider) works on a stack skeleton in + an isolated sandbox, with the model behind a budget-enforcing gateway; + template and single-prompt modes cover one-shot comparisons + ([generation](docs/GENERATION_PROCESS.md)), +2. **grades** it: the built app is deployed with no network and checked by + a hidden conformance suite the model never saw, +3. **runs it** in its own container so you can click through the live app, +4. **analyzes** it with the tools you install from the analyzer shop: 29 + static tools (ruff, bandit, semgrep, eslint, trivy, osv-scanner, + gitleaks, hadolint, …), an LLM code reviewer, and three DAST scanners + (OWASP ZAP, Nuclei, Wapiti) that attack the running app + ([analyzer guide](docs/ANALYZER_GUIDE.md)), +5. **compares** the results in statistics, rankings and study reports. + +Comparisons are run as **experiments**: apps × conditions (model, harness, +sampling preset, profile) × repeats, with a pre-registered baseline, a +reproducibility seed, a live monitor, cron schedules and post-run analysis +([experiments](docs/EXPERIMENTS.md)). A dedicated executor process runs every +job under leases and fair capacity lanes, so runs survive web restarts +([execution](docs/EXECUTION.md)). Study tooling covers deduplicated findings, +build-healing ablation, a manual-review sample and Markdown/LaTeX study +reports ([study tables](docs/STUDY_TABLES.md)). + +Outward, the platform exposes a REST API with personal tokens, an MCP server +for Claude, ChatGPT and other MCP clients, GitHub export and webhooks +([integrations](docs/INTEGRATIONS.md)). Multi-user support (email login, +optional MFA, per-user encrypted API keys) makes a shared instance practical. + +The stack: Django 6 + django-ninja + PostgreSQL + Redis (+ Celery for a few +background tasks) on the backend, SvelteKit (Svelte 5) on the frontend, +everything in Docker, with untrusted code confined to a Docker-in-Docker +sandbox. + +This began as a master's-thesis project and is maintained by one person. +Expect sharp edges, and feel free to file issues. ## 🚀 Quick start -You need Docker (with the Compose plugin), -[`just`](https://github.com/casey/just), and an -[OpenRouter API key](https://openrouter.ai/keys) — the key is only needed -for generation, browsing works without it. +You need Docker (with the Compose plugin and permission to run one +privileged container), [`just`](https://github.com/casey/just), Python 3 for +the bootstrap script, and an +[OpenRouter API key](https://openrouter.ai/keys) (or a local model server). +The key is only needed for generation; browsing works without it. ```bash git clone https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework.git cd AutoLLM_Code_Analyzer_rework -just bootstrap # .env files with generated secrets -just build && just up # build and start the stack -just manage seed_generation_templates # scaffolding templates & prompts -just manage createsuperuser +just bootstrap # .env files with generated secrets +just build && just up # build and start the stack ``` -Migrations run automatically when the django container starts. Open -, log in, and add your OpenRouter key in the UI (or -set `OPENROUTER_API_KEY` in `.envs/.local/.django` as a global fallback). +Migrations and seed data (analyzer tools, generation library, stacks, run +presets) are applied automatically when the django container starts. Open +: on an empty database the sign-up page creates the +initial admin account. Then add your OpenRouter key under +**Settings → API keys** and sync the model catalog from **Models**. | URL | Service | | -------------------------------- | ------------------------- | @@ -79,7 +96,8 @@ set `OPENROUTER_API_KEY` in `.envs/.local/.django` as a global fallback). | | Mailpit (captured emails) | | | Flower (Celery tasks) | -The full walkthrough is in [docs/QUICKSTART.md](docs/QUICKSTART.md). +The full walkthrough is in [docs/QUICKSTART.md](docs/QUICKSTART.md); the +[user guide](docs/USER_GUIDE.md) tours the UI page by page. ## 🌍 Deploy to a server @@ -87,9 +105,10 @@ The full walkthrough is in [docs/QUICKSTART.md](docs/QUICKSTART.md). bash <(curl -fsSL https://raw.githubusercontent.com/GrabowMar/AutoLLM_Code_Analyzer_rework/main/scripts/deploy.sh) ``` -The script handles first-time installs and in-place updates behind an -existing Caddy proxy; options are documented in the script header. For the -full production stack (Traefik with wildcard TLS, per-app subdomains), see +The script handles first-time installs and in-place updates of the local +stack behind an existing Caddy (or nginx) proxy; options are documented in +the script header. For the production compose stack (Traefik with wildcard +TLS, per-app subdomains) and its caveats, see [docs/deployment-guide.md](docs/deployment-guide.md). ## 🛠️ Development @@ -98,37 +117,38 @@ The `justfile` is the entry point for everything: ```bash just up / just down # start / stop the stack -just logs # tail container logs -just test # backend tests — pytest in Docker, same as CI +just logs [service] # tail container logs +just test [args] # backend tests: pytest in Docker, same as CI just manage # any manage.py command just build # rebuild images just prune # remove containers and volumes ``` -Linting and type checks: +Linting, type checks and frontend tests: ```bash uv run pre-commit run --all-files # ruff, djlint, prettier, and friends uv run mypy backend -cd frontend && npm run check # svelte-check +cd frontend && npm run check && npm test # svelte-check, vitest ``` How the code is laid out: ``` -backend/ Django apps, one per domain: generation, analysis, automation, - reports, rankings, runtime, users, ... +backend/ Django apps, one per domain: generation, execution, inference, + analysis, statistics, reports, rankings, runtime, integrations, ... config/ settings, root URLconf, Celery app, API root -frontend/ SvelteKit app — routes, components, API client +frontend/ SvelteKit app: routes, components, API client +relay/ the relay script that connects a GPU box's model server compose/ Dockerfiles for the local and production targets scripts/ bootstrap.py (env files), deploy.sh (server install) -docs/ deeper documentation — start at docs/README.md +docs/ deeper documentation, start at docs/README.md ``` Secrets live in `.envs/`; only `*.example` templates are committed, and `just bootstrap` derives the real files from them. -Docs under `docs/` are served live by the in-app viewer at `/docs` — the +Docs under `docs/` are served live by the in-app viewer at `/docs`. The [development guide](docs/development-guide.md) covers the workflow and the conventions for writing them. @@ -141,8 +161,8 @@ Bug reports, ideas, and PRs are welcome. The short version: prefixes: `feat:`, `fix:`, `chore:`, `docs:`, `refactor:`, `test:`. - Install the git hooks once — they run the same checks as CI: `uv sync && uv run pre-commit install`. -- Before pushing, run the three commands from - [Development](#-development) above (lint, `just test`, svelte-check); +- Before pushing, run the commands from + [Development](#-development) above (lint, `just test`, svelte-check and vitest); CI gates merging on them. - One logical change per PR; screenshots for UI changes. diff --git a/backend/analysis/api/schema.py b/backend/analysis/api/schema.py index a7ef6b1c..89973553 100644 --- a/backend/analysis/api/schema.py +++ b/backend/analysis/api/schema.py @@ -16,6 +16,8 @@ class AnalyzerToolSchema(Schema): category: str kind: str target_language: str + analysis_mode: str = "static" + technologies: list[str] = [] icon: str version: str docs_url: str @@ -31,6 +33,33 @@ class AnalyzerToolSchema(Schema): update_available: bool = False +class CatalogEntrySchema(Schema): + slug: str + label: str + description: str = "" + group: str = "" + + +class CatalogMetaSchema(Schema): + """Display vocabulary for the shop (from ``data/tools/catalog.yaml``).""" + + categories: list[CatalogEntrySchema] + modes: list[CatalogEntrySchema] + technology_groups: list[CatalogEntrySchema] + technologies: list[CatalogEntrySchema] + + +class AppCoverageSchema(Schema): + """How the user's generated apps map onto the catalog.""" + + apps_total: int + apps_considered: int + # {technology slug: number of apps containing it} + technologies: dict[str, int] + # {tool slug: number of apps containing at least one technology it scans} + tools: dict[str, int] + + # --- Workspace -------------------------------------------------------------- @@ -105,6 +134,76 @@ class RunCreateSchema(Schema): auto_start: bool = True +class BatchCreateSchema(Schema): + name: str = "" + tool_slugs: list[str] + generation_job_ids: list[str] + + +class BatchCreatedSchema(Schema): + batch_id: str + run_ids: list[str] + + +class RunPreviewRequestSchema(Schema): + tool_slugs: list[str] = [] + generation_job_ids: list[str] = [] + source_code: dict[str, str] | None = None + + +class PreviewTargetSchema(Schema): + job_id: str | None + label: str + files: int + lines: int + app_running: bool + + +class PreviewToolSchema(Schema): + slug: str + name: str + known: bool + installed: bool + install_status: str + mode: str + kind: str + # Targets the tool would run on / skip (nothing it reads; app not running). + runs: int + not_applicable: int + app_not_running: int + + +class RunPreviewSchema(Schema): + workspace_status: str + targets: list[PreviewTargetSchema] + tools: list[PreviewToolSchema] + tool_runs: int + warnings: list[str] + + +class RerunSchema(Schema): + name: str = "" + # Empty: all of the run's tools (or, with only_unmeasured, those that measured nothing). + tool_slugs: list[str] = [] + only_unmeasured: bool = False + + +class BulkActionSchema(Schema): + action: str + ids: list[str] + + +class BulkResultSchema(Schema): + affected: int + skipped: int + + +class RunStatsSchema(Schema): + total: int + active: int + by_status: dict[str, int] + + class ToolResultSchema(Schema): id: str tool_slug: str @@ -113,11 +212,23 @@ class ToolResultSchema(Schema): summary: dict[str, Any] metrics: dict[str, Any] = {} error_message: str + started_at: str | None = None + duration_seconds: float | None = None + + +class ToolResultDetailSchema(ToolResultSchema): + raw_output: dict[str, Any] + tool_name: str + tool_docs_url: str + findings_count: int class FindingSchema(Schema): id: str + # Normalized level (a heuristic for tools that rate nothing); display tool_severity. severity: str + # The level exactly as the tool printed it; "" when the tool rates nothing. + tool_severity: str = "" category: str confidence: str title: str @@ -129,20 +240,34 @@ class FindingSchema(Schema): code_snippet: str rule_id: str tool_slug: str + tool_specific_data: dict[str, Any] = {} -class AnalysisRunSchema(Schema): - id: str - name: str - status: str - tool_slugs: list[str] - summary: dict[str, Any] - error_message: str - generation_job_id: str | None - started_at: str | None - completed_at: str | None - created_at: str - results: list[ToolResultSchema] +class RunTargetSchema(Schema): + kind: str # "job" | "code" + label: str + job_id: str | None = None + job_status: str = "" + mode: str = "" + model_name: str = "" + model_id: str = "" + app_name: str = "" + app_slug: str = "" + stack_slug: str = "" + experiment_id: str | None = None + file_count: int | None = None + + +class RunProgressSchema(Schema): + total: int + done: int + running: list[str] + findings: int + + +class SourceFileSchema(Schema): + path: str + lines: int class AnalysisRunListSchema(Schema): @@ -151,7 +276,23 @@ class AnalysisRunListSchema(Schema): status: str tool_slugs: list[str] summary: dict[str, Any] + error_message: str = "" + generation_job_id: str | None = None + batch_id: str | None = None + target: RunTargetSchema created_at: str + started_at: str | None = None + completed_at: str | None = None + duration_seconds: float | None = None + # While running: waiting_for_workspace | preparing | running_tools. + phase: str = "" + # Active runs only. + progress: RunProgressSchema | None = None + + +class AnalysisRunSchema(AnalysisRunListSchema): + results: list[ToolResultSchema] + source_files: list[SourceFileSchema] = [] class PaginatedRunsSchema(Schema): @@ -168,3 +309,28 @@ class PaginatedFindingsSchema(Schema): page: int per_page: int pages: int + + +class FacetValueSchema(Schema): + value: str + count: int + label: str = "" + + +class LevelFacetSchema(Schema): + value: str + count: int + # 0 = most severe (the mapped rank of this label). + rank: int + + +class FindingFacetsSchema(Schema): + total: int + tools: list[FacetValueSchema] + levels: list[LevelFacetSchema] + unrated: int + categories: list[FacetValueSchema] + files: list[FacetValueSchema] + files_total: int + rules: list[FacetValueSchema] + rules_total: int diff --git a/backend/analysis/api/views/runs.py b/backend/analysis/api/views/runs.py index 57abb17b..a056cf17 100644 --- a/backend/analysis/api/views/runs.py +++ b/backend/analysis/api/views/runs.py @@ -1,42 +1,128 @@ -"""Analysis run endpoints (mounted under ``/analysis/``).""" +"""Analysis run endpoints (mounted under ``/analysis/``). + +Static ``runs//`` routes are registered before ``runs/{run_id}/``: +Django resolves the first matching path, so ``runs/{run_id}/`` declared first +would swallow ``runs/stats/`` and friends. +""" from __future__ import annotations +import contextlib +import uuid from typing import Any +from django.core.exceptions import ValidationError +from django.db.models import Case +from django.db.models import CharField +from django.db.models import Count +from django.db.models import F +from django.db.models import FloatField +from django.db.models import IntegerField +from django.db.models import Min +from django.db.models import Q +from django.db.models import Value +from django.db.models import When +from django.db.models.fields.json import KeyTextTransform +from django.db.models.functions import Cast +from django.db.models.functions import Lower from django.shortcuts import get_object_or_404 from ninja import Query from ninja.errors import HttpError from backend.analysis.api.schema import AnalysisRunSchema +from backend.analysis.api.schema import BatchCreatedSchema +from backend.analysis.api.schema import BatchCreateSchema +from backend.analysis.api.schema import BulkActionSchema +from backend.analysis.api.schema import BulkResultSchema +from backend.analysis.api.schema import FindingFacetsSchema from backend.analysis.api.schema import PaginatedFindingsSchema from backend.analysis.api.schema import PaginatedRunsSchema +from backend.analysis.api.schema import RerunSchema from backend.analysis.api.schema import RunCreateSchema +from backend.analysis.api.schema import RunPreviewRequestSchema +from backend.analysis.api.schema import RunPreviewSchema +from backend.analysis.api.schema import RunStatsSchema +from backend.analysis.api.schema import ToolResultDetailSchema from backend.analysis.api.views._router import router from backend.analysis.models import AnalysisRun +from backend.analysis.models import AnalyzerTool from backend.analysis.models import Finding +from backend.analysis.models import ToolResult from backend.analysis.services import runner from backend.analysis.services import workspace_service +from backend.analysis.services.serialize import iso +from backend.analysis.services.serialize import serialize_result from backend.common.pagination import paginate_queryset +from backend.generation.models import GenerationJob + +# A batch (and its preview) analyzes at most this many apps. +MAX_BATCH_APPS = 100 + +ACTIVE_STATUSES = (AnalysisRun.Status.PENDING, AnalysisRun.Status.RUNNING) + +_RUN_SORTS = { + "created_desc": ("-created_at",), + "created_asc": ("created_at",), + "name_asc": ("name", "-created_at"), + "duration_desc": (F("duration_seconds").desc(nulls_last=True), "-created_at"), + "findings_desc": (F("findings_total").desc(nulls_last=True), "-created_at"), +} +_SEVERITY_RANK = {"critical": 0, "high": 1, "medium": 2, "low": 3, "info": 4} +# Rated findings ordered by their (mapped) level, then those the tool did not rate. +_LEVEL_ORDER = Case( + When(tool_severity="", then=Value(9)), + *[When(severity=sev, then=Value(rank)) for sev, rank in _SEVERITY_RANK.items()], + default=Value(8), + output_field=IntegerField(), +) +_FINDING_SORTS = { + "severity": (_LEVEL_ORDER, "file_path", "line_number"), + "file": ("file_path", "line_number", _LEVEL_ORDER), + "tool": ("result__tool_slug", _LEVEL_ORDER, "file_path", "line_number"), + "rule": ("rule_id", "file_path", "line_number"), +} +# Facet lists longer than this are cut (the response says how many there were). +_FACET_LIMIT = 50 -def _iso(value) -> str | None: - return value.isoformat() if value else None +# --- Serialization ------------------------------------------------------------- -def serialize_result(result) -> dict[str, Any]: + +def _target(run: AnalysisRun) -> dict[str, Any]: + """What the run analyzed, for display: a generated app or pasted code.""" + job = run.generation_job if run.generation_job_id else None + if job is None: + files = run.source_code if isinstance(run.source_code, dict) else {} + if not files: # the analyzed app was deleted since (generation_job is SET_NULL) + return {"kind": "deleted", "label": "Deleted app"} + count = sum(1 for v in files.values() if isinstance(v, str) and v.strip()) + return { + "kind": "code", + "label": f"Pasted code ({count} file{'' if count == 1 else 's'})", + "file_count": count, + } + model = job.model if job.model_id else None + app = job.app_requirement if job.app_requirement_id else None + model_name = model.model_name if model else "" + app_name = app.name if app else "" + parts = [app_name or job.stack_slug or "App", model_name] return { - "id": str(result.id), - "tool_slug": result.tool_slug, - "category": result.category, - "status": result.status, - "summary": result.summary or {}, - "metrics": result.metrics or {}, - "error_message": result.error_message, + "kind": "job", + "label": " · ".join(p for p in parts if p), + "job_id": str(job.id), + "job_status": job.status, + "mode": job.mode, + "model_name": model_name, + "model_id": model.model_id if model else "", + "app_name": app_name, + "app_slug": app.slug if app else "", + "stack_slug": job.stack_slug or "", + "experiment_id": str(job.experiment_id) if job.experiment_id else None, } -def serialize_run(run: AnalysisRun) -> dict[str, Any]: +def _base(run: AnalysisRun) -> dict[str, Any]: return { "id": str(run.id), "name": run.name, @@ -45,41 +131,197 @@ def serialize_run(run: AnalysisRun) -> dict[str, Any]: "summary": run.summary or {}, "error_message": run.error_message, "generation_job_id": str(run.generation_job_id) if run.generation_job_id else None, - "started_at": _iso(run.started_at), - "completed_at": _iso(run.completed_at), - "created_at": _iso(run.created_at), - "results": [serialize_result(r) for r in run.results.all()], + "batch_id": str(run.batch_id) if run.batch_id else None, + "target": _target(run), + "created_at": iso(run.created_at), + "started_at": iso(run.started_at), + "completed_at": iso(run.completed_at), + "duration_seconds": run.duration_seconds, + "phase": (run.summary or {}).get("phase", "") if run.status == AnalysisRun.Status.RUNNING else "", } -def serialize_run_list(run: AnalysisRun) -> dict[str, Any]: +def _progress(runs: list[AnalysisRun]) -> dict[str, dict[str, Any]]: + """Tools done / running and findings so far for the active runs among *runs* (two queries).""" + active = [r for r in runs if r.status in ACTIVE_STATUSES] + if not active: + return {} + ids = [r.id for r in active] + progress: dict[str, dict[str, Any]] = { + str(r.id): {"total": len(r.tool_slugs or []), "done": 0, "running": [], "findings": 0} for r in active + } + for run_id, slug, status in ToolResult.objects.filter(run_id__in=ids).values_list("run_id", "tool_slug", "status"): + entry = progress[str(run_id)] + if status == ToolResult.Status.RUNNING: + entry["running"].append(slug) + elif status != ToolResult.Status.PENDING: + entry["done"] += 1 + for row in Finding.objects.filter(result__run_id__in=ids).values("result__run_id").annotate(n=Count("id")): + progress[str(row["result__run_id"])]["findings"] = row["n"] + return progress + + +def serialize_run_list(run: AnalysisRun, progress: dict[str, Any] | None = None) -> dict[str, Any]: + data = _base(run) + # Per-tool metrics (function lists, duplication blocks) belong to the detail view. + data["summary"] = {k: v for k, v in data["summary"].items() if k != "metrics_by_tool"} + return {**data, "progress": progress} + + +def serialize_run(run: AnalysisRun) -> dict[str, Any]: + data = _base(run) + data["progress"] = _progress([run]).get(str(run.id)) + data["results"] = [serialize_result(r) for r in run.results.all()] + # The files the tools see (pasted code, or the generated app's code), without contents. + files = runner._materialize(run.get_code_for_analysis()) # noqa: SLF001 + data["source_files"] = [{"path": path, "lines": content.count("\n") + 1} for path, content in files.items()] + return data + + +def _serialize_finding(f: Finding) -> dict[str, Any]: return { - "id": str(run.id), - "name": run.name, - "status": run.status, - "tool_slugs": run.tool_slugs or [], - "summary": run.summary or {}, - "created_at": _iso(run.created_at), + "id": str(f.id), + "severity": f.severity, + "tool_severity": f.tool_severity, + "category": f.category, + "confidence": f.confidence, + "title": f.title, + "description": f.description, + "suggestion": f.suggestion, + "file_path": f.file_path, + "line_number": f.line_number, + "column_number": f.column_number, + "code_snippet": f.code_snippet, + "rule_id": f.rule_id, + "tool_slug": f.result.tool_slug, + "tool_specific_data": f.tool_specific_data or {}, } +def _runs_qs(user): + return AnalysisRun.objects.filter(created_by=user).select_related( + "generation_job__model", + "generation_job__app_requirement", + ) + + +def _uuid(value: str, what: str = "id") -> uuid.UUID: + try: + return uuid.UUID(str(value)) + except ValueError as exc: + raise HttpError(400, f"Invalid {what}.") from exc + + +def _get_run(request, run_id: str, *, related: bool = True) -> AnalysisRun: + """The caller's run *run_id*; a malformed id is a 404 like an unknown one.""" + with contextlib.suppress(ValueError): + qs = _runs_qs(request.auth) if related else AnalysisRun.objects.filter(created_by=request.auth) + return get_object_or_404(qs, id=uuid.UUID(run_id)) + raise HttpError(404, "Not Found") + + +# --- Validation helpers -------------------------------------------------------- + + +def _validated_tools(slugs: list[str]) -> list[str]: + """*slugs* deduplicated in order; 400 when empty or naming unknown tools.""" + ordered = list(dict.fromkeys(s for s in slugs if s)) + if not ordered: + raise HttpError(400, "At least one tool must be selected.") + known = set(AnalyzerTool.objects.filter(slug__in=ordered).values_list("slug", flat=True)) + unknown = [s for s in ordered if s not in known] + if unknown: + raise HttpError(400, f"Unknown tool(s): {', '.join(unknown)}") + return ordered + + +def _accessible_jobs(user, job_ids: list[str]) -> list[GenerationJob]: + """The jobs *user* may analyze (their own; staff: any), in the given order; 404 if any is not.""" + ids = [] + for job_id in dict.fromkeys(job_ids): + try: + ids.append(uuid.UUID(str(job_id))) + except ValueError as exc: + raise HttpError(404, "Generation job not found.") from exc + jobs = GenerationJob.objects.all() if user.is_staff else GenerationJob.objects.filter(created_by=user) + found = {j.id: j for j in jobs.filter(id__in=ids).select_related("model", "app_requirement")} + if len(found) != len(ids): + raise HttpError(404, "Generation job not found.") + return [found[i] for i in ids] + + +def _require_job_access(user, job_id: str) -> None: + """404 unless *user* may analyze generation job *job_id* (their own; staff: any). + + Analyzing a job copies its code into the caller's workspace and stores the + findings under the caller, so another user's job must look nonexistent. + """ + jobs = GenerationJob.objects.all() if user.is_staff else GenerationJob.objects.filter(created_by=user) + try: + found = jobs.filter(id=job_id).exists() + except (ValidationError, ValueError): + found = False + if not found: + raise HttpError(404, "Generation job not found.") + + +def _default_name(job: GenerationJob | None) -> str: + if job is None: + return "Pasted code" + run = AnalysisRun(generation_job=job) + return _target(run)["label"] or "Analysis run" + + +# --- List, stats --------------------------------------------------------------- + + @router.get("/runs/", response=PaginatedRunsSchema) def list_runs( request, page: int = Query(1, ge=1), per_page: int = Query(20, ge=1, le=100), - status: str = Query(""), + status: str = Query("", description="Comma-separated statuses; 'active' = pending + running."), generation_job_id: str = Query(""), + batch_id: str = Query(""), + tool_slug: str = Query(""), + search: str = Query(""), + sort: str = Query("created_desc"), ): """List the current user's analysis runs.""" - qs = AnalysisRun.objects.filter(created_by=request.auth) + runner.reap_stale_runs() # runs whose executing thread died with a reload + qs = _runs_qs(request.auth) if status: - qs = qs.filter(status=status) + wanted = {s.strip() for s in status.split(",") if s.strip()} + if "active" in wanted: + wanted = (wanted - {"active"}) | set(ACTIVE_STATUSES) + qs = qs.filter(status__in=wanted) if generation_job_id: - qs = qs.filter(generation_job_id=generation_job_id) + qs = qs.filter(generation_job_id=_uuid(generation_job_id, "generation_job_id")) + if batch_id: + qs = qs.filter(batch_id=_uuid(batch_id, "batch_id")) + if tool_slug: + qs = qs.filter(tool_slugs__contains=[tool_slug]) + if search: + term = search.strip() + q = ( + Q(name__icontains=term) + | Q(generation_job__model__model_name__icontains=term) + | Q(generation_job__model__model_id__icontains=term) + | Q(generation_job__app_requirement__name__icontains=term) + | Q(generation_job__stack_slug__icontains=term) + ) + if len(term) >= 4 and all(c in "0123456789abcdef-" for c in term.lower()): + qs = qs.annotate(id_text=Cast("id", CharField()), job_text=Cast("generation_job_id", CharField())) + q |= Q(id_text__startswith=term.lower()) | Q(job_text__startswith=term.lower()) + qs = qs.filter(q) + if sort == "findings_desc": + qs = qs.annotate(findings_total=Cast(KeyTextTransform("total_findings", "summary"), FloatField())) + qs = qs.order_by(*_RUN_SORTS.get(sort, _RUN_SORTS["created_desc"])) page_qs, total, page, pages = paginate_queryset(qs, page, per_page) + runs = list(page_qs) + progress = _progress(runs) return { - "items": [serialize_run_list(r) for r in page_qs], + "items": [serialize_run_list(r, progress.get(str(r.id))) for r in runs], "total": total, "page": page, "per_page": per_page, @@ -87,6 +329,26 @@ def list_runs( } +@router.get("/runs/stats/", response=RunStatsSchema) +def run_stats(request): + """Run counts per status for the current user.""" + runner.reap_stale_runs() + by_status = dict( + AnalysisRun.objects.filter(created_by=request.auth) + .values_list("status") + .annotate(n=Count("id")) + .values_list("status", "n"), + ) + return { + "total": sum(by_status.values()), + "active": sum(by_status.get(s, 0) for s in ACTIVE_STATUSES), + "by_status": by_status, + } + + +# --- Create ---------------------------------------------------------------------- + + @router.post("/runs/", response=AnalysisRunSchema) def create_run(request, payload: RunCreateSchema): """Create an analysis run and (optionally) start it.""" @@ -94,85 +356,372 @@ def create_run(request, payload: RunCreateSchema): raise HttpError(400, "At least one tool must be selected.") if not payload.source_code and not payload.generation_job_id: raise HttpError(400, "Provide source_code or a generation_job_id.") + tool_slugs = _validated_tools(payload.tool_slugs) + + job = None + if payload.generation_job_id: + _require_job_access(request.auth, payload.generation_job_id) + job = GenerationJob.objects.select_related("model", "app_requirement").get(id=payload.generation_job_id) workspace = workspace_service.get_workspace(request.auth) run = AnalysisRun.objects.create( - name=payload.name or "Analysis run", + name=payload.name or _default_name(job), created_by=request.auth, workspace=workspace, - generation_job_id=payload.generation_job_id or None, - source_code=payload.source_code, - tool_slugs=payload.tool_slugs, + generation_job=job, + source_code=None if job else payload.source_code, + tool_slugs=tool_slugs, ) if payload.auto_start: runner.dispatch(str(run.id)) - return serialize_run(run) + return serialize_run(_get_run(request, str(run.id))) + + +@router.post("/runs/batch/", response=BatchCreatedSchema) +def create_batch(request, payload: BatchCreateSchema): + """One run per generated app, queued to execute one after another.""" + if not payload.generation_job_ids: + raise HttpError(400, "Select at least one app.") + if len(payload.generation_job_ids) > MAX_BATCH_APPS: + raise HttpError(400, f"At most {MAX_BATCH_APPS} apps per batch.") + tool_slugs = _validated_tools(payload.tool_slugs) + jobs = _accessible_jobs(request.auth, payload.generation_job_ids) + workspace = workspace_service.get_workspace(request.auth) + batch_id = uuid.uuid4() + runs = [ + AnalysisRun.objects.create( + name=payload.name or _default_name(job), + created_by=request.auth, + workspace=workspace, + generation_job=job, + tool_slugs=tool_slugs, + batch_id=batch_id, + ) + for job in jobs + ] + runner.dispatch_many([r.id for r in runs]) + return {"batch_id": str(batch_id), "run_ids": [str(r.id) for r in runs]} + + +@router.post("/runs/preview/", response=RunPreviewSchema) +def preview_runs(request, payload: RunPreviewRequestSchema): + """What a run over these apps / this code with these tools would do (free; nothing is created). + + Per tool: on how many targets it would run, and on how many it would be + skipped (no file it reads; app not running for dynamic tools). Per target: + file count and whether its app container is running. + """ + if len(payload.generation_job_ids) > MAX_BATCH_APPS: + raise HttpError(400, f"At most {MAX_BATCH_APPS} apps per batch.") + user = request.auth + tools = {t.slug: t for t in AnalyzerTool.objects.filter(slug__in=payload.tool_slugs)} + workspace = workspace_service.get_workspace(user) + installed = {it.tool.slug: it.status for it in workspace.installed_tools.select_related("tool")} + + targets: list[dict[str, Any]] = [] + file_maps: list[dict[str, str]] = [] + running_apps: list[bool] = [] + if payload.generation_job_ids: + jobs = _accessible_jobs(user, payload.generation_job_ids) + running = runner.running_app_containers(user, [j.id for j in jobs]) + for job in jobs: + files = runner._materialize(AnalysisRun(generation_job=job).get_code_for_analysis()) # noqa: SLF001 + app_running = str(job.id) in running + file_maps.append(files) + running_apps.append(app_running) + targets.append( + { + "job_id": str(job.id), + "label": _target(AnalysisRun(generation_job=job))["label"], + "files": len(files), + "lines": sum(c.count("\n") + 1 for c in files.values()), + "app_running": app_running, + }, + ) + elif payload.source_code: + files = runner._materialize(payload.source_code) # noqa: SLF001 + file_maps.append(files) + running_apps.append(False) + targets.append( + { + "job_id": None, + "label": "Pasted code", + "files": len(files), + "lines": sum(c.count("\n") + 1 for c in files.values()), + "app_running": False, + }, + ) + + tool_rows = [] + tool_runs = 0 + for slug in dict.fromkeys(payload.tool_slugs): + tool = tools.get(slug) + row = { + "slug": slug, + "name": tool.name if tool else slug, + "known": tool is not None, + "installed": installed.get(slug) == "installed", + "install_status": installed.get(slug, ""), + "mode": tool.analysis_mode if tool else "", + "kind": tool.kind if tool else "", + "runs": 0, + "not_applicable": 0, + "app_not_running": 0, + } + for files, app_running in zip(file_maps, running_apps, strict=True): + if tool is None: + continue + if tool.analysis_mode == "dynamic" and not app_running: + row["app_not_running"] += 1 + elif not runner.applicable_files(tool, files): + row["not_applicable"] += 1 + else: + row["runs"] += 1 + tool_runs += row["runs"] + tool_rows.append(row) + + warnings = [] + missing = [r["name"] for r in tool_rows if r["known"] and not r["installed"]] + if missing: + warnings.append(f"Not installed (they would fail): {', '.join(missing)}.") + unknown = [r["slug"] for r in tool_rows if not r["known"]] + if unknown: + warnings.append(f"Unknown tool(s): {', '.join(unknown)}.") + empty = [t["label"] for t in targets if t["files"] == 0] + if empty: + warnings.append(f"No code to analyze for: {', '.join(empty[:5])}{'…' if len(empty) > 5 else ''}.") + return { + "workspace_status": workspace.status, + "targets": targets, + "tools": tool_rows, + "tool_runs": tool_runs, + "warnings": warnings, + } + + +@router.post("/runs/bulk/", response=BulkResultSchema) +def bulk_action(request, payload: BulkActionSchema): + """Cancel or delete several runs. Active runs are never deleted (cancel them first).""" + if payload.action not in ("cancel", "delete"): + raise HttpError(400, "action must be 'cancel' or 'delete'.") + ids = [_uuid(i) for i in payload.ids[:500]] + runs = list(AnalysisRun.objects.filter(created_by=request.auth, id__in=ids).select_related("workspace__container")) + affected = 0 + if payload.action == "cancel": + for run in runs: + affected += int(runner.cancel(run)) + else: + deletable = [r.id for r in runs if r.status not in ACTIVE_STATUSES] + affected = len(deletable) + AnalysisRun.objects.filter(id__in=deletable).delete() + return {"affected": affected, "skipped": len(runs) - affected} + + +# --- One run -------------------------------------------------------------------- @router.get("/runs/{run_id}/", response=AnalysisRunSchema) def get_run(request, run_id: str): - run = get_object_or_404( - AnalysisRun.objects.prefetch_related("results"), - id=run_id, - created_by=request.auth, - ) - return serialize_run(run) + runner.reap_stale_runs() + return serialize_run(_get_run(request, run_id)) @router.post("/runs/{run_id}/cancel/", response=AnalysisRunSchema) def cancel_run(request, run_id: str): - run = get_object_or_404(AnalysisRun, id=run_id, created_by=request.auth) - if run.status in (AnalysisRun.Status.PENDING, AnalysisRun.Status.RUNNING): - run.status = AnalysisRun.Status.CANCELLED - run.save(update_fields=["status"]) - return serialize_run(run) + """Cancel a pending or running run: it stops before its next tool and the + in-flight tool's processes are killed; ``cancelled`` is final.""" + run = _get_run(request, run_id) + runner.cancel(run) + return serialize_run(_get_run(request, run_id)) + + +@router.post("/runs/{run_id}/rerun/", response=AnalysisRunSchema) +def rerun(request, run_id: str, payload: RerunSchema): + """Start a new run on the same code: all tools, the given ones, or only those that measured nothing.""" + source = _get_run(request, run_id) + if source.generation_job_id is None and not source.source_code: + raise HttpError(400, "This run's app was deleted; there is nothing to re-run.") + if payload.tool_slugs: + slugs = payload.tool_slugs + elif payload.only_unmeasured: + measured = { + r.tool_slug + for r in source.results.all() + if r.status == ToolResult.Status.COMPLETED + or (r.status == ToolResult.Status.SKIPPED and (r.summary or {}).get("skip_reason") == "not_applicable") + } + slugs = [s for s in source.tool_slugs if s not in measured] + if not slugs: + raise HttpError(400, "Every tool of this run measured the code; nothing to re-run.") + else: + slugs = source.tool_slugs + run = AnalysisRun.objects.create( + name=payload.name or source.name, + created_by=request.auth, + workspace=workspace_service.get_workspace(request.auth), + generation_job_id=source.generation_job_id, + source_code=source.source_code, + tool_slugs=_validated_tools(list(slugs)), + ) + runner.dispatch(str(run.id)) + return serialize_run(_get_run(request, str(run.id))) @router.delete("/runs/{run_id}/", response={200: dict}) def delete_run(request, run_id: str): - run = get_object_or_404(AnalysisRun, id=run_id, created_by=request.auth) + run = _get_run(request, run_id, related=False) + if run.status in ACTIVE_STATUSES: + raise HttpError(409, "The run is still active; cancel it first.") run.delete() return 200, {"status": "deleted"} +@router.get("/runs/{run_id}/results/{result_id}/", response=ToolResultDetailSchema) +def get_result(request, run_id: str, result_id: str): + """One tool's result with its raw output (command, exit code, stdout) for debugging.""" + run = _get_run(request, run_id, related=False) + try: + result = get_object_or_404(ToolResult, id=uuid.UUID(result_id), run=run) + except ValueError as exc: + raise HttpError(404, "Not Found") from exc + tool = AnalyzerTool.objects.filter(slug=result.tool_slug).first() + return { + **serialize_result(result), + "raw_output": result.raw_output or {}, + "tool_name": tool.name if tool else result.tool_slug, + "tool_docs_url": tool.docs_url if tool else "", + "findings_count": result.findings.count(), + } + + +# --- Findings --------------------------------------------------------------------- + + +def _findings_qs( + run: AnalysisRun, + *, + tool_slug: str = "", + tool_severity: str = "", + severity: str = "", + category: str = "", + file_path: str = "", + rule_id: str = "", + search: str = "", +): + qs = Finding.objects.filter(result__run=run).select_related("result") + if tool_slug: + qs = qs.filter(result__tool_slug=tool_slug) + if tool_severity == "unrated": + qs = qs.filter(tool_severity="") + elif tool_severity: + qs = qs.filter(tool_severity__iexact=tool_severity) + if severity: # normalized level (legacy / aggregate consumers) + qs = qs.filter(severity=severity) + if category: + qs = qs.filter(category=category) + if file_path: + qs = qs.filter(file_path=file_path) + if rule_id: + qs = qs.filter(rule_id=rule_id) + if search: + term = search.strip() + qs = qs.filter( + Q(title__icontains=term) + | Q(description__icontains=term) + | Q(file_path__icontains=term) + | Q(rule_id__icontains=term) + | Q(code_snippet__icontains=term), + ) + return qs + + @router.get("/runs/{run_id}/findings/", response=PaginatedFindingsSchema) def list_findings( request, run_id: str, page: int = Query(1, ge=1), per_page: int = Query(50, ge=1, le=200), - severity: str = Query(""), tool_slug: str = Query(""), + tool_severity: str = Query("", description="Level as the tool printed it (case-insensitive), or 'unrated'."), + severity: str = Query("", description="Normalized level (aggregate statistics); prefer tool_severity."), + category: str = Query(""), + file_path: str = Query(""), + rule_id: str = Query(""), + search: str = Query(""), + sort: str = Query("severity"), ): - run = get_object_or_404(AnalysisRun, id=run_id, created_by=request.auth) - qs = Finding.objects.filter(result__run=run).select_related("result") - if severity: - qs = qs.filter(severity=severity) - if tool_slug: - qs = qs.filter(result__tool_slug=tool_slug) + run = _get_run(request, run_id, related=False) + qs = _findings_qs( + run, + tool_slug=tool_slug, + tool_severity=tool_severity, + severity=severity, + category=category, + file_path=file_path, + rule_id=rule_id, + search=search, + ).order_by(*_FINDING_SORTS.get(sort, _FINDING_SORTS["severity"]), "id") page_qs, total, page, pages = paginate_queryset(qs, page, per_page) - items = [ - { - "id": str(f.id), - "severity": f.severity, - "category": f.category, - "confidence": f.confidence, - "title": f.title, - "description": f.description, - "suggestion": f.suggestion, - "file_path": f.file_path, - "line_number": f.line_number, - "column_number": f.column_number, - "code_snippet": f.code_snippet, - "rule_id": f.rule_id, - "tool_slug": f.result.tool_slug, - } - for f in page_qs - ] return { - "items": items, + "items": [_serialize_finding(f) for f in page_qs], "total": total, "page": page, "per_page": per_page, "pages": pages, } + + +@router.get("/runs/{run_id}/findings/facets/", response=FindingFacetsSchema) +def finding_facets(request, run_id: str, tool_slug: str = Query("")): + """Counts for the findings explorer: per tool, reported level, category, file and rule. + + Levels are the tools' own labels (case-folded), ordered by their mapped + rank; findings of tools that rate nothing are counted as ``unrated``. + With ``tool_slug`` every facet but ``tools`` is limited to that tool. + """ + run = _get_run(request, run_id, related=False) + every = Finding.objects.filter(result__run=run) + scoped = every.filter(result__tool_slug=tool_slug) if tool_slug else every + + def rows(qs, field, *, limit=None): + data = list(qs.values(field).annotate(n=Count("id")).order_by("-n", field)) + return [{"value": r[field] or "", "count": r["n"]} for r in (data[:limit] if limit else data)], len(data) + + tools, _ = rows(every, "result__tool_slug") + levels = list( + scoped.exclude(tool_severity="") + .annotate(label=Lower("tool_severity")) + .values("label") + .annotate( + n=Count("id"), + rank=Min(Case(*[When(severity=s, then=Value(r)) for s, r in _SEVERITY_RANK.items()], default=Value(5))), + ) + .order_by("rank", "-n"), + ) + categories, _ = rows(scoped.exclude(category=""), "category") + files, files_total = rows(scoped.exclude(file_path=""), "file_path", limit=_FACET_LIMIT) + rules, rules_total = rows(scoped.exclude(rule_id=""), "rule_id", limit=_FACET_LIMIT) + # One representative title per rule, so the rule facet is readable. + titles = dict( + scoped.filter(rule_id__in=[r["value"] for r in rules]) + .values("rule_id") + .annotate(t=Min("title")) + .values_list("rule_id", "t"), + ) + for r in rules: + r["label"] = titles.get(r["value"], "") + return { + "total": scoped.count(), + "tools": tools, + "levels": [{"value": r["label"], "count": r["n"], "rank": r["rank"]} for r in levels], + "unrated": scoped.filter(tool_severity="").count(), + "categories": categories, + "files": files, + "files_total": files_total, + "rules": rules, + "rules_total": rules_total, + } + + +# Re-exported for callers that serialized runs before the views were split. +__all__ = ["serialize_result", "serialize_run", "serialize_run_list"] diff --git a/backend/analysis/api/views/tools.py b/backend/analysis/api/views/tools.py index d11ef074..d486fb12 100644 --- a/backend/analysis/api/views/tools.py +++ b/backend/analysis/api/views/tools.py @@ -7,10 +7,14 @@ from django.shortcuts import get_object_or_404 from backend.analysis.api.schema import AnalyzerToolSchema +from backend.analysis.api.schema import AppCoverageSchema +from backend.analysis.api.schema import CatalogMetaSchema from backend.analysis.api.views._router import analyzers_router from backend.analysis.models import AnalyzerTool from backend.analysis.models import AnalyzerWorkspace from backend.analysis.models import InstalledTool +from backend.analysis.seeding import load_catalog_meta +from backend.analysis.services.coverage import app_coverage def install_state_for(user) -> dict[str, InstalledTool]: @@ -30,6 +34,8 @@ def serialize_tool(tool: AnalyzerTool, installs: dict[str, InstalledTool]) -> di "category": tool.category, "kind": tool.kind, "target_language": tool.target_language, + "analysis_mode": tool.analysis_mode, + "technologies": tool.technologies or [], "icon": tool.icon, "version": tool.version, "docs_url": tool.docs_url, @@ -53,6 +59,19 @@ def list_tools(request): return [serialize_tool(t, installs) for t in tools] +@analyzers_router.get("/catalog/", response=CatalogMetaSchema) +def catalog_meta(request): + """Shop vocabulary: categories, static/dynamic modes and scanned technologies.""" + meta = load_catalog_meta() + return {key: meta.get(key) or [] for key in ("categories", "modes", "technology_groups", "technologies")} + + +@analyzers_router.get("/coverage/", response=AppCoverageSchema) +def coverage(request): + """Technologies in the caller's recent generated apps and how many apps each tool fits.""" + return app_coverage(request.auth) + + @analyzers_router.get("/tools/{slug}/", response=AnalyzerToolSchema) def get_tool(request, slug: str): """Get a single catalog tool by slug.""" diff --git a/backend/analysis/apps.py b/backend/analysis/apps.py index 890be0a9..5f7bce9b 100644 --- a/backend/analysis/apps.py +++ b/backend/analysis/apps.py @@ -15,9 +15,11 @@ class AnalysisConfig(AppConfig): def ready(self) -> None: from django.db.models.signals import post_migrate - # Registers the dependency / web / extra lint parsers alongside the - # built-in ones in services.parsers. + # Registers the dependency / web / extra lint parsers and the DAST and + # multi-ecosystem scanner parsers alongside the built-in ones in + # services.parsers. from backend.analysis.services import extra_parsers # noqa: F401 + from backend.analysis.services import scanner_parsers # noqa: F401 post_migrate.connect( _seed_catalog, diff --git a/backend/analysis/data/tools/bandit.yaml b/backend/analysis/data/tools/bandit.yaml index 44632d15..f8665711 100644 --- a/backend/analysis/data/tools/bandit.yaml +++ b/backend/analysis/data/tools/bandit.yaml @@ -6,14 +6,19 @@ description: >- category: security kind: container target_language: python +analysis_mode: static +technologies: [python, flask, django] icon: shield-alert version: "1.7" docs_url: https://bandit.readthedocs.io/ install_cmd: "pip install --user --upgrade 'bandit==1.7.*'" verify_cmd: "bandit --version" uninstall_cmd: "pip uninstall -y bandit" -run_cmd: "bandit -r {target} -f json -q" +# B104 (bind 0.0.0.0) is skipped by default: the generation template requires it for +# the container, so it would add one identical finding to every app. +run_cmd: "skip={config:skip}; bandit -r {target} -f json -q --confidence-level {config:confidence} ${skip:+--skip \"$skip\"}" parser_key: bandit +file_patterns: ["*.py"] run_timeout: 120 install_timeout: 300 display_order: 10 @@ -27,8 +32,15 @@ config_schema: - { value: low, label: Low } - { value: medium, label: Medium } - { value: high, label: High } + - name: skip + type: string + label: Skipped tests + description: Comma-separated Bandit test ids to skip. + default: B104 + placeholder: B104,B101 default_config: confidence: low + skip: B104 sample_code: | import hashlib diff --git a/backend/analysis/data/tools/catalog.yaml b/backend/analysis/data/tools/catalog.yaml index 99b61a8e..be90024a 100644 --- a/backend/analysis/data/tools/catalog.yaml +++ b/backend/analysis/data/tools/catalog.yaml @@ -1,22 +1,71 @@ # Display metadata for the analyzer "shop". Tool definitions live in the # sibling *.yaml files (one per tool) and are seeded by `seed_analysis_tools`. -catalog_schema_version: 1 +# Served to the frontend by GET /api/analyzers/catalog/. +catalog_schema_version: 2 categories: - slug: security label: Security description: Find vulnerabilities and insecure patterns. + - slug: dependencies + label: Dependencies + description: Vulnerable, missing and unused packages. - slug: lint label: Lint & quality description: Catch bugs, style issues, and dead code. - - slug: performance - label: Performance - description: Surface performance problems. - slug: secrets label: Secrets description: Detect leaked credentials and secrets. - - slug: dependencies - label: Dependencies - description: Vulnerable, missing and unused packages. + - slug: performance + label: Performance + description: Surface performance problems. - slug: ai label: AI review description: LLM-assisted code review. + +# How a tool looks at the app: its source, or the app's running container. +modes: + - slug: static + label: Static + description: Reads the source code; works on any generated app. + - slug: dynamic + label: Dynamic + description: >- + Probes the running app over HTTP (DAST). Start the app's container + first; the scan is skipped otherwise. + +# What a tool scans. A tool lists every technology it understands in its +# `technologies:` field; the seed warns about slugs missing from this list. +# A technology may have no tool yet (e.g. Vue): the shop's coverage panel +# then reports it as a gap when generated apps contain it. +technology_groups: + - slug: language + label: Languages + - slug: framework + label: Frameworks & templates + - slug: ecosystem + label: Package ecosystems + - slug: infrastructure + label: Containers & IaC + - slug: runtime + label: Running app +technologies: + - { slug: python, label: Python, group: language } + - { slug: javascript, label: JavaScript, group: language } + - { slug: html, label: HTML, group: language } + - { slug: css, label: CSS, group: language } + - { slug: shell, label: Shell, group: language } + - { slug: generic, label: Any source, group: language } + - { slug: flask, label: Flask, group: framework } + - { slug: fastapi, label: FastAPI, group: framework } + - { slug: django, label: Django, group: framework } + - { slug: express, label: Express / Node.js, group: framework } + - { slug: react, label: React, group: framework } + - { slug: vue, label: Vue, group: framework } + - { slug: jinja, label: Jinja / Django templates, group: framework } + - { slug: pypi, label: pip / PyPI, group: ecosystem } + - { slug: npm, label: npm, group: ecosystem } + - { slug: docker, label: Docker, group: infrastructure } + - { slug: kubernetes, label: Kubernetes, group: infrastructure } + - { slug: terraform, label: Terraform, group: infrastructure } + - { slug: http, label: Web app (HTTP), group: runtime } + - { slug: rest-api, label: REST API, group: runtime } diff --git a/backend/analysis/data/tools/codespell.yaml b/backend/analysis/data/tools/codespell.yaml index 55c8cea4..79126385 100644 --- a/backend/analysis/data/tools/codespell.yaml +++ b/backend/analysis/data/tools/codespell.yaml @@ -6,13 +6,15 @@ description: >- category: lint kind: container target_language: any +analysis_mode: static +technologies: [generic] icon: spell-check version: "2" docs_url: https://github.com/codespell-project/codespell install_cmd: "pip install --user --upgrade 'codespell==2.*'" verify_cmd: "codespell --version" uninstall_cmd: "pip uninstall -y codespell" -run_cmd: "codespell {target} --disable-colors" +run_cmd: "codespell {target} --disable-colors --skip '*/node_modules/*,package-lock.json,*.lock,*.min.js,*.svg'" parser_key: codespell run_timeout: 60 install_timeout: 120 diff --git a/backend/analysis/data/tools/depcheck.yaml b/backend/analysis/data/tools/depcheck.yaml index 6bf58a5b..cd1bac79 100644 --- a/backend/analysis/data/tools/depcheck.yaml +++ b/backend/analysis/data/tools/depcheck.yaml @@ -6,6 +6,8 @@ description: >- category: dependencies kind: container target_language: javascript +analysis_mode: static +technologies: [javascript, npm] icon: package-check version: "1.4" docs_url: https://github.com/depcheck/depcheck @@ -20,6 +22,7 @@ run_cmd: | echo done parser_key: depcheck +file_patterns: ["package.json"] run_timeout: 120 install_timeout: 180 display_order: 40 diff --git a/backend/analysis/data/tools/deptry.yaml b/backend/analysis/data/tools/deptry.yaml index 42699a10..8f325349 100644 --- a/backend/analysis/data/tools/deptry.yaml +++ b/backend/analysis/data/tools/deptry.yaml @@ -7,6 +7,8 @@ description: >- category: dependencies kind: container target_language: python +analysis_mode: static +technologies: [python, pypi] icon: boxes version: "0.23" docs_url: https://deptry.com/ @@ -23,6 +25,7 @@ run_cmd: | echo done parser_key: deptry +file_patterns: ["requirements*.txt"] run_timeout: 120 install_timeout: 180 display_order: 30 diff --git a/backend/analysis/data/tools/detect-secrets.yaml b/backend/analysis/data/tools/detect-secrets.yaml index 228a6d15..0cb3cbb9 100644 --- a/backend/analysis/data/tools/detect-secrets.yaml +++ b/backend/analysis/data/tools/detect-secrets.yaml @@ -6,6 +6,8 @@ description: >- category: secrets kind: container target_language: any +analysis_mode: static +technologies: [generic] icon: key version: "1.5" docs_url: https://github.com/Yelp/detect-secrets diff --git a/backend/analysis/data/tools/djlint.yaml b/backend/analysis/data/tools/djlint.yaml new file mode 100644 index 00000000..90bd2554 --- /dev/null +++ b/backend/analysis/data/tools/djlint.yaml @@ -0,0 +1,47 @@ +slug: djlint +name: djLint +description: >- + Linter for HTML templates — Jinja (Flask) and Django templates as well as + plain HTML. Flags accessibility gaps, missing attributes, unsafe inline + markup and malformed template tags. +category: lint +kind: container +target_language: html +analysis_mode: static +technologies: [html, jinja] +icon: file-code +version: "1.36" +docs_url: https://www.djlint.com/docs/linter/ +install_cmd: >- + set -e; + python3 -m venv /home/analyzer/.local/opt/djlint; + /home/analyzer/.local/opt/djlint/bin/pip install --upgrade 'djlint==1.36.*'; + mkdir -p /home/analyzer/.local/bin; + ln -sf /home/analyzer/.local/opt/djlint/bin/djlint /home/analyzer/.local/bin/djlint +verify_cmd: "djlint --version" +uninstall_cmd: "rm -rf /home/analyzer/.local/opt/djlint /home/analyzer/.local/bin/djlint" +# A machine-readable line format; the progress bar lines are ignored by the parser. +run_cmd: "djlint {target} --lint --profile={config:profile} --linter-output-format '{filename}:{line}:{code}:{message}' 2>/dev/null" +parser_key: djlint +file_patterns: ["*.html", "*.jinja", "*.jinja2", "*.j2"] +run_timeout: 120 +install_timeout: 300 +display_order: 75 +config_schema: + - name: profile + type: select + label: Template profile + description: Template language the rules assume. + default: jinja + options: + - { value: jinja, label: Jinja (Flask) } + - { value: django, label: Django } + - { value: html, label: Plain HTML } +default_config: + profile: jinja +sample_filename: index.html +sample_code: | + + + {{ name|safe }} + diff --git a/backend/analysis/data/tools/eslint-react.yaml b/backend/analysis/data/tools/eslint-react.yaml index 9fac0d66..1f2e483d 100644 --- a/backend/analysis/data/tools/eslint-react.yaml +++ b/backend/analysis/data/tools/eslint-react.yaml @@ -7,6 +7,8 @@ description: >- category: lint kind: container target_language: javascript +analysis_mode: static +technologies: [javascript, react] icon: atom version: "7" docs_url: https://github.com/jsx-eslint/eslint-plugin-react @@ -17,6 +19,7 @@ run_cmd: | printf '%s' '{"root":true,"env":{"browser":true,"node":true,"es2022":true},"parserOptions":{"ecmaVersion":"latest","sourceType":"module","ecmaFeatures":{"jsx":true}},"settings":{"react":{"version":"18.2"}},"extends":["plugin:react/recommended","plugin:react/jsx-runtime","plugin:react-hooks/recommended"],"rules":{"react/prop-types":"off","react/no-danger":"warn"}}' > /tmp/eslint-react.json eslint {target} --ext .js,.jsx,.cjs,.mjs -f json --no-eslintrc -c /tmp/eslint-react.json --resolve-plugins-relative-to "$(npm root -g)" --ignore-pattern node_modules 2>/dev/null parser_key: eslint +file_patterns: ["*.js", "*.jsx", "*.cjs", "*.mjs"] run_timeout: 120 install_timeout: 300 display_order: 35 diff --git a/backend/analysis/data/tools/eslint-security.yaml b/backend/analysis/data/tools/eslint-security.yaml index c56e00c2..ef85e7aa 100644 --- a/backend/analysis/data/tools/eslint-security.yaml +++ b/backend/analysis/data/tools/eslint-security.yaml @@ -7,6 +7,8 @@ description: >- category: security kind: container target_language: javascript +analysis_mode: static +technologies: [javascript, express] icon: shield-check version: "1" docs_url: https://github.com/eslint-community/eslint-plugin-security @@ -18,6 +20,7 @@ run_cmd: | printf '%s' '{"root":true,"env":{"browser":true,"node":true,"es2022":true},"parserOptions":{"ecmaVersion":"latest","sourceType":"module","ecmaFeatures":{"jsx":true}},"plugins":["security"],"extends":["plugin:security/recommended"],"rules":{"security/detect-object-injection":"off"}}' > /tmp/eslint-security.json eslint {target} --ext .js,.jsx,.cjs,.mjs -f json --no-eslintrc -c /tmp/eslint-security.json --resolve-plugins-relative-to "$(npm root -g)" --ignore-pattern node_modules 2>/dev/null parser_key: eslint_security +file_patterns: ["*.js", "*.jsx", "*.cjs", "*.mjs"] run_timeout: 120 install_timeout: 300 display_order: 30 diff --git a/backend/analysis/data/tools/eslint.yaml b/backend/analysis/data/tools/eslint.yaml index 0f6fde73..293e7790 100644 --- a/backend/analysis/data/tools/eslint.yaml +++ b/backend/analysis/data/tools/eslint.yaml @@ -1,21 +1,29 @@ slug: eslint name: ESLint description: >- - The pluggable linter for JavaScript and TypeScript. Reports problematic - patterns and style violations in JS/TS code. + The pluggable linter for JavaScript. Reports undefined names and unused + variables in .js, .jsx, .cjs and .mjs files (TypeScript is not linted: + no TypeScript parser is installed). category: lint kind: container target_language: javascript +analysis_mode: static +technologies: [javascript] icon: braces version: "8" docs_url: https://eslint.org/docs/latest/ -install_cmd: "npm install -g eslint@8" +install_cmd: "npm install -g eslint@8 eslint-plugin-react@7" verify_cmd: "eslint --version" uninstall_cmd: "npm uninstall -g eslint" # es2022 is needed alongside browser/node: without it Promise, Set, etc. are -# "not defined" and no-undef floods the results with false positives. -run_cmd: "eslint {target} --ext .js,.jsx,.cjs,.mjs -f json --no-eslintrc --env browser,node,es2022 --parser-options=ecmaVersion:latest,sourceType:module,ecmaFeatures:{jsx:true} --rule '{\"no-undef\":\"error\",\"no-unused-vars\":\"warn\"}'" +# "not defined" and no-undef floods the results with false positives. The react +# plugin's jsx-uses-vars rules only mark JSX-referenced components as used (they +# never report): without them every imported component is "unused". +run_cmd: | + printf '%s' '{"root":true,"env":{"browser":true,"node":true,"es2022":true},"parserOptions":{"ecmaVersion":"latest","sourceType":"module","ecmaFeatures":{"jsx":true}},"settings":{"react":{"version":"18.2"}},"plugins":["react"],"rules":{"no-undef":"error","no-unused-vars":"warn","react/jsx-uses-vars":"error","react/jsx-uses-react":"error"}}' > /tmp/eslint-base.json + eslint {target} --ext .js,.jsx,.cjs,.mjs -f json --no-eslintrc -c /tmp/eslint-base.json --resolve-plugins-relative-to "$(npm root -g)" --ignore-pattern node_modules 2>/dev/null parser_key: eslint +file_patterns: ["*.js", "*.jsx", "*.cjs", "*.mjs"] run_timeout: 120 install_timeout: 300 display_order: 30 diff --git a/backend/analysis/data/tools/flake8.yaml b/backend/analysis/data/tools/flake8.yaml index 664af325..7efae181 100644 --- a/backend/analysis/data/tools/flake8.yaml +++ b/backend/analysis/data/tools/flake8.yaml @@ -6,6 +6,8 @@ description: >- category: lint kind: container target_language: python +analysis_mode: static +technologies: [python] icon: snowflake version: "7" docs_url: https://flake8.pycqa.org/ @@ -14,6 +16,7 @@ verify_cmd: "flake8 --version" uninstall_cmd: "pip uninstall -y flake8" run_cmd: "flake8 {target} --exit-zero --max-line-length 120 --exclude node_modules,.venv,venv 2>/dev/null" parser_key: flake8 +file_patterns: ["*.py"] run_timeout: 120 install_timeout: 180 display_order: 25 diff --git a/backend/analysis/data/tools/gitleaks.yaml b/backend/analysis/data/tools/gitleaks.yaml index a3a41966..aefd57c6 100644 --- a/backend/analysis/data/tools/gitleaks.yaml +++ b/backend/analysis/data/tools/gitleaks.yaml @@ -6,19 +6,23 @@ description: >- category: secrets kind: container target_language: any +analysis_mode: static +technologies: [generic] icon: key-round version: "8.18" docs_url: https://github.com/gitleaks/gitleaks +# sha256 from the release's gitleaks_8.18.4_checksums.txt. install_cmd: >- - mkdir -p /home/analyzer/.local/bin && - curl -sSfL - https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz - | tar -xz -C /home/analyzer/.local/bin gitleaks + set -e; + mkdir -p /home/analyzer/.local/bin && cd /tmp; + curl -sSfL -o gitleaks.tgz https://github.com/gitleaks/gitleaks/releases/download/v8.18.4/gitleaks_8.18.4_linux_x64.tar.gz; + echo "ba6dbb656933921c775ee5a2d1c13a91046e7952e9d919f9bac4cec61d628e7d gitleaks.tgz" | sha256sum -c -; + tar -xzf gitleaks.tgz -C /home/analyzer/.local/bin gitleaks && rm -f gitleaks.tgz verify_cmd: "gitleaks version" uninstall_cmd: "rm -f /home/analyzer/.local/bin/gitleaks" # Write the JSON report to a file and cat only that, sending gitleaks' banner # and logs to /dev/null — otherwise the ASCII banner/ANSI logs pollute stdout. -run_cmd: "gitleaks detect --no-git --source {target} --report-format json --report-path /tmp/gitleaks-report.json --exit-code 0 --redact --no-banner --log-level error >/dev/null 2>&1; cat /tmp/gitleaks-report.json" +run_cmd: "rm -f /tmp/gitleaks-report.json; gitleaks detect --no-git --source {target} --report-format json --report-path /tmp/gitleaks-report.json --exit-code 0 --redact --no-banner --log-level error >/dev/null 2>&1; cat /tmp/gitleaks-report.json" parser_key: gitleaks run_timeout: 180 install_timeout: 300 diff --git a/backend/analysis/data/tools/hadolint.yaml b/backend/analysis/data/tools/hadolint.yaml index e72e6535..f009f86d 100644 --- a/backend/analysis/data/tools/hadolint.yaml +++ b/backend/analysis/data/tools/hadolint.yaml @@ -6,13 +6,17 @@ description: >- category: lint kind: container target_language: any +analysis_mode: static +technologies: [docker] icon: container version: "2.12" docs_url: https://github.com/hadolint/hadolint +# sha256 from the release's hadolint-Linux-x86_64.sha256. install_cmd: >- - mkdir -p /home/analyzer/.local/bin && - curl -sSfL -o /home/analyzer/.local/bin/hadolint - https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64 && + set -e; + mkdir -p /home/analyzer/.local/bin; + curl -sSfL -o /home/analyzer/.local/bin/hadolint https://github.com/hadolint/hadolint/releases/download/v2.12.0/hadolint-Linux-x86_64; + echo "56de6d5e5ec427e17b74fa48d51271c7fc0d61244bf5c90e828aab8362d55010 /home/analyzer/.local/bin/hadolint" | sha256sum -c - || { rm -f /home/analyzer/.local/bin/hadolint; exit 1; }; chmod +x /home/analyzer/.local/bin/hadolint verify_cmd: "hadolint --version" uninstall_cmd: "rm -f /home/analyzer/.local/bin/hadolint" @@ -20,6 +24,7 @@ uninstall_cmd: "rm -f /home/analyzer/.local/bin/hadolint" # keeps the exit code at 0 and -r makes xargs a no-op when none exist. run_cmd: "find {target} -type f \\( -name Dockerfile -o -name 'Dockerfile.*' -o -name '*.dockerfile' \\) -print0 | xargs -0 -r hadolint -f json --no-fail" parser_key: hadolint +file_patterns: ["Dockerfile", "Dockerfile.*", "*.dockerfile"] run_timeout: 60 install_timeout: 120 display_order: 90 diff --git a/backend/analysis/data/tools/htmlhint.yaml b/backend/analysis/data/tools/htmlhint.yaml index a00e629f..5b477fef 100644 --- a/backend/analysis/data/tools/htmlhint.yaml +++ b/backend/analysis/data/tools/htmlhint.yaml @@ -6,6 +6,8 @@ description: >- category: lint kind: container target_language: html +analysis_mode: static +technologies: [html] icon: file-code version: "1" docs_url: https://htmlhint.com/ @@ -14,6 +16,7 @@ verify_cmd: "htmlhint --version" uninstall_cmd: "npm uninstall -g htmlhint" run_cmd: "htmlhint '{target}/**/*.html' --ignore '**/node_modules/**' --format json 2>/dev/null" parser_key: htmlhint +file_patterns: ["*.html"] run_timeout: 60 install_timeout: 120 display_order: 100 diff --git a/backend/analysis/data/tools/jscpd.yaml b/backend/analysis/data/tools/jscpd.yaml index 9b539581..33e71519 100644 --- a/backend/analysis/data/tools/jscpd.yaml +++ b/backend/analysis/data/tools/jscpd.yaml @@ -6,6 +6,8 @@ description: >- category: lint kind: container target_language: any +analysis_mode: static +technologies: [generic] icon: copy version: "4" docs_url: https://github.com/kucherenko/jscpd @@ -17,8 +19,9 @@ uninstall_cmd: "npm uninstall -g jscpd" # long-lived workspace container, so remove it first — otherwise a failed run # would serve the previous run's report as this run's result. jscpd's default # --max-lines of 1000 silently skips whole generated apps, so raise both size -# gates well past realistic generated-file sizes. -run_cmd: "rm -rf /tmp/jscpd; jscpd {target} --min-tokens 35 --max-lines 50000 --max-size 2mb --reporters json --output /tmp/jscpd --silent >/dev/null 2>&1; cat /tmp/jscpd/jscpd-report.json" +# gates well past realistic generated-file sizes. Lockfiles repeat package.json +# by construction, so they are not scanned. +run_cmd: "rm -rf /tmp/jscpd; jscpd {target} --ignore '**/package-lock.json,**/*.lock,**/node_modules/**,**/*.min.js' --min-tokens 35 --max-lines 50000 --max-size 2mb --reporters json --output /tmp/jscpd --silent >/dev/null 2>&1; cat /tmp/jscpd/jscpd-report.json" parser_key: jscpd output_kind: mixed run_timeout: 120 diff --git a/backend/analysis/data/tools/lizard.yaml b/backend/analysis/data/tools/lizard.yaml index 56c15c1a..60a6aa5d 100644 --- a/backend/analysis/data/tools/lizard.yaml +++ b/backend/analysis/data/tools/lizard.yaml @@ -7,6 +7,8 @@ description: >- category: performance kind: container target_language: any +analysis_mode: static +technologies: [python, javascript] icon: activity version: "1.17" docs_url: https://github.com/terryyin/lizard @@ -15,6 +17,7 @@ verify_cmd: "lizard --version" uninstall_cmd: "pip uninstall -y lizard" run_cmd: "lizard {target} --csv -l python -l javascript -l typescript -x '*/node_modules/*' 2>/dev/null" parser_key: lizard +file_patterns: ["*.py", "*.js", "*.jsx", "*.ts", "*.tsx"] output_kind: mixed run_timeout: 120 install_timeout: 120 diff --git a/backend/analysis/data/tools/llm_review.yaml b/backend/analysis/data/tools/llm_review.yaml index 469c4a3e..799d1f34 100644 --- a/backend/analysis/data/tools/llm_review.yaml +++ b/backend/analysis/data/tools/llm_review.yaml @@ -1,12 +1,15 @@ slug: llm_review name: AI Reviewer description: >- - An LLM-powered code reviewer (via OpenRouter) that reads the code and reports - security, bug, and quality findings in natural language. Requires an - OpenRouter API key. + An LLM-powered code reviewer that reads the code and reports security, bug, + and quality findings in natural language. Uses any model in your model + catalog (OpenRouter or a local endpoint); other model ids go to OpenRouter + and need an OpenRouter API key. category: ai kind: ai target_language: any +analysis_mode: static +technologies: [generic] icon: bot version: "" install_cmd: "" @@ -20,9 +23,9 @@ config_schema: - name: model type: string label: Model - description: OpenRouter model id for the review. - default: anthropic/claude-opus-4-8 - placeholder: anthropic/claude-opus-4-8 + description: Model id for the review (a catalog model, OpenRouter or local; otherwise sent to OpenRouter). + default: anthropic/claude-opus-5.5 + placeholder: anthropic/claude-opus-5.5 - name: max_tokens type: number label: Max tokens @@ -31,6 +34,6 @@ config_schema: min: 500 max: 16000 default_config: - model: anthropic/claude-opus-4-8 + model: anthropic/claude-opus-5.5 max_tokens: 8000 sample_code: "" diff --git a/backend/analysis/data/tools/mypy.yaml b/backend/analysis/data/tools/mypy.yaml index 271d52bf..9e60ca23 100644 --- a/backend/analysis/data/tools/mypy.yaml +++ b/backend/analysis/data/tools/mypy.yaml @@ -6,14 +6,22 @@ description: >- category: lint kind: container target_language: python +analysis_mode: static +technologies: [python] icon: check-check -version: "1" +version: "2.4" docs_url: https://mypy.readthedocs.io/ -install_cmd: "pip install --user --upgrade mypy" +# Pinned like the other pip tools: an unpinned install drifts to new major +# versions (new checks change the counts) between workspaces. +install_cmd: "pip install --user --upgrade 'mypy==2.4.*'" verify_cmd: "mypy --version" uninstall_cmd: "pip uninstall -y mypy" -run_cmd: "mypy {target} --no-error-summary --no-color-output --show-error-codes --ignore-missing-imports --no-pretty" +# --cache-dir: keep the cache out of /work, where detect-secrets/jscpd would scan it. +# --explicit-package-bases from the app root: multi-module apps (routers/admin.py) +# otherwise abort with "source file found twice under different module names". +run_cmd: "cd {target} && mypy . --explicit-package-bases --cache-dir /tmp/mypy-cache --no-error-summary --no-color-output --show-error-codes --ignore-missing-imports --no-pretty" parser_key: mypy +file_patterns: ["*.py"] run_timeout: 180 install_timeout: 300 display_order: 50 diff --git a/backend/analysis/data/tools/npm-audit.yaml b/backend/analysis/data/tools/npm-audit.yaml index 015fb462..76de60f0 100644 --- a/backend/analysis/data/tools/npm-audit.yaml +++ b/backend/analysis/data/tools/npm-audit.yaml @@ -7,6 +7,8 @@ description: >- category: dependencies kind: container target_language: javascript +analysis_mode: static +technologies: [javascript, npm] icon: package-x version: "10" docs_url: https://docs.npmjs.com/cli/commands/npm-audit @@ -14,19 +16,37 @@ docs_url: https://docs.npmjs.com/cli/commands/npm-audit install_cmd: "" verify_cmd: "npm --version" run_cmd: | + omit="$( [ {config:production_only} = true ] && echo --omit=dev )" cd {target} && find . -name package.json -not -path '*/node_modules/*' | sort | while IFS= read -r f; do rm -rf /tmp/npm-audit && mkdir -p /tmp/npm-audit cp "$f" /tmp/npm-audit/ lock="$(dirname "$f")/package-lock.json" [ -f "$lock" ] && cp "$lock" /tmp/npm-audit/ echo "### ${f#./}" - (cd /tmp/npm-audit && { [ -f package-lock.json ] || npm install --package-lock-only --ignore-scripts --no-audit --no-fund >/dev/null 2>&1; } && npm audit --json 2>/dev/null) + (cd /tmp/npm-audit && if [ ! -f package-lock.json ] && ! npm install --package-lock-only --ignore-scripts --no-audit --no-fund >/tmp/npm-install.log 2>&1; then + grep -E 'not in this registry|No matching version|npm error code' /tmp/npm-install.log | head -n 20 | sed 's/^/!! /' + else + npm audit --json $omit 2>/dev/null + fi) echo done parser_key: npm_audit +file_patterns: ["package.json"] run_timeout: 300 install_timeout: 60 display_order: 20 +# production_only: dev-only packages (vite, eslint, …) never ship with the +# built app, so their advisories (mostly dev-server issues) are left out by +# default. A failed lockfile resolution (a hallucinated package or version) is +# echoed as "!! " lines and reported as nonexistent-package/-version findings. +config_schema: + - name: production_only + type: boolean + label: Production dependencies only + description: Skip advisories that only affect devDependencies. + default: true +default_config: + production_only: true sample_filename: package.json sample_code: | { diff --git a/backend/analysis/data/tools/nuclei.yaml b/backend/analysis/data/tools/nuclei.yaml new file mode 100644 index 00000000..34ac4ca0 --- /dev/null +++ b/backend/analysis/data/tools/nuclei.yaml @@ -0,0 +1,44 @@ +slug: nuclei +name: Nuclei +description: >- + Template-based web scanner from ProjectDiscovery. Probes the running app with + community templates for misconfigurations, exposed files and panels, missing + security headers, CORS issues and common injection bugs. Requires the app's + container to be running. +category: security +kind: container +target_language: any +analysis_mode: dynamic +technologies: [http, rest-api] +icon: scan-search +version: "3.11" +docs_url: https://docs.projectdiscovery.io/tools/nuclei/overview +# Templates are fetched at install time; reinstall/update refreshes them. +install_cmd: >- + set -e; + mkdir -p /home/analyzer/.local/bin && cd /tmp; + curl -sSfL -o nuclei.zip https://github.com/projectdiscovery/nuclei/releases/download/v3.11.1/nuclei_3.11.1_linux_amd64.zip; + echo "ea63d4ae232808cd7c6bc00d0142428e231fab59dae01042246097d195835ab6 nuclei.zip" | sha256sum -c -; + python3 -c "import zipfile; zipfile.ZipFile('nuclei.zip').extract('nuclei', '/home/analyzer/.local/bin')"; + chmod +x /home/analyzer/.local/bin/nuclei && rm -f nuclei.zip; + nuclei -update-templates -no-color 2>&1 | tail -3; + test -d /home/analyzer/nuclei-templates +verify_cmd: "nuclei -version 2>&1 | grep -o 'Engine Version: v[0-9.]*'" +uninstall_cmd: "rm -rf /home/analyzer/.local/bin/nuclei /home/analyzer/nuclei-templates /home/analyzer/.config/nuclei /home/analyzer/.cache/nuclei" +# Intrusive / DoS / fuzzing templates are always excluded. +run_cmd: >- + nuclei -u {url} -tags {config:tags} -etags dos,fuzz,intrusive + -jsonl -omit-raw -silent -duc -no-color -rate-limit 50 -timeout 5 2>/dev/null +parser_key: nuclei +run_timeout: 900 +install_timeout: 600 +display_order: 6 +config_schema: + - name: tags + type: string + label: Template tags + description: Comma-separated template tags to run (fewer tags = faster scan). + default: "misconfig,exposure,tech,headers,xss,sqli,lfi,redirect,cors" + placeholder: "misconfig,exposure,xss" +default_config: + tags: "misconfig,exposure,tech,headers,xss,sqli,lfi,redirect,cors" diff --git a/backend/analysis/data/tools/osv-scanner.yaml b/backend/analysis/data/tools/osv-scanner.yaml new file mode 100644 index 00000000..0f20bda7 --- /dev/null +++ b/backend/analysis/data/tools/osv-scanner.yaml @@ -0,0 +1,38 @@ +slug: osv-scanner +name: OSV-Scanner +description: >- + Google's scanner for known vulnerabilities in dependencies, backed by the + OSV.dev database. Reads requirements.txt, package.json and lockfiles and + resolves transitive packages. Needs network access. +category: dependencies +kind: container +target_language: any +analysis_mode: static +technologies: [pypi, npm] +icon: package-search +version: "2.6" +docs_url: https://google.github.io/osv-scanner/ +install_cmd: >- + set -e; + mkdir -p /home/analyzer/.local/bin; + curl -sSfL -o /home/analyzer/.local/bin/osv-scanner https://github.com/google/osv-scanner/releases/download/v2.6.0/osv-scanner_linux_amd64; + echo "ca69b3d3cd08f889a49dc0a383122f71cc528b83803671df5fd874d97485b108 /home/analyzer/.local/bin/osv-scanner" | sha256sum -c -; + chmod +x /home/analyzer/.local/bin/osv-scanner +verify_cmd: "osv-scanner --version | head -1" +uninstall_cmd: "rm -f /home/analyzer/.local/bin/osv-scanner" +# Exits 1 when it finds vulnerabilities; the JSON on stdout is what counts. +# --no-resolve: osv-scanner's own transitive resolution of requirements.txt +# (via deps.dev) picks implausibly old versions (ecdsa 0.9.0 under a pinned +# python-jose), inventing critical CVEs. Scan only declared versions and real +# lockfiles; pip-audit covers Python transitive dependencies with pip itself. +run_cmd: "osv-scanner scan source -r --no-resolve --format json {target} 2>/dev/null" +parser_key: osv_scanner +file_patterns: ["requirements*.txt", "package.json", "package-lock.json", "poetry.lock", "Pipfile.lock", "yarn.lock", "pnpm-lock.yaml"] +run_timeout: 300 +install_timeout: 120 +display_order: 25 +config_schema: [] +default_config: {} +sample_filename: requirements.txt +sample_code: | + Jinja2==2.10 diff --git a/backend/analysis/data/tools/perflint.yaml b/backend/analysis/data/tools/perflint.yaml index 69055b51..fbff3e8a 100644 --- a/backend/analysis/data/tools/perflint.yaml +++ b/backend/analysis/data/tools/perflint.yaml @@ -7,6 +7,8 @@ description: >- category: performance kind: container target_language: python +analysis_mode: static +technologies: [python] icon: zap version: "0.8" docs_url: https://github.com/tonybaloney/perflint @@ -20,6 +22,7 @@ verify_cmd: "perflint --version" uninstall_cmd: "rm -rf /home/analyzer/.venvs/perflint /home/analyzer/.local/bin/perflint" run_cmd: "perflint {target} --output-format=json --exit-zero --ignore=node_modules,.venv 2>/dev/null" parser_key: perflint +file_patterns: ["*.py"] run_timeout: 180 install_timeout: 300 display_order: 30 diff --git a/backend/analysis/data/tools/pip-audit.yaml b/backend/analysis/data/tools/pip-audit.yaml index 38029034..62f90e72 100644 --- a/backend/analysis/data/tools/pip-audit.yaml +++ b/backend/analysis/data/tools/pip-audit.yaml @@ -7,6 +7,8 @@ description: >- category: dependencies kind: container target_language: python +analysis_mode: static +technologies: [python, pypi] icon: package-search version: "2" docs_url: https://github.com/pypa/pip-audit @@ -15,13 +17,22 @@ verify_cmd: "pip-audit --version" uninstall_cmd: "pip uninstall -y pip-audit" # One "### " section per requirements file; pip-audit resolves # transitive dependencies in a throwaway venv, so unpinned files work too. +# When resolution fails (a hallucinated package or version) pip-audit audits +# nothing: its errors are echoed as "!! " lines, which the parser turns into +# nonexistent-package/-version findings and a "not analyzed" coverage gap. run_cmd: | cd {target} && find . -name 'requirements*.txt' -not -path '*/node_modules/*' -not -path '*/.venv/*' | sort | while IFS= read -r f; do echo "### ${f#./}" - pip-audit -r "$f" -f json --progress-spinner off 2>/dev/null + out="$(pip-audit -r "$f" -f json --progress-spinner off 2>/tmp/pip-audit.err)" + if [ -n "$out" ]; then + printf '%s\n' "$out" + else + grep -E 'ERROR|from versions' /tmp/pip-audit.err | cut -c1-400 | head -n 20 | sed 's/^/!! /' + fi echo done parser_key: pip_audit +file_patterns: ["requirements*.txt"] run_timeout: 300 install_timeout: 300 display_order: 10 diff --git a/backend/analysis/data/tools/pylint.yaml b/backend/analysis/data/tools/pylint.yaml index 77ce8932..6b8c4234 100644 --- a/backend/analysis/data/tools/pylint.yaml +++ b/backend/analysis/data/tools/pylint.yaml @@ -6,19 +6,34 @@ description: >- category: lint kind: container target_language: python +analysis_mode: static +technologies: [python] icon: badge-check version: "3" docs_url: https://pylint.readthedocs.io/ install_cmd: "pip install --user --upgrade pylint" verify_cmd: "pylint --version" uninstall_cmd: "pip uninstall -y pylint" -run_cmd: "pylint {target} -f json --exit-zero --disable=import-error" +run_cmd: "pylint {target} -f json --exit-zero --disable={config:disable}" parser_key: pylint +file_patterns: ["*.py"] run_timeout: 180 install_timeout: 300 display_order: 40 -config_schema: [] -default_config: {} +# Docstring and too-few-public-methods conventions are disabled by default: on +# generated web apps (one-method data classes, many small route handlers) they +# outnumber every other message and drown real defects in per-app counts. +# unused-argument is disabled too: framework dependency injection +# (`user = Depends(require_admin)`, handler `request` args) makes nearly every +# hit on a web app a false positive. +config_schema: + - name: disable + type: string + label: Disabled messages + description: Comma-separated pylint message names or ids to disable. + default: import-error,missing-module-docstring,missing-class-docstring,missing-function-docstring,too-few-public-methods,unused-argument +default_config: + disable: import-error,missing-module-docstring,missing-class-docstring,missing-function-docstring,too-few-public-methods,unused-argument sample_code: | import os diff --git a/backend/analysis/data/tools/radon.yaml b/backend/analysis/data/tools/radon.yaml index 963a2d93..856a391c 100644 --- a/backend/analysis/data/tools/radon.yaml +++ b/backend/analysis/data/tools/radon.yaml @@ -7,6 +7,8 @@ description: >- category: performance kind: container target_language: python +analysis_mode: static +technologies: [python] icon: gauge version: "6" docs_url: https://radon.readthedocs.io/ @@ -17,6 +19,7 @@ uninstall_cmd: "pip uninstall -y radon" # compute complexity metrics; only E/F blocks become findings. run_cmd: "radon cc {target} -j" parser_key: radon +file_patterns: ["*.py"] output_kind: mixed run_timeout: 90 install_timeout: 180 diff --git a/backend/analysis/data/tools/retire.yaml b/backend/analysis/data/tools/retire.yaml new file mode 100644 index 00000000..f2979c66 --- /dev/null +++ b/backend/analysis/data/tools/retire.yaml @@ -0,0 +1,28 @@ +slug: retire +name: Retire.js +description: >- + Finds JavaScript libraries with known vulnerabilities, including copies + vendored or bundled into static files (old jQuery, Angular, Bootstrap…) + that package-manifest audits miss. +category: dependencies +kind: container +target_language: javascript +analysis_mode: static +technologies: [javascript, html, npm] +icon: history +version: "5" +docs_url: https://retirejs.github.io/retire.js/ +install_cmd: "npm install -g retire@5" +verify_cmd: "retire --version" +uninstall_cmd: "npm uninstall -g retire" +run_cmd: "retire --path {target} --outputformat json --exitwith 0 --ignore node_modules 2>/dev/null" +parser_key: retire +file_patterns: ["*.js", "*.html", "package.json"] +run_timeout: 180 +install_timeout: 300 +display_order: 30 +config_schema: [] +default_config: {} +sample_filename: index.html +sample_code: | + diff --git a/backend/analysis/data/tools/ruff.yaml b/backend/analysis/data/tools/ruff.yaml index 630a71e4..ab6f97fb 100644 --- a/backend/analysis/data/tools/ruff.yaml +++ b/backend/analysis/data/tools/ruff.yaml @@ -6,14 +6,18 @@ description: >- category: lint kind: container target_language: python +analysis_mode: static +technologies: [python] icon: sparkles version: "0.6" docs_url: https://docs.astral.sh/ruff/ install_cmd: "pip install --user --upgrade 'ruff==0.6.*'" verify_cmd: "ruff --version" uninstall_cmd: "pip uninstall -y ruff" -run_cmd: "ruff check {target} --output-format json" +# --no-cache: a .ruff_cache in /work would be scanned by detect-secrets/jscpd. +run_cmd: "sel={config:select}; ruff check {target} --no-cache --output-format json ${sel:+--select \"$sel\"}" parser_key: ruff +file_patterns: ["*.py"] run_timeout: 90 install_timeout: 180 display_order: 20 @@ -24,7 +28,8 @@ config_schema: description: Comma-separated rule codes to enable (leave blank for defaults). default: "" placeholder: "E,F,W" -default_config: {} +default_config: + select: "" sample_code: | import os import sys diff --git a/backend/analysis/data/tools/semgrep.yaml b/backend/analysis/data/tools/semgrep.yaml index d0dde0e1..eb4c180e 100644 --- a/backend/analysis/data/tools/semgrep.yaml +++ b/backend/analysis/data/tools/semgrep.yaml @@ -7,6 +7,8 @@ description: >- category: security kind: container target_language: any +analysis_mode: static +technologies: [python, javascript, flask, fastapi, django, express] icon: radar version: "1" docs_url: https://semgrep.dev/docs/ @@ -17,6 +19,7 @@ uninstall_cmd: "pip uninstall -y semgrep" # refuses when metrics are disabled. Packs are fetched from the registry once. run_cmd: "semgrep --config p/python --config p/javascript --json --quiet --metrics off --disable-version-check {target}" parser_key: semgrep +file_patterns: ["*.py", "*.js", "*.jsx", "*.ts", "*.tsx"] run_timeout: 300 install_timeout: 600 display_order: 20 diff --git a/backend/analysis/data/tools/shellcheck.yaml b/backend/analysis/data/tools/shellcheck.yaml index 85d9c979..965b35cc 100644 --- a/backend/analysis/data/tools/shellcheck.yaml +++ b/backend/analysis/data/tools/shellcheck.yaml @@ -6,6 +6,8 @@ description: >- category: lint kind: container target_language: shell +analysis_mode: static +technologies: [shell] icon: terminal version: "0.10" docs_url: https://www.shellcheck.net/ @@ -14,6 +16,7 @@ verify_cmd: "shellcheck --version" uninstall_cmd: "pip uninstall -y shellcheck-py" run_cmd: "find {target} -type f \\( -name '*.sh' -o -name '*.bash' \\) -not -path '*/node_modules/*' -print0 | xargs -0 -r shellcheck -f json 2>/dev/null" parser_key: shellcheck +file_patterns: ["*.sh", "*.bash"] run_timeout: 60 install_timeout: 120 display_order: 95 diff --git a/backend/analysis/data/tools/stylelint.yaml b/backend/analysis/data/tools/stylelint.yaml index 438df808..2d78e96d 100644 --- a/backend/analysis/data/tools/stylelint.yaml +++ b/backend/analysis/data/tools/stylelint.yaml @@ -7,6 +7,8 @@ description: >- category: lint kind: container target_language: css +analysis_mode: static +technologies: [css] icon: palette version: "16" docs_url: https://stylelint.io/ @@ -20,6 +22,7 @@ run_cmd: | stylelint '{target}/**/*.{css,scss}' --ignore-pattern '**/node_modules/**' --config /tmp/stylelintrc.json --config-basedir "$(npm root -g)" --allow-empty-input -f json --output-file /tmp/stylelint.json >/dev/null 2>&1 cat /tmp/stylelint.json 2>/dev/null parser_key: stylelint +file_patterns: ["*.css", "*.scss"] run_timeout: 60 install_timeout: 180 display_order: 110 diff --git a/backend/analysis/data/tools/trivy.yaml b/backend/analysis/data/tools/trivy.yaml new file mode 100644 index 00000000..7dfe6b0b --- /dev/null +++ b/backend/analysis/data/tools/trivy.yaml @@ -0,0 +1,46 @@ +slug: trivy +name: Trivy +description: >- + All-in-one scanner from Aqua Security: vulnerable packages in Python and + JavaScript manifests and lockfiles, misconfigurations in Dockerfiles, + Kubernetes manifests and Terraform, and hardcoded secrets. +category: security +kind: container +target_language: any +analysis_mode: static +technologies: [pypi, npm, docker, kubernetes, terraform] +icon: shield-half +version: "0.75" +docs_url: https://trivy.dev/latest/docs/ +# Checksum-pinned release; the vulnerability DB (~1.5 GB on disk) is fetched +# at install time so runs don't pay for it. +install_cmd: >- + set -e; + mkdir -p /home/analyzer/.local/bin && cd /tmp; + curl -sSfL -o trivy.tgz https://github.com/aquasecurity/trivy/releases/download/v0.75.0/trivy_0.75.0_Linux-64bit.tar.gz; + echo "c6e65abddb348e25f10549df887045629cf28cc72453cd1c63acb717316b3f3f trivy.tgz" | sha256sum -c -; + tar -xzf trivy.tgz -C /home/analyzer/.local/bin trivy && rm -f trivy.tgz; + trivy fs --quiet --download-db-only +verify_cmd: "trivy --version | head -1" +uninstall_cmd: "rm -rf /home/analyzer/.local/bin/trivy /home/analyzer/.cache/trivy" +run_cmd: "trivy fs --quiet --format json --scanners {config:scanners} --skip-dirs '**/node_modules' --exit-code 0 {target} 2>/dev/null" +parser_key: trivy +run_timeout: 300 +install_timeout: 900 +display_order: 15 +config_schema: + - name: scanners + type: select + label: Scanners + description: What to look for. Misconfig and secret scanning work offline. + default: "vuln,misconfig,secret" + options: + - { value: "vuln,misconfig,secret", label: "Vulnerabilities, misconfigurations, secrets" } + - { value: "vuln", label: "Vulnerable packages only" } + - { value: "misconfig,secret", label: "Misconfigurations and secrets" } +default_config: + scanners: "vuln,misconfig,secret" +sample_filename: requirements.txt +sample_code: | + flask==0.12.2 + Jinja2==2.10 diff --git a/backend/analysis/data/tools/vulture.yaml b/backend/analysis/data/tools/vulture.yaml index 5148b824..479c60ad 100644 --- a/backend/analysis/data/tools/vulture.yaml +++ b/backend/analysis/data/tools/vulture.yaml @@ -6,14 +6,20 @@ description: >- category: lint kind: container target_language: python +analysis_mode: static +technologies: [python] icon: skull version: "2" docs_url: https://github.com/jendrikseipp/vulture install_cmd: "pip install --user --upgrade 'vulture==2.*'" verify_cmd: "vulture --version" uninstall_cmd: "pip uninstall -y vulture" -run_cmd: "vulture {target} --min-confidence 60" +# Route handlers, validators and hooks are reached through their decorators, and +# `cls`/`request` are signature-mandated: without these ignores ~90% of the +# findings on a generated web app are false positives. +run_cmd: "vulture {target} --min-confidence 60 --ignore-decorators '@app.*,@*.route,@*.get,@*.post,@*.put,@*.patch,@*.delete,@*.websocket,@*.on_event,@*.exception_handler,@*.middleware,@*.errorhandler,@*.before_request,@*.after_request,@*.teardown_appcontext,@*.context_processor,@*.cli.command,@*.user_loader,@*.listens_for,@*validator,@*.hybrid_property,@*.setter,@*.expression,@lru_cache,@*.fixture' --ignore-names 'cls,request,exc,model_config,__tablename__,Config,orm_mode,from_attributes'" parser_key: vulture +file_patterns: ["*.py"] run_timeout: 90 install_timeout: 180 display_order: 60 diff --git a/backend/analysis/data/tools/wapiti.yaml b/backend/analysis/data/tools/wapiti.yaml new file mode 100644 index 00000000..66e07d46 --- /dev/null +++ b/backend/analysis/data/tools/wapiti.yaml @@ -0,0 +1,51 @@ +slug: wapiti +name: Wapiti +description: >- + Black-box web vulnerability scanner. Crawls the running app and fuzzes its + forms and parameters for XSS, SQL and command injection, path traversal, + open redirects, CRLF and CSRF, and checks cookie flags and security + headers. Requires the app's container to be running. +category: security +kind: container +target_language: any +analysis_mode: dynamic +technologies: [http, rest-api] +icon: bug +version: "3.2" +docs_url: https://wapiti-scanner.github.io/ +# Own virtualenv so its pinned dependencies never clash with other pip tools. +install_cmd: >- + set -e; + python3 -m venv /home/analyzer/.local/opt/wapiti; + /home/analyzer/.local/opt/wapiti/bin/pip install --upgrade 'wapiti3==3.2.*'; + mkdir -p /home/analyzer/.local/bin; + ln -sf /home/analyzer/.local/opt/wapiti/bin/wapiti /home/analyzer/.local/bin/wapiti +verify_cmd: 'v=$(wapiti --version 2>/dev/null | tail -1) && [ -n "$v" ] && echo "wapiti $v"' +uninstall_cmd: "rm -rf /home/analyzer/.local/opt/wapiti /home/analyzer/.local/bin/wapiti" +run_cmd: >- + rm -rf /tmp/wapiti && mkdir -p /tmp/wapiti && + wapiti -u {url}/ --scope folder -m {config:modules} + --max-scan-time {config:max_scan_seconds} --max-attack-time 60 + -f json -o /tmp/wapiti/report.json --store-session /tmp/wapiti/session + --flush-session --no-bugreport >/tmp/wapiti/wapiti.log 2>&1; + cat /tmp/wapiti/report.json +parser_key: wapiti +run_timeout: 900 +install_timeout: 600 +display_order: 7 +config_schema: + - name: modules + type: string + label: Attack modules + description: Comma-separated Wapiti modules (modules that call out to external servers are left out). + default: "backup,cookieflags,crlf,csp,csrf,exec,file,http_headers,methods,permanentxss,redirect,sql,xss" + - name: max_scan_seconds + type: number + label: Max scan time (seconds) + description: Upper bound for the whole scan. + default: 300 + min: 60 + max: 840 +default_config: + modules: "backup,cookieflags,crlf,csp,csrf,exec,file,http_headers,methods,permanentxss,redirect,sql,xss" + max_scan_seconds: 300 diff --git a/backend/analysis/data/tools/zap.yaml b/backend/analysis/data/tools/zap.yaml new file mode 100644 index 00000000..7cae2050 --- /dev/null +++ b/backend/analysis/data/tools/zap.yaml @@ -0,0 +1,65 @@ +slug: zap +name: OWASP ZAP +description: >- + Dynamic application security testing (DAST) against the running app: spiders + its pages, then passively and actively scans them for missing security + headers, XSS, SQL injection, information leaks and misconfigurations. + Requires the app's container to be running. +category: security +kind: container +target_language: any +analysis_mode: dynamic +technologies: [http, rest-api] +icon: globe +version: "2.17" +docs_url: https://www.zaproxy.org/docs/ +# Self-contained: ZAP plus its own Temurin JRE under ~/.local/opt (the base +# image has no Java and no root). Both downloads are checksum-pinned. +install_cmd: >- + set -e; + mkdir -p /home/analyzer/.local/opt /home/analyzer/.local/bin && cd /tmp; + curl -sSfL -o zap-jre.tgz https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.12%2B8/OpenJDK21U-jre_x64_linux_hotspot_21.0.12_8.tar.gz; + echo "8a379a67c91a3ae61ffb33d46e0a40c7ba35e70713c4db31cfca30492f792eff zap-jre.tgz" | sha256sum -c -; + curl -sSfL -o zap.tgz https://github.com/zaproxy/zaproxy/releases/download/v2.17.0/ZAP_2.17.0_Linux.tar.gz; + echo "efe799aaa3627db683b43f00c9c210aea0b75c00cc8f0a0f0434d12bb3ddde5a zap.tgz" | sha256sum -c -; + rm -rf /home/analyzer/.local/opt/zap /home/analyzer/.local/opt/zap-jre; + mkdir -p /home/analyzer/.local/opt/zap /home/analyzer/.local/opt/zap-jre; + tar -xzf zap-jre.tgz -C /home/analyzer/.local/opt/zap-jre --strip-components=1; + tar -xzf zap.tgz -C /home/analyzer/.local/opt/zap --strip-components=1; + rm -f zap.tgz zap-jre.tgz; + printf '#!/bin/sh\nexport JAVA_HOME=/home/analyzer/.local/opt/zap-jre\nexport PATH="$JAVA_HOME/bin:$PATH"\nexec /home/analyzer/.local/opt/zap/zap.sh -Xmx512m "$@"\n' > /home/analyzer/.local/bin/zap; + chmod +x /home/analyzer/.local/bin/zap +verify_cmd: 'v=$(zap -cmd -silent -dir /tmp/zap-verify -version 2>/dev/null | tail -1) && [ -n "$v" ] && echo "ZAP $v"' +uninstall_cmd: "rm -rf /home/analyzer/.local/opt/zap /home/analyzer/.local/opt/zap-jre /home/analyzer/.local/bin/zap" +# Quick scan = spider + active scan. A fresh -dir per run keeps no state +# between apps; the JSON report is the only thing printed. +run_cmd: >- + rm -rf /tmp/zap && mkdir -p /tmp/zap && + zap -cmd -silent -dir /tmp/zap/home + -config spider.maxDuration={config:spider_minutes} + -config scanner.maxScanDurationInMins={config:scan_minutes} + -config scanner.threadPerHost=4 + -quickurl {url}/ -quickout /tmp/zap/report.json >/tmp/zap/zap.log 2>&1; + cat /tmp/zap/report.json +parser_key: zap +run_timeout: 1200 +install_timeout: 900 +display_order: 5 +config_schema: + - name: spider_minutes + type: number + label: Spider time (minutes) + description: Upper bound for crawling the app before scanning. + default: 2 + min: 1 + max: 10 + - name: scan_minutes + type: number + label: Active scan time (minutes) + description: Upper bound for the active (attack) phase. + default: 5 + min: 1 + max: 15 +default_config: + spider_minutes: 2 + scan_minutes: 5 diff --git a/backend/analysis/migrations/0006_tool_mode_and_technologies.py b/backend/analysis/migrations/0006_tool_mode_and_technologies.py new file mode 100644 index 00000000..1ba564c4 --- /dev/null +++ b/backend/analysis/migrations/0006_tool_mode_and_technologies.py @@ -0,0 +1,28 @@ +# Generated by Django 6.0.6 on 2026-10-26 11:59 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('analysis', '0005_installed_tool_updates'), + ] + + operations = [ + migrations.AddField( + model_name='analyzertool', + name='analysis_mode', + field=models.CharField(choices=[('static', 'Static (source code)'), ('dynamic', 'Dynamic (running app)')], db_index=True, default='static', help_text="Static tools read the source; dynamic tools probe the app's running container.", max_length=20, verbose_name='analysis mode'), + ), + migrations.AddField( + model_name='analyzertool', + name='technologies', + field=models.JSONField(blank=True, default=list, help_text='Technology slugs the tool scans (vocabulary in data/tools/catalog.yaml).', verbose_name='technologies'), + ), + migrations.AlterField( + model_name='analyzertool', + name='run_cmd', + field=models.TextField(blank=True, default='', help_text="Template: '{target}' is replaced with the code path, '{url}' with the running app's base URL (dynamic tools) and '{config:}' with a shell-quoted option.", verbose_name='run command'), + ), + ] diff --git a/backend/analysis/migrations/0007_tool_file_patterns.py b/backend/analysis/migrations/0007_tool_file_patterns.py new file mode 100644 index 00000000..a7b7153a --- /dev/null +++ b/backend/analysis/migrations/0007_tool_file_patterns.py @@ -0,0 +1,18 @@ +# Generated by Django 6.0.6 on 2026-10-26 12:48 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('analysis', '0006_tool_mode_and_technologies'), + ] + + operations = [ + migrations.AddField( + model_name='analyzertool', + name='file_patterns', + field=models.JSONField(blank=True, default=list, help_text="Globs (on the file name, or the relative path when they contain '/') of the files the tool reads. A run whose code has none of them skips the tool as not applicable instead of recording a clean result. Empty: always applicable.", verbose_name='file patterns'), + ), + ] diff --git a/backend/analysis/migrations/0008_analysisrun_heartbeat_at.py b/backend/analysis/migrations/0008_analysisrun_heartbeat_at.py new file mode 100644 index 00000000..a22d6d3d --- /dev/null +++ b/backend/analysis/migrations/0008_analysisrun_heartbeat_at.py @@ -0,0 +1,17 @@ +# Generated by Django 6.0.6 on 2026-10-26 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("analysis", "0007_tool_file_patterns"), + ] + + operations = [ + migrations.AddField( + model_name="analysisrun", + name="heartbeat_at", + field=models.DateTimeField(blank=True, null=True, verbose_name="heartbeat at"), + ), + ] diff --git a/backend/analysis/migrations/0009_run_batch_tool_timing_tool_severity.py b/backend/analysis/migrations/0009_run_batch_tool_timing_tool_severity.py new file mode 100644 index 00000000..db6eed7e --- /dev/null +++ b/backend/analysis/migrations/0009_run_batch_tool_timing_tool_severity.py @@ -0,0 +1,58 @@ +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("analysis", "0008_analysisrun_heartbeat_at"), + ] + + operations = [ + migrations.AddField( + model_name="analysisrun", + name="batch_id", + field=models.UUIDField(blank=True, db_index=True, null=True, verbose_name="batch"), + ), + migrations.AddField( + model_name="toolresult", + name="started_at", + field=models.DateTimeField(blank=True, null=True, verbose_name="started at"), + ), + migrations.AddField( + model_name="toolresult", + name="duration_seconds", + field=models.FloatField(blank=True, null=True, verbose_name="duration (s)"), + ), + migrations.AddField( + model_name="finding", + name="tool_severity", + field=models.CharField( + blank=True, + default="", + help_text="The severity/level exactly as the tool printed it; empty when the tool rates nothing.", + max_length=40, + verbose_name="tool severity", + ), + ), + migrations.AlterField( + model_name="finding", + name="severity", + field=models.CharField( + choices=[ + ("critical", "Critical"), + ("high", "High"), + ("medium", "Medium"), + ("low", "Low"), + ("info", "Info"), + ], + db_index=True, + default="info", + help_text=( + "Normalized level for aggregate statistics; a heuristic for tools that rate nothing " + "(see tool_severity)." + ), + max_length=20, + verbose_name="severity", + ), + ), + ] diff --git a/backend/analysis/models.py b/backend/analysis/models.py index b0196032..435363d3 100644 --- a/backend/analysis/models.py +++ b/backend/analysis/models.py @@ -20,6 +20,9 @@ from django.db import models from django.utils.translation import gettext_lazy as _ +# Top-level ``result_data`` fields of non-file-map jobs that hold code. +_JOB_CODE_KEYS = frozenset({"backend", "backend_code", "frontend", "frontend_code", "content"}) + class ToolKind(models.TextChoices): CONTAINER = "container", _("Container CLI") @@ -44,6 +47,11 @@ class TargetLanguage(models.TextChoices): ANY = "any", _("Any") +class AnalysisMode(models.TextChoices): + STATIC = "static", _("Static (source code)") + DYNAMIC = "dynamic", _("Dynamic (running app)") + + class ToolOutputKind(models.TextChoices): FINDINGS = "findings", _("Findings") METRICS = "metrics", _("Metrics") @@ -75,6 +83,20 @@ class AnalyzerTool(models.Model): choices=TargetLanguage, default=TargetLanguage.ANY, ) + analysis_mode = models.CharField( + _("analysis mode"), + max_length=20, + choices=AnalysisMode, + default=AnalysisMode.STATIC, + db_index=True, + help_text=_("Static tools read the source; dynamic tools probe the app's running container."), + ) + technologies = models.JSONField( + _("technologies"), + default=list, + blank=True, + help_text=_("Technology slugs the tool scans (vocabulary in data/tools/catalog.yaml)."), + ) icon = models.CharField(_("icon"), max_length=100, blank=True, default="") version = models.CharField(_("version"), max_length=50, blank=True, default="") docs_url = models.URLField(_("documentation URL"), blank=True, default="") @@ -92,9 +114,22 @@ class AnalyzerTool(models.Model): _("run command"), blank=True, default="", - help_text=_("Template; '{target}' is replaced with the code path."), + help_text=_( + "Template: '{target}' is replaced with the code path, '{url}' with the running " + "app's base URL (dynamic tools) and '{config:}' with a shell-quoted option.", + ), ) parser_key = models.CharField(_("parser key"), max_length=50, blank=True, default="") + file_patterns = models.JSONField( + _("file patterns"), + default=list, + blank=True, + help_text=_( + "Globs (on the file name, or the relative path when they contain '/') of the files the " + "tool reads. A run whose code has none of them skips the tool as not applicable instead " + "of recording a clean result. Empty: always applicable.", + ), + ) output_kind = models.CharField( _("output kind"), max_length=20, @@ -283,6 +318,8 @@ class Status(models.TextChoices): ) source_code = models.JSONField(_("source code"), null=True, blank=True) tool_slugs = models.JSONField(_("tool slugs"), default=list, blank=True) + # Runs launched together (one per app) share a batch id. + batch_id = models.UUIDField(_("batch"), null=True, blank=True, db_index=True) status = models.CharField( _("status"), max_length=20, @@ -295,6 +332,10 @@ class Status(models.TextChoices): duration_seconds = models.FloatField(_("duration (s)"), null=True, blank=True) started_at = models.DateTimeField(_("started at"), null=True, blank=True) completed_at = models.DateTimeField(_("completed at"), null=True, blank=True) + # Refreshed by the executing thread while the run is ``running``; a stale + # heartbeat means that thread is gone (backend reload) and the run can be + # reaped (runner.reap_stale_runs). + heartbeat_at = models.DateTimeField(_("heartbeat at"), null=True, blank=True) created_at = models.DateTimeField(auto_now_add=True) updated_at = models.DateTimeField(auto_now=True) @@ -343,8 +384,9 @@ def get_code_for_analysis(self) -> dict[str, str]: if file_map: return file_map # Scaffolding/custom jobs: top-level string blobs - # (backend_code/frontend_code/content). - return {k: v for k, v in data.items() if isinstance(v, str)} + # (backend_code/frontend_code/content). Other string fields are + # metadata (finish_reason, error_detail, …), not code to scan. + return {k: v for k, v in data.items() if isinstance(v, str) and k in _JOB_CODE_KEYS} return {} @@ -372,6 +414,8 @@ class Status(models.TextChoices): choices=Status.choices, default=Status.PENDING, ) + started_at = models.DateTimeField(_("started at"), null=True, blank=True) + duration_seconds = models.FloatField(_("duration (s)"), null=True, blank=True) raw_output = models.JSONField(_("raw output"), default=dict, blank=True) summary = models.JSONField(_("summary"), default=dict, blank=True) metrics = models.JSONField(_("metrics"), default=dict, blank=True) @@ -387,6 +431,37 @@ class Meta: def __str__(self) -> str: return f"{self.tool_slug} ({self.status})" + @classmethod + def latest_ids_per_job(cls, job_ids=None) -> list: + """Per generation job and tool, the latest result of a finished run. + + A job's code never changes, so each tool's measurement of it stands + on its own: re-running one tool (or only the AI reviewer) must not + discard the other tools' results, as picking the latest *run* would. + A result that measured something (completed, or skipped as not + applicable) wins over a later one that did not (failed, app not + running); a tool that never measured the job keeps its latest result + so coverage can say why. + """ + measured = models.Case( + models.When(status=cls.Status.COMPLETED, then=1), + models.When(status=cls.Status.SKIPPED, summary__skip_reason="not_applicable", then=1), + default=0, + output_field=models.IntegerField(), + ) + qs = cls.objects.filter( + run__generation_job__isnull=False, + run__status__in=[AnalysisRun.Status.COMPLETED, AnalysisRun.Status.PARTIAL, AnalysisRun.Status.FAILED], + ).exclude(status__in=[cls.Status.PENDING, cls.Status.RUNNING]) + if job_ids is not None: + qs = qs.filter(run__generation_job_id__in=list(job_ids)) + return list( + qs.annotate(measured=measured) + .order_by("run__generation_job_id", "tool_slug", "-measured", "-created_at") + .distinct("run__generation_job_id", "tool_slug") + .values_list("id", flat=True), + ) + class Finding(models.Model): """A normalized issue produced by a tool.""" @@ -410,6 +485,16 @@ class Severity(models.TextChoices): choices=Severity.choices, default=Severity.INFO, db_index=True, + help_text=_( + "Normalized level for aggregate statistics; a heuristic for tools that rate nothing (see tool_severity).", + ), + ) + tool_severity = models.CharField( + _("tool severity"), + max_length=40, + blank=True, + default="", + help_text=_("The severity/level exactly as the tool printed it; empty when the tool rates nothing."), ) category = models.CharField(_("category"), max_length=50, blank=True, default="") confidence = models.CharField(_("confidence"), max_length=20, blank=True, default="medium") diff --git a/backend/analysis/seeding.py b/backend/analysis/seeding.py index 79aa8339..70a66b6f 100644 --- a/backend/analysis/seeding.py +++ b/backend/analysis/seeding.py @@ -20,6 +20,15 @@ logger = logging.getLogger(__name__) DATA_DIR = Path(__file__).resolve().parent / "data" / "tools" +CATALOG_FILE = DATA_DIR / "catalog.yaml" + + +def load_catalog_meta() -> dict: + """Display metadata from ``catalog.yaml``: categories, modes, technologies.""" + if not CATALOG_FILE.exists(): + return {} + return yaml.safe_load(CATALOG_FILE.read_text()) or {} + # Fields copied verbatim from YAML into the model (when present). _FIELDS = ( @@ -28,11 +37,14 @@ "category", "kind", "target_language", + "analysis_mode", + "technologies", "icon", "version", "docs_url", "install_cmd", "verify_cmd", + "file_patterns", "uninstall_cmd", "run_cmd", "parser_key", @@ -66,9 +78,10 @@ def seed_tools( emit(f"No data dir at {DATA_DIR}") return 0, 0 + known_techs = {t["slug"] for t in load_catalog_meta().get("technologies", [])} created = updated = 0 for path in sorted(DATA_DIR.glob("*.yaml")): - if path.name == "catalog.yaml": + if path == CATALOG_FILE: continue data = yaml.safe_load(path.read_text()) or {} slug = data.get("slug") or path.stem @@ -84,6 +97,9 @@ def seed_tools( else: updated += 1 emit(f"Updated tool: {slug}") + unknown = set(obj.technologies or []) - known_techs + if unknown: + logger.warning("Tool %r lists technologies missing from catalog.yaml: %s", slug, sorted(unknown)) if obj.kind == "container" and obj.parser_key and not parsers.has_parser(obj.parser_key): logger.warning( "Tool %r declares parser_key %r but no such parser is registered", diff --git a/backend/analysis/services/ai_runner.py b/backend/analysis/services/ai_runner.py index a0cd8881..ec6a1f2e 100644 --- a/backend/analysis/services/ai_runner.py +++ b/backend/analysis/services/ai_runner.py @@ -16,8 +16,10 @@ from typing import TYPE_CHECKING from typing import Any +from backend.analysis.services.base import SEVERITIES from backend.analysis.services.base import AnalyzerOutput from backend.analysis.services.base import FindingData +from backend.analysis.services.parsers import _label if TYPE_CHECKING: from backend.analysis.models import AnalyzerTool @@ -26,7 +28,7 @@ logger = logging.getLogger(__name__) # Default to the latest Claude model for the built-in reviewer. -DEFAULT_MODEL = "anthropic/claude-opus-4-8" +DEFAULT_MODEL = "anthropic/claude-opus-5.5" # Upper bound for a local-model call made inside a web request (tool "test"). INLINE_LOCAL_TIMEOUT_S = 180 @@ -55,7 +57,7 @@ def run( config = config or {} model = config.get("model") or tool.default_config.get("model") or DEFAULT_MODEL - code_blob = _format_code(code) + code_blob, omitted = _format_code(code) max_tokens = int(config.get("max_tokens") or tool.default_config.get("max_tokens") or 4000) messages = [ {"role": "system", "content": _SYSTEM_PROMPT}, @@ -90,7 +92,13 @@ def run( return AnalyzerOutput( findings=findings, raw_output={"model": model, "truncated": truncated, "response": content[:8000]}, - summary={"model": model, "finding_count": len(findings), "truncated": truncated}, + summary={ + "model": model, + "finding_count": len(findings), + "truncated": truncated, + **({"files_not_reviewed": omitted} if omitted else {}), + }, + warnings=[f"Not reviewed (prompt size limit): {', '.join(omitted)}"] if omitted else [], ) @@ -182,12 +190,41 @@ def call() -> dict[str, Any]: return pool.submit(call).result() -def _format_code(code: dict[str, str]) -> str: +# Prompt budget for the code bundle (characters). +MAX_CODE_CHARS = 60000 +# Generated, not written by the model: hashes and resolved versions only. +_LOCKFILES = frozenset({"package-lock.json", "yarn.lock", "pnpm-lock.yaml", "poetry.lock", "pipfile.lock"}) + + +def _review_order(name: str) -> tuple[int, str]: + from backend.reports.services.loc import is_source_file + + # Source first, then manifests/config/docs: when the budget runs out it + # is the least informative files that are left out, never the app code. + return (0 if is_source_file(name) else 1, name) + + +def _format_code(code: dict[str, str]) -> tuple[str, list[str]]: + """The review prompt's code bundle and the files that did not fit in it. + + Whole files only: a file cut mid-way would make the model report the + cut as a syntax error. + """ parts: list[str] = [] - for name, content in code.items(): - if content and content.strip(): - parts.append(f"# ===== {name} =====\n{content}") - return "\n\n".join(parts)[:60000] + omitted: list[str] = [] + used = 0 + for name in sorted(code, key=_review_order): + content = code[name] + base = name.rsplit("/", 1)[-1].lower() + if not content or not content.strip() or base in _LOCKFILES or ".min." in base: + continue + block = f"# ===== {name} =====\n{content}" + if used + len(block) > MAX_CODE_CHARS: + omitted.append(name) + continue + parts.append(block) + used += len(block) + 2 + return "\n\n".join(parts), omitted def _parse_findings(content: str) -> list[FindingData]: @@ -198,10 +235,15 @@ def _parse_findings(content: str) -> list[FindingData]: for item in payload: if not isinstance(item, dict): continue + severity = str(item.get("severity", "info")).strip().lower() + category = str(item.get("category", "quality")).lower() + tool_confidence = _label(item.get("confidence")) findings.append( FindingData( - severity=str(item.get("severity", "info")).lower(), - category=str(item.get("category", "quality")), + # Models drift off the requested vocabulary ("moderate", + # "Security "); keep severities/categories comparable. + severity=_SEVERITY_ALIASES.get(severity, severity if severity in SEVERITIES else "info"), + category=category.strip() if category.strip() in _CATEGORIES else "quality", title=str(item.get("title", "AI finding"))[:500], description=str(item.get("description", "")), suggestion=str(item.get("suggestion", "")), @@ -209,11 +251,19 @@ def _parse_findings(content: str) -> list[FindingData]: line_number=_as_int(item.get("line_number")), rule_id=str(item.get("rule_id", "")), confidence="medium", + # The label exactly as the model wrote it ("Moderate"); empty + # when it gave none. + tool_severity=_label(item.get("severity")), + tool_specific_data={"tool_confidence": tool_confidence} if tool_confidence else {}, ), ) return findings +_SEVERITY_ALIASES = {"moderate": "medium", "warning": "medium", "error": "high", "minor": "low", "major": "high"} +_CATEGORIES = frozenset({"security", "quality", "performance", "style", "best_practice"}) + + def _extract_json_array(content: str) -> list | None: content = content.strip() fence = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", content, re.DOTALL) diff --git a/backend/analysis/services/base.py b/backend/analysis/services/base.py index 9cb230bc..9487672e 100644 --- a/backend/analysis/services/base.py +++ b/backend/analysis/services/base.py @@ -31,6 +31,11 @@ class FindingData: rule_id: str = "" confidence: str = "medium" # high, medium, low tool_specific_data: dict[str, Any] = field(default_factory=dict) + # The severity/level exactly as the tool printed it ("HIGH", "error", + # "convention"); empty when the tool rates nothing. ``severity`` above is + # a normalization (a heuristic for unrated tools) that only aggregate + # statistics use: the UI shows this field, never an invented level. + tool_severity: str = "" @dataclass @@ -44,6 +49,12 @@ class ParsedOutput: findings: list[FindingData] = field(default_factory=list) metrics: dict[str, Any] = field(default_factory=dict) + # Coverage gaps (e.g. one of several manifests could not be audited): the + # result stands but is incomplete. + warnings: list[str] = field(default_factory=list) + # The output shows the tool did not measure anything: the result must be + # recorded as failed, not as a clean (zero-finding) one. + error: str | None = None @dataclass @@ -54,6 +65,7 @@ class AnalyzerOutput: metrics: dict[str, Any] = field(default_factory=dict) summary: dict[str, Any] = field(default_factory=dict) raw_output: dict[str, Any] = field(default_factory=dict) + warnings: list[str] = field(default_factory=list) error: str | None = None @property diff --git a/backend/analysis/services/coverage.py b/backend/analysis/services/coverage.py new file mode 100644 index 00000000..7c26823f --- /dev/null +++ b/backend/analysis/services/coverage.py @@ -0,0 +1,122 @@ +"""Which technologies the user's generated apps contain, and which tools fit them. + +The shop's purpose is to pick instruments for measuring LLM-generated code. A +tool that scans nothing the models produce reports "0 findings", which reads +as "clean" in comparisons; this module lets the shop show, per tool, how many +of the user's generated apps it can actually say something about. + +Detection is deliberately cheap (file names, extensions and a few import / +manifest substrings) over the code map an analysis run would scan. +""" + +from __future__ import annotations + +import re +from pathlib import PurePosixPath +from typing import Any + +from backend.analysis.models import AnalysisRun +from backend.analysis.models import AnalyzerTool + +# Most recent completed jobs considered: enough to describe what the user's +# models generate without scanning a whole archive on every page load. +MAX_APPS = 200 + +_EXT_TECH = { + ".py": {"python"}, + ".js": {"javascript"}, + ".mjs": {"javascript"}, + ".cjs": {"javascript"}, + ".jsx": {"javascript", "react"}, + ".ts": {"javascript"}, + ".tsx": {"javascript", "react"}, + ".vue": {"javascript", "vue"}, + ".html": {"html"}, + ".css": {"css"}, + ".sh": {"shell"}, + ".tf": {"terraform"}, +} +_PY_MANIFESTS = {"requirements.txt", "pyproject.toml", "pipfile", "setup.py", "requirements-dev.txt"} +_PY_FRAMEWORKS = { + "flask": re.compile(r"^\s*(from|import)\s+flask\b|^flask\b", re.IGNORECASE | re.MULTILINE), + "fastapi": re.compile(r"^\s*(from|import)\s+fastapi\b|^fastapi\b", re.IGNORECASE | re.MULTILINE), + "django": re.compile(r"^\s*(from|import)\s+django\b|^django\b", re.IGNORECASE | re.MULTILINE), +} +_EXPRESS_RE = re.compile(r"""require\(\s*['"]express['"]\s*\)|from\s+['"]express['"]|"express"\s*:""") +_REACT_RE = re.compile(r"""from\s+['"]react['"]|require\(\s*['"]react['"]\s*\)|"react"\s*:""") +_VUE_RE = re.compile(r"""from\s+['"]vue['"]|"vue"\s*:""") +_TEMPLATE_RE = re.compile(r"\{\{.*?\}\}|\{%.*?%\}", re.DOTALL) +_K8S_RE = re.compile(r"^apiVersion:.*^kind:", re.MULTILINE | re.DOTALL) +_WEB_BACKENDS = {"flask", "fastapi", "django", "express"} + + +def detect_technologies(files: dict[str, str]) -> set[str]: + """Technology slugs (catalog vocabulary) present in a ``{path: content}`` map.""" + found: set[str] = set() + for path, content in files.items(): + name = PurePosixPath(path).name.lower() + suffix = PurePosixPath(name).suffix + text = content if isinstance(content, str) else "" + found |= _EXT_TECH.get(suffix, set()) + if name in _PY_MANIFESTS or (name.startswith("requirements") and suffix == ".txt"): + found.add("pypi") + if name == "package.json": + found.add("npm") + if name == "dockerfile" or name.startswith("dockerfile.") or suffix == ".dockerfile": + found.add("docker") + if suffix == ".py" or name in _PY_MANIFESTS or name.startswith("requirements"): + found |= {fw for fw, rx in _PY_FRAMEWORKS.items() if rx.search(text)} + if suffix in (".js", ".mjs", ".cjs", ".ts", ".jsx", ".tsx") or name == "package.json": + if _EXPRESS_RE.search(text): + found.add("express") + if _REACT_RE.search(text): + found.add("react") + if _VUE_RE.search(text): + found.add("vue") + if suffix == ".html" and _TEMPLATE_RE.search(text): + found.add("jinja") + if suffix in (".yaml", ".yml") and _K8S_RE.search(text): + found.add("kubernetes") + if found & _WEB_BACKENDS: + # A web backend is something dynamic scanners can probe once it runs. + found |= {"http", "rest-api"} + elif "html" in found: + found.add("http") + if found: + found.add("generic") + return found + + +def app_coverage(user) -> dict[str, Any]: + """Technology and per-tool fit counts over *user*'s recent generated apps.""" + from backend.analysis.services.runner import _materialize + from backend.generation.models import GenerationJob + + jobs = ( + GenerationJob.objects.filter(created_by=user, status=GenerationJob.Status.COMPLETED) + .only("id", "result_data") + .order_by("-created_at")[:MAX_APPS] + ) + per_app: list[set[str]] = [] + for job in jobs: + code = AnalysisRun(generation_job=job).get_code_for_analysis() + techs = detect_technologies(_materialize(code)) + if techs: + per_app.append(techs) + + tech_counts: dict[str, int] = {} + for techs in per_app: + for slug in techs: + tech_counts[slug] = tech_counts.get(slug, 0) + 1 + + tool_counts: dict[str, int] = {} + for tool in AnalyzerTool.objects.filter(is_enabled=True).only("slug", "technologies"): + wanted = set(tool.technologies or []) + tool_counts[tool.slug] = sum(1 for techs in per_app if techs & wanted) + + return { + "apps_total": len(per_app), + "apps_considered": MAX_APPS, + "technologies": tech_counts, + "tools": tool_counts, + } diff --git a/backend/analysis/services/extra_parsers.py b/backend/analysis/services/extra_parsers.py index 85586115..12f8ef39 100644 --- a/backend/analysis/services/extra_parsers.py +++ b/backend/analysis/services/extra_parsers.py @@ -17,27 +17,94 @@ from backend.analysis.services.base import FindingData from backend.analysis.services.base import ParsedOutput +from backend.analysis.services.parsers import _compact +from backend.analysis.services.parsers import _float +from backend.analysis.services.parsers import _label from backend.analysis.services.parsers import _loads +from backend.analysis.services.parsers import _pyflakes_severity +from backend.analysis.services.parsers import failure_lines from backend.analysis.services.parsers import parse_eslint from backend.analysis.services.parsers import parse_pylint from backend.analysis.services.parsers import register +from backend.analysis.services.parsers import section_bodies -_SECTION_RE = re.compile(r"^### (.+)$", re.MULTILINE) _CWE_RE = re.compile(r"CWE-(\d+)", re.IGNORECASE) def _sections(raw: str) -> list[tuple[str, Any]]: - """``[(path, parsed_json)]`` for ``### path`` sections; unparsable bodies are dropped.""" - parts = _SECTION_RE.split(raw or "") + """``[(path, parsed_json)]`` for ``### path`` sections; unparsable bodies are dropped + (``parsers.parse`` reports them as manifests that were not analyzed).""" out: list[tuple[str, Any]] = [] - # split() yields [preamble, path1, body1, path2, body2, ...] - for path, body in zip(parts[1::2], parts[2::2], strict=False): + for path, body in section_bodies(raw): data = _loads(body) if data is not None: - out.append((path.strip(), data)) + out.append((path, data)) return out +# pip / npm resolution errors the audit commands echo as "!! " lines. +_PIP_NO_DIST_RE = re.compile(r"No matching distribution found for (\S+)") +_PIP_NO_VERSIONS_RE = re.compile(r"satisfies the requirement (\S+) \(from versions: none\)") +_NPM_NOT_IN_REGISTRY_RE = re.compile(r"'(\S+?)' is not in this registry") +_NPM_NO_VERSION_RE = re.compile(r"No matching version found for (\S+?)\.?$") +_REQ_NAME_RE = re.compile(r"^(@?[A-Za-z0-9_.\-/]+?)(?:\[.*\])?(?:[=<>!~@^].*)?$") + + +def _unresolvable(path: str, spec: str, *, exists: bool, ecosystem: str) -> FindingData: + """A declared dependency the package index cannot satisfy: the app cannot be installed. + + A package missing from the registry is the hallucinated-dependency + ("slopsquatting") pattern: anyone may later publish that name. + """ + m = _REQ_NAME_RE.match(spec) + name = m.group(1) if m else spec + if exists: + title = f"{spec}: no such version on {ecosystem}" + rule, severity = "nonexistent-version", "medium" + suggestion = f"Pin {name} to a released version" + else: + title = f"{name}: package does not exist on {ecosystem}" + rule, severity = "nonexistent-package", "high" + suggestion = f"Remove {name} or replace it with the intended, existing package" + return FindingData( + severity=severity, + category="dependency", + title=title, + description="The dependency cannot be resolved, so installing the app fails. " + + ("" if exists else "An unregistered package name can be claimed by anyone (hallucinated dependency)."), + suggestion=suggestion, + file_path=path, + rule_id=rule, + confidence="high", + tool_specific_data={"package": name, "requirement": spec, "ecosystem": ecosystem}, + ) + + +def _resolution_findings(raw: str, ecosystem: str) -> list[FindingData]: + findings: list[FindingData] = [] + for path, body in section_bodies(raw): + lines = failure_lines(body) + text = "\n".join(lines) + seen: set[str] = set() + if ecosystem == "PyPI": + missing = set(_PIP_NO_VERSIONS_RE.findall(text)) + for spec in _PIP_NO_DIST_RE.findall(text): + if spec not in seen: + seen.add(spec) + findings.append(_unresolvable(path, spec, exists=spec not in missing, ecosystem=ecosystem)) + else: + for spec in _NPM_NOT_IN_REGISTRY_RE.findall(text): + if spec not in seen: + seen.add(spec) + findings.append(_unresolvable(path, spec, exists=False, ecosystem=ecosystem)) + for line in lines: + hit = _NPM_NO_VERSION_RE.search(line) + if hit and hit.group(1) not in seen: + seen.add(hit.group(1)) + findings.append(_unresolvable(path, hit.group(1), exists=True, ecosystem=ecosystem)) + return findings + + def _json_docs(raw: str) -> list[Any]: """Every top-level JSON value in *raw* (xargs may run a tool more than once).""" decoder = json.JSONDecoder() @@ -67,10 +134,10 @@ def _cwe(value: Any) -> dict[str, str]: # --- Dependency vulnerabilities --------------------------------------------- -@register("pip_audit") +@register("pip_audit", sectioned=True) def parse_pip_audit(raw: str) -> list[FindingData]: """``pip-audit -f json``: ``{"dependencies": [{name, version, vulns: [...]}]}``.""" - findings: list[FindingData] = [] + findings: list[FindingData] = _resolution_findings(raw, "PyPI") for path, data in _sections(raw): if not isinstance(data, dict): continue @@ -89,21 +156,29 @@ def parse_pip_audit(raw: str) -> list[FindingData]: description = vuln.get("description") or "" findings.append( FindingData( - # The OSV records pip-audit returns carry no CVSS score. + # The OSV records pip-audit returns carry no severity + # or CVSS score: ``tool_severity`` stays empty. severity="medium", category="dependency", title=f"{name} {version}: {cve or vid}", description=description, - suggestion=f"Upgrade {name} to {' or '.join(fixes)}" if fixes else "No fixed release yet", + suggestion=f"Upgrade {name} to {' or '.join(fixes)}" if fixes else "", file_path=path, rule_id=vid, confidence="high", - tool_specific_data={ - "package": name, - "version": version, - "aliases": aliases, - "fix_versions": fixes, - }, + tool_specific_data=_compact( + { + "package": name, + "installed_version": version, + "fixed_versions": fixes, + "vuln_id": vid, + "aliases": aliases, + "manifest": path, + # Legacy keys. + "version": version, + "fix_versions": fixes, + }, + ), ), ) return findings @@ -112,14 +187,14 @@ def parse_pip_audit(raw: str) -> list[FindingData]: _NPM_SEVERITY = {"critical": "critical", "high": "high", "moderate": "medium", "low": "low", "info": "info"} -@register("npm_audit") +@register("npm_audit", sectioned=True) def parse_npm_audit(raw: str) -> list[FindingData]: """``npm audit --json`` (report v2): one finding per advisory. Entries whose ``via`` holds only package names are inherited from another vulnerable package and are already reported through that package. """ - findings: list[FindingData] = [] + findings: list[FindingData] = _resolution_findings(raw, "npm") for path, data in _sections(raw): if not isinstance(data, dict): continue @@ -140,22 +215,36 @@ def parse_npm_audit(raw: str) -> list[FindingData]: if key in seen: continue seen.add(key) + cvss = via.get("cvss") or {} + # npm prints score 0 / vectorString null for an unscored advisory. + score = _float(cvss.get("score")) or None + url = str(via.get("url") or "") + advisory_id = url.rsplit("/", 1)[-1] if url else "" findings.append( FindingData( severity=_NPM_SEVERITY.get(str(via.get("severity") or entry.get("severity")), "medium"), category="dependency", title=f"{pkg} {via.get('range', '')}: {via.get('title', 'Vulnerable dependency')}".strip(), - description=via.get("url", ""), + # npm audit's report has no advisory text, only its URL (in docs_url). + description="", suggestion=suggestion, file_path=path, - rule_id=str(via.get("url", "")).rsplit("/", 1)[-1] or str(via.get("source", "")), + rule_id=advisory_id or str(via.get("source", "")), confidence="high", - tool_specific_data={ - "package": pkg, - "direct": bool(entry.get("isDirect")), - "cvss": (via.get("cvss") or {}).get("score"), - **_cwe(via.get("cwe")), - }, + tool_severity=_label(via.get("severity") or entry.get("severity")), + tool_specific_data=_compact( + { + "package": pkg, + "direct": bool(entry.get("isDirect")), + "vuln_id": advisory_id, + "docs_url": url, + "cvss_score": score, + "cvss_vector": cvss.get("vectorString") if score else None, + "manifest": path, + "cvss": score, # legacy key + **_cwe(via.get("cwe")), + }, + ), ), ) return findings @@ -172,7 +261,7 @@ def parse_npm_audit(raw: str) -> list[FindingData]: } -@register("deptry") +@register("deptry", sectioned=True) def parse_deptry(raw: str) -> list[FindingData]: """``deptry --json-output``: ``[{error: {code, message}, module, location}]``.""" findings: list[FindingData] = [] @@ -187,13 +276,11 @@ def parse_deptry(raw: str) -> list[FindingData]: message = error.get("message", "") findings.append( FindingData( + # deptry rates nothing: ``tool_severity`` stays empty. severity=_DEPTRY_SEVERITY.get(code, "low"), category="dependency", title=message or f"{code} {item.get('module', '')}", description=message, - suggestion=( - f"Add the package providing '{item.get('module')}' to requirements" if code == "DEP001" else "" - ), file_path=prefix + str(loc.get("file") or ""), line_number=loc.get("line"), column_number=loc.get("column"), @@ -205,7 +292,7 @@ def parse_deptry(raw: str) -> list[FindingData]: return findings -@register("depcheck") +@register("depcheck", sectioned=True) def parse_depcheck(raw: str) -> list[FindingData]: """``depcheck --json``: unused ``dependencies``/``devDependencies`` and ``missing``.""" findings: list[FindingData] = [] @@ -221,7 +308,6 @@ def parse_depcheck(raw: str) -> list[FindingData]: category="dependency", title=f"'{pkg}' is imported but missing from package.json", description=f"Used in {len(files or [])} file(s); the build fails until it is declared.", - suggestion=f"npm install {pkg}", file_path=first, rule_id="missing-dependency", confidence="high", @@ -235,11 +321,10 @@ def parse_depcheck(raw: str) -> list[FindingData]: severity=severity, category="dependency", title=f"'{pkg}' is declared in {key} but never imported", - suggestion=f"Remove {pkg} from package.json if it is not needed", file_path=root + "package.json" if root else manifest, rule_id=f"unused-{key}", confidence="medium", - tool_specific_data={"package": pkg}, + tool_specific_data={"package": pkg, "manifest": manifest}, ), ) return findings @@ -250,16 +335,6 @@ def parse_depcheck(raw: str) -> list[FindingData]: _FLAKE8_LINE_RE = re.compile(r"^(?P.+?):(?P\d+):(?P\d+): (?P[A-Z]+\d+) (?P.*)$") -def _flake8_severity(code: str) -> str: - # E9: syntax/IO errors; F63/F7/F82: invalid comparisons, misplaced - # statements, undefined names: all crash or misbehave at runtime. - if code.startswith(("E9", "F63", "F7", "F82")): - return "high" - if code.startswith("F"): - return "medium" - return "low" - - @register("flake8") def parse_flake8(raw: str) -> list[FindingData]: """flake8 default ``path:row:col: CODE message`` lines.""" @@ -271,7 +346,7 @@ def parse_flake8(raw: str) -> list[FindingData]: code = m.group("code") findings.append( FindingData( - severity=_flake8_severity(code), + severity=_pyflakes_severity(code), category="quality" if code.startswith(("F", "E9", "C9")) else "style", title=m.group("msg"), description=m.group("msg"), @@ -292,6 +367,8 @@ def parse_perflint(raw: str) -> list[FindingData]: for f in findings: f.category = "performance" f.severity = "low" + # Perflint's checks (W8xxx) are a plugin's: pylint's message pages do not cover them. + f.tool_specific_data.pop("docs_url", None) return findings @@ -329,18 +406,26 @@ def parse_lizard(raw: str) -> ParsedOutput: functions.append({"file": rec["file"], "name": rec["function"], "line": start, "ccn": ccn, "nloc": nloc}) severity = next((sev for limit, sev in _LIZARD_THRESHOLDS if ccn > limit), None) if severity: + end_line = int(rec["end"]) if rec["end"].isdigit() else start findings.append( FindingData( + # Lizard rates nothing (only a CCN warning threshold): + # ``tool_severity`` stays empty. severity=severity, category="performance", title=f"Function '{rec['function']}' has cyclomatic complexity {ccn}", description=f"{nloc} lines of code, {params} parameter(s), {length} lines long.", - suggestion="Split it into smaller functions or simplify the branching", file_path=rec["file"], line_number=start, rule_id="CCN", confidence="high", - tool_specific_data={"ccn": ccn, "nloc": nloc, "end_line": int(rec["end"] or start)}, + tool_specific_data={ + "complexity": ccn, + "function": rec["function"], + "nloc": nloc, + "end_line": end_line, + "ccn": ccn, # legacy key + }, ), ) if not functions: @@ -420,11 +505,21 @@ def parse_htmlhint(raw: str) -> list[FindingData]: code_snippet=str(msg.get("evidence") or "")[:300], rule_id=rule.get("id", ""), confidence="high", + tool_severity=_label(msg.get("type")), + # htmlhint links each rule's page in its report. + tool_specific_data=_compact({"docs_url": rule.get("link")}), ), ) return findings +def _stylelint_docs_url(rule: str) -> str: + """The page of a core stylelint rule; plugin rules (``plugin/rule``) and syntax errors have none.""" + if re.fullmatch(r"[a-z][a-z0-9-]*", rule or ""): + return f"https://stylelint.io/user-guide/rules/{rule}" + return "" + + @register("stylelint") def parse_stylelint(raw: str) -> list[FindingData]: """``stylelint -f json``: ``[{source, warnings: [...], parseErrors: [...]}]``.""" @@ -433,6 +528,7 @@ def parse_stylelint(raw: str) -> list[FindingData]: for entry in doc if isinstance(doc, list) else []: for warning in (entry.get("warnings") or []) + (entry.get("parseErrors") or []): text = warning.get("text", "CSS issue") + rule = warning.get("rule", "parse-error") findings.append( FindingData( severity="medium" if warning.get("severity") == "error" else "low", @@ -443,8 +539,16 @@ def parse_stylelint(raw: str) -> list[FindingData]: file_path=entry.get("source", ""), line_number=warning.get("line"), column_number=warning.get("column"), - rule_id=warning.get("rule", "parse-error"), + rule_id=rule, confidence="high", + tool_severity=_label(warning.get("severity")), + tool_specific_data=_compact( + { + "docs_url": _stylelint_docs_url(rule) if rule != "parse-error" else "", + "end_line": warning.get("endLine"), + "end_column": warning.get("endColumn"), + }, + ), ), ) return findings @@ -466,12 +570,19 @@ def parse_shellcheck(raw: str) -> list[FindingData]: category="quality", title=item.get("message", "Shell issue"), description=item.get("message", ""), - suggestion=f"See https://www.shellcheck.net/wiki/SC{code}" if code else "", file_path=item.get("file", ""), line_number=item.get("line"), column_number=item.get("column"), rule_id=f"SC{code}" if code else "", confidence="high", + tool_severity=_label(item.get("level")), + tool_specific_data=_compact( + { + "docs_url": f"https://www.shellcheck.net/wiki/SC{code}" if code else None, + "end_line": item.get("endLine"), + "end_column": item.get("endColumn"), + }, + ), ), ) return findings diff --git a/backend/analysis/services/parsers.py b/backend/analysis/services/parsers.py index 1271d9d1..07fd5c96 100644 --- a/backend/analysis/services/parsers.py +++ b/backend/analysis/services/parsers.py @@ -20,11 +20,19 @@ logger = logging.getLogger(__name__) _PARSERS: dict[str, Callable[[str], list[FindingData] | ParsedOutput]] = {} +# Parsers of ``### ``-sectioned output (one report per manifest). +_SECTIONED: set[str] = set() +_SECTION_RE = re.compile(r"^### (.+)$", re.MULTILINE) +# Lines a sectioned run command prints when a manifest's tool run failed. +_FAILURE_LINE_PREFIX = "!! " -def register(key: str) -> Callable[[Callable], Callable]: + +def register(key: str, *, sectioned: bool = False) -> Callable[[Callable], Callable]: def _wrap(fn: Callable[[str], list[FindingData] | ParsedOutput]) -> Callable: _PARSERS[key] = fn + if sectioned: + _SECTIONED.add(key) return fn return _wrap @@ -35,15 +43,60 @@ def parse(parser_key: str, raw_output: str) -> ParsedOutput: fn = _PARSERS.get(parser_key) if fn is None: logger.warning("No parser registered for key %r", parser_key) - return ParsedOutput() + return ParsedOutput(error=f"No parser registered for {parser_key!r}") try: result = fn(raw_output or "") - except Exception: + except Exception as exc: logger.exception("Parser %r failed", parser_key) - return ParsedOutput() - if isinstance(result, ParsedOutput): - return result - return ParsedOutput(findings=result) + # A crashed parser measured nothing; never let it pass as a clean result. + return ParsedOutput(error=f"Could not parse the tool output ({type(exc).__name__}: {exc})"[:500]) + parsed = result if isinstance(result, ParsedOutput) else ParsedOutput(findings=result) + if parser_key in _SECTIONED: + _check_sections(raw_output or "", parsed) + return parsed + + +def section_bodies(raw: str) -> list[tuple[str, str]]: + """``[(path, body)]`` for each ``### `` section of *raw*.""" + parts = _SECTION_RE.split(raw or "") + # split() yields [preamble, path1, body1, path2, body2, ...] + return [(path.strip(), body) for path, body in zip(parts[1::2], parts[2::2], strict=False)] + + +def failure_lines(body: str) -> list[str]: + """The ``!! `` diagnostic lines a run command printed for a failed manifest.""" + return [ + line[len(_FAILURE_LINE_PREFIX) :].strip() + for line in body.splitlines() + if line.startswith(_FAILURE_LINE_PREFIX) + ] + + +def _check_sections(raw: str, parsed: ParsedOutput) -> None: + """Flag manifests whose section holds no report: they were not analyzed. + + Multi-manifest tools (pip-audit, npm audit, deptry, depcheck) run once + per manifest; a run that fails (an unresolvable pin, a package missing + from the registry) leaves an empty section. Counting that as "no + vulnerabilities" would make broken apps look clean. + + Each unanalyzed manifest is a coverage gap in ``warnings``; when none was + analyzed, ``error`` is set too (the result is ``failed``: not a + measurement). Findings the parser derived from the failure itself + (``nonexistent-package``/``-version``) are kept either way: the runner + stores them on the failed result. + """ + sections = section_bodies(raw) + failed = [] + for path, body in sections: + if _loads(body) is None: + reason = "; ".join(failure_lines(body))[:300] + failed.append(f"{path}: {reason}" if reason else path) + if not failed: + return + parsed.warnings.append("Not analyzed: " + " | ".join(failed)) + if len(failed) == len(sections): + parsed.error = "No manifest could be analyzed — " + " | ".join(failed) def has_parser(parser_key: str) -> bool: @@ -71,6 +124,71 @@ def _loads(raw: str) -> Any: return None +# --- Shared helpers --------------------------------------------------------- +# +# ``FindingData.tool_severity`` and ``tool_specific_data["tool_confidence"]`` +# hold the tool's own labels verbatim; ``severity``/``confidence`` are our +# normalization. ``tool_specific_data`` uses a shared key vocabulary the UI +# renders generically (rule_name, docs_url, references, cwe, owasp, end_line, +# end_column, fix, package, installed_version, fixed_versions, vuln_id, +# aliases, cvss_score, cvss_vector, manifest, url, method, param, evidence, +# attack, instances, other_info, secret_type, complexity, function, nloc, +# duplicate_of, lines, tokens) and only carries keys with a real value. + + +# Finding.tool_severity is a CharField(max_length=40). +_LABEL_MAX = 40 + + +def _label(value: Any) -> str: + """A severity/confidence label exactly as the tool printed it; "" when it gave none.""" + if value is None or isinstance(value, (dict, list, bool)): + return "" + return str(value).strip()[:_LABEL_MAX] + + +def _compact(data: dict[str, Any]) -> dict[str, Any]: + """*data* without the keys that hold no value (None, "", [], {}); 0 and False are values.""" + return {k: v for k, v in data.items() if v is not None and not (isinstance(v, (str, list, dict)) and not v)} + + +def _int(value: Any) -> int | None: + if isinstance(value, bool): + return None + try: + return int(value) + except (TypeError, ValueError): + return None + + +def _float(value: Any) -> float | None: + if isinstance(value, bool): + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + +_URL_RE = re.compile(r"https?://[^\s<>\"']+") + + +def _urls(*values: Any, limit: int = 20) -> list[str]: + """Distinct URLs found in *values* (strings or lists of strings), in order.""" + out: list[str] = [] + for value in values: + for item in value if isinstance(value, list) else [value]: + for found in _URL_RE.findall(str(item or "")): + url = found.rstrip(".,;)") + if url not in out: + out.append(url) + return out[:limit] + + +def _clip(value: Any, limit: int) -> str: + return str(value or "")[:limit] + + _CWE_RE = re.compile(r"CWE-?(\d+)", re.IGNORECASE) @@ -98,6 +216,7 @@ def parse_bandit(raw: str) -> list[FindingData]: conf_map = {"HIGH": "high", "MEDIUM": "medium", "LOW": "low"} findings: list[FindingData] = [] for item in data.get("results", []): + line_range = [n for n in item.get("line_range") or [] if isinstance(n, int)] findings.append( FindingData( severity=sev_map.get(item.get("issue_severity", "LOW"), "low"), @@ -109,24 +228,50 @@ def parse_bandit(raw: str) -> list[FindingData]: code_snippet=(item.get("code") or "")[:500], rule_id=item.get("test_id", ""), confidence=conf_map.get(item.get("issue_confidence", "MEDIUM"), "medium"), - tool_specific_data={ - "test_name": item.get("test_name", ""), - **_cwe_field((item.get("issue_cwe") or {}).get("id")), - }, + tool_severity=_label(item.get("issue_severity")), + tool_specific_data=_compact( + { + "rule_name": item.get("test_name"), + "test_name": item.get("test_name", ""), # legacy key + "tool_confidence": _label(item.get("issue_confidence")), + "docs_url": item.get("more_info"), + "end_line": max(line_range) if line_range else None, + **_cwe_field((item.get("issue_cwe") or {}).get("id")), + }, + ), ), ) return findings +def _pyflakes_severity(code: str) -> str: + """Severity of a pycodestyle/pyflakes code (shared by flake8 and Ruff).""" + # E9: syntax/IO errors; F63/F7/F82: invalid comparisons, misplaced + # statements, undefined names: all crash or misbehave at runtime. + if code.startswith(("E9", "F63", "F7", "F82")): + return "high" + if code in ("F401", "F841"): # unused import / variable: dead code, harmless + return "low" + if code.startswith("F"): + return "medium" + return "low" + + @register("ruff") def parse_ruff(raw: str) -> list[FindingData]: data = _loads(raw) or [] findings: list[FindingData] = [] for item in data: loc = item.get("location") or {} + end = item.get("end_location") or {} + code = item.get("code") or "" + url = str(item.get("url") or "") findings.append( FindingData( - severity="low", + # Ruff's codes are flake8's: rate them the same way so the + # two tools' severities stay comparable. Ruff itself rates + # nothing, so ``tool_severity`` stays empty. + severity=_pyflakes_severity(code), category="quality", title=item.get("message", "Lint issue"), description=item.get("message", ""), @@ -135,11 +280,32 @@ def parse_ruff(raw: str) -> list[FindingData]: column_number=loc.get("column"), rule_id=item.get("code") or "", confidence="high", + tool_specific_data=_compact( + { + # The rule doc URL ends in the rule's name (".../rules/unused-import"). + "rule_name": url.rstrip("/").rsplit("/", 1)[-1] if "/rules/" in url else None, + "docs_url": url, + "fix": (item.get("fix") or {}).get("message"), + "end_line": end.get("row"), + "end_column": end.get("column"), + }, + ), ), ) return findings +# What the ESLint CLI prints for a message's numeric severity. +_ESLINT_LEVELS = {1: "warning", 2: "error"} + + +def _eslint_docs_url(rule_id: str) -> str: + """The rule page of a core ESLint rule; plugin rules (``plugin/rule``) have none we can vouch for.""" + if rule_id and "/" not in rule_id and re.fullmatch(r"[a-z][a-z0-9-]*", rule_id): + return f"https://eslint.org/docs/latest/rules/{rule_id}" + return "" + + @register("eslint") def parse_eslint(raw: str) -> list[FindingData]: data = _loads(raw) or [] @@ -148,6 +314,10 @@ def parse_eslint(raw: str) -> list[FindingData]: for file_entry in data: file_path = file_entry.get("filePath", "") for msg in file_entry.get("messages", []): + rule_id = msg.get("ruleId") or "" + suggestions = [ + s.get("desc") for s in msg.get("suggestions") or [] if isinstance(s, dict) and s.get("desc") + ] findings.append( FindingData( severity=sev_map.get(msg.get("severity", 1), "low"), @@ -157,13 +327,32 @@ def parse_eslint(raw: str) -> list[FindingData]: file_path=file_path, line_number=msg.get("line"), column_number=msg.get("column"), - rule_id=msg.get("ruleId") or "", + rule_id=rule_id, confidence="high", + tool_severity=_ESLINT_LEVELS.get(msg.get("severity"), ""), + tool_specific_data=_compact( + { + "docs_url": _eslint_docs_url(rule_id), + "fix": "; ".join(suggestions), + "end_line": msg.get("endLine"), + "end_column": msg.get("endColumn"), + }, + ), ), ) return findings +_SEMGREP_CATEGORIES = { + "security": "security", + "correctness": "quality", + "best-practice": "best_practice", + "maintainability": "quality", + "performance": "performance", + "portability": "quality", +} + + @register("semgrep") def parse_semgrep(raw: str) -> list[FindingData]: data = _loads(raw) or {} @@ -174,27 +363,60 @@ def parse_semgrep(raw: str) -> list[FindingData]: start = item.get("start") or {} end = item.get("end") or {} message = extra.get("message") or item.get("check_id", "Semgrep finding") + metadata = extra.get("metadata") or {} + # The registry packs mix security rules with correctness/best-practice + # ones; only the former belong in security counts. + rule_category = str(metadata.get("category") or "security").lower() + check_id = item.get("check_id", "") + lines = extra.get("lines") or "" + owasp = metadata.get("owasp") or [] findings.append( FindingData( severity=sev_map.get(str(extra.get("severity", "INFO")).upper(), "low"), - category="security", + category=_SEMGREP_CATEGORIES.get(rule_category, "quality"), title=(message.splitlines() or ["Semgrep finding"])[0][:200], description=message, file_path=item.get("path", ""), line_number=start.get("line"), column_number=start.get("col"), - code_snippet=(extra.get("lines") or "")[:500], - rule_id=item.get("check_id", ""), + # Semgrep OSS without a login prints this placeholder instead of the code. + code_snippet="" if lines.strip() == _SEMGREP_NO_LINES else lines[:500], + rule_id=check_id, confidence=str((extra.get("metadata") or {}).get("confidence", "medium")).lower() or "medium", - tool_specific_data={ - "end_line": end.get("line"), - **_cwe_field((extra.get("metadata") or {}).get("cwe")), - }, + tool_severity=_label(extra.get("severity")), + tool_specific_data=_compact( + { + "rule_name": check_id.rsplit(".", 1)[-1] if check_id else None, + "tool_confidence": _label(metadata.get("confidence")), + "likelihood": _label(metadata.get("likelihood")), + "impact": _label(metadata.get("impact")), + "docs_url": metadata.get("source") or metadata.get("shortlink"), + "references": _urls(metadata.get("references")), + "owasp": [str(o) for o in (owasp if isinstance(owasp, list) else [owasp]) if o], + "fix": extra.get("fix") if isinstance(extra.get("fix"), str) else None, + "end_line": end.get("line"), + "end_column": end.get("col"), + **_cwe_field((extra.get("metadata") or {}).get("cwe")), + }, + ), ), ) return findings +_SEMGREP_NO_LINES = "requires login" + +# Pylint message type -> the folder of its message pages in the docs. +_PYLINT_DOC_DIRS = { + "convention": "convention", + "refactor": "refactor", + "warning": "warning", + "error": "error", + "fatal": "fatal", + "info": "information", +} + + @register("pylint") def parse_pylint(raw: str) -> list[FindingData]: data = _loads(raw) or [] @@ -209,6 +431,8 @@ def parse_pylint(raw: str) -> list[FindingData]: findings: list[FindingData] = [] for item in data: msg_type = str(item.get("type", "warning")).lower() + symbol = item.get("symbol") or "" + doc_dir = _PYLINT_DOC_DIRS.get(msg_type) findings.append( FindingData( severity=sev_map.get(msg_type, "low"), @@ -218,9 +442,22 @@ def parse_pylint(raw: str) -> list[FindingData]: file_path=item.get("path", ""), line_number=item.get("line"), column_number=item.get("column"), - rule_id=item.get("symbol") or item.get("message-id") or "", + rule_id=symbol or item.get("message-id") or "", confidence="high", - tool_specific_data={"obj": item.get("obj", "")}, + tool_severity=_label(item.get("type")), + tool_specific_data=_compact( + { + "obj": item.get("obj", ""), + "message_id": item.get("message-id"), + "docs_url": ( + f"https://pylint.readthedocs.io/en/stable/user_guide/messages/{doc_dir}/{symbol}.html" + if doc_dir and symbol + else None + ), + "end_line": item.get("endLine"), + "end_column": item.get("endColumn"), + }, + ), ), ) return findings @@ -238,6 +475,7 @@ def parse_mypy(raw: str) -> list[FindingData]: """Mypy has no stable JSON output; parse its ``file:line: level: msg`` text.""" sev_map = {"error": "medium", "warning": "low", "note": "info"} findings: list[FindingData] = [] + last: FindingData | None = None for raw_line in (raw or "").splitlines(): line = raw_line.rstrip() if not line: @@ -245,25 +483,37 @@ def parse_mypy(raw: str) -> list[FindingData]: m = _MYPY_LINE_RE.match(line) if not m: continue - # "note" lines are usually continuations of the preceding error. if m.group("level") == "note": + # A note continues the preceding error on the same line ("See + # ", "Possible overload variants: ..."): attach it there + # rather than reporting it on its own. + if last is not None and last.file_path == m.group("file") and last.line_number == int(m.group("line")): + _attach_mypy_note(last, m.group("msg").strip()) continue - findings.append( - FindingData( - severity=sev_map.get(m.group("level"), "low"), - category="quality", - title=m.group("msg").strip(), - description=m.group("msg").strip(), - file_path=m.group("file"), - line_number=int(m.group("line")), - column_number=int(m.group("col")) if m.group("col") else None, - rule_id=m.group("code") or "", - confidence="high", - ), + last = FindingData( + severity=sev_map.get(m.group("level"), "low"), + category="quality", + title=m.group("msg").strip(), + description=m.group("msg").strip(), + file_path=m.group("file"), + line_number=int(m.group("line")), + column_number=int(m.group("col")) if m.group("col") else None, + rule_id=m.group("code") or "", + confidence="high", + tool_severity=m.group("level"), ) + findings.append(last) return findings +def _attach_mypy_note(finding: FindingData, note: str) -> None: + urls = _urls(note) + if note.startswith("See ") and urls and "docs_url" not in finding.tool_specific_data: + finding.tool_specific_data["docs_url"] = urls[0] + elif note: + finding.description = f"{finding.description}\n{note}" + + @register("radon") def parse_radon(raw: str) -> ParsedOutput: """Radon ``cc -j`` output: ``{file: [block]}`` with nested methods/closures. @@ -305,7 +555,21 @@ def _walk(file_path: str, block: dict) -> None: column_number=block.get("col_offset"), rule_id=f"CC-{rank}", confidence="high", - tool_specific_data={"complexity": complexity, "endline": block.get("endline")}, + # Radon grades complexity (rank A-F) but reports no + # severity: the rank stays a data point. + tool_specific_data=_compact( + { + "complexity": complexity if isinstance(complexity, int) else None, + "rank": rank, + "function": ( + f"{block['classname']}.{block.get('name', '?')}" + if block.get("classname") + else block.get("name") + ), + "end_line": block.get("endline"), + "endline": block.get("endline"), # legacy key + }, + ), ), ) for child in (block.get("methods") or []) + (block.get("closures") or []): @@ -362,7 +626,7 @@ def parse_vulture(raw: str) -> list[FindingData]: line_number=int(m.group("line")), rule_id="-".join(msg.split()[:2]).lower(), confidence="high" if conf >= 90 else "medium" if conf >= 60 else "low", - tool_specific_data={"confidence_pct": conf}, + tool_specific_data={"tool_confidence": f"{m.group('conf')}%", "confidence_pct": conf}, ), ) return findings @@ -390,7 +654,9 @@ def parse_detect_secrets(raw: str) -> list[FindingData]: line_number=item.get("line_number"), rule_id=secret_type, confidence="medium", + # The baseline holds only a SHA-1 of the secret, never its value. tool_specific_data={ + "secret_type": secret_type, "hashed_secret": item.get("hashed_secret", ""), "is_verified": item.get("is_verified", False), }, @@ -426,11 +692,23 @@ def parse_jscpd(raw: str) -> ParsedOutput: code_snippet=(dup.get("fragment") or "")[:500], rule_id="duplicate-code", confidence="high", - tool_specific_data={ - "tokens": dup.get("tokens"), - "second_file": second.get("name", ""), - "second_start": second.get("start"), - }, + tool_specific_data=_compact( + { + "duplicate_of": _compact( + { + "file": second.get("name"), + "start_line": second.get("start"), + "end_line": second.get("end"), + }, + ), + "lines": lines or None, + "tokens": dup.get("tokens"), + "end_line": first.get("end"), + # Legacy keys (older rows, readers of the raw data). + "second_file": second.get("name", ""), + "second_start": second.get("start"), + }, + ), ), ) @@ -459,6 +737,7 @@ def parse_hadolint(raw: str) -> list[FindingData]: sev_map = {"error": "high", "warning": "medium", "info": "low", "style": "info"} findings: list[FindingData] = [] for item in data: + code = str(item.get("code") or "") findings.append( FindingData( severity=sev_map.get(str(item.get("level", "warning")).lower(), "low"), @@ -470,11 +749,22 @@ def parse_hadolint(raw: str) -> list[FindingData]: column_number=item.get("column"), rule_id=item.get("code", ""), confidence="high", + tool_severity=_label(item.get("level")), + tool_specific_data=_compact({"docs_url": _hadolint_docs_url(code)}), ), ) return findings +def _hadolint_docs_url(code: str) -> str: + """Hadolint's own rules (DLxxxx) and the ShellCheck rules it runs on RUN lines (SCxxxx).""" + if re.fullmatch(r"DL\d+", code): + return f"https://github.com/hadolint/hadolint/wiki/{code}" + if re.fullmatch(r"SC\d+", code): + return f"https://www.shellcheck.net/wiki/{code}" + return "" + + _CODESPELL_LINE_RE = re.compile( r"^(?P.+?):(?P\d+):\s+(?P\S+) ==> (?P.+)$", ) @@ -500,6 +790,7 @@ def parse_codespell(raw: str) -> list[FindingData]: line_number=int(m.group("line")), rule_id="typo", confidence="medium", + tool_specific_data={"fix": right.strip()}, ), ) return findings @@ -523,10 +814,38 @@ def parse_gitleaks(raw: str) -> list[FindingData]: file_path=item.get("File", ""), line_number=item.get("StartLine"), column_number=item.get("StartColumn"), - code_snippet=(item.get("Match") or item.get("Secret") or "")[:500], + code_snippet=_redacted_match(item)[:500], rule_id=rule, confidence="high", - tool_specific_data={"entropy": item.get("Entropy")}, + tool_specific_data=_compact( + { + "secret_type": rule, + "entropy": item.get("Entropy"), + "end_line": item.get("EndLine"), + "end_column": item.get("EndColumn"), + }, + ), ), ) return findings + + +_REDACTED = "REDACTED" + + +def _redacted_match(item: dict[str, Any]) -> str: + """Gitleaks' ``Match`` (the matched text around the secret) with the secret masked. + + The run command passes ``--redact``, so both ``Match`` and ``Secret`` come + back as ``REDACTED``. Should a report arrive unredacted (an edited + command, an older gitleaks), the secret value must still never be stored: + mask it in the match, and keep nothing when it cannot be masked (the + match holds only part of a multi-line secret). + """ + match = str(item.get("Match") or "") + secret = str(item.get("Secret") or "") + if not secret or secret == _REDACTED: + return match + if secret not in match: + return "" + return match.replace(secret, _REDACTED) diff --git a/backend/analysis/services/runner.py b/backend/analysis/services/runner.py index f10fd1c0..de5cca2a 100644 --- a/backend/analysis/services/runner.py +++ b/backend/analysis/services/runner.py @@ -2,10 +2,22 @@ from __future__ import annotations +import fnmatch import logging +import re +import shlex +import threading +import zlib +from collections import Counter +from contextlib import contextmanager +from datetime import timedelta from pathlib import Path +from pathlib import PurePosixPath +from django.conf import settings from django.db import connection +from django.db import transaction +from django.db.models import Q from django.utils import timezone from backend.analysis.models import AnalysisRun @@ -18,8 +30,10 @@ from backend.analysis.services.base import AnalyzerOutput from backend.analysis.services.base import ParsedOutput from backend.analysis.services.base import build_severity_counts +from backend.analysis.services.serialize import serialize_result from backend.common.threading import dispatch_in_thread from backend.realtime import events as realtime +from backend.runtime.models import ContainerInstance from backend.runtime.services import docker_manager logger = logging.getLogger(__name__) @@ -40,23 +54,173 @@ } +# Port generated apps listen on inside their container (see runtime.proxy). +APP_CONTAINER_PORT = 8000 + +# ToolResult.summary["skip_reason"] values. Only a not-applicable skip is a +# measurement (the app has nothing the tool reads, so zero findings is true); +# the others mean the tool said nothing about the app. +SKIP_NOT_APPLICABLE = "not_applicable" +SKIP_APP_NOT_RUNNING = "app_not_running" +SKIP_UNAVAILABLE = "unavailable" +# The run was cancelled before (or while) this tool ran; it measured nothing. +SKIP_CANCELLED = "cancelled" + +# A running run's executing thread refreshes ``heartbeat_at`` this often; a run +# whose heartbeat is older than ``STALE_AFTER`` has no live thread (runs execute +# on daemon threads of the web process, which a reload kills). +HEARTBEAT_INTERVAL_S = 30 +STALE_AFTER = timedelta(minutes=5) +INTERRUPTED_MESSAGE = ( + "Interrupted: the backend process executing this run stopped (restart or reload). Re-run the analysis." +) +QUEUE_INTERRUPTED_MESSAGE = ( + "Interrupted before it started: the backend process holding the queue stopped (restart or reload). " + "Re-run the analysis." +) + +# What a running run is doing between its tools (``summary["phase"]`` while +# running, and ``progress`` events): waiting for another run of the same user +# to release the workspace, copying the code in, then running tools. +PHASE_WAITING = "waiting_for_workspace" +PHASE_PREPARING = "preparing" +PHASE_TOOLS = "running_tools" + +# Prefixed to every tool command (``: ; ``) so cancel can find +# and kill that run's in-flight processes in the shared workspace container. +_CMD_MARKER = "llmlab-analysis-run" + +_CONFIG_PLACEHOLDER_RE = re.compile(r"\{config:([A-Za-z0-9_]+)\}") + + +def render_command(tool: AnalyzerTool, *, target: str, url: str = "", config: dict | None = None) -> str: + """Fill a tool's ``run_cmd`` template. + + ``{target}`` is the code directory, ``{url}`` the running app's base URL + (dynamic tools) and ``{config:}`` the user's option value (falling + back to the tool's default), shell-quoted because it is user input. + """ + values = {**(tool.default_config or {}), **(config or {})} + + def _option(match: re.Match) -> str: + value = values.get(match.group(1), "") + if isinstance(value, bool): + value = "true" if value else "false" + elif isinstance(value, list): + value = ",".join(str(v) for v in value) + return shlex.quote(str(value)) + + cmd = tool.run_cmd.replace("{target}", target).replace("{url}", url) + return _CONFIG_PLACEHOLDER_RE.sub(_option, cmd) + + +def resolve_app_url(run: AnalysisRun) -> tuple[str, str]: + """``(base_url, "")`` of the run's app as seen from the workspace, or ``("", reason)``. + + Only a running container of the run's own generation job, started by or + belonging to the run's owner, is ever a target: dynamic tools attack what + they scan, so they must never be pointed at anything else. + """ + if not run.generation_job_id: + return "", "Dynamic tools scan a running app; analyze a generated app (not pasted code) to use them." + container = running_app_containers(run.created_by, [run.generation_job_id]).get(str(run.generation_job_id)) + if container is None: + return "", "The app is not running. Start its container, then re-run the dynamic tools." + if getattr(settings, "DOCKER_APPS_NETWORK", ""): + # Workspace and app share the apps network: reach the app by name. + return f"http://{container.name}:{APP_CONTAINER_PORT}", "" + # Default bridge (both containers live on the same daemon): use the app's IP. + info = docker_manager.inspect(container.name) + if info.get("State", {}).get("Running") is not True: + return "", "The app's container is not running. Start it, then re-run the dynamic tools." + for net in (info.get("NetworkSettings", {}).get("Networks") or {}).values(): + ip = (net or {}).get("IPAddress") + if ip: + return f"http://{ip}:{APP_CONTAINER_PORT}", "" + return "", "Could not determine the app container's address." + + +def running_app_containers(user, job_ids) -> dict[str, ContainerInstance]: + """``{job id: newest RUNNING container}`` for the jobs that have one, as recorded in the database. + + Only a container of the job started by, or belonging to, *user* counts: + dynamic tools attack what they scan, so nothing else may be a target. + """ + containers = ContainerInstance.objects.filter( + Q(created_by=user) | Q(generation_job__created_by=user), + generation_job_id__in=list(job_ids), + status=ContainerInstance.Status.RUNNING, + ).order_by("generation_job_id", "-updated_at") + found: dict[str, ContainerInstance] = {} + for container in containers: + found.setdefault(str(container.generation_job_id), container) + return found + + def dispatch(run_id: str) -> None: """Run an analysis on a background daemon thread.""" - dispatch_in_thread(_execute, run_id) + dispatch_many([run_id]) + + +def dispatch_many(run_ids) -> None: + """Run analyses one after another on a single background daemon thread. + + Runs of one user share a workspace and execute serially anyway; one + thread per batch keeps a 50-app batch from parking 50 threads (and + database connections) on the workspace lock. While queued, the runs' + ``heartbeat_at`` is refreshed so :func:`reap_stale_runs` can tell a live + queue from one that died with its process. + """ + ids = [str(i) for i in run_ids] + if not ids: + return + AnalysisRun.objects.filter(pk__in=ids, status=AnalysisRun.Status.PENDING).update(heartbeat_at=timezone.now()) + # Requests are atomic: a thread started before the commit may not see the runs yet. + transaction.on_commit(lambda: dispatch_in_thread(_execute_queue, ids, name=f"analysis-queue-{ids[0]}")) + + +def _execute_queue(run_ids: list[str]) -> None: + try: + with _Heartbeat(run_ids, statuses=(AnalysisRun.Status.PENDING, AnalysisRun.Status.RUNNING)): + for run_id in run_ids: + _execute(run_id) + finally: + connection.close() + + +def _channel(run_id) -> str: + return f"analysis:{run_id}" def _publish(run: AnalysisRun) -> None: realtime.publish( - f"analysis:{run.id}", + _channel(run.id), { "type": "status", "status": run.status, "summary": run.summary, + "error_message": run.error_message, "updated_at": timezone.now().isoformat(), }, ) +def _publish_result(result: ToolResult) -> None: + """A tool started or finished (``result`` event; the payload is the API's tool result).""" + realtime.publish(_channel(result.run_id), {"type": "result", "result": serialize_result(result)}) + + +def _set_phase(run: AnalysisRun, phase: str) -> None: + """Record what the running *run* is doing between tools (``progress`` event).""" + summary = {"phase": phase} + AnalysisRun.objects.filter(pk=run.pk, status=AnalysisRun.Status.RUNNING).update( + summary=summary, + updated_at=timezone.now(), + ) + run.summary = summary + realtime.publish(_channel(run.id), {"type": "progress", "phase": phase}) + + def _execute(run_id: str) -> None: try: run = AnalysisRun.objects.select_related("created_by", "workspace").get(id=run_id) @@ -67,27 +231,235 @@ def _execute(run_id: str) -> None: execute(run) except Exception: logger.exception("AnalysisRun %s failed unexpectedly", run_id) - run.status = AnalysisRun.Status.FAILED - run.error_message = "Internal error during analysis" - run.completed_at = timezone.now() - run.save(update_fields=["status", "error_message", "completed_at"]) - _publish(run) + _fail(run, "Internal error during analysis") finally: connection.close() +def is_cancelled(run: AnalysisRun) -> bool: + """Whether *run* was cancelled (read from the database: cancel happens in another request).""" + return AnalysisRun.objects.filter(pk=run.pk, status=AnalysisRun.Status.CANCELLED).exists() + + +def _save_unless_cancelled(run: AnalysisRun, **fields) -> bool: + """Write *fields* to *run*, keeping ``status``/``error_message`` if it was cancelled. + + One conditional UPDATE, so a cancel that lands while the runner is + finishing is never overwritten. Returns False when the run was cancelled. + """ + now = timezone.now() + updated = ( + AnalysisRun.objects.filter(pk=run.pk) + .exclude(status=AnalysisRun.Status.CANCELLED) + .update(updated_at=now, **fields) + ) + if not updated: + fields = {k: v for k, v in fields.items() if k not in ("status", "error_message")} + if fields: + AnalysisRun.objects.filter(pk=run.pk).update(updated_at=now, **fields) + fields["status"] = AnalysisRun.Status.CANCELLED + for key, value in fields.items(): + setattr(run, key, value) + return bool(updated) + + +def _fail(run: AnalysisRun, message: str) -> None: + completed_at = timezone.now() + _save_unless_cancelled( + run, + status=AnalysisRun.Status.FAILED, + error_message=message, + summary={k: v for k, v in (run.summary or {}).items() if k != "phase"}, + completed_at=completed_at, + duration_seconds=(completed_at - run.started_at).total_seconds() if run.started_at else None, + ) + _publish(run) + + +def cancel(run: AnalysisRun) -> bool: + """Cancel a pending or running *run*; kill its in-flight tool in the workspace. + + The executing thread notices the cancellation between tools (and after + the killed tool returns) and stops; its final save keeps ``cancelled``. + An in-flight AI review (an HTTP call) cannot be interrupted; its result + is discarded when it returns. Returns False when the run had already finished. + """ + now = timezone.now() + updated = AnalysisRun.objects.filter( + pk=run.pk, + status__in=[AnalysisRun.Status.PENDING, AnalysisRun.Status.RUNNING], + ).update(status=AnalysisRun.Status.CANCELLED, completed_at=now, updated_at=now) + run.refresh_from_db() + if not updated: + return False + + def _after_commit() -> None: + _kill_in_flight(run) + _publish(run) + + # Kill only once "cancelled" is committed (requests are atomic): the runner + # reads the status when the killed tool returns and must see the cancel, + # or it records the kill as a tool failure and starts the next tool. + transaction.on_commit(_after_commit) + return True + + +def _kill_script(run_id) -> str: + """Shell that SIGKILLs the process group of every process whose command line + carries this run's marker. Each tool runs under ``timeout``, which leads its + own process group, so the group holds the tool and all its children. The + analyzer image has no procps, so this reads /proc. The marker is assembled + at run time so this script's own command line never matches it.""" + head, tail = _CMD_MARKER.split("-", 1) + return ( + f"m='{head}'; m=\"$m-{tail}:{run_id}\"; " + "me=$(sed 's/.*) //' /proc/$$/stat | cut -d' ' -f3); n=0; " + "for d in /proc/[0-9]*; do " + "c=$( { tr '\\0' ' ' < \"$d/cmdline\"; } 2>/dev/null) || continue; " + 'case "$c" in *"$m"*) ;; *) continue ;; esac; ' + "g=$( { sed 's/.*) //' \"$d/stat\" | cut -d' ' -f3; } 2>/dev/null); " + 'case "$g" in \'\'|0|1|"$me") continue ;; esac; ' + 'kill -KILL -"$g" 2>/dev/null && n=$((n+1)); ' + 'done; echo "killed $n"' + ) + + +def _kill_in_flight(run: AnalysisRun) -> None: + workspace = run.workspace + container = workspace.container if workspace else None + if container is None: + return + result = docker_manager.exec_in(container.name, _kill_script(run.id), timeout_s=15) + if result.get("exit_code") != 0: + logger.warning("Could not kill in-flight tools of AnalysisRun %s: %s", run.id, result) + + +class _Heartbeat: + """Refreshes ``heartbeat_at`` of runs on a side thread while they execute or + wait in a queue (a single tool can run for many minutes, so beats between + tools are not enough). Only runs still in one of *statuses* are touched.""" + + def __init__(self, run_ids, statuses=(AnalysisRun.Status.RUNNING,)) -> None: + self._run_ids = list(run_ids) if isinstance(run_ids, (list, tuple, set)) else [run_ids] + self._statuses = list(statuses) + self._stop = threading.Event() + self._thread = threading.Thread( + target=self._beat, + daemon=True, + name=f"analysis-heartbeat-{self._run_ids[0] if self._run_ids else ''}", + ) + + def __enter__(self): + self._thread.start() + return self + + def __exit__(self, *exc) -> None: + self._stop.set() + + def _beat(self) -> None: + try: + while not self._stop.wait(HEARTBEAT_INTERVAL_S): + try: + now = timezone.now() + # updated_at too: the System "clear stuck" action ages runs by it. + AnalysisRun.objects.filter(pk__in=self._run_ids, status__in=self._statuses).update( + heartbeat_at=now, + updated_at=now, + ) + except Exception: # noqa: BLE001 - a missed beat must not end the loop + logger.warning("Heartbeat of AnalysisRun(s) %s failed", self._run_ids, exc_info=True) + finally: + connection.close() + + +def reap_stale_runs(stale_after: timedelta = STALE_AFTER) -> int: + """Fail ``running`` runs whose executing thread is gone; return how many. + + A live run beats every ``HEARTBEAT_INTERVAL_S``, so only a run whose + heartbeat (or, for rows from before heartbeats, start) is older than + *stale_after* is reaped: a run genuinely in progress in any live + process is never touched. Its still-``running`` tool results fail too. + Queued (``pending``) runs whose queue stopped beating fail as well; a + pending run that was never queued (no heartbeat) is left alone. + """ + now = timezone.now() + cutoff = now - stale_after + stale = AnalysisRun.objects.filter(status=AnalysisRun.Status.RUNNING).filter( + Q(heartbeat_at__lt=cutoff) + | Q(heartbeat_at__isnull=True, started_at__lt=cutoff) + | Q(heartbeat_at__isnull=True, started_at__isnull=True, updated_at__lt=cutoff), + ) + queued = AnalysisRun.objects.filter(status=AnalysisRun.Status.PENDING, heartbeat_at__lt=cutoff) + queued_count = queued.update( + status=AnalysisRun.Status.FAILED, + error_message=QUEUE_INTERRUPTED_MESSAGE, + completed_at=now, + updated_at=now, + ) + ids = list(stale.values_list("id", flat=True)) + if not ids: + return queued_count + count = AnalysisRun.objects.filter(id__in=ids, status=AnalysisRun.Status.RUNNING).update( + status=AnalysisRun.Status.FAILED, + error_message=INTERRUPTED_MESSAGE, + completed_at=now, + updated_at=now, + ) + ToolResult.objects.filter(run_id__in=ids, status=ToolResult.Status.RUNNING).update( + status=ToolResult.Status.FAILED, + error_message=INTERRUPTED_MESSAGE, + ) + logger.warning("Reaped %s interrupted analysis run(s): %s", count, ids) + return count + queued_count + + def execute(run: AnalysisRun) -> AnalysisRun: """Synchronously execute *run* (also used directly by tests/automation).""" - run.status = AnalysisRun.Status.RUNNING - run.started_at = timezone.now() - run.save(update_fields=["status", "started_at"]) + now = timezone.now() + if not _save_unless_cancelled(run, status=AnalysisRun.Status.RUNNING, started_at=now, heartbeat_at=now): + return run # cancelled before it started _publish(run) + # One workspace container, one /work and fixed /tmp report paths: two + # runs of the same user must not interleave, or each scans the other's code. + with _Heartbeat(run.pk), _workspace_lock(run.workspace_id, on_wait=lambda: _set_phase(run, PHASE_WAITING)): + return _execute_locked(run) + + +@contextmanager +def _workspace_lock(workspace_id, on_wait=None): + """Serialize runs on one workspace (a session-level Postgres advisory lock). + + ``on_wait`` is called once when another run holds the lock, before blocking. + """ + if not workspace_id or connection.vendor != "postgresql": + yield + return + key = zlib.crc32(f"analysis-workspace:{workspace_id}".encode()) + with connection.cursor() as cursor: + cursor.execute("SELECT pg_try_advisory_lock(%s)", [key]) + if not cursor.fetchone()[0]: + if on_wait is not None: + on_wait() + cursor.execute("SELECT pg_advisory_lock(%s)", [key]) + try: + yield + finally: + with connection.cursor() as cursor: + cursor.execute("SELECT pg_advisory_unlock(%s)", [key]) + + +def _execute_locked(run: AnalysisRun) -> AnalysisRun: + if is_cancelled(run): # cancelled while waiting for the workspace + _finalize(run, [], []) + return run + _set_phase(run, PHASE_PREPARING) code = run.get_code_for_analysis() files = _materialize(code) truncated_inputs = _truncated_inputs(run) tools = _resolve_tools(run.tool_slugs) tool_configs = _resolve_tool_configs(run) + tool_versions = _resolve_tool_versions(run) container_name = "" needs_container = any(t.kind == "container" for t in tools.values()) @@ -97,58 +469,149 @@ def execute(run: AnalysisRun) -> AnalysisRun: # /work is reused across runs in the long-lived workspace container. # Clear it first so files (and tool caches) from a previous run don't # leak into this one's findings. - docker_manager.exec_in( + cleared = docker_manager.exec_in( container_name, f"find {workspace_service.WORK_DIR} -mindepth 1 -delete", timeout_s=30, ) + if cleared.get("exit_code") != 0: + # Stale files would be analyzed as part of this app. + msg = f"Could not clear the workspace: {(cleared.get('output') or cleared.get('error') or '')[:300]}" + raise RuntimeError(msg) if files: - docker_manager.copy_files_in( + copied = docker_manager.copy_files_in( container_name, files, dest=workspace_service.WORK_DIR, + owner=workspace_service.WORKSPACE_UID, ) + if "error" in copied: + msg = f"Could not copy the code into the workspace: {copied['error']}" + raise RuntimeError(msg) except RuntimeError as exc: - run.status = AnalysisRun.Status.FAILED - run.error_message = str(exc) - run.completed_at = timezone.now() - run.save(update_fields=["status", "error_message", "completed_at"]) - _publish(run) + _fail(run, str(exc)) return run + app_url, app_url_error = "", "" + if any(t.analysis_mode == "dynamic" for t in tools.values()): + app_url, app_url_error = resolve_app_url(run) + + _set_phase(run, PHASE_TOOLS) statuses: list[str] = [] + cancelled = False for slug in run.tool_slugs: tool = tools.get(slug) + cancelled = cancelled or is_cancelled(run) + skip = _skip(tool, files, cancelled=cancelled, app_url=app_url, app_url_error=app_url_error) + if skip is not None: + reason, message = skip + result = ToolResult.objects.create( + run=run, + tool_slug=slug, + category=tool.category if tool else "", + status=ToolResult.Status.SKIPPED, + error_message=message, + summary={"skip_reason": reason}, + ) + _publish_result(result) + statuses.append(result.status) + continue + + started_at = timezone.now() result = ToolResult.objects.create( run=run, tool_slug=slug, - category=tool.category if tool else "", + category=tool.category, status=ToolResult.Status.RUNNING, + started_at=started_at, ) - if tool is None or not tool.is_enabled: - result.status = ToolResult.Status.SKIPPED - result.error_message = "Tool unavailable" - result.save(update_fields=["status", "error_message"]) - statuses.append(result.status) - continue + _publish_result(result) if tool.kind == "ai": output = ai_runner.run(tool, files, tool_configs.get(slug) or {}, run.created_by) else: - output = _run_container_tool(tool, container_name) + output = _run_container_tool( + tool, + container_name, + config=tool_configs.get(slug), + app_url=app_url, + run_id=run.pk, + ) + result.duration_seconds = round((timezone.now() - started_at).total_seconds(), 3) + if is_cancelled(run): + # Killed by cancel (or finished after it): not a measurement. + cancelled = True + result.status = ToolResult.Status.SKIPPED + result.error_message = "Run cancelled while this tool was running" + result.summary = {"skip_reason": SKIP_CANCELLED} + result.save(update_fields=["status", "error_message", "summary", "duration_seconds"]) + _publish_result(result) + statuses.append(result.status) + continue + + if tool.kind != "ai" and tool_versions.get(slug): + output.summary["tool_version"] = tool_versions[slug] + # The options the findings were produced with (provenance for reports). + output.summary["config"] = {**(tool.default_config or {}), **(tool_configs.get(slug) or {})} _persist_result(result, output) + _publish_result(result) statuses.append(result.status) _finalize(run, statuses, truncated_inputs) return run +def _skip( + tool: AnalyzerTool | None, + files: dict[str, str], + *, + cancelled: bool, + app_url: str, + app_url_error: str, +) -> tuple[str, str] | None: + """``(skip_reason, message)`` when *tool* must not run in this run, else None.""" + if cancelled: + return SKIP_CANCELLED, "Run cancelled" + if tool is None or not tool.is_enabled: + return SKIP_UNAVAILABLE, "Tool unavailable" + if tool.analysis_mode == "dynamic" and not app_url: + return SKIP_APP_NOT_RUNNING, app_url_error + if not applicable_files(tool, files): + # Not "0 findings": the tool had nothing to read, so it measured nothing. + return SKIP_NOT_APPLICABLE, f"Not applicable: no files matching {', '.join(tool.file_patterns)}" + return None + + +def applicable_files(tool: AnalyzerTool, files: dict[str, str]) -> list[str]: + """The paths in *files* that *tool* reads (all of them when it declares no patterns).""" + patterns = tool.file_patterns or [] + if not patterns or tool.analysis_mode == "dynamic": + return list(files) or [""] + matched = [] + for path in files: + name = PurePosixPath(path).name + if any(fnmatch.fnmatchcase(path if "/" in pat else name, pat) for pat in patterns): + matched.append(path) + return matched + + def _resolve_tools(slugs: list[str]) -> dict[str, AnalyzerTool]: qs = AnalyzerTool.objects.filter(slug__in=list(slugs)) return {t.slug: t for t in qs} +def _resolve_tool_versions(run: AnalysisRun) -> dict[str, str]: + """Installed version per slug, recorded on each result so findings stay attributable + to the exact tool build after the workspace is updated.""" + if not run.workspace_id: + return {} + return { + it.tool.slug: it.installed_version or it.catalog_version + for it in run.workspace.installed_tools.select_related("tool") + } + + def _resolve_tool_configs(run: AnalysisRun) -> dict[str, dict]: """Per-user tool config, keyed by slug (from the workspace's installed tools).""" if not run.workspace_id: @@ -170,42 +633,120 @@ def _normalize_path(path: str) -> str: """One file, one identity: container tools report /work-prefixed paths, jscpd relative ones — strip the workspace prefix so findings group.""" path = path.strip() - path = path.removeprefix(workspace_service.WORK_DIR + "/") + # djlint drops the leading slash ("work/app.html"). + path = path.removeprefix(workspace_service.WORK_DIR + "/").removeprefix(workspace_service.WORK_DIR[1:] + "/") return path.removeprefix("./") -def _run_container_tool(tool: AnalyzerTool, container_name: str) -> AnalyzerOutput: +def _run_container_tool( + tool: AnalyzerTool, + container_name: str, + *, + config: dict | None = None, + app_url: str = "", + run_id=None, +) -> AnalyzerOutput: if not container_name: return AnalyzerOutput(error="Analyzer workspace unavailable") if not tool.run_cmd: return AnalyzerOutput(error="Tool has no run command") - cmd = tool.run_cmd.replace("{target}", workspace_service.WORK_DIR) + cmd = render_command(tool, target=workspace_service.WORK_DIR, url=app_url, config=config) + # The marker lets cancel() find this run's processes (see _kill_script). + marked = f": {_CMD_MARKER}:{run_id}; {cmd}" if run_id else cmd result = docker_manager.exec_in( container_name, - cmd, + marked, timeout_s=tool.run_timeout, workdir=workspace_service.WORK_DIR, ) if result.get("error") and result.get("exit_code") == -1: return AnalyzerOutput(error=result["error"]) raw = result.get("output", "") - parsed = parsers.parse(tool.parser_key, raw) if tool.parser_key else ParsedOutput() + exit_code = result.get("exit_code") + raw_output = { + "command": cmd, + "exit_code": exit_code, + "stdout": raw[:_STDOUT_STORE_LIMIT], + **({"target_url": app_url} if tool.analysis_mode == "dynamic" else {}), + } + error = _exit_error(tool, exit_code, raw) + parsed = ParsedOutput() + if error is None: + parsed = parsers.parse(tool.parser_key, raw) if tool.parser_key else ParsedOutput() + error = parsed.error + if error is None and exit_code not in (0, None) and not parsed.findings and not parsed.metrics: + # Linters exit non-zero *because* they found something; non-zero + # with nothing parsed means the tool itself failed. + error = f"Exited with code {exit_code} without a parsable report: {_tail(raw)}" + if error is not None: + # Keep what the parser did find (nonexistent-package/-version findings + # of a manifest whose audit could not run): stored on the failed + # result, which still does not count as a measurement (study tables). + return AnalyzerOutput( + error=error, + raw_output=raw_output, + findings=parsed.findings, + warnings=parsed.warnings, + ) return AnalyzerOutput( findings=parsed.findings, metrics=parsed.metrics, - raw_output={"exit_code": result.get("exit_code"), "stdout": raw[:_STDOUT_STORE_LIMIT]}, + raw_output=raw_output, + warnings=parsed.warnings, summary={"finding_count": len(parsed.findings)}, ) +def _tail(raw: str, limit: int = 300) -> str: + text = (raw or "").strip() + return ("…" + text[-limit:]) if len(text) > limit else (text or "(no output)") + + +def _exit_error(tool: AnalyzerTool, exit_code, raw: str) -> str | None: + """A failure the exit code alone proves (``timeout -s KILL`` exits 137).""" + if exit_code == 137: + return f"Killed after {tool.run_timeout}s (timeout) or out of memory; the result would be incomplete." + if exit_code in (126, 127): + return f"Command not found or not executable (exit {exit_code}) — reinstall the tool: {_tail(raw)}" + return None + + def _persist_result(result: ToolResult, output: AnalyzerOutput) -> None: + _store_findings(result, output) if output.has_error: result.status = ToolResult.Status.FAILED result.error_message = output.error or "" result.raw_output = output.raw_output - result.save(update_fields=["status", "error_message", "raw_output"]) + result.summary = {k: v for k, v in output.summary.items() if k in ("tool_version", "config")} + if output.findings: + result.summary["finding_count"] = len(output.findings) + result.summary["severity_counts"] = build_severity_counts(output.findings) + result.summary["reported_severity_counts"] = _reported_counts(output.findings) + if output.warnings: + result.summary["warnings"] = output.warnings + result.save(update_fields=["status", "error_message", "raw_output", "summary", "duration_seconds"]) return + result.status = ToolResult.Status.COMPLETED + result.raw_output = output.raw_output + result.metrics = output.metrics + result.summary = { + **output.summary, + "severity_counts": build_severity_counts(output.findings), + "reported_severity_counts": _reported_counts(output.findings), + **({"warnings": output.warnings} if output.warnings else {}), + } + result.save(update_fields=["status", "raw_output", "metrics", "summary", "duration_seconds"]) + + +def _reported_counts(findings) -> dict[str, int]: + """Findings per level as the tool printed it (case-folded); unrated ones are not counted.""" + counts = Counter((f.tool_severity or "").strip().lower() for f in findings) + counts.pop("", None) + return dict(counts) + + +def _store_findings(result: ToolResult, output: AnalyzerOutput) -> None: Finding.objects.bulk_create( [ Finding( @@ -222,18 +763,11 @@ def _persist_result(result: ToolResult, output: AnalyzerOutput) -> None: code_snippet=f.code_snippet, rule_id=f.rule_id[:100], tool_specific_data=f.tool_specific_data, + tool_severity=(f.tool_severity or "").strip()[:40], ) for f in output.findings ], ) - result.status = ToolResult.Status.COMPLETED - result.raw_output = output.raw_output - result.metrics = output.metrics - result.summary = { - **output.summary, - "severity_counts": build_severity_counts(output.findings), - } - result.save(update_fields=["status", "raw_output", "metrics", "summary"]) def _finalize(run: AnalysisRun, statuses: list[str], truncated_inputs: list[str] | None = None) -> None: @@ -253,11 +787,17 @@ def _finalize(run: AnalysisRun, statuses: list[str], truncated_inputs: list[str] status = AnalysisRun.Status.COMPLETED metrics_by_tool = {slug: metrics for slug, metrics in run.results.values_list("tool_slug", "metrics") if metrics} + # Levels exactly as the tools printed them (case-folded); unrated findings + # come from tools that rate nothing. ``severity_counts`` is the normalized + # (partly heuristic) level kept for aggregate statistics and thresholds. + reported = Counter((label or "").strip().lower() for label in findings.values_list("tool_severity", flat=True)) + unrated = reported.pop("", 0) - run.status = status - run.summary = { + summary = { "total_findings": findings.count(), "severity_counts": severity_counts, + "reported_severity_counts": dict(reported), + "unrated_findings": unrated, "metrics_by_tool": metrics_by_tool, "tools_run": len(statuses), "tools_completed": completed, @@ -266,11 +806,18 @@ def _finalize(run: AnalysisRun, statuses: list[str], truncated_inputs: list[str] if truncated_inputs: # The generated code was cut off at max_tokens; findings describe an # incomplete app, so surface that next to them. - run.summary["truncated_inputs"] = truncated_inputs - run.completed_at = timezone.now() - if run.started_at: - run.duration_seconds = (run.completed_at - run.started_at).total_seconds() - run.save(update_fields=["status", "summary", "completed_at", "duration_seconds"]) + summary["truncated_inputs"] = truncated_inputs + completed_at = timezone.now() + duration = (completed_at - run.started_at).total_seconds() if run.started_at else None + # A cancelled run keeps its status (cancel is sticky); it still gets the + # summary of what ran before the cancel. + _save_unless_cancelled( + run, + status=status, + summary=summary, + completed_at=completed_at, + duration_seconds=duration, + ) _publish(run) diff --git a/backend/analysis/services/scanner_parsers.py b/backend/analysis/services/scanner_parsers.py new file mode 100644 index 00000000..26f018f5 --- /dev/null +++ b/backend/analysis/services/scanner_parsers.py @@ -0,0 +1,612 @@ +"""Parsers for the DAST scanners and the multi-ecosystem static scanners. + +Dynamic tools (ZAP, Nuclei, Wapiti) probe a running app, so their findings +point at a URL path instead of a source line: ``file_path`` holds the path +(``/api/items``) so findings on the same endpoint group together, and the full +URL, HTTP method and parameter go into ``tool_specific_data``. + +Registered into the shared registry from ``AnalysisConfig.ready``. +""" + +from __future__ import annotations + +import html +import json +import re +from typing import Any +from urllib.parse import urlsplit + +from backend.analysis.services.base import FindingData +from backend.analysis.services.extra_parsers import _cwe +from backend.analysis.services.parsers import _clip +from backend.analysis.services.parsers import _compact +from backend.analysis.services.parsers import _cwe_field +from backend.analysis.services.parsers import _float +from backend.analysis.services.parsers import _label +from backend.analysis.services.parsers import _loads +from backend.analysis.services.parsers import _urls +from backend.analysis.services.parsers import register + +_TAG_RE = re.compile(r"<[^>]+>") + + +def _plain(text: Any) -> str: + """Strip the HTML markup ZAP puts in descriptions (``

…

``).""" + return html.unescape(_TAG_RE.sub("\n", str(text or ""))).strip() + + +def _url_path(url: str) -> str: + try: + return urlsplit(url).path or "/" + except ValueError: + return url + + +def _cvss_severity(score: Any) -> str: + try: + value = float(score) + except (TypeError, ValueError): + return "medium" + if value >= 9.0: + return "critical" + if value >= 7.0: + return "high" + if value >= 4.0: + return "medium" + return "low" if value > 0 else "info" + + +# --- Dynamic (DAST) --------------------------------------------------------- + +# The scanners report the same weakness under their own ids (ZAP plugin +# 10038, Nuclei http-missing-security-headers:content-security-policy, Wapiti +# csp). ``issue_key`` names it once so cross-tool dedup counts it once, and +# keeps Nuclei's per-header hits (one template id) apart. +_SECURITY_HEADERS = ( + ("content-security-policy", ("content-security-policy", "content security policy", "csp")), + ("x-frame-options", ("x-frame-options", "clickjacking")), + ("x-content-type-options", ("x-content-type-options", "mime type confusion")), + ("strict-transport-security", ("strict-transport-security", "strict transport security", "hsts")), + ("referrer-policy", ("referrer-policy",)), + ("permissions-policy", ("permissions-policy", "permissions policy")), + ("cross-origin-embedder-policy", ("cross-origin-embedder-policy",)), + ("cross-origin-opener-policy", ("cross-origin-opener-policy",)), + ("cross-origin-resource-policy", ("cross-origin-resource-policy",)), + ("x-permitted-cross-domain-policies", ("x-permitted-cross-domain-policies",)), +) +# Fingerprints, not weaknesses: ZAP's "Modern Web Application" (10109). +_ZAP_SKIP = frozenset({"10109"}) + + +def _dast_issue_key(*texts: Any) -> dict[str, str]: + """``{"issue_key": ...}`` for a missing security header or a CORS misconfiguration, else {}.""" + text = " ".join(str(t or "") for t in texts).lower() + if "cors" in text or "cross-domain misconfiguration" in text: + return {"issue_key": "cors-misconfiguration"} + if not any(word in text for word in ("missing", "not set")): + return {} + for header, needles in _SECURITY_HEADERS: + if any(needle in text for needle in needles): + return {"issue_key": f"missing-header:{header}"} + return {} + + +_ZAP_RISK = {"3": "high", "2": "medium", "1": "low", "0": "info"} +_ZAP_CONFIDENCE = {"4": "high", "3": "high", "2": "medium", "1": "low"} +# ZAP's own words for its codes, as its reports print them (``riskdesc`` +# "Medium (High)" = risk Medium, confidence High). +_ZAP_RISK_WORDS = {"3": "High", "2": "Medium", "1": "Low", "0": "Informational"} +_ZAP_CONFIDENCE_WORDS = {"4": "Confirmed", "3": "High", "2": "Medium", "1": "Low", "0": "False Positive"} +_ZAP_RISKDESC_RE = re.compile(r"^\s*([^()]+?)\s*\(([^()]+)\)\s*$") + + +def _zap_labels(alert: dict[str, Any]) -> tuple[str, str]: + """ZAP's risk and confidence words for *alert*: from ``riskdesc``, else from the codes.""" + m = _ZAP_RISKDESC_RE.match(str(alert.get("riskdesc") or "")) + if m: + return m.group(1), m.group(2) + return ( + _ZAP_RISK_WORDS.get(str(alert.get("riskcode", "")), ""), + _ZAP_CONFIDENCE_WORDS.get(str(alert.get("confidence", "")), ""), + ) + + +@register("zap") +def parse_zap(raw: str) -> list[FindingData]: + """ZAP's traditional JSON report: ``site[].alerts[]`` with ``instances[]``. + + One finding per alert and endpoint (instances of the same alert on the + same path, e.g. one per HTTP method, collapse). ``systemic`` alerts — the + same issue on every page, like a missing security header — are one + finding listing the affected URLs. Alerts with confidence 0 are ones a + user marked as false positives. + """ + data = _loads(raw) + if not isinstance(data, dict): + return [] + findings: list[FindingData] = [] + for site in data.get("site") or []: + for alert in site.get("alerts") or []: + confidence = str(alert.get("confidence", "")) + if confidence == "0" or str(alert.get("pluginid", "")) in _ZAP_SKIP: + continue + title = alert.get("name") or alert.get("alert") or "ZAP alert" + instances = alert.get("instances") or [{"uri": site.get("@name", "")}] + urls = list(dict.fromkeys(inst.get("uri", "") for inst in instances)) + if alert.get("systemic"): + instances = instances[:1] + risk_word, confidence_word = _zap_labels(alert) + plugin_id = str(alert.get("pluginid", "")) + references = _urls(_plain(alert.get("reference"))) + seen: set[str] = set() + for inst in instances: + path = _url_path(inst.get("uri", "")) + if path in seen: + continue + seen.add(path) + findings.append( + FindingData( + severity=_ZAP_RISK.get(str(alert.get("riskcode", "")), "info"), + category="security", + title=title, + description=_plain(alert.get("desc")), + suggestion=_plain(alert.get("solution")), + file_path=path, + code_snippet=str(inst.get("evidence") or "")[:500], + rule_id=plugin_id, + confidence=_ZAP_CONFIDENCE.get(confidence, "medium"), + tool_severity=risk_word, + tool_specific_data=_compact( + { + "tool_confidence": confidence_word, + "url": inst.get("uri", ""), + "method": inst.get("method", ""), + "param": inst.get("param", ""), + "attack": _clip(inst.get("attack"), 500), + "evidence": _clip(inst.get("evidence"), 500), + "other_info": _clip(_plain(inst.get("otherinfo") or alert.get("otherinfo")), 1000), + "instances": int(alert.get("count") or 1), + **({"urls": urls[:50]} if alert.get("systemic") else {}), + "references": references, + "docs_url": f"https://www.zaproxy.org/docs/alerts/{plugin_id}/" + if plugin_id.isdigit() + else None, + **_dast_issue_key(title), + **_cwe_field(int(alert["cweid"]) if str(alert.get("cweid", "")).isdigit() else None), + }, + ), + ), + ) + return findings + + +_NUCLEI_SEVERITY = {"critical": "critical", "high": "high", "medium": "medium", "low": "low", "info": "info"} + + +@register("nuclei") +def parse_nuclei(raw: str) -> list[FindingData]: + """``nuclei -jsonl``: one JSON object per line, one line per matcher hit.""" + findings: list[FindingData] = [] + for raw_line in (raw or "").splitlines(): + line = raw_line.strip() + if not line.startswith("{"): + continue + try: + hit = json.loads(line) + except json.JSONDecodeError: + continue + info = hit.get("info") or {} + tags = info.get("tags") or [] + if hit.get("template-id") == "tech-detect" or "tech" in (tags if isinstance(tags, list) else [tags]): + continue # technology fingerprinting, not a weakness + matcher = hit.get("matcher-name") or "" + name = info.get("name") or hit.get("template-id") or "Nuclei match" + matched = hit.get("matched-at") or hit.get("url") or "" + classification = info.get("classification") or {} + refs = info.get("reference") or [] + extracted = hit.get("extracted-results") or [] + evidence = ", ".join(str(e) for e in extracted)[:500] + cvss_score = _float(classification.get("cvss-score")) + cve_ids = classification.get("cve-id") or [] + cve_ids = [str(c).upper() for c in (cve_ids if isinstance(cve_ids, list) else [cve_ids]) if c] + findings.append( + FindingData( + severity=_NUCLEI_SEVERITY.get(str(info.get("severity", "")).lower(), "info"), + category="security", + title=f"{name}: {matcher}" if matcher else name, + description=str(info.get("description") or "").strip(), + suggestion=str(info.get("remediation") or "").strip(), + file_path=_url_path(matched), + code_snippet=evidence, + rule_id=hit.get("template-id", ""), + confidence="medium", + tool_severity=_label(info.get("severity")), + tool_specific_data=_compact( + { + "rule_name": info.get("name"), + "url": matched, + "matcher": matcher, + "tags": tags, + **_dast_issue_key(hit.get("template-id"), matcher), + "references": [str(r) for r in (refs if isinstance(refs, list) else [refs]) if r], + "docs_url": hit.get("template-url"), + "evidence": evidence, + "vuln_id": cve_ids[0] if cve_ids else None, + "aliases": cve_ids[1:], + "cvss_score": cvss_score, + "cvss_vector": classification.get("cvss-metrics") if cvss_score else None, + "curl": hit.get("curl-command"), + **_cwe(classification.get("cwe-id")), + }, + ), + ), + ) + return findings + + +_WAPITI_LEVEL = {4: "critical", 3: "high", 2: "medium", 1: "low", 0: "info"} + + +@register("wapiti") +def parse_wapiti(raw: str) -> list[FindingData]: + """Wapiti's JSON report. + + ``vulnerabilities`` are security issues; ``anomalies`` are server errors + the attacks provoked (a robustness signal). ``additionals`` (fingerprints, + allowed HTTP methods) are informational noise and skipped. + """ + data = _loads(raw) + if not isinstance(data, dict): + return [] + classifications = data.get("classifications") or {} + findings: list[FindingData] = [] + for section, category in (("vulnerabilities", "security"), ("anomalies", "reliability")): + for name, items in (data.get(section) or {}).items(): + meta = classifications.get(name) or {} + refs = meta.get("ref") or {} + cwe = next((_cwe(k) for k in refs if "CWE" in k), {}) + references = _urls(list(refs.values()) if isinstance(refs, dict) else refs) + for item in items or []: + level = item.get("level") + severity = _WAPITI_LEVEL.get(level, "medium") if section == "vulnerabilities" else "medium" + findings.append( + FindingData( + severity=severity, + category=category, + title=f"{name}: {item.get('info', '')}".rstrip(": ")[:500], + description=str(meta.get("desc") or "").strip(), + suggestion=str(meta.get("sol") or "").strip(), + file_path=item.get("path") or "/", + rule_id=item.get("module") or name, + confidence="medium", + # Wapiti's JSON has only the numeric level (4 critical .. 0 info). + tool_severity=_label(level), + tool_specific_data=_compact( + { + "rule_name": name, + "method": item.get("method", ""), + "param": item.get("parameter", ""), + "curl": item.get("curl_command", ""), + "wstg": item.get("wstg") or [], + "references": references, + **cwe, + **(_dast_issue_key(name, item.get("info")) if section == "vulnerabilities" else {}), + }, + ), + ), + ) + return findings + + +# --- Static multi-ecosystem scanners --------------------------------------- + +_TRIVY_SEVERITY = {"CRITICAL": "critical", "HIGH": "high", "MEDIUM": "medium", "LOW": "low", "UNKNOWN": "info"} + + +@register("trivy") +def parse_trivy(raw: str) -> list[FindingData]: + """``trivy fs --format json``: ``Results[]`` with vulns, misconfigs and secrets.""" + data = _loads(raw) + if not isinstance(data, dict): + return [] + findings: list[FindingData] = [] + for result in data.get("Results") or []: + target = result.get("Target", "") + for v in result.get("Vulnerabilities") or []: + pkg, version, fixed = v.get("PkgName", "?"), v.get("InstalledVersion", ""), v.get("FixedVersion", "") + vid = v.get("VulnerabilityID", "") + findings.append( + FindingData( + severity=_TRIVY_SEVERITY.get(v.get("Severity", ""), "medium"), + category="dependency", + title=f"{pkg} {version}: {vid}" + (f" — {v['Title']}" if v.get("Title") else ""), + description=v.get("Description", ""), + suggestion=f"Upgrade {pkg} to {fixed}" if fixed else "", + file_path=target, + rule_id=vid, + confidence="high", + tool_severity=_label(v.get("Severity")), + tool_specific_data=_compact( + { + "package": pkg, + "installed_version": version, + # "1.2.5, 2.0.1" when several release lines are fixed. + "fixed_versions": [f.strip() for f in str(fixed).split(",") if f.strip()], + "vuln_id": vid, + "aliases": [a for a in v.get("VendorIDs") or [] if a != vid], + **_trivy_cvss(v), + "docs_url": v.get("PrimaryURL"), + "references": _urls(v.get("References")), + "manifest": target, + # Legacy keys. + "version": version, + "fixed_version": fixed, + "url": v.get("PrimaryURL", ""), + **_cwe(v.get("CweIDs")), + }, + ), + ), + ) + for m in result.get("Misconfigurations") or []: + if m.get("Status") not in (None, "FAIL"): + continue + cause = m.get("CauseMetadata") or {} + lines = (cause.get("Code") or {}).get("Lines") or [] + findings.append( + FindingData( + severity=_TRIVY_SEVERITY.get(m.get("Severity", ""), "medium"), + category="security", + title=m.get("Title") or m.get("ID", "Misconfiguration"), + description=m.get("Message") or m.get("Description", ""), + suggestion=m.get("Resolution", ""), + file_path=target, + line_number=cause.get("StartLine") or None, + code_snippet="\n".join(str(ln.get("Content", "")) for ln in lines if ln.get("IsCause"))[:500], + rule_id=m.get("ID") or m.get("AVDID", ""), + confidence="high", + tool_severity=_label(m.get("Severity")), + tool_specific_data=_compact( + { + "type": m.get("Type", ""), + "docs_url": m.get("PrimaryURL"), + "references": _urls(m.get("References")), + "end_line": cause.get("EndLine") or None, + "url": m.get("PrimaryURL", ""), # legacy key + }, + ), + ), + ) + for s in result.get("Secrets") or []: + findings.append( + FindingData( + severity=_TRIVY_SEVERITY.get(s.get("Severity", ""), "high"), + category="secrets", + title=s.get("Title") or s.get("RuleID", "Secret"), + description=f"{s.get('Category', 'Secret')} detected in source.", + file_path=target, + line_number=s.get("StartLine") or None, + # Trivy masks the secret in Match ("KEY=\"****\"") and + # prints its value nowhere else. + code_snippet=str(s.get("Match", ""))[:500], + rule_id=s.get("RuleID", ""), + confidence="high", + tool_severity=_label(s.get("Severity")), + tool_specific_data=_compact( + { + "secret_type": s.get("RuleID"), + "end_line": s.get("EndLine") or None, + "cwe": "CWE-798", + }, + ), + ), + ) + return findings + + +def _trivy_cvss(vuln: dict[str, Any]) -> dict[str, Any]: + """``cvss_score``/``cvss_vector`` from Trivy's per-source ``CVSS`` map. + + The source Trivy took ``Severity`` from (``SeveritySource``) comes first, + then NVD; within a source the newest CVSS version. Score and vector + always come from the same entry. + """ + cvss = vuln.get("CVSS") + if not isinstance(cvss, dict): + return {} + for source in dict.fromkeys([vuln.get("SeveritySource"), "nvd", *cvss]): + entry = cvss.get(source) if source else None + if not isinstance(entry, dict): + continue + for version in ("V40", "V3", "V2"): + score = _float(entry.get(f"{version}Score")) + if score: + return {"cvss_score": score, "cvss_vector": entry.get(f"{version}Vector")} + return {} + + +@register("osv_scanner") +def parse_osv_scanner(raw: str) -> list[FindingData]: + """``osv-scanner scan source --format json``. + + ``packages[].groups`` collapses aliases of one vulnerability (PYSEC/GHSA/ + CVE) and carries the max CVSS score; one finding per package and group. + """ + data = _loads(raw) + if not isinstance(data, dict): + return [] + findings: list[FindingData] = [] + # A manifest can appear twice (direct "lockfile" pass + transitive pass). + seen: set[tuple[str, str, str, str]] = set() + for result in data.get("results") or []: + path = (result.get("source") or {}).get("path", "") + for pkg in result.get("packages") or []: + meta = pkg.get("package") or {} + name, version = meta.get("name", "?"), meta.get("version", "") + vulns = {v.get("id"): v for v in pkg.get("vulnerabilities") or []} + for group in pkg.get("groups") or []: + ids = group.get("ids") or [] + key = (path, name, version, ids[0] if ids else "") + if key in seen: + continue + seen.add(key) + aliases = group.get("aliases") or ids + # One group = the same vulnerability in several databases + # (PYSEC + GHSA): each record carries part of the data (GHSA: + # summary, CWE, severity label; PYSEC: often only details). + records = [vulns[i] for i in dict.fromkeys([*ids, *aliases]) if i in vulns] + cve = next((a for a in aliases if str(a).startswith("CVE-")), "") + ghsa = next((a for a in aliases if str(a).startswith("GHSA-")), "") + fixed = sorted( + { + ev["fixed"] + for rec in records + for aff in rec.get("affected") or [] + if (aff.get("package") or {}).get("name", name).lower() == name.lower() + for rng in aff.get("ranges") or [] + for ev in rng.get("events") or [] + if ev.get("fixed") + }, + ) + summary = next((r["summary"] for r in records if r.get("summary")), "") + db_specific = [r.get("database_specific") or {} for r in records] + max_severity = _label(group.get("max_severity")) + vectors = { + str(s.get("score")) + for r in records + for s in r.get("severity") or [] + if str(s.get("type", "")).startswith("CVSS") and s.get("score") + } + findings.append( + FindingData( + severity=_cvss_severity(group.get("max_severity")), + category="dependency", + title=f"{name} {version}: {cve or ghsa or (ids[0] if ids else 'vulnerability')}" + + (f" — {summary}" if summary else ""), + description=next((r["details"] for r in records if r.get("details")), "") or summary, + suggestion=f"Upgrade {name} to {' or '.join(fixed)}" if fixed else "", + file_path=path, + rule_id=ids[0] if ids else "", + confidence="high", + # The advisory's own label (GitHub's "MODERATE") when a + # record has one, else the CVSS score osv-scanner + # prints in its severity column. + tool_severity=next( + (_label(d.get("severity")) for d in db_specific if _label(d.get("severity"))), + max_severity, + ), + tool_specific_data=_compact( + { + "package": name, + "installed_version": version, + "ecosystem": meta.get("ecosystem", ""), + "fixed_versions": fixed, + "vuln_id": ids[0] if ids else None, + "aliases": aliases, + "cvss_score": _float(max_severity), + # Only when unambiguous: max_severity may come + # from any of the records' vectors. + "cvss_vector": next(iter(vectors)) if len(vectors) == 1 else None, + "references": _urls( + [ref.get("url") for r in records for ref in r.get("references") or []], + ), + "manifest": path, + # Legacy keys. + "version": version, + "fix_versions": fixed, + "cvss": max_severity or None, + **next( + (_cwe(d.get("cwe_ids")) for d in db_specific if _cwe(d.get("cwe_ids"))), + {}, + ), + }, + ), + ), + ) + return findings + + +_RETIRE_SEVERITY = {"critical": "critical", "high": "high", "medium": "medium", "low": "low"} + + +@register("retire") +def parse_retire(raw: str) -> list[FindingData]: + """``retire --outputformat json``: ``data[].results[].vulnerabilities[]``.""" + data = _loads(raw) + if not isinstance(data, dict): + return [] + findings: list[FindingData] = [] + for entry in data.get("data") or []: + path = entry.get("file", "") + for res in entry.get("results") or []: + component, version = res.get("component", "?"), res.get("version", "") + for vuln in res.get("vulnerabilities") or []: + ids = vuln.get("identifiers") or {} + cves = ids.get("CVE") or [] + ref = cves[0] if cves else ids.get("githubID") or ids.get("issue") or ids.get("bug") or "" + below = vuln.get("below") + github_id = ids.get("githubID") + findings.append( + FindingData( + severity=_RETIRE_SEVERITY.get(str(vuln.get("severity", "")).lower(), "medium"), + category="dependency", + title=f"{component} {version}: {ids.get('summary') or ref or 'known vulnerability'}"[:500], + description="\n".join(vuln.get("info") or []), + suggestion=f"Upgrade {component} to {below} or later" if below else "", + file_path=path, + rule_id=str(ref), + confidence="high", + tool_severity=_label(vuln.get("severity")), + tool_specific_data=_compact( + { + "package": component, + "installed_version": version, + "fixed_versions": [str(below)] if below else [], + "vuln_id": str(ref), + "aliases": [a for a in [*cves, github_id] if a and a != ref], + "references": _urls(vuln.get("info")), + "detection": res.get("detection", ""), + "version": version, # legacy key + **_cwe(vuln.get("cwe")), + }, + ), + ), + ) + return findings + + +# djLint's {line} expands to "line:column". +_DJLINT_LINE_RE = re.compile(r"^(?P.+?):(?P\d+)(?::(?P\d+))?:(?P[A-Z]\d{3}):(?P.+)$") +# Suggestions rather than defects (meta description / keywords). +_DJLINT_INFO = {"H030", "H031"} + + +@register("djlint") +def parse_djlint(raw: str) -> list[FindingData]: + """``djlint --lint --linter-output-format '{filename}:{line}:{code}:{message}'``. + + Progress-bar and blank lines are ignored. djLint prints absolute paths + without the leading slash (``work/app/templates/x.html``). + """ + findings: list[FindingData] = [] + for line in (raw or "").splitlines(): + m = _DJLINT_LINE_RE.match(line.strip()) + if not m: + continue + code = m.group("code") + path = m.group("file").lstrip("/").removeprefix("work/") + findings.append( + FindingData( + # djLint's lint output carries no level: ``tool_severity`` stays empty. + severity="info" if code in _DJLINT_INFO else "low", + category="quality", + title=m.group("msg").strip(), + file_path=path, + line_number=int(m.group("line")), + column_number=int(m.group("col")) if m.group("col") else None, + rule_id=code, + confidence="high", + # The rule table (djLint has no per-rule pages). + tool_specific_data={"docs_url": "https://www.djlint.com/docs/linter/#rules"}, + ), + ) + return findings diff --git a/backend/analysis/services/serialize.py b/backend/analysis/services/serialize.py new file mode 100644 index 00000000..0c63c1d5 --- /dev/null +++ b/backend/analysis/services/serialize.py @@ -0,0 +1,24 @@ +"""JSON shapes of runs and tool results, shared by the API and the live (SSE) events.""" + +from __future__ import annotations + +from typing import Any + + +def iso(value) -> str | None: + return value.isoformat() if value else None + + +def serialize_result(result) -> dict[str, Any]: + """One tool's result within a run (no raw output: see the result detail endpoint).""" + return { + "id": str(result.id), + "tool_slug": result.tool_slug, + "category": result.category, + "status": result.status, + "summary": result.summary or {}, + "metrics": result.metrics or {}, + "error_message": result.error_message, + "started_at": iso(result.started_at), + "duration_seconds": result.duration_seconds, + } diff --git a/backend/analysis/services/tool_tester.py b/backend/analysis/services/tool_tester.py index f81b44aa..bc5bb0bf 100644 --- a/backend/analysis/services/tool_tester.py +++ b/backend/analysis/services/tool_tester.py @@ -9,6 +9,7 @@ from backend.analysis.services import ai_runner from backend.analysis.services import parsers +from backend.analysis.services import runner from backend.analysis.services import tool_installer from backend.analysis.services import workspace_service from backend.runtime.services import docker_manager @@ -53,6 +54,17 @@ def test_tool( "raw_output": {}, } + if tool.analysis_mode == "dynamic": + # No sample to scan: a dynamic tool needs a running app, which only a + # real analysis run of a generated app provides. + return { + "available": True, + "message": f"{message or 'Tool available'} — dynamic tool: it scans a running app during analysis runs.", + "findings": [], + "metrics": {}, + "raw_output": {}, + } + findings: list[dict[str, Any]] = [] metrics: dict[str, Any] = {} raw: dict[str, Any] = {} @@ -61,12 +73,15 @@ def test_tool( container_name = workspace_service.require_ready_container(workspace) ext = ".py" if tool.target_language == "python" else ".js" filename = tool.sample_filename or f"sample{ext}" + # Fresh dir per test: a previous tool's sample must not be scanned too. + docker_manager.exec_in(container_name, "rm -rf /tmp/analyzer-test", timeout_s=30) docker_manager.copy_files_in( container_name, {filename: sample}, dest="/tmp/analyzer-test", + owner=workspace_service.WORKSPACE_UID, ) - cmd = tool.run_cmd.replace("{target}", "/tmp/analyzer-test") + cmd = runner.render_command(tool, target="/tmp/analyzer-test", config=config) result = docker_manager.exec_in( container_name, cmd, diff --git a/backend/analysis/services/workspace_service.py b/backend/analysis/services/workspace_service.py index bc97e13e..b7c26930 100644 --- a/backend/analysis/services/workspace_service.py +++ b/backend/analysis/services/workspace_service.py @@ -31,6 +31,8 @@ BUILD_TIMEOUT_SECONDS = 600 START_POLL_TIMEOUT = 60 WORK_DIR = "/work" +# uid of the non-root ``analyzer`` user in the analyzer-base image. +WORKSPACE_UID = 1000 def ensure_base_image() -> tuple[bool, str]: diff --git a/backend/analysis/tests/test_ai_runner.py b/backend/analysis/tests/test_ai_runner.py index 0da1a5fb..720b67a3 100644 --- a/backend/analysis/tests/test_ai_runner.py +++ b/backend/analysis/tests/test_ai_runner.py @@ -88,3 +88,36 @@ def test_truncated_but_salvageable_response_keeps_findings(self) -> None: assert not output.has_error assert len(output.findings) == 1 assert output.summary["truncated"] is True + + +def test_review_bundle_skips_lockfiles_and_puts_source_first(monkeypatch): + monkeypatch.setattr(ai_runner, "MAX_CODE_CHARS", 200) + code = { + "README.md": "r" * 150, + "frontend/package-lock.json": "{" * 10_000, + "frontend/src/App.jsx": "export default 1;", + "app.py": "print(1)", + } + blob, omitted = ai_runner._format_code(code) + assert "package-lock" not in blob + assert blob.index("app.py") < blob.index("App.jsx") + # Whole files only; what does not fit is reported, never silently cut. + assert omitted == ["README.md"] + + +def test_off_vocabulary_severity_and_category_are_normalized(): + findings = ai_runner._parse_findings( + '[{"severity": "Moderate", "category": "Security", "title": "x"}, {"severity": "urgent", "category": "ux"}]', + ) + assert [(f.severity, f.category) for f in findings] == [("medium", "security"), ("info", "quality")] + + +def test_model_severity_label_is_kept_verbatim(): + findings = ai_runner._parse_findings( + '[{"severity": "Moderate", "confidence": " High ", "title": "x"}, {"title": "no rating"}]', + ) + rated, unrated = findings + assert (rated.severity, rated.tool_severity) == ("medium", "Moderate") + assert rated.tool_specific_data == {"tool_confidence": "High"} + # A finding the model did not rate shows no severity of its own. + assert (unrated.severity, unrated.tool_severity, unrated.tool_specific_data) == ("info", "", {}) diff --git a/backend/analysis/tests/test_api.py b/backend/analysis/tests/test_api.py index 471e10e7..a6fe16e9 100644 --- a/backend/analysis/tests/test_api.py +++ b/backend/analysis/tests/test_api.py @@ -22,6 +22,24 @@ def test_list_tools(api_client): assert "hidden" not in slugs +def test_tools_expose_mode_and_technologies(api_client): + resp = api_client.get("/api/analyzers/tools/zap/") + assert resp.status_code == 200 + body = resp.json() + assert body["analysis_mode"] == "dynamic" + assert "http" in body["technologies"] + + +def test_catalog_meta(api_client): + resp = api_client.get("/api/analyzers/catalog/") + assert resp.status_code == 200 + body = resp.json() + assert {m["slug"] for m in body["modes"]} == {"static", "dynamic"} + assert {"python", "docker", "http"} <= {t["slug"] for t in body["technologies"]} + assert all(t["group"] for t in body["technologies"]) + assert body["categories"][0]["slug"] == "security" + + def test_get_workspace_creates_record(api_client, user): resp = api_client.get("/api/analyzers/workspace/") assert resp.status_code == 200 @@ -94,3 +112,48 @@ def test_install_tool_dispatches(api_client, monkeypatch): assert resp.status_code == 200 assert resp.json()["tool_slug"] == "ruff" assert resp.json()["status"] == "installing" + + +def _job_create(api_client, job_id): + return api_client.post( + "/api/analysis/runs/", + data={"tool_slugs": ["bandit"], "generation_job_id": str(job_id)}, + content_type="application/json", + ) + + +def test_create_run_rejects_another_users_job(api_client, user, monkeypatch): + from backend.generation.tests.factories import GenerationJobFactory + from backend.users.tests.factories import UserFactory + + monkeypatch.setattr(runner, "dispatch", lambda run_id: None) + foreign = GenerationJobFactory(created_by=UserFactory()) + assert _job_create(api_client, foreign.id).status_code == 404 + assert _job_create(api_client, "not-a-uuid").status_code == 404 + assert not AnalysisRun.objects.exists() + + own = GenerationJobFactory(created_by=user) + assert _job_create(api_client, own.id).status_code == 200 + + +def test_staff_may_analyze_any_job(api_client, user, monkeypatch): + from backend.generation.tests.factories import GenerationJobFactory + from backend.users.tests.factories import UserFactory + + monkeypatch.setattr(runner, "dispatch", lambda run_id: None) + user.is_staff = True + user.save() + foreign = GenerationJobFactory(created_by=UserFactory()) + assert _job_create(api_client, foreign.id).status_code == 200 + + +def test_cancel_endpoint_marks_cancelled(api_client, user): + from backend.analysis.tests.factories import AnalysisRunFactory + + run = AnalysisRunFactory(created_by=user, status="running") + resp = api_client.post(f"/api/analysis/runs/{run.id}/cancel/") + assert resp.status_code == 200 + assert resp.json()["status"] == "cancelled" + run.refresh_from_db() + assert run.status == AnalysisRun.Status.CANCELLED + assert run.completed_at is not None diff --git a/backend/analysis/tests/test_coverage.py b/backend/analysis/tests/test_coverage.py new file mode 100644 index 00000000..f3a48a9a --- /dev/null +++ b/backend/analysis/tests/test_coverage.py @@ -0,0 +1,112 @@ +"""Shop coverage: technologies in generated apps and which tools fit them.""" + +from __future__ import annotations + +import json + +import pytest + +from backend.analysis.models import ToolResult +from backend.analysis.services import runner +from backend.analysis.services.coverage import app_coverage +from backend.analysis.services.coverage import detect_technologies +from backend.analysis.tests.factories import AnalysisRunFactory +from backend.analysis.tests.factories import AnalyzerToolFactory +from backend.analysis.tests.factories import AnalyzerWorkspaceFactory +from backend.analysis.tests.factories import InstalledToolFactory +from backend.generation.models import GenerationJob +from backend.generation.tests.factories import GenerationJobFactory + +pytestmark = pytest.mark.django_db + +FLASK_REACT = { + "backend/app.py": "from flask import Flask\napp = Flask(__name__)\n", + "backend/requirements.txt": "flask==3.0.0\n", + "backend/templates/index.html": "

{{ name }}

", + "frontend/package.json": json.dumps({"dependencies": {"react": "18.2.0"}}), + "frontend/src/App.jsx": "import React from 'react'\nexport default () =>
\n", + "frontend/src/index.css": "body{}", +} + + +def test_detect_flask_react_app(): + assert detect_technologies(FLASK_REACT) == { + "python", + "flask", + "pypi", + "html", + "jinja", + "npm", + "javascript", + "react", + "css", + "http", + "rest-api", + "generic", + } + + +def test_detect_fastapi_vue_and_infra(): + files = { + "main.py": "from fastapi import FastAPI\n", + "web/App.vue": "", + "Dockerfile": "FROM python:3.12\n", + "deploy.sh": "echo hi", + "k8s/app.yaml": "apiVersion: v1\nkind: Service\n", + } + techs = detect_technologies(files) + assert {"fastapi", "vue", "javascript", "docker", "shell", "kubernetes", "rest-api"} <= techs + assert "flask" not in techs + + +def test_detect_empty_map(): + assert detect_technologies({}) == set() + + +def test_app_coverage_counts_apps_and_tool_fit(user): + GenerationJobFactory(created_by=user, status=GenerationJob.Status.COMPLETED, result_data={"files": FLASK_REACT}) + GenerationJobFactory( + created_by=user, + status=GenerationJob.Status.COMPLETED, + result_data={"files": {"main.py": "from fastapi import FastAPI\n"}}, + ) + # Not counted: failed, or someone else's. + GenerationJobFactory(created_by=user, status=GenerationJob.Status.FAILED, result_data={"files": FLASK_REACT}) + GenerationJobFactory(status=GenerationJob.Status.COMPLETED, result_data={"files": FLASK_REACT}) + + cov = app_coverage(user) + + assert cov["apps_total"] == 2 + assert cov["technologies"]["python"] == 2 + assert cov["technologies"]["react"] == 1 + assert cov["technologies"]["fastapi"] == 1 + assert cov["tools"]["ruff"] == 2 + assert cov["tools"]["eslint"] == 1 + assert cov["tools"]["hadolint"] == 0 + + +def test_coverage_endpoint(api_client): + resp = api_client.get("/api/analyzers/coverage/") + assert resp.status_code == 200 + body = resp.json() + assert body["apps_total"] == 0 + assert "ruff" in body["tools"] + + +def test_results_record_installed_tool_version(user, monkeypatch): + tool = AnalyzerToolFactory(slug="ruff-fixture", parser_key="ruff", run_cmd="ruff {target}") + workspace = AnalyzerWorkspaceFactory(user=user) + InstalledToolFactory(workspace=workspace, tool=tool, installed_version="ruff 0.6.9") + run = AnalysisRunFactory( + created_by=user, + workspace=workspace, + tool_slugs=["ruff-fixture"], + source_code={"a.py": "x = 1"}, + ) + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr(runner.docker_manager, "copy_files_in", lambda *a, **k: {"status": "ok"}) + monkeypatch.setattr(runner.docker_manager, "exec_in", lambda *a, **k: {"exit_code": 0, "output": "[]"}) + + runner.execute(run) + + assert ToolResult.objects.get(run=run).summary["tool_version"] == "ruff 0.6.9" diff --git a/backend/analysis/tests/test_dynamic_tools.py b/backend/analysis/tests/test_dynamic_tools.py new file mode 100644 index 00000000..c566f71c --- /dev/null +++ b/backend/analysis/tests/test_dynamic_tools.py @@ -0,0 +1,174 @@ +"""Dynamic (DAST) tools: target resolution, command rendering, skip behaviour.""" + +from __future__ import annotations + +import json + +import pytest + +from backend.analysis.models import AnalysisRun +from backend.analysis.models import AnalyzerTool +from backend.analysis.models import Finding +from backend.analysis.models import ToolResult +from backend.analysis.services import runner +from backend.analysis.tests.factories import AnalysisRunFactory +from backend.analysis.tests.factories import AnalyzerToolFactory +from backend.analysis.tests.factories import AnalyzerWorkspaceFactory +from backend.analysis.tests.factories import InstalledToolFactory +from backend.generation.tests.factories import GenerationJobFactory +from backend.runtime.models import ContainerInstance +from backend.runtime.tests.factories import ContainerInstanceFactory +from backend.users.tests.factories import UserFactory + +pytestmark = pytest.mark.django_db + + +def _dynamic_tool(**kwargs) -> AnalyzerTool: + defaults = { + "slug": "dast-fixture", + "kind": "container", + "analysis_mode": "dynamic", + "parser_key": "nuclei", + "run_cmd": "nuclei -u {url} -tags {config:tags}", + "default_config": {"tags": "misconfig"}, + } + return AnalyzerToolFactory(**{**defaults, **kwargs}) + + +def _running_app(job, user, name="llm-app1") -> ContainerInstance: + return ContainerInstanceFactory( + name=name, + generation_job=job, + created_by=user, + status=ContainerInstance.Status.RUNNING, + ) + + +def _inspect_with_ip(ip="172.17.0.5", running=True): + return lambda name: { + "State": {"Running": running}, + "NetworkSettings": {"Networks": {"bridge": {"IPAddress": ip}}}, + } + + +# --- render_command ---------------------------------------------------------- + + +def test_render_command_fills_target_url_and_quoted_options(): + tool = AnalyzerToolFactory.build( + run_cmd="scan {target} --url {url}/ --tags {config:tags} --n {config:n} --flag {config:missing}", + default_config={"tags": "a,b", "n": 3}, + ) + cmd = runner.render_command(tool, target="/work", url="http://10.0.0.2:8000", config={"tags": "x; rm -rf /"}) + assert cmd == "scan /work --url http://10.0.0.2:8000/ --tags 'x; rm -rf /' --n 3 --flag ''" + + +# --- resolve_app_url --------------------------------------------------------- + + +def test_resolve_requires_generation_job(user): + run = AnalysisRunFactory(created_by=user, source_code={"a.py": "x"}) + url, reason = runner.resolve_app_url(run) + assert url == "" + assert "generated app" in reason + + +def test_resolve_reports_app_not_running(user): + job = GenerationJobFactory(created_by=user) + ContainerInstanceFactory(generation_job=job, created_by=user, status=ContainerInstance.Status.STOPPED) + run = AnalysisRunFactory(created_by=user, generation_job=job) + url, reason = runner.resolve_app_url(run) + assert url == "" + assert "not running" in reason + + +def test_resolve_uses_container_ip_on_default_bridge(user, monkeypatch, settings): + settings.DOCKER_APPS_NETWORK = "" + job = GenerationJobFactory(created_by=user) + _running_app(job, user) + monkeypatch.setattr(runner.docker_manager, "inspect", _inspect_with_ip("172.17.0.9")) + run = AnalysisRunFactory(created_by=user, generation_job=job) + assert runner.resolve_app_url(run) == ("http://172.17.0.9:8000", "") + + +def test_resolve_uses_container_name_on_apps_network(user, settings): + settings.DOCKER_APPS_NETWORK = "apps" + job = GenerationJobFactory(created_by=user) + _running_app(job, user, name="llm-app2") + run = AnalysisRunFactory(created_by=user, generation_job=job) + assert runner.resolve_app_url(run) == ("http://llm-app2:8000", "") + + +def test_resolve_never_targets_another_users_app(user, settings): + settings.DOCKER_APPS_NETWORK = "apps" + other = UserFactory() + job = GenerationJobFactory(created_by=other) + _running_app(job, other) + run = AnalysisRunFactory(created_by=user, generation_job=job) + url, reason = runner.resolve_app_url(run) + assert url == "" + assert reason + + +# --- execute ----------------------------------------------------------------- + + +def _stub_workspace(monkeypatch, exec_calls, output=""): + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr(runner.docker_manager, "copy_files_in", lambda *a, **k: {"status": "ok"}) + + def _exec(name, cmd, **kwargs): + exec_calls.append(cmd) + return {"exit_code": 0, "output": output} + + monkeypatch.setattr(runner.docker_manager, "exec_in", _exec) + + +def test_execute_skips_dynamic_tool_when_app_not_running(user, monkeypatch): + _dynamic_tool() + job = GenerationJobFactory(created_by=user, result_data={"backend_code": "print(1)"}) + run = AnalysisRunFactory( + created_by=user, + workspace=AnalyzerWorkspaceFactory(user=user), + generation_job=job, + tool_slugs=["dast-fixture"], + ) + calls: list[str] = [] + _stub_workspace(monkeypatch, calls) + + runner.execute(run) + run.refresh_from_db() + + result = ToolResult.objects.get(run=run, tool_slug="dast-fixture") + assert result.status == ToolResult.Status.SKIPPED + assert "not running" in result.error_message + assert not any("nuclei" in c for c in calls) + assert run.status == AnalysisRun.Status.COMPLETED + + +def test_execute_runs_dynamic_tool_against_app_with_user_config(user, monkeypatch, settings): + settings.DOCKER_APPS_NETWORK = "apps" + tool = _dynamic_tool() + workspace = AnalyzerWorkspaceFactory(user=user) + InstalledToolFactory(workspace=workspace, tool=tool, config={"tags": "exposure"}) + job = GenerationJobFactory(created_by=user, result_data={"backend_code": "print(1)"}) + _running_app(job, user, name="llm-app3") + run = AnalysisRunFactory(created_by=user, workspace=workspace, generation_job=job, tool_slugs=["dast-fixture"]) + hit = { + "template-id": "git-config", + "info": {"name": "Git Config Disclosure", "severity": "medium"}, + "matched-at": "http://llm-app3:8000/.git/config", + } + calls: list[str] = [] + _stub_workspace(monkeypatch, calls, output=json.dumps(hit)) + + runner.execute(run) + + # Prefixed with the run's marker so cancel can kill it (runner._kill_script). + assert f": llmlab-analysis-run:{run.pk}; nuclei -u http://llm-app3:8000 -tags exposure" in calls + result = ToolResult.objects.get(run=run, tool_slug="dast-fixture") + assert result.status == ToolResult.Status.COMPLETED + assert result.raw_output["target_url"] == "http://llm-app3:8000" + assert result.raw_output["command"] == "nuclei -u http://llm-app3:8000 -tags exposure" + finding = Finding.objects.get(result=result) + assert (finding.file_path, finding.severity) == ("/.git/config", "medium") diff --git a/backend/analysis/tests/test_extra_parsers.py b/backend/analysis/tests/test_extra_parsers.py index 1346038c..81b8c758 100644 --- a/backend/analysis/tests/test_extra_parsers.py +++ b/backend/analysis/tests/test_extra_parsers.py @@ -38,7 +38,8 @@ def test_new_tools_seeded_with_parsers(): assert parsers.has_parser(tool.parser_key), slug assert tool.icon, slug assert tool.sample_code, slug - assert AnalyzerTool.objects.filter(category="dependencies").count() == 4 + dependency_tools = set(AnalyzerTool.objects.filter(category="dependencies").values_list("slug", flat=True)) + assert {"pip-audit", "npm-audit", "deptry", "depcheck", "osv-scanner", "retire"} <= dependency_tools def test_pip_audit_dedupes_repeated_vulns_per_section(): @@ -127,7 +128,7 @@ def test_flake8_severity_by_code(): "garbage line\n" ) sev = {f.rule_id: f.severity for f in parsers.parse("flake8", raw).findings} - assert sev == {"F401": "medium", "E402": "low", "F821": "high"} + assert sev == {"F401": "low", "E402": "low", "F821": "high"} def test_lizard_metrics_and_complex_functions(): @@ -248,3 +249,287 @@ def test_empty_output_yields_nothing(): out = parsers.parse(key, "") assert out.findings == [], key assert out.metrics == {}, key + + +def test_failed_manifest_is_a_coverage_gap_not_a_clean_result(): + ok = '{"dependencies": [{"name": "flask", "version": "3.1.0", "vulns": []}]}' + raw = f"### requirements.txt\n{ok}\n### backend/requirements.txt\n!! ERROR: network unreachable\n" + parsed = parsers.parse("pip_audit", raw) + assert parsed.error is None + assert parsed.warnings == ["Not analyzed: backend/requirements.txt: ERROR: network unreachable"] + + parsed = parsers.parse("pip_audit", "### requirements.txt\n\n") + assert parsed.error.startswith("No manifest could be analyzed") + + +def test_pip_audit_reports_hallucinated_packages_and_versions(): + raw = ( + "### requirements.txt\n" + "!! ERROR: Could not find a version that satisfies the requirement flask-magic-auth==1.0 " + "(from versions: none)\n" + "!! ERROR: No matching distribution found for flask-magic-auth==1.0\n" + "### api/requirements.txt\n" + "!! ERROR: No matching distribution found for flask==99.0.0\n" + ) + parsed = parsers.parse("pip_audit", raw) + by_rule = {f.rule_id: f for f in parsed.findings} + assert by_rule["nonexistent-package"].tool_specific_data["package"] == "flask-magic-auth" + assert by_rule["nonexistent-package"].severity == "high" + assert by_rule["nonexistent-version"].file_path == "api/requirements.txt" + # Neither manifest was audited for vulnerabilities: the result is failed + # (not a measurement), but the hallucinated dependencies are kept and the + # gap is recorded as a warning (the runner stores both on the result). + assert parsed.error.startswith("No manifest could be analyzed") + assert parsed.warnings[0].startswith("Not analyzed: requirements.txt") + + +def test_npm_audit_reports_hallucinated_packages(): + raw = ( + "### frontend/package.json\n" + "!! npm error code E404\n" + "!! npm error 404 'react-super-grid-zz@^1.0.0' is not in this registry.\n" + ) + findings = parsers.parse("npm_audit", raw).findings + assert [(f.rule_id, f.tool_specific_data["package"]) for f in findings] == [ + ("nonexistent-package", "react-super-grid-zz"), + ] + + +def test_parser_crash_is_an_error(monkeypatch): + monkeypatch.setitem(parsers._PARSERS, "boom", lambda raw: 1 / 0) + assert "ZeroDivisionError" in parsers.parse("boom", "x").error + + +def test_ruff_and_flake8_rate_codes_alike(): + ruff = json.dumps( + [ + {"code": "F401", "message": "unused", "filename": "a.py", "location": {"row": 1}}, + {"code": "F821", "message": "undefined", "filename": "a.py", "location": {"row": 2}}, + ], + ) + assert [f.severity for f in parsers.parse("ruff", ruff).findings] == ["low", "high"] + + +def test_semgrep_category_comes_from_rule_metadata(): + raw = json.dumps( + { + "results": [ + { + "check_id": "a.sec", + "path": "a.py", + "start": {"line": 1}, + "extra": {"metadata": {"category": "security"}}, + }, + { + "check_id": "a.bp", + "path": "a.py", + "start": {"line": 2}, + "extra": {"metadata": {"category": "best-practice"}}, + }, + ], + }, + ) + assert [f.category for f in parsers.parse("semgrep", raw).findings] == ["security", "best_practice"] + + +# --- Tool's own severity and enriched tool_specific_data --------------------- + + +def test_npm_audit_keeps_its_label_and_drops_unscored_cvss(): + report = { + "auditReportVersion": 2, + "vulnerabilities": { + "react-router": { + "severity": "moderate", + "isDirect": False, + "via": [ + { + "source": 1124268, + "title": "Open redirect", + "url": "https://github.com/advisories/GHSA-wrjc-x8rr-h8h6", + "severity": "moderate", + "cwe": ["CWE-601"], + "cvss": {"score": 0, "vectorString": None}, + "range": ">=6.0.0 <7.18.0", + }, + { + "source": 1124272, + "title": "Constructor injection", + "url": "https://github.com/advisories/GHSA-337j-9hxr-rhxg", + "severity": "high", + "cwe": ["CWE-470"], + "cvss": {"score": 6.1, "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:C/C:L/I:L/A:N"}, + "range": ">=6.4.0 <7.18.0", + }, + ], + "fixAvailable": False, + }, + }, + } + raw = f"### frontend/package.json\n{json.dumps(report)}\n" + unscored, scored = parsers.parse("npm_audit", raw).findings + assert (unscored.tool_severity, scored.tool_severity) == ("moderate", "high") + # The advisory URL is a link, not a description. + assert unscored.description == "" + assert unscored.tool_specific_data == { + "package": "react-router", + "direct": False, + "vuln_id": "GHSA-wrjc-x8rr-h8h6", + "docs_url": "https://github.com/advisories/GHSA-wrjc-x8rr-h8h6", + "manifest": "frontend/package.json", + "cwe": "CWE-601", + } + assert scored.tool_specific_data["cvss_score"] == 6.1 + assert scored.tool_specific_data["cvss_vector"].startswith("CVSS:3.1/") + assert scored.tool_specific_data["cvss"] == 6.1 # legacy key + + +def test_pip_audit_has_no_severity_and_standard_dependency_keys(): + vuln = {"id": "PYSEC-2026-2151", "fix_versions": ["3.1.3"], "aliases": ["CVE-2026-27205"], "description": "d"} + nofix = {"id": "PYSEC-2024-271", "fix_versions": [], "aliases": [], "description": "e"} + report = {"dependencies": [{"name": "flask", "version": "3.0.0", "vulns": [vuln, nofix]}]} + fixed, unfixed = parsers.parse("pip_audit", f"### requirements.txt\n{json.dumps(report)}\n").findings + assert fixed.tool_severity == "" # pip-audit's JSON rates nothing + assert fixed.tool_specific_data == { + "package": "flask", + "installed_version": "3.0.0", + "fixed_versions": ["3.1.3"], + "vuln_id": "PYSEC-2026-2151", + "aliases": ["CVE-2026-27205"], + "manifest": "requirements.txt", + "version": "3.0.0", + "fix_versions": ["3.1.3"], + } + assert unfixed.suggestion == "" # no invented "no fixed release" text + assert "fixed_versions" not in unfixed.tool_specific_data + + +def test_lint_tools_report_their_levels_and_docs(): + shell = json.dumps( + [ + { + "file": "/work/x.sh", + "line": 2, + "endLine": 2, + "column": 8, + "endColumn": 12, + "level": "style", + "code": 2006, + "message": "Use $(...)", + "fix": None, + }, + ], + ) + [sh] = parsers.parse("shellcheck", shell).findings + assert sh.tool_severity == "style" + assert sh.suggestion == "" + assert sh.tool_specific_data == { + "docs_url": "https://www.shellcheck.net/wiki/SC2006", + "end_line": 2, + "end_column": 12, + } + + css = json.dumps( + [ + { + "source": "/work/x.css", + "warnings": [ + { + "line": 1, + "column": 5, + "endLine": 1, + "endColumn": 9, + "rule": "property-no-unknown", + "severity": "error", + "text": 'Unexpected unknown property "colr" (property-no-unknown)', + }, + {"line": 3, "column": 1, "rule": "CssSyntaxError", "severity": "error", "text": "Unclosed block"}, + ], + }, + ], + ) + known, syntax = parsers.parse("stylelint", css).findings + assert known.tool_severity == "error" + assert known.tool_specific_data["docs_url"] == "https://stylelint.io/user-guide/rules/property-no-unknown" + assert "docs_url" not in syntax.tool_specific_data + + html = json.dumps( + [ + { + "file": "/work/index.html", + "messages": [ + { + "type": "warning", + "message": "The id value [ a ] must be unique.", + "line": 1, + "col": 52, + "rule": { + "id": "id-unique", + "description": "The value of id attributes must be unique.", + "link": "https://htmlhint.com/rules/id-unique", + }, + }, + ], + }, + ], + ) + [h] = parsers.parse("htmlhint", html).findings + assert h.tool_severity == "warning" + assert h.tool_specific_data == {"docs_url": "https://htmlhint.com/rules/id-unique"} + + +def test_unrated_dependency_and_complexity_tools(): + deptry = [ + {"error": {"code": "DEP001", "message": "'yaml' imported but missing"}, "module": "yaml", "location": {}}, + ] + [dep] = parsers.parse("deptry", f"### .\n{json.dumps(deptry)}\n").findings + assert (dep.tool_severity, dep.suggestion) == ("", "") + + depcheck = {"dependencies": ["uuid"], "devDependencies": [], "missing": {"axios": ["/work/src/App.jsx"]}} + found = parsers.parse("depcheck", f"### frontend/package.json\n{json.dumps(depcheck)}\n").findings + assert {f.tool_severity for f in found} == {""} + assert {f.suggestion for f in found} == {""} + assert all(f.tool_specific_data["manifest"] == "frontend/package.json" for f in found) + + lizard = '30,26,300,4,60,"route@10-70@/work/app.py","/work/app.py","route","route( a, b, c, d )",10,70\n' + [cx] = parsers.parse("lizard", lizard).findings + assert (cx.tool_severity, cx.suggestion) == ("", "") + assert cx.tool_specific_data == {"complexity": 26, "function": "route", "nloc": 30, "end_line": 70, "ccn": 26} + + +def test_perflint_keeps_pylint_level_without_core_docs_link(): + raw = json.dumps( + [ + { + "type": "warning", + "line": 307, + "column": 4, + "endLine": 308, + "endColumn": 60, + "path": "app.py", + "symbol": "use-list-copy", + "message": "Use a list copy instead of a for-loop", + "message-id": "W8402", + "obj": "board_view", + }, + ], + ) + [f] = parsers.parse("perflint", raw).findings + assert f.tool_severity == "warning" + assert f.tool_specific_data == {"obj": "board_view", "message_id": "W8402", "end_line": 308, "end_column": 60} + + +def test_eslint_security_plugin_rules_keep_level_and_cwe(): + raw = json.dumps( + [ + { + "filePath": "/work/x.jsx", + "messages": [ + {"ruleId": "security/detect-child-process", "severity": 1, "message": "child", "line": 2}, + ], + }, + ], + ) + [f] = parsers.parse("eslint_security", raw).findings + assert f.tool_severity == "warning" + assert f.tool_specific_data == {"cwe": "CWE-78"} diff --git a/backend/analysis/tests/test_parsers.py b/backend/analysis/tests/test_parsers.py index 76ae4f9c..a162368c 100644 --- a/backend/analysis/tests/test_parsers.py +++ b/backend/analysis/tests/test_parsers.py @@ -262,8 +262,9 @@ def test_detect_secrets_parser(): assert f.category == "secrets" assert f.rule_id == "AWS Access Key" assert f.line_number == 2 - assert f.tool_specific_data == {"hashed_secret": "abc123", "is_verified": False} + assert f.tool_specific_data == {"secret_type": "AWS Access Key", "hashed_secret": "abc123", "is_verified": False} assert f.code_snippet == "" + assert f.tool_severity == "" # detect-secrets rates nothing def test_jscpd_parser_truncates_fragment(): @@ -400,3 +401,322 @@ def test_bandit_and_semgrep_record_cwe(): assert semgrep.findings[0].tool_specific_data["cwe"] == "CWE-89" no_cwe = parse("bandit", json.dumps({"results": [{"test_id": "B101"}]})) assert "cwe" not in no_cwe.findings[0].tool_specific_data + + +# --- Tool's own severity / confidence and enriched tool_specific_data -------- +# +# ``tool_severity`` is the label exactly as the tool printed it, "" for tools +# that rate nothing; ``severity`` stays our normalization for statistics. + + +def test_bandit_keeps_its_own_labels_and_rule_docs(): + raw = json.dumps( + { + "results": [ + { + "filename": "/work/app.py", + "issue_severity": "MEDIUM", + "issue_confidence": "LOW", + "issue_text": "Possible SQL injection vector through string-based query construction.", + "issue_cwe": {"id": 89, "link": "https://cwe.mitre.org/data/definitions/89.html"}, + "test_id": "B608", + "test_name": "hardcoded_sql_expressions", + "line_number": 407, + "line_range": [406, 407, 408], + "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b608_hardcoded_sql_expressions.html", + "code": "db.execute(f'...')", + }, + ], + }, + ) + [f] = parsers.parse("bandit", raw).findings + assert (f.severity, f.confidence) == ("medium", "low") # normalization unchanged + assert f.tool_severity == "MEDIUM" + data = f.tool_specific_data + assert data["tool_confidence"] == "LOW" + assert data["rule_name"] == "hardcoded_sql_expressions" + assert data["docs_url"].endswith("b608_hardcoded_sql_expressions.html") + assert (data["end_line"], data["cwe"]) == (408, "CWE-89") + + +def test_ruff_rates_nothing_but_reports_docs_and_fix(): + raw = json.dumps( + [ + { + "code": "F401", + "message": "`typing.List` imported but unused", + "filename": "/work/app.py", + "location": {"row": 8, "column": 20}, + "end_location": {"row": 8, "column": 24}, + "fix": {"applicability": "safe", "edits": [], "message": "Remove unused import: `typing.List`"}, + "url": "https://docs.astral.sh/ruff/rules/unused-import", + }, + ], + ) + [f] = parsers.parse("ruff", raw).findings + assert f.severity == "low" # heuristic normalization, kept for statistics + assert f.tool_severity == "" # ruff prints no level + assert "tool_confidence" not in f.tool_specific_data + assert f.tool_specific_data == { + "rule_name": "unused-import", + "docs_url": "https://docs.astral.sh/ruff/rules/unused-import", + "fix": "Remove unused import: `typing.List`", + "end_line": 8, + "end_column": 24, + } + + +def test_flake8_and_vulture_labels(): + [flake] = parsers.parse("flake8", "/work/app.py:8:1: F401 'typing.List' imported but unused\n").findings + assert flake.tool_severity == "" + [dead] = parsers.parse("vulture", "schemas.py:25: unused variable 'items' (60% confidence)\n").findings + assert dead.tool_severity == "" + assert dead.tool_specific_data["tool_confidence"] == "60%" + + +def test_eslint_levels_core_rule_docs_and_suggestions(): + raw = json.dumps( + [ + { + "filePath": "/work/x.js", + "messages": [ + { + "ruleId": "no-empty", + "severity": 2, + "message": "Empty block statement.", + "line": 3, + "column": 8, + "endLine": 3, + "endColumn": 10, + "suggestions": [ + { + "desc": "Add comment inside empty block statement.", + "fix": {"range": [1, 1], "text": ""}, + }, + ], + }, + {"ruleId": "react-hooks/exhaustive-deps", "severity": 1, "message": "deps", "line": 9}, + {"ruleId": None, "fatal": True, "severity": 2, "message": "Parsing error: x", "line": 1}, + ], + }, + ], + ) + core, plugin, fatal = parsers.parse("eslint", raw).findings + assert [f.tool_severity for f in (core, plugin, fatal)] == ["error", "warning", "error"] + assert core.tool_specific_data == { + "docs_url": "https://eslint.org/docs/latest/rules/no-empty", + "fix": "Add comment inside empty block statement.", + "end_line": 3, + "end_column": 10, + } + # Plugin rules get no guessed documentation link. + assert "docs_url" not in plugin.tool_specific_data + assert fatal.tool_specific_data == {} + + +def test_semgrep_labels_metadata_and_login_placeholder(): + raw = json.dumps( + { + "results": [ + { + "check_id": "python.flask.security.audit.app-run-param-config.avoid_app_run_with_bad_host", + "path": "/work/app.py", + "start": {"line": 484, "col": 5}, + "end": {"line": 484, "col": 52}, + "extra": { + "message": "Running flask app with host 0.0.0.0 could expose the server publicly.", + "metadata": { + "cwe": ["CWE-668: Exposure of Resource to Wrong Sphere"], + "owasp": ["A01:2021 - Broken Access Control"], + "category": "security", + "references": ["https://owasp.org/Top10/A01_2021-Broken_Access_Control"], + "likelihood": "HIGH", + "impact": "MEDIUM", + "confidence": "LOW", + "source": "https://semgrep.dev/r/python.flask.security.audit.app-run-param-config", + "shortlink": "https://sg.run/eLby", + }, + "severity": "WARNING", + "fingerprint": "requires login", + "lines": "requires login", + }, + }, + ], + }, + ) + [f] = parsers.parse("semgrep", raw).findings + assert (f.severity, f.tool_severity) == ("medium", "WARNING") + assert f.code_snippet == "" # semgrep OSS placeholder, not code + data = f.tool_specific_data + assert data["tool_confidence"] == "LOW" + assert data["rule_name"] == "avoid_app_run_with_bad_host" + assert data["docs_url"] == "https://semgrep.dev/r/python.flask.security.audit.app-run-param-config" + assert data["references"] == ["https://owasp.org/Top10/A01_2021-Broken_Access_Control"] + assert data["owasp"] == ["A01:2021 - Broken Access Control"] + assert (data["likelihood"], data["impact"]) == ("HIGH", "MEDIUM") + assert (data["end_line"], data["end_column"], data["cwe"]) == (484, 52, "CWE-668") + + +def test_pylint_message_type_and_docs_url(): + raw = json.dumps( + [ + { + "type": "convention", + "obj": "", + "line": 251, + "column": 0, + "endLine": None, + "endColumn": None, + "path": "app.py", + "symbol": "line-too-long", + "message": "Line too long (102/100)", + "message-id": "C0301", + }, + { + "type": "info", + "obj": "f", + "line": 3, + "column": 0, + "path": "app.py", + "symbol": "locally-disabled", + "message": "Locally disabling x", + "message-id": "I0011", + }, + ], + ) + conv, info = parsers.parse("pylint", raw).findings + assert (conv.tool_severity, info.tool_severity) == ("convention", "info") + assert conv.tool_specific_data == { + "message_id": "C0301", + "docs_url": "https://pylint.readthedocs.io/en/stable/user_guide/messages/convention/line-too-long.html", + } + # Pylint's "info" messages live under "information" in its docs. + assert info.tool_specific_data["docs_url"].endswith("/messages/information/locally-disabled.html") + + +def test_mypy_attaches_notes_to_their_error(): + raw = ( + 'app.py:50: error: Variable "app.Base" is not valid as a type [valid-type]\n' + "app.py:50: note: See https://mypy.readthedocs.io/en/stable/common_issues.html#variables-vs-type-aliases\n" + 'app.py:60: error: No overload variant of "get" matches argument types "int" [call-overload]\n' + "app.py:60: note: Possible overload variants:\n" + "app.py:99: note: Revealed type is 'int'\n" + ) + first, second = parsers.parse("mypy", raw).findings + assert first.tool_severity == "error" + assert first.tool_specific_data["docs_url"].endswith("#variables-vs-type-aliases") + assert second.description.endswith("\nPossible overload variants:") + assert "docs_url" not in second.tool_specific_data + + +def test_hadolint_levels_and_rule_docs(): + raw = json.dumps( + [ + {"file": "Dockerfile", "line": 1, "code": "DL3007", "level": "warning", "message": "latest"}, + {"file": "Dockerfile", "line": 3, "code": "SC2164", "level": "style", "message": "cd || exit"}, + ], + ) + dl, sc = parsers.parse("hadolint", raw).findings + assert (dl.tool_severity, sc.tool_severity) == ("warning", "style") + assert dl.tool_specific_data["docs_url"] == "https://github.com/hadolint/hadolint/wiki/DL3007" + assert sc.tool_specific_data["docs_url"] == "https://www.shellcheck.net/wiki/SC2164" + + +def test_codespell_fix_and_jscpd_duplicate_location(): + [typo] = parsers.parse("codespell", "/work/App.jsx:1353: floatIn ==> floating, float in\n").findings + assert typo.tool_specific_data == {"fix": "floating, float in"} + raw = json.dumps( + { + "duplicates": [ + { + "lines": 15, + "tokens": 124, + "fragment": "x", + "firstFile": {"name": "src/App.jsx", "start": 2023, "end": 2037}, + "secondFile": {"name": "src/App.jsx", "start": 1818, "end": 1829}, + }, + ], + }, + ) + [dup] = parsers.parse("jscpd", raw).findings + assert dup.tool_severity == "" + data = dup.tool_specific_data + assert data["duplicate_of"] == {"file": "src/App.jsx", "start_line": 1818, "end_line": 1829} + assert (data["lines"], data["tokens"], data["end_line"]) == (15, 124, 2037) + assert (data["second_file"], data["second_start"]) == ("src/App.jsx", 1818) # legacy keys kept + + +def test_radon_complexity_data(): + raw = json.dumps( + { + "app.py": [ + { + "type": "class", + "name": "Task", + "rank": "A", + "complexity": 3, + "lineno": 1, + "endline": 90, + "methods": [ + { + "type": "method", + "name": "to_dict", + "classname": "Task", + "rank": "F", + "complexity": 44, + "lineno": 5, + "endline": 80, + }, + ], + }, + ], + }, + ) + [f] = parsers.parse("radon", raw).findings + assert f.tool_severity == "" # a complexity rank, not a severity + assert f.tool_specific_data == { + "complexity": 44, + "rank": "F", + "function": "Task.to_dict", + "end_line": 80, + "endline": 80, + } + + +def test_gitleaks_never_stores_the_secret_value(): + secret = "ghp_1234567890abcdefghijklmnopqrstuvwxyz" # noqa: S105 - a fake token gitleaks:allow + item = { + "RuleID": "github-pat", + "Description": "Uncovered a GitHub Personal Access Token", + "File": "config.py", + "StartLine": 2, + "EndLine": 2, + "StartColumn": 16, + "EndColumn": 57, + "Entropy": 5.17, + } + # As the run command asks for it (--redact) ... + [redacted] = parsers.parse( + "gitleaks", + json.dumps([{**item, "Match": 'TOKEN = "REDACTED"', "Secret": "REDACTED"}]), + ).findings + assert redacted.code_snippet == 'TOKEN = "REDACTED"' + assert redacted.tool_specific_data == { + "secret_type": "github-pat", + "entropy": 5.17, + "end_line": 2, + "end_column": 57, + } + # ... and a report that arrives unredacted: the value is masked anyway. + leaky = [ + {**item, "Match": f'GITHUB_TOKEN = "{secret}"', "Secret": secret}, + {**item, "Match": "", "Secret": secret}, + {**item, "Match": "token\n", "Secret": "token\n\nrest"}, + ] + findings = parsers.parse("gitleaks", json.dumps(leaky)).findings + assert findings[0].code_snippet == 'GITHUB_TOKEN = "REDACTED"' + assert findings[1].code_snippet == "" + assert findings[2].code_snippet == "" # only part of the secret: nothing to mask it with + for f in findings: + assert secret not in repr(f) + assert "rest" not in f.code_snippet diff --git a/backend/analysis/tests/test_run_lifecycle.py b/backend/analysis/tests/test_run_lifecycle.py new file mode 100644 index 00000000..717d9ef1 --- /dev/null +++ b/backend/analysis/tests/test_run_lifecycle.py @@ -0,0 +1,175 @@ +"""Live feedback of a run: per-tool timing and events, phases, the run queue and its reaping.""" + +from __future__ import annotations + +import json +from datetime import timedelta + +import pytest +from django.utils import timezone + +from backend.analysis.models import AnalysisRun +from backend.analysis.models import ToolResult +from backend.analysis.services import runner +from backend.analysis.tests.factories import AnalysisRunFactory +from backend.analysis.tests.factories import AnalyzerToolFactory +from backend.analysis.tests.factories import AnalyzerWorkspaceFactory + +pytestmark = pytest.mark.django_db + + +@pytest.fixture +def events(monkeypatch): + sent: list[tuple[str, dict]] = [] + monkeypatch.setattr(runner.realtime, "publish", lambda channel, event: sent.append((channel, event))) + return sent + + +def _run_with_tools(monkeypatch, user, *, ruff_output): + AnalyzerToolFactory(slug="fixture-ruff", kind="container", parser_key="ruff", run_cmd="ruff {target}") + # Reads only HTML: not applicable to a Python-only app. + AnalyzerToolFactory(slug="fixture-html", kind="container", parser_key="ruff", file_patterns=["*.html"]) + run = AnalysisRunFactory( + created_by=user, + workspace=AnalyzerWorkspaceFactory(user=user), + tool_slugs=["fixture-ruff", "fixture-html"], + source_code={"app.py": "import os\n"}, + ) + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr(runner.docker_manager, "copy_files_in", lambda *a, **k: {"status": "ok"}) + monkeypatch.setattr( + runner.docker_manager, + "exec_in", + lambda name, cmd, **k: ( + {"exit_code": 0, "output": ""} if cmd.startswith("find ") else {"exit_code": 1, "output": ruff_output} + ), + ) + return run + + +RUFF_OUTPUT = json.dumps( + [ + { + "code": "F401", + "message": "`os` imported but unused", + "filename": "/work/app.py", + "location": {"row": 1, "column": 8}, + "end_location": {"row": 1, "column": 10}, + }, + ], +) + + +def test_execute_times_tools_and_streams_their_results(monkeypatch, user, events): + run = _run_with_tools(monkeypatch, user, ruff_output=RUFF_OUTPUT) + runner.execute(run) + run.refresh_from_db() + + ran = run.results.get(tool_slug="fixture-ruff") + skipped = run.results.get(tool_slug="fixture-html") + assert ran.status == ToolResult.Status.COMPLETED + assert ran.started_at is not None + assert ran.duration_seconds is not None + assert ran.duration_seconds >= 0 + # A tool that never ran has no timing. + assert skipped.status == ToolResult.Status.SKIPPED + assert skipped.started_at is None + assert skipped.duration_seconds is None + + channel = f"analysis:{run.id}" + assert {c for c, _ in events} == {channel} + phases = [e["phase"] for _, e in events if e["type"] == "progress"] + assert phases == [runner.PHASE_PREPARING, runner.PHASE_TOOLS] + results = [(e["result"]["tool_slug"], e["result"]["status"]) for _, e in events if e["type"] == "result"] + assert results == [("fixture-ruff", "running"), ("fixture-ruff", "completed"), ("fixture-html", "skipped")] + statuses = [e["status"] for _, e in events if e["type"] == "status"] + assert statuses == ["running", "completed"] + + +def test_finalize_counts_levels_as_the_tools_reported_them(monkeypatch, user, events): + run = _run_with_tools(monkeypatch, user, ruff_output=RUFF_OUTPUT) + runner.execute(run) + run.refresh_from_db() + finding = run.results.get(tool_slug="fixture-ruff").findings.get() + # Ruff rates nothing: the stored normalized level is a heuristic, the reported one is empty. + assert finding.tool_severity == "" + assert run.summary["unrated_findings"] == 1 + assert run.summary["reported_severity_counts"] == {} + assert "phase" not in run.summary + + +def test_reported_levels_are_stored_and_counted(monkeypatch, user, events): + from backend.analysis.services.base import AnalyzerOutput + from backend.analysis.services.base import FindingData + + AnalyzerToolFactory(slug="fixture-ai", kind="ai", parser_key="") + run = AnalysisRunFactory(created_by=user, tool_slugs=["fixture-ai"], source_code={"app.py": "x = 1\n"}) + output = AnalyzerOutput( + findings=[ + FindingData(severity="high", category="security", title="a", tool_severity="HIGH"), + FindingData(severity="high", category="security", title="b", tool_severity="high"), + FindingData(severity="low", category="quality", title="c"), + ], + ) + monkeypatch.setattr(runner.ai_runner, "run", lambda *a, **k: output) + runner.execute(run) + run.refresh_from_db() + assert sorted(run.results.get().findings.values_list("tool_severity", flat=True)) == ["", "HIGH", "high"] + assert run.summary["reported_severity_counts"] == {"high": 2} + assert run.summary["unrated_findings"] == 1 + + +def test_dispatch_many_queues_runs_on_one_thread(monkeypatch, user, django_capture_on_commit_callbacks): + queued = [AnalysisRunFactory(created_by=user) for _ in range(3)] + started: list = [] + monkeypatch.setattr(runner, "dispatch_in_thread", lambda fn, *args, **kw: started.append((fn, args))) + with django_capture_on_commit_callbacks(execute=True): + runner.dispatch_many([r.id for r in queued]) + # The thread starts only after the creating transaction commits. + assert started == [] + + assert len(started) == 1 + fn, args = started[0] + assert fn is runner._execute_queue + assert args == ([str(r.id) for r in queued],) + # Queued runs carry a heartbeat, so a dead queue can be told from a live one. + for run in queued: + run.refresh_from_db() + assert run.heartbeat_at is not None + + +def test_execute_queue_runs_in_order(monkeypatch, user): + order: list[str] = [] + monkeypatch.setattr(runner, "_execute", order.append) + runner._execute_queue(["a", "b", "c"]) + assert order == ["a", "b", "c"] + + +def test_reaper_fails_queued_runs_whose_queue_died(user): + old = timezone.now() - runner.STALE_AFTER - timedelta(minutes=1) + dead_queue = AnalysisRunFactory(created_by=user, status="pending", heartbeat_at=old) + live_queue = AnalysisRunFactory(created_by=user, status="pending", heartbeat_at=timezone.now()) + never_queued = AnalysisRunFactory(created_by=user, status="pending", heartbeat_at=None) + + assert runner.reap_stale_runs() == 1 + for run in (dead_queue, live_queue, never_queued): + run.refresh_from_db() + assert dead_queue.status == AnalysisRun.Status.FAILED + assert dead_queue.error_message == runner.QUEUE_INTERRUPTED_MESSAGE + assert live_queue.status == never_queued.status == AnalysisRun.Status.PENDING + + +def test_failed_preparation_drops_the_phase(monkeypatch, user, events): + run = _run_with_tools(monkeypatch, user, ruff_output=RUFF_OUTPUT) + + def broken(ws): + msg = "workspace down" + raise RuntimeError(msg) + + monkeypatch.setattr(runner.workspace_service, "require_ready_container", broken) + runner.execute(run) + run.refresh_from_db() + assert run.status == AnalysisRun.Status.FAILED + assert run.error_message == "workspace down" + assert "phase" not in run.summary + assert run.duration_seconds is not None diff --git a/backend/analysis/tests/test_runner.py b/backend/analysis/tests/test_runner.py index 2f7b3010..f707a86c 100644 --- a/backend/analysis/tests/test_runner.py +++ b/backend/analysis/tests/test_runner.py @@ -58,7 +58,10 @@ def test_execute_container_tool(monkeypatch, user): monkeypatch.setattr( runner.docker_manager, "exec_in", - lambda *a, **k: {"exit_code": 1, "output": bandit_out}, + # The /work cleanup succeeds; bandit exits 1 because it found something. + lambda name, cmd, **k: ( + {"exit_code": 0, "output": ""} if cmd.startswith("find ") else {"exit_code": 1, "output": bandit_out} + ), ) runner.execute(run) @@ -254,3 +257,276 @@ def test_truncated_generation_input_flagged_in_summary(monkeypatch, user): runner.execute(run) run.refresh_from_db() assert run.summary["truncated_inputs"] == ["frontend"] + + +def _container_run(monkeypatch, user, tool_kw, exec_result, source=None): + """Run one container tool whose command returns *exec_result* (cleanup succeeds).""" + AnalyzerToolFactory(slug="fixture-tool", kind="container", **tool_kw) + run = AnalysisRunFactory( + created_by=user, + workspace=AnalyzerWorkspaceFactory(user=user), + tool_slugs=["fixture-tool"], + source_code=source or {"app.py": "import os\n"}, + ) + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr(runner.docker_manager, "copy_files_in", lambda *a, **k: {"status": "ok"}) + monkeypatch.setattr( + runner.docker_manager, + "exec_in", + lambda name, cmd, **k: {"exit_code": 0, "output": ""} if cmd.startswith("find ") else exec_result, + ) + runner.execute(run) + run.refresh_from_db() + return run, run.results.get() + + +@pytest.mark.parametrize( + ("exec_result", "message"), + [ + ({"exit_code": 137, "output": "partial"}, "timeout"), + ({"exit_code": 127, "output": "sh: ruff: not found"}, "not found"), + ({"exit_code": 2, "output": "ruff: error: bad flag"}, "Exited with code 2"), + ], +) +def test_tool_failures_are_not_clean_results(monkeypatch, user, exec_result, message): + run, result = _container_run(monkeypatch, user, {"parser_key": "ruff", "run_cmd": "ruff {target}"}, exec_result) + assert result.status == "failed" + assert message in result.error_message + assert run.status == AnalysisRun.Status.FAILED + assert result.raw_output["command"] == "ruff /work" + + +def test_unparsable_output_with_findings_exit_still_completes(monkeypatch, user): + # Linters exit 1 because they found something: that is a result, not a failure. + out = json.dumps([{"code": "F401", "message": "unused", "filename": "/work/app.py", "location": {"row": 1}}]) + _, result = _container_run( + monkeypatch, + user, + {"parser_key": "ruff", "run_cmd": "ruff {target}"}, + {"exit_code": 1, "output": out}, + ) + assert result.status == "completed" + assert result.findings.count() == 1 + assert result.summary["config"] == {} + + +def test_tool_without_applicable_files_is_skipped(monkeypatch, user): + run, result = _container_run( + monkeypatch, + user, + {"parser_key": "eslint", "run_cmd": "eslint {target}", "file_patterns": ["*.js", "*.jsx"]}, + {"exit_code": 0, "output": "[]"}, + ) + assert result.status == "skipped" + assert result.summary["skip_reason"] == runner.SKIP_NOT_APPLICABLE + assert run.status == AnalysisRun.Status.COMPLETED + + +def test_applicable_files_match_names_and_paths(): + tool = AnalyzerToolFactory.build(file_patterns=["requirements*.txt", "frontend/*.json"]) + files = {"backend/requirements-dev.txt": "", "frontend/package.json": "", "app.py": ""} + assert sorted(runner.applicable_files(tool, files)) == ["backend/requirements-dev.txt", "frontend/package.json"] + assert runner.applicable_files(AnalyzerToolFactory.build(file_patterns=[]), {"a.py": ""}) == ["a.py"] + + +def test_workspace_cleanup_failure_fails_the_run(monkeypatch, user): + AnalyzerToolFactory(slug="fixture-tool", kind="container", parser_key="ruff", run_cmd="ruff {target}") + run = AnalysisRunFactory( + created_by=user, + workspace=AnalyzerWorkspaceFactory(user=user), + tool_slugs=["fixture-tool"], + source_code={"app.py": "x = 1\n"}, + ) + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr( + runner.docker_manager, + "exec_in", + lambda *a, **k: {"exit_code": 1, "output": "find: cannot delete '/work/frontend': Permission denied"}, + ) + runner.execute(run) + run.refresh_from_db() + # Stale files from the previous run would otherwise be scanned as this app's. + assert run.status == AnalysisRun.Status.FAILED + assert "Could not clear the workspace" in run.error_message + + +def test_job_metadata_fields_are_not_analyzed(user): + from backend.generation.tests.factories import GenerationJobFactory + + job = GenerationJobFactory(created_by=user, result_data={"content": "print(1)", "finish_reason": "stop"}) + run = AnalysisRunFactory(created_by=user, source_code=None, generation_job=job) + assert run.get_code_for_analysis() == {"content": "print(1)"} + + +def test_single_manifest_hallucinated_dependency_is_stored_on_failed_result(monkeypatch, user): + # One requirements.txt whose only problem is a package that does not exist: + # the audit could not run (failed, not measured), but the finding is real. + raw = "### requirements.txt\n!! ERROR: No matching distribution found for flask-magic-auth==1.0\n" + run, result = _container_run( + monkeypatch, + user, + {"parser_key": "pip_audit", "run_cmd": "audit {target}", "file_patterns": ["requirements*.txt"]}, + {"exit_code": 0, "output": raw}, + source={"requirements.txt": "flask-magic-auth==1.0\n"}, + ) + assert result.status == "failed" + assert result.error_message.startswith("No manifest could be analyzed") + assert [f.rule_id for f in result.findings.all()] == ["nonexistent-version"] + assert result.summary["finding_count"] == 1 + assert result.summary["warnings"][0].startswith("Not analyzed: requirements.txt") + assert run.summary["total_findings"] == 1 + + +def _tool_run(user, slugs): + for slug in slugs: + AnalyzerToolFactory(slug=slug, kind="container", parser_key="ruff", run_cmd="ruff {target}") + return AnalysisRunFactory( + created_by=user, + workspace=AnalyzerWorkspaceFactory(user=user), + tool_slugs=slugs, + source_code={"app.py": "import os\n"}, + ) + + +def test_cancel_during_a_tool_stops_the_run_and_sticks(monkeypatch, user): + run = _tool_run(user, ["fixture-a", "fixture-b"]) + monkeypatch.setattr(runner.workspace_service, "require_ready_container", lambda ws: "cname") + monkeypatch.setattr(runner.docker_manager, "copy_files_in", lambda *a, **k: {"status": "ok"}) + commands = [] + + def fake_exec(name, cmd, **k): + commands.append(cmd) + if cmd.startswith("find "): + return {"exit_code": 0, "output": ""} + # The user cancels while the first tool runs; cancel kills it (exit 137). + AnalysisRun.objects.filter(pk=run.pk).update(status=AnalysisRun.Status.CANCELLED) + return {"exit_code": 137, "output": ""} + + monkeypatch.setattr(runner.docker_manager, "exec_in", fake_exec) + runner.execute(run) + run.refresh_from_db() + + assert run.status == AnalysisRun.Status.CANCELLED + results = {r.tool_slug: r for r in run.results.all()} + assert {r.status for r in results.values()} == {"skipped"} + assert {r.summary["skip_reason"] for r in results.values()} == {runner.SKIP_CANCELLED} + # The second tool never ran; the first carried the cancel marker. + assert len(commands) == 2 + assert commands[1].startswith(f": llmlab-analysis-run:{run.pk}; ruff /work") + assert results["fixture-a"].raw_output == {} + assert run.summary["tools_run"] == 2 + + +def test_cancelled_before_start_never_runs(monkeypatch, user): + run = _tool_run(user, ["fixture-a"]) + AnalysisRun.objects.filter(pk=run.pk).update(status=AnalysisRun.Status.CANCELLED) + monkeypatch.setattr(runner.docker_manager, "exec_in", lambda *a, **k: pytest.fail("must not run")) + runner.execute(run) + run.refresh_from_db() + assert run.status == AnalysisRun.Status.CANCELLED + assert not run.results.exists() + + +def test_finalize_and_failure_do_not_overwrite_cancelled(user): + run = _tool_run(user, ["fixture-a"]) + AnalysisRun.objects.filter(pk=run.pk).update(status=AnalysisRun.Status.CANCELLED) + runner._finalize(run, ["completed"]) + run.refresh_from_db() + assert run.status == AnalysisRun.Status.CANCELLED + assert run.summary["tools_completed"] == 1 + runner._fail(run, "boom") + run.refresh_from_db() + assert run.status == AnalysisRun.Status.CANCELLED + assert run.error_message == "" + + +def test_cancel_kills_in_flight_tool_in_workspace(monkeypatch, user, django_capture_on_commit_callbacks): + from backend.runtime.tests.factories import ContainerInstanceFactory + + run = _tool_run(user, ["fixture-a"]) + run.workspace.container = ContainerInstanceFactory(name="analyzer-ws") + run.workspace.save() + AnalysisRun.objects.filter(pk=run.pk).update(status=AnalysisRun.Status.RUNNING) + calls = [] + monkeypatch.setattr( + runner.docker_manager, + "exec_in", + lambda name, cmd, **k: calls.append((name, cmd)) or {"exit_code": 0, "output": "killed 1"}, + ) + with django_capture_on_commit_callbacks() as callbacks: + assert runner.cancel(run) is True + assert run.status == AnalysisRun.Status.CANCELLED + # The kill waits for the cancel to be committed (the runner must see it when the tool dies). + assert calls == [] + for callback in callbacks: + callback() + assert calls[0][0] == "analyzer-ws" + # The script assembles the marker at run time, so it never matches itself. + assert f"llmlab-analysis-run:{run.pk}" not in calls[0][1] + assert str(run.pk) in calls[0][1] + # Already finished: nothing to cancel. + assert runner.cancel(run) is False + + +def test_kill_script_kills_only_the_marked_process_group(): + import shutil + import subprocess + import time + from pathlib import Path + + timeout_bin, sh_bin = shutil.which("timeout"), shutil.which("sh") + if not timeout_bin or not sh_bin or not Path("/proc/self/stat").exists(): + pytest.skip("needs coreutils timeout and /proc") + marker = "llmlab-analysis-run" + # Same shape as a tool exec: timeout -s KILL sh -c ": :; ". + tool = subprocess.Popen([timeout_bin, "-s", "KILL", "60", sh_bin, "-c", f": {marker}:r1; sleep 30"]) # noqa: S603 + other = subprocess.Popen([timeout_bin, "-s", "KILL", "60", sh_bin, "-c", f": {marker}:r2; sleep 30"]) # noqa: S603 + try: + time.sleep(0.3) + out = subprocess.run( # noqa: S603 + [sh_bin, "-c", runner._kill_script("r1")], + capture_output=True, + text=True, + timeout=15, + check=False, + ) + assert out.stdout.strip() == "killed 1" + assert tool.wait(timeout=5) == -9 + assert other.poll() is None + finally: + other.kill() + tool.kill() + + +def test_reap_stale_runs_fails_only_runs_without_a_live_heartbeat(user): + from datetime import timedelta + + from django.utils import timezone + + from backend.analysis.models import ToolResult + + now = timezone.now() + old = now - runner.STALE_AFTER - timedelta(minutes=1) + dead = AnalysisRunFactory(created_by=user, status="running", started_at=old, heartbeat_at=old) + ToolResult.objects.create(run=dead, tool_slug="semgrep", status="running") + legacy = AnalysisRunFactory(created_by=user, status="running", started_at=old, heartbeat_at=None) + # Long-running but beating: a live thread in some process owns it. + alive = AnalysisRunFactory(created_by=user, status="running", started_at=old, heartbeat_at=now) + pending = AnalysisRunFactory(created_by=user, status="pending") + + assert runner.reap_stale_runs() == 2 + for run in (dead, legacy, alive, pending): + run.refresh_from_db() + assert dead.status == legacy.status == AnalysisRun.Status.FAILED + assert dead.error_message == runner.INTERRUPTED_MESSAGE + assert dead.results.get().status == "failed" + assert alive.status == AnalysisRun.Status.RUNNING + assert pending.status == AnalysisRun.Status.PENDING + + +def test_execute_records_a_heartbeat(monkeypatch, user): + run = _tool_run(user, ["fixture-a"]) + monkeypatch.setattr(runner, "_execute_locked", lambda r: r) + runner.execute(run) + run.refresh_from_db() + assert run.heartbeat_at is not None diff --git a/backend/analysis/tests/test_runs_api.py b/backend/analysis/tests/test_runs_api.py new file mode 100644 index 00000000..bdc239b8 --- /dev/null +++ b/backend/analysis/tests/test_runs_api.py @@ -0,0 +1,273 @@ +"""The runs API behind the analysis pages: list/stats, batches, preview, re-run, bulk actions, findings.""" + +from __future__ import annotations + +import pytest + +from backend.analysis.models import AnalysisRun +from backend.analysis.services import runner +from backend.analysis.tests.factories import AnalysisRunFactory +from backend.analysis.tests.factories import AnalyzerToolFactory +from backend.analysis.tests.factories import FindingFactory +from backend.analysis.tests.factories import ToolResultFactory +from backend.generation.tests.factories import GenerationJobFactory +from backend.runtime.tests.factories import ContainerInstanceFactory +from backend.users.tests.factories import UserFactory + +pytestmark = pytest.mark.django_db + + +def _post(client, path, data): + return client.post(f"/api/analysis{path}", data=data, content_type="application/json") + + +@pytest.fixture +def no_dispatch(monkeypatch): + calls: list = [] + monkeypatch.setattr(runner, "dispatch", lambda run_id: calls.append([run_id])) + monkeypatch.setattr(runner, "dispatch_many", lambda ids: calls.append([str(i) for i in ids])) + return calls + + +def _job(user, files=None): + return GenerationJobFactory( + created_by=user, + status="completed", + result_data={"files": files or {"app.py": "import os\n", "templates/index.html": "

hi

\n"}}, + ) + + +# --- list / stats ------------------------------------------------------------- + + +def test_list_shows_target_and_live_progress(api_client, user): + job = _job(user) + active = AnalysisRunFactory( + created_by=user, + generation_job=job, + source_code=None, + status="running", + tool_slugs=["a", "b", "c"], + summary={"phase": runner.PHASE_TOOLS}, + ) + ToolResultFactory(run=active, tool_slug="a", status="completed") + FindingFactory(result=ToolResultFactory(run=active, tool_slug="b", status="running")) + summary = {"total_findings": 3, "metrics_by_tool": {}} + done = AnalysisRunFactory(created_by=user, status="completed", summary=summary) + + items = {i["id"]: i for i in api_client.get("/api/analysis/runs/").json()["items"]} + + live = items[str(active.id)] + assert live["target"]["kind"] == "job" + assert live["target"]["job_id"] == str(job.id) + assert live["phase"] == runner.PHASE_TOOLS + assert live["progress"] == {"total": 3, "done": 1, "running": ["b"], "findings": 1} + finished = items[str(done.id)] + assert finished["progress"] is None + assert finished["target"]["kind"] == "code" + assert "metrics_by_tool" not in finished["summary"] + + +def test_list_filters_and_sorts(api_client, user): + few = AnalysisRunFactory(created_by=user, name="alpha", status="completed", summary={"total_findings": 2}) + many = AnalysisRunFactory(created_by=user, name="beta", status="failed", summary={"total_findings": 9}) + pending = AnalysisRunFactory(created_by=user, name="gamma", status="pending", tool_slugs=["semgrep"]) + AnalysisRunFactory(name="alpha") # someone else's + + def ids(query): + return [i["id"] for i in api_client.get(f"/api/analysis/runs/?{query}").json()["items"]] + + assert ids("search=alph") == [str(few.id)] + assert ids(f"search={str(many.id)[:8]}") == [str(many.id)] + assert ids("status=active") == [str(pending.id)] + assert set(ids("status=completed,failed")) == {str(few.id), str(many.id)} + assert ids("tool_slug=semgrep") == [str(pending.id)] + assert ids("sort=findings_desc")[:2] == [str(many.id), str(few.id)] + + +def test_stats_count_the_callers_runs(api_client, user): + AnalysisRunFactory(created_by=user, status="running") + AnalysisRunFactory(created_by=user, status="completed") + AnalysisRunFactory(status="running") + body = api_client.get("/api/analysis/runs/stats/").json() + assert body == {"total": 2, "active": 1, "by_status": {"running": 1, "completed": 1}} + + +# --- create / batch / preview -------------------------------------------------- + + +def test_create_rejects_unknown_tools(api_client, no_dispatch): + resp = _post(api_client, "/runs/", {"tool_slugs": ["nope"], "source_code": {"app.py": "x"}}) + assert resp.status_code == 400 + assert "nope" in resp.json()["detail"] + assert not no_dispatch + + +def test_create_names_the_run_after_its_app(api_client, user, no_dispatch): + AnalyzerToolFactory(slug="bandit") + job = _job(user) + body = _post(api_client, "/runs/", {"tool_slugs": ["bandit"], "generation_job_id": str(job.id)}).json() + assert body["name"] == body["target"]["label"] + assert body["source_files"] == [{"path": "app.py", "lines": 2}, {"path": "templates/index.html", "lines": 2}] + assert no_dispatch == [[body["id"]]] + + +def test_batch_queues_one_run_per_app(api_client, user, no_dispatch): + AnalyzerToolFactory(slug="bandit") + jobs = [_job(user), _job(user)] + resp = _post( + api_client, + "/runs/batch/", + {"tool_slugs": ["bandit", "bandit"], "generation_job_ids": [str(j.id) for j in jobs]}, + ) + assert resp.status_code == 200, resp.content + body = resp.json() + runs = list(AnalysisRun.objects.filter(batch_id=body["batch_id"])) + assert {str(r.generation_job_id) for r in runs} == {str(j.id) for j in jobs} + assert all(r.tool_slugs == ["bandit"] for r in runs) + # One queue for the whole batch, in the order given. + assert no_dispatch == [body["run_ids"]] + listed = api_client.get(f"/api/analysis/runs/?batch_id={body['batch_id']}").json() + assert listed["total"] == 2 + + +def test_batch_refuses_other_users_apps(api_client, no_dispatch): + AnalyzerToolFactory(slug="bandit") + foreign = _job(UserFactory()) + resp = _post(api_client, "/runs/batch/", {"tool_slugs": ["bandit"], "generation_job_ids": [str(foreign.id)]}) + assert resp.status_code == 404 + assert not AnalysisRun.objects.exists() + assert not no_dispatch + + +def test_preview_says_where_each_tool_runs(api_client, user): + AnalyzerToolFactory(slug="py", file_patterns=["*.py"]) + AnalyzerToolFactory(slug="css", file_patterns=["*.css"]) + AnalyzerToolFactory(slug="dast", analysis_mode="dynamic") + up = _job(user) + down = _job(user) + ContainerInstanceFactory(created_by=user, generation_job=up, status="running") + + resp = _post( + api_client, + "/runs/preview/", + {"tool_slugs": ["py", "css", "dast"], "generation_job_ids": [str(up.id), str(down.id)]}, + ) + assert resp.status_code == 200, resp.content + body = resp.json() + targets = {t["job_id"]: t for t in body["targets"]} + assert targets[str(up.id)]["app_running"] is True + assert targets[str(down.id)]["app_running"] is False + assert targets[str(up.id)]["files"] == 2 + tools = {t["slug"]: t for t in body["tools"]} + assert (tools["py"]["runs"], tools["py"]["not_applicable"]) == (2, 0) + assert (tools["css"]["runs"], tools["css"]["not_applicable"]) == (0, 2) + assert (tools["dast"]["runs"], tools["dast"]["app_not_running"]) == (1, 1) + assert body["tool_runs"] == 3 + # Nothing is installed in this user's workspace. + assert any("Not installed" in w for w in body["warnings"]) + assert not AnalysisRun.objects.exists() + + +# --- re-run / delete / bulk -------------------------------------------------------- + + +def test_rerun_only_the_tools_that_measured_nothing(api_client, user, no_dispatch): + for slug in ("ok", "broken", "na", "down"): + AnalyzerToolFactory(slug=slug) + source = AnalysisRunFactory(created_by=user, status="partial", tool_slugs=["ok", "broken", "na", "down"]) + ToolResultFactory(run=source, tool_slug="ok", status="completed") + ToolResultFactory(run=source, tool_slug="broken", status="failed") + ToolResultFactory(run=source, tool_slug="na", status="skipped", summary={"skip_reason": "not_applicable"}) + ToolResultFactory(run=source, tool_slug="down", status="skipped", summary={"skip_reason": "app_not_running"}) + + resp = _post(api_client, f"/runs/{source.id}/rerun/", {"only_unmeasured": True}) + assert resp.status_code == 200, resp.content + body = resp.json() + assert body["tool_slugs"] == ["broken", "down"] + assert body["id"] != str(source.id) + assert no_dispatch == [[body["id"]]] + + full = _post(api_client, f"/runs/{source.id}/rerun/", {}).json() + assert full["tool_slugs"] == ["ok", "broken", "na", "down"] + + +def test_rerun_with_everything_measured_is_refused(api_client, user, no_dispatch): + AnalyzerToolFactory(slug="ok") + source = AnalysisRunFactory(created_by=user, status="completed", tool_slugs=["ok"]) + ToolResultFactory(run=source, tool_slug="ok", status="completed") + assert _post(api_client, f"/runs/{source.id}/rerun/", {"only_unmeasured": True}).status_code == 400 + + +def test_active_runs_are_not_deleted(api_client, user): + running = AnalysisRunFactory(created_by=user, status="running") + done = AnalysisRunFactory(created_by=user, status="completed") + assert api_client.delete(f"/api/analysis/runs/{running.id}/").status_code == 409 + + body = _post(api_client, "/runs/bulk/", {"action": "delete", "ids": [str(running.id), str(done.id)]}).json() + assert body == {"affected": 1, "skipped": 1} + assert list(AnalysisRun.objects.values_list("id", flat=True)) == [running.id] + + +def test_bulk_cancel(api_client, user, monkeypatch): + monkeypatch.setattr(runner, "_kill_in_flight", lambda run: None) + pending = AnalysisRunFactory(created_by=user, status="pending") + done = AnalysisRunFactory(created_by=user, status="completed") + foreign = AnalysisRunFactory(status="pending") + ids = [str(pending.id), str(done.id), str(foreign.id)] + body = _post(api_client, "/runs/bulk/", {"action": "cancel", "ids": ids}).json() + assert body == {"affected": 1, "skipped": 1} + pending.refresh_from_db() + foreign.refresh_from_db() + assert pending.status == "cancelled" + assert foreign.status == "pending" + + +# --- results / findings ------------------------------------------------------------- + + +def test_result_detail_exposes_the_raw_output(api_client, user): + run = AnalysisRunFactory(created_by=user) + result = ToolResultFactory(run=run, raw_output={"command": "bandit -r /work", "exit_code": 1, "stdout": "{}"}) + body = api_client.get(f"/api/analysis/runs/{run.id}/results/{result.id}/").json() + assert body["raw_output"]["exit_code"] == 1 + assert body["findings_count"] == 0 + other = ToolResultFactory() + assert api_client.get(f"/api/analysis/runs/{other.run_id}/results/{other.id}/").status_code == 404 + + +def _findings_run(user): + run = AnalysisRunFactory(created_by=user, status="completed") + bandit = ToolResultFactory(run=run, tool_slug="bandit") + ruff = ToolResultFactory(run=run, tool_slug="ruff", category="lint") + FindingFactory(result=bandit, severity="high", tool_severity="HIGH", file_path="app.py", rule_id="B602") + FindingFactory(result=bandit, severity="low", tool_severity="LOW", file_path="app.py", rule_id="B101") + # Ruff rates nothing: its normalized "high" is a heuristic and must not lead. + FindingFactory(result=ruff, severity="high", tool_severity="", file_path="util.py", rule_id="F821", title="x") + return run + + +def test_findings_filter_by_the_reported_level(api_client, user): + run = _findings_run(user) + url = f"/api/analysis/runs/{run.id}/findings/" + assert [f["rule_id"] for f in api_client.get(url).json()["items"]] == ["B602", "B101", "F821"] + assert [f["rule_id"] for f in api_client.get(f"{url}?tool_severity=high").json()["items"]] == ["B602"] + unrated = api_client.get(f"{url}?tool_severity=unrated").json()["items"] + assert [(f["rule_id"], f["tool_severity"]) for f in unrated] == [("F821", "")] + assert [f["rule_id"] for f in api_client.get(f"{url}?search=util").json()["items"]] == ["F821"] + assert [f["rule_id"] for f in api_client.get(f"{url}?sort=file").json()["items"]][-1] == "F821" + + +def test_facets_count_reported_levels_and_unrated(api_client, user): + run = _findings_run(user) + body = api_client.get(f"/api/analysis/runs/{run.id}/findings/facets/").json() + assert body["total"] == 3 + assert [(level["value"], level["count"]) for level in body["levels"]] == [("high", 1), ("low", 1)] + assert body["unrated"] == 1 + assert {t["value"]: t["count"] for t in body["tools"]} == {"bandit": 2, "ruff": 1} + assert body["files"][0] == {"value": "app.py", "count": 2, "label": ""} + + scoped = api_client.get(f"/api/analysis/runs/{run.id}/findings/facets/?tool_slug=ruff").json() + assert scoped["total"] == 1 + assert scoped["levels"] == [] + assert len(scoped["tools"]) == 2 # the tool facet itself stays unscoped diff --git a/backend/analysis/tests/test_scanner_parsers.py b/backend/analysis/tests/test_scanner_parsers.py new file mode 100644 index 00000000..a070ac72 --- /dev/null +++ b/backend/analysis/tests/test_scanner_parsers.py @@ -0,0 +1,627 @@ +"""Parsers for the DAST scanners and Trivy / OSV-Scanner / Retire.js / djLint. + +Fixtures are trimmed from real runs of the pinned tool versions against a +generated Flask + React sample app. +""" + +from __future__ import annotations + +import json + +from backend.analysis.services import parsers + + +def _parse(key: str, raw: str): + return parsers.parse(key, raw).findings + + +def test_zap_collapses_systemic_alerts_and_strips_html(): + report = { + "@programName": "ZAP", + "site": [ + { + "@name": "http://172.17.0.2:8000", + "alerts": [ + { + "pluginid": "10038", + "name": "Content Security Policy (CSP) Header Not Set", + "riskcode": "2", + "confidence": "3", + "desc": "

CSP is an added layer & more.

", + "solution": "

Set the header.

", + "instances": [ + {"uri": "http://172.17.0.2:8000/", "method": "GET", "param": "", "evidence": ""}, + {"uri": "http://172.17.0.2:8000/login", "method": "GET", "param": "", "evidence": ""}, + ], + "count": "2", + "systemic": True, + "cweid": "693", + }, + { + "pluginid": "40018", + "name": "SQL Injection", + "riskcode": "3", + "confidence": "2", + "desc": "

SQLi

", + "solution": "", + "instances": [ + {"uri": "http://172.17.0.2:8000/api/items?q=1", "method": "GET", "param": "q"}, + {"uri": "http://172.17.0.2:8000/api/items?q=1", "method": "POST", "param": "q"}, + ], + "count": "2", + "systemic": False, + "cweid": "89", + }, + {"pluginid": "1", "name": "Marked FP", "riskcode": "3", "confidence": "0", "instances": []}, + ], + }, + ], + } + findings = _parse("zap", json.dumps(report)) + assert [(f.title, f.severity, f.file_path) for f in findings] == [ + ("Content Security Policy (CSP) Header Not Set", "medium", "/"), + ("SQL Injection", "high", "/api/items"), + ] + csp, sqli = findings + assert csp.description == "CSP is an added layer & more." + assert csp.tool_specific_data["urls"] == ["http://172.17.0.2:8000/", "http://172.17.0.2:8000/login"] + assert csp.tool_specific_data["cwe"] == "CWE-693" + assert sqli.tool_specific_data["param"] == "q" + assert sqli.confidence == "medium" + + +def test_zap_without_report_is_empty(): + assert _parse("zap", "") == [] + + +def test_nuclei_jsonl(): + hits = [ + { + "template-id": "http-missing-security-headers", + "info": { + "name": "HTTP Missing Security Headers", + "severity": "info", + "tags": ["misconfig", "headers"], + "classification": {"cwe-id": ["cwe-693"]}, + }, + "matcher-name": "content-security-policy", + "matched-at": "http://172.17.0.2:8000", + }, + { + "template-id": "git-config", + "info": {"name": "Git Config Disclosure", "severity": "medium", "reference": "https://example.org"}, + "matched-at": "http://172.17.0.2:8000/.git/config", + "extracted-results": ["[core]"], + }, + ] + raw = "noise\n" + "\n".join(json.dumps(h) for h in hits) + "\n" + findings = _parse("nuclei", raw) + assert [(f.title, f.severity, f.file_path) for f in findings] == [ + ("HTTP Missing Security Headers: content-security-policy", "info", "/"), + ("Git Config Disclosure", "medium", "/.git/config"), + ] + assert findings[0].tool_specific_data["cwe"] == "CWE-693" + assert findings[1].code_snippet == "[core]" + assert findings[1].tool_specific_data["references"] == ["https://example.org"] + + +def test_wapiti_vulnerabilities_and_anomalies(): + report = { + "classifications": { + "Cross Site Scripting": { + "desc": "XSS desc", + "sol": "Escape output", + "ref": {"CWE-79: Improper Neutralization": "https://cwe.mitre.org/data/definitions/79.html"}, + }, + "Internal Server Error": {"desc": "500", "sol": "Handle errors", "ref": {}}, + }, + "vulnerabilities": { + "Cross Site Scripting": [ + { + "method": "GET", + "path": "/search", + "info": "XSS via q", + "level": 3, + "parameter": "q", + "module": "xss", + }, + ], + "SQL Injection": [], + }, + "anomalies": { + "Internal Server Error": [ + {"method": "POST", "path": "/api/items", "info": "500 on fuzz", "level": 3, "module": "exec"}, + ], + }, + "additionals": {"HTTP Methods": [{"path": "/", "info": "OPTIONS", "level": 0}]}, + } + findings = _parse("wapiti", json.dumps(report)) + assert [(f.category, f.severity, f.file_path, f.rule_id) for f in findings] == [ + ("security", "high", "/search", "xss"), + ("reliability", "medium", "/api/items", "exec"), + ] + assert findings[0].tool_specific_data["cwe"] == "CWE-79" + assert findings[0].suggestion == "Escape output" + + +def test_trivy_vulns_misconfigs_and_secrets(): + report = { + "Results": [ + { + "Target": "backend/requirements.txt", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2019-10906", + "PkgName": "Jinja2", + "InstalledVersion": "2.10", + "FixedVersion": "2.10.1", + "Severity": "HIGH", + "Title": "sandbox escape", + "CweIDs": ["CWE-693"], + }, + ], + }, + { + "Target": "backend/Dockerfile", + "Misconfigurations": [ + { + "ID": "DS-0001", + "Title": "':latest' tag used", + "Message": "Specify a tag", + "Resolution": "Add a tag", + "Severity": "MEDIUM", + "Status": "FAIL", + "CauseMetadata": { + "StartLine": 1, + "Code": {"Lines": [{"Number": 1, "Content": "FROM python:latest", "IsCause": True}]}, + }, + }, + {"ID": "DS-0002", "Title": "passed", "Severity": "HIGH", "Status": "PASS"}, + ], + }, + { + "Target": "app.py", + "Secrets": [ + { + "RuleID": "aws-access-key-id", + "Title": "AWS Access Key", + "Severity": "CRITICAL", + "StartLine": 3, + "Match": "KEY=****", + }, + ], + }, + ], + } + findings = _parse("trivy", json.dumps(report)) + assert [(f.category, f.severity, f.file_path, f.line_number) for f in findings] == [ + ("dependency", "high", "backend/requirements.txt", None), + ("security", "medium", "backend/Dockerfile", 1), + ("secrets", "critical", "app.py", 3), + ] + assert findings[0].suggestion == "Upgrade Jinja2 to 2.10.1" + assert findings[1].code_snippet == "FROM python:latest" + + +def test_osv_scanner_groups_and_dedupes_sources(): + package = { + "package": {"name": "jinja2", "version": "2.10", "ecosystem": "PyPI"}, + "vulnerabilities": [ + { + "id": "PYSEC-2019-217", + "summary": "", + "affected": [ + { + "package": {"name": "jinja2"}, + "ranges": [{"events": [{"introduced": "0"}, {"fixed": "2.10.1"}]}], + }, + ], + }, + ], + "groups": [ + { + "ids": ["PYSEC-2019-217", "GHSA-462w-v97r-4m45"], + "aliases": ["CVE-2019-10906", "GHSA-462w-v97r-4m45", "PYSEC-2019-217"], + "max_severity": "8.6", + }, + ], + } + source = {"path": "/work/backend/requirements.txt", "type": "lockfile"} + report = {"results": [{"source": source, "packages": [package]}, {"source": source, "packages": [package]}]} + findings = _parse("osv_scanner", json.dumps(report)) + assert len(findings) == 1 + f = findings[0] + assert (f.severity, f.title, f.rule_id) == ("high", "jinja2 2.10: CVE-2019-10906", "PYSEC-2019-217") + assert f.suggestion == "Upgrade jinja2 to 2.10.1" + + +def test_retire_vendored_library(): + report = { + "version": "5.7.0", + "data": [ + { + "file": "/work/frontend/static/jquery.js", + "results": [ + { + "component": "jquery", + "version": "1.8.3", + "detection": "filecontent", + "vulnerabilities": [ + { + "info": ["https://nvd.nist.gov/vuln/detail/CVE-2012-6708"], + "below": "1.9.0b1", + "severity": "medium", + "identifiers": {"summary": "Selector interpreted as HTML", "CVE": ["CVE-2012-6708"]}, + "cwe": ["CWE-79"], + }, + ], + }, + ], + }, + ], + } + findings = _parse("retire", json.dumps(report)) + assert len(findings) == 1 + f = findings[0] + assert (f.severity, f.rule_id, f.title) == ( + "medium", + "CVE-2012-6708", + "jquery 1.8.3: Selector interpreted as HTML", + ) + assert f.tool_specific_data["cwe"] == "CWE-79" + assert f.suggestion == "Upgrade jquery to 1.9.0b1 or later" + + +def test_djlint_line_format_ignores_progress_output(): + raw = ( + "\rLinting 1/1 files ━━━━ 00:00\n\n" + "work/backend/templates/index.html:1:0:H005:Html tag should have lang attribute.\n" + "work/backend/templates/index.html:2:36:H013:Img tag should have an alt attribute.\n" + "work/backend/templates/index.html:1:0:H030:Consider adding a meta description.\n" + ) + findings = _parse("djlint", raw) + assert [(f.rule_id, f.severity, f.file_path, f.line_number, f.column_number) for f in findings] == [ + ("H005", "low", "backend/templates/index.html", 1, 0), + ("H013", "low", "backend/templates/index.html", 2, 36), + ("H030", "info", "backend/templates/index.html", 1, 0), + ] + + +def test_dast_issue_keys_and_fingerprints(): + from backend.analysis.services import parsers as registry + + zap = json.dumps( + { + "site": [ + { + "@name": "http://app", + "alerts": [ + { + "pluginid": "10038", + "name": "Content Security Policy (CSP) Header Not Set", + "riskcode": "2", + "confidence": "3", + "instances": [{"uri": "http://app/"}], + }, + { + "pluginid": "10109", + "name": "Modern Web Application", + "riskcode": "0", + "confidence": "2", + "instances": [{"uri": "http://app/"}], + }, + ], + }, + ], + }, + ) + zap_findings = registry.parse("zap", zap).findings + assert [f.tool_specific_data.get("issue_key") for f in zap_findings] == ["missing-header:content-security-policy"] + + nuclei = "\n".join( + json.dumps(h) + for h in ( + { + "template-id": "http-missing-security-headers", + "matcher-name": "x-frame-options", + "matched-at": "http://app/", + "info": {"name": "Missing headers", "severity": "info", "tags": ["misconfig"]}, + }, + { + "template-id": "tech-detect", + "matcher-name": "uvicorn", + "matched-at": "http://app/", + "info": {"name": "Wappalyzer", "severity": "info", "tags": ["tech"]}, + }, + ) + ) + keys = [f.tool_specific_data.get("issue_key") for f in registry.parse("nuclei", nuclei).findings] + assert keys == ["missing-header:x-frame-options"] + + +# --- Tool's own severity / confidence and enriched tool_specific_data -------- + + +def test_zap_risk_and_confidence_words_and_instance_details(): + report = { + "site": [ + { + "@name": "http://app:8000", + "alerts": [ + { + "pluginid": "10098", + "name": "Cross-Domain Misconfiguration", + "riskcode": "2", + "confidence": "2", + "riskdesc": "Medium (Medium)", + "desc": "

CORS

", + "solution": "

Restrict origins.

", + "otherinfo": "", + "reference": "

https://vulncat.fortify.com/en/detail

https://owasp.org/cors

", + "instances": [ + { + "uri": "http://app:8000/api", + "method": "GET", + "param": "", + "attack": "", + "evidence": "Access-Control-Allow-Origin: *", + "otherinfo": "The CORS misconfiguration permits cross-domain reads.", + }, + ], + "count": "1", + "cweid": "264", + }, + # No riskdesc: ZAP's words for the codes. + { + "pluginid": "40018", + "name": "SQL Injection", + "riskcode": "3", + "confidence": "4", + "instances": [], + }, + ], + }, + ], + } + cors, sqli = _parse("zap", json.dumps(report)) + assert (cors.severity, cors.tool_severity) == ("medium", "Medium") + assert cors.suggestion == "Restrict origins." + data = cors.tool_specific_data + assert data["tool_confidence"] == "Medium" + assert data["evidence"] == "Access-Control-Allow-Origin: *" + assert data["other_info"] == "The CORS misconfiguration permits cross-domain reads." + assert data["references"] == ["https://vulncat.fortify.com/en/detail", "https://owasp.org/cors"] + assert data["docs_url"] == "https://www.zaproxy.org/docs/alerts/10098/" + assert (data["url"], data["method"], data["instances"]) == ("http://app:8000/api", "GET", 1) + # Empty in the report: no key. + assert "param" not in data + assert "attack" not in data + assert (sqli.tool_severity, sqli.tool_specific_data["tool_confidence"]) == ("High", "Confirmed") + + +def test_nuclei_labels_template_link_and_cvss(): + hit = { + "template-id": "CVE-2021-41773", + "template-url": "https://cloud.projectdiscovery.io/public/CVE-2021-41773", + "info": { + "name": "Apache 2.4.49 - Path Traversal", + "severity": "high", + "reference": ["https://nvd.nist.gov/vuln/detail/CVE-2021-41773"], + "classification": { + "cve-id": ["cve-2021-41773"], + "cwe-id": ["cwe-22"], + "cvss-metrics": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", + "cvss-score": 7.5, + }, + "tags": ["cve", "lfi"], + }, + "matched-at": "http://app:8000/cgi-bin/.%2e/etc/passwd", + "extracted-results": ["root:x:0:0"], + "curl-command": "curl -X 'GET' 'http://app:8000/cgi-bin/.%2e/etc/passwd'", + } + [f] = _parse("nuclei", json.dumps(hit)) + assert f.tool_severity == "high" + data = f.tool_specific_data + assert data["docs_url"] == "https://cloud.projectdiscovery.io/public/CVE-2021-41773" + assert data["rule_name"] == "Apache 2.4.49 - Path Traversal" + assert (data["vuln_id"], data["cvss_score"], data["cwe"]) == ("CVE-2021-41773", 7.5, "CWE-22") + assert data["cvss_vector"].startswith("CVSS:3.1/") + assert data["evidence"] == "root:x:0:0" + assert data["curl"].startswith("curl ") + + +def test_wapiti_numeric_level_and_classification_references(): + report = { + "classifications": { + "Cross Site Scripting": { + "desc": "XSS", + "sol": "Escape output", + "ref": { + "OWASP: Cross Site Scripting": "https://owasp.org/xss", + "CWE-79: Improper Neutralization": "https://cwe.mitre.org/data/definitions/79.html", + }, + }, + }, + "vulnerabilities": { + "Cross Site Scripting": [ + { + "method": "GET", + "path": "/search", + "info": "XSS via q", + "level": 3, + "parameter": "q", + "module": "xss", + }, + ], + }, + } + [f] = _parse("wapiti", json.dumps(report)) + assert (f.severity, f.tool_severity) == ("high", "3") + assert f.tool_specific_data["rule_name"] == "Cross Site Scripting" + assert f.tool_specific_data["references"] == [ + "https://owasp.org/xss", + "https://cwe.mitre.org/data/definitions/79.html", + ] + + +def test_trivy_labels_cvss_and_fix_versions(): + report = { + "Results": [ + { + "Target": "requirements.txt", + "Vulnerabilities": [ + { + "VulnerabilityID": "CVE-2026-27205", + "VendorIDs": ["GHSA-68rp-wp8r-4726"], + "PkgName": "Flask", + "InstalledVersion": "3.0.0", + "FixedVersion": "2.3.4, 3.1.3", + "SeveritySource": "ghsa", + "PrimaryURL": "https://avd.aquasec.com/nvd/cve-2026-27205", + "Severity": "LOW", + "CVSS": { + "ghsa": {"V40Vector": "CVSS:4.0/AV:N", "V40Score": 2.3}, + "nvd": {"V3Vector": "CVSS:3.1/AV:N", "V3Score": 4.3}, + }, + "References": ["https://nvd.nist.gov/vuln/detail/CVE-2026-27205"], + "CweIDs": ["CWE-524"], + }, + {"VulnerabilityID": "CVE-1", "PkgName": "x", "InstalledVersion": "1", "Severity": "UNKNOWN"}, + ], + }, + { + "Target": "config.py", + "Secrets": [ + { + "RuleID": "aws-access-key-id", + "Category": "AWS", + "Severity": "CRITICAL", + "Title": "AWS Access Key ID", + "StartLine": 1, + "EndLine": 1, + "Match": 'AWS_ACCESS_KEY_ID = "********************"', + }, + ], + }, + ], + } + vuln, unknown, secret = _parse("trivy", json.dumps(report)) + assert (vuln.severity, vuln.tool_severity) == ("low", "LOW") + data = vuln.tool_specific_data + # Score and vector from the source Trivy took the severity from. + assert (data["cvss_score"], data["cvss_vector"]) == (2.3, "CVSS:4.0/AV:N") + assert data["fixed_versions"] == ["2.3.4", "3.1.3"] + assert (data["installed_version"], data["vuln_id"], data["aliases"]) == ( + "3.0.0", + "CVE-2026-27205", + ["GHSA-68rp-wp8r-4726"], + ) + assert data["docs_url"] == "https://avd.aquasec.com/nvd/cve-2026-27205" + assert (data["version"], data["fixed_version"]) == ("3.0.0", "2.3.4, 3.1.3") # legacy keys + assert (unknown.severity, unknown.tool_severity, unknown.suggestion) == ("info", "UNKNOWN", "") + assert secret.tool_severity == "CRITICAL" + assert secret.suggestion == "" # no invented remediation + assert secret.tool_specific_data == {"secret_type": "aws-access-key-id", "end_line": 1, "cwe": "CWE-798"} + + +def test_osv_reads_label_and_cwe_from_every_record_of_a_group(): + package = { + "package": {"name": "flask", "version": "3.0.0", "ecosystem": "PyPI"}, + "vulnerabilities": [ + { + "id": "PYSEC-2026-2151", + "details": "PYSEC details", + "references": [{"type": "FIX", "url": "https://github.com/pallets/flask/commit/089cb86"}], + "severity": [{"type": "CVSS_V3", "score": "CVSS:3.1/AV:N/AC:L/PR:N/UI:R/S:U/C:L/I:N/A:N"}], + "affected": [{"package": {"name": "flask"}, "ranges": [{"events": [{"fixed": "3.1.3"}]}]}], + }, + { + "id": "GHSA-68rp-wp8r-4726", + "summary": "Flask session does not add `Vary: Cookie` header", + "database_specific": {"cwe_ids": ["CWE-524"], "severity": "LOW"}, + "references": [{"type": "ADVISORY", "url": "https://nvd.nist.gov/vuln/detail/CVE-2026-27205"}], + "severity": [{"type": "CVSS_V4", "score": "CVSS:4.0/AV:N/AC:L/AT:P/PR:N/UI:P/VC:L"}], + }, + ], + "groups": [ + { + "ids": ["PYSEC-2026-2151", "GHSA-68rp-wp8r-4726"], + "aliases": ["CVE-2026-27205", "GHSA-68rp-wp8r-4726", "PYSEC-2026-2151"], + "max_severity": "4.3", + }, + ], + } + report = {"results": [{"source": {"path": "/work/requirements.txt"}, "packages": [package]}]} + [f] = _parse("osv_scanner", json.dumps(report)) + assert f.tool_severity == "LOW" # the GHSA record's label + assert f.title == "flask 3.0.0: CVE-2026-27205 — Flask session does not add `Vary: Cookie` header" + assert f.description == "PYSEC details" + data = f.tool_specific_data + assert data["cwe"] == "CWE-524" + assert data["cvss_score"] == 4.3 + assert "cvss_vector" not in data # two records, two vectors: ambiguous + assert data["references"] == [ + "https://github.com/pallets/flask/commit/089cb86", + "https://nvd.nist.gov/vuln/detail/CVE-2026-27205", + ] + assert (data["vuln_id"], data["fixed_versions"], data["manifest"]) == ( + "PYSEC-2026-2151", + ["3.1.3"], + "/work/requirements.txt", + ) + + +def test_osv_without_label_falls_back_to_its_cvss_column(): + package = { + "package": {"name": "jinja2", "version": "2.10", "ecosystem": "PyPI"}, + "vulnerabilities": [{"id": "PYSEC-1", "severity": [{"type": "CVSS_V3", "score": "CVSS:3.1/AV:N"}]}], + "groups": [{"ids": ["PYSEC-1"], "aliases": ["PYSEC-1"], "max_severity": "8.6"}], + } + unrated = { + "package": {"name": "x", "version": "1", "ecosystem": "PyPI"}, + "vulnerabilities": [{"id": "PYSEC-2"}], + "groups": [{"ids": ["PYSEC-2"], "aliases": ["PYSEC-2"], "max_severity": ""}], + } + report = {"results": [{"source": {"path": "requirements.txt"}, "packages": [package, unrated]}]} + scored, plain = _parse("osv_scanner", json.dumps(report)) + assert (scored.tool_severity, scored.tool_specific_data["cvss_vector"]) == ("8.6", "CVSS:3.1/AV:N") + assert (plain.tool_severity, plain.suggestion) == ("", "") + assert "cvss_score" not in plain.tool_specific_data + + +def test_retire_and_djlint_labels(): + report = { + "data": [ + { + "file": "/work/static/jquery.js", + "results": [ + { + "component": "jquery", + "version": "1.8.3", + "vulnerabilities": [ + { + "info": ["https://nvd.nist.gov/vuln/detail/CVE-2012-6708"], + "below": "1.9.0b1", + "severity": "medium", + "identifiers": {"CVE": ["CVE-2012-6708"], "githubID": "GHSA-2pqj-h3vj-pqgw"}, + }, + ], + }, + ], + }, + ], + } + [f] = _parse("retire", json.dumps(report)) + assert f.tool_severity == "medium" + data = f.tool_specific_data + assert (data["installed_version"], data["fixed_versions"], data["vuln_id"]) == ( + "1.8.3", + ["1.9.0b1"], + "CVE-2012-6708", + ) + assert data["aliases"] == ["GHSA-2pqj-h3vj-pqgw"] + assert data["references"] == ["https://nvd.nist.gov/vuln/detail/CVE-2012-6708"] + + [dj] = _parse("djlint", "work/static/index.html:2:0:H031:Consider adding meta keywords.\n") + assert (dj.tool_severity, dj.suggestion) == ("", "") + assert dj.tool_specific_data == {"docs_url": "https://www.djlint.com/docs/linter/#rules"} diff --git a/backend/analysis/tests/test_seed.py b/backend/analysis/tests/test_seed.py index 1fb6e15b..0cabc4c2 100644 --- a/backend/analysis/tests/test_seed.py +++ b/backend/analysis/tests/test_seed.py @@ -45,3 +45,50 @@ def test_output_kind_seeded_from_yaml(): def test_performance_category_has_an_enabled_tool(): call_command("seed_analysis_tools") assert AnalyzerTool.objects.filter(category="performance", is_enabled=True).exists() + + +def test_tool_technologies_come_from_the_catalog_vocabulary(): + from backend.analysis.seeding import load_catalog_meta + + call_command("seed_analysis_tools") + meta = load_catalog_meta() + known = {t["slug"] for t in meta["technologies"]} + groups = {g["slug"] for g in meta["technology_groups"]} + assert all(t["group"] in groups for t in meta["technologies"]) + for tool in AnalyzerTool.objects.all(): + assert tool.technologies, f"{tool.slug} lists no technologies" + assert set(tool.technologies) <= known, tool.slug + + +def test_dynamic_tools_seeded_with_url_placeholder(): + call_command("seed_analysis_tools") + dynamic = AnalyzerTool.objects.filter(analysis_mode="dynamic") + assert {"zap", "nuclei", "wapiti"} <= set(dynamic.values_list("slug", flat=True)) + for tool in dynamic: + assert "{url}" in tool.run_cmd, tool.slug + assert "{target}" not in tool.run_cmd, tool.slug + assert AnalyzerTool.objects.get(slug="trivy").analysis_mode == "static" + + +def test_config_placeholders_have_defaults(): + import re + + call_command("seed_analysis_tools") + for tool in AnalyzerTool.objects.filter(kind="container"): + names = set(re.findall(r"\{config:([A-Za-z0-9_]+)\}", tool.run_cmd)) + schema_names = {f["name"] for f in tool.config_schema or []} + assert names <= set(tool.default_config or {}), tool.slug + assert names <= schema_names, tool.slug + + +def test_release_binary_downloads_are_checksummed(): + """Every tool that downloads a release binary verifies its published sha256.""" + import yaml + + from backend.analysis.seeding import DATA_DIR + + for path in sorted(DATA_DIR.glob("*.yaml")): + data = yaml.safe_load(path.read_text()) + install = (data.get("install_cmd") if isinstance(data, dict) else "") or "" + if "/releases/download/" in install: + assert "sha256sum -c" in install, path.name diff --git a/backend/automation/tasks.py b/backend/automation/tasks.py index 12bd4f14..f77de5dd 100644 --- a/backend/automation/tasks.py +++ b/backend/automation/tasks.py @@ -1,10 +1,11 @@ -"""Celery tasks for the automation execution engine. - -Workers are configured in ``docker-compose.local.yml``. -Verify a worker is running: ``docker ps | grep celery``. - -If the Celery broker is unreachable, ``trigger_run`` in ``services.py`` -falls back to a daemon thread automatically. +"""Celery tasks of the retired automation pipelines (legacy, kept for existing data). + +The automation API router is unmounted and nothing enqueues or schedules these +tasks any more: experiment schedules and post-run steps now run inside the +executor (see docs/BACKGROUND_SERVICES.md). They remain registered with the +legacy models. ``trigger_run`` in +``services.py`` (no longer reachable from the API) still falls back to a +daemon thread when the broker is unreachable. """ from __future__ import annotations diff --git a/backend/credentials/services/resolver.py b/backend/credentials/services/resolver.py index 24dd5436..c79ac604 100644 --- a/backend/credentials/services/resolver.py +++ b/backend/credentials/services/resolver.py @@ -20,7 +20,7 @@ from backend.users.models import User -SETTINGS_LINK_REMEDIATION = "Add or update your OpenRouter API key in Settings → API Access." +SETTINGS_LINK_REMEDIATION = "Add or update your OpenRouter API key in Settings → API keys." class MissingApiKeyError(Exception): diff --git a/backend/docs/services.py b/backend/docs/services.py index 97891269..cf315ee3 100644 --- a/backend/docs/services.py +++ b/backend/docs/services.py @@ -20,11 +20,13 @@ "QUICKSTART": "Getting Started", "README": "Getting Started", "development-guide": "Getting Started", + "USER_GUIDE": "Getting Started", "ARCHITECTURE": "Architecture", "ANALYSIS_PIPELINE": "Architecture", "BACKGROUND_SERVICES": "Architecture", "GENERATION_PROCESS": "Architecture", "EXPERIMENTS": "Architecture", + "EXECUTION": "Architecture", "STUDY_TABLES": "Reference", "architecture": "Architecture", "api-reference": "Reference", @@ -33,6 +35,8 @@ "MODELS_REFERENCE": "Reference", "TEMPLATE_PACKAGES": "Reference", "AUTOMATION_GUIDE": "Reference", + "LOCAL_MODELS": "Reference", + "INTEGRATIONS": "Reference", "deployment-guide": "Operations", "TROUBLESHOOTING": "Operations", } diff --git a/backend/export/services/review_sample.py b/backend/export/services/review_sample.py index e3b0de17..9af14c90 100644 --- a/backend/export/services/review_sample.py +++ b/backend/export/services/review_sample.py @@ -336,6 +336,8 @@ def build_review_sample( "line": f.line_number, "end_line": (f.tool_specific_data or {}).get("end_line"), "cwe": (f.tool_specific_data or {}).get("cwe") or "", + "package": (f.tool_specific_data or {}).get("package") or "", + "issue_key": (f.tool_specific_data or {}).get("issue_key") or "", "title": f.title, } for f in findings diff --git a/backend/generation/api/views/templates.py b/backend/generation/api/views/templates.py index 31fa06c9..11dfd147 100644 --- a/backend/generation/api/views/templates.py +++ b/backend/generation/api/views/templates.py @@ -13,6 +13,7 @@ from backend.generation.api.schema import StackWriteSchema from backend.generation.api.views._router import router from backend.generation.models import AppRequirementTemplate +from backend.generation.services.data_lint import app_spec_errors from backend.generation.services.packages import visible_app_templates_for from backend.generation.services.versioning import content_hash from backend.runtime.services.dockerfile_gen import generate_dockerfile @@ -274,6 +275,8 @@ def create_app_template(request, payload: AppRequirementCreateSchema): # Slugs are unique across every owner; don't say whose it is. if AppRequirementTemplate.objects.filter(slug=payload.slug).exists(): return 400, {"detail": f"App spec slug {payload.slug!r} is already in use."} + if errors := app_spec_errors(payload.dict(), label="App spec"): + return 400, {"detail": "; ".join(errors), "errors": errors} return 200, AppRequirementTemplate.objects.create( **payload.dict(), created_by=request.auth, @@ -285,13 +288,17 @@ def get_app_template(request, slug: str): return get_object_or_404(visible_app_templates_for(request.auth), slug=slug) -@router.put("/app-specs/{slug}/", response=AppRequirementTemplateSchema) +@router.put("/app-specs/{slug}/", response={200: AppRequirementTemplateSchema, 400: dict}) def update_app_template(request, slug: str, payload: AppRequirementCreateSchema): template = _mutable_template_or_403(request.auth, slug=slug) + # The slug never changes (and may predate the snake_case rule). + body = {**payload.dict(), "slug": template.slug} + if errors := app_spec_errors(body, label="App spec", check_slug=False): + return 400, {"detail": "; ".join(errors), "errors": errors} for attr, value in payload.dict(exclude={"slug"}).items(): setattr(template, attr, value) template.save() - return template + return 200, template @router.delete("/app-specs/{slug}/") diff --git a/backend/generation/migrations/0021_job_help_texts.py b/backend/generation/migrations/0021_job_help_texts.py new file mode 100644 index 00000000..70506634 --- /dev/null +++ b/backend/generation/migrations/0021_job_help_texts.py @@ -0,0 +1,36 @@ +# Generated by Django 6.0.6 on 2026-10-26 13:57 + +from django.db import migrations +from django.db import models + + +class Migration(migrations.Migration): + dependencies = [ + ("generation", "0020_analysis_plan"), + ] + + operations = [ + migrations.AlterField( + model_name="generationjob", + name="bundle_key", + field=models.CharField( + blank=True, + db_index=True, + default="", + help_text='Denormalized slicing key: "profile-slug@version" for template modes; for agent mode the method key "[profile@v/]agent:@" plus "+opts:/+llm:/+budget:" suffixes for non-defaults', + max_length=220, + verbose_name="bundle key", + ), + ), + migrations.AlterField( + model_name="generationjob", + name="stack_slug", + field=models.CharField( + blank=True, + default="", + help_text="Canonical or alias slug of a runtime.Stack row (blank = the default stack)", + max_length=100, + verbose_name="stack slug", + ), + ), + ] diff --git a/backend/generation/models.py b/backend/generation/models.py index 3d402732..f21f9777 100644 --- a/backend/generation/models.py +++ b/backend/generation/models.py @@ -789,7 +789,7 @@ class Lane(models.TextChoices): max_length=100, blank=True, default="", - help_text="Canonical or alias slug from runtime/scaffolding/manifest.json", + help_text="Canonical or alias slug of a runtime.Stack row (blank = the default stack)", ) app_requirement = models.ForeignKey( AppRequirementTemplate, @@ -836,7 +836,10 @@ class Lane(models.TextChoices): blank=True, default="", db_index=True, - help_text='Denormalized "profile-slug@version" slicing key', + help_text=( + 'Denormalized slicing key: "profile-slug@version" for template modes; for agent mode the method key ' + '"[profile@v/]agent:@" plus "+opts:/+llm:/+budget:" suffixes for non-defaults' + ), ) # Experiment membership (a job is one run/cell of a designed experiment) diff --git a/backend/generation/services/agent/runner.py b/backend/generation/services/agent/runner.py index 5105447d..93b170ea 100644 --- a/backend/generation/services/agent/runner.py +++ b/backend/generation/services/agent/runner.py @@ -429,7 +429,7 @@ def _first_llm_error(calls: list[dict], result) -> str: def _remediation(calls: list[dict]) -> str: statuses = {c.get("status") for c in calls} if 401 in statuses or 403 in statuses: - return "Check the OpenRouter API key in Settings → API Access." + return "Check the OpenRouter API key in Settings → API keys." if 404 in statuses: return "No provider serves this model under the run's routing policy; relax llm_params.provider." if 402 in statuses: diff --git a/backend/generation/services/data_lint.py b/backend/generation/services/data_lint.py index 82dbc187..873a81bb 100644 --- a/backend/generation/services/data_lint.py +++ b/backend/generation/services/data_lint.py @@ -67,6 +67,23 @@ def _spec_errors(validator: jsonschema.Draft202012Validator, data: Any, label: s ] +def app_spec_errors(spec: dict[str, Any], *, label: str = "app spec", check_slug: bool = True) -> list[str]: + """Errors of one app spec against ``schema.json`` (used by the API and package import). + + Empty optional values (``difficulty: ""``) count as absent. With + ``check_slug=False`` the slug's snake_case pattern is not enforced: specs + created before the schema, and slugs the importer renames (``slug-2``), + keep working. + """ + data = {k: v for k, v in spec.items() if not (k == "difficulty" and not v)} + errors = [] + for err in sorted(_spec_validator().iter_errors(data), key=lambda e: list(e.path)): + if not check_slug and list(err.path) == ["slug"] and err.validator == "pattern": + continue + errors.append(f"{label}: {'.'.join(str(p) for p in err.path) or ''}: {err.message}") + return errors + + def lint_requirement_specs() -> list[str]: """Validate every ``data/requirements/*.json`` against ``schema.json``.""" errors: list[str] = [] diff --git a/backend/generation/services/experiments.py b/backend/generation/services/experiments.py index fe23df9d..367592f9 100644 --- a/backend/generation/services/experiments.py +++ b/backend/generation/services/experiments.py @@ -961,16 +961,21 @@ def executor_state() -> dict[str, bool]: def _code_stale() -> bool: - """Whether the executor runs older code than this process (Track A exposes the comparison).""" - try: - from backend.execution import capacity - except ImportError: - return False - check = getattr(capacity, "code_stale", None) - if check is None: - return False + """Whether the live leader executor runs older code than this process.""" + from backend.execution import capacity + from backend.execution.models import ExecutorInstance + try: - return bool(check()) + leader = ( + ExecutorInstance.objects.filter( + is_leader=True, + stopped_at__isnull=True, + last_seen_at__gt=timezone.now() - timedelta(seconds=_EXECUTOR_STALE_SECONDS), + ) + .order_by("-last_seen_at") + .first() + ) + return bool(leader is not None and leader.code_version and leader.code_version != capacity.code_version()) except Exception: # noqa: BLE001 — a rollup must not fail on an advisory flag logger.debug("code_stale check failed", exc_info=True) return False @@ -1499,7 +1504,14 @@ def _new_job( if is_agent: apply_agent_snapshot_to_job(job, save=False) else: - apply_snapshot_to_job(job, save=False) + # Template mode only: the agent harness has its own budget and + # retry loop, so continuation/repair do not apply to agent cells. + apply_snapshot_to_job( + job, + save=False, + continuation_limit=experiment.continuation_limit, + enable_repair=experiment.enable_repair, + ) except Exception as exc: # noqa: BLE001 — isolate one bad cell from the launch logger.warning("Snapshot build failed for experiment job %s: %s", job.id, exc) job.status = GenerationJob.Status.FAILED @@ -1819,10 +1831,21 @@ def normalize_post_run(spec: dict | None, mode: str) -> dict[str, Any]: report = spec.get("report") or {} slugs = list(dict.fromkeys(str(s) for s in analysis.get("tool_slugs") or [])) if slugs: - known = set(AnalyzerTool.objects.filter(slug__in=slugs).values_list("slug", flat=True)) - unknown = [s for s in slugs if s not in known] + from backend.analysis.models import AnalysisMode + + modes = dict(AnalyzerTool.objects.filter(slug__in=slugs).values_list("slug", "analysis_mode")) + unknown = [s for s in slugs if s not in modes] if unknown: raise ExperimentError(400, f"Unknown analyzer {', '.join(unknown)}", "unknown_analyzer") + # Post-run analysis runs on the generated source; dynamic analyzers + # need a deployed, running app, which post-run never starts. + dynamic = [s for s in slugs if modes[s] == AnalysisMode.DYNAMIC] + if dynamic: + raise ExperimentError( + 400, + f"Dynamic analyzer(s) {', '.join(dynamic)} need a running app; post-run analysis is static only.", + "dynamic_analyzer", + ) enabled = bool(analysis.get("enabled", False)) if enabled and not slugs: raise ExperimentError(400, "Post-run analysis needs at least one analyzer (tool_slugs).", "no_analyzers") @@ -2516,7 +2539,11 @@ def _refreeze_drifted(experiment: Experiment, user: AbstractUser, conditions: li if job.mode == GenerationJob.Mode.AGENT: apply_agent_snapshot_to_job(job) else: - apply_snapshot_to_job(job) + apply_snapshot_to_job( + job, + continuation_limit=experiment.continuation_limit, + enable_repair=experiment.enable_repair, + ) _append_history( experiment, { diff --git a/backend/generation/services/job_cloning.py b/backend/generation/services/job_cloning.py index 70552439..d452769e 100644 --- a/backend/generation/services/job_cloning.py +++ b/backend/generation/services/job_cloning.py @@ -18,6 +18,7 @@ from backend.generation.services.profile_resolver import build_custom_snapshot from backend.generation.services.profile_resolver import bundle_key_from_snapshot from backend.generation.services.profile_resolver import experiment_cell_seed +from backend.generation.services.profile_resolver import generation_options from backend.generation.services.profile_resolver import run_fingerprint if TYPE_CHECKING: @@ -248,7 +249,9 @@ def clone_job( if original.mode == GenerationJob.Mode.AGENT: apply_agent_snapshot_to_job(new_job) elif original.mode == GenerationJob.Mode.SCAFFOLDING and original.app_requirement: - apply_snapshot_to_job(new_job) + # Keep the original's continuation/repair options (an experiment's). + limit, repair = generation_options(original.resolved_bundle) + apply_snapshot_to_job(new_job, continuation_limit=limit, enable_repair=repair) elif original.mode == GenerationJob.Mode.CUSTOM: # Execution reads the frozen llm section; without it a retry # would fall back to the legacy columns and drop llm_params. diff --git a/backend/generation/services/openrouter_client.py b/backend/generation/services/openrouter_client.py index 943d112c..1e4c13eb 100644 --- a/backend/generation/services/openrouter_client.py +++ b/backend/generation/services/openrouter_client.py @@ -59,7 +59,7 @@ def _build_auth_error(status_code: int, body: str) -> OpenRouterError: f"API error {status_code}: {body[:500]}", status_code=status_code, user_facing_message=("OpenRouter rejected the API key used for this request."), - remediation=("Update or rotate your OpenRouter API key in Settings → API Access."), + remediation=("Update or rotate your OpenRouter API key in Settings → API keys."), ) diff --git a/backend/generation/services/orchestrator.py b/backend/generation/services/orchestrator.py index b22656d1..db6531c8 100644 --- a/backend/generation/services/orchestrator.py +++ b/backend/generation/services/orchestrator.py @@ -34,17 +34,20 @@ from backend.generation.services.openrouter_client import OpenRouterError from backend.generation.services.output_guard import call_with_continuations from backend.generation.services.output_guard import clamped_max_tokens +from backend.generation.services.profile_resolver import DEFAULT_CONTINUATION_LIMIT +from backend.generation.services.profile_resolver import DEFAULT_ENABLE_REPAIR +from backend.generation.services.profile_resolver import generation_options from backend.generation.services.prompt_renderer import PromptRenderer from backend.runtime.services.scaffolding import get_stack_config from backend.runtime.services.scaffolding import resolve_stack_slug logger = logging.getLogger(__name__) -# Continuation/repair are per-experiment config in the planned Experiment -# entity (not yet implemented); these are the defaults every job gets until -# that config exists. -DEFAULT_CONTINUATION_LIMIT = 1 -DEFAULT_ENABLE_REPAIR = True +# Continuation/repair (template modes): an experiment's continuation_limit +# and enable_repair are frozen into each job's snapshot (``generation``, +# only when non-default) and read back in ``_run_scaffolding``; every other +# job runs with the service defaults (DEFAULT_CONTINUATION_LIMIT / +# DEFAULT_ENABLE_REPAIR, overridable on the instance). def _is_cancelled(job: GenerationJob) -> bool: @@ -337,6 +340,11 @@ def _call(messages: list[dict], stage_name: str) -> dict: return self._call_llm(job, model_id, messages, stage=stage_name) bundle = job.resolved_bundle if isinstance(job.resolved_bundle, dict) else {} + continuation_limit, enable_repair = generation_options( + bundle, + continuation_limit=self.continuation_limit, + enable_repair=self.enable_repair, + ) backend_messages = self.renderer.render_backend_messages( app_requirement=app_req, resolved_bundle=bundle if bundle else None, @@ -346,7 +354,7 @@ def _call(messages: list[dict], stage_name: str) -> dict: _call, backend_messages, stage="backend", - limit=self.continuation_limit, + limit=continuation_limit, ) backend_elapsed = time.time() - start for k in total_usage: @@ -371,7 +379,7 @@ def _call(messages: list[dict], stage_name: str) -> dict: _call, frontend_messages, stage="frontend", - limit=self.continuation_limit, + limit=continuation_limit, ) frontend_elapsed = time.time() - start2 for k in total_usage: @@ -383,7 +391,7 @@ def _call(messages: list[dict], stage_name: str) -> dict: py_validation = validate_python_files(files) fe_validation = validate_frontend_files(files) repair_used = False - if self.enable_repair and not (py_validation.passed and fe_validation.passed): + if enable_repair and not (py_validation.passed and fe_validation.passed): files, repair_used = self._run_repair_round(job, model_id, files, py_validation, fe_validation) py_validation = validate_python_files(files) fe_validation = validate_frontend_files(files) diff --git a/backend/generation/services/packages/importers.py b/backend/generation/services/packages/importers.py index 90a96e0c..d092ac71 100644 --- a/backend/generation/services/packages/importers.py +++ b/backend/generation/services/packages/importers.py @@ -432,6 +432,12 @@ def _import_bundle( payload = _validate_bundle_payload(raw_bundle) # A stack renamed on import keeps its profiles pointing at it. payload["scaffolding_slug"] = (stack_slug_map or {}).get(payload["scaffolding_slug"], payload["scaffolding_slug"]) + # An unknown stack would silently fall back to the default one at run time. + from backend.runtime.services.scaffolding import is_known_stack_slug + + if not is_known_stack_slug(payload["scaffolding_slug"]): + msg = f"Bundle {payload['slug']} references unknown stack {payload['scaffolding_slug']!r}" + raise ValueError(msg) normalized_refs = [] for raw_ref in payload["block_refs"]: ref = _validate_block_ref(raw_ref) @@ -689,6 +695,14 @@ def _validate_app_template_payload(raw_template: Any) -> dict[str, Any]: msg = f"App template {slug}: difficulty must be one of basic, standard, advanced" raise ValueError(msg) payload["difficulty"] = difficulty + # The same contract the seeder's specs and POST /app-specs/ meet (the + # slug pattern aside: older specs and import renames use hyphens). + from backend.generation.services.data_lint import app_spec_errors + + errors = app_spec_errors(payload, label=f"App template {slug}", check_slug=False) + if errors: + msg = "; ".join(errors[:5]) + (f" (+{len(errors) - 5} more)" if len(errors) > 5 else "") + raise ValueError(msg) return payload @@ -708,8 +722,12 @@ def _validate_block_payload(raw_block: Any) -> dict[str, Any]: if not content: msg = f"Block {slug} is missing content" raise ValueError(msg) + block_type = str(raw_block.get("block_type", "")).strip() or ContentBlock.BlockType.PROMPT_TONE + if block_type not in ContentBlock.BlockType.values: + msg = f"Block {slug} has unknown block_type {block_type!r} (one of {', '.join(ContentBlock.BlockType.values)})" + raise ValueError(msg) return { - "block_type": str(raw_block.get("block_type", "")).strip() or ContentBlock.BlockType.PROMPT_TONE, + "block_type": block_type, "slug": slug, "version": int(raw_block.get("version", 1)), "name": name, diff --git a/backend/generation/services/profile_resolver.py b/backend/generation/services/profile_resolver.py index 5168269c..6bf71b17 100644 --- a/backend/generation/services/profile_resolver.py +++ b/backend/generation/services/profile_resolver.py @@ -535,9 +535,57 @@ def snapshot_for_scaffolding_job(job: GenerationJob) -> dict[str, Any]: return snapshot -def apply_snapshot_to_job(job: GenerationJob, *, save: bool = True) -> GenerationJob: +# Template-mode generation options (an experiment's ``continuation_limit`` / +# ``enable_repair``). The defaults are what every job ran with before they +# were configurable, and they are never written to the snapshot, so a +# default job's snapshot and fingerprint stay byte-identical. +DEFAULT_CONTINUATION_LIMIT = 1 +DEFAULT_ENABLE_REPAIR = True + + +def attach_generation_options( + snapshot: dict[str, Any], + *, + continuation_limit: int = DEFAULT_CONTINUATION_LIMIT, + enable_repair: bool = DEFAULT_ENABLE_REPAIR, +) -> dict[str, Any]: + """Record non-default continuation/repair options in ``snapshot["generation"]`` (and re-fingerprint).""" + options: dict[str, Any] = {} + if int(continuation_limit) != DEFAULT_CONTINUATION_LIMIT: + options["continuation_limit"] = int(continuation_limit) + if bool(enable_repair) != DEFAULT_ENABLE_REPAIR: + options["enable_repair"] = bool(enable_repair) + if options: + snapshot["generation"] = options + snapshot["run_fingerprint"] = run_fingerprint(snapshot) + return snapshot + + +def generation_options( + snapshot: dict[str, Any] | None, + *, + continuation_limit: int = DEFAULT_CONTINUATION_LIMIT, + enable_repair: bool = DEFAULT_ENABLE_REPAIR, +) -> tuple[int, bool]: + """``(continuation_limit, enable_repair)`` a template-mode job runs with (the given defaults when unset).""" + options = snapshot.get("generation") if isinstance(snapshot, dict) else None + options = options if isinstance(options, dict) else {} + return ( + int(options.get("continuation_limit", continuation_limit)), + bool(options.get("enable_repair", enable_repair)), + ) + + +def apply_snapshot_to_job( + job: GenerationJob, + *, + save: bool = True, + continuation_limit: int = DEFAULT_CONTINUATION_LIMIT, + enable_repair: bool = DEFAULT_ENABLE_REPAIR, +) -> GenerationJob: """Build snapshot; set seed, ``resolved_bundle``, and slicing keys on job.""" snapshot = snapshot_for_scaffolding_job(job) + attach_generation_options(snapshot, continuation_limit=continuation_limit, enable_repair=enable_repair) job.experiment_seed = snapshot.get("seed") job.resolved_bundle = snapshot job.prompt_hash = snapshot.get("prompt_hash") or "" @@ -597,6 +645,9 @@ def run_fingerprint(snapshot: dict[str, Any]) -> str: }, "scaffolding_slug": snapshot.get("scaffolding_slug"), } + # Non-default continuation/repair options only (absent for default runs). + if snapshot.get("generation"): + payload["generation"] = snapshot["generation"] # Skeleton content identity: two runs on different stack versions are # not exact replays of each other. stack = snapshot.get("stack") or {} diff --git a/backend/generation/tests/test_experiment_service.py b/backend/generation/tests/test_experiment_service.py index 06f41cdc..89a9d834 100644 --- a/backend/generation/tests/test_experiment_service.py +++ b/backend/generation/tests/test_experiment_service.py @@ -106,6 +106,29 @@ def test_launch_creates_one_job_per_cell(self, experiment, condition, user): assert GenerationJob.objects.filter(experiment=experiment).count() == 4 assert _enqueued(dispatch) == 4 + def test_default_generation_options_leave_snapshot_unchanged(self, experiment, condition, user): + with mock.patch("backend.generation.services.experiments.enqueue_jobs"): + experiments_svc.launch_experiment(experiment, user) + for job in GenerationJob.objects.filter(experiment=experiment): + assert "generation" not in job.resolved_bundle + + def test_continuation_limit_and_repair_are_frozen_into_jobs(self, experiment, condition, user): + from backend.generation.services.profile_resolver import generation_options + from backend.generation.services.profile_resolver import run_fingerprint + + experiment.continuation_limit = 3 + experiment.enable_repair = False + experiment.save(update_fields=["continuation_limit", "enable_repair"]) + with mock.patch("backend.generation.services.experiments.enqueue_jobs"): + experiments_svc.launch_experiment(experiment, user) + job = GenerationJob.objects.filter(experiment=experiment).first() + assert job.resolved_bundle["generation"] == {"continuation_limit": 3, "enable_repair": False} + assert generation_options(job.resolved_bundle) == (3, False) + # The options are part of the run identity. + assert job.resolved_bundle["run_fingerprint"] == run_fingerprint(job.resolved_bundle) + bare = {k: v for k, v in job.resolved_bundle.items() if k != "generation"} + assert run_fingerprint(bare) != job.resolved_bundle["run_fingerprint"] + def test_launch_sets_stack_slug_from_bundle(self, experiment, condition, user): """resolve_stack_slug() reads job.stack_slug — must not be blank.""" with mock.patch("backend.generation.services.experiments.enqueue_jobs"): diff --git a/backend/generation/tests/test_experiments_v2_api.py b/backend/generation/tests/test_experiments_v2_api.py index fe9ae6fb..1e4ac2f0 100644 --- a/backend/generation/tests/test_experiments_v2_api.py +++ b/backend/generation/tests/test_experiments_v2_api.py @@ -1077,3 +1077,22 @@ def test_rejects_unknown_mode(self, api, user): body = _create_body(_models(1), _apps(), build_healing="sometimes") res = api.post(f"{API}/experiments/", data=json.dumps(body), content_type="application/json") assert res.status_code == 422 + + +def test_executor_state_flags_a_leader_on_older_code(): + from backend.execution import capacity + from backend.execution.models import ExecutorInstance + + assert svc.executor_state() == {"online": False, "draining": False, "code_stale": False} + leader = ExecutorInstance.objects.create( + id="exec:1", + hostname="h", + pid=1, + code_version="old", + is_leader=True, + last_seen_at=timezone.now(), + ) + assert svc.executor_state() == {"online": True, "draining": False, "code_stale": True} + leader.code_version = capacity.code_version() + leader.save() + assert svc.executor_state()["code_stale"] is False diff --git a/backend/generation/tests/test_multifile_and_robustness.py b/backend/generation/tests/test_multifile_and_robustness.py index 804b568e..2acab326 100644 --- a/backend/generation/tests/test_multifile_and_robustness.py +++ b/backend/generation/tests/test_multifile_and_robustness.py @@ -468,4 +468,25 @@ def test_repair_round_keeps_original_when_fix_still_invalid(self, _no_docker, mo assert "broken(:" in job.result_data["files"]["app.py"] assert job.result_data["validation"]["passed"] is False + + def test_snapshot_generation_options_override_service_defaults(self, _no_docker, monkeypatch): + """An experiment's continuation_limit/enable_repair, frozen in the snapshot, take effect.""" + job = self._make_job() + job.resolved_bundle = {"generation": {"continuation_limit": 0, "enable_repair": False}} + service = self._service( + monkeypatch, + [ + _api_response("```python:app.py\ndef broken(:\n", finish_reason="length"), + _api_response("```jsx:App.jsx\nexport default function App() {}\n```"), + ], + ) + service.enable_repair = True + service.continuation_limit = 1 + + service._run_scaffolding(job) + + # No continuation call and no repair call: only the two stage calls ran. + assert job.metrics["continuations_used"] == 0 + assert job.result_data["backend_truncated"] is True + assert job.metrics["repair_used"] is False assert job.metrics["repair_used"] is False diff --git a/backend/generation/tests/test_post_run.py b/backend/generation/tests/test_post_run.py index c72e8e73..a35141e3 100644 --- a/backend/generation/tests/test_post_run.py +++ b/backend/generation/tests/test_post_run.py @@ -120,6 +120,18 @@ def test_unknown_analyzer_is_refused(self): with pytest.raises(ExperimentError, match="at least one analyzer"): normalize_post_run({"analysis": {"enabled": True, "tool_slugs": []}}, "agent") + def test_dynamic_analyzer_is_refused(self, tool): + AnalyzerTool.objects.update_or_create( + slug="zap-test", + defaults={"name": "ZAP (test)", "analysis_mode": "dynamic"}, + ) + with pytest.raises(ExperimentError, match="running app") as err: + normalize_post_run({"analysis": {"enabled": True, "tool_slugs": ["bandit", "zap-test"]}}, "agent") + assert err.value.code == "dynamic_analyzer" + assert "bandit" not in str(err.value) + out = normalize_post_run({"analysis": {"enabled": True, "tool_slugs": ["bandit"]}}, "agent") + assert out["analysis"]["tool_slugs"] == ["bandit"] + class TestAdvance: def test_analyses_run_sequentially_for_jobs_without_one_then_the_report(self, user, tool): diff --git a/backend/generation/tests/test_profile_api.py b/backend/generation/tests/test_profile_api.py index c7b4743d..6bdb1ee0 100644 --- a/backend/generation/tests/test_profile_api.py +++ b/backend/generation/tests/test_profile_api.py @@ -224,10 +224,13 @@ def test_bundle_import_renames_conflicting_block_and_bundle(client): def test_template_package_export_and_import(client): owner = UserFactory() client.force_login(owner) + # Import checks specs against data/requirements/schema.json. + spec_fields = {k: v for k, v in SPEC_BODY.items() if k not in ("name", "slug")} app_template = AppRequirementTemplateFactory( is_default=False, created_by=owner, slug="shared-app", + **spec_fields, ) block = ContentBlockFactory( is_system=False, @@ -239,7 +242,7 @@ def test_template_package_export_and_import(client): is_default=False, created_by=owner, slug="shared-bundle-export", - scaffolding_slug="shared-stack", + scaffolding_slug="flask-react", block_refs=[{"type": block.block_type, "slug": block.slug, "version": block.version}], ) @@ -302,10 +305,10 @@ def test_import_skips_legacy_prompt_templates_section(client): "category": "Productivity", "description": "From an old export", "backend_requirements": ["Store items"], - "frontend_requirements": [], + "frontend_requirements": ["Item list"], "admin_requirements": [], - "api_endpoints": [], - "data_model": {}, + "api_endpoints": [{"method": "GET", "path": "/api/items", "description": "List"}], + "data_model": {"name": "Item", "fields": {"id": "integer"}}, "admin_api_endpoints": [], }, ], diff --git a/backend/generation/tests/test_template_packages.py b/backend/generation/tests/test_template_packages.py index 896cbaf4..4f8819cf 100644 --- a/backend/generation/tests/test_template_packages.py +++ b/backend/generation/tests/test_template_packages.py @@ -292,7 +292,7 @@ def test_reimporting_own_package_reuses_identical_stack_and_profile(): def test_import_rejects_invalid_profile_llm_config_atomically(): client = _client(UserFactory()) package = _package( - app_specs=[{"name": "Atomic", "slug": "atomic_app"}], + app_specs=[_spec("atomic_app")], profiles=[ { "name": "Hot", @@ -391,7 +391,7 @@ def test_v2_package_imports_with_no_presets(): package = { "kind": "llm-lab-template-package", "template_package_schema_version": 2, - "assets": {"app_specs": [{"name": "Old", "slug": "old_v2_app"}], "blocks": [], "profiles": [], "stacks": []}, + "assets": {"app_specs": [_spec("old_v2_app")], "blocks": [], "profiles": [], "stacks": []}, } response = _import(client, package) @@ -616,3 +616,87 @@ def test_uploaded_packages_never_read_content_files(): assert response.status_code == 400 assert "requires name and slug" in response.json()["detail"] + + +def _spec(slug: str, **extra) -> dict: + """A minimal app spec that meets data/requirements/schema.json.""" + return { + "name": slug.replace("_", " ").title(), + "slug": slug, + "category": "CRUD", + "description": "Track things", + "backend_requirements": ["Thing model"], + "frontend_requirements": ["Thing list"], + "api_endpoints": [{"method": "GET", "path": "/api/things", "description": "List"}], + "data_model": {"name": "Thing", "fields": {"id": "integer"}}, + **extra, + } + + +def test_import_validates_app_specs_against_the_schema(): + client = _client(UserFactory()) + response = _import(client, _package(app_specs=[{"name": "Thin", "slug": "thin_app"}])) + assert response.status_code == 400 + assert "App template thin_app" in response.json()["detail"] + assert "backend_requirements" in response.json()["detail"] + + ok = _import(client, _package(app_specs=[_spec("valid_app")])) + assert ok.status_code == 200, ok.content + assert ok.json()["app_specs"] == ["valid_app"] + + +def test_import_rejects_unknown_block_type(): + client = _client(UserFactory()) + block = {"slug": "weird-block", "name": "Weird", "content": "x", "block_type": "nonsense"} + response = _import(client, _package(blocks=[block])) + assert response.status_code == 400 + assert "unknown block_type 'nonsense'" in response.json()["detail"] + assert not ContentBlock.objects.filter(slug="weird-block").exists() + + +def test_import_rejects_profile_on_unknown_stack(): + client = _client(UserFactory()) + profile = { + "name": "Lost", + "slug": "lost-profile", + "scaffolding_slug": "no-such-stack", + "block_refs": [{"type": "prompt_tone", "slug": "tone-standard"}], + } + response = _import(client, _package(profiles=[profile])) + assert response.status_code == 400 + assert "unknown stack 'no-such-stack'" in response.json()["detail"] + assert not GenerationProfile.objects.filter(slug="lost-profile").exists() + + +def test_app_spec_api_validates_against_the_schema(): + client = _client(UserFactory()) + bad = client.post( + "/api/generation/app-specs/", + data=json.dumps({"name": "Thin", "slug": "thin_spec"}), + content_type="application/json", + ) + assert bad.status_code == 400 + assert bad.json()["errors"] + assert "backend_requirements" in bad.json()["detail"] + + hyphen = client.post( + "/api/generation/app-specs/", + data=json.dumps({**_spec("x"), "slug": "Not-Snake"}), + content_type="application/json", + ) + assert hyphen.status_code == 400 + assert "slug" in hyphen.json()["detail"] + + created = client.post( + "/api/generation/app-specs/", + data=json.dumps(_spec("ok_spec")), + content_type="application/json", + ) + assert created.status_code == 200, created.content + emptied = client.put( + "/api/generation/app-specs/ok_spec/", + data=json.dumps({**_spec("ok_spec"), "api_endpoints": []}), + content_type="application/json", + ) + assert emptied.status_code == 400 + assert "api_endpoints" in emptied.json()["detail"] diff --git a/backend/integrations/mcp/tools.py b/backend/integrations/mcp/tools.py index db792b44..66fb27b0 100644 --- a/backend/integrations/mcp/tools.py +++ b/backend/integrations/mcp/tools.py @@ -187,7 +187,7 @@ class Curated: { "name": "find_api_operations", "description": ( - "Search every LLM Lab REST operation (225+) by keyword and/or tag. Use when no dedicated " + "Search all LLM Lab REST operations by keyword and/or tag. Use when no dedicated " "tool fits; then describe_api_operation and call_api." ), "inputSchema": { diff --git a/backend/integrations/mcp/views.py b/backend/integrations/mcp/views.py index e6a79e2a..64b22e7c 100644 --- a/backend/integrations/mcp/views.py +++ b/backend/integrations/mcp/views.py @@ -70,7 +70,7 @@ def _handle(request: HttpRequest, raw_token: str, *, url_token: bool) -> HttpRes return _unauthorized("invalid, expired or revoked token") scopes = set(token.scopes or []) if url_token and URL_TOKEN_SCOPE not in scopes: - return _unauthorized("tokens in the URL need the 'mcp' scope; create one under Settings → Integrations") + return _unauthorized("tokens in the URL need the 'mcp' scope; create one under Settings → MCP server") token.mark_used(request.META.get("REMOTE_ADDR", "")) if len(request.body) > MAX_REQUEST_BYTES: diff --git a/backend/integrations/services/github.py b/backend/integrations/services/github.py index c73c7fbe..67c84fc6 100644 --- a/backend/integrations/services/github.py +++ b/backend/integrations/services/github.py @@ -91,7 +91,7 @@ class GitHubError(APIError): def _token_for(user: User) -> str: cred = get_user_credential_for(user, Provider.GITHUB) if cred is None: - msg = "No GitHub token connected. Add one in Settings → API Access (provider 'github')." + msg = "No GitHub token connected. Add one in Settings → GitHub." raise GitHubError(msg, status_code=400) try: token = cred.get_secret() diff --git a/backend/llm_models/api/views.py b/backend/llm_models/api/views.py index 41f64919..7e55940c 100644 --- a/backend/llm_models/api/views.py +++ b/backend/llm_models/api/views.py @@ -4,8 +4,10 @@ import csv import json +import logging from io import StringIO +from django.db import transaction from django.db.models import Avg from django.db.models import Q from django.http import HttpResponse @@ -30,6 +32,10 @@ from backend.llm_models.services import normalize_model_identifier from backend.llm_models.services import refresh_model_from_openrouter from backend.llm_models.services import sync_models_from_openrouter +from backend.llm_models.services.enrichment import fetch_provider_endpoints +from backend.llm_models.services.insights import model_insights + +logger = logging.getLogger(__name__) router = Router(tags=["models"]) @@ -336,7 +342,19 @@ def sync_models(request): api_key = _resolve_openrouter_catalog_key(request) if not isinstance(api_key, str): return api_key - return Status(200, sync_models_from_openrouter(api_key)) + result = sync_models_from_openrouter(api_key) + transaction.on_commit(_queue_hf_enrichment) + return Status(200, result) + + +def _queue_hf_enrichment() -> None: + """Hugging Face lookups take minutes for the whole catalog: run them on a worker.""" + from backend.llm_models.tasks import enrich_catalog_task + + try: + enrich_catalog_task.delay() + except Exception: # noqa: BLE001 — any broker/kombu error; the sync already succeeded + logger.warning("Could not queue catalog enrichment", exc_info=True) @router.post("/import/", response=ModelImportResultSchema) @@ -406,7 +424,12 @@ def get_model(request, slug: str): @router.post("/detail/{slug}/refresh/", response={200: LLMModelSchema, 403: dict}) def refresh_model(request, slug: str): - """Refresh a single model from the OpenRouter model catalog.""" + """Refresh a single model from the OpenRouter model catalog (staff only). + + Catalog rows are shared by every user, so rewriting one is gated like + sync/import/delete. + """ + _require_staff(request) model = _get_model_by_identifier(request, slug) if model.endpoint_id is not None and not model.endpoint.is_system: raise HttpError(400, "Local models are refreshed from their endpoint (Endpoints page).") @@ -430,13 +453,26 @@ def delete_model(request, slug: str): @router.get("/detail/{slug}/related/", response=list[LLMModelListSchema]) def get_related_models(request, slug: str, limit: int = Query(10, ge=1, le=20)): - """Get models from the same provider, excluding the current model.""" + """Get models from the same provider and source (catalog or local), excluding the current model.""" model = _get_model_by_identifier(request, slug) - related = ( - _base_queryset(request, include_unavailable=False) - .filter(provider=model.provider) - .exclude(pk=model.pk) - .order_by("-cost_efficiency", "model_name")[:limit] - ) + qs = _base_queryset(request, include_unavailable=False).filter(provider=model.provider).exclude(pk=model.pk) + qs = _apply_endpoint_filter(qs, "local" if model.is_local else "openrouter") + related = qs.order_by("-cost_efficiency", "model_name")[:limit] online: dict[int, bool] = {} return [LLMModelListSchema.from_model(m, online) for m in related] + + +@router.get("/detail/{slug}/insights/", response=dict) +def get_model_insights(request, slug: str): + """What this platform measured for the model (caller's jobs) and its place in the catalog.""" + model = _get_model_by_identifier(request, slug) + return model_insights(model, request.auth) + + +@router.get("/detail/{slug}/endpoints/", response=dict) +def get_model_endpoints(request, slug: str, refresh: bool = Query(False)): # noqa: FBT003 + """Providers serving the model on OpenRouter (live price, uptime, latency; cached 10 min).""" + model = _get_model_by_identifier(request, slug) + if model.endpoint_id is not None and not model.endpoint.is_system: + return {"available": False, "fetched_at": None, "endpoints": []} + return fetch_provider_endpoints(model, use_cache=not refresh) diff --git a/backend/llm_models/management/__init__.py b/backend/llm_models/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/llm_models/management/commands/__init__.py b/backend/llm_models/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/llm_models/management/commands/enrich_models.py b/backend/llm_models/management/commands/enrich_models.py new file mode 100644 index 00000000..ab9a4782 --- /dev/null +++ b/backend/llm_models/management/commands/enrich_models.py @@ -0,0 +1,23 @@ +"""Attach models.dev and Hugging Face metadata to the OpenRouter catalog. + +Usage: + python manage.py enrich_models [--no-hf] [--force-hf] +""" + +from __future__ import annotations + +from django.core.management.base import BaseCommand + +from backend.llm_models.services.enrichment import enrich_catalog + + +class Command(BaseCommand): + help = "Enrich catalog models with models.dev and Hugging Face metadata." + + def add_arguments(self, parser): + parser.add_argument("--no-hf", action="store_true", help="Skip Hugging Face lookups.") + parser.add_argument("--force-hf", action="store_true", help="Refetch Hugging Face data even when fresh.") + + def handle(self, *args, **options): + result = enrich_catalog(include_hf=not options["no_hf"], force_hf=options["force_hf"]) + self.stdout.write(self.style.SUCCESS(f"Enrichment done: {result}")) diff --git a/backend/llm_models/services/enrichment.py b/backend/llm_models/services/enrichment.py new file mode 100644 index 00000000..f8ca2b4e --- /dev/null +++ b/backend/llm_models/services/enrichment.py @@ -0,0 +1,327 @@ +"""Third-party model metadata beyond the OpenRouter catalog row. + +Three public, keyless sources: + +* **models.dev** (``https://models.dev/api.json``) — an open model database; + its ``openrouter`` provider is keyed by OpenRouter model id and adds the + knowledge cutoff, release / last-updated dates, open-weights flag, family, + reasoning options and context-tiered pricing. Stored in + ``metadata["models_dev"]``. +* **Hugging Face Hub** (``/api/models/``) for models whose payload names + a ``hugging_face_id`` — license, parameter count, downloads, likes, base + model, architecture, arXiv papers. Stored in ``metadata["huggingface"]``. +* **OpenRouter endpoints** (``/api/v1/models///endpoints``) — + the providers serving a model with their own price, quantization, limits, + uptime, latency and throughput. Live operational data, so it is cached + briefly instead of stored. + +The catalog sync keeps the stored keys (``ENRICHMENT_KEYS``) when it rewrites +``metadata``, so enrichment survives re-syncs. +""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime +from typing import Any + +import requests +from django.core.cache import cache +from django.utils import timezone + +from backend.llm_models.models import LLMModel + +logger = logging.getLogger(__name__) + +MODELS_DEV_URL = "https://models.dev/api.json" +HF_API_URL = "https://huggingface.co/api/models/{hf_id}" +OPENROUTER_ENDPOINTS_URL = "https://openrouter.ai/api/v1/models/{path}/endpoints" + +ENRICHMENT_KEYS = ("models_dev", "huggingface") + +MODELS_DEV_CACHE_KEY = "llm_models:models_dev:v1" +MODELS_DEV_CACHE_SECONDS = 6 * 3600 +ENDPOINTS_CACHE_SECONDS = 600 +HF_REFRESH_AFTER_SECONDS = 7 * 24 * 3600 +HF_REQUEST_PAUSE_SECONDS = 0.2 +REQUEST_TIMEOUT = 20 + +_MODELS_DEV_FIELDS = ( + "name", + "family", + "knowledge", + "release_date", + "last_updated", + "open_weights", + "reasoning", + "reasoning_options", + "tool_call", + "structured_output", + "temperature", + "attachment", + "modalities", + "limit", + "cost", + "status", +) + + +def _now_iso() -> str: + return timezone.now().isoformat() + + +def _get_json(url: str) -> Any | None: + try: + resp = requests.get(url, timeout=REQUEST_TIMEOUT, headers={"Accept": "application/json"}) + resp.raise_for_status() + return resp.json() + except (requests.RequestException, ValueError) as exc: + # Gated / renamed Hub repos answer 401/404 routinely; one line, no traceback. + logger.warning("Model enrichment request failed: %s (%s)", url, exc) + return None + + +# ── models.dev ────────────────────────────────────────────────────────────── + + +def fetch_models_dev(*, use_cache: bool = True) -> dict[str, dict]: + """models.dev's OpenRouter models keyed by OpenRouter model id (``{}`` when unreachable).""" + if use_cache: + cached = cache.get(MODELS_DEV_CACHE_KEY) + if cached is not None: + return cached + data = _get_json(MODELS_DEV_URL) + if not isinstance(data, dict): + return {} + models = ((data.get("openrouter") or {}).get("models")) or {} + catalog = {str(key): value for key, value in models.items() if isinstance(value, dict)} + if catalog: + cache.set(MODELS_DEV_CACHE_KEY, catalog, MODELS_DEV_CACHE_SECONDS) + return catalog + + +def models_dev_entry(catalog: dict[str, dict], model_id: str) -> dict | None: + """The normalized models.dev record for ``model_id`` (a ``:variant`` falls back to its base model).""" + entry = catalog.get(model_id) + if entry is None and ":" in model_id: + entry = catalog.get(model_id.split(":", 1)[0]) + if entry is None: + return None + out = {key: entry[key] for key in _MODELS_DEV_FIELDS if key in entry} + out["id"] = entry.get("id") or model_id + out["fetched_at"] = _now_iso() + return out + + +# ── Hugging Face ──────────────────────────────────────────────────────────── + + +def _hf_license(info: dict) -> str: + card = info.get("cardData") or {} + lic = card.get("license") + if isinstance(lic, list): + lic = lic[0] if lic else "" + if lic: + return str(lic) + for tag in info.get("tags") or []: + if isinstance(tag, str) and tag.startswith("license:"): + return tag.split(":", 1)[1] + return "" + + +def _as_list(value: Any) -> list[str]: + if value is None or value == "": + return [] + if isinstance(value, list): + return [str(v) for v in value if v] + return [str(value)] + + +def normalize_hf_info(info: dict) -> dict: + """The fields of a Hub model record the model page shows.""" + card = info.get("cardData") or {} + config = info.get("config") or {} + safetensors = info.get("safetensors") or {} + tags = [t for t in info.get("tags") or [] if isinstance(t, str)] + gated = info.get("gated") + return { + "id": info.get("id") or info.get("modelId") or "", + "author": info.get("author") or "", + "license": _hf_license(info), + "license_name": card.get("license_name") or "", + "pipeline_tag": info.get("pipeline_tag") or "", + "library_name": info.get("library_name") or card.get("library_name") or "", + "downloads": int(info.get("downloads") or 0), + "downloads_all_time": info.get("downloadsAllTime"), + "likes": int(info.get("likes") or 0), + "parameters": safetensors.get("total"), + "parameters_by_dtype": safetensors.get("parameters") or {}, + "architectures": _as_list(config.get("architectures")), + "model_type": config.get("model_type") or "", + "base_models": _as_list(card.get("base_model")), + "languages": _as_list(card.get("language")), + "arxiv": [t.split(":", 1)[1] for t in tags if t.startswith("arxiv:")], + "gated": bool(gated) and gated != "False", + "created_at": info.get("createdAt"), + "last_modified": info.get("lastModified"), + "spaces_count": len(info.get("spaces") or []), + "fetched_at": _now_iso(), + } + + +def fetch_hf_info(hf_id: str) -> dict | None: + hf_id = (hf_id or "").strip().strip("/") + if not hf_id or hf_id.count("/") != 1: + return None + data = _get_json(HF_API_URL.format(hf_id=hf_id)) + return normalize_hf_info(data) if isinstance(data, dict) and not data.get("error") else None + + +def _hf_is_fresh(meta: dict) -> bool: + fetched = (meta.get("huggingface") or {}).get("fetched_at") + if not fetched: + return False + try: + age = timezone.now() - datetime.fromisoformat(fetched) + except (TypeError, ValueError): + return False + return age.total_seconds() < HF_REFRESH_AFTER_SECONDS + + +# ── Enrichment entry points ───────────────────────────────────────────────── + + +def _is_catalog_model(model: LLMModel) -> bool: + return model.endpoint_id is None + + +def enrich_model( + model: LLMModel, + *, + catalog: dict[str, dict] | None = None, + include_hf: bool = True, + force_hf: bool = False, +) -> dict[str, bool]: + """Attach models.dev / Hugging Face metadata to one catalog model. + + Returns which sources were written (``{"models_dev": …, "huggingface": …}``). + """ + written = {"models_dev": False, "huggingface": False} + if not _is_catalog_model(model): + return written + meta = dict(model.metadata or {}) + + md_catalog = catalog if catalog is not None else fetch_models_dev() + entry = models_dev_entry(md_catalog, model.model_id) if md_catalog else None + if entry is not None: + meta["models_dev"] = entry + written["models_dev"] = True + + hf_id = str((model.capabilities_json or {}).get("hugging_face_id") or "") + if include_hf and hf_id and (force_hf or not _hf_is_fresh(meta)): + info = fetch_hf_info(hf_id) + if info is not None: + meta["huggingface"] = info + written["huggingface"] = True + + if any(written.values()): + model.metadata = meta + model.save(update_fields=["metadata", "updated_at"]) + return written + + +def enrich_catalog(*, include_hf: bool = True, force_hf: bool = False) -> dict[str, int]: + """Enrich every OpenRouter catalog model; HF lookups are rate-paced and skipped while fresh.""" + catalog = fetch_models_dev(use_cache=False) + counts = {"models_dev": 0, "huggingface": 0} + for model in LLMModel.objects.filter(endpoint__isnull=True).iterator(): + written = enrich_model(model, catalog=catalog, include_hf=include_hf, force_hf=force_hf) + for key, value in written.items(): + counts[key] += value + if written["huggingface"]: + time.sleep(HF_REQUEST_PAUSE_SECONDS) + return { + "models_dev_matched": counts["models_dev"], + "huggingface_fetched": counts["huggingface"], + "models_dev_size": len(catalog), + } + + +def preserved_enrichment(model_ids: list[str]) -> dict[str, dict]: + """``{model_id: {key: value}}`` of stored enrichment, for the sync to carry over.""" + out: dict[str, dict] = {} + rows = LLMModel.objects.filter(endpoint__isnull=True, model_id__in=model_ids).values_list("model_id", "metadata") + for model_id, meta in rows: + kept = {k: meta[k] for k in ENRICHMENT_KEYS if isinstance(meta, dict) and k in meta} + if kept: + out[model_id] = kept + return out + + +# ── OpenRouter provider endpoints (live) ──────────────────────────────────── + + +def _endpoints_path(model: LLMModel) -> str: + payload = model.capabilities_json or {} + details = ((payload.get("links") or {}).get("details")) or "" + prefix = "/api/v1/models/" + if details.startswith(prefix) and details.endswith("/endpoints"): + return details[len(prefix) : -len("/endpoints")] + return model.model_id.split(":", 1)[0] + + +def _float_or_none(value: Any) -> float | None: + try: + return None if value in (None, "") else float(value) + except (TypeError, ValueError): + return None + + +def _normalize_endpoint(raw: dict) -> dict: + pricing = raw.get("pricing") or {} + return { + "provider_name": raw.get("provider_name") or raw.get("name") or "", + "tag": raw.get("tag") or "", + "quantization": raw.get("quantization") or "", + "context_length": raw.get("context_length"), + "max_completion_tokens": raw.get("max_completion_tokens"), + "max_prompt_tokens": raw.get("max_prompt_tokens"), + "input_price_per_million": (_float_or_none(pricing.get("prompt")) or 0.0) * 1_000_000, + "output_price_per_million": (_float_or_none(pricing.get("completion")) or 0.0) * 1_000_000, + "cache_read_per_million": ( + p * 1_000_000 if (p := _float_or_none(pricing.get("input_cache_read"))) is not None else None + ), + "discount": _float_or_none(pricing.get("discount")) or 0.0, + "status": raw.get("status"), + "uptime_last_5m": _float_or_none(raw.get("uptime_last_5m")), + "uptime_last_30m": _float_or_none(raw.get("uptime_last_30m")), + "uptime_last_1d": _float_or_none(raw.get("uptime_last_1d")), + "latency_last_30m": raw.get("latency_last_30m"), + "throughput_last_30m": raw.get("throughput_last_30m"), + "supported_parameters": raw.get("supported_parameters") or [], + "supports_implicit_caching": bool(raw.get("supports_implicit_caching")), + "native_tools": sorted((raw.get("native_tools") or {}).keys()), + "service_tier": raw.get("service_tier"), + } + + +def fetch_provider_endpoints(model: LLMModel, *, use_cache: bool = True) -> dict[str, Any]: + """Providers serving ``model`` on OpenRouter; ``available`` False when it could not be fetched.""" + path = _endpoints_path(model) + cache_key = f"llm_models:endpoints:v1:{path}" + if use_cache: + cached = cache.get(cache_key) + if cached is not None: + return cached + data = _get_json(OPENROUTER_ENDPOINTS_URL.format(path=path)) + raw = ((data or {}).get("data") or {}).get("endpoints") if isinstance(data, dict) else None + if not isinstance(raw, list): + return {"available": False, "fetched_at": _now_iso(), "endpoints": []} + result = { + "available": True, + "fetched_at": _now_iso(), + "endpoints": [_normalize_endpoint(e) for e in raw if isinstance(e, dict)], + } + cache.set(cache_key, result, ENDPOINTS_CACHE_SECONDS) + return result diff --git a/backend/llm_models/services/insights.py b/backend/llm_models/services/insights.py new file mode 100644 index 00000000..a61c0112 --- /dev/null +++ b/backend/llm_models/services/insights.py @@ -0,0 +1,458 @@ +"""Per-model research data for the model page. + +Everything this platform measured about one model for one user — generation +outcomes, resource use, conformance results, analysis findings, the model's +place in the rankings — plus where its catalog attributes and external +benchmark scores sit relative to the rest of the catalog. + +Local numbers are scoped to the requesting user's jobs (the same scope as +the jobs list and /statistics); catalog percentiles are global. +""" + +from __future__ import annotations + +import statistics as pystats +from bisect import bisect_left +from bisect import bisect_right +from collections import Counter +from typing import TYPE_CHECKING +from typing import Any + +from django.core.cache import cache +from django.db.models import Count +from django.db.models import Min + +from backend.analysis.models import AnalysisRun +from backend.analysis.models import AnalyzerTool +from backend.analysis.models import Finding +from backend.generation.models import GenerationJob +from backend.llm_models.models import LLMModel +from backend.rankings.models import BenchmarkResult +from backend.statistics.services.inference import wilson_interval + +if TYPE_CHECKING: + from django.contrib.auth.models import AbstractBaseUser + +SEVERITIES = ("critical", "high", "medium", "low", "info") +RANKINGS_CACHE_SECONDS = 60 +CATALOG_CACHE_SECONDS = 600 +TOP_RULES = 10 +TOP_TEMPLATES = 12 + +# Artificial Analysis indices OpenRouter embeds in the catalog payload. +AA_INDICES = ("intelligence_index", "coding_index", "agentic_index") + + +def _round(value: float | None, digits: int = 4) -> float | None: + return None if value is None else round(value, digits) + + +def _mean(values: list[float]) -> float | None: + return round(pystats.fmean(values), 4) if values else None + + +def _median(values: list[float]) -> float | None: + return round(pystats.median(values), 4) if values else None + + +def _rate(successes: int, n: int) -> dict[str, Any]: + ci = wilson_interval(successes, n) if n else None + return { + "successes": successes, + "n": n, + "rate": round(successes / n, 4) if n else None, + "ci_low": _round(ci[0]) if ci else None, + "ci_high": _round(ci[1]) if ci else None, + } + + +def _percentile_rank(sorted_values: list[float], value: float | None) -> float | None: + """Share of the catalog strictly below ``value`` plus half the ties, 0..1.""" + if value is None or not sorted_values: + return None + below = bisect_left(sorted_values, value) + ties = bisect_right(sorted_values, value) - below + return round((below + ties / 2) / len(sorted_values), 4) + + +# ── Generation ────────────────────────────────────────────────────────────── + + +def _num(metrics: dict, key: str) -> float | None: + value = metrics.get(key) + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None + + +def _generation_stats(jobs) -> dict[str, Any]: + total = jobs.count() + by_status = dict(jobs.values_list("status").annotate(c=Count("id")).values_list("status", "c")) + by_mode = dict(jobs.values_list("mode").annotate(c=Count("id")).values_list("mode", "c")) + completed = int(by_status.get(GenerationJob.Status.COMPLETED, 0)) + finished = completed + int(by_status.get(GenerationJob.Status.FAILED, 0)) + + series: dict[str, list[float]] = { + k: [] + for k in ( + "cost", + "duration", + "prompt_tokens", + "completion_tokens", + "reasoning_tokens", + "cached_tokens", + "total_tokens", + "llm_calls", + "lines_of_code", + "pass_rate", + "output_tps", + ) + } + engines: Counter[str] = Counter() + providers: Counter[str] = Counter() + terminations: Counter[str] = Counter() + functional_status: Counter[str] = Counter() + functional_passed = functional_evaluated = 0 + repair_used = continuations = 0 + total_cost = 0.0 + + rows = jobs.filter(status=GenerationJob.Status.COMPLETED).values_list("metrics", "duration_seconds") + for metrics, duration in rows: + if not isinstance(metrics, dict): + continue + for key in ( + "prompt_tokens", + "completion_tokens", + "reasoning_tokens", + "cached_tokens", + "total_tokens", + "llm_calls", + "lines_of_code", + ): + if (v := _num(metrics, key)) is not None: + series[key].append(v) + if (cost := _num(metrics, "cost")) is not None: + series["cost"].append(cost) + total_cost += cost + dur = duration if duration else _num(metrics, "total_duration") + if dur: + series["duration"].append(float(dur)) + if (out := _num(metrics, "completion_tokens")) is not None: + series["output_tps"].append(out / float(dur)) + if metrics.get("engine"): + engines[str(metrics["engine"]).split("@", 1)[0]] += 1 + for provider, n in (metrics.get("providers") or {}).items(): + if isinstance(n, (int, float)): + providers[str(provider)] += int(n) + if metrics.get("termination"): + terminations[str(metrics["termination"])] += 1 + repair_used += bool(metrics.get("repair_used")) + continuations += int(metrics.get("continuations_used") or 0) + functional = metrics.get("functional") + if isinstance(functional, dict) and functional: + functional_evaluated += 1 + functional_passed += bool(functional.get("passed")) + functional_status[ + str(functional.get("status") or ("passed" if functional.get("passed") else "failed")) + ] += 1 + if isinstance(functional.get("pass_rate"), (int, float)): + series["pass_rate"].append(float(functional["pass_rate"])) + + prompt_sum = sum(series["prompt_tokens"]) + loc_sum = sum(series["lines_of_code"]) + + return { + "jobs_total": total, + "by_status": by_status, + "by_mode": by_mode, + "completed": completed, + "success": _rate(completed, finished), + "functional": { + **_rate(functional_passed, functional_evaluated), + "mean_pass_rate": _mean(series["pass_rate"]), + "by_status": dict(functional_status), + }, + "resources": { + "total_cost": round(total_cost, 4), + "mean_cost": _mean(series["cost"]), + "cost_per_kloc": round(total_cost / loc_sum * 1000, 4) if loc_sum and series["cost"] else None, + "mean_duration": _mean(series["duration"]), + "median_duration": _median(series["duration"]), + "mean_prompt_tokens": _mean(series["prompt_tokens"]), + "mean_completion_tokens": _mean(series["completion_tokens"]), + "mean_reasoning_tokens": _mean(series["reasoning_tokens"]), + "mean_total_tokens": _mean(series["total_tokens"]), + "cache_hit_ratio": round(sum(series["cached_tokens"]) / prompt_sum, 4) + if prompt_sum and series["cached_tokens"] + else None, + "mean_llm_calls": _mean(series["llm_calls"]), + "mean_lines_of_code": _mean(series["lines_of_code"]), + "median_output_tps": _median(series["output_tps"]), + }, + "repair_rate": round(repair_used / completed, 4) if completed else None, + "continuations_used": continuations, + "engines": dict(engines.most_common()), + "providers": dict(providers.most_common()), + "terminations": dict(terminations.most_common()), + "first_job_at": jobs.order_by("created_at").values_list("created_at", flat=True).first(), + "last_job_at": jobs.order_by("-created_at").values_list("created_at", flat=True).first(), + } + + +def _template_breakdown(jobs) -> list[dict[str, Any]]: + rows = ( + jobs.exclude(app_requirement__isnull=True) + .values("app_requirement__slug", "app_requirement__name") + .annotate(n=Count("id")) + .order_by("-n")[:TOP_TEMPLATES] + ) + out: list[dict[str, Any]] = [] + for row in rows: + scoped = jobs.filter(app_requirement__slug=row["app_requirement__slug"]) + completed = passed = evaluated = 0 + for status, metrics in scoped.values_list("status", "metrics"): + completed += status == GenerationJob.Status.COMPLETED + functional = (metrics or {}).get("functional") if isinstance(metrics, dict) else None + if status == GenerationJob.Status.COMPLETED and isinstance(functional, dict) and functional: + evaluated += 1 + passed += bool(functional.get("passed")) + out.append( + { + "slug": row["app_requirement__slug"], + "name": row["app_requirement__name"], + "jobs": int(row["n"]), + "completed": completed, + "functional": _rate(passed, evaluated), + }, + ) + return out + + +def _experiments(jobs) -> list[dict[str, Any]]: + rows = ( + jobs.exclude(experiment__isnull=True) + .values("experiment_id", "experiment__name", "experiment__status", "experiment__created_at") + .annotate(n=Count("id")) + .order_by("-experiment__created_at")[:20] + ) + return [ + { + "id": str(r["experiment_id"]), + "name": r["experiment__name"], + "status": r["experiment__status"], + "jobs": int(r["n"]), + "created_at": r["experiment__created_at"], + } + for r in rows + ] + + +# ── Analysis ──────────────────────────────────────────────────────────────── + + +def _analysis_stats(jobs) -> dict[str, Any]: + job_ids = list(jobs.values_list("id", flat=True)) + latest = AnalysisRun.latest_ids_per_job(job_ids) if job_ids else [] + findings = Finding.objects.filter(result__run_id__in=latest) + ai_slugs = AnalyzerTool.ai_slugs() + + static = dict.fromkeys(SEVERITIES, 0) + ai = dict.fromkeys(SEVERITIES, 0) + by_tool: dict[str, dict[str, Any]] = {} + for row in findings.values("result__tool_slug", "severity").annotate(c=Count("id")): + tool, sev, c = row["result__tool_slug"], row["severity"], int(row["c"]) + target = ai if tool in ai_slugs else static + if sev in target: + target[sev] += c + entry = by_tool.setdefault( + tool, + {"tool": tool, "ai": tool in ai_slugs, "total": 0, **dict.fromkeys(SEVERITIES, 0)}, + ) + entry["total"] += c + if sev in SEVERITIES: + entry[sev] += c + + # Titles vary with the matched code; Min() picks one representative per rule. + top_rules = [ + { + "tool": r["result__tool_slug"], + "rule_id": r["rule_id"], + "title": r["title"] or r["rule_id"], + "severity": r["severity"], + "count": int(r["c"]), + } + for r in findings.exclude(rule_id="") + .values("result__tool_slug", "rule_id", "severity") + .annotate(c=Count("id"), title=Min("title")) + .order_by("-c")[:TOP_RULES] + ] + + return { + "analyzed_jobs": len(latest), + "findings_static": static, + "findings_ai": ai, + "by_tool": sorted(by_tool.values(), key=lambda t: -t["total"]), + "top_rules": top_rules, + } + + +# ── Rankings ──────────────────────────────────────────────────────────────── + + +def _rankings_for(user: AbstractBaseUser | None) -> list[dict[str, Any]]: + from backend.rankings.services import aggregate_rankings + + key = f"llm_models:insights:rankings:{getattr(user, 'pk', 'anon')}" + rows = cache.get(key) + if rows is None: + rows = aggregate_rankings(user=user) + cache.set(key, rows, RANKINGS_CACHE_SECONDS) + return rows + + +def _position(rows: list[dict[str, Any]], model_id: str, field: str) -> dict[str, Any] | None: + scored = sorted((r for r in rows if r.get(field) is not None), key=lambda r: -(r[field] or 0.0)) + for index, row in enumerate(scored): + if row["model_id"] == model_id: + return {"rank": index + 1, "of": len(scored), "score": row[field]} + return None + + +def _ranking_stats(model: LLMModel, user: AbstractBaseUser | None) -> dict[str, Any] | None: + rows = _rankings_for(user) + row = next((r for r in rows if r["model_id"] == model.model_id), None) + if row is None: + return None + exercised = [r for r in rows if r.get("apps")] + return { + "scores": { + key: row.get(key) + for key in ( + "composite_score", + "mss_score", + "empirical_quality_score", + "benchmark_score", + "cost_efficiency_score", + "accessibility_score", + "adoption_score", + ) + }, + "positions": { + "composite": _position(rows, model.model_id, "composite_score"), + "mss": _position(rows, model.model_id, "mss_score"), + "empirical": _position(exercised, model.model_id, "empirical_quality_score"), + "benchmark": _position( + [r for r in rows if (r.get("benchmark_score") or 0) > 0], + model.model_id, + "benchmark_score", + ), + }, + "variance": row.get("variance"), + "local_loc": row.get("local_loc", 0), + "n_trials": row.get("n_trials", 0), + "models_exercised": len(exercised), + } + + +# ── Catalog context & external benchmarks ─────────────────────────────────── + + +def _catalog_distributions() -> dict[str, list[float]]: + key = "llm_models:insights:catalog:v1" + cached = cache.get(key) + if cached is not None: + return cached + dists: dict[str, list[float]] = { + "input_price": [], + "output_price": [], + "context_window": [], + "max_output_tokens": [], + **{f"aa_{k}": [] for k in AA_INDICES}, + } + rows = LLMModel.objects.filter(endpoint__isnull=True).values_list( + "input_price_per_token", + "output_price_per_token", + "context_window", + "max_output_tokens", + "is_free", + "capabilities_json", + ) + for inp, out, ctx, max_out, is_free, payload in rows: + if not is_free and inp > 0: + dists["input_price"].append(inp * 1_000_000) + if not is_free and out > 0: + dists["output_price"].append(out * 1_000_000) + if ctx > 0: + dists["context_window"].append(float(ctx)) + if max_out > 0: + dists["max_output_tokens"].append(float(max_out)) + aa = ((payload or {}).get("benchmarks") or {}).get("artificial_analysis") or {} + for k in AA_INDICES: + if isinstance(aa.get(k), (int, float)): + dists[f"aa_{k}"].append(float(aa[k])) + for values in dists.values(): + values.sort() + cache.set(key, dists, CATALOG_CACHE_SECONDS) + return dists + + +def _catalog_context(model: LLMModel) -> dict[str, Any]: + dists = _catalog_distributions() + paid = not model.is_free + + def entry(name: str, value: float | None) -> dict[str, Any]: + values = dists[name] + return { + "value": value, + "percentile": _percentile_rank(values, value), + "n": len(values), + "median": _median(values), + } + + aa = ((model.capabilities_json or {}).get("benchmarks") or {}).get("artificial_analysis") or {} + provider_models = LLMModel.objects.filter(endpoint__isnull=True, provider=model.provider).count() + return { + "input_price": entry("input_price", model.input_price_per_token * 1_000_000 if paid else None), + "output_price": entry("output_price", model.output_price_per_token * 1_000_000 if paid else None), + "context_window": entry("context_window", float(model.context_window) or None), + "max_output_tokens": entry("max_output_tokens", float(model.max_output_tokens) or None), + "artificial_analysis": { + k: entry(f"aa_{k}", float(aa[k]) if isinstance(aa.get(k), (int, float)) else None) for k in AA_INDICES + }, + "catalog_size": LLMModel.objects.filter(endpoint__isnull=True).count(), + "provider_models": provider_models, + } + + +def _local_benchmarks(model: LLMModel) -> list[dict[str, Any]]: + rows = BenchmarkResult.objects.filter(model_id=model.model_id).order_by("benchmark") + out: list[dict[str, Any]] = [] + for row in rows: + others = sorted(BenchmarkResult.objects.filter(benchmark=row.benchmark).values_list("score", flat=True)) + out.append( + { + "benchmark": row.benchmark, + "score": row.score, + "source": row.source, + "fetched_at": row.fetched_at, + "percentile": _percentile_rank(others, row.score), + "n": len(others), + }, + ) + return out + + +# ── Entry point ───────────────────────────────────────────────────────────── + + +def model_insights(model: LLMModel, user: AbstractBaseUser | None) -> dict[str, Any]: + jobs = GenerationJob.objects.filter(model=model) + if user is not None and getattr(user, "is_authenticated", False): + jobs = jobs.filter(created_by=user) + return { + "generation": _generation_stats(jobs), + "templates": _template_breakdown(jobs), + "experiments": _experiments(jobs), + "analysis": _analysis_stats(jobs), + "ranking": _ranking_stats(model, user), + "catalog": _catalog_context(model), + "benchmarks": _local_benchmarks(model), + } diff --git a/backend/llm_models/services/sync.py b/backend/llm_models/services/sync.py index e3f87e10..28d1feac 100644 --- a/backend/llm_models/services/sync.py +++ b/backend/llm_models/services/sync.py @@ -7,6 +7,9 @@ from django.conf import settings from backend.llm_models.models import LLMModel +from backend.llm_models.services.enrichment import enrich_model +from backend.llm_models.services.enrichment import fetch_models_dev +from backend.llm_models.services.enrichment import preserved_enrichment from backend.llm_models.services.helpers import _build_canonical_slug from backend.llm_models.services.helpers import _update_or_create_model from backend.llm_models.services.helpers import normalize_model_identifier @@ -55,6 +58,9 @@ def upsert_openrouter_models(models_payload: list[dict]) -> int: Returns count upserted. """ upserted = 0 + # Third-party enrichment (models.dev, Hugging Face) is not in the payload; + # carry it over so a re-sync doesn't wipe it. + preserved = preserved_enrichment([str(m.get("id")) for m in models_payload if m.get("id")]) for model_data in models_payload: model_id = model_data.get("id") if not model_id: @@ -168,6 +174,7 @@ def upsert_openrouter_models(models_payload: list[dict]) -> int: meta["architecture_instruct_type"] = arch.get("instruct_type") meta["architecture_input_modalities"] = arch.get("input_modalities") meta["architecture_output_modalities"] = arch.get("output_modalities") + meta.update(preserved.get(model_id, {})) defaults = { "provider": provider, @@ -218,7 +225,11 @@ def refresh_model_from_openrouter(model: LLMModel, api_key: str) -> bool: str(model_data.get("canonical_slug") or ""), ) if model_id.lower() in candidates or canonical_slug in candidates: - return upsert_openrouter_models([model_data]) == 1 + if upsert_openrouter_models([model_data]) != 1: + return False + model.refresh_from_db() + enrich_model(model, force_hf=True) + return True return False @@ -229,4 +240,14 @@ def sync_models_from_openrouter(api_key: str) -> dict: if not models_payload: return {"fetched": 0, "upserted": 0} upserted = upsert_openrouter_models(models_payload) + _enrich_from_models_dev() return {"fetched": len(models_payload), "upserted": upserted} + + +def _enrich_from_models_dev() -> None: + """One models.dev download covers the whole catalog; Hugging Face runs separately (``enrich_models``).""" + catalog = fetch_models_dev(use_cache=False) + if not catalog: + return + for model in LLMModel.objects.filter(endpoint__isnull=True).iterator(): + enrich_model(model, catalog=catalog, include_hf=False) diff --git a/backend/llm_models/tasks.py b/backend/llm_models/tasks.py new file mode 100644 index 00000000..dd3ef713 --- /dev/null +++ b/backend/llm_models/tasks.py @@ -0,0 +1,13 @@ +"""Celery tasks for the model catalog.""" + +from __future__ import annotations + +from celery import shared_task + + +@shared_task(acks_late=True) +def enrich_catalog_task(include_hf: bool = True) -> dict: + """Attach models.dev / Hugging Face metadata to every catalog model (see ``services.enrichment``).""" + from backend.llm_models.services.enrichment import enrich_catalog + + return enrich_catalog(include_hf=include_hf) diff --git a/backend/llm_models/tests/test_api.py b/backend/llm_models/tests/test_api.py index 9668cf75..0d9f4fc8 100644 --- a/backend/llm_models/tests/test_api.py +++ b/backend/llm_models/tests/test_api.py @@ -320,7 +320,18 @@ def test_compare_models_preserves_order(self, api_client): @pytest.mark.django_db class TestRefreshAPI: @patch("backend.llm_models.api.views.refresh_model_from_openrouter") - def test_refresh_model(self, mock_refresh, api_client, user): + def test_refresh_is_staff_only(self, mock_refresh, api_client, user): + """Catalog rows are shared, so a regular user can't rewrite one.""" + _set_user_openrouter_key(user, "user-key") + model = LLMModelFactory(model_id="openai/gpt-4o", canonical_slug="openai_gpt-4o") + + resp = api_client.post(f"/api/models/detail/{model.canonical_slug}/refresh/") + + assert resp.status_code == HTTPStatus.FORBIDDEN + mock_refresh.assert_not_called() + + @patch("backend.llm_models.api.views.refresh_model_from_openrouter") + def test_refresh_model(self, mock_refresh, staff_api_client, user): _set_user_openrouter_key(user, "user-key") model = LLMModelFactory( model_id="openai/gpt-4o", @@ -336,7 +347,7 @@ def refresh_side_effect(instance, api_key): mock_refresh.side_effect = refresh_side_effect - resp = api_client.post(f"/api/models/detail/{model.canonical_slug}/refresh/") + resp = staff_api_client.post(f"/api/models/detail/{model.canonical_slug}/refresh/") assert resp.status_code == HTTPStatus.OK, resp.content assert resp.json()["model_name"] == "New Name" @@ -344,13 +355,13 @@ def refresh_side_effect(instance, api_key): @override_settings(OPENROUTER_API_KEY="global-key", OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=True) @patch("backend.llm_models.api.views.refresh_model_from_openrouter") - def test_refresh_requires_user_key_even_when_global_fallback_exists(self, mock_refresh, api_client): + def test_refresh_requires_user_key_even_when_global_fallback_exists(self, mock_refresh, staff_api_client): model = LLMModelFactory( model_id="openai/gpt-4o", canonical_slug="openai_gpt-4o", ) - resp = api_client.post(f"/api/models/detail/{model.canonical_slug}/refresh/") + resp = staff_api_client.post(f"/api/models/detail/{model.canonical_slug}/refresh/") assert resp.status_code == HTTPStatus.FORBIDDEN data = resp.json() diff --git a/backend/llm_models/tests/test_enrichment.py b/backend/llm_models/tests/test_enrichment.py new file mode 100644 index 00000000..23dbfe68 --- /dev/null +++ b/backend/llm_models/tests/test_enrichment.py @@ -0,0 +1,206 @@ +from __future__ import annotations + +from http import HTTPStatus +from unittest.mock import Mock +from unittest.mock import patch + +import pytest +from django.core.cache import cache + +from backend.generation.tests.factories import GenerationJobFactory +from backend.llm_models.services.enrichment import enrich_model +from backend.llm_models.services.enrichment import fetch_provider_endpoints +from backend.llm_models.services.enrichment import models_dev_entry +from backend.llm_models.services.enrichment import normalize_hf_info +from backend.llm_models.services.sync import upsert_openrouter_models +from backend.llm_models.tests.factories import LLMModelFactory + +MODELS_DEV = { + "openrouter": { + "models": { + "acme/coder-1": { + "id": "acme/coder-1", + "family": "coder", + "knowledge": "2025-08-31", + "release_date": "2026-02-17", + "open_weights": True, + "limit": {"context": 128000, "output": 8192}, + "cost": {"input": 1, "output": 2}, + "unexpected": "dropped", + }, + }, + }, +} + +HF_RECORD = { + "id": "acme/Coder-1", + "author": "acme", + "downloads": 1234, + "likes": 56, + "pipeline_tag": "text-generation", + "tags": ["transformers", "license:apache-2.0", "arxiv:2412.19437"], + "cardData": {"base_model": "acme/Coder-1-Base", "language": ["en"]}, + "config": {"architectures": ["CoderForCausalLM"], "model_type": "coder"}, + "safetensors": {"total": 7_000_000_000, "parameters": {"BF16": 7_000_000_000}}, + "gated": "False", +} + + +def _response(payload): + resp = Mock() + resp.raise_for_status.return_value = None + resp.json.return_value = payload + return resp + + +@pytest.fixture(autouse=True) +def _clear_cache(): + cache.clear() + yield + cache.clear() + + +def test_models_dev_entry_keeps_known_fields_and_falls_back_from_variant(): + catalog = MODELS_DEV["openrouter"]["models"] + entry = models_dev_entry(catalog, "acme/coder-1:free") + assert entry is not None + assert entry["knowledge"] == "2025-08-31" + assert entry["open_weights"] is True + assert "unexpected" not in entry + assert models_dev_entry(catalog, "acme/other") is None + + +def test_normalize_hf_info_reads_license_params_and_papers(): + info = normalize_hf_info(HF_RECORD) + assert info["license"] == "apache-2.0" + assert info["parameters"] == 7_000_000_000 + assert info["arxiv"] == ["2412.19437"] + assert info["base_models"] == ["acme/Coder-1-Base"] + assert info["architectures"] == ["CoderForCausalLM"] + assert info["gated"] is False + + +@pytest.mark.django_db +@patch("backend.llm_models.services.enrichment.requests.get") +def test_enrich_model_stores_models_dev_and_hf(mock_get): + model = LLMModelFactory(model_id="acme/coder-1", capabilities_json={"hugging_face_id": "acme/Coder-1"}) + mock_get.side_effect = lambda url, **_: _response(MODELS_DEV if "models.dev" in url else HF_RECORD) + + written = enrich_model(model) + + assert written == {"models_dev": True, "huggingface": True} + model.refresh_from_db() + assert model.metadata["models_dev"]["release_date"] == "2026-02-17" + assert model.metadata["huggingface"]["downloads"] == 1234 + + +@pytest.mark.django_db +@patch("backend.llm_models.services.enrichment.requests.get") +def test_enrich_model_survives_unreachable_sources(mock_get): + import requests + + model = LLMModelFactory(model_id="acme/coder-1", capabilities_json={"hugging_face_id": "acme/Coder-1"}) + mock_get.side_effect = requests.ConnectionError("offline") + + assert enrich_model(model) == {"models_dev": False, "huggingface": False} + + +@pytest.mark.django_db +def test_catalog_resync_keeps_enrichment(): + model = LLMModelFactory( + model_id="acme/coder-1", + metadata={"models_dev": {"knowledge": "2025-08-31"}, "huggingface": {"likes": 3}, "stale": 1}, + ) + with patch("backend.llm_models.services.enrichment.requests.get"): + upsert_openrouter_models([{"id": "acme/coder-1", "name": "Acme: Coder 1", "pricing": {}}]) + + model.refresh_from_db() + assert model.metadata["models_dev"] == {"knowledge": "2025-08-31"} + assert model.metadata["huggingface"] == {"likes": 3} + assert "stale" not in model.metadata + assert model.metadata["openrouter_name"] == "Acme: Coder 1" + + +@pytest.mark.django_db +@patch("backend.llm_models.services.enrichment.requests.get") +def test_provider_endpoints_use_details_link_and_normalize(mock_get): + model = LLMModelFactory( + model_id="acme/coder-1", + capabilities_json={"links": {"details": "/api/v1/models/acme/coder-1-20260101/endpoints"}}, + ) + mock_get.return_value = _response( + { + "data": { + "endpoints": [ + { + "provider_name": "Acme Cloud", + "quantization": "fp8", + "context_length": 128000, + "pricing": {"prompt": "0.000001", "completion": "0.000002"}, + "uptime_last_30m": 99.5, + "native_tools": {"openrouter:web_search": {}}, + }, + ], + }, + }, + ) + + result = fetch_provider_endpoints(model) + + assert mock_get.call_args.args[0].endswith("/models/acme/coder-1-20260101/endpoints") + assert result["available"] is True + endpoint = result["endpoints"][0] + assert endpoint["provider_name"] == "Acme Cloud" + assert endpoint["input_price_per_million"] == pytest.approx(1.0) + assert endpoint["native_tools"] == ["openrouter:web_search"] + # Second call is served from the cache. + fetch_provider_endpoints(model) + assert mock_get.call_count == 1 + + +@pytest.mark.django_db +class TestModelInsightsAPI: + def test_insights_summarize_own_jobs_only(self, client, user): + client.force_login(user) + model = LLMModelFactory(model_id="acme/coder-1", canonical_slug="acme_coder-1") + metrics = { + "cost": 0.5, + "prompt_tokens": 1000, + "cached_tokens": 500, + "completion_tokens": 200, + "lines_of_code": 2000, + "engine": "opencode@1.0", + "functional": {"passed": True, "pass_rate": 1.0, "status": "passed"}, + } + GenerationJobFactory(created_by=user, model=model, status="completed", metrics=metrics, duration_seconds=100) + GenerationJobFactory(created_by=user, model=model, status="failed") + GenerationJobFactory(model=model, status="completed", metrics=metrics) # another user's job + + resp = client.get("/api/models/detail/acme_coder-1/insights/") + + assert resp.status_code == HTTPStatus.OK + data = resp.json() + gen = data["generation"] + assert gen["jobs_total"] == 2 + assert gen["success"]["rate"] == pytest.approx(0.5) + assert gen["functional"]["successes"] == 1 + assert gen["resources"]["cache_hit_ratio"] == pytest.approx(0.5) + assert gen["resources"]["cost_per_kloc"] == pytest.approx(0.25) + assert gen["engines"] == {"opencode": 1} + assert data["ranking"]["positions"]["composite"]["rank"] >= 1 + assert data["catalog"]["catalog_size"] == 1 + + def test_insights_requires_auth(self, client): + LLMModelFactory(model_id="acme/coder-1", canonical_slug="acme_coder-1") + assert client.get("/api/models/detail/acme_coder-1/insights/").status_code == HTTPStatus.UNAUTHORIZED + + @patch("backend.llm_models.services.enrichment.requests.get") + def test_endpoints_route(self, mock_get, client, user): + client.force_login(user) + LLMModelFactory(model_id="acme/coder-1", canonical_slug="acme_coder-1") + mock_get.return_value = _response({"data": {"endpoints": []}}) + + resp = client.get("/api/models/detail/acme_coder-1/endpoints/") + + assert resp.status_code == HTTPStatus.OK + assert resp.json()["available"] is True diff --git a/backend/rankings/services/aggregator.py b/backend/rankings/services/aggregator.py index 5a8f022b..2e9e288c 100644 --- a/backend/rankings/services/aggregator.py +++ b/backend/rankings/services/aggregator.py @@ -23,6 +23,7 @@ from backend.rankings.services.scoring import compute_cost_efficiency_score from backend.rankings.services.scoring import compute_empirical_quality from backend.rankings.services.scoring import compute_mss +from backend.reports.services.loc import loc_from_result_data if TYPE_CHECKING: from django.contrib.auth.models import AbstractBaseUser @@ -88,35 +89,55 @@ def _local_app_stats( "functional_pass_rate": None, } - # LOC + smoke results live in job metrics; roll both up so the empirical - # quality score can normalize findings by code volume, and keep per-job - # values for the variance stats. + # Only each job's latest result per tool — otherwise re-analyzing a job + # double-counts its findings in the rollup (and re-running a single tool + # would drop the others). + from backend.analysis.models import ToolResult + + latest_result_ids = ToolResult.latest_ids_per_job() + analyzed_job_ids = set( + ToolResult.objects.filter(id__in=latest_result_ids).values_list( + "run__generation_job_id", + flat=True, + ), + ) + + # Source LOC (result_data) + smoke results (metrics); roll both up so the + # empirical quality score can normalize findings by code volume, and keep + # per-job values for the variance stats. LOC is summed over analyzed jobs + # only: findings exist only for those, so the density's numerator and + # denominator describe the same apps. functional_rates: dict[str, list[float]] = {} job_loc: dict[Any, int] = {} job_model: dict[Any, str] = {} - for job_id, model_id, metrics in jobs.filter(status="completed").values_list( + for job_id, model_id, metrics, result_data in jobs.filter(status="completed").values_list( "id", "model__model_id", "metrics", + "result_data", ): if model_id not in out or not isinstance(metrics, dict): continue - loc = int(metrics.get("lines_of_code") or 0) - out[model_id]["local_loc"] += loc + # Jobs without stored code (legacy rows) fall back to the generation-time count. + loc = loc_from_result_data(result_data)["total_loc"] or int(metrics.get("lines_of_code") or 0) + if job_id in analyzed_job_ids: + out[model_id]["local_loc"] += loc job_loc[job_id] = loc job_model[job_id] = model_id functional = metrics.get("functional") - if isinstance(functional, dict) and "pass_rate" in functional: - functional_rates.setdefault(model_id, []).append(float(functional["pass_rate"])) + # ``pass_rate`` is null when the suite never ran (infra_error): that + # job is ungraded, so it is left out of the mean rather than scored 0. + if isinstance(functional, dict) and functional.get("pass_rate") is not None: + try: + rate = float(functional["pass_rate"]) + except (TypeError, ValueError): + rate = None + if rate is not None: + functional_rates.setdefault(model_id, []).append(rate) for model_id, rates in functional_rates.items(): out[model_id]["functional_pass_rate"] = round(sum(rates) / len(rates), 4) - # Only each job's latest finished run — otherwise re-analyzing a job - # double-counts its findings in the rollup. - from backend.analysis.models import AnalysisRun - - latest_run_ids = AnalysisRun.latest_ids_per_job() - base = Finding.objects.filter(result__run_id__in=latest_run_ids).exclude( + base = Finding.objects.filter(result_id__in=latest_result_ids).exclude( result__run__generation_job__model__model_id__isnull=True, ) if user is not None and getattr(user, "is_authenticated", False): @@ -160,12 +181,6 @@ def _local_app_stats( # Per-trial variance over analyzed jobs: weighted findings density per # KLOC and smoke pass rate. Jobs never analyzed are excluded — a missing # analysis is not a clean bill of health. - analyzed_job_ids = set( - AnalysisRun.objects.filter(id__in=latest_run_ids).values_list( - "generation_job_id", - flat=True, - ), - ) densities: dict[str, list[float]] = {} for job_id in analyzed_job_ids: model_id = job_model.get(job_id) diff --git a/backend/rankings/tests/test_services.py b/backend/rankings/tests/test_services.py index 6f4a6224..4aebfc60 100644 --- a/backend/rankings/tests/test_services.py +++ b/backend/rankings/tests/test_services.py @@ -522,3 +522,21 @@ def test_aggregate_rankings_variance_none_below_two_trials(): # completed but never analyzed: no density samples at all assert row["variance"]["n_jobs"] == 0 assert row["variance"]["density_per_kloc_stdev"] is None + + +def test_aggregate_rankings_skips_ungraded_pass_rate(): + """An infra_error evaluation stores ``pass_rate: None``: excluded from the mean, not counted as 0.""" + user = UserFactory() + model = LLMModelFactory(model_id="infra/model") + for functional in ( + {"status": "passed", "passed": True, "pass_rate": 0.8}, + {"status": "infra_error", "passed": False, "pass_rate": None}, + ): + GenerationJobFactory( + created_by=user, + model=model, + status=GenerationJob.Status.COMPLETED, + metrics={"lines_of_code": 100, "functional": functional}, + ) + row = next(r for r in services.aggregate_rankings() if r["model_id"] == model.model_id) + assert row["functional_pass_rate"] == 0.8 diff --git a/backend/realtime/authz.py b/backend/realtime/authz.py index f2bbbce3..741578dc 100644 --- a/backend/realtime/authz.py +++ b/backend/realtime/authz.py @@ -12,8 +12,8 @@ | ``endpoint:`` | ``can_use_endpoint``; non-owners get ``:public`` | | ``endpoint::public`` | ``can_use_endpoint`` | | ``analysis:`` | the analysis run's creator or staff | -| ``runtime:`` | the container's creator or staff | -| ``capacity``, ``dashboard``| any authenticated user (aggregate payloads only) | +| ``runtime:`` | the container's creator, its job's owner, or staff| +| ``capacity`` | any authenticated user (aggregate payloads only) | """ from __future__ import annotations @@ -33,7 +33,7 @@ "analysis": re.compile(rf"^analysis:({_UUID}|[1-9][0-9]{{0,18}})$"), "runtime": re.compile(rf"^runtime:({_UUID}|[1-9][0-9]{{0,18}})$"), } -AGGREGATE_CHANNELS = frozenset({"capacity", "dashboard"}) +AGGREGATE_CHANNELS = frozenset({"capacity"}) def _owned(model, object_id: str, user, owner_field: str = "created_by") -> bool: @@ -61,6 +61,16 @@ def _owner_model(prefix: str): return ContainerInstance +def _may_see(prefix: str, object_id: str, user) -> bool: + """Whether *user* may watch the ``:`` object.""" + if prefix == "runtime": + # Same visibility as the runtime API: creator, generation-job owner, or staff. + from backend.runtime.services.container_service import visible_containers + + return visible_containers(user).filter(pk=object_id).exists() + return _owned(_owner_model(prefix), object_id, user) + + def _endpoint_channel(user, endpoint_id: str, *, public: bool) -> str | None: from backend.inference.access import can_use_endpoint from backend.inference.access import is_endpoint_owner @@ -94,7 +104,7 @@ def resolve_channel(user, channel: str) -> str | None: # Owner-checked objects; their primary keys are UUIDs, so integer ids never match. if not _UUID_RE.match(object_id): return None - return channel if _owned(_owner_model(prefix), object_id, user) else None + return channel if _may_see(prefix, object_id, user) else None def authorize_channels(user, channels: list[str]) -> list[str] | None: diff --git a/backend/realtime/events.py b/backend/realtime/events.py index fa2b7f7b..6b1621ae 100644 --- a/backend/realtime/events.py +++ b/backend/realtime/events.py @@ -26,7 +26,7 @@ def publish(channel: str, event: dict) -> None: - ``generation:`` - ``analysis:`` - ``runtime:`` - - ``dashboard`` (broadcast) + - ``capacity`` (broadcast) """ try: payload = json.dumps( diff --git a/backend/realtime/tests/test_events.py b/backend/realtime/tests/test_events.py index 4b7178fb..75f8fd7e 100644 --- a/backend/realtime/tests/test_events.py +++ b/backend/realtime/tests/test_events.py @@ -40,11 +40,11 @@ def test_publish_includes_channel_and_timestamp(self, mock_get_redis): mock_redis = MagicMock() mock_get_redis.return_value = mock_redis - publish("dashboard", {"type": "update"}) + publish("capacity", {"type": "update"}) _, payload_arg = mock_redis.publish.call_args[0] parsed = json.loads(payload_arg) - assert parsed["channel"] == "dashboard" + assert parsed["channel"] == "capacity" assert "published_at" in parsed @patch("backend.realtime.events._get_redis") diff --git a/backend/realtime/tests/test_sse_async.py b/backend/realtime/tests/test_sse_async.py index 6c3eed5b..216aa4e0 100644 --- a/backend/realtime/tests/test_sse_async.py +++ b/backend/realtime/tests/test_sse_async.py @@ -19,6 +19,7 @@ from backend.inference.models import LLMEndpoint from backend.realtime import authz from backend.realtime.subscribers import event_stream +from backend.runtime.tests.factories import ContainerInstanceFactory from backend.users.tests.factories import UserFactory STREAM = "/api/realtime/stream" @@ -72,10 +73,21 @@ def test_foreign_unknown_and_malformed_channels_are_refused(self): ): assert authz.resolve_channel(stranger, channel) is None, channel + def test_runtime_channel_follows_container_visibility(self): + builder, job_owner, stranger = UserFactory(), UserFactory(), UserFactory() + job = GenerationJobFactory(created_by=job_owner) + container = ContainerInstanceFactory(created_by=builder, generation_job=job) + channel = f"runtime:{container.id}" + assert authz.resolve_channel(builder, channel) == channel + assert authz.resolve_channel(job_owner, channel) == channel + assert authz.resolve_channel(UserFactory(is_staff=True), channel) == channel + assert authz.resolve_channel(stranger, channel) is None + def test_aggregate_channels_need_only_a_login(self): user = UserFactory() assert authz.resolve_channel(user, "capacity") == "capacity" - assert authz.resolve_channel(user, "dashboard") == "dashboard" + # Nothing publishes a "dashboard" channel, so it is refused like any unknown name. + assert authz.resolve_channel(user, "dashboard") is None assert authz.resolve_channel(None, "capacity") is None def test_endpoint_non_owners_are_mapped_to_the_public_variant(self): diff --git a/backend/reports/services/generators.py b/backend/reports/services/generators.py index c742e35e..8d0bea21 100644 --- a/backend/reports/services/generators.py +++ b/backend/reports/services/generators.py @@ -153,7 +153,8 @@ def _experiment_stats(jobs_qs, *, exclude_truncated: bool = True) -> dict[str, A if metrics.get("total_tokens"): tokens.append(float(metrics["total_tokens"])) functional = metrics.get("functional") - if isinstance(functional, dict) and "pass_rate" in functional: + # null pass_rate = ungraded (infra_error): excluded, not scored 0. + if isinstance(functional, dict) and functional.get("pass_rate") is not None: functional_rates.append(float(functional["pass_rate"])) functional_passed += 1 if functional.get("passed") else 0 diff --git a/backend/reports/services/loc.py b/backend/reports/services/loc.py index b869af7f..1904cabc 100644 --- a/backend/reports/services/loc.py +++ b/backend/reports/services/loc.py @@ -14,12 +14,24 @@ ".jsx": "//", ".tsx": "//", ".js": "//", + ".mjs": "//", + ".cjs": "//", ".ts": "//", ".vue": "//", ".svelte": "//", ".css": "/*", + ".scss": "//", ".html": " B[workspace lock] + B --> C[materialize code, note truncation] + C --> D{container tools selected?} + D -- yes --> E[ensure container, wipe /work, copy files] + D -- no --> F + E --> F[resolve app URL if a dynamic tool is selected] + F --> G[per tool: skip, AI review or container exec] + G --> H[parse, persist findings and metrics] + H --> I[finalize run status and summary] +``` + +`runner.execute`: + +1. Marks the run `running` and publishes its status on the `analysis:` realtime channel (`/api/realtime/stream`). +2. Takes the workspace lock, a Postgres advisory lock per workspace: runs of one user execute one at a time because they share `/work` and the tools' fixed `/tmp` report paths. +3. Materializes the code. A job's `result_data.files` map is used as-is; older scaffolding/custom jobs contribute their `backend`, `backend_code`, `frontend`, `frontend_code` and `content` strings, with an extension added from the key (`backend_code` -> `.py`, `frontend_code` -> `.jsx`, `frontend` -> `.js`, otherwise `.py`). Parts cut off at generation time (`backend_truncated`, `frontend_truncated`) are recorded in the run's `summary.truncated_inputs`, so syntax errors at the cut are not mistaken for model defects. +4. If any selected tool is a container tool: makes sure the container is running (provisioning it if needed), deletes everything under `/work` and copies the files in, owned by uid 1000. If the container cannot be obtained, or the wipe or the copy fails, the whole run fails: stale files would otherwise be scanned as part of this app. +5. If any selected tool is dynamic, resolves the app's URL once (see the [Analyzer guide](/docs/ANALYZER_GUIDE)). +6. For each slug in `tool_slugs`, in order, creates a `ToolResult` and either skips it (unknown or disabled tool, dynamic tool without a running app, no applicable files), runs the AI reviewer (`services/ai_runner.py`), or renders `run_cmd` (`{target}` -> `/work`, `{url}`, `{config:}`) and executes it in `/work` under `timeout -s KILL `. +7. Parses stdout with the parser registered under the tool's `parser_key`, records `summary.tool_version` (container tools) and `summary.config` (defaults merged with the user's options), and stores the findings, with paths normalized (`/work/` and `./` prefixes stripped), and the metrics. +8. Finalizes the run: `failed` when no tool completed and at least one failed, `partial` when some failed, otherwise `completed`. Skipped tools never fail a run, so a run whose tools were all skipped is `completed`. `summary` holds `total_findings`, `severity_counts` (normalized), `reported_severity_counts` (levels as the tools printed them, case-folded), `unrated_findings` (findings of tools that rate nothing), `metrics_by_tool`, `tools_run`, `tools_completed`, `tools_failed` and, when relevant, `truncated_inputs`. Each tool result's `summary` carries its own `finding_count`, `severity_counts` and `reported_severity_counts`. + +`POST /api/analysis/runs//cancel/` marks a `pending` or `running` run `cancelled`, and `cancelled` is final: the runner never overwrites it. Cancel also kills the tool that is running. Every tool command carries a `: llmlab-analysis-run:;` prefix, and cancel SIGKILLs the process group of every process in the workspace whose command line has that marker. The runner checks for cancellation before each tool and after each tool returns. The killed tool and every tool not yet started are recorded as `skipped` with `summary.skip_reason: cancelled`, which does not count as measured. The run's `summary` still describes what ran before the cancel. An AI review in flight (an HTTP call) cannot be interrupted, so its response is discarded when it arrives. Results of cancelled runs never feed study tables, reports or rankings. + +`POST /api/analysis/runs/` with a `generation_job_id` returns 404 unless the job is yours. Staff can analyze any job. + +### Runs API + +| Endpoint | Purpose | +| --- | --- | +| `GET /api/analysis/runs/` | List with `status` (comma-separated; `active` = pending + running), `search` (name, app, model, stack, ID prefix), `batch_id`, `generation_job_id`, `tool_slug`, `sort` (`created_desc`, `created_asc`, `name_asc`, `duration_desc`, `findings_desc`). Items carry the `target` (app, model, stack, or pasted code), `phase` and, for active runs, `progress` (tools done / running, findings so far). | +| `GET /api/analysis/runs/stats/` | Run counts per status | +| `POST /api/analysis/runs/` | One run (a job or pasted code); unknown tool slugs are a 400 | +| `POST /api/analysis/runs/batch/` | One run per job (at most 100), queued on one thread | +| `POST /api/analysis/runs/preview/` | Free dry run: per tool, on how many targets it would run or be skipped (no file it reads, app not running); per target, files, lines and whether its app container runs | +| `POST /api/analysis/runs//rerun/` | A new run on the same code: all tools, `tool_slugs`, or `only_unmeasured` (failed, cancelled, unavailable, app not running) | +| `POST /api/analysis/runs/bulk/` | `cancel` or `delete` several runs; active runs are never deleted | +| `DELETE /api/analysis/runs//` | 409 while the run is active (cancel it first) | +| `GET /api/analysis/runs//results//` | One tool's raw output: command, exit code, stored stdout (first 30,000 characters), target URL | +| `GET /api/analysis/runs//findings/` | Findings with `tool_slug`, `tool_severity` (a reported label or `unrated`), `category`, `file_path`, `rule_id`, `search`, `sort` (`severity`, `file`, `tool`, `rule`) | +| `GET /api/analysis/runs//findings/facets/` | Counts per tool, reported level, category, file and rule (top 50) | -1. Marks the run `running` and publishes to the `analysis:` SSE channel. -2. Materializes the target code into files and records which parts were truncated at generation time (surfaced on the result so you don't chase phantom syntax errors). -3. Ensures the workspace container is ready, wipes `/work` from the previous run, and copies the files in. -4. Runs each tool: `run_cmd` with `{target}` replaced by `/work`, bounded by the tool's `run_timeout`. AI tools skip the container and go through `ai_runner.py` instead. -5. Parses stdout with the parser registered under the tool's `parser_key` (`parsers.py`), persists findings and metrics, and finalizes the run status from the per-tool statuses (`completed`, `partial`, or `failed`). +## Severity as reported -Per-user tool configuration (from the workspace's installed tools, shaped by each tool's `config_schema`) is resolved at run time and passed to the tool. +Tools rate findings in their own words (bandit `HIGH`, ESLint `error`, pylint `convention`, ZAP `Medium`, npm audit `moderate`) or not at all (ruff, flake8, vulture, gitleaks, jscpd, pip-audit, …). Every finding stores both: + +- `tool_severity`: the label exactly as the tool printed it, or empty when the tool rates nothing. This is what the analysis pages display; a finding of an unrated tool shows no level. +- `severity`: a normalization to critical / high / medium / low / info. For unrated tools it is a heuristic (for example ruff and flake8 codes are mapped by family). Statistics, study tables, reports and automation thresholds still use it. + +Parsers also store what the tools report about each finding in `tool_specific_data` under standard keys: `rule_name`, `docs_url`, `references`, `cwe`, `owasp`, `end_line`, `end_column`, `fix`, `tool_confidence`; for dependency vulnerabilities `package`, `installed_version`, `fixed_versions`, `vuln_id`, `aliases`, `cvss_score`, `cvss_vector`, `manifest`; for scanners `url`, `method`, `param`, `evidence`, `attack`, `instances`, `other_info`; `secret_type` (never the secret), `complexity`, `function`, `nloc`, and `duplicate_of` for clones. Older tool-specific keys (trivy `fixed_version`, osv `fix_versions`, jscpd `second_file`) are kept next to them. + +## What a tool result means + +A tool that measured nothing must never pass for a clean result, because zero findings reads as "no defects" in every comparison. Each `ToolResult` is one of: + +| Status | Meaning | Counts as measured? | +| --- | --- | --- | +| `completed` | The tool ran and its report parsed. Zero findings is a real zero. | Yes | +| `skipped`, `summary.skip_reason: not_applicable` | The app has none of the files the tool reads (no `requirements*.txt` for pip-audit, no `.js` for ESLint). | Yes (the true count is zero) | +| `skipped`, `skip_reason: app_not_running` | A dynamic tool, and the run analyzes pasted code or the app's container was not running. | No | +| `skipped`, `skip_reason: unavailable` | The tool is unknown or disabled in the catalog. | No | +| `skipped`, `skip_reason: cancelled` | The run was cancelled before this tool ran, or while it ran (the tool was killed). | No | +| `failed` | Killed (exit 137: `run_timeout` or out of memory), not installed (exit 126/127), a non-zero exit with nothing parsed, no parser registered for the `parser_key`, a parser exception, every manifest of a multi-manifest tool failed, the workspace was unavailable, or an AI response that was not a findings array. | No | + +A `completed` result can still carry `summary.warnings`, for example when one of several `requirements.txt` files could not be audited, or when the AI reviewer had to leave files out. Every container result stores the exact command and exit code in `raw_output` (`command`, `exit_code`, the first 30,000 characters of stdout, and `target_url` for dynamic tools), the installed tool version in `summary.tool_version` and the effective options in `summary.config`, so findings stay attributable after the workspace is updated. The runs API returns each result's `summary`, `metrics` and `error_message`; `raw_output` stays on the row (visible in the Django admin). + +When a dependency cannot be resolved, pip-audit and npm audit audit nothing for that manifest. The run command echoes the resolver errors as `!! ` lines, and the parser turns them into `nonexistent-package` findings (high severity: the hallucinated-dependency or "slopsquatting" pattern, since anyone can register that name) or `nonexistent-version` findings (medium), next to a "not analyzed" warning for the manifest. pip stops at the first unresolvable requirement, so at most one is reported per file. + +These findings are always stored. If at least one other manifest of the app was audited, the result is `completed` with the warning. If no manifest could be audited (the common case of a single `requirements.txt` or `package.json`), the result is `failed`: the vulnerability audit did not run, so the result is not a measurement. Its error message and `summary.warnings` name the unresolvable requirement, and the `nonexistent-*` findings are stored on the failed result anyway (`summary.finding_count`, `summary.severity_counts`), so they appear in the run's findings and in `summary.total_findings`. + +> [!NOTE] +> Study tables count a tool only for apps it measured, so a failed audit is never read as zero vulnerabilities. For the same reason, the `nonexistent-*` findings on such a failed result do not reach the common-instrument tables: pip-audit or npm-audit appears under `coverage.tools_partial` with a `failed` count. To count hallucinated dependencies in every app, export with `--all-tools` or read the findings directly. ## The AI reviewer -`llm_review` is a tool with `kind: ai`: instead of a container command, `ai_runner.py` sends the code to an LLM through OpenRouter using the run owner's API key — the same key resolution as generation. Its findings land in the same `Finding` table as every other tool, so AI review is comparable and exportable like the rest. +`llm_review` is the one tool with `kind: ai`. Instead of a container command, `ai_runner.py` sends the code to an LLM with temperature 0.2 and asks for a JSON array of findings. + +- **Model.** The tool's `model` option (default `anthropic/claude-opus-5.5`) and `max_tokens` (default 8000). A model id that exists in the user's model catalog resolves through the inference layer, which covers OpenRouter and local endpoints ([Local models](/docs/LOCAL_MODELS)); any other id is sent to OpenRouter with the run owner's API key, using the same key resolution as generation. +- **Input.** Files are introduced by `# ===== =====` markers, source files first. Lockfiles and `.min.` bundles are skipped, files are never cut mid-way, and the bundle stops at 60,000 characters; files that do not fit are listed in `summary.files_not_reviewed` and in a warning. +- **Output.** Severities and categories are normalized to the platform vocabulary (`moderate` -> `medium`; categories `security`, `quality`, `performance`, `style`, `best_practice`). Complete findings are salvaged from a response cut off at `max_tokens`; a response with no parsable array is a `failed` result, never a clean one. +- **Status of its findings.** They land in the same `Finding` table and are exportable like the rest, but they are opinions, not measurements: study tables keep them out unless `include_ai` is set, the review sample puts them on their own sheet, and rankings collect them apart from the empirical score. + +## Measurement rules for comparisons + +- **Common instrument set.** Study tables (`statistics/services/study.py`, used by the study report, `GET /api/statistics/study/breakdowns/` and `export_study_tables`) count findings only from tools that *measured* every analyzed app in scope. The `coverage` block lists `tools_common` and, under `tools_partial`, each dropped tool with `apps_measured`, `apps_missing` and a count per reason (`failed`, `app_not_running`, ...). Without this, an app where Semgrep timed out, or that was not running for the DAST scanners, would look cleaner than its peers. Pass `common_tools_only=false` (CLI: `--all-tools`) to count everything. See [Study tables](/docs/STUDY_TABLES). +- **Source LOC.** Per-kLOC rates divide by source LOC (`reports/services/loc.py`): non-blank lines that do not start with the file type's comment marker, in `.py`, `.js`, `.jsx`, `.ts`, `.tsx`, `.mjs`, `.cjs`, `.vue`, `.svelte`, `.css`, `.scss`, `.html` and `.sh` files, excluding `.min.` bundles and `node_modules/`. Lockfiles, manifests, docs and configs are not code the model wrote; a generated `package-lock.json` alone can outweigh the whole app. The generation-time `metrics.lines_of_code` is used only as a fallback in rankings, for legacy jobs without stored code. +- **Cross-tool deduplication** (`statistics/services/dedup.py`) merges one defect reported by several tools: Bandit, Ruff `S` rules and Semgrep; the lint rules Ruff, flake8, Pylint, Vulture, Mypy and ESLint share; the secret scanners; eslint-react's `react/no-danger` and the eslint-security rules that Semgrep also covers (eval, child processes, non-literal and unsafe regular expressions, pseudo-random bytes, non-literal file paths); the known-vulnerability scanners (pip-audit, npm audit, osv-scanner, Trivy, retire.js), which match on the vulnerable package within one project directory because each names the advisory differently (PYSEC/GHSA/CVE); and DAST findings about the same missing security header or CORS problem. +- **Defaults that keep counts about the code.** Three are per-user options, recorded per result in `summary.config`: Bandit skips B104 (the template requires binding `0.0.0.0` in the container); Pylint disables `import-error`, the docstring messages, `too-few-public-methods` and `unused-argument`; npm audit counts production dependencies only. The rest are fixed in each tool's `run_cmd`: Vulture ignores framework entry points (route, validator and hook decorators) and signature-mandated names, ESLint marks JSX-referenced components as used, Ruff and mypy keep their caches out of `/work` (mypy runs from the app root with `--explicit-package-bases`, so multi-module apps type-check), osv-scanner checks declared versions only (`--no-resolve`), Nuclei's tech-detect and ZAP's fingerprint alert 10109 are dropped as non-defects, and jscpd, codespell and the AI reviewer skip lockfiles and minified bundles. + +## Known limits + +- ESLint, eslint-security and eslint-react read `.js`, `.jsx`, `.cjs` and `.mjs` only. `.ts`/`.tsx` files reach only Semgrep and Lizard, and `.vue`/`.svelte` single-file components are counted in LOC but linted by no tool, which understates issue density for those stacks. +- Dependency audits resolve unpinned requirements at analysis time, against the advisory databases of that day. Compare apps analyzed in the same period, or re-analyze them together. +- Tools installed from release binaries (Trivy, osv-scanner, Gitleaks, Hadolint, Nuclei, ZAP's runtime) download `linux-amd64`/`x86_64` builds and do not install on an arm64 Docker host. ## Where results go -Findings and metrics feed the statistics dashboards, [model rankings](/docs/ARCHITECTURE), and reports; aggregations count only the **latest completed run per generation job**, so re-analyzing never double-counts. Raw findings are downloadable as CSV/JSON/SARIF via the export endpoints ([API reference](/docs/api-reference)). +Findings and metrics feed the Statistics page, [model rankings](/docs/ARCHITECTURE), reports and the [study tables](/docs/STUDY_TABLES). Rankings, the study tables and the study report count **one result per (generation job, tool)** (`ToolResult.latest_ids_per_job`): the latest result from a finished run, where a result that measured the app (`completed`, or `skipped` as `not_applicable`) wins over a later one that did not (`failed`, `app_not_running`). Re-analyzing never double-counts, and re-running one tool, or only the AI reviewer, replaces just that tool's findings instead of discarding the app's other results. The legacy report generators, the model page insights, execution accounting (per run by design) and the review sample still pick the latest finished run per job (`AnalysisRun.latest_ids_per_job`). The Statistics dashboard's overview, severity and top-findings tiles count the findings of every run. Raw findings are downloadable as CSV, JSON or SARIF from `/api/export/findings.csv`, `/api/export/findings.json` and `/api/export/findings.sarif` ([API reference](/docs/api-reference)). -For the tool catalog and how to add a tool, see the [Analyzer guide](/docs/ANALYZER_GUIDE). +For the tool catalog, the YAML format and how to add a tool, see the [Analyzer guide](/docs/ANALYZER_GUIDE). diff --git a/docs/ANALYZER_GUIDE.md b/docs/ANALYZER_GUIDE.md index 25e22ccb..5740b70c 100644 --- a/docs/ANALYZER_GUIDE.md +++ b/docs/ANALYZER_GUIDE.md @@ -1,29 +1,94 @@ # Analyzer guide -The analysis tool catalog: what each tool does, how tools are defined in YAML, and how to add one. +The 33-tool analyzer catalog and shop: what each tool scans, how tools fit your apps, the tool YAML format and how to add a tool. ## Tool catalog -Fourteen tools, defined one YAML file per tool in `backend/analysis/data/tools/`: - -| Tool | Category | Target | Finds | -| --- | --- | --- | --- | -| bandit | security | Python | Insecure patterns, known-vulnerable calls | -| semgrep | security | polyglot | Rule-based security/correctness patterns | -| detect-secrets | secrets | any | Hardcoded credentials | -| gitleaks | secrets | any | Leaked secrets and keys | -| ruff | lint | Python | Bugs, unused imports, style | -| pylint | lint | Python | Deeper static analysis | -| mypy | lint | Python | Type errors | -| eslint | lint | JS/TS | Bugs and style in frontend code | -| codespell | lint | any | Typos in code and comments | -| vulture | lint | Python | Dead code | -| hadolint | lint | Dockerfile | Dockerfile mistakes | -| radon | performance | Python | Complexity/maintainability **metrics** | -| jscpd | performance | polyglot | Copy-paste duplication **metrics** | -| llm_review | ai | any | LLM code review via OpenRouter | - -Most tools emit findings only; radon and jscpd are `output_kind: mixed` — findings plus a structured metrics JSON channel on the tool result. Display categories come from `catalog.yaml`. +The catalog has 33 tools, one YAML file per tool in `backend/analysis/data/tools/`: 29 deterministic static tools, 1 AI reviewer and 3 dynamic (DAST) scanners. The shop groups them three ways: by **category** (what kind of problem), by **mode** (static or dynamic) and by the **technologies** they scan. The vocabulary for all three lives in `data/tools/catalog.yaml` and is served by `GET /api/analyzers/catalog/`. + +"Reads" is the tool's `file_patterns`: a run whose code contains none of these files skips the tool as not applicable (see Applicability below). "always" means the tool declares no patterns. + +### Static tools (read the source) + +| Tool | Category | Scans | Reads | Finds | +| --- | --- | --- | --- | --- | +| bandit | security | Python, Flask, Django | `*.py` | Insecure calls, hardcoded secrets, weak crypto | +| trivy | security | pip/PyPI, npm, Docker, Kubernetes, Terraform | always | Vulnerable packages, Dockerfile/IaC misconfigurations, secrets | +| semgrep | security | Python, JavaScript, Flask, FastAPI, Django, Express | `*.py`, `*.js`, `*.jsx`, `*.ts`, `*.tsx` | Rule-based security and correctness patterns (`p/python`, `p/javascript`) | +| eslint-security | security | JavaScript, Express | `*.js`, `*.jsx`, `*.cjs`, `*.mjs` | eval, ReDoS, unsafe child processes, timing-unsafe comparisons | +| pip-audit | dependencies | Python, pip/PyPI | `requirements*.txt` | Vulnerable Python packages; nonexistent packages and versions | +| npm-audit | dependencies | JavaScript, npm | `package.json` | Vulnerable JavaScript packages; nonexistent packages and versions | +| osv-scanner | dependencies | pip/PyPI, npm | `requirements*.txt`, `package.json`, lockfiles | Vulnerable packages from OSV.dev, at the declared versions (`--no-resolve`: its own resolution invented old versions and bogus critical CVEs) | +| deptry | dependencies | Python, pip/PyPI | `requirements*.txt` | Missing, unused and misplaced Python dependencies | +| retire | dependencies | JavaScript, HTML, npm | `*.js`, `*.html`, `package.json` | Vulnerable JS libraries, including vendored copies | +| depcheck | dependencies | JavaScript, npm | `package.json` | Missing and unused JavaScript dependencies | +| ruff | lint | Python | `*.py` | Bugs, unused imports, style | +| flake8 | lint | Python | `*.py` | pyflakes + pycodestyle + mccabe | +| eslint | lint | JavaScript | `*.js`, `*.jsx`, `*.cjs`, `*.mjs` | Undefined names, unused variables | +| eslint-react | lint | JavaScript, React | `*.js`, `*.jsx`, `*.cjs`, `*.mjs` | Rules of Hooks, missing keys, unsafe links, `dangerouslySetInnerHTML` | +| pylint | lint | Python | `*.py` | Errors, code smells, refactoring candidates | +| mypy | lint | Python | `*.py` | Type errors | +| vulture | lint | Python | `*.py` | Dead code | +| codespell | lint | any source | always | Typos in code and comments | +| djlint | lint | HTML, Jinja/Django templates | `*.html`, `*.jinja`, `*.jinja2`, `*.j2` | Template accessibility and markup problems | +| jscpd | lint | any source | always | Copy-paste duplication (**metrics** + findings) | +| hadolint | lint | Docker | `Dockerfile`, `Dockerfile.*`, `*.dockerfile` | Dockerfile mistakes | +| shellcheck | lint | Shell | `*.sh`, `*.bash` | Quoting, globbing and portability bugs | +| htmlhint | lint | HTML | `*.html` | Missing doctype/title, duplicate ids, unclosed tags | +| stylelint | lint | CSS | `*.css`, `*.scss` | Invalid CSS, duplicate selectors | +| gitleaks | secrets | any source | always | Leaked secrets and keys | +| detect-secrets | secrets | any source | always | Hardcoded credentials | +| radon | performance | Python | `*.py` | Cyclomatic complexity, maintainability (**metrics** + findings) | +| lizard | performance | Python, JavaScript | `*.py`, `*.js`, `*.jsx`, `*.ts`, `*.tsx` | Complexity and function length (**metrics** + findings) | +| perflint | performance | Python | `*.py` | Performance anti-patterns | +| llm_review | ai | any source | always | LLM code review (`kind: ai`, see [The AI reviewer](/docs/ANALYSIS_PIPELINE)) | + +Every other tool is `output_kind: findings`; radon, jscpd and lizard are `output_kind: mixed`: findings plus a structured metrics JSON on the tool result (radon's `average_complexity` and `max_complexity` feed the study report's code metrics). + +### Dynamic tools (probe the running app, DAST) + +| Tool | Category | Scans | Finds | Timeout | +| --- | --- | --- | --- | --- | +| zap | security | Web app, REST API | OWASP ZAP: spider, passive and active scan (headers, XSS, SQLi, leaks, misconfigurations) | 1200 s | +| nuclei | security | Web app, REST API | Template checks: misconfigurations, exposures, headers, CORS, common injections | 900 s | +| wapiti | security | Web app, REST API | Crawls and fuzzes forms and parameters: XSS, SQL and command injection, CRLF, CSRF, cookie flags | 900 s | + +Dynamic tools need the generated app's container to be **running** when the analysis starts (start it from the application page). The runner resolves the target from the run's generation job only: a running `ContainerInstance` of that job, created by the run's owner or belonging to the owner's job, reached from the analyzer workspace at `http://:8000` (by name on `DOCKER_APPS_NETWORK`, else by its bridge IP). If the app isn't running, or the run analyzes pasted code, the result is `skipped` with `skip_reason: app_not_running`; the analysis wizard drops dynamic tools for pasted code. Findings point at URL paths (`file_path` = `/api/items`), with the full URL, method and parameter in `tool_specific_data`. + +Experiment post-run analysis does not offer dynamic tools: evaluation deploys each app in a throwaway sandbox and removes it, so no app is running when the post-run analysis starts. + +## The analyzer shop + +The Analyzers page (`/analyzers`) has three tabs: + +| Tab | What it does | +| --- | --- | +| Shop | The catalog with search plus category, mode and technology filters, per-tool fit counts, single install and an "Install N shown" bulk install | +| Installed | Installed tools with status, version and install log; per-tool options (from `config_schema`), test, update, "update all", uninstall | +| Workspace | Provision, start, stop or delete the analyzer workspace ([Analysis pipeline](/docs/ANALYSIS_PIPELINE)) | + +### Choosing tools for your apps + +The shop's **What your generated apps contain** panel (`GET /api/analyzers/coverage/`, `services/coverage.py`) scans your 200 most recent completed generation jobs for technologies (file extensions, manifests, framework imports, template syntax) and shows, per technology, how many apps contain it and which installed tools scan it. A technology no installed tool targets is flagged as a gap. Each tool row shows how many of your apps contain one of its technologies. A tool that fits none of them (Hadolint, when generated apps contain no Dockerfile) can only report zero findings, which says nothing about the code's quality. + +### Tool versions + +Each YAML pins a `version`; an install records the catalog version it was made from (`catalog_version`) and the version its `verify_cmd` reports (`installed_version`). When the catalog pin changes, the installed tool shows `update_available`. Updating a tool can change what it reports, so the shop asks for confirmation before a version update: finish a study on one version. Each tool result records the version it ran with in `summary.tool_version`. + +### Analyzer API + +All under `/api/analyzers/` (see the [API reference](/docs/api-reference)): + +| Endpoint | Purpose | +| --- | --- | +| `GET tools/`, `GET tools//` | Enabled catalog tools with the caller's install state | +| `GET catalog/` | Categories, modes, technology groups and technologies | +| `GET coverage/` | Technology and per-tool fit counts over your recent apps | +| `GET workspace/`, `POST workspace/provision/`, `start/`, `stop/`, `DELETE workspace/` | Workspace lifecycle | +| `GET workspace/tools/`, `POST workspace/tools/` (`tool_slug`), `POST workspace/tools/bulk/` (`tool_slugs`) | List and install; installs run in the background | +| `POST workspace/tools/update-all/` (`only_outdated`), `POST workspace/tools//update/` | Re-run installs to update | +| `PUT workspace/tools//` (`config`), `DELETE workspace/tools//` | Per-user options, uninstall | +| `POST workspace/tools//test/` | Verify the install and run the tool on its sample | ## Tool YAML format @@ -32,39 +97,103 @@ Abridged from `ruff.yaml`: ```yaml slug: ruff name: Ruff -category: lint # display category from catalog.yaml +description: >- + Extremely fast Python linter. +category: lint # categories in catalog.yaml kind: container # container = runs in the workspace; ai = ai_runner -target_language: python -install_cmd: "pip install --user 'ruff==0.6.*'" +target_language: python # python | javascript | html | css | shell | any +analysis_mode: static # static (reads {target}) | dynamic (probes {url}) +technologies: [python] # slugs from catalog.yaml `technologies` +icon: sparkles +version: "0.6" # catalog pin; a change offers an update to installs +docs_url: https://docs.astral.sh/ruff/ +install_cmd: "pip install --user --upgrade 'ruff==0.6.*'" verify_cmd: "ruff --version" -run_cmd: "ruff check {target} --output-format json" # {target} -> /work -parser_key: ruff # which parser in services/parsers.py reads stdout +uninstall_cmd: "pip uninstall -y ruff" +run_cmd: "sel={config:select}; ruff check {target} --no-cache --output-format json ${sel:+--select \"$sel\"}" +parser_key: ruff # which registered parser reads stdout +file_patterns: ["*.py"] # files the tool reads; none present -> skipped run_timeout: 90 -config_schema: # renders as a settings form in the UI +install_timeout: 180 +display_order: 20 +config_schema: # renders as an options form on the installed tool - name: select type: string label: Rule selection default: "" -sample_code: | # used by the "test tool" action + placeholder: "E,F,W" +default_config: + select: "" +sample_code: | # used by the test action (static tools) import os ... ``` +| Field | Default | Notes | +| --- | --- | --- | +| `slug`, `name`, `description` | file stem for `slug` | `slug` is the upsert key | +| `category` | `lint` | `security`, `dependencies`, `lint`, `secrets`, `performance`, `ai` | +| `kind` | `container` | `ai` skips install and runs through `ai_runner.py` | +| `analysis_mode` | `static` | `dynamic` tools get `{url}` and ignore `file_patterns` | +| `output_kind` | `findings` | `findings`, `metrics` or `mixed` | +| `run_timeout`, `install_timeout` | 120, 300 seconds | `run_timeout` is enforced with `timeout -s KILL` | +| `sample_filename` | `sample.py` or `sample.js` | For filename-sensitive tools (`Dockerfile`, `requirements.txt`) | +| `display_order` | 100 | Shop ordering | +| `is_enabled` | true | Disabled tools leave the shop and are skipped as `unavailable` | + +`config_schema` entries take `name`, `type` (`string`, `number`, `boolean` or `select` with `options: [{value, label}]`), `label`, `description`, `default` and optionally `placeholder`, `min` and `max`. + +`run_cmd` placeholders: + +- `{target}`: the code directory (`/work`). +- `{url}`: the running app's base URL, without a trailing slash (dynamic tools). +- `{config:}`: the user's value for a `config_schema` field, falling back to `default_config`. Values are shell-quoted (booleans become `true`/`false`, lists are comma-joined), so they can be used as plain arguments. Every placeholder needs a `config_schema` field and a `default_config` entry (`tests/test_seed.py` checks this). + +Commands run as the unprivileged `analyzer` user under `sh -c` (no root, no apt). Tools with many Python dependencies (djlint, wapiti) install into their own venv under `~/.local/opt/` and symlink the entry point into `~/.local/bin`, so their pins never clash with other pip tools. Release binaries are pinned to a version, and every one of them (Trivy, osv-scanner, Gitleaks, Hadolint, Nuclei and ZAP with its JRE) is verified with `sha256sum -c` against the checksum the release publishes before use. A test (`test_release_binary_downloads_are_checksummed`) fails for a new tool that downloads a release asset without a check. + +### Applicability: `file_patterns` + +`file_patterns` lists the files a tool reads, as case-sensitive globs. A pattern without `/` matches the file name (`"*.py"`, `"Dockerfile"`); a pattern with `/` matches the path relative to the code root (`"templates/*.html"`). If none of a run's files match, the tool is not run: its result is `skipped` with `summary.skip_reason = "not_applicable"`, instead of a clean-looking 0 findings that would count as "no issues" in model comparisons. An empty list means always applicable, and dynamic tools ignore the field (they scan the running app). + +### Failure rules (no silent clean results) + +The runner marks a tool result `failed`, never `completed` with 0 findings, when: + +- the command exits **137** (killed by the `run_timeout` or out of memory), +- it exits **126/127** (tool not installed or not executable: reinstall it), +- it exits non-zero and the parser extracted neither findings nor metrics, +- no parser is registered under the `parser_key`, or the parser raises an exception, or +- the parser itself reports an error (every manifest of a sectioned tool failed). + +A non-zero exit *with* parsed findings is normal (many linters exit 1 when they find something). Every result records the exact command in `raw_output.command` and the effective configuration (defaults plus the user's options) in `summary.config`, alongside `summary.tool_version`. + +### Tools that scan several manifests + +Dependency tools (pip-audit, npm audit, deptry, depcheck) run once per manifest and print one `### ` header followed by that manifest's JSON report. Register their parser with `register(key, sectioned=True)`; the registry then checks the output per section: + +- an empty or unparsable section counts as "not analyzed" and is listed in `summary.warnings`; +- if every section fails, the whole result is `failed`, but any findings the parser derived from the failure are still stored on it; +- the run command may print `!! ` lines inside a section to say why that manifest failed. pip-audit and npm audit use these to turn resolution errors into `nonexistent-package`/`nonexistent-version` findings: a requirement on a package or version that doesn't exist is a hallucinated dependency (they are stored even when the result is `failed`; see [What a tool result means](/docs/ANALYSIS_PIPELINE)). + +### Keep `/work` clean + +`/work` holds only the code under analysis, and tools such as detect-secrets and jscpd scan everything in it. A tool must not write caches or reports there: point caches at `/tmp` (mypy uses `--cache-dir /tmp/mypy-cache`, ruff runs with `--no-cache`) and write report files under `/tmp` (`/tmp/jscpd/`, `/tmp/gitleaks-report.json`). + ## Seeding -The catalog upserts into the `AnalyzerTool` table, keyed by slug: +The YAML files upsert into the `AnalyzerTool` table, keyed by slug (`backend/analysis/seeding.py`): -- automatically after every `migrate` (post-migrate hook in the analysis app), and +- automatically after every `migrate` (post-migrate hook in the analysis app; a seeding error is logged and never breaks `migrate`), and - manually via `just manage seed_analysis_tools`. -Seeding is idempotent; edits to the YAML land on the next migrate or manual seed. In tests, run `just test --create-db` after changing tool YAML — the reused test database keeps the old catalog. +Seeding is idempotent; edits to the YAML land on the next migrate or manual seed. It never deletes a tool whose YAML file was removed: set `is_enabled: false` to retire one. The seed logs a warning for technology slugs missing from `catalog.yaml` and for a container tool whose `parser_key` has no registered parser. In tests, run `just test --create-db` after changing tool YAML: the reused test database keeps the old catalog. ## Adding a tool -1. Drop a new `.yaml` in `backend/analysis/data/tools/` with the fields above. The tool must be installable inside the analyzer-base container by `install_cmd` and print machine-readable output from `run_cmd`. -2. If existing parsers don't fit its output, register a parser under your `parser_key` in `backend/analysis/services/parsers.py`. -3. Reseed, install the tool into your workspace from the Analyzers page, and use the test action (runs `run_cmd` against the YAML's `sample_code`) to check the parse end to end. +1. Drop a new `.yaml` in `backend/analysis/data/tools/` with the fields above. The tool must be installable inside the analyzer-base container by `install_cmd` and print machine-readable output from `run_cmd`. Add any new technology to `catalog.yaml`, and set `file_patterns` so the tool is skipped (not "clean") on apps it can't read. +2. If no existing parser fits its output, register one under your `parser_key` with `@register("")` in `services/parsers.py`, `extra_parsers.py` (dependency, extra lint and web parsers) or `scanner_parsers.py` (DAST and multi-ecosystem scanners). A parser returns a list of `FindingData` or a `ParsedOutput` (findings, metrics, warnings, error). Set `tool_specific_data` keys the deduplicator understands where they apply: `cwe`, `end_line`, `package` (vulnerability scanners) and `issue_key`. +3. Reseed, install the tool into your workspace from the Analyzers page, and use the test action to check the parse end to end: it copies `sample_code` (as `sample_filename`) into `/tmp/analyzer-test` and runs `run_cmd` there with a timeout of at most 60 seconds. For a dynamic tool the test action only verifies the install; run it against a started app to check the parse. For the AI reviewer it makes a real model call. -## Per-run configuration +## Per-user configuration -Each `config_schema` entry becomes a form field on the installed tool. Values are stored per user on the workspace's installed tool and passed to the tool at run time — see [Analysis pipeline](/docs/ANALYSIS_PIPELINE) for where that happens. +Each `config_schema` entry becomes a form field on the installed tool. Values are stored per user on the workspace's `InstalledTool.config` (seeded from `default_config` at install) and reach the tool through `{config:}` placeholders in `run_cmd`, or directly for the AI reviewer (`model`, `max_tokens`). Tools with options today: bandit (`confidence`, `skip`), pylint (`disable`), ruff (`select`), npm-audit (`production_only`), trivy (`scanners`), djlint (`profile`), llm_review (`model`, `max_tokens`), zap (`spider_minutes`, `scan_minutes`), nuclei (`tags`) and wapiti (`modules`, `max_scan_seconds`). See [Analysis pipeline](/docs/ANALYSIS_PIPELINE) for the rest of the run. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 39457f93..4265ffc6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,107 +1,124 @@ # Architecture -How the pieces fit together: the Django API, the SvelteKit frontend, the Docker surfaces for untrusted code, and where each kind of work runs. +How the pieces fit together: the Django API, the SvelteKit frontend, the executor, the Docker sandbox for untrusted code, and where each kind of work runs. ## System overview ```mermaid flowchart LR B[Browser] --> FE["SvelteKit frontend :8000"] - FE -->|"/api proxy"| DJ["Django + django-ninja :8001"] + FE -->|"/api, /_allauth, /admin, /media, /apps"| DJ["django (web, ninja API) :8001"] + MC["MCP clients / scripts"] -->|"Bearer token"| DJ + RL["Relay on a GPU box"] -->|"/api/relay/v1/"| DJ DJ --> PG[(Postgres)] - DJ --> RD[(Redis)] - DJ -->|threads| GEN["Generation / analysis runs"] - RD --> CW["Celery worker + beat"] - DJ -->|"mTLS :2376"| DK["apps-dind (sandbox Docker daemon)"] + DJ -->|"doorbell, pub/sub"| RD[(Redis)] + EX["executor (run_executor)"] --> PG + RD --> EX + RD --> CW["celeryworker + celerybeat"] + EX -->|"mTLS :2376"| DK["apps-dind (sandbox Docker daemon)"] + DJ -->|"mTLS :2376"| DK DK --> AW["Analyzer workspaces (per user)"] - DK --> SA["Generated sample apps"] - GEN --> OR[["OpenRouter"]] + DK --> SA["Generated apps, agent sandboxes"] + EX --> OR[["OpenRouter / cloud / local LLM endpoints"]] ``` -The browser only ever talks to the SvelteKit server, which proxies `/api` to Django. Django owns all state (Postgres) and publishes progress events to Redis, which the frontend consumes over an SSE stream. +The browser only talks to the SvelteKit server, which proxies `/api/`, `/_allauth/`, `/admin/`, `/media/` and `/apps/` to Django (`frontend/src/hooks.server.ts`). Django owns all state in Postgres; Redis carries executor doorbells, Celery messages and the pub/sub events behind the SSE stream. ## Backend apps -Fifteen Django apps under `backend/`, one responsibility each: +Eighteen Django apps under `backend/` (plus `backend/contrib/sites` and the non-app helpers in `backend/common/`): | App | Role | | --- | --- | -| `users`, `tokens`, `credentials` | Accounts (allauth headless, MFA), Bearer API tokens, encrypted provider keys | -| `llm_models` | Model catalog synced from OpenRouter | -| `generation` | Generation jobs and templates — see [Generation process](/docs/GENERATION_PROCESS) | -| `analysis` | Tool catalog, analyzer workspaces, runs, findings — see [Analysis pipeline](/docs/ANALYSIS_PIPELINE) | -| `statistics`, `rankings`, `reports` | Aggregations, model scoring/benchmarks, saved reports (`rpt_` public IDs) | -| `runtime` | Runs generated apps as Docker containers; routing and proxying | -| `realtime` | SSE event stream backed by Redis pub/sub | -| `export` | CSV / JSON / SARIF downloads | -| `automation` | Pipelines, batches, cron schedules | -| `docs`, `system` | This docs viewer; staff-only system snapshots | +| `users` | Accounts (allauth headless, MFA), first-run admin bootstrap, profile and activity | +| `tokens` | Personal API tokens (`llml_…`) with optional scopes | +| `credentials` | Encrypted per-user provider keys: OpenRouter, Hugging Face, GitHub | +| `llm_models` | Model catalog (OpenRouter sync plus models served by your endpoints), enrichment, per-model insights — see [Models reference](/docs/MODELS_REFERENCE) | +| `inference` | LLM endpoints (OpenRouter, Ollama, OpenAI-compatible; direct or via relay), relay protocol and tokens, the internal LLM proxy — see [Local models](/docs/LOCAL_MODELS) | +| `execution` | The executor process: scheduler, lanes, pools, leases, reconciliation, capacity — see [Execution](/docs/EXECUTION) | +| `generation` | Jobs (agent, scaffolding, custom; legacy copilot), templates, stacks, blocks, profiles, packages, presets, evaluation, experiments with their schedules and post-runs — see [Generation process](/docs/GENERATION_PROCESS), [Experiments](/docs/EXPERIMENTS) | +| `analysis` | Analyzer tool catalog, per-user workspaces, runs, findings — see [Analysis pipeline](/docs/ANALYSIS_PIPELINE) | +| `statistics` | Dashboard and study aggregates — see [Study tables](/docs/STUDY_TABLES) | +| `rankings` | Model scores, benchmarks, severity-weight sensitivity | +| `reports` | Saved reports (`rpt_` public IDs) | +| `runtime` | Builds and runs generated apps as containers; the `/apps//` proxy and subdomain middleware | +| `realtime` | SSE stream over Redis pub/sub, per-channel authorization | +| `export` | CSV / JSON / SARIF downloads, review-sample ZIPs | +| `integrations` | MCP server, cloud provider presets, GitHub export, notification webhooks — see [Integrations](/docs/INTEGRATIONS) | +| `docs` | This docs viewer (Markdown from `docs/`) | +| `system` | Staff-only host, container, Redis, Celery and DB snapshots plus maintenance actions | +| `automation` | Retired pipelines, batches and schedules. Its models, tables and Celery tasks remain, but its API router is unmounted and `/automation` in the UI redirects to `/experiments`. Scheduling and post-run analysis now belong to experiments. | ## Request path and auth -The REST API is a single django-ninja `NinjaAPI` mounted at `/api` (`config/api.py`), with two interchangeable auth schemes: Bearer API tokens (`backend/tokens/auth.py`) and Django session cookies. The interactive OpenAPI UI at `/api/docs` is staff-only. One endpoint lives outside ninja because it streams: `GET /api/realtime/stream` (`config/urls.py`). +The REST API is one django-ninja `NinjaAPI` mounted at `/api/` (`config/api.py`) with two auth schemes, Bearer API tokens (`backend/tokens/auth.py`) and Django session cookies. Some routers accept only the session (see [API reference](/docs/api-reference#authentication)). The OpenAPI UI (`/api/docs`) and schema (`/api/openapi.json`) are staff-only. -Views raise HTTP errors directly; services raise `backend.common.exceptions.*`, mapped to `{"detail": ...}` responses by the exception handler in `config/api.py`. +Plain Django views live outside ninja in `config/urls.py`: -## Execution model: threads vs Celery +| Path | What | +| --- | --- | +| `/api/realtime/stream` | SSE stream (async view; ninja cannot stream) | +| `/api/relay/v1/…` | Relay protocol for local-model relays (`backend/inference/relay/urls.py`) | +| `/api/mcp`, `/api/mcp/t/` | MCP server (`backend/integrations/mcp/views.py`) | +| `/apps//` | Reverse proxy to a running generated app (`backend/runtime/proxy.py`) | +| `/_allauth/`, `/accounts/` | allauth headless and browser flows | + +Views raise ninja `HttpError`; services raise `backend.common.exceptions.*`, which the exception handler in `config/api.py` maps to `{"detail": ...}` responses. + +## Where work runs -The load-bearing nuance of this codebase: +| Work | Runs in | Started by | +| --- | --- | --- | +| Every generation job (agent sandboxes, scaffolding and custom LLM calls), evaluation, re-evaluation | `executor` service, worker pools `sandbox`, `llm`, `eval` | `backend/generation/services/dispatcher.py` → `backend/execution/api.py` writes rows and rings the `executor:wake` doorbell | +| Experiment schedules (cron top-up, add replicates, clone and run) and post-runs (analyze, then report) | `executor`, pools `schedule` and `post` (ticked every 30 s) | `backend/generation/services/experiment_automation.py` | +| LLM calls to direct endpoints (one pump thread per concurrency slot), the internal LLM proxy for agent gateways | `executor` | `backend/inference/pump.py`, `backend/inference/proxy_server.py` | +| LLM calls to relay endpoints | Queued as `LLMRequest` rows. The relay on the GPU box claims them by long-polling `/api/relay/v1/poll` on `django` | `backend/inference/relay/` | +| Interactive analysis runs, workspace provisioning and tool installs, container build/start/stop actions, report generation | Daemon threads inside the `django` process (`backend/common/threading.py:dispatch_in_thread`) | `analysis/services/runner.py`, `analysis/api/views/workspace.py`, `analysis/api/views/installed.py`, `runtime/services/container_service.py`, `reports/services/report_service.py` | +| Webhook delivery and the 60 s webhook sweep, Hugging Face catalog enrichment | Celery worker, beat (`django_celery_beat` database scheduler) | `backend/integrations/tasks.py`, `backend/llm_models/tasks.py` | -- **Interactive generation and analysis runs execute on daemon threads inside the django process** (`backend/common/threading.dispatch_in_thread`, used by `generation/services/dispatcher.py` and `analysis/services/runner.py`). The stack works with no Celery worker at all — but in-flight runs die if the django container restarts. -- **Celery handles automation**: pipeline runs (`run_pipeline_task`), batches, and the schedule tick (`run_scheduler_tick` via beat). If the broker is unreachable, automation falls back to a daemon thread too. +Consequences: -Details in [Background services](/docs/BACKGROUND_SERVICES). +- The web process never runs a generation job. If the executor is down, jobs stay queued (the UI shows the executor as offline) and resume when it returns. Exactly one executor runs. It holds a Postgres advisory lock, uses leases so jobs survive restarts, and has no autoreload: restart it to pick up backend changes, draining first. See [Execution](/docs/EXECUTION). +- Thread-dispatched work dies with the `django` container: a restart or autoreload during an interactive analysis run or a report kills it. The system page has maintenance actions to clear stuck runs. +- Celery is only needed for notifications and enrichment. Without a worker, webhooks queue up and the Hugging Face pass is skipped. + +Details of each service are in [Background services](/docs/BACKGROUND_SERVICES). ## Docker surfaces -Django never touches the host Docker daemon. Locally it drives `apps-dind`, a sandbox Docker-in-Docker service, over mutual TLS (`APPS_DOCKER_HOST=tcp://apps-dind:2376`, client cert from `APPS_DOCKER_CERT_PATH`), and two kinds of containers run nested inside it: +Locally, no service touches the host Docker daemon. Django and the executor drive `apps-dind`, a Docker-in-Docker sandbox, over mutual TLS (`APPS_DOCKER_HOST=tcp://apps-dind:2376`, client cert from `APPS_DOCKER_CERT_PATH`). Nested inside it run: + +1. **Analyzer workspaces**: one long-lived, hardened container per user, built from `backend/analyzer-base:latest` (`backend/analysis/images/analyzer-base`). Tools are installed once and reused. Each run's code is copied into `/work` after the previous run's files are wiped. Dynamic tools (e.g. ZAP, Nuclei, Wapiti) scan only a running container of the analyzed app belonging to the run's owner. +2. **Agent sandboxes**: where a coding agent builds an app. The sandbox's gateway reaches LLMs through the executor's internal LLM proxy on the internal `llmproxy` network (`LLM_PROXY_ADDR`, default `172.30.99.2:8090`). +3. **Generated apps**: built and run for evaluation (hidden conformance suite, on an internal network) and on demand by the `runtime` app for clicking through and dynamic analysis. -1. **Analyzer workspaces** — one long-lived, hardened container per user, built from the `backend/analyzer-base:latest` image (`backend/analysis/images/analyzer-base`). Analysis tools are installed into it once and reused across runs; each run's code is copied into `/work` after wiping the previous run's files. -2. **Generated sample apps** — each completed generation job can be built and started as its own container by the `runtime` app. Locally an app's port is published only inside `apps-dind`, and Django's path proxy (`/apps//`) forwards to `apps-dind:`; in production `AppSubdomainProxyMiddleware` serves each app at `.` behind Traefik — see [Deployment guide](/docs/deployment-guide). +How apps are reached depends on the routing mode: `APPS_PROXY_PATH` (local default, Django proxies `/apps//` to `apps-dind:`), `APPS_SUBDOMAIN_PROXY` (`AppSubdomainProxyMiddleware` serves `.`), or the Traefik bridge (`TRAEFIK_DYNAMIC_DIR` + `DOCKER_APPS_NETWORK`, used by `docker-compose.production.yml`). See [Deployment guide](/docs/deployment-guide). -`apps-dind` fences its nested containers off from private address space (the stack's own network with Postgres/Redis/Django, other host networks, cloud metadata) with `DOCKER-USER` rules set by `compose/local/apps-dind/entrypoint.sh`; public internet stays open for package installs. Replies to connections Django opens into an app still flow. +`apps-dind` fences its nested containers off from private address space (the stack's own network, other host networks, cloud metadata) with `DOCKER-USER` rules in `compose/local/apps-dind/entrypoint.sh`. Public internet stays open for package installs, and the only private exception is the gateway egress subnet to the LLM proxy. Replies to connections Django opens into an app still flow. -The local stack is the compose project `llmlab`. Every service has a memory/CPU/PID cap and `oom_score_adj: 500`, so on a shared host the stack throttles itself and is the OOM killer's first pick. `apps-dind`'s cap (4 GB, 4 CPUs) covers all nested apps together. Inside it, logs rotate, build cache is capped at 5 GB, images unused for a week are pruned hourly, and removing an app also deletes its image. The devcontainer has no host Docker socket: its `docker` CLI is pointed at `apps-dind` too. +> [!WARNING] +> `docker-compose.production.yml` has no `apps-dind`: Django and the executor mount the host `/var/run/docker.sock`, and apps run on the `llm_apps` network. Local-model agent runs are therefore off there (`LLM_LOCAL_AGENT_RUNS` defaults to `bool(APPS_DOCKER_HOST)`). + +The local stack is the compose project `llmlab`. Every service has a memory/CPU/PID cap and `oom_score_adj: 500`, so on a shared host the stack throttles itself and is the OOM killer's first pick. `apps-dind`'s cap (`APPS_DIND_MEM_LIMIT`, default 4 GB, 4 CPUs) covers all nested containers together, and the executor's slot settings are validated against it. Inside `apps-dind`, logs rotate, build cache is capped at 5 GB, images unused for a week are pruned hourly, and removing an app also deletes its image. The devcontainer has no host Docker socket: its `docker` CLI points at `apps-dind` too. ## Data flow -A full experiment is: generate an app per model → run it → analyze it → compare. +An experiment generates an app per model, evaluates it, analyzes it and compares the results. -1. A generation job calls OpenRouter with the job owner's key and writes structured code artifacts. -2. An analysis run executes the selected tools against that code and persists normalized `Finding` rows plus per-tool metrics. -3. Statistics, rankings, and reports aggregate over findings and jobs — counting only the latest completed run per job — and everything is downloadable via the export endpoints ([API reference](/docs/api-reference)). +1. A generation job calls its model's endpoint through the executor. OpenRouter calls use the job owner's key; see [Models reference](/docs/MODELS_REFERENCE#api-key-resolution). The job writes its code artifacts. +2. Evaluation builds and runs the app in `apps-dind` and grades it with the hidden conformance suite, writing `metrics.functional`. +3. An analysis run executes the selected tools against that code and persists normalized `Finding` rows plus per-tool metrics. For experiments, the post-run does this automatically. +4. Statistics, rankings and reports aggregate over findings and jobs, counting one result per job and tool: the latest one that measured the app (see [Analysis pipeline](/docs/ANALYSIS_PIPELINE#where-results-go)). Everything is downloadable through the export endpoints ([API reference](/docs/api-reference#export-endpoints)). ## Scoring: measurement vs. decision aid -Rankings keep two kinds of numbers strictly apart (`backend/rankings/services/`): - -- **Empirical quality (measured).** Severity-weighted findings from the deterministic - container tools only, normalized per KLOC of generated code and blended with the - smoke pass rate. Findings from AI-kind tools (`AnalyzerTool.kind == "ai"`, e.g. the - LLM reviewer) are opinions with their own run-to-run variance, so they never enter - this score — they are reported separately as `ai_findings`. Each score carries - `n_trials` and between-trial spread (stdev of per-job findings density and smoke - pass rate); a single completed trial is an anecdote, not a result. -- **MSS (decision aid).** An opinionated composite of model metadata — adoption, - public benchmarks, cost, accessibility — with asserted weights. Useful for picking - a model, but not a measurement made here. **Composite** blends MSS with empirical - quality when local measurements exist. - -Because the baseline severity weights (`SEVERITY_WEIGHTS` in -`rankings/services/constants.py`) are asserted rather than derived, the platform also -reports a sensitivity analysis: the empirical ranking is recomputed under the -alternative schemes in `SEVERITY_WEIGHT_SCHEMES` and compared with Kendall's tau plus -the list of rank swaps (`GET /api/rankings/sensitivity/`, also embedded in -template-comparison and comprehensive reports). Tau near 1.0 across schemes means the -ordering does not hinge on the particular weight choice. - -The "smoke pass rate" is exactly that: `/api/health` plus the template's declared GET -endpoints. It shows a generated app starts and responds, not that it is functionally -correct. The statistics page's model comparison delegates to the same rankings -aggregation, so the two pages cannot disagree. - -Every job also carries `prompt_hash`/`bundle_key`, and rankings/statistics/reports -accept them as optional filters — a comparison across models is only valid if every -model was generated from the same prompt version; without an explicit filter, jobs -from different template edits on the same app are pooled together silently. See -[Experiments](/docs/EXPERIMENTS) for running a designed, apples-to-apples comparison. +Rankings keep two kinds of numbers apart (`backend/rankings/services/`): + +- **Empirical quality (measured).** Severity-weighted findings from the deterministic container tools only, normalized per KLOC of generated code and blended with the functional pass rate. Findings from AI-kind tools (`AnalyzerTool.kind == "ai"`, e.g. the LLM reviewer) are opinions with their own run-to-run variance. They never enter this score and are reported separately as `ai_findings`. Each score carries `n_trials` and between-trial spread (stdev of per-job findings density and pass rate). A single completed trial is an anecdote, not a result. +- **MSS (decision aid).** An opinionated composite of model metadata (adoption, public benchmarks, cost, accessibility) with asserted weights. It helps when picking a model but is not a measurement made here. **Composite** blends MSS with empirical quality when local measurements exist. + +The baseline severity weights (`SEVERITY_WEIGHTS` in `rankings/services/constants.py`) are asserted, not derived. To account for that, the platform reports a sensitivity analysis: the empirical ranking is recomputed under the alternative schemes in `SEVERITY_WEIGHT_SCHEMES` and compared with Kendall's tau and the list of rank swaps (`GET /api/rankings/sensitivity/`, also embedded in template-comparison and comprehensive reports). Tau near 1.0 across schemes means the ordering does not hinge on the weight choice. + +The functional pass rate is `metrics.functional.pass_rate`. Evaluation writes it from the conformance suite. A manual re-probe (`POST /api/runtime/jobs/{job_id}/smoke/`, `/api/health` plus the template's declared GET endpoints) overwrites it with the weaker smoke result. The statistics page's model comparison uses the same rankings aggregation, so the two pages cannot disagree. + +Every job also carries `prompt_hash`/`bundle_key`, and rankings, statistics and reports accept them as optional filters. A comparison across models is only valid if every model was generated from the same prompt version. Without an explicit filter, jobs from different template edits on the same app are pooled together. See [Experiments](/docs/EXPERIMENTS) for running a designed, apples-to-apples comparison. diff --git a/docs/AUTOMATION_GUIDE.md b/docs/AUTOMATION_GUIDE.md index e481b210..7ba7f58b 100644 --- a/docs/AUTOMATION_GUIDE.md +++ b/docs/AUTOMATION_GUIDE.md @@ -1,6 +1,17 @@ # Automation guide -Automation is now part of each experiment: **Schedules** and **Post-run** on the experiment's Automation tab (`/experiments/{id}`). The `/automation` pages redirect to `/experiments`, and the `/api/automation/` router is no longer mounted. +Automation is part of each experiment: cron schedules and post-run steps under the experiment's **Operations → Automation** section replace the retired pipeline hub. + +## Where it lives now + +On an experiment (`/experiments/{id}?tab=operations`; the old `?tab=automation` still lands there), section 5 Operations → Automation has two cards: + +- **Schedules**: cron triggers in a timezone (`top_up`, `add_replicates`, `clone_and_run`), with an enable switch, the next run, the last result and Run now. +- **Post-run**: the steps that run once the experiment completes (analyzers over the completed jobs, then the experiment report), with the post-run history and Run now. + +The composer's Automation section sets the post-run steps and one optional schedule when you create the experiment; add more schedules later in that section. Archived experiments are read-only there. How schedules fire (15-minute minimum interval, no burst catch-up, skip while active) and how post-runs run (once per completion, sequential analyses, owner-scoped report) is in [Experiments → Automation](/docs/EXPERIMENTS#automation). + +## Migrating from pipelines | What you used to build | What to use now | |---|---| @@ -8,6 +19,8 @@ Automation is now part of each experiment: **Schedules** and **Post-run** on the | A batch fanning a pipeline out over models × templates | One experiment whose conditions are the cross-product (models × harness presets, or models × profiles) | | `analyze` and `report` steps after generation | The experiment's post-run: analyzers over the completed jobs, then the experiment report | -See [Experiments → Automation](/docs/EXPERIMENTS#automation) for how schedules fire (cron in a timezone, a 15-minute minimum interval, no burst catch-up, skip while active) and how post-runs run (once per completion, sequential analyses, owner-scoped report). +## What is left of the old app -The `backend.automation` app, its tables and its engine stay in the codebase for existing data; nothing creates new pipeline runs. +- `/automation` and every page under it redirect (308) to `/experiments`. +- The `/api/automation/` router is not mounted (`config/api.py`). +- `backend.automation` stays installed for existing data: its models (`Pipeline`, `PipelineStep`, `PipelineRun`, `PipelineStepRun`, `Batch`, `BatchItem`, `Schedule`) and their admin pages, the `engine/` package, the Celery tasks `run_pipeline_task` and `run_scheduler_tick`, and the `runautomationscheduler` management command. No UI or API creates new pipeline runs, and no beat entry for `run_scheduler_tick` ships with the code (beat uses the database scheduler, so only a manually added periodic task would run it). diff --git a/docs/BACKGROUND_SERVICES.md b/docs/BACKGROUND_SERVICES.md index e612f925..510f50f7 100644 --- a/docs/BACKGROUND_SERVICES.md +++ b/docs/BACKGROUND_SERVICES.md @@ -1,45 +1,105 @@ # Background services -Which work runs where: in-process threads, Celery workers, beat schedules, and the realtime event stream. +Which work runs where: the executor process, threads inside the web process, Celery workers and beat, and the Redis-backed realtime stream. -## Threads vs Celery +## Who runs what -| Work | Runs on | Needs a worker? | +| Work | Runs in | Survives a `django` restart? | | --- | --- | --- | -| Interactive generation jobs | Daemon thread in django (`dispatch_job`) | No | -| Interactive analysis runs | Daemon thread in django (`runner.dispatch`) | No | -| Automation pipeline runs | Celery `run_pipeline_task` (thread fallback if broker is down) | Preferably | -| Automation schedules | Celery beat `run_scheduler_tick` (or a management command) | Yes | +| Generation jobs (agent, scaffolding, custom) and their evaluation | `executor` service, claimed from Postgres | Yes | +| Experiment schedules (top up, add replicates, clone and run) | `executor`, checked every 30 s | Yes | +| Experiment post-runs (analyses over completed jobs, then the report) | `executor` post-run pool | Yes; a lost lease is re-claimed | +| Internal LLM proxy and direct-endpoint pumps for local models | `executor` threads | Yes | +| Interactive analysis runs | Daemon thread in `django` (`analysis.services.runner.dispatch`) | No | +| Analyzer workspace provisioning, tool installs | Daemon threads in `django` | No | +| Report generation, container start/stop/build actions | Daemon threads in `django` | No | +| Catalog enrichment after a model sync | Celery `enrich_catalog_task` | Yes (queued in Redis) | +| Outbound webhook deliveries | Celery `deliver_webhook`, retried with backoff | Yes | +| Webhook sweep for missed status transitions | Celery beat, every 60 s | Yes | -The trade-off of the thread model: the stack is fully functional without any Celery worker, but **in-flight interactive runs die if the django container restarts** — they resurface as stuck `running` rows, not as retried jobs. +The web process never runs generation jobs. Views write rows (`queued_at`, lane, priority) and, after the transaction commits, push a token onto the Redis list `executor:wake` (the doorbell) so the executor claims them without waiting for its next tick. See [Execution engine](/docs/EXECUTION) for the scheduler, slots, leases and drain procedure. -## Celery services +Threads in `django` die with the process. In the local stack that includes every `uvicorn --reload` triggered by a backend edit. A killed analysis run stays `running`; cancel it, or use **System → Maintenance → Clear Stuck Analysis Tasks** (staff) to fail runs untouched for an hour. + +## The executor + +One instance, compose service `executor`, command `python manage.py run_executor` after `python manage.py wait_for_migrations` (it never migrates itself). It uses the same image and env files as `django` and talks to the apps daemon for sandboxes and evaluation containers. + +| Thread or pool | Job | +| --- | --- | +| Main loop | Wait on `executor:wake` up to `EXECUTOR_TICK_SECONDS` (2 s), requeue lost jobs, claim work, reconcile every 15 s, schedules and post-runs every 30 s, purge old LLM request rows daily | +| Heartbeat | Every `EXECUTOR_HEARTBEAT_SECONDS` (20 s): instance `last_seen_at`, lease renewal for jobs it is running | +| `sandbox`, `eval`, `llm`, `post`, `schedule` pools | Agent runs, evaluations, scaffolding/custom jobs, post-runs, schedule actions | +| LLM proxy | `inference.proxy_server` on `LLM_PROXY_BIND` (`172.30.99.2:8090` on the internal `llmproxy` network) | +| Direct pumps | Workers that forward queued LLM requests to direct local endpoints | -Three compose services share the django image (`docker-compose.local.yml`): +It has `restart: unless-stopped` and exits with code 1 when it loses the Postgres leader lock, when the LLM proxy cannot bind, or when migrations are still pending after 10 minutes. It does not autoreload; restart it after backend edits. Check it with: + +```bash +just manage executor status # online, draining, running jobs, safe_to_restart +just manage executor drain # stop claiming before a restart +just manage executor resume +``` + +Staff can also see and change its slots in the capacity strip and `GET`/`PATCH /api/execution/settings/`. + +## Celery services -- `celeryworker` — executes automation tasks. Broker and result backend are Redis (`REDIS_URL`). -- `celerybeat` — periodic scheduler, using `django_celery_beat`'s database scheduler, so periodic tasks are editable in the Django admin. -- `flower` — task dashboard on :5555, credentials generated by `just bootstrap`. +Three compose services share the django image and the Redis broker (`REDIS_URL`, also the result backend): -## The automation scheduler +| Service | Command | Role | +| --- | --- | --- | +| `celeryworker` | `celery -A config.celery_app worker` | Runs the tasks below. Locally `CELERY_WORKER_CONCURRENCY=2` | +| `celerybeat` | `celery -A config.celery_app beat` | Periodic tasks from `django_celery_beat`'s database scheduler, editable in the Django admin | +| `flower` | `celery ... flower` | Task dashboard on :5555, login from `CELERY_FLOWER_USER` / `CELERY_FLOWER_PASSWORD` | -Cron [schedules](/docs/AUTOMATION_GUIDE) fire when something calls `scheduler.tick()`, which checks for enabled schedules whose `next_run_at` has passed. Two ways to make that happen: +| Task | Module | Triggered by | +| --- | --- | --- | +| `enrich_catalog_task` | `backend/llm_models/tasks.py` | `POST /api/models/sync/`, after the commit. Also available synchronously as `manage.py enrich_models` | +| `deliver_webhook` | `backend/integrations/tasks.py` | A webhook event for an enabled subscription. Up to 3 retries at 15, 60 and 240 s on 5xx, 429 or network errors | +| `sweep_webhook_events` | `backend/integrations/tasks.py` | Beat, every 60 s. Registered as the periodic task "integrations: sweep webhook events" after every migrate. Emits events for terminal transitions from the last 30 minutes that bypassed `post_save`, and re-dispatches deliveries that never got a first attempt | +| `run_pipeline_task`, `run_scheduler_tick` | `backend/automation/tasks.py` | Legacy automation; nothing calls or schedules them any more | +| `get_users_count` | `backend/users/tasks.py` | Template example, unused | -- **Celery beat** running the `run_scheduler_tick` task — the production recommendation. -- `just manage runautomationscheduler` — a standalone loop that ticks every 30 seconds, for setups without beat. +Without a worker the app still works: model sync succeeds without enrichment, and webhook deliveries wait in Redis (and stay `pending` in the database) until a worker starts. Without beat the sweep stops, so deliveries that could not be queued (Redis down) are never retried. -If neither runs, schedules simply never fire; pipelines can still be triggered manually. +> [!NOTE] +> Experiment schedules do not need Celery beat. The old `run_scheduler_tick` / `runautomationscheduler` path only served legacy automation pipelines; schedules now fire inside the executor. See [Experiments](/docs/EXPERIMENTS#automation) and [Automation guide](/docs/AUTOMATION_GUIDE). ## Realtime events (SSE) -Progress updates reach the browser over server-sent events, not polling: +Progress reaches the browser over server-sent events, not polling. + +- Publishers call `backend.realtime.events.publish(channel, event)`, which pushes JSON into Redis pub/sub. Helpers wrap the experiment, endpoint and capacity channels. +- `GET /api/realtime/stream?channels=a,b` (or repeated `channels=`) is an async Django view mounted outside ninja in `config/urls.py`. It authorizes every channel, closes its database connection, then streams from Redis on the event loop, so an open tab holds neither a worker thread nor a Postgres connection. +- Responses: 401 unauthenticated, 400 with no channels or more than 20, 403 when any channel is malformed, unknown or not yours. + +| Channel | Who may subscribe | Published by | +| --- | --- | --- | +| `generation:` | The job's creator or staff | Job status and results | +| `experiment:` | The experiment's creator or staff | Executor tick events, job transitions | +| `endpoint:` | Users who may use the endpoint; non-owners get `endpoint::public` | Local endpoint state | +| `analysis:` | The run's creator or staff | Analysis run status | +| `runtime:` | The container's creator, the owner of its generation job, or staff | Container actions | +| `capacity` | Any authenticated user | Executor slot usage, at most every 5 s | + +## Redis roles -- Publishers call `backend/realtime/events.publish(channel, event)`, which pushes JSON into Redis pub/sub. Channels: `generation:`, `analysis:`, `runtime:`, and the `dashboard` broadcast. -- `GET /api/realtime/stream?channels=generation:,analysis:` is a plain Django streaming view mounted outside ninja (`config/urls.py`) because ninja can't stream. It requires an authenticated session and at least one channel. +| Use | Keys | +| --- | --- | +| Celery broker and results | Celery defaults on `REDIS_URL` | +| Realtime pub/sub | The channels above | +| Executor doorbell | List `executor:wake` | +| LLM request queue doorbells | `backend/inference/doorbell.py` keys (prefix `INFERENCE_REDIS_PREFIX`, empty by default) | +| Django cache | Production only; local settings use an in-memory cache | + +Postgres stays the source of truth: a Redis outage delays the executor by at most one tick and drops live updates, but loses no jobs. ## Runtime housekeeping Two management commands keep the container fleet honest, useful from cron in production: -- `just manage sync_container_status` — reconciles DB container state with what Docker actually reports. -- `just manage reconcile_app_routes` — rewrites the per-app Traefik route files (see [Deployment guide](/docs/deployment-guide)). +- `just manage sync_container_status`: updates `ContainerInstance.status` from the live Docker state. +- `just manage reconcile_app_routes`: rewrites the per-app Traefik route files and drops stale ones ([Deployment guide](/docs/deployment-guide)). + +The executor's reconciler separately reaps lost agent sandboxes and evaluation containers on the apps daemon. **System → Maintenance** offers "Purge Orphan Containers" for container rows whose Docker container is gone, and "Clear Stuck Generation Jobs" as a manual backstop for executor-owned jobs whose lease expired long ago. diff --git a/docs/EXECUTION.md b/docs/EXECUTION.md index 779d9d09..013cb377 100644 --- a/docs/EXECUTION.md +++ b/docs/EXECUTION.md @@ -1,69 +1,48 @@ # Execution engine -Every generation job (agent, scaffolding, custom), its evaluation, experiment -schedules and post-runs run in one process: the **executor** -(`python manage.py run_executor`, compose service `executor`). The web process -(`django`) only writes rows and rings a Redis doorbell after the transaction -commits. Code lives in `backend/execution/`. +Every generation job, its evaluation, experiment schedules and post-runs run in one leader process, the executor, which claims work from Postgres under leases. + +The executor is `python manage.py run_executor` (compose service `executor`). The web process (`django`) only writes rows and rings a Redis doorbell (`executor:wake`) after the transaction commits; nothing runs in the web process. Code lives in `backend/execution/`. How experiments decide the run order is in [Experiments](/docs/EXPERIMENTS#scheduling-policies); local endpoints and the relay are in [Local models](/docs/LOCAL_MODELS). The other background services (Celery, beat) are in [Background services](/docs/BACKGROUND_SERVICES). ## Process | Part | What it does | |---|---| -| Leader lock | `pg_try_advisory_lock(EXECUTOR_LEADER_LOCK_KEY)` on a dedicated connection at start; a second instance exits with code 1. The lock is re-checked in `pg_locks` every tick; if it is gone (Postgres restart, dropped connection) the executor stops claiming, expires its own leases and exits with code 1; the compose service has `restart: unless-stopped`, so Docker starts it again. It is a guard, not high availability: exactly one executor runs. | -| Main loop | `BLPOP executor:wake` for at most `EXECUTOR_TICK_SECONDS` (2 s), then: lost-job check → `Scheduler.tick` → submit claims to pools. `reconcile()` every 15 s, schedules and post-runs every 30 s, LLM request purge daily. `ExecutorSettings` is re-read every tick. A Redis error sleeps one tick (never busy-spins); Postgres stays the only source of truth. | +| Leader lock | `pg_try_advisory_lock(EXECUTOR_LEADER_LOCK_KEY)` on a dedicated connection at start; a second instance exits with code 1. The lock is re-checked in `pg_locks` every tick; if it is gone (Postgres restart, dropped connection) the executor stops claiming, expires its own leases and exits with code 1, and Docker starts it again. It is a guard, not high availability: exactly one executor runs. | +| Main loop | `BLPOP executor:wake` for at most `EXECUTOR_TICK_SECONDS` (2 s), then: lost-job check → `Scheduler.tick` → submit claims to pools. `reconcile()` every 15 s, schedules and post-runs every 30 s, a daily LLM request purge (clears `body`/`response` of rows older than `LLM_REQUEST_RETENTION_DAYS`, 7). `ExecutorSettings` is re-read every tick. A Redis error sleeps one tick (never busy-spins); Postgres stays the only source of truth. | | Heartbeat | Every `EXECUTOR_HEARTBEAT_SECONDS` (20 s): `ExecutorInstance.last_seen_at` + status blob (`config`: effective slots, open circuits, blocked reasons). Renews `lease_expires_at` (+`EXECUTOR_LEASE_SECONDS`, 90 s) **only for jobs in the in-process registry**, and leases of running post-runs. | -| Pools | `sandbox` (agent runs), `eval` (evaluations), `llm` (scaffolding / custom), `post` (post-runs and schedule actions). Admission uses `min(slots − used in DB, really free threads)`. | -| Live events | Every 2 s at most, only when changed: `tick` per active experiment, `endpoint:` state per local endpoint (owner payload plus the redacted `:public` copy). `capacity` at most every 5 s; its `executor.code_stale` is true once the code on disk differs from what this executor started with. | -| LLM proxy, direct pumps | `inference.proxy_server.serve` and `inference.pump.run_direct_pumps` threads, started after leadership is won (see `docs/LOCAL_MODELS.md`). The proxy is required: if it cannot bind `LLM_PROXY_BIND` or stops, the executor exits with code 1 (a restart loop in `docker compose ps`/logs, not a silently "online" executor whose local agent runs all fail). The pumps supervise their own threads. | +| Pools | `sandbox` (agent runs), `eval` (evaluations), `llm` (scaffolding / custom), `post` (post-runs) and `schedule` (schedule actions: one thread of short DB work). Admission uses `min(slots − used in DB, really free threads)`. | +| Live events | Every 2 s at most, only when changed: `tick` per active experiment, `endpoint:` state per local endpoint (owner payload plus the redacted `endpoint::public` copy). `capacity` at most every 5 s; its `executor.code_stale` is true once the code on disk differs from what this executor started with. | +| LLM proxy, direct pumps | `inference.proxy_server.serve` and `inference.pump.run_direct_pumps` threads, started after leadership is won (see [Local models](/docs/LOCAL_MODELS#5-how-a-call-flows)). The proxy is required: if it cannot bind `LLM_PROXY_BIND` or stops, the executor exits with code 1 (a restart loop in `docker compose ps`/logs, not a silently "online" executor whose local agent runs all fail). The pumps supervise their own threads. | -The executor runs with `CONN_MAX_AGE=0`; every worker closes its DB connection -when it finishes, so idle threads hold none. +The executor runs with `CONN_MAX_AGE=0`; every worker closes its DB connection when it finishes, so idle threads hold none. `ExecutorInstance.code_version` is a sha1 over path, mtime and size of `backend/**/*.py`; the capacity API and experiment status compare the live leader's value with the web process's (`code_stale`). ### Restart policy and exits -The `executor` service (both compose files) has `restart: unless-stopped`. It -exits with code 1, and Docker restarts it, when: a second instance holds the -leader lock; the lock is lost (Postgres restart, dropped lock connection); the -LLM proxy cannot bind; or `wait_for_migrations` gives up after 600 s (the -executor never migrates: locally django's `/start` does, which on dev1 only runs -once VS Code attaches; production `/start` does not migrate at all, so run -`manage.py migrate` after a deploy). Nothing is lost while it is down: jobs stay -`PENDING` and its leases are requeued as `executor_lost` on the next start. +The `executor` service (both compose files) has `restart: unless-stopped`. It exits with code 1, and Docker restarts it, when: a second instance holds the leader lock; the lock is lost (Postgres restart, dropped lock connection); the LLM proxy cannot bind; or `wait_for_migrations` gives up after 600 s. The executor never migrates: locally django's `/start` does; production `/start` does not migrate at all, so run `manage.py migrate` after a deploy (see [Deployment guide](/docs/deployment-guide)). Nothing is lost while it is down: jobs stay `PENDING` and its leases are requeued as `executor_lost` on the next start. ### Compose commands -Commands below use bare `docker compose`. There is no default compose file in -the repo: run them through `just` (which exports -`COMPOSE_FILE=docker-compose.local.yml`), set `COMPOSE_FILE` yourself, or pass -the same `-f` files the stack was started with (dev1: the local file plus the -devcontainer overlays). Mixing a different file set recreates containers. +Commands below use bare `docker compose`. There is no default compose file in the repo: run them through `just` (which exports `COMPOSE_FILE=docker-compose.local.yml`), set `COMPOSE_FILE` yourself, or pass the same `-f` files the stack was started with. Mixing a different file set recreates containers. -Locally the executor runs `/app/compose/local/django/start-executor` from the -bind-mounted repo, so it starts on an existing `llmlab/django:local` image. The -image's baked-in `/start` (django) picks up changes such as -`--timeout-graceful-shutdown 5` only after a rebuild (`just build`, or -`docker compose build django`); a rebuild marks django, celeryworker, -celerybeat and flower for recreation on the next `up`. +Locally the executor runs `bash /app/compose/local/django/start-executor` from the bind-mounted repo, so it starts on an existing `llmlab/django:local` image and picks up script changes without a rebuild. Production runs the image's `/start-executor`. The django image's baked-in `/start` picks up changes only after a rebuild (`just build`, or `docker compose build django`); a rebuild marks django, celeryworker, celerybeat and flower for recreation on the next `up`. ### Drain and restart -Backend edits do not reload the executor (no `--reload`). The capacity strip -shows "Executor runs older code" when the web process's code version differs -from the running executor's. To restart safely: +Backend edits do not reload the executor (no `--reload`). The capacity strip shows "Executor on old code" when the web process's code version differs from the running executor's. To restart safely: ```bash -docker compose exec django python manage.py executor drain # stop claiming jobs, post-runs and schedules -docker compose exec django python manage.py executor status # wait for "safe_to_restart": true +docker compose exec django /entrypoint python manage.py executor drain # stop claiming jobs, post-runs and schedules +docker compose exec django /entrypoint python manage.py executor status # wait for "safe_to_restart": true docker compose restart executor -docker compose exec django python manage.py executor resume +docker compose exec django /entrypoint python manage.py executor resume ``` -`SIGTERM` (a plain restart) gives LLM calls 20 s, then expires every lease the -instance holds (jobs, post-runs, pump-leased LLM requests) and sets -`stopped_at`. Running sandboxes are not awaited: the next start reaps their -containers and requeues the jobs as `executor_lost` (same snapshot and seed; -this does not consume the experiment's infra retries). +`docker compose exec` skips the image's entrypoint, which is what builds `DATABASE_URL`, so call `/entrypoint` explicitly (or use `just manage executor status`, which starts a one-off container). + +`safe_to_restart` is true when no job and no post-run is running. Drain is the `draining` flag of `ExecutorSettings` (also settable through the settings API); pending jobs report `draining` as their blocked reason. + +`SIGTERM` (a plain restart) gives LLM calls 20 s, then expires every lease the instance holds (jobs, post-runs, pump-leased LLM requests) and sets `stopped_at`. Running sandboxes are not awaited: the next start reaps their containers and requeues the jobs as `executor_lost` (same snapshot and seed; this does not consume the experiment's infra retries). ## Job lifecycle @@ -74,76 +53,63 @@ PENDING (queued_at set) ──claim──► RUNNING/running (lease) ──agent └───────── attempt+1, run_after ── COMPLETED / FAILED ◄─────────────────── RUNNING/evaluating (lease) ``` -- `queued_at IS NOT NULL` means the executor owns the row. Thread-era rows - (`queued_at IS NULL`) are never touched by the reconciler; requeue them - explicitly (`execution.api.requeue_legacy_jobs`). -- Lanes: `(experiment or adhoc:, endpoint)`, strict head-of-line in the - frozen `sequence` (experiments) or enqueue order (ad-hoc). Across lanes the - scheduler is work-conserving: priority, then fair share per user, then per - experiment. -- Admission rules (blocked reason in brackets): free sandbox / llm thread - (`sandbox_full`, `llm_threads_full`); endpoint enabled, online, circuit - closed and below its job cap (`endpoint_offline`, `circuit_open`, - `endpoint_full`); the live session serves the frozen `(name, digest)` - (`model_not_served`); residency K (`residency_wait`, with a starvation guard - after `EXECUTOR_STARVATION_S` that drains the endpoint for a swap); the - experiment's `max_concurrency` (`experiment_cap`); Ollama prefetch - (`warming_model`); `draining`. `retry_backoff` marks requeued jobs waiting - for `run_after`. -- Every write a worker makes goes through `transitions.guarded_save` - (`WHERE status='running' AND lease_owner=me AND attempt=a`); a 0-row result - raises `LeaseLost` and the worker stops quietly. `finish_stage` runs as - separate autocommit steps (job → LLM responses → batch → events → - experiment completion → wake). -- Cancel (`transitions.cancel_jobs`): PENDING and `phase=running` become - CANCELLED at once; the running worker keeps its `lease_owner` (and its slot) - until it notices and exits. Jobs waiting for or in evaluation finish. +- `queued_at IS NOT NULL` means the executor owns the row. Thread-era rows (`queued_at IS NULL`) are never touched by the reconciler; requeue them explicitly (`execution.api.requeue_legacy_jobs`, or `POST …/jobs/requeue-legacy/` on an experiment). +- Lanes: `(experiment or adhoc:, endpoint)`, strict head-of-line in the frozen `sequence` (experiments) or enqueue order (ad-hoc). Across lanes the scheduler is work-conserving. Lane heads are ordered by: priority, then fair share per user, then per experiment, then endpoint load (active jobs over what the endpoint can take, so the OpenRouter lane does not take every free thread while local endpoints idle), then model affinity (already resident), then a warm model, then launch time. +- Only the chosen rows are locked (`FOR UPDATE SKIP LOCKED`) and claimed with one conditional UPDATE; rows a concurrent cancel or pause holds are reconsidered on the next tick. +- Every write a worker makes goes through `transitions.guarded_save` (`WHERE status='running' AND lease_owner=me AND attempt=a`); a 0-row result raises `LeaseLost` and the worker stops quietly. `finish_stage` runs as separate autocommit steps (job → LLM responses → batch → events → experiment completion → wake). +- Cancel (`transitions.cancel_jobs`): PENDING and `phase=running` become CANCELLED at once; the running worker keeps its `lease_owner` (and its slot) until it notices and exits. Jobs waiting for or in evaluation finish. + +### Blocked reasons + +A pending job that is not admitted reports why (tick events, `GET /api/execution/queue/`, the Overview tab's endpoint lanes): + +| Reason | Meaning | +|---|---| +| `sandbox_full`, `llm_threads_full` | No free sandbox slot / LLM job thread. | +| `endpoint_offline` | The endpoint is disabled or has no live relay session / fresh inventory. | +| `circuit_open` | Three infra failures on the endpoint within 5 minutes; closed again after 60 s. | +| `endpoint_full` | The endpoint is at its job cap (`max_active_jobs`, else `max_concurrency`). | +| `model_not_served` | The live session does not serve the frozen `(name, digest)`. | +| `residency_wait` | Starting this model would exceed the endpoint's resident-model limit K. After `EXECUTOR_STARVATION_S` (600 s) the starvation guard drains the endpoint for a swap. | +| `warming_model` | An Ollama prefetch (warm-up) of the model is in progress. | +| `experiment_cap` | The experiment's `max_concurrency` is reached. | +| `draining` | The executor is draining. | +| `retry_backoff` | A requeued job waiting for its `run_after`. | ### Slots and the memory rule -Staff edit slots at `/api/execution/settings/` (or the admin). The rule: +`ExecutorSettings` (singleton, re-read every tick) holds: + +| Setting | Default | Bounds | +|---|---|---| +| `sandbox_slots` | `AGENT_MAX_CONCURRENT_RUNS` (else 1) | 0..8 | +| `eval_slots` | 1 | 0..4 | +| `llm_job_threads` | 16 | 1..32 | +| `post_run_slots` | 1 | 0..1 (the analyzer workspace is shared) | +| `draining` | false | | + +Staff edit them on `/endpoints` (capacity card), at `PATCH /api/execution/settings/`, or in the admin. The rule: ``` sandbox_slots × (AGENT_SANDBOX_MEM_LIMIT + 256m) + eval_slots × EVAL_SLOT_MEM + post_run_slots × ANALYSIS_WORKSPACE_MEM + APPS_DIND_APP_RESERVE ≤ 0.95 × APPS_DIND_MEM_LIMIT ``` -`APPS_DIND_MEM_LIMIT` is authoritative (keep it equal to apps-dind's -`mem_limit`). A PATCH that breaks the rule is refused with the arithmetic, -except for the minimal one-sandbox / one-evaluation configuration. When even -that does not fit (the default 4 g apps-dind), sandbox runs and evaluations -**take turns** (`exclusive_eval`), as they did before the executor, and -post-runs wait while either is busy. A rule broken only by the post-run term -is allowed; post-runs then wait while sandbox and evaluation are busy. +Defaults: `AGENT_SANDBOX_MEM_LIMIT` 3g, `EVAL_SLOT_MEM` 1536m, `ANALYSIS_WORKSPACE_MEM` 1g, `APPS_DIND_APP_RESERVE` 1g, `APPS_DIND_MEM_LIMIT` 4g. `APPS_DIND_MEM_LIMIT` is authoritative (keep it equal to apps-dind's `mem_limit`; `docker info` inside apps-dind reports host RAM). A PATCH that breaks the rule is refused with the arithmetic (`{detail, arithmetic, mem_limit, required}`), except for the minimal one-sandbox / one-evaluation configuration. When even that does not fit (the default 4g apps-dind), sandbox runs and evaluations **take turns** (`exclusive_eval`), and post-runs wait while either is busy. A rule broken only by the post-run term is allowed; post-runs then wait while sandbox and evaluation are busy. ## Leases and reconciliation -`reconcile()` runs at leader start and every 15 s (each step isolated): - -0. **Lost jobs** (every tick): owned by this executor, not in the registry, - stage entered > 30 s ago → `infra_retry(executor_lost)`; a lost evaluation - goes back to `eval_queued`. -1. **Expired leases** (and every lease owned by another executor id: only one - leader exists): sandbox runs have their containers reaped, then - `executor_lost`; evaluations go back to `eval_queued` at most - `EVAL_MAX_REQUEUES` (2) times, then end ungraded (`evaluation.status = - infra_error`); cancelled rows just release the lease. -2. **Orphan containers** (every 4th pass): containers labelled - `llmlab.agent-run` / `llmlab.evaluation` whose job is not running under a - live lease and not in the registry, older than 2 minutes (`llmlab.started`). -3. LLM request leases (`inference.requests.requeue_expired`), and pump leases - of dead instances (`requeue_pump_orphans`). -4. Stale relay sessions end (`timeout`, `inference.tokens.reap_stale_sessions`: - acked leases use an attempt); a standby session is promoted. -5. Sessions of expired relay tokens end (`token_expired`, - `inference.tokens.reap_expired_tokens`). +`reconcile()` runs at leader start and every 15 s (each step isolated: one failing never stops the rest): + +0. **Lost jobs** (every tick): owned by this executor, not in the registry, stage entered > 30 s ago → `infra_retry(executor_lost)`; a lost evaluation goes back to `eval_queued`. +1. **Expired leases** (and every lease owned by another executor id: only one leader exists): sandbox runs have their containers reaped, then `executor_lost`; evaluations go back to `eval_queued` at most `EVAL_MAX_REQUEUES` (2) times, then end ungraded (`evaluation.status = infra_error`); cancelled rows just release the lease. +2. **Orphan containers** (every 4th pass): containers labelled `llmlab.agent-run` / `llmlab.evaluation` whose job is not running under a live lease and not in the registry, older than 2 minutes (`llmlab.started`). +3. Pump leases of dead instances (`requeue_pump_orphans`), and LLM request leases (`inference.requests.requeue_expired`). +4. Stale relay sessions end (`timeout`, `inference.tokens.reap_stale_sessions`: acked leases use an attempt); a standby session is promoted. +5. Sessions of expired relay tokens end (`token_expired`, `inference.tokens.reap_expired_tokens`). 6. Executor instances silent for 90 s are marked stopped. -7. Completion sweep: running experiments with no active job complete - (`maybe_complete_experiment`, idempotent: one post-run per completion); - pending jobs and generation runs of cancelled experiments are cancelled - (rows an experiment cancel skipped while a claim held them). -8. Schedules more than one period overdue are fast-forwarded (no burst; - `experiment_automation.fast_forward_schedules`). +7. Completion sweep: running experiments with no active job complete (`maybe_complete_experiment`, idempotent: one post-run per completion); pending jobs and generation runs of cancelled experiments are cancelled (rows an experiment cancel skipped while a claim held them). +8. Schedules more than one period overdue are fast-forwarded (no burst; `experiment_automation.fast_forward_schedules`). ## Retries @@ -154,101 +120,63 @@ is allowed; post-runs then wait while sandbox and evaluation are busy. | `infra` | sandbox/apps-dind failures; no successful LLM call and a transport error (gateway 502, proxy 503/504, relay lease expiry); scaffolding `LLMAccessError` with a retryable code before any successful call | `experiment.infra_retries` (default 2; ad-hoc 2), backoff `30 s × 2^n` | | `infra_interrupted` | local endpoints only: the run ended after the proxy returned an infra error, or a scaffolding call failed after a successful one | as `infra`; exhausted → COMPLETED with validity `invalid: infra_interrupted` | -A retry requeues the **same row** (`attempt + 1`, same snapshot, seed and -cell): pairing survives. Artifacts of the attempt are deleted; agent evidence -of attempt `n > 0` lives under `GENERATION_RUNS_ROOT//a/`; sandbox -names carry `-a`. Anything else (model 4xx, OpenRouter runs with a -successful call) is a sample or a failure, never retried. Three infra -failures on one endpoint within 5 minutes open its circuit for 60 s. +A retry requeues the **same row** (`attempt + 1`, same snapshot, seed and cell): pairing survives. Artifacts of the attempt are deleted; agent evidence of attempt `n > 0` lives under `GENERATION_RUNS_ROOT//a/`; sandbox names carry `-a`. Anything else (model 4xx, OpenRouter runs with a successful call) is a sample or a failure, never retried. `infra` and `infra_interrupted` failures feed the endpoint's circuit breaker (3 within 5 minutes open it for 60 s). -The executor only claims jobs of **running** experiments. A manual Retry of -a failed or cancelled cell (`POST /jobs/{id}/retry/`) therefore reopens a -completed or cancelled experiment, as a top-up does; a paused one stays -paused until it is resumed. +The executor only claims jobs of **running** experiments. A manual Retry of a failed or cancelled cell (`POST /api/generation/jobs/{id}/retry/`) therefore reopens a completed or cancelled experiment, as a top-up does; a paused one stays paused until it is resumed. -Validity notes added by the executor (kept through the evaluation stage): `infra_retried` (warn, attempt > 0), -`relay_interruptions` (info), `contended` (warn, LLM queue wait > 10 % of the -wall clock), `infra_interrupted` (exclude). +Validity notes added by the executor (kept through the evaluation stage): `infra_retried` (warn, attempt > 0), `relay_interruptions` (info), `contended` (warn, LLM queue wait > 10 % of the wall clock), `infra_interrupted` (exclude). ## Timeout chain (local models) -Per endpoint, from `request_timeout_s` (default 900 s; `inference.timeouts.chain`): -request deadline = timeout × 3 + 600 s; gateway socket timeout = timeout + 300 s; -proxy non-stream wait = gateway − 60 s (then 504, the gateway retries and -re-attaches by dedupe key); harness client timeout = gateway × 5 + 60 s. +Per endpoint, from `request_timeout_s` (default 900 s, 30..7200; `inference.timeouts.chain`): + +| Timeout | Formula | Default | +|---|---|---| +| Request deadline | timeout × 3 (max attempts) + 600 s | 3300 s | +| Gateway socket timeout | timeout + 300 s | 1200 s | +| Proxy non-stream wait | gateway − 60 s (then 504; the gateway retries and re-attaches by dedupe key) | 1140 s | +| Harness client timeout | gateway × 5 (4 gateway retries + 1) + 60 s | 6060 s | + +The chain guarantees the harness never gives up before the endpoint does. ## Network settings -The compose files derive the executor's `LLM_PROXY_BIND` -(`${LLM_PROXY_ADDR}:8090`), `LLM_PROXY_UPSTREAM` and `LLM_GATEWAY_EGRESS_SUBNET` -from the same variables apps-dind's firewall uses. To move the networks (e.g. -a subnet collision on the host), set them together in the shell or `.env` -used by compose: `LLMPROXY_SUBNET`, `LLM_PROXY_ADDR` and -`APPS_DIND_LLMPROXY_ADDR` (both inside that subnet), and optionally -`LLM_GATEWAY_EGRESS_SUBNET`. Recreate apps-dind and the executor afterwards. +The compose files derive the executor's `LLM_PROXY_BIND` (`${LLM_PROXY_ADDR}:8090`, default `172.30.99.2:8090`), `LLM_PROXY_UPSTREAM` and `LLM_GATEWAY_EGRESS_SUBNET` (default `10.213.0.0/24`) from the same variables apps-dind's firewall uses. To move the networks (e.g. a subnet collision on the host), set them together in the shell or `.env` used by compose: `LLMPROXY_SUBNET`, `LLM_PROXY_ADDR` and `APPS_DIND_LLMPROXY_ADDR` (both inside that subnet), and optionally `LLM_GATEWAY_EGRESS_SUBNET`. Recreate apps-dind and the executor afterwards. ## Production compose -`docker-compose.production.yml` has no apps-dind: sandboxes and gateways run on -the host daemon, which cannot reach the internal `llmproxy` network. Local-model -**agent** runs are therefore refused there (`LLM_LOCAL_AGENT_RUNS`, default -on only when `APPS_DOCKER_HOST` is set; the job fails at once with a -non-retryable error). OpenRouter agent runs and local-model scaffolding runs -(the executor calls the relay/endpoint itself) work. Traefik routes `/api/` -straight to django, so the adapter-node `BODY_SIZE_LIMIT` does not apply there. +`docker-compose.production.yml` has no apps-dind: sandboxes and gateways run on the host daemon, which cannot reach the internal `llmproxy` network. Local-model **agent** runs are therefore refused there (`LLM_LOCAL_AGENT_RUNS`, default on only when `APPS_DOCKER_HOST` is set; the job fails at once with a non-retryable error). OpenRouter agent runs and local-model scaffolding runs (the executor calls the relay/endpoint itself) work. Traefik routes `/api/` straight to django, so the adapter-node `BODY_SIZE_LIMIT` does not apply there. ## apps-dind firewall exception -Gateways of local-model agent runs join the nested network `llmlab-gw-egress` -(bridge `llmlab-gwegr`, `10.213.0.0/24`, inter-container traffic off). The only -new path out of apps-dind is `llmlab-gwegr → 172.30.99.2:8090` (the executor's -proxy on the internal `llmproxy` network); the rest of that subnet and -apps-dind's own `llmproxy` address are rejected (`compose/local/apps-dind/entrypoint.sh`). +Gateways of local-model agent runs join the nested network `llmlab-gw-egress` (bridge `llmlab-gwegr`, `10.213.0.0/24`, inter-container traffic off). The only new path out of apps-dind is `llmlab-gwegr → :8090` (the executor's proxy on the internal `llmproxy` network); the rest of that subnet and apps-dind's own `llmproxy` address are rejected (`compose/local/apps-dind/entrypoint.sh`). ## API -- `GET /api/execution/capacity/`: executor status (online, leader, stale code, - draining and what is left), slots, memory limits and warnings, lane usage, - visible endpoints (other tenants' models only as counts), own experiments - (staff: all) with an ETA from the last 30 minutes' throughput. -- `GET /api/execution/queue/?experiment_id&endpoint&limit`: the caller's next - pending jobs with `blocked_reason` (a dry run of the scheduler, no locks) - and `others_ahead` per endpoint (a count only). -- `GET/PATCH /api/execution/settings/` (staff). +All under `/api/execution/`: + +| Route | Who | What | +|---|---|---| +| `GET /capacity/` | any user | Executor status (online, leader, `code_stale`, draining and what is left), effective slots, memory limits and warnings, lane usage, visible endpoints (other tenants' models only as counts), own experiments (staff: all) with an ETA from the last 30 minutes' throughput. Patched live by the SSE `capacity` channel. | +| `GET /queue/?experiment_id&endpoint&limit` | any user | The caller's next pending jobs (limit 1..200, default 50) with `blocked_reason` (a dry run of the scheduler, no locks) and `others_ahead` per endpoint (a count only). | +| `GET/PATCH /settings/` | staff | Slot settings and drain; a PATCH that breaks the memory rule is a 400 with the arithmetic. | ## Test patch targets -- Views keep `backend.generation.api.views.custom.dispatch_job`, - `…custom.has_resolvable_key`, `…jobs.dispatch_job` and - `…jobs.dispatch_evaluation` (all now enqueue: nothing runs in the web process). +- Views keep `backend.generation.api.views.custom.dispatch_job`, `…custom.has_resolvable_key`, `…jobs.dispatch_job` and `…jobs.dispatch_evaluation` (all now enqueue: nothing runs in the web process). - To keep the doorbell out of Redis: `backend.execution.api.wake`. -- To assert ordering, read `GenerationJob.sequence` / `queued_at` (there is no - `run_queue` any more); experiment launches set `sequence` from - `execution.sequence.cell_sequence`. -- To run jobs in a test: `Executor(sync_pools=True, start_proxy=False, - start_pumps=False, redis_client=)`, then `acquire_leadership()`, - `startup()` and `tick()` (or `run()` with `max_ticks` / `stop_when_idle`); - or `GenerationService().execute(job)` (inline claim, evaluation inline). -- LLM calls: `backend.inference.chat.chat` and - `backend.inference.access.resolve_access`; agent sandboxes: - `backend.generation.services.agent.runner.AgentSandbox` and - `backend.inference.gateway_wiring.for_job`; evaluation: - `backend.generation.services.evaluation.runner.evaluate_job` (takes a - `save=` keyword). - -## Manual smoke (dev1; needs the user's go-ahead to recreate apps-dind and start `executor`) - -1. `docker compose up -d executor` (see "Compose commands" for the file set; - the executor starts on the existing image, no `--build` needed); - `manage.py executor status` shows - `online: true`; `/api/execution/capacity/` shows the leader and no stale code. -2. Launch a 1-model × 1-app OpenRouter agent experiment: the job goes - pending → running → eval_queued → evaluating → completed; the sandbox slot - frees before evaluation starts. -3. `docker compose restart executor` during a run: the job is requeued as - `executor_lost` (attempt 1) and completes; no orphan containers remain. -4. Local endpoint: see `docs/LOCAL_MODELS.md` (relay, fake Ollama), then run - one agent experiment per harness and check `calls.jsonl`, lanes and evaluation. -5. In apps-dind: a container on `bridge` cannot reach `172.30.99.2:8090`; one on - `llmlab-gw-egress` can; from that network `postgres:5432`, `django:8000` and - `172.30.99.3:2376` are refused, and another gateway's `:8080` is unreachable. +- To assert ordering, read `GenerationJob.sequence` / `queued_at` (there is no `run_queue` any more); experiment launches set `sequence` from `execution.sequence.cell_sequence`. +- To run jobs in a test: `Executor(sync_pools=True, start_proxy=False, start_pumps=False, redis_client=)`, then `acquire_leadership()`, `startup()` and `tick()` (or `run()` with `max_ticks` / `stop_when_idle`); or `GenerationService().execute(job)` (inline claim, evaluation inline). +- LLM calls: `backend.inference.chat.chat` and `backend.inference.access.resolve_access`; agent sandboxes: `backend.generation.services.agent.runner.AgentSandbox` and `backend.inference.gateway_wiring.for_job`; evaluation: `backend.generation.services.evaluation.runner.evaluate_job` (takes a `save=` keyword). + +More on running the test suite is in the [Development guide](/docs/development-guide). + +## Manual smoke test + +Recreating apps-dind and starting the executor interrupts running work; do it when the stack is idle. + +1. `docker compose up -d executor` (see "Compose commands" for the file set; the executor starts on the existing image, no `--build` needed); `manage.py executor status` (through `/entrypoint`, as above) shows `online: true`; `/api/execution/capacity/` shows the leader and no stale code. +2. Launch a 1-model × 1-app OpenRouter agent experiment: the job goes pending → running → eval_queued → evaluating → completed; the sandbox slot frees before evaluation starts. +3. `docker compose restart executor` during a run: the job is requeued as `executor_lost` (attempt 1) and completes; no orphan containers remain. +4. Local endpoint: see [Local models](/docs/LOCAL_MODELS) (relay, fake Ollama), then run one agent experiment per harness and check `calls.jsonl`, lanes and evaluation. +5. In apps-dind: a container on `bridge` cannot reach `:8090`; one on `llmlab-gw-egress` can; from that network `postgres:5432`, `django:8000` and the apps-dind daemon port are refused, and another gateway's `:8080` is unreachable. diff --git a/docs/EXPERIMENTS.md b/docs/EXPERIMENTS.md index 0e4ffec8..f189e673 100644 --- a/docs/EXPERIMENTS.md +++ b/docs/EXPERIMENTS.md @@ -1,8 +1,24 @@ # Experiments -An experiment is a designed run: **apps × conditions × repeats**, with a name, a hypothesis, a pre-registered baseline and a reproducibility seed. A condition is a model plus a harness preset (agent mode) or a model plus a profile (scaffolding mode). Experiments are the research entry point; the [sample generator](/docs/GENERATION_PROCESS) stays available for one-off, ad-hoc generation. +An experiment is a designed comparison of **apps × conditions × repeats** with a hypothesis, a pre-registered baseline and a reproducibility seed. -Everything lives under **Experiments** (`/experiments`). Automation (cron schedules, post-run analysis and reports) is part of each experiment; the old `/automation` hub redirects here. Where jobs run is described in [Execution](/docs/EXECUTION); local models in [Local models](/docs/LOCAL_MODELS). +A condition is a model plus a harness preset (agent mode) or a model plus a prompt profile (scaffolding mode). Experiments are the research entry point; the [sample generator](/docs/GENERATION_PROCESS) stays available for one-off, ad-hoc generation. + +Everything lives under **Experiments** (`/experiments`). Automation (cron schedules, post-run analysis and reports) is part of each experiment; the old `/automation` pages redirect here (see [Automation guide](/docs/AUTOMATION_GUIDE)). Where and in which order jobs run is described in [Execution](/docs/EXECUTION); local models and endpoints in [Local models](/docs/LOCAL_MODELS). + +## In the UI + +A task-oriented walkthrough of the app is in the [User guide](/docs/USER_GUIDE); this section is the reference. + +| Page | What it does | +|---|---| +| `/experiments` | Your experiments, most recently updated first. Filters in the URL (`?q`, `status` = Active / Drafts / Finished / Archived / All, `mode`, `endpoint`, `page`). A capacity strip on top shows the executor, lane usage and endpoints (capacity warnings behind a chip). Active rows update live over SSE; the list refetches every 15 s while any row is active. | +| `/experiments/new` | The composer, in ten numbered sections: Basics, Method, Models (grouped by endpoint), Harness presets or Prompt profiles, Apps, Replication, Sampling & budget, Ablations, Execution, Automation. A sticky rail (on wide screens) holds the section index and the **Plan**: the server preview (per-endpoint load, estimated hours, bottleneck, cost bound, issues, comparability check) with Save draft / Launch. URL parameters: `?from=` clones a design, `?edit=` edits a draft, `?models=` preselects models (from an endpoint page), `?from_batch=` starts from a batch's grid. Unsaved create-mode edits persist in the session. `/experiments/create` redirects here. | +| `/experiments/{id}` | A study record. Masthead: accession id (`EXP-` + the first UUID block), title, status and dates, the hypothesis, a spec strip (design formula `k conditions × n apps × r repeats = N cells`, baseline, outcome, randomisation), lifecycle actions (Launch, Pause, Resume, Cancel, Archive, Clone, Top up failed, JSON/CSV export) and, once launched, the data-collection strip (progress, slot lanes, ETA, bottleneck). Five numbered sections, kept in `?tab=`: **1 Abstract** (`summary`, the default: generated abstract, key findings and limitations from the pre-registered analysis, Figure 1 pass rate with 95% CIs, Figure 2 the condition × app grid, why runs failed, the record panel; drafts show a launch estimate), **2 Method** (`method`: factors, conditions, what is held constant, comparability, procedure — read-only), **3 Results** (`results`: analysis specification, leaderboard, forest plot of paired deltas vs the baseline, per-app table, notes; options in the URL), **4 Data** (`data`: every job, filterable, current attempts or retried only, Retry failed, the collection log of batches, dataset downloads), **5 Operations** (`operations`: capacity and one lane per endpoint, activity feed, execution limits — priority, concurrency, retries, preloading and repeats can change while it runs —, schedules and post-run). The old names `overview`, `design`, `runs` and `automation` still resolve. Results and Data are disabled for drafts. Archived experiments are read-only. | + +Using a model on an endpoint owned by someone else (a shared endpoint) requires ticking a consent note in the Models section: prompts and generated code pass through that owner's machine. + +`build_healing` and `analysis_plan` are set through the API only; the composer does not expose them yet. ## Design @@ -10,15 +26,22 @@ Everything lives under **Experiments** (`/experiments`). Automation (cron schedu `Experiment` (`backend/generation/models.py`) holds: -- `name`, `slug` (unique per owner), `description`, `hypothesis`. -- `status`: `draft → running ⇄ paused → completed | cancelled`, and `archived`. A top-up reopens a completed or cancelled experiment. -- `app_requirements`: only app specs you can see (the catalog's and your own). An id you cannot see is a 404, never dropped silently. -- `repeats` per cell and an optional `base_seed`. -- `mode`: `agent` (default; a sandboxed harness builds the app and the platform grades it) or `scaffolding` (template prompts). -- Settings every cell shares: `stack_slug` (every condition runs on it; null keeps the legacy per-profile stack), the pinned `sampling_preset` / `budget_preset`, `llm_defaults` and `agent_config`. -- Execution settings: `scheduling_policy` (`auto`, `interleaved`, `blocked`, `grouped`), `priority` (−10..10; non-staff are clamped to −5..0), `max_concurrency` (cap on this experiment's running jobs), `infra_retries` (0..5, default 2) and `execution_options` (`warmup`, `max_resident_models`). -- `post_run`: what happens after the runs (evaluation, analyzers, report); see [Automation](#automation). -- `baseline_condition`: the pre-registered baseline. It defaults to the first condition and is frozen at launch. +| Field | Meaning | +|---|---| +| `name`, `slug`, `description`, `hypothesis` | `slug` is unique per owner. | +| `status` | `draft → running ⇄ paused → completed \| cancelled`, plus `archived`. A top-up reopens a completed or cancelled experiment. | +| `pause_reason` | `user` or `digest_changed` (set by the endpoint inventory). `endpoint_removed` exists in the enum for data compatibility but is never set: deleting an endpoint that conditions use is refused with a 409. | +| `app_requirements` | App specs you can see (the catalog's and your own). An id you cannot see is a 404, never dropped silently. | +| `repeats`, `base_seed` | Repeats per cell (1..20 at create) and an optional seed (0..2³¹−1). | +| `mode` | `agent` (default; a sandboxed harness builds the app and the platform grades it) or `scaffolding` (template prompts). | +| `stack_slug`, `sampling_preset`, `budget_preset`, `llm_defaults`, `agent_config` | Settings every cell shares. A blank stack keeps the legacy per-profile stack. | +| `build_healing` | `full` (default), `no_deps` or `none`; copied to every job at launch. Scaffolding builds only (agent output is built verbatim). See [Study tables → Build healing](/docs/STUDY_TABLES#build-healing). | +| `continuation_limit` (0..5), `enable_repair` | Scaffolding settings: how many continuation calls a response cut off at `max_tokens` gets, and whether files that fail static checks get a repair round. Launch freezes non-default values into each job's snapshot (`resolved_bundle.generation`, part of `run_fingerprint`); the defaults (1 and true) leave the snapshot unchanged. Agent-mode cells ignore them (the harness has its own budget). | +| `scheduling_policy`, `priority`, `max_concurrency`, `infra_retries`, `execution_options` | Execution settings: policy `auto` / `interleaved` / `blocked` / `grouped`; priority −10..10 (non-staff are clamped to −5..0); a cap on this experiment's running jobs (1..64, null = none); infra retries 0..5 (default 2); options `warmup` (Ollama preloading, default on) and `max_resident_models` (only tightens an endpoint's K). | +| `post_run` | What happens after the runs; see [Automation](#automation). | +| `analysis_plan` | Optional pre-registered analysis plan; editable while draft, frozen at launch (see below). | +| `baseline_condition` | The pre-registered baseline. Defaults to the first condition and is frozen at launch. | +| `cloned_from`, `created_by_schedule` | Provenance of clones. | ### Conditions: one call, a cross-product @@ -33,7 +56,7 @@ Everything lives under **Experiments** (`/experiments`). Automation (cron schedu "extra": [{"model_id": 11, "harness_preset_id": 7, "label": "cold", "param_overrides": {"temperature": 0}}], "baseline": {"model_id": 11, "harness_preset_id": 2} }, - "execution": {"scheduling_policy": "auto", "priority": 0}, + "execution": {"scheduling_policy": "auto", "priority": 0, "options": {"warmup": true}}, "post_run": {"evaluate": true, "report": {"enabled": true, "exclude_truncated": true}}, "schedules": [{"action": "top_up", "cron": "0 3 * * *", "timezone": "Europe/Warsaw"}] } @@ -42,7 +65,7 @@ Everything lives under **Experiments** (`/experiments`). Automation (cron schedu - `cross` expands as a product: agent mode is models × harness choices (`harness_preset_ids` and/or bare `harnesses`, which run that harness's system default preset); scaffolding mode is models × `profile_ids` (explicit profiles, no "Auto"). - `extra` adds single conditions, typically ablations. Sampling parameters in `cross` apply identically to every model; per-model parameter changes exist only as `extra` entries, and `param_overrides` always shows in `differs_on`. - Conditions get auto-labels: `"{model} · {harness}@{preset}"`, plus `" · ±{hash6}"` for parameter ablations. -- Dedupe is O(N): every candidate is resolved once (`experiment_layers`) and keyed on what it runs (model, profile, harness, normalised options, overrides). Two candidates that run the same thing (e.g. harness `aider` and the `aider-default` preset) are one arm. Create drops the duplicate; `POST …/conditions/bulk/` returns it in `skipped` with `reason: "duplicate"`. A profile on another stack is skipped as `"ineligible"`. +- Dedupe: every candidate is resolved once (`experiment_layers`) and keyed on what it runs (model, profile, harness, normalised options, overrides). Two candidates that run the same thing (e.g. harness `aider` and the `aider-default` preset) are one arm. Create drops the duplicate; `POST …/conditions/bulk/` returns it in `skipped` with `reason: "duplicate"`. A profile on another stack is skipped as `"ineligible"`. - The condition table holds **one row per (model, profile, harness, harness preset)**. A parameter-only ablation of an existing arm therefore needs its own harness preset. - Models resolve through `LLMModel.objects.visible_to(user)`: an invisible or unknown model is a 404 and nothing is written. - A tool-less model on a tool-calling harness is not refused when the condition is added; it is the blocking preview issue `models_without_tools`. @@ -58,25 +81,29 @@ Later wins. One helper (`services/experiments.py:experiment_layers`) resolves th - **Harness options**: the condition's harness preset. - **Stack**: the experiment's `stack_slug`, else the condition profile's stack, else the platform default. +### Analysis plan + +`analysis_plan` (validated by `services/analysis_plan.py`, 400 `invalid_analysis_plan` otherwise) states before launch what will be tested: `research_questions` (up to 20, each with a metric and direction `two-sided` / `greater` / `less`), `primary_metric` and `secondary_metrics` (`deploy_rate`, `pass_rate`, `functional_passed`, `unique_issues_per_app`, `issues_per_kloc`, `security_per_app`, `cost_per_app`), `alpha` (0 < α < 0.5, default 0.05), `correction` (`holm`, `bh`, `none`), `ranking_criteria`, `exclusions`, `min_apps_per_condition` and `notes`. The first launch freezes it with a sha256 of its canonical JSON (`analysis_plan_hash`, `analysis_plan_frozen_at`); PATCH on a launched experiment ignores it. A clone copies the plan unfrozen. The study report labels declared analyses confirmatory and everything else exploratory; see [Study tables](/docs/STUDY_TABLES#pre-registered-analysis-plan). + ## Preview -`POST /experiments/preview/` takes an unsaved create body and returns the plan without writing anything (the composer's plan rail). `GET /experiments/{id}/preview/` does the same for a saved experiment. Besides the matrix size, cost estimate and `launch_errors`, the preview has: +`POST /experiments/preview/` takes an unsaved create body and returns the plan without writing anything (the composer's Plan). `GET /experiments/{id}/preview/` (`POST` is an alias kept for older clients) does the same for a saved experiment. Besides the matrix size, cost estimate and `launch_errors`, the preview has: -- **`comparability`** (agent mode): per condition, what will actually run: harness, version, preset and options, sampling policy, forced params, output cap, provider routing (`n/a` for local models), seed support, dropped params, context window, stack, profile, overrides, and where it runs (`endpoint`, `digest`, `num_ctx`, `api_flavor`, `quantization`). `differs_on` names the columns that vary, so an unintended difference shows before anything runs. -- **`issues`**: blocking ones stop a launch: `missing_api_key`, `endpoint_disabled`, `endpoint_forbidden`, `model_unavailable`, `models_without_tools`, `digest_drift` (top-up only), plus every invalid preset, stack mismatch or resolution error. Non-blocking: `endpoint_offline`, `model_not_served`, `foreign_endpoint`, `num_ctx_small`, `num_ctx_server_default`, `model_thrash_risk`, `time_confounded`, `block_grouped`, `digest_unknown`, `same_digest_multiple_endpoints`, `wall_clock_tight`. -- **`endpoints`**: one row per endpoint the conditions use, with liveness, job cap, resident-model limit K, the models, the cells, the median run time (`est_run_s_p50`: the last 50 completed jobs of the model in this mode, else of the endpoint) and `est_hours = cells × p50 / min(job_cap, slots)`. -- **`schedule`**: the effective policy per endpoint, the order seed and the estimated model loads per endpoint. -- **`capacity`**, **`estimate`** (`total_hours`, `bottleneck`) and **`cost`** (`openrouter_upper_bound_usd`, `local_cells`, `kind`: `budget_upper_bound`, `heuristic`, `local_zero` or `mixed`). +- **`comparability`** (agent mode): per condition, what will actually run: harness, version, preset and options, sampling policy, forced params, output cap, provider routing (`n/a` for local models), seed support, dropped params, context window, stack, profile, overrides, and where it runs (`endpoint` slug, `openrouter` for the system endpoint; `digest`, `num_ctx`, `api_flavor`, `quantization`). `differs_on` names the columns that vary, so an unintended difference shows before anything runs. +- **`issues`**, each `{code, detail, blocking, model_id, endpoint, condition_id}`. Blocking ones stop a launch: `missing_api_key`, `endpoint_disabled`, `endpoint_forbidden`, `model_unavailable`, `models_without_tools`, `digest_drift` (launched experiments only), plus every invalid preset, stack mismatch or resolution error. Non-blocking: `endpoint_offline`, `model_not_served`, `foreign_endpoint`, `params_dropped`, `support_unknown`, `num_ctx_small` (agent context below 16 384), `num_ctx_server_default`, `model_thrash_risk`, `time_confounded`, `block_grouped`, `digest_unknown`, `same_digest_multiple_endpoints`, `wall_clock_tight`. +- **`endpoints`**: one row per endpoint the conditions use, with liveness and sessions, `max_concurrency`, `job_cap`, `resident_limit` (K), the models, the pending cells, the median run time (`est_run_s_p50`: the last 50 completed jobs of the model in this mode, else of the endpoint) and `est_hours = cells × p50 / min(job_cap, slots)`. +- **`schedule`**: the effective policy per endpoint, its reason, the order seed and the estimated model loads per endpoint. +- **`capacity`** (slots, `est_parallel_runs`), **`estimate`** (`total_hours`, `bottleneck`: `sandbox` or `endpoint:`) and **`cost`** (`openrouter_upper_bound_usd`, `local_cells`, `kind`: `budget_upper_bound`, `heuristic`, `local_zero` or `mixed`). ## Launch and execution `POST /experiments/{id}/launch/` (body `{mode: "launch" | "top_up", condition_ids?}`) or `POST /experiments/?launch=true` (create and launch in one call; a failed launch keeps the draft and returns 400 with `experiment_id`). -1. Every targeted condition is resolved first. A blocking issue refuses the whole launch with `400 {detail, code, errors[]}` and nothing is written. -2. A draft becomes `running`, gets `launched_at`, its baseline defaults to the first condition, and each local condition pins its model's digest (`pinned_digest`). -3. One job is created per uncovered cell, in bulk (chunks of 500). A cell is covered by a live or completed job; failed and cancelled cells are re-run by a later top-up. -4. Each job gets its frozen snapshot, its cell seed and its execution fields: `sequence` (its position in the run order), `lane` (`sandbox` for agent runs, `llm` for scaffolding), `endpoint` (NULL for OpenRouter), `priority` and `execution.policy`. Then the jobs are handed to the executor (`execution.api.enqueue_jobs`), which claims them strictly in sequence order per (experiment, endpoint) lane. -5. The batch (`GenerationBatch`, linked by `experiment` and `kind` = `launch`, `top_up`, `retry` or `schedule`) records the design and `request.schedule`: requested and effective policies, order seed, K per endpoint, estimated swaps and a `history` of later policy and digest changes. +1. Every targeted condition is resolved first. A blocking issue refuses the whole launch with `400 {detail, code, errors[]}` and nothing is written. An experiment without conditions (`no_conditions`) or apps (`no_apps`), or an archived one, is refused too. +2. A draft becomes `running`, gets `launched_at`, freezes its analysis plan, its baseline defaults to the first condition, and each local condition pins its model's digest (`pinned_digest`). +3. One job is created per uncovered cell, in bulk (chunks of 500). A cell is covered by a live or completed job; failed and cancelled cells are re-run by a later top-up. A cell whose snapshot cannot be built is created `failed` and runs nothing. +4. Each job gets its frozen snapshot, its cell seed, `build_healing` and its execution fields: `sequence` (its position in the run order), `lane` (`sandbox` for agent runs, `llm` for scaffolding), `endpoint` (NULL for OpenRouter), `priority` and `execution.policy`. Then the jobs are handed to the executor (`execution.api.enqueue_jobs`), which claims them strictly in sequence order per (experiment, endpoint) lane. +5. The batch (`GenerationBatch`, linked by `experiment`; `kind` = `launch`, `top_up`, `retry` when every filled cell was attempted before, or `schedule` for schedule fires) records the design and `request.schedule`: requested and effective policies, order seed, K per endpoint, estimated swaps and a `history` of later policy and digest changes. The launch response carries `jobs_created`, `policy_effective_by_endpoint` and the non-blocking `warnings`. **Top-up** is the same call on a launched experiment. It is idempotent: only uncovered cells get jobs, optionally only for `condition_ids`. A completed or cancelled experiment is reopened; a paused one stays paused. Raising `repeats` after launch (allowed: repeats only increase) and topping up adds new repeat blocks after the existing ones. @@ -100,29 +127,39 @@ With `base_seed` set, each cell's seed is `profile_resolver.derive_experiment_se - agent mode: `(app_id, repeat_index)`, so every condition of an (app, repeat) cell shares its seed (common random numbers); - scaffolding mode: `(condition_id, app_id, repeat_index)`, the historical derivation. -Seeds do not depend on the run order or the policy. A top-up re-derives the same seed for every cell it fills; infra retries reuse the same row, snapshot and seed. Retrying a failed or cancelled job from its job page takes over the job's cell and its seed (and its run position); it is refused when the cell is already filled again. Replays and reruns are extra runs outside the design. +Seeds do not depend on the run order or the policy. A top-up re-derives the same seed for every cell it fills; infra retries reuse the same row, snapshot and seed. Retrying a failed or cancelled job from its job page (`POST /api/generation/jobs/{id}/retry/`) takes over the job's cell and its seed (and its run position); it is refused when the cell is already filled again. Replays and reruns are extra runs outside the design. ### Pause, resume, cancel - **Pause** (`POST …/pause/`): `running → paused`. The executor stops claiming the experiment's jobs; running jobs finish. -- **Resume** (`POST …/resume/`, body `{accept_digest_drift?}`): `paused → running`. When a local model's weights changed (`pause_reason: digest_changed`, set by the endpoint inventory), resume needs `accept_digest_drift: true`: the pending jobs' snapshots are re-frozen to the new digest with the same seeds, the conditions re-pin the digest, and the change is recorded in the schedule history. Results then flag those conditions `mixed_fingerprints`. The alternative is a clone. -- **Cancel** (`POST …/cancel/`, body `{scope: "all" | "pending"}`): pending jobs are cancelled at once, running generations are signalled, and jobs already waiting for or in evaluation finish and are graded. Scope `all` also ends the experiment as `cancelled`. A draft, completed or archived experiment has nothing to cancel (400 `not_active`). -- **Archive** is refused while jobs are active, and disables the experiment's schedules. **Delete** works on drafts and on finished experiments without active jobs. +- **Resume** (`POST …/resume/`, body `{accept_digest_drift?}`): `paused → running`. When a local model's weights changed (`pause_reason: digest_changed`), resume needs `accept_digest_drift: true`: the pending jobs' snapshots are re-frozen to the new digest with the same seeds, the conditions re-pin the digest, and the change is recorded in the schedule history. Results then flag those conditions `mixed_fingerprints`. The alternative is a clone ("Clone with current digest" in the UI). +- **Cancel** (`POST …/cancel/`, body `{scope: "all" | "pending"}`): pending jobs are cancelled at once, running generations are signalled, and jobs already waiting for or in evaluation finish and are graded. Scope `all` also ends the experiment as `cancelled`. A draft, completed or archived experiment has nothing to cancel (400 `not_active`). The response counts `cancelled_pending`, `signalled_running` and `finishing_evaluations`. +- **Archive** is refused while jobs are active (`active_jobs`), and disables the experiment's schedules. **Delete** works on drafts and on finished experiments without active jobs; their jobs stay, unlinked. - **Requeue legacy jobs** (`POST …/jobs/requeue-legacy/`) hands jobs created before the executor existed (pending or running rows with no `queued_at`) to the executor. -- **Clone** (`POST …/clone/`, body `{name?, slug?, include_schedules, include_post_run, overrides?, launch?}`) copies the design into a new draft (`cloned_from` set; default slug `slugify(name)[:40]-YYYYMMDD-HHMM`, with `-2`, `-3`… on collision). Conditions point at the same model rows, whose digests are current. +- **Clone** (`POST …/clone/`, body `{name?, slug?, include_schedules, include_post_run, overrides?, launch?}`) copies the design into a new draft (`cloned_from` set; default name `" (copy)"`, default slug `slugify(name)[:40]-YYYYMMDD-HHMM`, with `-2`, `-3`… on collision). Conditions point at the same model rows, whose digests are current. -PATCH on a launched experiment takes only `name`, `description`, `hypothesis`, `priority` (also re-prioritises pending jobs and their queued LLM requests), `max_concurrency`, `infra_retries`, `execution_options`, `post_run`, `repeats` (increase only) and `scheduling_policy` (paused only). The baseline is frozen at launch. +PATCH on a launched experiment takes only `name`, `description`, `hypothesis`, `priority` (also re-prioritises pending jobs and their queued LLM requests), `max_concurrency`, `infra_retries`, `execution_options`, `post_run`, `repeats` (increase only) and `scheduling_policy` (paused only). Everything else, including the baseline and the analysis plan, is frozen at launch. ## Monitoring -- `GET /experiments/` lists your experiments, paginated (`page`, `per_page` ≤ 100), filterable by `status` (`active` = running + paused, `finished` = completed + cancelled, `draft`, `archived` or an exact status), `mode`, `search` and `endpoint` (a slug, `openrouter` or `local`), ordered by `-updated_at`, `name` or `-launched_at`. Each row carries progress over the current attempt of every cell, the cost so far, an ETA, the endpoints it uses and its next scheduled run. -- `GET /experiments/{id}/status/` returns rollups without the grid: counts, cells (`done`, `failed_final`, `in_flight`, `retried`), throughput, ETA and bottleneck, per-condition progress and pass rate, per-endpoint lanes (slots, active jobs, resident and loaded models, queue, tokens/s; other tenants' models appear only as counts and `last_error` only for the endpoint owner), executor lanes and the latest post-run. -- `GET /experiments/{id}/cells/` returns the compact grid: `[condition_index, app_index, repeat, state, job_id, pass_rate, attempt]` per cell, where the state is one letter: `p` pending, `b` retry backoff, `w` waiting for a model prefetch, `r` running, `q` waiting for evaluation, `e` evaluating, `c` passed, `f` graded but not passed, `u` completed but ungraded, `x` failed, `k` cancelled. +- `GET /experiments/` lists your experiments, paginated (`page`, `per_page` ≤ 100, default 25), filterable by `status` (`active` = running + paused, `finished` = completed + cancelled, or an exact status such as `draft` or `archived`), `mode`, `search` and `endpoint` (a slug, `openrouter` or `local`), ordered by `-updated_at` (default), `name` or `-launched_at`. Each row carries progress over the current attempt of every cell, the cost so far, an ETA, the endpoints it uses and its next scheduled run. +- `GET /experiments/{id}/status/` returns rollups without the grid: counts, cells (`done`, `failed_final`, `in_flight`, `retried`), throughput, ETA and bottleneck, per-condition progress and pass rate, per-endpoint lanes (slots, active jobs, resident and loaded models, queue, tokens/s; other tenants' models appear only as counts and `last_error` only for the endpoint owner), executor state (`online`, `draining`, `code_stale`: the live leader runs older code than the web process) and the latest post-run. +- `GET /experiments/{id}/cells/` returns the compact grid: `[condition_index, app_index, repeat, state, job_id, pass_rate, attempt]` per cell, where the state is one letter: + + | Letter | State | Letter | State | + |---|---|---|---| + | `p` | pending | `c` | passed every check | + | `b` | retry backoff (`run_after` in the future) | `f` | graded, not passed | + | `w` | waiting for a model prefetch | `u` | completed, ungraded | + | `r` | running | `x` | failed | + | `q` | waiting for evaluation | `k` | cancelled | + | `e` | evaluating | | | + - Live updates arrive over SSE on `experiment:`: `job`, `tick`, `state` and `post_run` events (see [API reference](/docs/api-reference#realtime-stream)). ## Results and statistics -`GET /experiments/{id}/results/?baseline=&exclude_invalid=false&exclude_truncated=true&common_cells=true&fingerprint=all|majority&metric=pass_rate` is computed by `generation/services/experiment_results.py:compute`. The Results tab, the results matrix and the auto-generated `experiment_report` all use this one implementation. +`GET /experiments/{id}/results/?baseline=&exclude_invalid=false&exclude_truncated=true&common_cells=true&fingerprint=all|majority&metric=pass_rate` is computed by `generation/services/experiment_results.py:compute`; `pass_rate` is the only supported metric (anything else is 400 `invalid_results_query`). The Results section, the results matrix and the `experiment_report` all use this one implementation. - **The unit of analysis is the app.** Repeats of one (condition, app) share the app (and, in agent mode, the seed), so they are strongly correlated. Each (condition, app) collapses to the mean of its graded repeats; a condition's score is the mean of its app means. - **Usable cells**: the current attempt (latest job) of a cell counts when it is completed and graded (`metrics.functional.pass_rate` not null). Missing cells are counted by reason in `n_missing`: `failed` (failed or cancelled), `ungraded`, `excluded_invalid`, `excluded_truncated`, `not_common`. @@ -136,13 +173,13 @@ PATCH on a launched experiment takes only `name`, `description`, `hypothesis`, ` - **Mixed fingerprints**: conditions whose current jobs span several `config_fingerprint`s (after accepting a digest change, or a prompt version change) report `mixed_fingerprints`; `fingerprint=majority` keeps only the majority fingerprint's cells. - **Timings** confound the model with the machine and its load; each row names its `timing_endpoint_slug`, and a note warns when conditions span several endpoints. -`GET /experiments/{id}/export/?format=json|csv` exports the resolved configuration (with preset provenance), the conditions and every job with its seed, fingerprints, endpoint, digest, `num_ctx`, attempt and execution record (relay session details are left out). CSV has one row per job. +`GET /experiments/{id}/export/?format=json|csv` exports the resolved configuration (with preset provenance, build healing and the analysis plan with its hash), the conditions and every job with its seed, fingerprints, endpoint, digest, `num_ctx`, attempt and execution record (relay session details are left out). CSV has one row per job. For paper-ready tables over an experiment (outcome funnel, deduplicated findings, planned comparisons) use the study report; see [Study tables](/docs/STUDY_TABLES). ## Automation ### Schedules -A schedule (`ExperimentSchedule`) fires an action on a cron expression in its own timezone. The minimum interval is 15 minutes. +A schedule (`ExperimentSchedule`) fires an action on a cron expression in its own timezone. The minimum interval is 15 minutes (`SCHEDULE_MIN_INTERVAL_S`). | Action | Effect | |---|---| @@ -150,9 +187,9 @@ A schedule (`ExperimentSchedule`) fires an action on a cron expression in its ow | `add_replicates` | `repeats += params.replicates` (1..5), then top up: new repeat blocks run after the existing ones. | | `clone_and_run` | Clone the experiment (name from `params.name_template`, default `"{name} {date}"`; `created_by_schedule` set) and launch the clone. | -The executor checks for due schedules every 30 s. It claims them with `SKIP LOCKED` (two ticks never fire one schedule twice) and computes the next run from `max(now, next_run_at)`, so an executor that was down never fires a burst of catch-up runs. With `skip_if_active` (default) a schedule whose experiment still has pending or running jobs, or is paused, records `{outcome: "skipped"}`. The action itself runs in the executor's post-run pool. Errors are recorded in `last_result`, never raised; `top_up` and `add_replicates` record `digest_drift` when a local model's weights changed since launch, and a refused launch also rolls back the replicates it would have added. `top_up` and `add_replicates` never launch a draft (`skipped`, "experiment not launched yet"), and a `top_up` with no failed, cancelled or new cell is `skipped` ("nothing to top up") instead of creating an empty batch. Draining the executor stops schedule firing. +The executor checks for due schedules every 30 s. It claims them with `SKIP LOCKED` (two ticks never fire one schedule twice) and computes the next run from `max(now, next_run_at)` (DST-safe), so an executor that was down never fires a burst of catch-up runs. With `skip_if_active` (default) a fire is recorded as `{outcome: "skipped"}` when the experiment still has pending or running jobs or is paused, and for `clone_and_run` also while a clone it launched earlier is still active. The action runs on the executor's own single-thread schedule pool. Errors are recorded in `last_result` (`{outcome: fired|skipped|error, batch_id?, experiment_id?, jobs_created?, detail}`), never raised; `top_up` and `add_replicates` record `digest_drift` when a local model's weights changed since launch, and a refused launch also rolls back the replicates it would have added. `top_up` and `add_replicates` never launch a draft (`skipped`, "experiment not launched yet"), and a `top_up` with no failed, cancelled or new cell is `skipped` ("nothing to top up") instead of creating an empty batch. Draining the executor stops schedule firing. -Routes: `GET/POST …/schedules/`, `PATCH/DELETE …/schedules/{sid}/`, `POST …/schedules/{sid}/run-now/` (fires the action now), and `GET /generation/cron-preview/?cron=&tz=` (validity and the next five runs). +Routes: `GET/POST …/schedules/`, `PATCH/DELETE …/schedules/{sid}/`, `POST …/schedules/{sid}/run-now/` (fires the action now), and `GET /api/generation/cron-preview/?cron=&tz=` (validity and the next five runs). ### Post-run @@ -165,26 +202,27 @@ Routes: `GET/POST …/schedules/`, `PATCH/DELETE …/schedules/{sid}/`, `POST ``` - `evaluate` grades each job right after its run (the hidden conformance suite); it is a per-job step, not a post-run step. -- When the experiment completes (exactly once per completion, even under concurrent job ends) an `ExperimentPostRun` is created with a frozen copy of the configuration, if analysis or the report is enabled. `POST …/post-run/run/` queues one by hand (400 while one is active). -- The executor claims post-runs one at a time (leased). Analyses run **sequentially** (the analyzer workspace is shared) over the completed current-attempt jobs that have no analysis yet (`only_passing`: pass rate 1 only); a failed analysis is counted and the rest continue. Then the `experiment_report` is generated synchronously, owned by the experiment's owner. Progress is published as `post_run` events. -- When the apps daemon's memory cannot fit the analyzer workspace next to full sandbox and evaluation lanes (see [Execution](/docs/EXECUTION)), post-runs wait until those lanes have room. +- Analysis needs at least one analyzer (`no_analyzers`); unknown slugs are a 400 (`unknown_analyzer`), and so are dynamic analyzers (`analysis_mode: dynamic`, e.g. ZAP, Nuclei, Wapiti; `dynamic_analyzer`): they need a running app, which post-run analysis does not start. The UI only offers installed static and AI analyzers. See [Analysis pipeline](/docs/ANALYSIS_PIPELINE). +- When the experiment completes (exactly once per completion, even under concurrent job ends) an `ExperimentPostRun` (trigger `completed`) is created with a frozen copy of the configuration, if analysis or the report is enabled. `POST …/post-run/run/` queues one by hand (trigger `manual`; 400 while one is active). +- The executor claims post-runs one at a time (leased; status `pending → running → completed | failed | cancelled`). Analyses run **sequentially** (the analyzer workspace is shared) over the completed current-attempt jobs that have no analysis yet (`only_passing`: pass rate 1 only); a failed analysis is counted (`progress.analysis_failed`) and the rest continue. Then the `experiment_report` is generated synchronously, owned by the experiment's owner. Progress is published as `post_run` events. A running post-run whose lease expired (executor restart) is claimed again and resumed. +- When the apps daemon's memory cannot fit the analyzer workspace next to full sandbox and evaluation lanes (see [Execution](/docs/EXECUTION#slots-and-the-memory-rule)), post-runs wait until those lanes have room. `GET/PUT …/post-run/` reads and sets the configuration and lists recent post-runs. ### Reports -The `experiment_report` (`config: {"experiment_id", "exclude_truncated"}`) is owner-scoped: only the experiment's owner (or staff) can generate it. It embeds the leaderboard and the paired-vs-baseline analysis from `experiment_results.compute`, plus per-condition code statistics (findings, LOC, cost, duration). It no longer diffs every condition pair. +The `experiment_report` (`config: {"experiment_id", "exclude_truncated"}`) is owner-scoped: only the experiment's owner (or staff) can generate it. It embeds the leaderboard and the paired-vs-baseline analysis from `experiment_results.compute`, plus per-condition code statistics (findings, LOC, cost, duration). It does not diff every condition pair. The broader `study_report` (same scoping) is described in [Study tables](/docs/STUDY_TABLES#study-report). ## API summary -All routes are under `/api/generation/` and owner-scoped (another user's experiment, condition or schedule is a 404). +All routes are under `/api/generation/` and owner-scoped (another user's experiment, condition or schedule is a 404). Full request and response shapes are in [API reference](/docs/api-reference). | Route | Purpose | |---|---| | `GET/POST /experiments/` | list (paginated) / create with conditions (`?launch=true`) | | `POST /experiments/preview/` | stateless preview of an unsaved design | | `GET/PATCH/DELETE /experiments/{id}/` | detail / update / delete | -| `GET/POST/PUT /experiments/{id}/conditions/`, `POST …/conditions/bulk/`, `DELETE …/conditions/{cid}/` | conditions | +| `GET/POST/PUT /experiments/{id}/conditions/`, `POST …/conditions/bulk/`, `DELETE …/conditions/{cid}/` | conditions (draft only) | | `GET/POST /experiments/{id}/preview/` | preview | | `POST /experiments/{id}/launch/` | launch or top up | | `POST /experiments/{id}/pause/`, `resume/`, `cancel/`, `archive/`, `clone/` | lifecycle | diff --git a/docs/GENERATION_PROCESS.md b/docs/GENERATION_PROCESS.md index f480fc60..c69bf0f8 100644 --- a/docs/GENERATION_PROCESS.md +++ b/docs/GENERATION_PROCESS.md @@ -1,297 +1,352 @@ # Generation process -How a sample app gets made: the primary **agent** mode (a coding-agent harness builds the app in a sandbox and the platform grades it with a hidden conformance suite), the secondary template modes, and what each run records so results stay comparable. +How a sample app gets made: in the primary **agent** mode a coding-agent harness builds the app in a sandbox and the platform grades it with a hidden conformance suite; the secondary template modes are also covered, along with what each run records so results stay comparable. ## Modes -`GenerationJob.mode` (`backend/generation/models.py`) selects the strategy: +`GenerationJob.mode` (`backend/generation/models.py`) selects the strategy. The Sample Generator (`/sample-generator?mode=agent|scaffolding|custom`) has one composer per live mode. -| Mode | What it does | Use it for | -| --- | --- | --- | -| `agent` (default) | A coding-agent **harness** (OpenCode, mini-swe-agent or Aider) works on the stack skeleton inside an isolated sandbox until it finishes or its budget runs out; the platform then builds and grades the result (`services/agent/`, `services/evaluation/`) | (model × harness) comparisons on realistic, multi-file, self-tested apps | -| `scaffolding` | A generation profile renders backend/frontend prompts; two LLM calls fill a known skeleton | One-shot "how well does a model write code from a prompt" comparisons | -| `custom` | Your prompt goes straight to the model | Ad-hoc probing | -| `copilot` | *Retired.* The in-process aider loop. Kept only so historical jobs still load; `POST /jobs/copilot/` answers 410. (Aider itself lives on as a sandboxed agent harness) | — | +| Mode | UI label | What it does | Use it for | +| --- | --- | --- | --- | +| `agent` (default) | Agent | A coding-agent **harness** (OpenCode, mini-swe-agent or Aider) works on the stack skeleton inside an isolated sandbox until it finishes or its budget runs out. The platform then builds and grades the result (`services/agent/`, `services/evaluation/`) | (model × harness) comparisons on realistic, multi-file, self-tested apps | +| `scaffolding` | Template scaffolding | A generation profile renders backend/frontend prompts, and two LLM calls fill a known skeleton | One-shot "how well does a model write code from a prompt" comparisons | +| `custom` | Single prompt | Your system and user prompt go straight to one model. Nothing is added or graded | Ad-hoc probing | +| `copilot` | none | *Retired.* The in-process aider loop. Kept only so historical jobs still load, and `POST /jobs/copilot/` answers 410 `mode_retired`. Aider itself lives on as a sandboxed agent harness | none | + +Every mode runs on the **executor**, not in the web process (see [Job lifecycle](#job-lifecycle-all-modes) and [Execution engine](/docs/EXECUTION)). ## Agent mode ### What runs where ``` -Django ──(apps daemon, mTLS)──▶ apps-dind - run: llmlab-gateway- bridge + job net holds the real OpenRouter key - llmlab-agent- job net only harness + workspace, no key - evaluate: llmlab-eval-app- eval net only the built sample, no egress - llmlab-eval-tester-… eval net only hidden conformance suite +executor ──(apps daemon, mTLS)──▶ apps-dind + run: llmlab-gateway--a bridge + job net holds the real API key + llmlab-agent--a job net only harness + workspace, no key + evaluate: llmlab-eval-app- eval net only the built sample, no egress + llmlab-eval-tester- eval net only hidden conformance suite ``` -- **Per-job sandbox** (`services/agent/sandbox.py`). There is one internal Docker network per job. The agent container is attached only to that network, so its only route is the gateway. It runs unprivileged (all caps dropped, `no-new-privileges`) with memory, CPU and PID caps that are frozen into the snapshot. Everything is a nested container on apps-dind, never the host daemon; it is labelled with the job id and removed in `finally`. *Purge orphan containers* in System → Maintenance reaps leftovers from a crashed process. -- **Images** (`images/`, `services/agent/images.py`). Every harness image builds `FROM images/agent-base`: Node 22.23.3 + Python 3.11 + git on an exactly pinned base release. So the tools an agent finds are byte-identical across harnesses; only the harness layer differs. Python harnesses install a fully locked dependency set (`requirements.lock`) into a private venv that the agent's own `python3`/`pip` never sees. Tags are content hashes of the image directory and build args; the harness tag covers the base tag. The snapshot records the tags at job creation, and a job refuses to run if an image definition changed since. -- **LLM gateway** (`images/llm-gateway/gateway.py`, stdlib only). The harness is configured with a dummy key and the gateway as its OpenAI/OpenRouter base URL. The gateway: +`-a` is the attempt number. An infrastructure retry gets fresh containers and networks, so the leftovers of an earlier attempt never collide with it. + +- **Per-job sandbox** (`services/agent/sandbox.py`). Each job gets its own internal Docker network. The agent container is attached only to that network, so its only route is the gateway. It runs unprivileged (all caps dropped, `no-new-privileges`, 1024 PIDs), and its memory and CPU caps are frozen into the snapshot. Everything runs as a nested container on apps-dind, never on the host daemon. Resources are labelled with the job id and removed in `finally`. Leftovers of a crashed process are reaped by the executor's reconciler and by *Purge Orphan Containers* in System → Maintenance. +- **Images** (`backend/generation/images/`, `services/agent/images.py`). Every harness image builds `FROM agent-base`, which is `node:22.23.3-bookworm-slim` plus Debian's Python 3, git and ripgrep. The tools an agent finds are therefore byte-identical across harnesses, and only the harness layer differs. Python harnesses install a fully locked dependency set (`requirements.lock`) into a private venv that the agent's own `python3`/`pip` never sees. Tags (`llmlab/:`) hash the image directory and its build args, and a harness tag covers the base tag. The snapshot records the tags at job creation. A job refuses to run if an image definition has changed since then. +- **LLM gateway** (`images/llm-gateway/gateway.py`, stdlib only). The harness gets a dummy key and the gateway as its OpenAI/OpenRouter base URL. The gateway: - pins the model, - applies the job's sampling policy (below), - forces the job's seed, provider routing and per-call output cap, - - retries transient upstream failures itself (408/429/5xx, connection errors, with backoff), - - counts only successful calls toward the step budget, - - answers a non-retryable 400 once the budget is spent, + - retries transient upstream failures itself (408/429/5xx and connection errors, with backoff), + - counts only successful calls toward the request budget, + - answers a non-retryable 400 `budget_exhausted` once the budget is spent (a 400 rather than a 429, because harnesses retry 429s), - logs every exchange. -- **Egress.** Under network policy `packages` (the default), the gateway also tunnels HTTPS `CONNECT` to `pypi.org`, `files.pythonhosted.org` and `registry.npmjs.org`. Everything else is refused and logged. Under policy `none`, nothing is reachable. +- **Egress.** Under network policy `packages` (the default) the gateway also tunnels HTTPS `CONNECT` to `pypi.org`, `files.pythonhosted.org` and `registry.npmjs.org`. Everything else is refused and logged. Under `none`, nothing is reachable. +- **Local models.** For a model on a local endpoint (Ollama or an OpenAI-compatible server), the gateway is the `llm-gateway-local` overlay on the dedicated `llmlab-gw-egress` network. Its only upstream is the executor's internal LLM proxy (`backend/inference/gateway_wiring.py`). Such runs need apps-dind (`LLM_LOCAL_AGENT_RUNS`, on when `APPS_DOCKER_HOST` is set), and they fail fast without it. See [Local models](/docs/LOCAL_MODELS). ### Harnesses (`services/agent/harnesses/`) A harness is the agent program that drives the model. Benchmarks compare (harness, model) pairs: - **Within a harness**, every model gets byte-identical treatment. Anything a harness would normally switch per model family is pinned: prompts, edit formats, temperatures. -- **Across harnesses**, each keeps its own native prompts, frozen; the harness's design is what is being compared. All of them get the same sandbox image, the same brief, the same gateway policy and the same budgets. +- **Across harnesses**, each keeps its own native prompts, frozen, because the harness's design is what is being compared. All of them get the same sandbox image, brief, gateway policy and budgets. | Harness | Paradigm | Tool calling | What is pinned beyond the release | | --- | --- | --- | --- | -| `opencode` ([OpenCode](https://opencode.ai) 1.18.35) | Full IDE-style agent: read/edit/write/grep/glob/bash/todo/task tools | required | Its own model-agnostic default prompt (captured verbatim in `data/agent/harness_prompts/`) for the `build` agent **and** the `general` subagent; OpenCode otherwise picks a per-family prompt for both. Web fetch/search and skills off; config/AGENTS.md discovery, plugins and auto-update off; session title passed in (no hidden title call); `doom_loop` explicitly denied | -| `mini-swe-agent` ([mini-swe-agent](https://mini-swe-agent.com) 2.4.6) | Minimal research baseline: one `bash` tool, fresh subshell per command | required | Native `mini.yaml` templates; `yolo`, `step_limit` = request budget, cost limit off (the gateway owns cost) | -| `aider` ([Aider](https://aider.chat) 0.86.2) | Edit-format pair programmer, one headless `--message` turn plus its native lint-fix reflection | not needed | Edit format `diff` for every model by default (Aider otherwise picks it per model); weak/editor model = the run's model; the same model-metadata file for every model (context window, output cap), since Aider's own database does not know gateway-routed ids; repo map 1024 tokens by default; no commits or shell-command suggestions; all scaffold sources added to the chat | +| `opencode` ([OpenCode](https://opencode.ai) 1.18.35) | Full IDE-style agent with read/edit/write/grep/glob/bash/todo/task tools | required | Its own model-agnostic default prompt, captured verbatim in `data/agent/harness_prompts/`, for the `build` agent **and** the `general` subagent (OpenCode otherwise picks a per-family prompt for both). Web fetch/search and skills are off, as are config/AGENTS.md discovery, plugins and auto-update. The session title is passed in, so there is no hidden title call, and `doom_loop` is explicitly denied | +| `mini-swe-agent` ([mini-swe-agent](https://mini-swe-agent.com) 2.4.6) | Minimal research baseline: one `bash` tool, a fresh subshell per command | required | Native `mini.yaml` templates, `yolo`, `step_limit` set to the request budget, and the cost limit off (the gateway owns cost) | +| `aider` ([Aider](https://aider.chat) 0.86.2) | Edit-format pair programmer: one headless `--message` turn plus its native lint-fix reflection | not needed | Edit format `diff` for every model by default (Aider otherwise picks it per model), with the weak/editor model set to the run's model. Every model gets the same model-metadata file (context window, output cap), because Aider's own database does not know gateway-routed ids. Repo map is 1024 tokens by default. No commits or shell-command suggestions, and all scaffold sources are added to the chat | -**Harness options.** A few pinned values can be changed, but only through a harness preset (below) and only where the pinned source was checked to apply the value identically to every model family (each adapter's docstring records what was checked): +**Harness options.** A few pinned values can be changed, but only through a harness preset (below). An option is offered only where the pinned source was checked to apply it identically to every model family; each adapter's docstring records the check. | Harness | Option | Values (default) | | --- | --- | --- | | `aider` | `edit_format` | `diff` \| `whole` \| `udiff` \| `diff-fenced` (`diff`) | | `aider` | `map_tokens` | 0–8192 (1024); 0 disables the repo map | -| `mini-swe-agent` | `command_timeout_seconds` | 10–600 (30), mini's `environment.timeout` | +| `mini-swe-agent` | `command_timeout_seconds` | 10–600 (30): mini's `environment.timeout` | | `opencode` | none | OpenCode picks its editing tools by model id, so tool toggles would be model-specific | -Options are rendered only into the arguments and config a harness already emits, and only when they differ from the default, so a default section is byte-identical to one from before options existed. `GET /generation/agent/config/` lists every harness with its `option_schema` and `default_options`. +Options are rendered only into the arguments and config a harness already emits, and only when they differ from the default, so a default harness section is byte-identical to one from before options existed. `GET /api/generation/agent/config/` lists every harness with its `option_schema` and `default_options`. -Every harness gets the brief from a file or stdin, never argv. Agents restart their dev server with "kill every process mentioning `app.py`", and a brief in argv once made OpenCode kill itself. Harness artifacts (Aider's `.aider*` caches, histories) never enter the sample. +Every harness reads the brief from a file or stdin, never from argv. Agents restart their dev server with "kill every process mentioning `app.py`", and a brief in argv once made OpenCode kill itself. Harness artifacts (Aider's `.aider*` caches, histories) never enter the sample. ### The brief and the evaluation contract -The brief (`data/agent/brief.md.jinja2`, `BRIEF_VERSION` 3) is the only task-specific text an agent receives, identical for every harness and model: +The brief (`data/agent/brief.md.jinja2`, `BRIEF_VERSION` 3) is the only task-specific text an agent receives. It is identical for every harness and model: -- an **autonomy** statement: nobody will answer questions, finish with a summary, -- the **app spec**, verbatim: requirements, data model, every endpoint with its request/response shapes. A task `description` sent with an app spec follows the spec's own description as an **Additional context** section; without an app spec the description *is* the task (a free-form task, which has no conformance suite and is not graded), -- an **accounts contract** (`services/evaluation/contract.py`). Specs say *which* writes need a signed-in user but almost never *how* to sign in, so each agent invented its own scheme and write endpoints were ungradable. Unless the spec defines its own auth endpoints, the app exposes `POST /api/auth/register` and `POST /api/auth/login` (→ `{"token"}`) and accepts `Authorization: Bearer`. Endpoints reserved for signed-in users answer 401 anonymously. An admin account's password comes from `ADMIN_PASSWORD`; the evaluation sets it to a fresh random value, so a hard-coded credential fails, -- the **runtime contract**: entry point and start command, port, `/api/health`, SPA served for non-API paths, SQLite in an initially empty `./data/`, pinned dependencies, platform-owned build files. Builtin stacks take this text from `data/agent/stacks.yaml`; a user-authored stack states the command its generated Dockerfile runs (`python ` or `uvicorn :app …`, from its `server_kind`), -- the notice that the result is **checked automatically** against all of the above (the checks themselves stay hidden), +- an **autonomy** statement: nobody will answer questions, and the agent finishes with a summary, +- the **app spec**, verbatim: requirements, data model, and every endpoint with its request/response shapes. A task `description` sent along with an app spec appears after the spec's own description as an **Additional context** section. Without an app spec the description *is* the task: a free-form task, which has no conformance suite and is not graded, +- an **accounts contract** (`services/evaluation/contract.py`). Specs say *which* writes need a signed-in user but almost never *how* to sign in, so agents used to invent their own schemes and write endpoints could not be graded. Unless the spec defines its own auth endpoints, the app must expose `POST /api/auth/register` and `POST /api/auth/login` (returning `{"token"}`) and accept `Authorization: Bearer`. Endpoints reserved for signed-in users answer 401 to anonymous requests. Admin endpoints live under `/api/admin/` and use an admin account whose password comes from `ADMIN_PASSWORD`. The evaluation sets it to a fresh random value, so a hard-coded credential fails, +- the **runtime contract**: entry point and start command, port, `GET /api/health`, the SPA served for non-API paths, SQLite in an initially empty `./data/`, pinned dependencies, and platform-owned build files. Builtin stacks take this text from `data/agent/stacks.yaml`. A user-authored stack states the command its generated Dockerfile runs (`python ` or `uvicorn :app …`, from its `server_kind`), +- a notice that the result is **checked automatically** against all of the above (the checks themselves stay hidden), - the **budget**, with the request budget stated as the primary limit. There are no length targets and no style coaching. ### One run, step by step (`services/agent/runner.py`) -1. **Snapshot at job creation** (`services/agent/snapshot.py`). Everything is rendered and frozen into `resolved_bundle`: brief, contract, harness section, image tags, sampling and gateway policy, routing, budget, sandbox caps and the stack version. A run never re-reads a template, and older snapshot schemas are refused rather than replayed approximately. -2. **Queueing.** `RunQueue` admits runs to the sandbox slots (`AGENT_MAX_CONCURRENT_RUNS`, default 1) in **FIFO order of dispatch**, so the order an experiment shuffles its jobs into is the order they run in. Cancellation is polled while waiting; a dead reservation never blocks the queue. +1. **Snapshot at job creation** (`services/agent/snapshot.py`). Everything is rendered and frozen into `resolved_bundle`: brief, contract, harness section, image tags, sampling and gateway policy, routing, budget, sandbox caps, the stack version and, for local models, the endpoint section. A run never re-reads a template. Snapshots older than `AGENT_SNAPSHOT_SCHEMA_VERSION` (2) are refused rather than replayed approximately; *Retry* re-creates them under the current procedure. +2. **Queueing.** The job joins the executor's **sandbox lane** and is admitted in its frozen `sequence` order. The order an experiment or batch shuffles its jobs into is therefore the order they run in. Concurrency is `ExecutorSettings.sandbox_slots`, which starts at `AGENT_MAX_CONCURRENT_RUNS`, can be edited by staff at runtime, and is clamped by apps-dind memory. 3. **Seeding.** The stack skeleton is copied into `/workspace` and committed as `scaffold`. Dependency/build directories and `data/` go into `.git/info/exclude`. -4. **The agent run.** `/out/harness.sh` runs under `timeout`. A heartbeat touches the job row every minute. -5. **Collection.** The run records the patch, diffstat and name-status, and the workspace is classified against the seed (`added` / `modified` / `scaffold`). Binaries, oversized files and harness artifacts are dropped. Tampering with platform-owned files is flagged. -6. **Evaluation** (same slot, `services/evaluation/runner.py`; see below). -7. **Evidence.** Everything goes to `GENERATION_RUNS_ROOT//` with sha256s in the job row: - - `llm_calls.jsonl.gz`, the authoritative log of every request and response, with the policy diff per call, - - the harness's own report (event stream / trajectory / chat log), - - `changes.patch`, `workspace.tar.gz` and the logs. - - Download at `GET /generation/jobs//run-record/`; this is refused if the file no longer matches its sha. +4. **The agent run.** `/out/harness.sh` runs under `timeout`. Once a minute the run checks whether it still owns the job (not cancelled, requeued or reclaimed) and touches the job's `updated_at`. Lease renewal is the executor heartbeat's job. +5. **Collection.** The run records the patch, diffstat and name-status, and classifies the workspace against the seed (`added` / `modified` / `scaffold`). Binaries, oversized files and harness artifacts are dropped, and tampering with platform-owned files is flagged. +6. **Evidence.** Everything is written to `GENERATION_RUNS_ROOT//`, or `/a/` for a retried attempt, with sha256s recorded in the job row. The files are `llm_calls.jsonl.gz` (the authoritative log of every request and response, with the policy diff per call), `gateway_events.jsonl`, `harness_events.jsonl.gz`, `harness.log`, `harness_report.tar.gz` (the harness's own trajectory or chat log), `changes.patch`, `name_status.txt`, `workspace.tar.gz` and `container.log`. Each call is also stored as a compact `GenerationArtifact` (`agent_call_`). Download a file at `GET /api/generation/jobs//run-record/`. The request is refused (409) if the file no longer matches its sha. +7. **Evaluation.** With `AGENT_AUTO_EVALUATE` the sandbox slot is released and the job waits (`phase=eval_queued`) for the executor's eval pool (`ExecutorSettings.eval_slots`), which grades it (`phase=evaluating`, below). ### Platform evaluation (hidden conformance suite) -Every sample is graded the same way by the platform, never by its own report: +Every sample is graded the same way by the platform, never by its own report (`services/evaluation/runner.py`): -1. **Build:** `prepare_build_dir(job)` + the stack's own Dockerfile on apps-dind. One retry happens only if the log shows a registry/network signature, and repeated infra failures are `infra_error`, not a failed sample. -2. **Run:** the image runs on an internal network (no egress), with memory/CPU caps and `ADMIN_PASSWORD` random. -3. **Grade:** `images/conformance-tester/tester.py` (stdlib, own container on the same network) checks only what the brief states, in a fixed order: +1. **Build:** `prepare_build_dir(job)` plus the stack's own Dockerfile on apps-dind. The build is retried once, and only when the log shows a registry/network signature. A repeated infra failure is `infra_error`, not a failed sample. +2. **Run:** the image runs on an internal network with no egress, under memory/CPU caps and with a random `ADMIN_PASSWORD`. +3. **Grade:** `images/conformance-tester/tester.py` (stdlib, `VERSION` 3, in its own container on the same network) checks only what the brief states, in a fixed order: - runtime: health, - frontend: `/` and a deep link serve the SPA, - auth: register, login, admin login, anonymous writes rejected, - api/admin: every contract endpoint answers 2xx with the documented fields. - Ids come from the app's own responses. Destructive or state-changing item checks each get a freshly created resource, so no check depends on another's side effects. Request bodies are synthesised deterministically from the spec's samples. + Ids come from the app's own responses. Each destructive or state-changing check gets a freshly created resource, so no check depends on another's side effects. Request bodies are synthesised deterministically from the spec's samples. 4. **Record:** - - `result_data.evaluation`: status, summary, every check, build log tail, the dependencies pip actually resolved, the app log tail, and grader identity (`evaluator_version`, `tester_version`, `tester_sha256`, `contract_version`), - - `metrics.functional`: `pass_rate`, endpoints ok/checked. + - `result_data.evaluation`: status, summary, every check, build log tail and healing report, the dependencies pip actually resolved, the app log tail, and the grader identity (`evaluator_version`, `tester_version`, `tester_sha256`, `contract_version`), + - `metrics.functional`: `source: "conformance"`, `status`, `passed`, `pass_rate` (null for `infra_error`), `health_ok`, `endpoints_ok` / `endpoints_checked`, `deployed`, `healing_mode` / `healing_changed`, and the grader identity. A later container-start smoke probe writes `metrics.smoke` and never replaces a conformance verdict. -Statuses: - -- `passed` / `failed`: the suite ran. -- `build_failed` / `start_failed`: a sample outcome. -- `infra_error`: exclude and re-evaluate. -- `not_applicable`: no app spec. +| Status | Meaning | +| --- | --- | +| `passed` / `failed` | The suite ran (`deployed: true`) | +| `build_failed` / `start_failed` | A sample outcome: it does not build, or never answers its health check (`deployed: false`, pass rate 0) | +| `infra_error` | The platform could not evaluate it. Exclude it and re-evaluate | +| `not_applicable` | No app spec to grade against | -`POST /generation/jobs//evaluate/` re-grades any completed job through the same queue, after a grader change, an `infra_error`, or for template-mode samples. Template-mode prompts never stated the accounts contract (`contract_stated_in_prompt: false`), so don't compare those scores with agent-mode ones. +`POST /api/generation/jobs//evaluate/` re-grades any completed job with an app spec through the same eval pool, for example after a grader change, after an `infra_error`, or for an ad-hoc template-mode sample. Template-mode prompts never stated the accounts contract (`contract_stated_in_prompt: false`), so do not compare their scores with agent-mode ones. ### Outcome semantics A run that made at least one successful LLM call is a **sample**: the job is `completed`, and `result_data.agent.termination` says how it ended. -| termination | meaning | +| termination | Meaning | | --- | --- | -| `completed` | the agent stopped by itself | -| `budget_exhausted` | a budget was hit (`budget_exhausted` names which: `max_requests`, `max_total_tokens`, `max_cost_usd`, `max_attempts`) | -| `timeout` | wall-clock limit hit | -| `resource_limit` | killed at the sandbox memory cap | -| `harness_killed` | a process inside the sandbox (the agent's own commands) signalled the harness | -| `harness_error` | the harness exited non-zero for another reason | +| `completed` | The agent stopped by itself | +| `budget_exhausted` | A budget was hit; `agent.budget_exhausted` names which one (`max_requests`, `max_total_tokens`, `max_cost_usd`, `max_attempts`) | +| `timeout` | Wall-clock limit hit | +| `resource_limit` | Killed at the sandbox memory cap | +| `harness_killed` | A process inside the sandbox (the agent's own commands) signalled the harness | +| `harness_error` | The harness exited non-zero for another reason | -Do not drop budget stops or timeouts from an analysis. That biases comparisons toward models that finish quickly. A run with **no** successful LLM call is an infrastructure failure (bad key, no provider matching the routing, sandbox down): the job is `failed` with the upstream error. +Do not drop budget stops or timeouts from an analysis: that biases comparisons toward models that finish quickly. -**Validity notes** (`result_data.agent.validity`) list what an analysis must know before pooling a sample: +A run with **no** successful LLM call ends in one of two ways. A transport or platform failure (upstream 502/503/504, proxy or endpoint unavailable, apps daemon down) is **retried** by the executor as a new attempt with the same snapshot and seed: up to 2 times for ad-hoc jobs, `Experiment.infra_retries` times for experiments, with exponential backoff. Any other failure (a bad key, no provider matching the routing) fails the job, with the upstream error and a remediation hint in `result_data`. -- `exclude`: the platform could not evaluate it. -- `warn`: valid but not model-neutral or not as configured. Examples: a USD or token cap bound the run (expensive or verbose models get fewer steps), a wall-clock stop (provider latency), a param the model can't take, a different served model id, the memory cap, or responses cut off at the per-call output cap (`output_truncated`; single-turn harnesses such as Aider are most exposed). -- `info`: context, such as the request budget binding, failed LLM calls, harness self-kills, blocked egress, or edits to platform files. +**Validity notes** (`result_data.agent.validity.notes`, `{code, severity, detail}`) list what an analysis must know before pooling a sample. `valid` is false only when there is an `exclude` note. -`valid` is false only for `exclude` notes. +| Severity | Codes | +| --- | --- | +| `exclude`: not a valid sample of the condition | `evaluation_infra_error`, `infra_interrupted` (a local-endpoint run kept being interrupted by infrastructure errors) | +| `warn`: valid, but not model-neutral or not as configured | `budget:max_cost_usd`, `budget:max_total_tokens`, `budget:max_attempts`, `wall_clock`, `resource_limit`, `harness_error`, `params_not_applied` (a configured param the model cannot take), `served_model_differs`, `output_truncated` (responses cut off at the per-call output cap; single-turn harnesses such as Aider are most exposed), `not_evaluated`, `infra_retried` (attempt 2 or later), `contended` (more than 10% of the wall clock spent waiting for endpoint request slots) | +| `info`: context | `budget:max_requests` (the model-neutral limit), `params_not_applied` when only the seed was dropped, `harness_killed`, `llm_errors`, `protected_files_modified`, `egress_denied`, `relay_interruptions` | ### Building and running the result -Agent results set `result_data.materialize = "verbatim"`. The runtime writes the collected files over the pinned skeleton exactly as the agent left them: no entry renaming, no Flask patching, no dependency inference. It restores the platform-owned `Dockerfile`, `.dockerignore` and `docker-compose.yml`. Builtin stack Dockerfiles copy the whole backend tree and honour `package-lock.json`. +Agent results set `result_data.materialize = "verbatim"`. The runtime writes the collected files over the pinned skeleton exactly as the agent left them: no entry renaming, no Flask patching, no dependency inference. Build healing never applies to them. The runtime does restore the platform-owned `Dockerfile`, `.dockerignore` and `docker-compose.yml`. Builtin stack Dockerfiles copy the whole backend tree and honour `package-lock.json`. ## Research design: what keeps agent samples comparable | Threat | Control | | --- | --- | -| Harness switches its system prompt per model family (OpenCode: build agent *and* `general` subagent) | OpenCode's own model-agnostic default prompt, captured verbatim per release and pinned for both; Aider's edit format pinned (`diff` unless a harness preset picks another) for every model | -| Harness or model-family sampling defaults (OpenCode sets per-family temperatures, Aider sends 0, others send nothing) | Sampling policy `provider_default` (default): the gateway strips every sampling/reasoning knob the harness sends. Only params the run configures explicitly reach the model, forced on every call. `harness_native` is available and recorded | -| One harness sends no `max_tokens` and gets a provider default, another sends its own | The gateway forces one per-call output cap (`min(max_tokens, model limit)`) on every call | -| Params a model can't take silently ignored (e.g. `seed`/`temperature` on some closed models) | Forwarded only if OpenRouter lists them for that model; otherwise recorded in `gateway.dropped_params` and in the experiment comparability report. A model with no supported-parameter list in the catalog gets its params forwarded unverified (`llm.supported_params_source: "unknown"`, preview warning `support_unknown`): the default routing's `require_parameters` makes a provider that can't honour one fail the run rather than ignore it, and default runs on such models keep their seed | -| Model "X" served by different providers at fp4…bf16, some ignoring `seed` | Default routing: `require_parameters` + quantization ≥ fp8 (or unknown, for closed models); the serving provider of every call is recorded (`agent.providers`) | -| Model alias repointed mid-study | The dated canonical slug is recorded per job; served model ids per call; mismatches are flagged | -| Hidden auxiliary calls (titles, "small model", weak model) | Title passed in; small/weak/editor model = the run's model; the gateway pins the model on every call and counts override attempts | -| A flaky provider costs a model steps; harnesses retry differently | Gateway retries transient failures for every harness; only successful calls count toward `max_llm_requests`; attempts are capped separately | -| Budgets that favour some models (USD favours cheap ones, tokens terse ones, wall clock fast providers) | The **request budget** binds by default (100). USD (5), tokens (30M) and wall clock (60 min) are safety nets; a run stopped by one carries a `warn` note | -| Time confounds (provider load, routing, rate limits drift over hours) | Randomised complete blocks: repeat by repeat, cells shuffled within a repeat with a seed derived from the experiment. The FIFO queue keeps dispatch order = run order, and each job records its run order and queue time | +| Harness switches its system prompt per model family (OpenCode does this for the build agent *and* the `general` subagent) | OpenCode's own model-agnostic default prompt, captured verbatim per release and pinned for both. Aider's edit format pinned for every model (`diff` unless a harness preset picks another) | +| Harness or model-family sampling defaults (OpenCode sets per-family temperatures, Aider sends 0, others send nothing) | Sampling policy `provider_default` (the default): the gateway strips every sampling/reasoning knob the harness sends. Only params the run configures explicitly reach the model, and they are forced on every call. `harness_native` is available and recorded | +| One harness sends no `max_tokens` and gets a provider default, another sends its own | The gateway forces one per-call output cap, `min(max_tokens, model limit)` (default 32000), on every call | +| Params a model can't take are silently ignored (e.g. `seed`/`temperature` on some closed models) | Params are forwarded only if OpenRouter lists them for that model; otherwise they are recorded in `gateway.dropped_params` and in the experiment comparability report. A model with no supported-parameter list in the catalog gets its params forwarded unverified (`llm.supported_params_source: "unknown"`, preview warning `support_unknown`). The default routing's `require_parameters` then makes a provider that cannot honour one fail the run rather than ignore it. Local models use the endpoint's static parameter table (`endpoint_static`) | +| Model "X" served by different providers at fp4…bf16, some ignoring `seed` | Default routing (`AGENT_DEFAULT_PROVIDER_ROUTING`): `require_parameters`, quantization fp8 or better (or `unknown`, for closed models), fallbacks allowed. The serving provider of every call is recorded (`agent.providers`) | +| Model alias repointed mid-study | The dated canonical slug is recorded per job and the served model id per call. Mismatches are flagged | +| Hidden auxiliary calls (titles, "small model", weak model) | The title is passed in, and the small/weak/editor model is the run's model. The gateway pins the model on every call and counts override attempts | +| A flaky provider costs a model steps, and harnesses retry differently | The gateway retries transient failures for every harness. Only successful calls count toward `max_llm_requests`, and attempts are capped separately | +| Budgets that favour some models (USD favours cheap ones, tokens terse ones, wall clock fast providers) | The **request budget** binds by default (100). USD (5), tokens (30M) and wall clock (60 min) are safety nets, and a run stopped by one carries a `warn` note | +| Time confounds (provider load, routing and rate limits drift over hours) | Randomised complete blocks: repeat by repeat, with cells shuffled within a repeat by a seeded RNG. The executor keeps dispatch order equal to run order, and each job records its run order and queue time (`agent.schedule`) | | Seeds confounded with conditions | Common random numbers: every condition of an (app, repeat) cell shares one seed | -| Unequal sandboxes across harnesses | One base image with an exact pinned release; harness packages locked and isolated in a private venv; memory/CPU caps frozen per job | -| Dependency drift inside a harness (an unpinned install once resolved a LiteLLM that would not import) | Fully locked `requirements.lock` per Python harness; content-addressed image tags recorded per job; a changed definition makes old jobs refuse to run | -| Agent kills its own harness while restarting its server | Brief via file/stdin only; `harness.sh` exec'd under `timeout`; remaining self-kills are recorded as `harness_killed` | -| Agent's requests to its own app captured by the egress proxy | Only `HTTPS_PROXY` is set, and loopback is in `NO_PROXY` | -| Agent looks things up online, or the run depends on internet state | No web tools; egress limited to package registries (or none) | -| Agent reconfigures its harness or reads stray instructions | Config/AGENTS.md/CLAUDE.md discovery, plugins and external skills off | -| Agent games its own evaluation | Hidden, platform-run conformance suite; platform-owned build files restored; random admin password; app evaluated with no egress | -| Ungradable or inconsistent requirements (no auth API in the specs) | A shared, versioned accounts/runtime contract stated in every brief and checked by the suite | -| Grading artefacts (one check's side effects failing another) | Fresh resources for destructive checks; deterministic order and bodies; grader versioned and hashed in every result | +| Unequal sandboxes across harnesses | One base image with an exact pinned release. Harness packages are locked and isolated in a private venv, and memory/CPU caps are frozen per job | +| Dependency drift inside a harness (an unpinned install once resolved a LiteLLM that would not import) | A fully locked `requirements.lock` per Python harness. Content-addressed image tags are recorded per job, and a changed definition makes old jobs refuse to run | +| Agent kills its own harness while restarting its server | The brief goes in via file/stdin only, and `harness.sh` is exec'd under `timeout`. Remaining self-kills are recorded as `harness_killed` | +| The agent's requests to its own app captured by the egress proxy | Only `HTTPS_PROXY` is set, and loopback is in `NO_PROXY` | +| Agent looks things up online, or the run depends on internet state | No web tools. Egress is limited to package registries (or none) | +| Agent reconfigures its harness or reads stray instructions | Config/AGENTS.md/CLAUDE.md discovery, plugins and external skills are off | +| Agent games its own evaluation | A hidden, platform-run conformance suite. Platform-owned build files are restored, the admin password is random, and the app is evaluated with no egress | +| Ungradable or inconsistent requirements (no auth API in the specs) | A shared, versioned accounts/runtime contract, stated in every brief and checked by the suite | +| Grading artefacts (one check's side effects failing another) | Fresh resources for destructive checks, a deterministic order and request bodies, and a grader that is versioned and hashed in every result | +| Infrastructure failures counted as model failures | Transport failures with no successful call are retried under the same snapshot and seed, and evaluation `infra_error`s are excluded, not scored | | Prompt coaching that favours some models (length targets, style rules) | The brief states only the spec and the contract | | Key exfiltration into generated code | The agent never holds the real key | Known residual variation, recorded but not controlled: -- Per-model behaviour inside the pinned harness releases, which no setting removes (the limit of "byte-identical treatment within a harness"): +- Per-model behaviour inside the pinned harness releases, which no setting removes (this is the limit of "byte-identical treatment within a harness"): - OpenCode appends an environment block to its system prompt that names the model and today's date, and gives GPT-family model ids an `apply_patch` tool instead of `edit`/`write`. - - Aider applies generic settings by model-name substring (reminder placement, lazy/overeager prompt additions, `use_temperature`). Every model reaches it as `openai//`, so those still apply. + - Aider applies generic settings by model-name substring (reminder placement, lazy/overeager prompt additions, `use_temperature`). Every model reaches it as `openai//`, so those settings still apply. - Capabilities differ by design: a param a model does not take is dropped (and recorded), and the output cap and context window follow the model. The run preview lists this per model before launch. -- Providers within the routing policy may still differ in serving stack; stratify by `agent.providers`. -- Debian packages in `agent-base` resolve when the image is built; the built image id is recorded per run. -- Package registries change over time under policy `packages`. Pinned dependency files limit this; `resolved_python_deps` records what the evaluated build actually installed. Policy `none` removes it. -- mini-swe-agent's native 30 s per-command timeout and Aider's single-turn headless mode are harness properties and part of what is compared. An Aider turn that writes the whole app in one response can hit the per-call output cap and apply nothing; this is observed with DeepSeek V4.1 Flash and flagged `output_truncated`. -- Without a model-metadata file Aider refreshes its model database from GitHub at start-up (blocked, recorded as `egress_denied`); with the metadata file it does not. -- Request budgets are not compute-equal across harnesses (Aider needs few calls, mini-swe-agent many). For cross-harness comparisons, report cost and tokens alongside outcomes. +- Providers within the routing policy may still differ in serving stack, so stratify by `agent.providers`. +- Debian packages in `agent-base` resolve when the image is built. The built image id is recorded per run. +- Package registries change over time under policy `packages`. Pinned dependency files limit this, and `resolved_python_deps` records what the evaluated build actually installed. Policy `none` removes the variation. +- mini-swe-agent's native 30 s per-command timeout and Aider's single-turn headless mode are harness properties and part of what is compared. An Aider turn that writes the whole app in one response can hit the per-call output cap and apply nothing. This has been observed with DeepSeek V4.1 Flash and is flagged `output_truncated`. +- Without a model-metadata file, Aider refreshes its model database from GitHub at start-up (blocked, and recorded as `egress_denied`). With the metadata file it does not. +- Request budgets are not compute-equal across harnesses: Aider needs few calls, mini-swe-agent many. For cross-harness comparisons, report cost and tokens alongside outcomes. -Agent runs are not deterministic even with a seed (tool timing, provider batching), so **repeats are the unit of analysis**: use `trials` (or experiment `repeats`) of 3 or more, with a `base_seed` so the seed matrix and the run order are reproducible. +Agent runs are not deterministic even with a seed (tool timing, provider batching), so **repeats are the unit of analysis**. Use `trials` (or experiment `repeats`) of 3 or more, with a `base_seed` so that the seed matrix and the run order are reproducible. ### Fairness rules -These hold for every agent batch and experiment; the code enforces them and the tests pin them: +These hold for every agent batch and experiment. The code enforces them and the tests pin them: -1. **Within a harness**, every model gets a byte-identical harness section except the derived per-model fields (model slug, context window, output cap, step limit). Harness presets carry no per-model keys. -2. **Default runs never change identity.** With the default presets, `prompt_hash`, `run_fingerprint` and the `agent`, `gateway`, `harness` and brief values equal what they were before presets existed (`backend/generation/tests/test_default_run_golden.py`, captured from the pre-preset code). -3. **Batch- or experiment-wide:** the stack, the budget preset (limits and network policy) and the sampling preset (sampling params and sampling policy). What may vary per condition: the model and the harness preset; in experiments also `param_overrides` and the profile, which comparability always reports. +1. **Within a harness**, every model gets a byte-identical harness section except for the derived per-model fields (model slug, context window, output cap, step limit). Harness presets carry no per-model keys. +2. **Default runs never change identity.** With the default presets, `prompt_hash`, `run_fingerprint` and the `agent`, `gateway`, `harness` and brief values equal what they were before presets existed (`backend/generation/tests/test_default_run_golden.py`). The local-endpoint additions are absent for OpenRouter models, so those snapshots are byte-identical too. +3. **Batch- or experiment-wide:** the stack, the budget preset (limits and network policy) and the sampling preset (sampling params and sampling policy). What may vary per condition: the model and the harness preset, and in experiments also `param_overrides` and the profile, which comparability always reports. 4. The brief stays harness- and model-agnostic. -5. Under `provider_default`, only explicitly configured params reach the model; unsupported ones are dropped and recorded, and the preview shows them per model. +5. Under `provider_default`, only explicitly configured params reach the model. Unsupported ones are dropped and recorded, and the preview shows them per model. 6. Not tunable per run: harness versions, sandbox resources, the egress allowlist, protected files, the evaluator, and the default provider routing. Routing changes only through an explicit inline `llm_params.provider`, never through a preset. -7. Snapshots are immutable. Presets are versioned and immutable in content; jobs record which presets they used. +7. Snapshots are immutable. Presets are versioned and immutable in content, and jobs record which presets they used. 8. Preset values are copied into the job's own fields at creation (`llm_params`, `agent_config`, `harness_options`), so retry, replay and rerun reproduce the configuration without reading the preset again. ### Run presets -A run preset (`RunPreset`, `services/presets.py`) is a named, versioned piece of agent-run configuration. There are three kinds; each is normalized before it is stored or hashed: +A run preset (`RunPreset`, `services/presets.py`) is a named, versioned piece of agent-run configuration. There are three kinds, and each is normalized before it is stored or hashed: | Kind | Config | Notes | | --- | --- | --- | -| `harness` | `{"harness": , "options": {}}` | Options at their defaults are dropped, so `{}` and `{"edit_format": "diff"}` are the same preset content. A user preset with every option at its default is refused: use `-default` | -| `sampling` | `{"llm_params": {...}, "sampling_policy"?: ...}` | Model-agnostic sampling settings (temperature, top_p, reasoning effort, output cap, …) applied identically to every model. `provider`, `stop` and `response_format` are refused: routing and output formatting are not sampling properties | -| `budget` | sparse subset of `wall_clock_seconds`, `max_llm_requests`, `max_total_tokens`, `max_cost_usd`, `network_policy` | `{}` = the platform defaults (the `AGENT_*` settings) | - -- **Versions.** A content change creates version + 1; a name or description change does not. `content_hash` = sha256 of the canonical `{kind, config}` (integral floats hash as ints, so `0` and `0.0` agree). Delete archives every version; jobs keep their references (`PROTECT`). -- **Slugs** are global per kind, whoever owns them: `^[a-z0-9][a-z0-9-]{1,80}$`, and `*-default` plus every system preset file's slug are reserved. A taken slug is a plain "slug already in use", never naming the owner. -- **System presets** live in `data/presets/-.yaml`: `harness/opencode-default`, `harness/mini-swe-agent-default`, `harness/aider-default`, `sampling/provider-default`, `budget/platform-default`. They are seeded first in `seed_all`, idempotent by content hash (a changed file becomes a new version). A request-time guard (`ensure_system_presets`) recreates missing defaults, so the default run path never depends on seeding having worked. -- **Stored presets are re-validated** whenever a run uses them; a preset a harness release no longer accepts is an `invalid_preset` error in the preview and a 400 on create, never a crash. +| `harness` | `{"harness": , "options": {}}` | Options at their defaults are dropped, so `{}` and `{"edit_format": "diff"}` are the same content. A user preset with every option at its default is refused: use `-default` | +| `sampling` | `{"llm_params": {...}, "sampling_policy"?: ...}` | Model-agnostic sampling settings (temperature, top_p, reasoning effort, output cap, …) applied identically to every model. `provider`, `stop` and `response_format` are refused, because routing and output formatting are not sampling properties | +| `budget` | a sparse subset of `wall_clock_seconds`, `max_llm_requests`, `max_total_tokens`, `max_cost_usd`, `network_policy` | `{}` means the platform defaults (the `AGENT_*` settings) | + +- **Versions.** A content change creates version + 1; a name or description change does not. `content_hash` is the sha256 of the canonical `{kind, config}`, with integral floats hashed as ints so that `0` and `0.0` agree. Delete archives every version, and jobs keep their references (`PROTECT`). +- **Slugs** are global per kind, whoever owns them, and must match `^[a-z0-9][a-z0-9-]{1,80}$`. Slugs ending in `-default` and every system preset slug are reserved. A taken slug gets a plain "slug already in use" error that never names the owner. +- **System presets** live in `data/presets/-.yaml`: `harness/opencode-default`, `harness/mini-swe-agent-default`, `harness/aider-default`, `sampling/provider-default` and `budget/platform-default`. They are seeded by `seed_all`, idempotently by content hash (a changed file becomes a new version). A request-time guard (`ensure_system_presets`) recreates missing defaults, so the default run path never depends on seeding having worked. +- **Stored presets are re-validated** whenever a run uses them. A preset that a harness release no longer accepts is an `invalid_preset` error in the preview and a 400 on create, never a crash. - **Starter presets.** The `agent-benchmark-starter` package ([Template packages](/docs/TEMPLATE_PACKAGES)) adds `aider-whole-file`, `aider-no-repo-map`, `mini-long-commands`, `temperature-zero`, `reasoning-high`, `smoke` (30 requests, 60 min, $3) and `extended` (250 requests, 180 min, $25). Their USD and wall-clock caps stay generous safety nets, so the request budget still binds. +- In the UI, presets live in Library → Presets (Harness / Sampling / Budget), and the agent composer can save its current settings as a new preset. -How a batch applies them (later wins), copied into job fields at creation: +How a batch applies them, copied into the job's fields at creation (later layers win): - `job.agent_config` = budget preset → the sampling preset's `sampling_policy` → inline `agent_config` (+ the harness), - `job.llm_params` = legacy `max_tokens` → profile `llm_config` → sampling preset `llm_params` → inline `llm_params`, -- `job.harness_options` = the harness preset's options (never part of `agent_config` or the snapshot's `agent` section). +- `job.harness_options` = the harness preset's options (never part of `agent_config` or of the snapshot's `agent` section). The `harness_preset`, `sampling_preset` and `budget_preset` foreign keys on the job, and `snapshot.presets` (`{id, slug, version, content_hash, name}` per kind), are provenance only. Leaving a preset out means the system default for its kind. ### Identity keys -- `prompt_hash`: brief sha + contract version. The *task* identity, shared across harnesses, models and seeds; the grouping key for comparisons. The brief states the budget, so changing the budget changes it. +- `prompt_hash`: the brief sha plus the contract version. This is the *task* identity, shared across harnesses, models and seeds, and the grouping key for comparisons. The brief states the budget, so changing the budget changes it. - `bundle_key` (`method_key` in the snapshot): `[profile@version/]agent:@`, plus a content-addressed suffix for each part that departs from the defaults: - `+opts:`: non-default harness options, - - `+llm:`: explicit LLM params other than `max_tokens` and `seed`, and a non-default sampling policy, + - `+llm:`: explicit LLM params other than `max_tokens` and `seed`, or a non-default sampling policy, - `+budget:`: an effective budget (limits or network policy) that differs from the platform defaults. - Hashing content rather than preset names means renaming a preset (or importing it under another slug) never splits a slice, and a default run keeps the key it always had. The *method* identity and condition slicing key. -- `run_fingerprint`: prompt hash + model + gateway policy + harness section + image tags + agent config + stack content hash. Equal fingerprints mean exact replays. -- `config_fingerprint`: the same inputs minus the seed. The *condition* identity that replicates (trials) share, used to group trials in batch views. Jobs created before presets have none. A rerun keeps the original's frozen gateway policy (only the forwarded seed changes, never what today's catalog says the model supports), so it has the original's `config_fingerprint`. -- Numbers enter these keys in their declared type: float sampling params (`temperature`, `top_p`, `min_p`, the penalties) and `max_cost_usd` are frozen as floats, integer limits as ints, whichever layer set them. A preset stored as `temperature: 0` and an inline `0.0` therefore fingerprint alike. (The keys hash plain JSON on purpose: folding `5.0` into `5` would change every default run's fingerprint.) + This is the *method* identity and the condition slicing key. It hashes content rather than preset names, so renaming a preset (or importing it under another slug) never splits a slice, and a default run keeps the key it always had. +- `run_fingerprint`: prompt hash + model + gateway policy + harness section + image tags + agent config + stack slug and content hash, plus, for local models, `llm_source` (which weights on which server flavour). Equal fingerprints mean exact replays. +- `config_fingerprint`: the same inputs minus the seed. This is the *condition* identity that replicates (trials) share, used to group trials in batch views. A rerun keeps the original's frozen gateway policy (only the forwarded seed changes, never what today's catalog says the model supports), so it has the original's `config_fingerprint`. +- Numbers enter these keys in their declared type, whichever layer set them. Float sampling params (`temperature`, `top_p`, `min_p`, the penalties) and `max_cost_usd` are frozen as floats, and integer limits as ints, so a preset stored as `temperature: 0` and an inline `0.0` fingerprint alike. The keys hash plain JSON on purpose: folding `5.0` into `5` would change every default run's fingerprint. When you change something, version it: -- `BRIEF_VERSION` (`services/agent/brief.py`) for the brief (v3 added the "Additional context" section and user-stack start commands; briefs without either are byte-identical to v2). -- `CONTRACT_VERSION` for the contract. -- The tester's `VERSION` / `EVALUATOR_VERSION` for grading. -- `SANDBOX_REVISION` (`harnesses/base.py`) when the sandbox environment changes without an image change. -- To move OpenCode to a new release, capture its default prompt into `data/agent/harness_prompts/opencode--default.md` first; jobs refuse to snapshot otherwise. +- `BRIEF_VERSION` (`services/agent/brief.py`, currently 3) for the brief. v3 added the "Additional context" section and user-stack start commands; briefs without either are byte-identical to v2. +- `CONTRACT_VERSION` (`services/evaluation/contract.py`, 1) for the contract. +- The tester's `VERSION` (3) or `EVALUATOR_VERSION` (1) for grading. +- `SANDBOX_REVISION` (`harnesses/base.py`, 4) when the sandbox environment changes without an image change. +- To move OpenCode to a new release, first capture its default prompt into `data/agent/harness_prompts/opencode--default.md`. Jobs refuse to snapshot otherwise. ### Configuration | Setting | Default | | | --- | --- | --- | | `AGENT_DEFAULT_HARNESS` | `opencode` | | -| `GENERATION_DEFAULT_STACK` | `flask-react` | stack when neither the request nor a profile names one (agent and scaffolding) | -| `AGENT_OPENCODE_VERSION` / `AGENT_MINI_SWE_AGENT_VERSION` / `AGENT_AIDER_VERSION` | `1.18.35` / `2.4.6` / `0.86.2` | releases baked into the images | -| `AGENT_MAX_LLM_REQUESTS` | 100 | the binding budget (successful calls) | -| `AGENT_WALL_CLOCK_SECONDS` | 3600 | safety net | -| `AGENT_MAX_TOTAL_TOKENS` | 30M | safety net | -| `AGENT_MAX_COST_USD` | 5.0 | safety net | -| `AGENT_SAMPLING_POLICY` | `provider_default` | or `harness_native` | -| `AGENT_NETWORK_POLICY` | `packages` | or `none` | -| `AGENT_AUTO_EVALUATE` | true | grade every sample right after its run | -| `AGENT_MAX_CONCURRENT_RUNS` | 1 | sandbox slots per process (runs and evaluations share them) | -| `AGENT_SANDBOX_MEM_LIMIT` / `_NANO_CPUS` | `3g` / 2 CPUs | must fit apps-dind's own cap | - -API (prefix `/api/generation`): - -- `POST /jobs/agent/` takes `{stack_slug?, app_requirement_ids, description, model_ids, harness_preset_ids, sampling_preset_id?, budget_preset_id?, llm_params?, agent_config?, max_tokens?, profile_id?, trials, base_seed?}`. It creates tasks × models × harness presets × trials jobs in one batch, with one seed per (task, trial) cell shared by every model and harness preset (`derive_experiment_seed(base_seed, app slug or "task", trial)`) and blocked random order. - - The stack is `stack_slug`, else the profile's stack, else `GENERATION_DEFAULT_STACK`; whatever its source, it must exist and be usable by the requester. - - Without presets or harnesses the default harness's system preset runs. The legacy `harnesses: []` maps each name to its system default preset; it cannot be combined with `harness_preset_ids`, nor can `agent_config.harness`. +| `GENERATION_DEFAULT_STACK` | `flask-react` | The stack used when neither the request nor a profile names one (agent and scaffolding) | +| `AGENT_OPENCODE_VERSION` / `AGENT_MINI_SWE_AGENT_VERSION` / `AGENT_AIDER_VERSION` | `1.18.35` / `2.4.6` / `0.86.2` | Releases baked into the images | +| `AGENT_MAX_LLM_REQUESTS` | 100 | The binding budget (successful calls) | +| `AGENT_WALL_CLOCK_SECONDS` | 3600 | Safety net | +| `AGENT_MAX_TOTAL_TOKENS` | 30,000,000 | Safety net | +| `AGENT_MAX_COST_USD` | 5.0 | Safety net | +| `AGENT_SAMPLING_POLICY` | `provider_default` | Or `harness_native` | +| `AGENT_NETWORK_POLICY` | `packages` | Or `none` | +| `AGENT_AUTO_EVALUATE` | true | Grade every sample right after its run | +| `AGENT_MAX_CONCURRENT_RUNS` | 1 | *Initial* `ExecutorSettings.sandbox_slots`, editable at runtime by staff; not frozen into snapshots | +| `AGENT_SANDBOX_MEM_LIMIT` / `AGENT_SANDBOX_NANO_CPUS` | `3g` / 2 CPUs | Frozen into snapshots. Each sandbox must fit apps-dind's own cap (`APPS_DIND_MEM_LIMIT`), so raise both together | +| `AGENT_DEFAULT_PROVIDER_ROUTING` | `require_parameters`, quantizations fp8/fp16/bf16/fp32/unknown, fallbacks on | Used when a run sets no explicit `provider` | +| `LLM_LOCAL_AGENT_RUNS` | on when `APPS_DOCKER_HOST` is set | Allow agent runs on local-endpoint models | +| `GENERATION_RUNS_ROOT` | `MEDIA_ROOT/generation_runs` | Run evidence | + +Executor tuning (`EXECUTOR_*`, eval and LLM slots) is covered in [Execution engine](/docs/EXECUTION). + +### API + +All paths are under `/api/generation` ([API reference](/docs/api-reference)). + +- `POST /jobs/agent/` takes `{stack_slug?, app_requirement_ids, description, model_ids, harness_preset_ids, sampling_preset_id?, budget_preset_id?, llm_params?, agent_config?, max_tokens?, profile_id?, trials (1–10), base_seed?}`. It creates (app specs, or one free-form task) × models × harness presets × trials jobs in one `GenerationBatch`. With a `base_seed`, every (task, trial) cell shares one seed across all models and harness presets (`batch_cell_seed`). Jobs are enqueued trial by trial, in a seeded random order within each trial; the order seed is `base_seed`, or a random seed when none is given. + - The stack is `stack_slug`, else the profile's stack, else `GENERATION_DEFAULT_STACK`. Whatever its source, it must exist and be usable by the requester. + - Without presets or harnesses, the default harness's system preset runs. The legacy `harnesses: []` maps each name to that harness's system default preset. It cannot be combined with `harness_preset_ids`, and neither can `agent_config.harness`. - Any error-level problem refuses the whole batch with `400 {detail, code, errors: [...]}` before anything is written. -- `POST /jobs/agent/preview/` takes the same body and writes nothing. It returns the run count, the effective agent config and LLM params with the source of each value, every harness preset with its effective options, a per-model table (tool support, context window and its source, output cap, forced and dropped params, whether the seed is forwarded), the maximum cost (budget cap × runs) and warnings grouped by severity: errors `no_tasks`, `no_models`, `models_without_tools`, `missing_api_key`, `duplicate_harness_config`, `invalid_preset`, `reasoning_exceeds_output_cap`, `resolution_failed`; warnings `params_dropped` (the seed never counts), `support_unknown` (the catalog lists no supported parameters for a model, so its params are forwarded unverified), `harness_native_sampling`, `context_window_fallback`, `cost_cap_may_bind`, `wall_clock_may_bind`, `whole_file_small_output`, `large_batch`, `ungraded_task`; infos `non_default_harness_options`, `budget_changes_brief`, `single_trial`, `random_seeds`. -- `GET /agent/config/` returns the defaults, limits, policies, the default stack and output cap, and every harness with its option schema. `GET /agent/harnesses/` lists the harnesses. -- Presets: `GET /presets/?kind=&include_archived=` (latest version per slug: system and your own), `POST /presets/validate/`, `POST /presets/`, `GET /presets//` (any visible version), `GET /presets///versions/`, `PUT /presets///` (content change → new version) and `DELETE /presets///` (archive). -- `POST /jobs//evaluate/` re-grades a job. +- `POST /jobs/agent/preview/` takes the same body and writes nothing. It returns: + - the run count, the effective agent config and LLM params with the source of each value, and every harness preset with its effective options, + - a per-model table: tool support, context window and its source, output cap, forced and dropped params, and whether the seed is forwarded, + - the maximum cost (budget cap × runs), + - issues grouped by severity: + - errors: `no_tasks`, `no_models`, `models_without_tools`, `missing_api_key`, `duplicate_harness_config`, `invalid_preset`, `reasoning_exceeds_output_cap`, `resolution_failed`, and for local models `endpoint_disabled`, `endpoint_forbidden` and `model_unavailable`, + - warnings: `params_dropped` (the seed never counts), `support_unknown`, `harness_native_sampling`, `context_window_fallback`, `cost_cap_may_bind`, `wall_clock_may_bind`, `whole_file_small_output`, `large_batch`, `ungraded_task`, + - local-model warnings: `endpoint_offline`, `model_not_served`, `foreign_endpoint`, + - infos: `non_default_harness_options`, `budget_changes_brief`, `single_trial`, `random_seeds`. +- `GET /agent/config/` returns the defaults, limits, network and sampling policies, the default provider routing, the default stack and output cap, and every harness with its option schema. `GET /agent/harnesses/` lists the harnesses. +- Presets: `GET /presets/?kind=&include_archived=` (the latest version per slug, system and your own), `POST /presets/validate/`, `POST /presets/`, `GET /presets//` (any visible version), `GET /presets///versions/`, `PUT /presets///` (a content change creates a new version) and `DELETE /presets///` (archives). +- `POST /jobs//evaluate/` re-grades a completed job. `GET /jobs//run-record/` downloads run evidence. ## Template modes (scaffolding, custom) -The Generation Library concepts still drive scaffolding mode: +The Generation Library (`/templates`) holds the concepts that drive scaffolding mode: + +- **App specs** (`AppRequirementTemplate`) say what to build: requirements, data model, API endpoints, admin endpoints, difficulty. There is one mutable row per slug, and the slug is unique across owners. System specs are seeded from `data/requirements/*.json` and checked against `data/requirements/schema.json` by the data lint; `POST`/`PUT /app-specs/` and package import check the same schema. Users see the catalog plus their own specs. Agent mode uses the same specs. +- **Generation profiles** (`GenerationProfile`) are the prompt material for template scaffolding: versioned content blocks, a stack and an `llm_config`. In agent mode a profile contributes only its stack and `llm_config`; its prompt blocks do not apply. +- **Content blocks** (`ContentBlock`) are versioned prompt fragments, keyed by `(slug, version)`, of type `prompt_stage`, `prompt_tone`, `prompt_rules`, `requirement`, `api_schema`, `scaffold_hint`, `validation` or `eval_rubric`. +- **Stacks** (`runtime.Stack`) are versioned skeletons: the agent's starting workspace and the build recipe for every mode. The builtins are `flask-react`, `fastapi-react`, `fastapi-vue` and `generic-python`. Users can author their own stacks with a generated Dockerfile, from base images in `STACK_ALLOWED_BASE_IMAGES`. With `STACK_REQUIRE_APPROVAL` on, a non-staff stack needs staff approval before other users' jobs can run on it. + +System profiles come from `data/blocks/catalog.yaml`, and per-app pilots from `data/requirements/manifests/`. They name system blocks by slug only, and the seeder pins each to the latest system block version, so a prompt-file edit reaches every profile that uses it as a new profile version. A new system block version always takes the next free version number, so a user's own version of a system block never collides with the seeder. `is_default` marks the default profile *of its stack* (at most one per stack). The global default, used when no stack is given, is the catalog's `default_bundle_slug` (`system-scaffolding-standard`, on `flask-react`). Profiles, blocks, app specs, user stacks and presets travel between installs as [template packages](/docs/TEMPLATE_PACKAGES). -- **App specs** (`AppRequirementTemplate`): what to build, validated against `data/requirements/schema.json`. Agent mode uses the same specs. -- **Generation profiles** (`GenerationProfile`): prompt material for template scaffolding — versioned content blocks + stack + `llm_config`. In agent mode a profile contributes only its stack and `llm_config`; its prompt blocks do not apply. -- **Stacks** (`runtime.Stack`): versioned skeletons. The agent's starting workspace and the build recipe for every mode. +**Scaffolding** renders prompts from the profile's blocks (`prompt_renderer.py`) and calls the model twice: once for the backend, then for the frontend with a scan of the generated backend. A response cut off at `max_tokens` gets one continuation call. The code blocks are parsed into files, and files that fail static checks get one repair round (a fix is kept only if it validates). An experiment's `continuation_limit` and `enable_repair` change both; non-default values are frozen into the job snapshot (`generation`) and its `run_fingerprint`. Its snapshot keys (`prompt_hash`, `bundle_key` = `profile-slug@version`, `run_fingerprint`) mean the same as in agent mode. `POST /jobs/scaffolding/` takes `{stack_slug?, app_requirement_ids, model_ids, profile_id?, sampling_preset_id?, temperature, max_tokens, llm_params?, trials, build_healing}` and creates app specs × models × trials jobs in one batch. Without a `profile_id`, each app spec gets its per-app pilot profile for the stack if one is seeded, else the stack's default profile. An explicit `stack_slug` that disagrees with the profile's stack is a 400, and so is a stack whose automatic choice falls back to a profile for another stack. -System profiles come from `data/blocks/catalog.yaml` (and per-app pilots from `data/requirements/manifests/`). They name system blocks by slug only; the seeder pins each to the latest system block version, so a prompt-file edit reaches every profile that uses it as a new profile version. A new system version always takes the next free version number, so a user's own version of a system block never collides with the seeder. `is_default` marks the default profile *of its stack* (at most one per stack); the global default, used when no stack is given, is the catalog's `default_bundle_slug` (`system-scaffolding-standard`). Profiles, blocks, app specs, stacks and presets travel between installs as [template packages](/docs/TEMPLATE_PACKAGES). +**Single prompt** (`POST /jobs/custom/`, `{model_id, system_prompt, user_prompt, temperature, max_tokens, sampling_preset_id?, llm_params?, seed?}`) sends one request and stores the raw response. Nothing is parsed, built or graded. -Scaffolding renders prompts from the profile's blocks (`prompt_renderer.py`), calls the model twice (backend, then frontend with a scan of the backend), parses the code blocks into files, and runs one repair round on files that fail static checks. Its snapshot keys (`prompt_hash`, `bundle_key`, `run_fingerprint`) mean the same as in agent mode. +**Sampling** in the template modes starts from the **mode default** (`temperature` 0.3, `max_tokens` 32000). It then layers profile `llm_config` → sampling preset `llm_params` → experiment `llm_defaults` → condition `param_overrides` → per-run `llm_params` (`services/llm_params.py`, `services/sampling.py`). A sampling preset applies only when one is chosen: there is no implicit default here, because the mode default already differs from provider defaults. Only the preset's `llm_params` count; its `sampling_policy` is an agent-mode setting. The snapshot records the preset as provenance. `POST /jobs/llm-preview/ {mode, profile_id?, sampling_preset_id?, llm_params?}` returns the effective params with the source of each value (`mode_default`, `profile`, `preset`, `inline`). -Sampling in the template modes starts from the **mode default** (`temperature` 0.3, `max_tokens` 32000), then layers profile `llm_config` → sampling preset `llm_params` → experiment `llm_defaults` → condition `param_overrides` → per-run `llm_params` (`services/llm_params.py`, `services/sampling.py`). A sampling preset applies only when one is chosen (no implicit default here, since the mode default already differs from provider defaults), and only its `llm_params` count; its `sampling_policy` is an agent-mode setting. The snapshot records the preset as provenance. `POST /jobs/llm-preview/ {mode, profile_id?, sampling_preset_id?, llm_params?}` returns the effective params with the source of each (`mode_default`, `profile`, `preset`, `inline`). An explicit scaffolding `stack_slug` that disagrees with the profile's stack is a 400. +**Build healing** (`GenerationJob.build_healing`, `Experiment.build_healing`) decides how much the build repairs template-mode output before Docker sees it: + +| Value | What the build does | +| --- | --- | +| `full` (default) | Infers missing requirements, applies the runtime patches (sqlite path, port, app context, unknown lucide icons), and fills a missing entry file with a placeholder | +| `no_deps` | Applies the patches but leaves `requirements.txt` as the model wrote it | +| `none` | Builds the output exactly as written, so the deploy rate is attributable to the model alone | + +Healing is not part of the snapshot: healed and raw builds share a fingerprint. Every evaluation records what changed (`metrics.functional.healing_mode` / `healing_changed`, `result_data.evaluation.build.healing`). `manage.py healing_ablation --experiment [--modes none,no_deps]` re-evaluates unchanged outputs under other modes into `result_data.healing_ablation`, for a paired estimate of the pipeline's contribution ([Study tables](/docs/STUDY_TABLES)). + +**Grading.** Scaffolding jobs of an experiment are graded by the conformance suite after generation, unless the experiment's `post_run.evaluate` is off. Ad-hoc scaffolding jobs are graded on demand through `POST /jobs//evaluate/`. ## Job lifecycle (all modes) -1. The API view creates the job(s), freezes the snapshot, and calls `dispatcher.dispatch_job`, which runs `GenerationService.execute` on a daemon thread. -2. Status moves `pending → running → completed | failed | cancelled`. Progress streams over the `generation:` SSE channel. While running, `GenerationJob.phase` records `queued` (waiting for a sandbox slot), `running` or `evaluating`, so lists and counts survive a reload; it is cleared when the job ends. A cancel request always wins, even if it races the end of a run. -3. Agent and scaffolding creates, and experiment launches, group their jobs in a `GenerationBatch` that records the request (base seed, trials, stack, preset ids, apps, models), so a batch's seed matrix and run order can be audited. A batch's status is derived from its jobs: `pending`/`running` while any job is active, then `completed` (all completed), `cancelled` (all cancelled), `failed` (none completed) or `partial`. `GET /batches/`, `GET /batches//`, `GET /batches//jobs/` and `POST /batches//cancel/` serve the batch views; `GET /jobs/` filters by `batch_id` and `harness`. -4. The OpenRouter key is the **job owner's**, resolved by `credentials/services/resolver.py`. -5. A completed job can be built and started by the runtime app, then analyzed ([analysis pipeline](/docs/ANALYSIS_PIPELINE)). +1. The API view creates the job(s) and freezes the snapshot. `dispatcher.dispatch_job` / `enqueue_jobs` then marks the rows as executor-owned (`queued_at`, lane, endpoint, priority, `sequence`) and wakes the executor after the transaction commits. Agent jobs go to the `sandbox` lane, and scaffolding and custom jobs to the `llm` lane. The web process never runs a job: the executor (`manage.py run_executor`) claims jobs under a lease and runs `GenerationService.execute` (`services/orchestrator.py`) in a worker thread. See [Execution engine](/docs/EXECUTION). +2. Status moves `pending → running → completed | failed | cancelled`. While a job is `running`, `GenerationJob.phase` is `running` (sandbox or LLM calls), `eval_queued` (generation done, waiting for an eval slot) or `evaluating`. `queued` appears only on legacy thread-dispatch rows. Progress streams over the `generation:` SSE channel. A cancel always wins, even if it races the end of a run, but a job that is already being evaluated finishes on its own. +3. Agent and scaffolding creates, and experiment launches, group their jobs in a `GenerationBatch`. The batch records the request (base seed, run-order seed, trials, stack, preset ids, apps, models, inline overrides), so its seed matrix and run order can be audited. `GenerationBatch.kind` is `adhoc`, `launch`, `top_up`, `retry` or `schedule`. A batch's status is derived from its jobs: `pending`/`running` while any job is active, then `completed` (all completed, or no jobs), `cancelled` (all cancelled), `failed` (none completed) or `partial`. +4. LLM access is the **job owner's**, resolved by `inference.access.resolve_access`: the owner's OpenRouter key (`credentials/services/resolver.py`), or a local endpoint the owner may use. +5. A completed job can be built and started by the runtime app, then analyzed ([Analysis pipeline](/docs/ANALYSIS_PIPELINE)). + +**Cloning a finished job:** -For designed runs (models × harness presets × apps × repeats) see [Experiments](/docs/EXPERIMENTS). Experiments run in agent mode by default (`Experiment.mode`), with the stack and the sampling and budget presets set experiment-wide and a harness preset per condition; launches use common random numbers and blocked randomised order. The preview reports the budget cap as an upper-bound cost and a comparability report per condition. +| Action | Endpoint | Allowed from | Snapshot | Seed | Batch / experiment cell | +| --- | --- | --- | --- | --- | --- | +| Retry | `POST /jobs//retry/` | `failed`, `cancelled` | Re-resolved from current templates | The cell's seed (common random numbers), or a fresh one if the job never had a seed | Takes over the original's cell and batch; refuses if the cell is already filled; reopens a finished experiment | +| Replay | `POST /jobs//replay/` | Any finished job | Copied verbatim | Same seed | No batch and no cell (a reproducibility check) | +| Rerun | `POST /jobs//rerun/` | Any finished job | Copied, reseeded | Fresh seed; gateway policy kept | Joins the batch, no cell (an extra, unpaired trial) | + +Other job and batch endpoints: `GET /jobs/` (filters include `mode`, `status`, `model_id`, `batch_id`, `harness`, `experiment_id`, `condition_id`, `endpoint_id`, `app_requirement_id`, `repeat_index`, `phase`, `current`), `GET /jobs/stats/`, `GET /jobs//`, `POST /jobs//cancel/`, `DELETE /jobs//` (not while running), `GET /jobs//artifacts/`, `GET /jobs//export/`, `GET /batches/?mode=&experiment=`, `GET /batches//`, `GET /batches//jobs/` and `POST /batches//cancel/`. + +## In the UI + +| Page | What it does | +| --- | --- | +| `/sample-generator` | The composer, with one tab per mode (Agent, Template scaffolding, Single prompt) and a live run preview. `?model=` and `?app=` preselect, `?from_job=` starts from a job's settings, and `?from_batch=` reproduces a batch's whole grid with a new seed. The draft survives a reload | +| `/sample-generator/batches` | Every batch with its progress, mean pass rate and cost | +| `/sample-generator/batches/` | The results matrix and every run of one batch, with Cancel and a link to re-generate the grid | +| `/templates` | The Generation Library: app specs, stacks, presets (harness / sampling / budget), profiles and blocks, plus the Packages dialog | +| `/applications/` | One job: overview, Agent run summary (agent jobs), prompts, files, container, tools, analyses, logs, artifacts and metrics, with retry/replay/rerun and evaluate actions | +| `/applications//failure` | Error details and LLM artifacts of a failed job | + +For designed runs (models × harness presets × apps × repeats) see [Experiments](/docs/EXPERIMENTS). Experiments run in agent mode by default (`Experiment.mode`, either `agent` or `scaffolding`). The stack and the sampling and budget presets are set experiment-wide, with a harness preset per condition. Launches use common random numbers and a blocked randomised order, and the preview reports the budget cap as an upper-bound cost along with a comparability report per condition. diff --git a/docs/INTEGRATIONS.md b/docs/INTEGRATIONS.md index 8aa46dd7..a4aa10c8 100644 --- a/docs/INTEGRATIONS.md +++ b/docs/INTEGRATIONS.md @@ -1,119 +1,82 @@ # Integrations -LLM Lab connects outward in four ways, all managed under **Integrations** in the UI -(`/integrations`) and the API at `/api/integrations/`: +LLM Lab connects outward in four ways, managed under **Settings → Connections** (`/users/settings`) and the API at `/api/integrations/`. -| Integration | What it gives you | -|---|---| -| [MCP server](#1-mcp-server) | Claude, ChatGPT, Cursor, VS Code, Codex, Gemini CLI … drive LLM Lab with tools | -| [Cloud LLM providers](#2-cloud-llm-providers) | Claude API, OpenAI, Gemini, Mistral, Groq, DeepSeek, xAI, Together, Fireworks, Cerebras, NVIDIA as endpoints next to OpenRouter | -| [GitHub](#3-github) | Export a generated app to a repo (one commit) or Gist; findings as SARIF in code scanning | -| [Notifications](#4-notifications-webhooks) | Slack, Discord, Teams, ntfy, Telegram or signed JSON webhooks when jobs, analyses, experiments and reports finish | +| Integration | What it gives you | Settings section | +|---|---|---| +| [MCP server](#1-mcp-server) | Claude, ChatGPT, Cursor, VS Code, Codex, Gemini CLI, Windsurf … drive LLM Lab with tools | `#mcp` | +| [Cloud LLM providers](#2-cloud-llm-providers) | Claude API, OpenAI, Gemini, Mistral, Groq, DeepSeek, xAI, Together, Fireworks, Cerebras, NVIDIA NIM as endpoints next to OpenRouter | `#providers` | +| [GitHub](#3-github) | Export a generated app to a repo (one commit) or Gist; findings as SARIF in code scanning | `#github` | +| [Notifications](#4-notifications-webhooks) | Slack, Discord, Teams, ntfy, Telegram or signed JSON webhooks when jobs, analyses, experiments and reports finish | `#notifications` | + +The old `/integrations` page redirects to these sections (`?tab=github` → `#github`). ## 1. MCP server -`/api/mcp` is a [Model Context Protocol](https://modelcontextprotocol.io) server -(Streamable HTTP, stateless JSON responses; protocol versions 2024-11-05 → 2025-11-25). +`/api/mcp` is a [Model Context Protocol](https://modelcontextprotocol.io) server (Streamable HTTP, stateless JSON responses, no server-initiated stream). It speaks protocol versions `2025-11-25`, `2025-06-18`, `2025-03-26` and `2024-11-05`, and answers unknown versions with the latest. Code: `backend/integrations/mcp/`. -**Auth.** Send an LLM Lab API token: `Authorization: Bearer llml_…`. Every tool call is -replayed in-process against the REST API *as that token's user*, so permissions, ownership -and validation are exactly the API's. +**Auth.** Send an LLM Lab API token: `Authorization: Bearer llml_…`. Every tool call is replayed in-process against the REST API *as that token's user* (`mcp/dispatch.py`), so permissions, ownership and validation are exactly the API's. Token minting (`/api/tokens/`), first-run bootstrap and `/api/mcp` itself are unreachable from MCP. -| Token scopes | Effect | +| Token scopes | Effect on MCP | |---|---| | *(none)* | full access, header auth only | -| `mcp` | may also be embedded in the URL: `/api/mcp/t/` — for web connectors that cannot send headers (claude.ai, ChatGPT). A token without `mcp` is refused there. | -| `read` | read-only: write tools are hidden, `call_api` allows GET only | +| `mcp` | may also be embedded in the URL: `/api/mcp/t/`, for web connectors that cannot send headers (claude.ai, ChatGPT). A token without `mcp` is refused there. | +| `read` (or `mcp:read`) | read-only: write tools are hidden, `call_api` allows GET only | + +Create a connection token in **Settings → MCP server** (it mints one via `POST /api/tokens/` and fills it into the snippets) or under **Settings → Personal tokens**. Treat URL tokens like passwords, since they appear in proxy logs. Prefer `["mcp", "read"]` for connectors you share. -Create tokens under **Integrations → MCP server**; treat URL tokens like passwords (they -appear in proxy logs) and prefer `["mcp", "read"]` for connectors you share. +> [!WARNING] +> Scopes only restrict the MCP endpoint. The same token sent as a Bearer header to the REST API has its user's full rights. -**Tools.** ~50 tools: +**Tools.** 52 in total (`mcp/tools.py`): -- *curated* — `search_models`, `list_app_templates`, `generate_apps`, `create_agent_jobs`, - `get_generation_job`, `build_app`, `analyze`, `list_findings`, `rankings`, `dashboard`, - `preview_experiment`, `create_experiment`, `experiment_results`, `generate_report`, … Each - maps to one REST operation and takes its input schema from the OpenAPI definition, so tools - never drift from the API; -- *generic* — `find_api_operations` → `describe_api_operation` → `call_api` reach every one - of the 225+ operations the UI uses; -- *docs* — `search_docs`, `read_doc`. +- *47 curated tools*, each mapped to one REST operation with its input schema taken from the OpenAPI definition, so tools never drift from the API. They cover account and catalog (`whoami`, `search_models`, `get_model`, `compare_models`, `list_app_templates`, `list_stacks`) and generation (`generate_apps`, `generate_custom`, `preview_agent_jobs`, `create_agent_jobs`, `list_generation_jobs`, `get_generation_job`, `get_job_artifacts`, `generation_stats`, `cancel_generation_job`, `retry_generation_job`). Runtime: `build_app`, `smoke_test_app`, `list_containers`, `container_logs`, `start_container`, `stop_container`. Analysis: `list_analyzers`, `analyze`, `list_analysis_runs`, `get_analysis_run`, `list_findings`, `cancel_analysis_run`. Results: `rankings`, `top_models`, `dashboard`, `top_findings`, `list_reports`, `generate_report`, `get_report_data`. Experiments: `list_experiments`, `preview_experiment`, `create_experiment`, `get_experiment`, `experiment_status`, `experiment_results`, `launch_experiment`, `pause_experiment`, `resume_experiment`, `cancel_experiment`. Infrastructure: `list_endpoints`, `execution_queue`. +- *3 generic tools*: `find_api_operations` → `describe_api_operation` → `call_api` reach every REST operation not excluded above (the tool descriptions do not hard-code a count). +- *2 docs tools*: `search_docs`, `read_doc`. -`?toolset=core` (e.g. `/api/mcp?toolset=core`) lists only 20 core tools (~3.5k tokens of -definitions instead of ~12k) for clients with a small tool budget. +A `read` token sees 33 tools. `?toolset=core` (e.g. `/api/mcp?toolset=core`) lists only 20 core tools (about 13k characters of definitions instead of about 48k) for clients with a small tool budget. Combined with a `read` token, that leaves 18. **Resources**: every docs page (`llmlab://docs/`) and `llmlab://openapi.json`. -**Prompts**: `compare_models`, `triage_findings`, `experiment_report`, `platform_health`. +**Prompts**: `compare_models` (`task`, `models`), `triage_findings` (`run_id`), `experiment_report` (`experiment_id`), `platform_health`. -**Connecting clients** — *Integrations → MCP server* renders these with your token filled in -(`GET /api/integrations/mcp/config/`): +**Connecting clients.** **Settings → MCP server** renders ready-to-paste snippets with your token filled in (`GET /api/integrations/mcp/config/`): ```bash # Claude Code claude mcp add --transport http llm-lab https:///api/mcp --header "Authorization: Bearer " ``` -- **claude.ai / Claude Desktop / mobile**: Settings → Connectors → Add custom connector → - `https:///api/mcp/t/`. -- **ChatGPT**: Settings → Apps & Connectors → Advanced → Developer mode → Create; the URL-token - URL, authentication "No authentication". -- **Cursor / VS Code / Gemini CLI / Windsurf / Codex CLI**: JSON/TOML snippets on the page. -- **Claude API** (MCP connector, beta `mcp-client-2025-11-20`) and **OpenAI Responses API** - (`{"type": "mcp", …}` tool): code snippets on the page. These call the server from the - provider's cloud, so the instance must be publicly reachable. +- **claude.ai / Claude Desktop / mobile**: Settings → Connectors → Add custom connector → `https:///api/mcp/t/`. +- **ChatGPT**: Settings → Apps & Connectors → Advanced → Developer mode → Create. Use the URL-token URL with authentication set to "No authentication". +- **Cursor / VS Code / Gemini CLI / Windsurf / Codex CLI**: JSON/TOML snippets on the page, plus an MCP Inspector command. +- **Claude API** (MCP connector, beta `mcp-client-2025-11-20`) and **OpenAI Responses API** (`{"type": "mcp", …}` tool): code snippets on the page. These call the server from the provider's cloud, so the instance must be publicly reachable. -Generation and agent runs spend LLM credits; the server instructions ask the model to confirm -with the user and to prefer the free `preview_*` tools first. +Generation and agent runs spend LLM credits. The server instructions ask the model to confirm with the user first and to prefer the free preview tools. ## 2. Cloud LLM providers -`GET /api/integrations/providers/` lists presets; **Connect** (staff only, like every direct -endpoint) validates the key against the provider's `/models`, creates a private direct -`openai_compatible` endpoint (key encrypted at rest), runs the inventory and hides non-chat -models (embeddings, TTS, image, moderation …). Models appear as `local-/` and can -be used in experiments and agent runs like any other endpoint (see `LOCAL_MODELS.md`). +`GET /api/integrations/providers/` lists the presets (open to every user) with any endpoints already connected from them. `POST /{preset_id}/test/` checks a key without saving it. `POST /{preset_id}/connect/` validates the key against the provider's `/models`, creates a private direct `openai_compatible` endpoint (key encrypted at rest), runs the inventory and hides non-chat models (embeddings, TTS, image, moderation …). Test and connect are staff-only, like every direct endpoint. Models appear as `local-/` and can be used in experiments and agent runs like any other endpoint (see [Local models](/docs/LOCAL_MODELS)). -Provider quirks handled by the executor's upstream caller: +Provider quirks handled by the upstream caller (`backend/inference/upstream.py`): -- **Claude API** (`api.anthropic.com`, OpenAI-compatible layer): `anthropic-version` is sent, - and `temperature` / `top_p` / `top_k` are dropped because current Claude models reject them. - Sampling settings therefore have no effect on Claude endpoints. -- **Gemini** (`…/v1beta/openai`): requests go to `/chat/completions` and `/models` without the - `/v1` prefix. +- **Claude API** (`api.anthropic.com`, OpenAI-compatible layer): `anthropic-version` is sent, and `temperature` / `top_p` / `top_k` are dropped because current Claude models reject them. Sampling settings therefore have no effect on Claude endpoints. +- **Gemini** (`…/v1beta/openai`): requests go to `/chat/completions` and `/models` without the `/v1` prefix. -Caveats: hiding non-chat models happens once, at connect time; `/v1/models` is fetched without -pagination; cloud endpoints rarely report context windows. Requests are executed by the -**executor** service — it must be running for cloud endpoints to answer. +Caveats: non-chat models are hidden only once, at connect time. `/v1/models` is fetched without pagination. Cloud endpoints rarely report context windows. Requests are executed by the **executor** service, which must be running for cloud endpoints to answer ([Execution](/docs/EXECUTION)). ## 3. GitHub -Store a token with `PUT /api/credentials/github/` (`{"api_key": ""}`; classic scopes -`repo`, plus `gist` and `security_events` as needed; fine-grained: Administration + Contents -write). `GET /api/integrations/github/status/` shows the connected login. +Store a token in **Settings → GitHub** or with `PUT /api/credentials/github/` (`{"api_key": ""}`). A classic token needs scope `repo`, plus `gist` and `security_events` as needed. A fine-grained token needs Administration and Contents write. `GET /api/integrations/github/status/` shows the connected login, the token type and (classic tokens only) its scopes. -`POST /api/integrations/github/export/` -`{application_id, target: "repo"|"gist", repo_name, private, description?, upload_sarif, include_scaffold}` -creates the repo and pushes the app (stack skeleton + generated code, junk and files > 1 MB -skipped) as one commit with an LLM Lab README (model, template, findings summary). With -`upload_sarif`, the app's findings go to GitHub code scanning (needs a public repo or GitHub -Advanced Security). Gists hold only the generated files, flattened. +`POST /api/integrations/github/export/` takes `{application_id, target: "repo"|"gist", repo_name, private, description?, upload_sarif, include_scaffold}`. `application_id` is the generation job UUID. For a repo, it creates the repository and pushes the app (stack skeleton plus generated code) as one commit. Junk and files over 1 MB are skipped, with a 50 MB total cap. The commit adds an LLM Lab README (model, template, findings summary), written as `LLM_LAB_EXPORT.md` if the app has its own `README.md`. With `upload_sarif`, the app's findings go to GitHub code scanning, which needs a public repo or GitHub Advanced Security. Gists hold only the generated files, flattened, up to 50 files and 1 MB in total. ## 4. Notifications (webhooks) -`/api/integrations/webhooks/` — up to 25 subscriptions per user, each a kind (`slack`, -`discord`, `teams`, `ntfy`, `telegram`, `generic`), a target and a set of events: -`generation.{completed,failed,cancelled}`, `analysis.{completed,failed,cancelled}`, -`experiment.{completed,cancelled}`, `report.{completed,failed}`. Jobs inside an experiment -only notify with `include_experiment_jobs`. - -- URLs and secrets are encrypted and write-only. `generic` deliveries carry - `X-LLMLab-Event`, `X-LLMLab-Delivery` and, with a secret, - `X-LLMLab-Signature: sha256=`. -- Delivery is a Celery task (10 s timeout, 3 retries with backoff), de-duplicated per - `(subscription, event)`. Events are caught by `post_save` and by a 60 s sweep - (`integrations: sweep webhook events` beat task), since the executor finishes jobs with bulk - updates. -- Targets must be public https (http only with `DEBUG`); private, loopback and platform - addresses are refused, connections go to the vetted IP and redirects are not followed. - Allow a LAN ntfy/Teams host with `INFERENCE_DIRECT_ALLOWED_TARGETS`. -- `POST /{id}/test/` sends a sample event; `GET /{id}/deliveries/` shows the last 50. +`/api/integrations/webhooks/` allows up to 25 subscriptions per user. Each has a kind (`slack`, `discord`, `teams`, `ntfy`, `telegram`, `generic`), a target and a set of events (`GET /event-types/`): `generation.{completed,failed,cancelled}`, `analysis.{completed,failed,cancelled}`, `experiment.{completed,cancelled}`, `report.{completed,failed}`. Jobs inside an experiment only notify with `include_experiment_jobs`. + +- URLs and secrets are encrypted and write-only. Telegram takes a bot token (as the secret) and `telegram_chat_id` instead of a URL. ntfy takes an optional access token. `generic` deliveries carry `X-LLMLab-Event`, `X-LLMLab-Delivery` and, with a secret, `X-LLMLab-Signature: sha256=`. +- Delivery is a Celery task (`deliver_webhook`) with a 10 s timeout and 3 retries on 5xx, 429 or network errors (backoff 15 s, 60 s, 240 s). It is de-duplicated per `(subscription, event)`. Events are caught by `post_save` and by a 60 s sweep (`integrations: sweep webhook events` beat task, registered after `migrate`), since the executor finishes jobs with bulk updates. +- Targets must be public https (http only with `DEBUG`). Private, loopback and platform addresses are refused, connections go to the vetted IP, and redirects are not followed. Allow a LAN ntfy or Teams host with `INFERENCE_DIRECT_ALLOWED_TARGETS`. +- `POST /{id}/test/` sends a sample event. `GET /{id}/deliveries/` shows the last 50. + +Without a running Celery worker, deliveries stay pending ([Background services](/docs/BACKGROUND_SERVICES)). diff --git a/docs/LOCAL_MODELS.md b/docs/LOCAL_MODELS.md index 7f0e5b7a..adf3c5e7 100644 --- a/docs/LOCAL_MODELS.md +++ b/docs/LOCAL_MODELS.md @@ -1,39 +1,44 @@ # Local models: endpoints, the relay and direct servers -LLM Lab runs experiments on OpenRouter models and on models you host -yourself: Ollama or any OpenAI-compatible server (vLLM, llama.cpp server, -LM Studio). Every place models run is an **endpoint** (`/endpoints` in the -UI, `/api/inference/endpoints/` in the API). The OpenRouter endpoint is -built in; you add your own. +LLM Lab runs experiments on models you host yourself (Ollama or any OpenAI-compatible server such as vLLM, llama.cpp server or LM Studio) through endpoints, next to the built-in OpenRouter endpoint. -This page covers setup, the settings that matter for fair experiments, how -requests flow, the security model and troubleshooting. Execution internals -(the executor, slots, retries) are in `docs/EXECUTION.md`. +Every place models run is an **endpoint** (`/endpoints` in the UI, `/api/inference/endpoints/` in the API). This page covers setup, the settings that matter for fair experiments, how requests flow, the security model and troubleshooting. Execution internals (the executor, slots, retries, the timeout chain) are in [Execution](/docs/EXECUTION); how experiments use endpoints is in [Experiments](/docs/EXPERIMENTS). The OpenRouter catalog itself is covered in [Models reference](/docs/MODELS_REFERENCE). ## 1. Two ways to connect | | **Relay** (recommended) | **Direct** | |---|---|---| -| Who can create it | any user | staff only | +| Who can create it | any user (up to 100 relay endpoints per non-staff owner) | staff only | | Network | the GPU box dials out over HTTPS; works behind NAT and firewalls | the executor container calls `base_url`; the host must route to it (e.g. Tailscale on the host) | | What runs on the GPU box | `relay/llmlab_relay.py` next to the model server | only the model server | -| Credentials | an endpoint-scoped relay token (rotatable, revocable) | optional API key, stored encrypted | +| Credentials | endpoint-scoped relay tokens (rotatable, revocable; at most 20 live per endpoint for non-staff) | optional API key, stored encrypted | -Both feed the same durable request queue, so caps, priorities and -cancellation behave identically. +Both feed the same durable request queue (`LLMRequest`), so caps, priorities and cancellation behave identically. ## 2. Add a relay endpoint -1. **Endpoints → Add endpoint**: kind (Ollama / OpenAI-compatible), name, - slug (immutable: it is part of every model id, `local-/`), - slots (`max_concurrency`), max resident models (K), request timeout, - `num_ctx` cap and keep-alive. -2. The token is shown **once**. Store it in a file with mode 0600; it is - never placed in a command line. -3. Follow the **Install** tab (Python, Docker or systemd). Every command - verifies the script's sha256 shown in the UI. See `relay/README.md`. -4. The page turns green when the relay connects and lists the discovered - models ("12 models · 9 with tools"). +1. **Endpoints → Add endpoint** (`/endpoints/new`) is a three-step wizard: **Server** (kind: Ollama or OpenAI-compatible; relay or direct), **Settings** (name, slug, slots, limits, timeouts; see the table below) and **Connect**. +2. On Connect the relay token is shown **once**. Store it in a file with mode 0600; it is never placed in a command line. +3. Follow the install instructions on the same step (Python, Docker or systemd; later also on the endpoint's **Tokens** tab). Every command verifies the script's sha256. The commands point at `LLMLAB_PUBLIC_URL` (or, when unset, the scheme and host of your request). See `relay/README.md`. +4. The page turns green when the relay connects and lists the discovered models ("N models · M with tools"). A **Test** button probes the server. + +### Endpoint settings + +| Field | Default | Meaning | +|---|---|---| +| `slug` | from the name | `^[a-z0-9][a-z0-9-]{1,38}$`; `openrouter` and `local` are reserved. Immutable: it is part of every model id (`local-/`) and of the fingerprint. | +| `kind` | | `ollama` or `openai_compatible`. | +| `connection` | `relay` | `relay` or `direct` (staff). | +| `api_flavor` | `native` | Ollama only: `native` (`/api/chat`) or `openai` (`/v1/chat/completions`). OpenAI-compatible endpoints are always `openai`. | +| `max_concurrency` (slots) | 1 | In-flight upstream requests, 1..256. | +| `max_active_jobs` | null (= slots) | Job cap: running jobs on this endpoint, 1..256. | +| `max_resident_models` (K) | null (unlimited) | Models the server holds at once, 1..64. | +| `request_timeout_s` | 900 | Per upstream call, 30..7200; drives the [timeout chain](/docs/EXECUTION#timeout-chain-local-models). | +| `keep_alive` | `30m` | Ollama `keep_alive` sent with each request. | +| `num_ctx_cap` | 32768 | Ollama: effective `num_ctx = min(model context, cap)`. | +| `assume_tools` | false | OpenAI-compatible servers report no capabilities; this sets the default tool flag. | +| `enabled` | true | Disabled endpoints claim nothing. | +| `visibility` | `private` | `shared` is staff-only (see Sharing). | ### Ollama settings on the GPU box @@ -46,168 +51,120 @@ Environment="OLLAMA_MAX_LOADED_MODELS=" # = the endpoint's max resident m Environment="OLLAMA_KEEP_ALIVE=30m" ``` -- **Slots.** `max_concurrency` is enforced by the server when a request is - claimed, for scaffolding calls, agent calls, probes and AI-review calls - together. Match it to `OLLAMA_NUM_PARALLEL`. For strict reproducibility - use 1: Ollama batches parallel requests, which can break seed - determinism. -- **Residency (K).** The scheduler never starts a model that would exceed K - resident models on the endpoint. With the default `blocked` policy each - model's cells run back to back, so a model loads once per repeat block. - `interleaved` maximises temporal fairness but reloads models constantly - when K is small (the composer warns `model_thrash_risk`). -- **Warm-up (prefetch).** For Ollama, before a lane's next model starts, the - executor sends one `/api/generate` with an empty prompt and admits the job - once the model is loaded, so load time never counts against an agent's - wall clock (`execution.prefetch_load_s` records it). -- **`num_ctx` and VRAM.** Ollama's default context is small; LLM Lab always - sends `options.num_ctx = min(model context, endpoint num_ctx cap)` (or a - per-model override). Large contexts cost VRAM: if a load fails with - "requires more system memory" the run gets `upstream_oom`; lower the cap - or K. The relay's `--max-num-ctx` clamps it locally as a last resort. - The effective `num_ctx` is part of the run fingerprint. -- **API flavor.** `native` (default for Ollama) uses `/api/chat`, which - honours `num_ctx`, `keep_alive` and every sampling option. `openai` uses - `/v1/chat/completions` and cannot set `num_ctx`; then set - `OLLAMA_CONTEXT_LENGTH` on the server yourself. +- **Slots.** `max_concurrency` is enforced by the server when a request is claimed, for scaffolding calls, agent calls, probes and AI-review calls together. A relay session's capacity is the smaller of its `--parallel` and the endpoint's slots. Match both to `OLLAMA_NUM_PARALLEL`. For strict reproducibility use 1: Ollama batches parallel requests, which can break seed determinism. +- **Residency (K).** The scheduler never starts a model that would exceed K resident models on the endpoint (`residency_wait`). An experiment's `execution_options.max_resident_models` can only tighten it. With the `blocked` order (the `auto` default for a finite K) each model's cells run back to back, so a model loads once per repeat block. `interleaved` maximises temporal fairness but reloads models constantly when K is small (the composer warns `model_thrash_risk`). +- **Warm-up (prefetch).** For Ollama, before a lane's next model starts, the executor sends one `/api/generate` with an empty prompt and admits the job once the model is loaded, so load time never counts against an agent's wall clock (`execution.prefetch_load_s` records it). Turn it off per experiment with `execution_options.warmup: false`. +- **`num_ctx` and VRAM.** Ollama's default context is small; LLM Lab always sends `options.num_ctx = min(model context, endpoint num_ctx cap)` (or a per-model override). Large contexts cost VRAM: if a load fails with "requires more system memory" the run gets `upstream_oom`; lower the cap or K. The relay's `--max-num-ctx` clamps it locally as a last resort. The effective `num_ctx` is part of the run fingerprint; agent runs below 16 384 tokens get the `num_ctx_small` warning. +- **API flavor.** `native` (default for Ollama) uses `/api/chat`, which honours `num_ctx`, `keep_alive` and every sampling option. `openai` uses `/v1/chat/completions` and cannot set `num_ctx`; then set `OLLAMA_CONTEXT_LENGTH` on the server yourself (the preview warns `num_ctx_server_default`). ### One endpoint per machine -Slots and K describe one machine, so create one endpoint per GPU box. A -second relay on the same endpoint (different `relay_id`, e.g. a second box -by mistake, or a replacement) connects as **standby**: it gets no work and -takes over when the active relay disconnects or times out (120 s). To spread -30 models over several boxes, create an endpoint per box and use "Balance -across endpoints" in the composer. +Slots and K describe one machine, so create one endpoint per GPU box. A second relay on the same endpoint (different `relay_id`, e.g. a second box by mistake, or a replacement) connects as **standby**: it gets no work and takes over when the active relay disconnects or times out (120 s). To spread models over several boxes, create an endpoint per box and use **Balance across endpoints** in the composer's Models section. ## 3. Direct endpoints (staff) -A direct endpoint is called from the executor container. `base_url` is the -server root (e.g. `http://100.101.102.103:11434`). It is validated against -SSRF rules: +A direct endpoint is called from the executor container. `base_url` is the server root (e.g. `http://100.101.102.103:11434`). It is validated against SSRF rules (`backend/inference/security.py`): - http/https only, no credentials, query or fragment in the URL; -- compose service names (`postgres`, `redis`, `django`, `apps-dind`, …), - `localhost` and metadata hosts are rejected; -- every resolved address must not be loopback, link-local, multicast, - reserved, inside a network the executor is attached to, the Docker host - (default gateway) or one of `INFERENCE_HOST_PUBLIC_IPS`; -- private, CGNAT and LAN addresses (Tailscale is `100.64.0.0/10`) need an - explicit `INFERENCE_DIRECT_ALLOWED_TARGETS` entry `cidr:port`, e.g. - `INFERENCE_DIRECT_ALLOWED_TARGETS=100.64.0.0/10:11434`; -- public addresses are allowed on any port except 22, 25, 2375-2376, 5432 - and 6379. - -At call time the host is re-resolved and re-validated on every connection, -the socket connects to the vetted IP (no DNS rebinding), redirects are never -followed and TLS verifies the original hostname. The executor has no -`extra_hosts`; `host.docker.internal` is not a supported target. Pumps in -the executor refresh the model list every 120 s (or on **Refresh models**). +- compose service names (`postgres`, `redis`, `django`, `apps-dind`, `executor`, `frontend`, …), `localhost` and metadata hosts are rejected (`INFERENCE_BLOCKED_HOSTS`); +- every resolved address must not be loopback, link-local, multicast, reserved, inside a network the executor is attached to, the Docker host (default gateway), one of `INFERENCE_HOST_PUBLIC_IPS` or in `INFERENCE_EXTRA_BLOCKED_CIDRS`; +- private, CGNAT and LAN addresses (Tailscale is `100.64.0.0/10`) need an explicit `INFERENCE_DIRECT_ALLOWED_TARGETS` entry `cidr:port`, e.g. `INFERENCE_DIRECT_ALLOWED_TARGETS=100.64.0.0/10:11434`; +- public addresses are allowed on any port except 22, 25, 2375, 2376, 5432 and 6379. + +At call time the host is re-resolved and re-validated on every connection, the socket connects to the vetted IP (no DNS rebinding), redirects are never followed and TLS verifies the original hostname. The executor has no `extra_hosts`; `host.docker.internal` is not a supported target. Pumps in the executor refresh the model list every 120 s (or on **Refresh models**); a direct endpoint without a fresh inventory for 10 minutes counts as offline. ## 4. Models -Discovered models appear in the model picker grouped by endpoint, as -`local-/`, free and priced 0. Each row records the exact -served name (`qwen2.5-coder:32b`), the digest of its weights, family, -parameter size and quantization, and the capabilities Ollama reports -(`tools`, `vision`, `thinking`). Models with no `tools` capability cannot -drive tool-calling harnesses (OpenCode, mini-swe-agent); Aider works with -any model. On the endpoint's Models tab you can hide models, override -`num_ctx` or the tool flag (OpenAI-compatible servers report no -capabilities; the endpoint's "assume tools" sets the default). - -**Digest changes.** When a model's weights change upstream (`ollama pull`), -the old digest is kept in the model's history and every running experiment -with pending runs that froze the old digest is paused (`digest_changed`). -Resume with "accept digest drift" (pending runs are re-frozen and the -results are flagged as mixed) or clone the experiment. Nothing ever runs on -different weights than its snapshot recorded: the relay is only given -requests for `(name, digest)` pairs it currently serves. - -**Fingerprints.** A local run's fingerprint includes -`llm_source = {kind, api_flavor, served name, digest, num_ctx}` and the -model id embeds the endpoint slug, so the same weights on two machines are -two conditions (no cross-endpoint pooling in v1). OpenRouter runs are -unchanged. +Discovered models appear in the model picker grouped by endpoint, as `local-/`, free and priced 0. Each row records the exact served name (`qwen2.5-coder:32b`), the digest of its weights, family, parameter size and quantization, and the capabilities Ollama reports (`tools`, `vision`, `thinking`). Models with no `tools` capability cannot drive tool-calling harnesses (the preview blocks with `models_without_tools`); Aider works with any model. On the endpoint's **Models** tab you can hide models, override `num_ctx` or the tool flag, rename the display name, run a one-token test against a model and (Ollama) unload it. A link from there opens the composer with those models preselected. + +**Digest changes.** When a model's weights change upstream (`ollama pull`), the old digest is kept in the model's history and every running experiment with pending runs that froze the old digest is paused (`digest_changed`). Resume with "accept digest drift" (pending runs are re-frozen and the results are flagged as mixed) or clone the experiment. Nothing ever runs on different weights than its snapshot recorded: the relay is only given requests for `(name, digest)` pairs it currently serves (`model_not_served` otherwise). + +**Fingerprints.** A local run's fingerprint includes `llm_source = {kind, api_flavor, served name, digest, num_ctx}` and the model id embeds the endpoint slug, so the same weights on two machines are two conditions (`same_digest_multiple_endpoints`; no cross-endpoint pooling). OpenRouter runs are unchanged. ## 5. How a call flows -- **Scaffolding / repairs / AI review:** the executor (or the analysis - worker) creates an `LLMRequest` row and waits; the endpoint's active relay - claims it on its next long-poll (`POST /api/relay/v1/poll`, 25 s), runs it - against the local server and posts the result. -- **Agent runs:** the agent talks to its per-job gateway exactly as with - OpenRouter. For local models the gateway is the `llm-gateway-local` - overlay, which forwards to the executor's internal LLM proxy (an internal - Docker network; a narrow firewall exception allows only the gateway egress - network to reach it, on one IP and port). The proxy turns each call into - an `LLMRequest`, built from the job's frozen snapshot, and answers when the - relay delivers. Streaming harnesses get synthesized SSE with keep-alive - pings; non-streaming harnesses ride a 504 re-attach chain, so a long - queue never breaks a run. -- **Timeouts** form one chain derived from the endpoint's request timeout - (default 900 s): request deadline 3300 s, gateway socket 1200 s, proxy wait - 1140 s, harness client 6060 s. The chain guarantees the harness never gives - up before the endpoint does. -- **Priorities.** Experiment calls first; probes −20; AI-review analysis and - inventory −40. +```mermaid +flowchart LR + subgraph Server + E[Executor / analysis worker] -->|LLMRequest row| Q[(Postgres queue)] + G[Per-job gateway] --> P[Executor LLM proxy] -->|LLMRequest row| Q + D[Direct pump] -->|claims| Q + end + R[Relay on GPU box] -->|POST /api/relay/v1/poll| Q + R --> O[Ollama / OpenAI-compatible server] + D --> O2[Direct server] +``` + +- **Scaffolding / repairs / AI review:** the executor (or the analysis worker) creates an `LLMRequest` row and waits; the endpoint's active relay claims it on its next long-poll (`POST /api/relay/v1/poll`, 25 s), runs it against the local server and posts the result. For a direct endpoint, a pump thread in the executor claims and runs it. +- **Agent runs:** the agent talks to its per-job gateway exactly as with OpenRouter. For local models the gateway is the `llm-gateway-local` overlay, which forwards to the executor's internal LLM proxy (an internal Docker network; a narrow firewall exception allows only the gateway egress network to reach it, on one IP and port). The proxy turns each call into an `LLMRequest`, built from the job's frozen snapshot, and answers when the relay delivers. Streaming harnesses get synthesized SSE with keep-alive pings; non-streaming harnesses ride a 504 re-attach chain, so a long queue never breaks a run. +- **Timeouts** form one chain derived from the endpoint's request timeout (default 900 s): request deadline 3300 s, gateway socket 1200 s, proxy wait 1140 s, harness client 6060 s (see [Execution](/docs/EXECUTION#timeout-chain-local-models)). +- **Priorities.** A request's claim order is its priority plus a source offset: experiment calls (agent proxy, scaffolding, warm-up) 0, probes −20, AI-review analysis and inventory −40. An experiment's priority is copied to its requests. +- **Attempts.** A request has at most 3 attempts; a request body is cleared on completion, and responses are cleared once consumed or by the daily purge. ### Failure handling | Situation | What happens | |---|---| -| Relay offline | Jobs wait (status `endpoint_offline`); the launch preview warns. | +| Relay offline | Jobs wait (`endpoint_offline`); the launch preview warns. | | Relay restarts mid-request | The request returns to the queue (attempt +1) and runs again; the run records a relay interruption. | | A poll response is lost (proxy restart) | Claims carry a 60 s ack lease; unacknowledged claims return to the queue within a minute without using an attempt. | | Cancel | The relay is told within one poll and closes the upstream socket; Ollama stops generating. | | Network blip (< ~2 min) | The relay retries every few seconds while requests run; its session (120 s TTL) survives and running generations finish. | | Endpoint disabled | Relays stay connected and idle (polling every 15 s); nothing is claimed. | +| Repeated infra failures | Three within 5 minutes open the endpoint's circuit for 60 s (`circuit_open`). | ## 6. Tokens and sessions -- Format `llr_.`; only a sha256 of the secret is stored. - The prefix identifies the token in the UI. -- **Rotate** creates a new token; the old one keeps working for the grace - period (default 24 h, up to 7 days). Replace the token file and send the - relay `SIGHUP` (`systemctl reload llmlab-relay`), or just restart it. -- **Revoke** takes effect immediately: its sessions end, their requests - return to the queue and the relay aborts its running generations and exits - with code 3 (systemd will not - restart it; Docker with `--on-unauthorized wait` idles). -- The endpoint page lists sessions with hostname, version, client IP, - capacity and last-seen time. A new IP or hostname for a token is worth a - look. +- Format `llr_.`; only a sha256 of the secret is stored. The prefix identifies the token in the UI. +- **Rotate** creates a new token; the old one keeps working for the grace period (`grace_minutes`, default 1440 = 24 h, 0..10080 = 7 days). Replace the token file and send the relay `SIGHUP` (`systemctl reload llmlab-relay`), or just restart it. +- **Revoke** takes effect immediately: its sessions end, their requests return to the queue and the relay aborts its running generations and exits with code 3 (systemd will not restart it; Docker with `--on-unauthorized wait` idles). +- The endpoint's Overview tab lists sessions with role (active / standby), hostname, version, client IP, capacity and last-seen time, and flags a new hostname for a token. A new IP or hostname is worth a look. ### Client IP trust -Relays reach Django through Caddy and the SvelteKit proxy, which appends to -`X-Forwarded-For`. Django takes the **right-most** hop that is not in -`TRUSTED_PROXY_CIDRS` (set it to the frontend container and Caddy -addresses). Requests that bypass Caddy (the directly published frontend -port) can only spoof hops to the left of the frontend's own entry, so the -recorded IP is trustworthy only for traffic through Caddy. Production -adapter-node needs `BODY_SIZE_LIMIT` ≥ 16M for agent-sized results. +Relays reach Django through Caddy and the SvelteKit proxy, which appends to `X-Forwarded-For`. Django takes the **right-most** hop that is not in `TRUSTED_PROXY_CIDRS` (set it to the frontend container and Caddy addresses). Requests that bypass Caddy (the directly published frontend port) can only spoof hops to the left of the frontend's own entry, so the recorded IP is trustworthy only for traffic through Caddy. Production adapter-node needs `BODY_SIZE_LIMIT` ≥ 16M for agent-sized results. ## 7. Sharing -Endpoints are private. Staff can mark one **shared**; then every user can -run models on it. Non-owners see a consent banner in the composer ("Prompts -and generated code pass through a machine run by …") and never see -operational details: base URL, key prefix, sessions, IPs, hostnames and -errors are shown only to the owner and staff (REST and live updates). +Endpoints are private. Staff can mark one **shared**; then every user can run models on it. Non-owners must tick a consent note in the composer ("Prompts and generated code … pass through machines run by …"; preview issue `foreign_endpoint`) and never see operational details: base URL, key prefix, sessions, IPs, hostnames and errors are shown only to the owner and staff (REST and live updates, where non-owners get the `endpoint::public` channel). ## 8. Security model -- The relay holds no secrets except its own token and runs only allowlisted - inference paths; the script is pinned by sha256 and never self-updates. -- A stolen token lets an attacker read prompts and answer requests for that - endpoint's runs. Mitigations: endpoint-scoped tokens, 0600 token files, - instant revocation, session visibility. -- The internal LLM proxy accepts only per-job signed tokens that name the - job and attempt, valid only while that attempt holds a live lease. +- The relay holds no secrets except its own token and runs only allowlisted inference paths; the script is pinned by sha256 and never self-updates. +- A stolen token lets an attacker read prompts and answer requests for that endpoint's runs. Mitigations: endpoint-scoped tokens, 0600 token files, instant revocation, session visibility. +- The relay protocol is rate-limited per token (`RELAY_RATE_LIMIT_PER_S`, 20/s; 429) and caps bodies (8 MB, 32 MB decompressed; 413 `too_large` / `too_complex`). +- The internal LLM proxy accepts only per-job signed tokens that name the job and attempt, valid only while that attempt holds a live lease; endpoint access is re-checked for agent jobs. - Direct endpoints are staff-only and SSRF-validated on every connection. -## 9. Troubleshooting +## 9. API + +Endpoint routes are under `/api/inference/`; endpoints you cannot see are 404, mutations need the owner or staff. + +| Route | Purpose | +|---|---| +| `GET/POST /endpoints/` | list visible endpoints / create (a relay endpoint's first token is returned once) | +| `GET/PATCH/DELETE /endpoints/{id}/` | detail / update (`slug` immutable, `api_key: ""` clears the key) / delete (409 `in_use` once it has runs or requests: disable it instead) | +| `POST /endpoints/{id}/refresh/` | request a model inventory resync | +| `POST /endpoints/{id}/test/`, `GET …/test/{request_id}/` | probe (`/api/version` or `/v1/models`, or a one-token chat with `model_id`) and its result | +| `POST /endpoints/{id}/unload/` | unload a model (Ollama, `keep_alive: 0`) | +| `GET /endpoints/{id}/models/`, `PATCH …/models/{pk}/` | discovered models (`?include_hidden`) / hide, `num_ctx_override`, tool flag, display name | +| `GET/POST /endpoints/{id}/tokens/`, `POST …/tokens/{tid}/rotate/`, `POST …/tokens/{tid}/revoke/` | relay tokens | +| `GET /endpoints/{id}/sessions/` | relay sessions | +| `GET /endpoints/{id}/metrics/?minutes=` | per-minute in-flight time, completions, tokens and errors (5..1440 minutes, default 60) | +| `GET /endpoints/{id}/install/` | install commands with the server URL, slots and sha256 filled in | + +The relay protocol lives outside the ninja API at `/api/relay/v1/` (relay token as `Authorization: Bearer llr_…`): + +| Route | Purpose | +|---|---| +| `POST hello` | open a session (version check: 426 below `RELAY_MIN_VERSION`; 409 `kind_mismatch`); active or standby | +| `POST inventory` | push the model list (≤ 500 models) and loaded models | +| `POST poll` | long-poll (25 s) for claims; carries in-flight acks | +| `POST requests//result` | deliver a result | +| `POST bye` | end the session | +| `GET relay.py`, `GET Dockerfile`, `GET llmlab-relay.service` | the relay files, with `X-Relay-Version` and `X-Relay-Sha256` headers | + +## 10. Troubleshooting | Symptom | Check | |---|---| @@ -215,8 +172,12 @@ errors are shown only to the owner and staff (REST and live updates). | `401` / exit code 3 | token revoked, expired after rotation, or the wrong file | | `409 kind_mismatch` | `--kind` differs from the endpoint's kind | | `426` | the relay is older than `RELAY_MIN_VERSION`: fetch and verify the new script | +| `429` | the relay exceeds the per-token rate limit (`RELAY_RATE_LIMIT_PER_S`) | | Endpoint stays offline | relay logs; the relay must reach `https:///api/relay/v1/hello` | | Models listed but jobs wait (`model_not_served`) | the model was removed or re-pulled with other weights; see Digest changes | +| Jobs wait with `residency_wait` | K is reached; raise `max_resident_models` (and `OLLAMA_MAX_LOADED_MODELS`) or use the `blocked` order | | `upstream_oom` | lower the `num_ctx` cap or max resident models | -| Runs hit their wall clock | slow hardware: raise the agent wall clock or the request timeout; watch tokens/s on the endpoint page | +| Runs hit their wall clock | slow hardware: raise the agent wall clock or the request timeout (the preview warns `wall_clock_tight`); watch tokens/s on the endpoint page | | Relay polls but never gets work | the session is `standby` (another relay holds the endpoint) or the endpoint is disabled | + +More general problems are in [Troubleshooting](/docs/TROUBLESHOOTING). diff --git a/docs/MODELS_REFERENCE.md b/docs/MODELS_REFERENCE.md index d26b37f8..36941a7a 100644 --- a/docs/MODELS_REFERENCE.md +++ b/docs/MODELS_REFERENCE.md @@ -1,28 +1,73 @@ # Models reference -How the model catalog is populated from OpenRouter and how API keys are resolved when a job runs. +How the model catalog is populated and enriched, what the model page measures, and how OpenRouter keys are resolved when a job runs. ## The model catalog -`LLMModel` (`backend/llm_models/models.py`) stores one row per OpenRouter model: the `model_id` (e.g. `openai/gpt-4o`), provider, display name, capability flags (function calling, vision, streaming, JSON mode), context window and max output tokens, per-token pricing, and the full raw OpenRouter payload for anything else. A derived `cost_efficiency` score (0–1, free models score 1.0) feeds the rankings. +`LLMModel` (`backend/llm_models/models.py`) holds two kinds of rows: + +| Rows | `endpoint` | `model_id` | Populated by | +| --- | --- | --- | --- | +| OpenRouter catalog | empty (or the system OpenRouter endpoint) | the OpenRouter id, e.g. `openai/gpt-4o` | catalog sync / import | +| Models on your own endpoints (Ollama, OpenAI-compatible, cloud providers) | that `LLMEndpoint` | `local-/` | the endpoint's inventory, see [Local models](/docs/LOCAL_MODELS) | + +Each row stores provider, display name, capability flags (function calling, vision, streaming, JSON mode), context window and max output tokens, per-token pricing, the raw payload (`capabilities_json`) and structured `metadata`. Endpoint models also carry `served_model_name`, digest, quantization, parameter size, family, an optional `num_ctx_override`, and `is_available` / `is_hidden`. A derived `cost_efficiency` score (0 to 1, free models score 1.0) feeds the rankings. + +Visibility goes through `LLMModel.objects.visible_to(user)`: everyone sees the OpenRouter catalog plus models of endpoints they may use, and staff see every row. An invisible model is a 404 everywhere, including job and condition creation. ## Syncing from OpenRouter -`sync_models_from_openrouter` fetches the OpenRouter model list and upserts it — new models appear, existing ones update in place. Trigger it from the Models page in the UI (or via the `/api/models/` sync endpoint). The sync itself needs a resolvable OpenRouter key for the requesting user, so set up a key first. +`sync_models_from_openrouter` (`backend/llm_models/services/sync.py`) fetches the OpenRouter model list and upserts it: new models appear and existing ones update in place, keeping stored enrichment. Trigger it from the Models page or with `POST /api/models/sync/`. + +- **Staff only.** The sync uses the staff user's **own** stored OpenRouter key. The global `OPENROUTER_API_KEY` fallback is deliberately not used here, so without a personal key the call answers `403` with a `remediation` hint. +- `POST /api/models/detail//refresh/` (staff) re-fetches one catalog model under the same key rule and forces a Hugging Face refetch. Endpoint models refresh from their endpoint instead. +- `POST /api/models/import/` (staff) loads models from an exported JSON or a raw OpenRouter payload. `GET /api/models/export/?format=csv|json` exports what you can see. + +An empty models page after a fresh install means nobody has synced yet. + +## Enrichment beyond OpenRouter + +The model page combines the catalog row with three public, keyless sources (`backend/llm_models/services/enrichment.py`): -An empty models page after a fresh install just means nobody has synced yet. +| Source | Adds | Stored | +| --- | --- | --- | +| [models.dev](https://models.dev) | release / last-updated date, knowledge cutoff, family, open-weights flag, reasoning options, long-context price tiers | `metadata.models_dev`, refreshed on every catalog sync (download cached 6 h) | +| [Hugging Face Hub](https://huggingface.co) (models with a `hugging_face_id`) | license, parameter count, architecture, base model, downloads, likes, arXiv papers | `metadata.huggingface`, refetched after 7 days | +| OpenRouter endpoints API | every provider serving the model: price, quantization, limits, uptime, latency, throughput | not stored: fetched live, cached 10 min | + +The OpenRouter payload also carries Artificial Analysis indices and Design Arena Elo scores (`capabilities_json.benchmarks`), shown on the Benchmarks tab with catalog percentiles. + +A sync queues the Hugging Face pass as the Celery task `enrich_catalog_task` after commit. Without a Celery worker it never runs, so run it by hand with `just manage enrich_models`. Use `--no-hf` to do only models.dev, or `--force-hf` to refetch fresh entries. + +## Model page data + +| Endpoint | Returns | +| --- | --- | +| `GET /api/models/detail//` | The catalog row | +| `GET /api/models/detail//insights/` | What the platform measured for **your** jobs with this model: build success and conformance with Wilson intervals, tokens, cost, findings by severity and tool, per-template results, experiments, ranking positions, plus global catalog percentiles (price, context, Artificial Analysis indices) and local benchmark results (`backend/llm_models/services/insights.py`) | +| `GET /api/models/detail//endpoints/?refresh=` | Live OpenRouter provider list (empty for endpoint models) | +| `GET /api/models/detail//related/` | Other models from the same provider and source | +| `GET /api/models/comparison/?models=a,b,…` | Side-by-side data for up to 6 models | + +The list endpoint `GET /api/models/` filters by `search`, `provider`, `capability`, `free_only`, `price_range` (`free`, `low`, `medium`, `high`), `context_range` (`small`, `medium`, `large`, `xlarge`), `used_in_apps`, and `endpoint` (a slug, `local` or `openrouter`). It sorts with `sort_by` / `sort_dir`. Hidden and unavailable endpoint models are left out unless `include_unavailable=true`. ## API key resolution -Every OpenRouter call — generation, model sync, the `llm_review` analyzer — resolves its key per user via `backend/credentials/services/resolver.py`: +Every OpenRouter call made for a user resolves its key through `backend/credentials/services/resolver.py`. That covers generation jobs (including the key handed to an agent sandbox's gateway), experiment launch checks, and the `llm_review` analyzer. -1. The user's own key, stored encrypted via **Settings → API Credentials**, wins if set. +1. The user's own key, stored encrypted under **Settings → API keys** (`PUT /api/credentials/openrouter/`), wins if set. 2. Otherwise the deployment-wide `OPENROUTER_API_KEY` env var is used, but only while `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK` is `True` (the default). -3. Otherwise the call fails with a "No OpenRouter API key is configured for this account" error that points at the settings page. +3. Otherwise the call fails with "No OpenRouter API key is configured for this account." Experiment launch reports this as `missing_api_key`. + +Catalog sync and refresh skip step 2 (see above). Models on your own endpoints need no OpenRouter key: their access is resolved per endpoint by `backend/inference/access.py`. > [!TIP] > On shared deployments set `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=False` so each user's spend goes on their own key. -## API +The credentials API also stores `huggingface` and `github` keys. The GitHub key powers [GitHub export](/docs/INTEGRATIONS#3-github). + +## See also -`/api/models/` lists and filters the catalog and exposes the sync action; see the [API reference](/docs/api-reference). Model choice happens at generation time — see [Generation process](/docs/GENERATION_PROCESS). +- [API reference](/docs/api-reference) for the `/api/models/` router +- [Generation process](/docs/GENERATION_PROCESS) for where models are chosen +- [Local models](/docs/LOCAL_MODELS) for endpoints and their inventories diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 3f5dabad..ed254125 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -4,20 +4,22 @@ Get a local LLM Lab instance running with Docker Compose, from clone to your fir ## Prerequisites -- Docker with the Compose plugin, able to run one privileged container: analyzer workspaces and generated sample apps run nested inside the `apps-dind` service (Docker-in-Docker), never on the host daemon. -- [just](https://github.com/casey/just) and Python 3 on the host (only used to run the bootstrap script; everything else runs in containers). -- An [OpenRouter API key](https://openrouter.ai/keys). Not needed to boot the stack, but nothing can be generated or AI-reviewed without one. +- Docker with the Compose plugin, able to run one privileged container. Generated apps, agent sandboxes and analyzer workspaces run as nested containers inside the `apps-dind` service (Docker-in-Docker), never on the host daemon. +- [just](https://github.com/casey/just) and Python 3 on the host. Python only runs the bootstrap script; everything else runs in containers. +- Enough memory: the services' caps in `docker-compose.local.yml` add up to about 15 GB (apps-dind alone 4 GB). Idle use is far lower, but agent runs and app builds need several GB. +- An [OpenRouter API key](https://openrouter.ai/keys). Not needed to boot the stack, but nothing can be generated or AI-reviewed without one, unless you connect a local model server instead (see [Local models](/docs/LOCAL_MODELS)). ## Bootstrap ```bash -git clone && cd AutoLLM_Code_Analyzer_rework +git clone https://github.com/GrabowMar/AutoLLM_Code_Analyzer_rework.git +cd AutoLLM_Code_Analyzer_rework just bootstrap ``` -`just bootstrap` runs `scripts/bootstrap.py`: it copies `.envs/.local/.django.example` and `.envs/.local/.postgres.example` to their real names and fills in random Flower and Postgres credentials. It is idempotent and never overwrites existing `.env` files. +`just bootstrap` runs `scripts/bootstrap.py`. It copies `.envs/.local/.django.example` and `.envs/.local/.postgres.example` to `.envs/.local/.django` and `.envs/.local/.postgres` and replaces the `changeme` placeholders with random Flower and Postgres credentials. It never overwrites an existing file. -Optionally edit `.envs/.local/.django` now to set `OPENROUTER_API_KEY` as a global fallback key — or skip it and add a personal key per user in the UI later (see [Models reference](/docs/MODELS_REFERENCE)). +Optionally set `OPENROUTER_API_KEY` in `.envs/.local/.django` as a global fallback key. You can skip this and add a personal key per user in the UI later (see [Models reference](/docs/MODELS_REFERENCE)). ## Build and start @@ -28,31 +30,30 @@ just up | Service | URL / port | Role | | --- | --- | --- | -| frontend | http://localhost:8000 | SvelteKit dev server — **this is the app** | -| django | http://localhost:8001 | API backend (the frontend proxies `/api` to it) | -| mailpit | http://localhost:8025 | Catches all outgoing email (verification links) | -| flower | http://localhost:5555 | Celery dashboard (credentials from bootstrap) | -| postgres, redis, celeryworker, celerybeat | internal | Database, broker, background workers | +| `frontend` | http://localhost:8000 | SvelteKit dev server. **This is the app.** | +| `django` | http://localhost:8001 | API backend; the frontend proxies `/api`, `/_allauth`, `/admin`, `/media` and `/apps` to it | +| `mailpit` | http://localhost:8025 | Catches all outgoing email | +| `flower` | http://localhost:5555 | Celery dashboard (credentials from bootstrap, in `.envs/.local/.django`) | +| `executor` | internal | Runs every generation job, evaluation, experiment schedule and post-run | +| `apps-dind` | internal | Sandbox Docker daemon for apps, sandboxes and workspaces | +| `postgres`, `redis`, `celeryworker`, `celerybeat` | internal | Database, broker and pub/sub, background tasks | -On startup the django container runs `python manage.py migrate` automatically, and both the analyzer tool catalog and the generation library (stacks, app specs, blocks, profiles) seeds itself after every migrate. One thing is **not** automatic: - -```bash -just manage createsuperuser -``` +On startup the django container runs `python manage.py migrate`. After every migrate the analyzer tool catalog, the generation library (run presets, app specs, prompts, blocks, profiles), the built-in stacks and the system OpenRouter endpoint seed themselves. The executor waits for migrations to finish before it starts. ## First run -1. Open http://localhost:8000 and sign in (or register — the verification email lands in mailpit at :8025). -2. Add your OpenRouter key under **Settings → API Credentials**. It is stored encrypted and takes precedence over the global fallback key. -3. Sync the model catalog from the Models page so there is something to generate with. -4. Create a generation job (pick a model, a scaffolding template or a custom prompt) and wait for it to finish. -5. Open the job and start an analysis run. The first run provisions your personal analyzer workspace container and installs the selected tools, so it takes a while; later runs reuse it. +1. Open http://localhost:8000. On an empty database the sign-up page creates the **initial admin account**: a superuser whose email is `admin@` (`admin@example.com` with the template's `DJANGO_DOMAIN`), which you then sign in with. `just manage createsuperuser` works too if you prefer your own email. Later sign-ups create normal accounts; locally email verification is optional and the mails land in mailpit. +2. Add your OpenRouter key under **Settings → API keys** (`/users/settings#credentials`). It is stored encrypted and takes precedence over the global fallback key. +3. Open **Models** and sync the catalog. Sync is staff-only and uses the staff user's own key, not the global fallback. +4. Generate something: a single app from **Sample Generator**, or a planned set of runs from **Experiments** (see [Experiments](/docs/EXPERIMENTS)). The executor picks the job up within a few seconds. +5. Open the finished app under **Applications** and start an analysis. The first run provisions your personal analyzer workspace container (building the `backend/analyzer-base:latest` image) and installs the selected tools, so it takes a while; later runs reuse it. > [!NOTE] -> Generation and analysis run on background threads inside the django container. Restarting the container kills in-flight runs — see [Troubleshooting](/docs/TROUBLESHOOTING). +> Generation jobs run in the `executor` service and survive web restarts. Interactive analysis runs, workspace provisioning and tool installs still run on threads inside the `django` container, so restarting it (or a backend code reload) kills those. See [Background services](/docs/BACKGROUND_SERVICES). ## Next steps -- [Development guide](/docs/development-guide) — just targets, tests, env vars, code layout. -- [Architecture](/docs/ARCHITECTURE) — how the pieces fit together. -- [Troubleshooting](/docs/TROUBLESHOOTING) — when something above didn't work. +- [User guide](/docs/USER_GUIDE): the app page by page. +- [Development guide](/docs/development-guide): just targets, tests, env vars, code layout. +- [Architecture](/docs/ARCHITECTURE): how the pieces fit together. +- [Troubleshooting](/docs/TROUBLESHOOTING): when something above did not work. diff --git a/docs/README.md b/docs/README.md index b9bbad8f..99a56f23 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,38 +1,47 @@ # LLM Lab documentation -Guides and reference for running, using, and extending LLM Lab — the platform that generates one application per LLM, runs each in a Docker sandbox, and scores the code with 14 analysis tools. +Guides and reference for running, using and extending LLM Lab, the platform that has each LLM build the same application, grades and runs every app in a Docker sandbox, and scores the code with 33 analysis tools. ## What is LLM Lab -You pick a requirement template and a set of models from OpenRouter. For each model, the platform generates a complete application, can run it live in its own container, analyzes the code with the tools you select, and aggregates findings into statistics, reports, and per-model rankings. Everything works from the web UI or unattended through automation pipelines. +You pick an app spec and a set of models, from OpenRouter or from your own Ollama / OpenAI-compatible server. For each model the platform builds a complete application (in the primary agent mode, a coding-agent harness works in an isolated sandbox), grades it with a hidden conformance suite, can run it live in its own container, analyzes the code with the tools you install, and aggregates the results into statistics, rankings and study reports. Designed comparisons run as experiments, executed by a dedicated executor process; everything is also reachable over the REST API and an MCP server. ## Documentation map | Doc | What it covers | | --- | --- | +| **Getting started** | | | [Quickstart](/docs/QUICKSTART) | From clone to your first generated and analyzed app | -| [Development guide](/docs/development-guide) | just targets, tests, env vars, code layout, doc conventions | -| [Architecture](/docs/ARCHITECTURE) | How the pieces fit: apps, auth, threads vs Celery, Docker surfaces | -| [Generation process](/docs/GENERATION_PROCESS) | The generation modes, run presets, fairness rules, identity keys, and the job lifecycle | -| [Experiments](/docs/EXPERIMENTS) | Designed runs: conditions, presets, comparability, launch and export | -| [Study tables](/docs/STUDY_TABLES) | Pre-registered plans, the study report (Markdown/LaTeX), healing ablation, deduplicated findings, review sample, tool accounting | -| [Template packages](/docs/TEMPLATE_PACKAGES) | The package format, import/export rules, and starter packages | -| [Analysis pipeline](/docs/ANALYSIS_PIPELINE) | Analyzer workspaces, tool execution, findings | -| [Background services](/docs/BACKGROUND_SERVICES) | Celery, the scheduler, and the SSE event stream | -| [Automation guide](/docs/AUTOMATION_GUIDE) | Pipelines, the DSL, batches, and schedules | -| [Analyzer guide](/docs/ANALYZER_GUIDE) | The 14 tools, tool YAML format, adding a tool | -| [Models reference](/docs/MODELS_REFERENCE) | Model catalog sync and API key resolution | -| [API reference](/docs/api-reference) | Auth, conventions, routers, exports, SSE | +| [User guide](/docs/USER_GUIDE) | The web UI page by page, and the workflows that connect the pages | +| [Development guide](/docs/development-guide) | just targets, devcontainer, tests, env vars, code layout, doc conventions | +| **Architecture** | | +| [Architecture](/docs/ARCHITECTURE) | How the pieces fit: apps, auth, where work runs, the Docker sandbox | +| [Generation process](/docs/GENERATION_PROCESS) | Agent and template modes, harnesses, run presets, grading, healing, identity keys, the job lifecycle | +| [Experiments](/docs/EXPERIMENTS) | Designed runs: conditions, comparability, launch, monitor, results, schedules, post-run, export | +| [Execution](/docs/EXECUTION) | The executor: leases, pools and lanes, fair-share ordering, blocked reasons, capacity settings, drain | +| [Analysis pipeline](/docs/ANALYSIS_PIPELINE) | Analyzer workspaces, tool execution, result statuses, the AI reviewer, measurement rules | +| [Background services](/docs/BACKGROUND_SERVICES) | Executor, web-process threads, Celery, and the SSE event stream | +| **Reference** | | +| [Analyzer guide](/docs/ANALYZER_GUIDE) | The 33 tools and the analyzer shop, tool YAML format, adding a tool | +| [Study tables](/docs/STUDY_TABLES) | Analysis plans, study report (Markdown/LaTeX), healing ablation, deduplicated findings, review sample, accounting | +| [Template packages](/docs/TEMPLATE_PACKAGES) | The package format, import/export rules, starter packages | +| [Local models](/docs/LOCAL_MODELS) | Endpoints for Ollama and OpenAI-compatible servers, direct connections and the relay | +| [Models reference](/docs/MODELS_REFERENCE) | Model catalog sync, enrichment, the model page, API key resolution | +| [Integrations](/docs/INTEGRATIONS) | MCP server, provider connections, GitHub export, webhooks | +| [API reference](/docs/api-reference) | Auth, conventions, routers, exports, relay, MCP, SSE | +| [Automation guide](/docs/AUTOMATION_GUIDE) | Where the retired automation hub went: experiment schedules and post-runs | +| **Operations** | | | [Deployment guide](/docs/deployment-guide) | The deploy script and the production Traefik stack | | [Troubleshooting](/docs/TROUBLESHOOTING) | Symptom-first fixes for common failures | ## Where things live ```text -backend/ Django apps, one per domain (generation, analysis, automation, …) +backend/ Django apps, one per domain (generation, execution, inference, analysis, …) config/ settings, API root, URLs, Celery app frontend/ SvelteKit app +relay/ relay script for connecting a model server behind NAT compose/ Dockerfiles for local and production stacks scripts/ bootstrap.py and deploy.sh -docs/ these pages — served live by the in-app viewer +docs/ these pages, served live by the in-app viewer ``` diff --git a/docs/STUDY_TABLES.md b/docs/STUDY_TABLES.md index 5a64e32e..7fa22e13 100644 --- a/docs/STUDY_TABLES.md +++ b/docs/STUDY_TABLES.md @@ -1,108 +1,150 @@ # Study tables -Tools for reporting a study's results: the effect of build healing, cross-tool deduplication of findings, a manual-review sample, and one way of counting tools and executions. Every table describes a *scope* of generation jobs (one experiment, one batch, or all of yours) and counts each job's latest finished analysis run only. +Reporting a study: build-healing ablation, deduplicated findings, manual-review sample, execution accounting, analysis plans and the study report. + +Every table describes a *scope* of generation jobs (one experiment, one batch, or every job you own) and counts one tool result per job and tool (the latest that measured the app, `ToolResult.latest_ids_per_job`), so re-analyses never double-count and re-running a single tool only replaces that tool's findings. In the API and the UI the scope is limited to your own jobs (staff see everyone's); the `manage.py` commands below see every user's jobs in scope. AI-reviewer findings are opinions, not measurements, and are left out unless asked for. How analyses run and what each tool result status means is in [Analysis pipeline](/docs/ANALYSIS_PIPELINE); experiments themselves are in [Experiments](/docs/EXPERIMENTS). ## Build healing Before a scaffolding or custom job's output is built, the platform can repair it. The setting is `build_healing` on the job, and the same field on an experiment is copied to every job at launch: -| mode | dependency inference | runtime patches | missing-file placeholders | +| Mode | Dependency inference | Runtime patches | Missing-file placeholders | | --- | --- | --- | --- | -| `full` (default) | requirements inferred from imports are appended to `requirements.txt` | sqlite path, port 8000, `app.app_context()`, unknown lucide icons | a missing entry or component is stubbed | +| `full` (default) | Requirements inferred from imports are appended to `requirements.txt` | Flask entry file: sqlite path, port 8000, `app.app_context()`; React components: unknown lucide icons | A missing entry file or frontend component is stubbed | | `no_deps` | — | yes | yes | -| `none` | — | — | a file the model never wrote is **removed**, including the skeleton's own stub | +| `none` | — | — | A file the model never wrote is **removed**, including the skeleton's own stub | Agent-mode output is always built verbatim, so it is never healed (`applicable: false`). -Every build records what it changed: `result_data.evaluation.build.healing` holds `{mode, applicable, changed, added_requirements, patches}`. The conformance outcome adds `metrics.functional.deployed` (the app built and answered its health check), `healing_mode` and `healing_changed`. +Every build records what it changed: `result_data.evaluation.build.healing` holds `{mode, applicable, changed, added_requirements, patches}` (patches as `:`). The conformance outcome adds `metrics.functional.deployed` (the app built and answered its health check), `healing_mode` and `healing_changed`. **Paired ablation.** Measuring healing by generating twice confounds it with sampling variance. Instead, re-evaluate the *same* output without healing: ```bash python manage.py healing_ablation --experiment --modes none,no_deps +# or: --batch , or --jobs ...; --force re-runs modes a job already has; --limit N ``` -Results land in `result_data.healing_ablation[mode]`; the job's own evaluation is not touched. The command builds on apps-dind one job at a time, so don't run it while agent runs are live. `GET /api/statistics/study/healing/?experiment_id=` returns the deploy rate per mode, overall and per model/template: +`--modes` defaults to `none`. Results land in `result_data.healing_ablation[mode]`; the job's own evaluation is not touched. The command builds on apps-dind one job at a time, so don't run it while agent runs are live ([Execution](/docs/EXECUTION)). Agent jobs are skipped. + +`GET /api/statistics/study/healing/?experiment_id=` (or `batch_id=`) returns the deploy rate per mode against the `full` baseline, overall (`totals`) and `by_model` / `by_template`. For every other mode ``: -- `rescued_vs_none`: apps that deploy only because they were healed, -- `broken_vs_none`: apps that healing broke, -- `healing_effect_pp_vs_none`: the net effect, in percentage points, -- `mcnemar_p_vs_none`: McNemar's exact test on the discordant pairs, +| Key | Meaning | +| --- | --- | +| `n_`, `deploy_rate_` | Apps evaluated under that mode, and the share that deployed (percent) | +| `paired_` | Apps evaluated under both `full` and `` | +| `rescued_vs_` | Apps that deploy only because they were healed | +| `broken_vs_` | Apps that healing broke | +| `healing_effect_pp_vs_` | The net effect, in percentage points | +| `mcnemar_p_vs_` | McNemar's exact test on the discordant pairs | -plus which packages were added and which patches fired. +The response also lists which packages were added (`added_requirements`), which patches fired (`patches`), one row per app (`per_app`), and `not_applicable` (agent builds). ## Deduplicated findings -Tools overlap: Ruff's `S` rules mirror Bandit, Semgrep covers much of Bandit, detect-secrets and Gitleaks both flag the same key, and Ruff, Pylint and Vulture all report the same unused import. Raw finding counts therefore count one defect several times. +Tools overlap: Ruff's `S` rules mirror Bandit, Semgrep covers much of Bandit, detect-secrets and Gitleaks both flag the same key, Ruff, flake8, Pylint and Vulture all report the same unused import, and pip-audit, osv-scanner and Trivy report the same vulnerable package under different advisory ids. Raw finding counts therefore count one defect several times. + +`statistics/services/dedup.py` maps each finding to an *issue class*, in this order: + +1. an `issue_key` the parser already set (DAST findings: `missing-header:
`, `cors-misconfiguration`); +2. for the vulnerability scanners (pip-audit, npm audit, osv-scanner, Trivy, retire.js), `vulnerable-dependency:`, matched within one project directory rather than one file, so `requirements.txt`, `package.json` and a lockfile meet; +3. an explicit rule equivalence (Bandit ids and Ruff's `S` mirrors, the shared Ruff/flake8/Pylint/Vulture/Mypy/ESLint lint rules, eslint-react `react/no-danger` and the eslint-security rules Semgrep also covers, keyed by the reporting tool's slug (`eslint-security:security/detect-eval-with-expression`), Semgrep rule-name keywords, every secret-scanner finding as `hardcoded-secret`); +4. the finding's CWE (Bandit, Semgrep and the DAST scanners record it); +5. otherwise its own `tool:rule`, which never merges across tools. + +Within one app, findings of the same class in the same file whose lines are within `line_tolerance` (default 2, Semgrep ranges included) form one issue; file-level findings merge only with other file-level ones. A single tool's findings merge only when they are exact duplicates (same rule, same line), so two separate secrets on adjacent lines stay two issues. An issue takes its members' highest severity. -`statistics/services/dedup.py` maps each finding to an *issue class*. The class comes from an explicit rule equivalence where one is known, otherwise from the finding's CWE (Bandit and Semgrep findings record it), otherwise from its own `tool:rule`, which never merges across tools. Within one app, findings of the same class in the same file whose lines are within `line_tolerance` (default 2, Semgrep ranges included) form one issue. A single tool's findings merge only when they are exact duplicates. +Only tools that measured every analyzed app in scope are counted (see [Measurement rules](/docs/ANALYSIS_PIPELINE)): a tool that failed, timed out or found the app not running on some apps is dropped, and the `coverage` block says which tools were dropped and why. A dependency audit that could not run because a dependency does not exist counts as `failed` here, even though its `nonexistent-package`/`nonexistent-version` findings are stored on the result; use `--all-tools` (`common_tools_only=false`) to count them. Results of cancelled runs are never counted. In a run reaped after a backend restart, tools that finished before the interruption keep their results, and the interrupted tool counts as `failed`. -`GET /api/statistics/study/breakdowns/?experiment_id=&include_ai=false&line_tolerance=2` returns: +`GET /api/statistics/study/breakdowns/?experiment_id=&batch_id=&include_ai=false&line_tolerance=2&common_tools_only=true&include_issues=false` returns: -| key | content | +| Key | Content | | --- | --- | -| `per_app` | raw findings, unique issues, multi-tool issues, unique issues by severity, issues per kLOC | -| `by_model`, `by_template`, `by_model_template` | the same, aggregated, with mean/median/sd per app | -| `by_tool` | raw findings, unique issues, issues only this tool found (`exclusive`), issues shared with other tools | -| `by_tool_rule` | which tool finds which kind of issue (rule → issue class, apps affected, severity) | -| `tool_pairs` | issues both tools reported, plus their agreement (Jaccard) on the issue classes both can detect | +| `params`, `coverage` | The settings used; `tools_common` and, per dropped tool, `apps_measured`, `apps_missing` and a count per reason | +| `totals` | The scope-wide summary, including `duplicate_findings` and `multi_tool_issues` | +| `per_app` | Job labels (model, template, mode, repeat, healing), source `loc`, `raw_findings`, `unique_issues`, `multi_tool_issues`, `security_unique`, `unique_`, `unique_per_kloc` | +| `by_model`, `by_template`, `by_model_template` | The same, aggregated, with mean, median and sd of unique issues per app | +| `by_tool` | Raw findings, unique issues, issues only this tool found (`exclusive`), issues shared with other tools (`shared`), apps with findings, raw counts by severity | +| `by_tool_rule` | Which tool finds which kind of issue: rule, issue class, CWE, example title, findings, apps affected, maximum severity | +| `tool_pairs` | Issues both tools reported (`both`, `only_a`, `only_b`) and their agreement (Jaccard) on the issue classes both reported somewhere | +| `issues` | Only with `include_issues=true`: every cluster with its tools, rule ids and finding ids | -AI-reviewer findings are left out unless `include_ai=true`. Pass `include_issues=true` to get every cluster with its finding ids. +AI-reviewer findings are left out unless `include_ai=true`; `common_tools_only=false` counts every tool. ## Manual review sample -To check the automated verdicts against a human, export a seeded, stratified sample: +To check the automated verdicts against a human, export a seeded, stratified sample of completed jobs that have code: ```bash python manage.py export_review_sample --experiment --size 30 --stratify model --seed 0 --out sample.zip -# or: GET /api/export/review-sample.zip?experiment_id=&size=30&stratify=model,template&seed=0 +# also: --batch , --per-stratum N (instead of proportional allocation), --stratify '' | model | template | model,template +# or: GET /api/export/review-sample.zip?experiment_id=&size=30&stratify=model,template&seed=0 ``` -The ZIP holds `apps/S001/code/…`, each app's `REQUIREMENTS.md`, and blank sheets in `labels/`: +Without `--per-stratum`, `--size` is allocated proportionally across strata (largest remainder, at least one per stratum). The ZIP holds a `README.md` with instructions, `apps/S001/code/…` and each app's `REQUIREMENTS.md`, and blank sheets in `labels/`: -- `ai_findings.csv`: TP/FP for each AI-reviewer finding, -- `static_issues.csv`: TP/FP for deduplicated static issues, up to 25 per app, -- `requirements.csv`: yes/partial/no for each requirement, -- `apps.csv`: works/partial/broken and a 1–5 quality score. +| Sheet | Rows | Label | +| --- | --- | --- | +| `ai_findings.csv` | Every AI-reviewer finding | `human_verdict`: TP, FP or unsure | +| `static_issues.csv` | Deduplicated static issues (all static tools), up to 25 per app drawn with the seed | `human_verdict`: TP, FP or unsure | +| `requirements.csv` | Each backend, frontend and admin requirement of the app's template | `human_met`: yes, partial or no | +| `apps.csv` | One per app | `human_functional`: works, partial or broken; `human_quality`: 1–5 | -Model, template and the automated outcomes are kept only in `key.csv` (blind by default; `--no-blind` turns that off). `manifest.json` records the seed, the strata and the population. +Model, template, healing mode and the automated outcomes (conformance status and pass rate, deployed, analysis run id) are kept only in `key.csv`; sample folders are `S001`… in shuffled order (blind by default; `--no-blind` or `blind=false` names them by job id). `manifest.json` records the seed, the sampling method, the strata and the population. Score the filled-in sheets: ```bash -python manage.py score_review_sample sample.zip --json scores.json +python manage.py score_review_sample sample.zip --json scores.json # a ZIP or an unpacked folder # or: POST /api/export/review-sample/score/ (multipart "file") ``` -The scores report the precision of AI and static findings (with 95% Wilson intervals, by severity, category, tool and issue class), Cohen's κ between the human functional verdict and the conformance outcome, and Spearman ρ between human requirement compliance and the conformance pass rate. +| Score | What it compares | +| --- | --- | +| `ai_findings`, `static_issues` | Precision of the findings with a 95% Wilson interval, overall and by severity and category (static issues also by tool and issue class) | +| `functional` | Human "works" vs conformance `passed`: agreement, Cohen's κ and the confusion counts (infra errors excluded) | +| `compliance` | Mean human requirement compliance (yes 1, partial 0.5, no 0) and its Spearman ρ with the conformance pass rate | +| `quality` | Mean of the 1–5 scores | ## Tool count and execution accounting -- `GET /api/statistics/study/tools/` lists every catalog analyzer (enabled or not) and the platform's own runtime evaluators (conformance tester, smoke test), with the counts a tool table needs: `catalog_registered`, `catalog_enabled`, deterministic vs AI, by category, and `total_enabled`. -- `GET /api/statistics/study/accounting/?experiment_id=` counts what actually ran, not tools × apps: - - apps generated, analyzed, evaluated and deployed, - - per tool: executions, completed/failed/skipped and apps covered (latest runs, and separately all runs including re-analyses), - - `planned_static` (tools the runs selected) next to the executed count, - - dynamic evaluations by status, and the healing-ablation evaluations. +- `GET /api/statistics/study/tools/` lists every catalog analyzer (enabled or not) and the platform's own runtime evaluators (conformance tester, smoke test), with the counts a tool table needs: `catalog_registered`, `catalog_enabled`, `catalog_enabled_deterministic`, `catalog_enabled_ai`, `platform_evaluators`, `enabled_by_category`, `enabled_by_kind`, and `total_enabled` (enabled analyzers plus evaluators: 35 with the shipped catalog of 33). The catalog itself is in the [Analyzer guide](/docs/ANALYZER_GUIDE). +- `GET /api/statistics/study/accounting/?experiment_id=` (or `batch_id=`) counts what actually ran, never tools × apps: + - `apps`: jobs by status, generated, analyzed, evaluated (infra errors excluded) and deployed; + - `analysis_runs`: latest and all runs, all runs by status; + - `tools`: per tool, executions, completed, failed, skipped, unfinished and apps covered, for the latest runs and separately for all runs including re-analyses; + - `evaluations`: conformance and smoke executions by status, and the healing-ablation evaluations per mode; + - `executions`: `planned_static` (tool selections of the latest runs) next to the executed `static` count (every analyzer execution, DAST scanners included), `static_completed`, `dynamic` (conformance plus smoke evaluations) and `total`. To export everything for a paper: ```bash -python manage.py export_study_tables --experiment --out results/ +python manage.py export_study_tables --out results/ [--experiment ] [--batch ] [--include-ai] [--line-tolerance 2] [--all-tools] ``` -This writes one CSV per table above, plus `study_summary.json` with the totals, the healing summary, the tool counts and the accounting. +| File | Content | +| --- | --- | +| `issues_per_app.csv`, `issues_by_model.csv`, `issues_by_template.csv`, `issues_by_model_template.csv` | Deduplicated issue tables | +| `issues_by_tool.csv`, `issues_by_tool_rule.csv`, `tool_overlap.csv`, `issue_clusters.csv` | Per-tool tables, tool pairs and every cluster | +| `healing_by_model.csv`, `healing_by_template.csv`, `healing_per_app.csv` | The healing ablation | +| `tool_inventory.csv`, `tool_executions.csv` | Catalog analyzers plus evaluators; per-tool executions of the latest runs | +| `study_summary.json` | Scope, issue params and totals, the healing summary, the tool counts and the full accounting | ## Pre-registered analysis plan -An experiment can carry an `analysis_plan` (create/PATCH body, draft only). It records: +An experiment can carry an `analysis_plan`, set in the experiment create or PATCH body while it is a draft (there is no plan editor in the UI yet; see the [API reference](/docs/api-reference)). The plan is validated and stored in canonical form: -- the research questions, -- the primary and secondary metrics, -- `alpha`, -- the correction (`holm`, `bh` or `none`), -- the ranking criteria and weights, -- the exclusion rules. +| Field | Values | +| --- | --- | +| `primary_metric` | Required; one of the metrics below | +| `secondary_metrics` | Metrics below | +| `research_questions` | Up to 20 `{id, question, hypothesis, metric, direction}`; `direction` is `two-sided` (default), `greater` or `less`; `metric` defaults to the primary one | +| `alpha` | Between 0 and 0.5, default 0.05 | +| `correction` | `holm` (default), `bh` or `none` | +| `ranking_criteria` | `[{key, direction, weight, label?}]`, keys from the metrics below, `direction` `benefit` or `cost`; weights are renormalized | +| `exclusions` | `{truncated: true}` by default; infra errors are always excluded | +| `min_apps_per_condition` | Default 0 (no minimum); planned comparisons skip a condition with fewer apps | +| `notes` | Recorded with the plan | Allowed metrics: `deploy_rate`, `pass_rate`, `functional_passed`, `unique_issues_per_app`, `issues_per_kloc`, `security_per_app`, `cost_per_app`. @@ -121,24 +163,34 @@ Allowed metrics: `deploy_rate`, `pass_rate`, `functional_passed`, `unique_issues } ``` -Launch freezes the plan. `analysis_plan_frozen_at` and `analysis_plan_hash` (sha256 of the canonical JSON) are set, and the plan cannot be edited afterwards; clone the experiment to change it. The study report treats every metric the plan declared as confirmatory and everything else as exploratory. It also uses the plan's alpha, correction and ranking weights. +Launch freezes the plan: `analysis_plan_frozen_at` and `analysis_plan_hash` (sha256 of the canonical JSON, sorted keys, no whitespace) are set, and the plan cannot be edited afterwards; clone the experiment to change it. The study report then uses the plan's alpha, correction and ranking weights, marks the planned comparisons and the per-model outcomes confirmatory, marks the ranking confirmatory when it uses the plan's criteria, and keeps everything else (correlations, code metrics) exploratory. + +The plan's `exclusions` decide the study report's sample: with a plan, `exclusions.truncated` replaces the report's `exclude_truncated` setting (a differing setting is ignored and flagged `plan_exclusions` in `validity`; `protocol.settings.exclusions_source` says which applied). Infra errors never count as evaluated. A condition (or the baseline) with fewer apps that have a value for the metric than `min_apps_per_condition` is listed in `planned_comparisons` with `skipped: "below_min_apps"` and no p-values, stays out of the multiple-comparison correction, and is flagged `below_min_apps`. ## Study report -Report type `study_report` (Reports → New → Study Report, or `manage.py study_report --experiment --out study.md`). Scope it to one experiment or batch, or leave it on all of your apps. Export it as Markdown, LaTeX (`longtable` + `booktabs`) or JSON with `GET /api/reports//export/?format=md|tex|json`, or with the buttons on the report page. +Report type `study_report`: Reports → New → **Study Report**, or `python manage.py study_report --experiment --out study.md`. In the UI, scope it to one experiment or leave it on all of your apps, and choose the seed, app-bootstrap rank intervals and whether AI-reviewer findings count. The report config also accepts `batch_id`, `line_tolerance`, `exclude_truncated` (default true) and `ranking_criteria` through `POST /api/reports/generate/`. + +The CLI takes `--format md|tex|json` (default: from the `--out` suffix), `--experiment`, `--batch`, `--user ` (without it, every user's jobs in scope; with only `--experiment`, the experiment's owner), `--seed`, `--include-ai`, `--no-bootstrap` and `--save` (also store a Report row so it appears in the UI). + +Export a stored report as Markdown, LaTeX (`longtable` + `booktabs`) or JSON with `GET /api/reports//export/?format=md|tex|json`, or with the buttons on the report page. | Section | What it answers | | --- | --- | -| `protocol` | What ran: models, templates, modes, stacks, prompt versions, cells by trial count, tool and evaluator versions, healing modes, seed, plan hash, and a fingerprint of the data | -| `definitions` | *Generated*, *deployed* and *functionally correct* are different outcomes; each is defined | -| `funnel` | generated → analyzed → evaluated → deployed → passes every check, per model and template, with Wilson 95% intervals; also correct given deployed | -| `issues` | Deduplicated issue tables (see above) | -| `security_bounds` | Per severity: issues, apps affected (with CI) and, for a zero count, the exact 95% upper bound on the per-app rate (zero in 20 apps is ≤ 13.9%, not 0) | -| `code_metrics` | LOC (backend/frontend), files, backend share and radon complexity per app, model and template; how much models differ on the same template (CV); app-level Spearman of size/complexity vs issues | -| `correlations` | Model parameters vs outcomes: Spearman ρ, exact (n ≤ 8) or 10k-draw permutation p, bootstrap CI, Holm and BH across the whole family, leave-one-out range with the most influential model, and a sign-flip flag. Nominal and effective (endpoint-limited) context windows are separate predictors. Always marked exploratory | -| `ranking` | TOPSIS and WSM under the plan's (or default) weights. Sensitivity: equal weights, ±50% one-at-a-time, Monte Carlo weights (near the chosen ones and arbitrary) with 95% rank intervals and P(top 3), and app-bootstrap rank intervals. LOC is not a criterion, so verbosity is not rewarded twice | -| `planned_comparisons` | Experiments with a plan only: each declared metric, each condition vs the pre-registered baseline, blocked on the app; sign and Wilcoxon tests adjusted as the plan says. Marked interim while the experiment runs | -| `healing`, `accounting`, `tool_inventory` | The healing ablation and what actually ran | -| `validity` | Threats found in *this* data: few models or apps, imbalance, truncation, missing evaluations, mixed prompts or modes, single stack, healing without an ablation, zero severe findings, tool failures, effective ≠ nominal context, unstable correlations, duplicate share, no plan | - -Every resampling step takes the report's `seed`, so a report regenerates identically. +| `protocol` | What ran: scope, experiment design (hypothesis, repeats, base seed, baseline, healing, plan and its hash), models, templates, modes, stacks, number of prompt versions, apps per model, trials per cell, tool and evaluator versions (per tool, `installed_versions` and `configs` as recorded by the counted results), `instrument_coverage` (the common-instrument coverage block), healing modes, report settings, and a fingerprint of the data | +| `definitions`, `metric_definitions` | *Generated*, *deployed* and *functionally correct* are different outcomes; each is defined, and so is every plan metric | +| `funnel` | generated -> analyzed -> evaluated -> deployed -> passes every check, per model and template, with Wilson 95% intervals; also correct given deployed | +| `model_outcomes` | The plan metrics per model | +| `issues` | The deduplicated issue tables (see above), without the cluster list | +| `security_bounds` | Per severity: issues, apps affected (with CI) and, for a zero count, the exact 95% upper bound on the per-app rate (zero in 20 apps is at most 13.9%, not 0); per model, critical/high counts and bounds | +| `code_metrics` | Source LOC (backend/frontend), files, backend share and radon complexity per app, model and template; how much models differ on the same template (CV); app-level Spearman of size and complexity vs issues and pass rate | +| `correlations` | Model parameters (nominal and effective context window, output limit, input and output price) vs deploy rate, pass rate, issues per kLOC and security issues per app: Spearman ρ, exact (n ≤ 8) or 10,000-draw permutation p, bootstrap CI, Holm and BH across the whole family, leave-one-out range with the most influential model, and a sign-flip flag. Always exploratory | +| `ranking` | TOPSIS and WSM under the plan's (or default) weights, with Kendall's τ between them. Sensitivity: equal weights, ±50% one-at-a-time with the rank reversals each causes, Monte Carlo weights (Dirichlet near the chosen weights and flat) with median rank, 95% rank intervals, P(rank 1) and P(top 3), and app-bootstrap rank intervals. LOC is not a criterion, so verbosity is not rewarded twice | +| `planned_comparisons` | Experiments with a plan and a baseline condition only: each declared metric (primary, secondary and research-question metrics; `role` `primary`, `secondary` or `research_question`, with the questions' ids and directions), each condition vs the baseline, paired on the app; sign and two-sided Wilcoxon tests, adjusted per metric as the plan says. Conditions below `min_apps_per_condition` are skipped. Marked interim while the experiment is not completed, cancelled or archived | +| `healing`, `accounting`, `tool_inventory` | The healing ablation, what actually ran, and the tool counts | +| `per_app` | One row per job with every per-app quantity above | +| `validity` | Threats found in *this* data, each with the sections it affects: few models or apps, unequal apps per model, truncation, infra errors, missing evaluations, mixed prompt versions or modes, a single stack, healing without an unhealed ablation, zero severe findings, AI findings excluded, tool failures, effective ≠ nominal context, correlations that flip sign, duplicate share, no plan, a report setting overridden by the plan's exclusions, conditions below the plan's minimum apps | + +Default ranking criteria, when neither the plan nor the report config sets them: deploy rate 0.30, pass rate 0.30, issues per kLOC 0.15, security issues per app 0.15 and cost per app 0.10 (the last three as costs). Every resampling step takes the report's `seed`, so a report regenerates identically. + +Experiment post-run automation generates an `experiment_report`, not a study report; see [Experiments](/docs/EXPERIMENTS). diff --git a/docs/TEMPLATE_PACKAGES.md b/docs/TEMPLATE_PACKAGES.md index bb9aeb79..97b40dda 100644 --- a/docs/TEMPLATE_PACKAGES.md +++ b/docs/TEMPLATE_PACKAGES.md @@ -1,6 +1,6 @@ # Template packages -A template package is a portable JSON or YAML file of Generation Library content: app specs, content blocks, generation profiles, user-authored stacks and run presets. Packages move that content between installs and users. Starter packages, shipped with the app, use the same format. In the UI it is Library → Packages; the code is `backend/generation/services/packages/`. +A template package is a portable JSON or YAML file of Generation Library content: app specs, content blocks, generation profiles, user-authored stacks and run presets. Packages move that content between installs and users. Starter packages, shipped with the app, use the same format. In the UI it is the **Packages** dialog of the Generation Library (`/templates`), with Starter packages, Export and Import tabs. The code is in `backend/generation/services/packages/`. ## The format (v3) @@ -47,8 +47,9 @@ Each section is a list; a missing section counts as empty. - Only assets you can see are exported (system rows and your own; for stacks, approved stacks and your own). - A profile exports at its latest version and brings along the blocks it references and the **user stack it targets**, so the package imports elsewhere without a dangling stack. Builtin stacks are never exported. - A referenced block you cannot see (another user's private block, or one that no longer exists) is left out, and the package lists it in a top-level `omitted_blocks: [{profile, type, slug, version, reason}]` (present only when something was left out; bundle packages carry it too). The profile keeps the reference, so importing it elsewhere needs that block to exist there. The Library's Packages dialog warns before the export. Creating or editing a profile refuses such refs (400 `Unknown block v`). -- Presets export at their latest visible version. -- An export with nothing in it is a 400. +- Presets export at their latest visible version. A block ref without a `version` exports version 1. Unknown slugs and refs are skipped silently. +- An export with nothing in it is a 400 (`Select at least one asset to export.`). +- The response is a file download (`generation-template-package.json` or `.yaml`). ## Import @@ -56,10 +57,12 @@ Each section is a list; a missing section counts as empty. The response names the rows the package resolved to, created or reused, per section: `{"app_specs": [slug], "blocks": ["slug:version"], "profiles": [slug], "stacks": [slug], "presets": ["kind/slug:version"]}`. Any problem is a `400 {detail}` and nothing is written, because the whole import is one transaction. -Assets import in dependency order: app specs → blocks → stacks → profiles → presets. Everything is validated as if it were created through the API: +Accepted schema versions are 1, 2 and 3; anything else is refused. Assets import in dependency order: app specs → blocks → stacks → profiles → presets. Each entry is validated: -- **Profiles.** `llm_config` follows the same rules as `POST /profiles/`. Every block ref must resolve to a block in the package or a block you can see. A ref without a `version` takes the package's own block of that slug, else the latest system version, else your latest own version. A ref without a `type` takes the block's type. A missing `scaffolding_slug` means `GENERATION_DEFAULT_STACK`. -- **Stacks** are re-validated against this install's base-image allowlist and file rules, always use a generated Dockerfile, and go through the approval queue like locally created ones. +- **App specs** must be valid against `data/requirements/schema.json` (name, category, description, at least one backend and frontend requirement and API endpoint, a data model with fields), except that the slug need not be snake_case (older specs and renamed imports use hyphens). `difficulty`, when present, must be `""`, `basic`, `standard` or `advanced`. App spec slugs are unique across all owners. `POST /app-specs/` checks the same schema, slug pattern included, and `PUT` checks it without the slug; both answer `400 {detail, errors}`. +- **Blocks** need a `slug`, `name` and `content`. A missing `block_type` defaults to `prompt_tone`, and a missing `version` to 1; any other `block_type` must be one of `ContentBlock.BlockType`. +- **Profiles.** `llm_config` follows the same rules as `POST /profiles/`. Every block ref must resolve to a block in the package or a block you can see. A ref without a `version` takes the package's own block of that slug, else the latest system version, else your latest own version. A ref without a `type` takes the block's type. A missing `scaffolding_slug` means `GENERATION_DEFAULT_STACK`; any other must be a known stack (or alias) after the remapping below, a stack in the same package included. +- **Stacks** are re-validated against this install's base-image allowlist (`STACK_ALLOWED_BASE_IMAGES`) and file rules, and always use a generated Dockerfile. An unknown `patch_profile` becomes `none`, and an unknown `server_kind` becomes `python`. They go through the approval queue like locally created ones: an import is approved at once when the importer is staff or `STACK_REQUIRE_APPROVAL` is off. - **Presets** are normalized and validated like `POST /presets/`. For example, a sampling preset may not set `provider`, `stop` or `response_format`, and options must be ones the harness offers. ### Conflicts @@ -68,9 +71,9 @@ One rule holds for every asset: content you can already see that is **identical* | Asset | `rename` | `overwrite` (your own item) | `error` | | --- | --- | --- | --- | -| App spec (one mutable row per slug) | new slug `slug-2` | updated in place | refused | -| Block (`slug` + `version`) | new slug, same version | **new version** of the slug; the package's profiles follow it | refused | -| Profile | new slug, version 1 | **new version** | refused | +| App spec (one mutable row per slug, slugs global) | new slug `slug-2` | updated in place | refused | +| Block (`slug` + `version`) | new slug `slug-2`, same version | **new version** of the slug; the package's profiles follow it | refused | +| Profile | new slug `slug-2`, version 1 | **new version** | refused | | Stack | new slug `slug-2` | renamed too (stack versions are immutable) | refused | | Preset | new slug `slug-2` | **new version**; a name/description difference alone updates the metadata | refused (`slug already in use`) | diff --git a/docs/TROUBLESHOOTING.md b/docs/TROUBLESHOOTING.md index fe792458..78a4c9ac 100644 --- a/docs/TROUBLESHOOTING.md +++ b/docs/TROUBLESHOOTING.md @@ -1,56 +1,122 @@ # Troubleshooting -Symptoms, causes, and fixes for the most common local and production failures. +Symptoms, causes and fixes for the most common local and production failures. -## "No OpenRouter API key is configured for this account" +Commands below assume the local stack through `just` (or `docker compose` with `COMPOSE_FILE=docker-compose.local.yml`). Subsystem pages have their own troubleshooting sections: [Local models](/docs/LOCAL_MODELS#10-troubleshooting), [Execution engine](/docs/EXECUTION), [Analyzer guide](/docs/ANALYZER_GUIDE). -Key resolution is per user: your own key from **Settings → API Credentials** wins; otherwise the global `OPENROUTER_API_KEY` applies only while `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=True`. Add a personal key, or set the global one in `.envs/.local/.django` and restart. Details in [Models reference](/docs/MODELS_REFERENCE). +## Setup and access -## A run is stuck in "running" after a restart +### `manage.py` fails with "Set the DATABASE_URL environment variable" -Interactive generation and analysis execute on threads inside the django container — restarting it kills in-flight runs, which stay `running` in the DB. There is no automatic retry; start a new run. (Automation pipeline runs go through Celery and retry.) +`DATABASE_URL` is assembled from the `POSTGRES_*` variables by the image's `/entrypoint`, which `docker compose exec` skips. Use `just manage ` (a one-off container that goes through the entrypoint), or `docker compose exec django /entrypoint python manage.py `. -## Analysis runs fail to start +### Port already in use -- The django container must reach the `apps-dind` sandbox daemon: check `just logs apps-dind` for `API listen on [::]:2376`, and that `docker compose exec apps-dind docker ps` lists an `analyzer-*` container once provisioned. Workspaces and sample apps live there, not in the host's `docker ps`. -- The first-ever run builds the `backend/analyzer-base:latest` image, which can take minutes or fail on network hiccups — `just logs django` shows the build output. -- Install tools **one at a time** from the Analyzers page; parallel installs can race workspace provisioning. +The local stack binds 8000 (frontend), 8001 (django), 8025 (mailpit) and 5555 (flower). Stop whatever holds the port, or override the mapping in a compose override file rather than editing `docker-compose.local.yml`. -## Automation schedules never fire +### No verification email arrives -Schedules only fire when something ticks the scheduler: the `celeryworker` **and** `celerybeat` services must be up (or run `just manage runautomationscheduler` as a substitute loop). Manual pipeline triggers still work without them. See [Background services](/docs/BACKGROUND_SERVICES). +Locally nothing sends real email: everything lands in mailpit at http://localhost:8025, and verification is optional (`ACCOUNT_EMAIL_VERIFICATION=optional`). If the link in the email points at the wrong host, set `FRONTEND_PUBLIC_ORIGIN` to the URL you actually open. In production, see the email gap in [Deployment guide](/docs/deployment-guide#path-b-production-compose). -## No verification email arrives +### 403, CSRF or CORS errors from a non-localhost address -Locally, nothing sends real email — everything lands in mailpit at http://localhost:8025. If the link inside the email points at the wrong host, fix `FRONTEND_PUBLIC_ORIGIN` to match the URL you actually open in the browser. +`DJANGO_ALLOWED_HOSTS`, `CORS_ALLOWED_ORIGINS` and `CSRF_TRUSTED_ORIGINS` in `.envs/.local/.django` must all include the origin you browse from; restart the stack after changing them. Production reads `DJANGO_CORS_ALLOWED_ORIGINS` and `DJANGO_CSRF_TRUSTED_ORIGINS` instead (note the prefix). -## 403 / CSRF / CORS errors from a non-localhost address +### Bearer token works everywhere except export URLs -All three of `DJANGO_ALLOWED_HOSTS`, `CORS_ALLOWED_ORIGINS`, and `CSRF_TRUSTED_ORIGINS` in `.envs/.local/.django` must include the origin you're browsing from. Restart the stack after changing them. +By design: `/api/export/*` endpoints check the session user only. Download exports from a logged-in browser session; see [API reference](/docs/api-reference). -## Bearer token works everywhere except export URLs +### Every API request hangs after a backend edit -By design: `/api/export/*` endpoints authenticate via session cookie only. Download exports from a logged-in browser session — see [API reference](/docs/api-reference). +`uvicorn --reload` can wait forever for open SSE streams before the old worker exits. The image's `/start` passes `--timeout-graceful-shutdown 5` to prevent this; images built before that change need `just build` and a recreate of `django`. As a stopgap, `docker compose restart django` (this kills running analyses). -## Port already in use +## Models and keys -The local stack binds 8000 (frontend), 8001 (django), 8025 (mailpit), and 5555 (flower). Stop whatever holds the port or adjust the mapping in `docker-compose.local.yml`. +### "No OpenRouter API key is configured for your account" -## Model list is empty +Key resolution is per user: your own key from **Settings → API keys** wins; otherwise the global `OPENROUTER_API_KEY` applies only while `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=True`. Add a personal key, or set the global one in `.envs/.local/.django` and restart. Details in [Models reference](/docs/MODELS_REFERENCE). -The catalog isn't seeded — it syncs from OpenRouter on demand. Open the Models page and sync (needs a resolvable API key for your user). +### Model list is empty, or sync returns 403 -## Generation UI offers almost nothing +The catalog is not seeded; it syncs from OpenRouter on demand. Sync is staff-only and always uses the staff user's **personal** key, never the global fallback, so add a key under Settings first. Model metadata enrichment runs afterwards on `celeryworker`; without a worker the models appear but lack models.dev / Hugging Face details (`just manage enrich_models` runs it inline). -Generation templates seed automatically after every migrate. If the UI still looks empty, check the django container logs for a "Generation template seeding failed" warning — malformed YAML/JSON under `backend/generation/data/` is caught and logged, not raised, so migrate still succeeds but seeding silently no-ops. Fix the data file and run `just manage seed_generation_templates` to retry. See [Quickstart](/docs/QUICKSTART). +### Runs on `:free` OpenRouter models never finish -## Production: a generated app's subdomain doesn't resolve +Free models are rate-limited upstream and often answer 429. Agent runs then make no progress until they hit their wall-clock budget. Use a paid model for anything you need to finish. -Check, in order: the DNS wildcard `*.APPS_DOMAIN` points at the host; Traefik obtained the wildcard cert (DNS-01 credentials in `.envs/.production/.traefik`); the route file exists — `python manage.py reconcile_app_routes` rewrites them. See [Deployment guide](/docs/deployment-guide). +## Generation and experiments + +### Jobs stay pending + +Generation jobs only run in the `executor` service. Check, in order: + +1. It is running: `docker compose ps executor`, `just logs executor`. +2. It is not drained: `just manage executor status` shows `"draining": false`; otherwise `just manage executor resume`. +3. It is not waiting for migrations: its log says "Waiting for N migration(s)" until django's `migrate` finishes. In the devcontainer, django's `/start` (and so `migrate`) only runs once an editor attaches. +4. Capacity is free: the capacity strip on Experiments shows the executor state (online, draining, offline, on old code) and used versus total slots per lane. Slots are also limited by the apps-dind memory rule ([Execution engine](/docs/EXECUTION#slots-and-the-memory-rule)). + +### The executor keeps restarting + +It exits with code 1 when another instance holds the leader lock, when the lock connection drops (for example a Postgres restart), when the LLM proxy cannot bind `LLM_PROXY_BIND`, or when migrations are still pending after 10 minutes. `just logs executor` names the reason. A bind failure usually means the `llmproxy` network or its fixed address is missing; recreate the executor with the same compose files the stack was started with. See [Execution engine](/docs/EXECUTION#restart-policy-and-exits). + +### The capacity strip says "Executor on old code" + +The executor does not autoreload. Drain it, wait for `safe_to_restart`, restart, resume ([Execution engine](/docs/EXECUTION#drain-and-restart)). + +### Local-model agent runs fail but OpenRouter runs work + +Agent sandboxes reach a local model through the executor's LLM proxy via apps-dind's `llmproxy` attachment and firewall exception. An `apps-dind` container created before that network existed lacks both; recreate it (`docker compose up -d --force-recreate apps-dind`), which stops every sample app running inside it. In the production compose stack there is no apps-dind, so these runs are refused by design (`LLM_LOCAL_AGENT_RUNS` is off). Scaffolding runs on local models work in both. + +### Generation library looks empty + +Library data seeds after every migrate. If the UI is still empty, look for "Generation template seeding failed" or "Run preset seeding failed" in `just logs django`: malformed YAML or JSON under `backend/generation/data/` is logged, not raised, so `migrate` still succeeds. Validate with `just manage lint_generation_data`, fix the file, then `just manage seed_generation_templates`. + +### Experiment schedules never fire + +Schedules fire inside the executor (checked every 30 s), not through Celery beat. They do not fire while the executor is drained or down, they skip a fire while the experiment still has active jobs (when "skip if active" is on), and missed fires are not replayed in a burst. Cron expressions must be at least 15 minutes apart. See [Experiments](/docs/EXPERIMENTS#automation). + +## Analysis and apps + +### An analysis run is stuck in "running" + +Interactive analysis runs execute on threads inside the django container. Restarting it, or a `uvicorn --reload` after a backend edit, kills them. A live run refreshes a heartbeat every 30 seconds, so within about 5 minutes of the restart the orphaned run is marked `failed` ("Interrupted: …") the next time runs are listed or opened, or when staff use **System → Maintenance → Clear Stuck Analysis Tasks**. Start a new run. Cancelling stops a live run before its next tool and kills the tool in flight. Post-run analyses of experiments run in the executor and are re-claimed after a restart. + +### Analysis runs fail to start + +- Django must reach the `apps-dind` daemon: `just logs apps-dind` should show `API listen on [::]:2376`, and `docker compose exec apps-dind docker ps` should list an `analyzer-*` container once provisioned. Workspaces and sample apps live there, not in the host's `docker ps`. +- The first run builds the `backend/analyzer-base:latest` image, which can take minutes or fail on network hiccups; `just logs django` shows the build. +- Install tools one at a time from the Analyzers page; concurrent installs into a workspace that is still provisioning can race. +- Dynamic tools need the app's container running; start it first. See [Analyzer guide](/docs/ANALYZER_GUIDE). + +### A generated app shows a blank page under `/apps//` + +Path-proxy mode (the local default, `APPS_PROXY_PATH=True`) serves the app under a prefix, but most generated SPAs load root-absolute `/assets/...` and call `/api/...`, which hit the platform instead. The app itself is usually fine. Switch to subdomain routing ([Deployment guide](/docs/deployment-guide#sample-app-routing)) to view it properly. + +### apps-dind runs out of memory or disk + +apps-dind is capped at `APPS_DIND_MEM_LIMIT` (4 GB) and hosts sample apps, agent sandboxes (3 GB each by default) and analyzer workspaces together. Stop apps you no longer need before agent runs. It prunes unused images older than a week every hour; to reclaim more, `docker compose exec apps-dind docker system prune`. + +### Webhooks are not delivered + +Deliveries run on `celeryworker`; check `just logs celeryworker` and that the subscription is enabled. Deliveries that could not even be queued are re-dispatched by the beat sweep, so `celerybeat` must run too. See [Integrations](/docs/INTEGRATIONS). + +## Production + +### Production containers start with debug settings or crash on `debug_toolbar` + +`docker-compose.production.yml` sets `DJANGO_SETTINGS_MODULE=config.settings.production` on every Django-image service; without it `config/asgi.py`, `manage.py` and the Celery app fall back to `config.settings.local`, which imports `debug_toolbar` (not in the production image). If you run the image some other way, or a service overrides `environment:`, set it there. `DJANGO_SECRET_KEY` is the only other required variable; `SENTRY_DSN` is optional. + +### The site returns 502 or 404 behind Traefik + +Traefik routes the frontend to `frontend:3000`, so `PORT` in `.envs/.production/.frontend` must stay `3000`. The admin is routed at whatever `DJANGO_ADMIN_URL` is (Traefik reads it from `.django` at start), so restart Traefik after changing it. + +### A generated app's subdomain does not resolve + +Check, in order: the DNS wildcard `*.APPS_DOMAIN` points at the host; Traefik obtained the wildcard certificate (DNS-01 credentials in `.envs/.production/.traefik`, `TRAEFIK_DNS_PROVIDER` set); the route file exists (`python manage.py reconcile_app_routes` rewrites them). See [Deployment guide](/docs/deployment-guide). ## Start over ```bash -just prune # removes containers AND volumes — deletes the database +just prune # removes containers AND volumes: database, apps-dind images and certs just bootstrap && just build && just up ``` diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md new file mode 100644 index 00000000..e9f08ff2 --- /dev/null +++ b/docs/USER_GUIDE.md @@ -0,0 +1,449 @@ +# User guide + +A page-by-page tour of the LLM Lab web UI and the workflows that connect the pages. + +For installing and starting the stack, see [Quickstart](/docs/QUICKSTART). This guide starts once you can sign in. + +## Layout and navigation + +Every signed-in page has a left sidebar, a header with breadcrumbs, a theme toggle and an account menu. + +The sidebar groups pages in workflow order. The breadcrumb's first, unlinked crumb is the group name. + +| Group | Pages | Purpose | +| --- | --- | --- | +| Overview | Dashboard | Activity across the lab at a glance | +| Generate | Models, Endpoints, Library, Sample Generator, Experiments | Choose models and where they run, define what to build, start runs | +| Evaluate | Applications, Analysis, Analyzers, Runtime | Inspect generated apps, run analyzers, manage containers | +| Insights | Rankings, Statistics, Reports | Compare models and export results | +| Help | Docs | This documentation | +| Admin (staff only) | System | Host, worker and database health, maintenance | + +- **Collapse** at the bottom of the sidebar shrinks it to icons. On phones the sidebar opens from the menu button in the header. +- The account menu (your initial, top right) has **View profile**, **Account settings**, **Appearance**, **API keys**, **Integrations** and **Sign out**. +- **About** (`/about`) and **Privacy & data** (`/privacy`) are not in the sidebar; the footer at the bottom of every page links to **About**, **Privacy** and **Docs**. The privacy page is also linked from Settings, under **Privacy & data**. It lists the cookies and browser storage the app uses, what the server keeps and what leaves the instance (LLM providers, GitHub exports, notification channels); account deletion or a full server-side export goes through the instance administrator. + +### Old links that redirect + +| Old address | Lands on | +| --- | --- | +| `/dashboard` | Dashboard (`/`) | +| `/generation` | Sample Generator, Agent mode (the query string is kept) | +| `/login` | `/auth/login` | +| `/settings` | Settings (`/users/settings`) | +| `/integrations?tab=…` | The matching Settings section, for example `#github` or `#notifications` | +| `/automation` and anything under it | Experiments. Automation now lives on each experiment's **Operations** section | +| `/experiments/create` | `/experiments/new` (query kept) | +| `/sample-generator?job=` | That run's application page | + +## Typical workflows + +```mermaid +flowchart LR + K[Settings: API keys] --> M[Models: Sync] + M --> G[Sample Generator] + E[Endpoints: local model] --> G + G --> A[Applications] + A --> R[Container: Build / Start] + A --> N[Analysis: New analysis] + G --> X[Experiments] + X --> RS[Results tab] + N --> I[Rankings / Statistics] + RS --> P[Reports: Study Report] +``` + +### First app: keys, models, generate, run, analyze, compare + +1. **Add your keys.** Open Settings → **API keys** and click **Add key** for OpenRouter (and optionally Hugging Face, for model metadata). Keys are checked with the provider, stored encrypted and never shown again. Until you add one, jobs use the deployment-wide key if there is one. The Sample Generator shows a banner when no usable OpenRouter key exists. +2. **Sync the model catalog.** On **Models**, click **Sync**. Staff also find **Sync models** in the Dashboard's maintenance menu. +3. **Generate an app.** Open **Sample Generator**, keep the **Agent** mode, pick one or more app specs under **Task**, pick models, keep the default harness preset and click **Start N runs**. Every combination of task, model, harness and trial becomes one run in a batch. +4. **Follow the batch.** You land on the batch page. Its **Results** matrix fills in as runs finish. Open a run to get its application page. +5. **Run the app.** On the application page, click **Build**, then **Open** to preview it. **Container** shows ports and status, and **Tools** sends HTTP requests to the running app. +6. **Analyze it.** Install analyzers once on **Analyzers** → **Shop**. Then click **New Analysis** on the application's **Analyses** tab, or go to **Analysis** → **New analysis**. +7. **Compare.** Select models on **Models** and click **Compare**. You can also check **Rankings** and **Statistics**, or generate a report. + +More detail: [Generation process](/docs/GENERATION_PROCESS), [Execution](/docs/EXECUTION), [Analysis pipeline](/docs/ANALYSIS_PIPELINE). + +### Designing and running an experiment + +1. **Experiments** → **New experiment**. Work through the numbered sections: **Basics**, **Method**, **Models**, **Harness presets** (or **Prompt profiles** for scaffolding), **Apps**, **Replication**, **Sampling & budget**, **Ablations**, **Execution**, **Automation**. +2. Watch the **Plan** rail on the right. It shows conditions, cells, where they run, cost, bottleneck and a **Comparability** table. Problems that block launch are listed there. +3. Pick the **Pre-registered baseline** in the **Harness presets** (or **Prompt profiles**) section. Results compare every condition against it. +4. Click **Save draft** to keep the design, or **Launch** once the plan is ready. +5. The experiment page reads like a paper. **Abstract** summarises the design, the progress of data collection and the key findings (with limitations), and plots the pass rate per condition. **Method** lists the factors, conditions, what is held constant and the comparability check. **Results** has the leaderboard, the comparison with the baseline (forest plot) and the per-app table. **Data** lists every run (open or retry one) and downloads the dataset. **Operations** shows endpoints, activity, execution limits and automation. +6. When it finishes, you can **Top up failed** cells, **Clone** the design, **Archive** it, or click **Generate report** in the Results section. + +More detail: [Experiments](/docs/EXPERIMENTS). + +### Adding a local model endpoint + +1. **Endpoints** → **Add endpoint**. The wizard has three steps: **Server**, **Settings**, **Connect**. +2. **Server**: choose the **Server kind** (**Ollama** or **OpenAI-compatible**) and the **Connection** (**Relay**, recommended, or **Direct**, staff only). +3. **Settings**: name, slug (it cannot be changed later), capacity (**Max active jobs**, **Max resident models**, **Request timeout (s)**, **num_ctx cap**, **Keep-alive**), tool calling, and sharing (**Private** or **Shared**; only staff can set **Shared**). +4. **Connect**: for a relay, copy the relay token from the dialog (it is shown once). Run the install commands on the model machine and wait for "Waiting for the relay to connect…" to turn green. For a direct endpoint, use **Test connection**. +5. Open the endpoint and go to **Models**. Select discovered models and click **Use N in new experiment**. They also appear in the Sample Generator's model picker. + +To add a cloud provider instead (Anthropic, OpenAI, Groq, Mistral and others), use Settings → **LLM providers**. LLM Lab then creates an endpoint and imports its models. + +More detail: [Local models](/docs/LOCAL_MODELS). + +### Building a study report + +1. Run an experiment to completion with **Evaluate every run** on. Optionally turn on **Static analysis when the experiment completes** under its **Operations → Automation** section. +2. **Reports** → **New report** → **Study Report**. +3. Optionally choose an **Experiment** (default **All my apps**). With an experiment, its pre-registered analysis plan decides which comparisons are confirmatory. +4. Set a **Random seed** so the report regenerates identically. Optionally tick **Bootstrap rank intervals over apps (slower)** and **Count AI-reviewer findings as issues**. +5. Click **Generate report**. When it completes, download it as **Markdown**, **LaTeX** or **JSON**. + +More detail: [Study tables](/docs/STUDY_TABLES). + +## Overview + +### Dashboard (`/`) + +Shows lab-wide activity. + +- **Header stats**: models in use, apps, analyses and findings. Each one links to its page. +- **Buttons**: **Generate app**, **New analysis**, **Refresh dashboard**, and, for staff only, a **Maintenance** menu with **Sync models**, **Clear caches**, **Clear stuck jobs** and **Open system panel**. +- **Panels**: + - **Analysis Activity** (trend) and **Finding Severity**. + - **Analyzer Health**: one tile per analyzer. + - **Recent Analyses** and **Recent Generation Jobs**, each with **View all**. + - **Recent Activity** feed. + +## Generate + +### Models (`/models`) + +The catalog of models you can generate with. + +- **Filters**: + - **Quick** filters: **Free**, **Vision**, **Functions**, **128K+ Context**, **Efficient**, **Used in Apps**. + - Capability, **Price**, **Context** and **Provider** filters, and **Sort**. +- **Actions**: **Sync** refreshes the catalog from OpenRouter, **Export** downloads it, **Import** opens the import page, **Rankings** opens Rankings. +- **Compare**: tick models in the table, then click **Compare (N)**. + +**Model detail** (`/models/`) has these tabs: + +- **Overview**: about, links to OpenRouter, models.dev and Hugging Face, and how this model has done on this platform. +- **Results**: ranking, resource use, code findings by tool, results per app spec, and the experiments it was part of. +- **Benchmarks** and **Capabilities**. +- **Pricing & providers**. +- **Applications**: apps generated with this model, and usage. +- **Raw data**: **Copy JSON** and **Export**. + +The header has **Copy model ID**, a link to OpenRouter, export as JSON, **Refresh metadata from OpenRouter** (staff only), **Compare** and **Generate app**. A notice appears when OpenRouter plans to retire the model. + +**Compare** (`/models/compare`) puts the selected models side by side. + +- **Sections**: **Core Metrics**, **Pricing Deltas** and **Capabilities Comparison**. +- **Baseline**: **First Selected**, **Average** or **Median**. +- **Other controls**: **Highlight** marks the best value in each row, **Share Link** copies a link to this comparison. + +**Import** (`/models/import`) uploads a model JSON file. It accepts this app's own model export or a raw OpenRouter model payload. + +Reference: [Models reference](/docs/MODELS_REFERENCE). + +### Endpoints (`/endpoints`) + +Shows where models run: OpenRouter, plus your own Ollama or OpenAI-compatible servers. + +- **Capacity card**: the shared execution slots. Staff also see **Drain** and **Slot settings**. +- **Endpoint cards**: active jobs, queued requests, tokens/s and loaded models, with **Open** for details. +- **Add endpoint**: opens the three-step wizard described in [Adding a local model endpoint](#adding-a-local-model-endpoint). + +**Endpoint detail** (`/endpoints/`) has these tabs. Which ones you see depends on the endpoint type and on whether you can edit it. + +- **Overview**: live numbers (active jobs, queued requests, tokens/s), **Loaded models** with **Unload**, **Relay sessions** and **Recent errors**. +- **Models**: the discovered models. Use **Refresh models**, **Show hidden**, **Test** or **Unload** a model, hide a model from pickers, override num_ctx per model, and **Use N in new experiment**. +- **Tokens** (relay endpoints only): relay tokens with **New token**, **Rotate** (the old token keeps working for a grace period) and **Revoke**. **Install the relay** shows the setup commands. +- **Settings**: edit the endpoint, and **Disable or delete**. An endpoint that has runs cannot be deleted, only disabled. + +Reference: [Local models](/docs/LOCAL_MODELS). + +### Library (`/templates`) + +The Generation Library holds everything a sample run is assembled from. A side menu groups its sections. + +| Group | Section | What it holds | +| --- | --- | --- | +| What to build | **App specs** | Requirements, data model and API contract. Agent runs on a spec are graded by a hidden conformance suite. | +| Where it runs | **Stacks** | Project skeletons (Dockerfile, port, entry point). Stacks you create need an admin's approval before others can run them. | +| How agents run | **Harness**, **Sampling**, **Budget** | Versioned presets. Changing one saves a new version, and every job records the version it used. **Show archived** lists older ones. | +| Template prompts | **Profiles**, **Blocks** | Used only by template scaffolding. Profiles are ordered prompt blocks for one stack. Blocks are versioned prompt text. | + +- Each section has a search box and a **New …** button (for example **New app spec**, **New stack**, **New profile**, **New block**). +- Catalog app specs are read-only for non-staff users. Use **Duplicate to edit** to make your own copy. +- **Packages** (header button) has three tabs: + - **Starter packages**. + - **Export**: bundles app specs, profiles, blocks, user stacks and presets into one file. + - **Import**: accepts a package as JSON or YAML. + +Reference: [Template packages](/docs/TEMPLATE_PACKAGES). + +### Sample Generator (`/sample-generator`) + +Starts generation runs. The **Generation mode** switch has three options. + +| Mode | What it does | +| --- | --- | +| **Agent** (Recommended) | An agent harness builds each app in a sandbox. Runs on app specs are graded by hidden tests. | +| **Template scaffolding** | A prompt profile fills a stack template in one pass, per app spec and model. | +| **Single prompt** | One raw system and user prompt sent to one model. Nothing is added and nothing is graded. | + +- **Agent** sections: + - **Task**: **App specs**, or **Free-form task (not graded)**, plus optional **Additional context**. + - **Models** and **Harnesses**. + - **Model settings**: sampling settings, with **Overrides for this batch** and **Save as preset…**. + - **Budget & repeats**: **Trials per cell** and **Base seed**. +- **Template scaffolding** sections: **Task & prompts** (app specs, **Stack**, **Profile**), **Models**, **Model settings** and **Repeats**. +- **Single prompt** sections: **Prompt** (**System prompt**, **User prompt**), **Model** and **Model settings** (with an optional **Seed**). Start it with **Start generation**. +- **Run summary** (right-hand panel) resolves every run before anything starts: models, harness presets, sampling, budget per run, per-model parameter support and **Max cost**. +- **Header buttons**: **Batches** and **Library**. The **?** help explains the terms App spec, Harness, Preset and Stack. + +**Batches** (`/sample-generator/batches`) lists every generation batch with **Status**, **Progress**, **Design**, **Pass rate**, **Cost** and **Created**. You can filter by **Mode** and **Source**. + +**Batch detail** (`/sample-generator/batches/`): + +- **Results**: a matrix of task × condition, with a dot per trial. It warns when trials ran under different configurations. +- **Runs**: every run in trial order. +- **Actions**: **Cancel**, **New batch like this** (same grid, new seed) and **Open experiment** (when the batch belongs to one). + +Reference: [Generation process](/docs/GENERATION_PROCESS). + +### Experiments (`/experiments`) + +Designed model comparisons, run in parallel across your endpoints. + +- **List**: search, filter by **Status**, **Method** and **Endpoint**. +- **Header buttons**: **Endpoints** and **New experiment**. + +**New experiment** (`/experiments/new`) is the composer described in [Designing and running an experiment](#designing-and-running-an-experiment). + +- **Section notes**: + - **Method**: generation method and stack. + - **Replication**: **Repeats per cell** and **Base seed**. + - **Ablations**: **Add ablation**, one model with deliberately different sampling. + - **Execution**: **Run order** (**Auto**, **Interleaved**, **Per repeat**, **Grouped**), **Priority** (**Low**, **Normal**, **High**; High is staff only), **Max concurrent runs**, **Infra retries**, **Max resident models** and **Preload Ollama models before their runs**. + - **Automation**: **After the run** settings and a **Schedule**. +- **Buttons**: **Start over** resets the draft. Opening the composer from **Clone** or **Edit** pre-fills it, and **Edit** shows **Save changes**. + +**Experiment detail** (`/experiments/`): + +- **Header actions** depend on status: + - Draft: **Edit**, **Delete**, **Launch**. + - Running: **Pause**, **Cancel**. + - Paused: **Resume**, **Cancel**. + - Completed or cancelled: **Archive**, **Clone**, **Top up failed**. + - Any status: export as **JSON** or **CSV**. +- **Live strip**: done, in flight and failed cells, cells/h, ETA, tok/s, cost and bottleneck. +- **Overview**: endpoint lanes, a cell grid (one square per run), and **Activity**. +- **Results**: + - Choose a **Baseline**. Picking one other than the pre-registered baseline is labelled exploratory. + - Toggles: **Common cells only**, **Exclude invalid**, **Exclude truncated**. + - Views: **Leaderboard**, **Against the baseline**, **Per app**. + - **Generate report**. +- **Runs**: every run with filters (**Current attempts only**, **Retried only**), plus **Launch history**. +- **Design**: the frozen design, conditions grouped by endpoint, **Comparability** and **Execution**. +- **Automation**: **Post-run** (analysis and report after all cells finish, with history) and **Schedules** (**Add schedule** with the actions **Top up failed cells**, **Add replicates** or **Clone and run** on a cron expression). + +> [!NOTE] +> If a model's weights change on its endpoint after launch, the experiment pauses. You can choose **Clone with current digest** or **Resume anyway**. + +Reference: [Experiments](/docs/EXPERIMENTS), [Execution](/docs/EXECUTION). + +## Evaluate + +### Applications (`/applications`) + +Every generated application (generation job). + +- **Filters**: + - **Status**. + - **Mode**: **Agent**, **Custom**, **Scaffolding**, **Copilot (legacy)**. + - **Container**: **Running**, **Stopped**, **Building**, **None**. + - **Model**, **Sort**, and page size. +- **Header buttons**: **Refresh**, **Batches** and **Generate New**. + +**Application detail** (`/applications/`): + +- **Header actions**: **Cancel**, **Retry**, **Re-generate** (a new batch with this run's settings), download code, export the run as JSON, **Export to GitHub**, delete, **Copy job ID**. +- **Container controls**: **Build**, **Open**, **Start**, **Stop**, **Restart**, **Rebuild**, **Remove**. +- **Tabs**: + - **Overview**. + - **Agent Run** (agent jobs): wall time, calls, cost, platform evaluation with **Evaluate** or **Re-evaluate**, validity notes and reproducibility data. + - **Prompts**. + - **Files & Code**: file explorer and key artifacts. + - **Container**: status, image and ports. + - **Tools**: **API Tester**, **Quick Commands**, **Environment**. + - **Analyses**: this app's runs, with **New Analysis**. + - **Logs**. + - **API Scan** (scaffolding jobs with a backend scan). + - **Dependencies**. + - **Artifacts**: stored LLM requests and responses. + - **Copilot Iterations** (legacy copilot jobs). + - **Cost & Metrics**. + +A failed job links to **Failure Details** (`/applications//failure`). It shows the error message, generation errors, build output, request artifacts, a timeline, and **Re-generate**. + +Reference: [Execution](/docs/EXECUTION). + +### Analysis (`/analysis`) + +Runs your analyzer workspace's tools on generated apps and shows their results live. + +- **List**: every run with what it analyzed (app · model · stack), **Status** (with a progress bar and the running tool while active), **Findings** (total, plus the levels the tools reported), **Tools** (measured / total), **Duration** and **Created**. Filter by status (counts on each chip), search by name, app, model, stack or ID, and sort by findings, duration or date. Select rows to cancel or delete them in bulk; re-run a finished run from its row. Links scope the list to a batch (`?batch=`), an app (`?job=`) or a tool (`?tool=`). Active runs update live. +- **Header buttons**: **Analyzers** and **New analysis**. + +**New analysis** (`/analysis/create`) is one page: + +1. **What to analyze**: **Generated apps** (search, an **App running** filter, multi-select; each app gets its own run, queued one after another) or **Paste code** as named files (names decide which tools apply). +2. **Tools**: installed tools by category, with quick sets (**All installed**, **Static only**, **Security & deps**). Each selected tool says where it would run ("Runs on all 3 apps", "Nothing to read — would be skipped", "App not running"). Tools that aren't installed are listed but disabled. The selection is remembered. +3. **Name**: optional. + +The **Run summary** shows apps × tools = tool runs, how many would be skipped, files and lines to scan, and warnings. Linking `/analysis/create?generation_job_id=` preselects an app. + +**Run detail** (`/analysis/`): + +- While running: a progress strip (one segment per tool, the running tool and its clock, findings so far, live-connection state) and **Cancel run**. +- **Overview**: findings, tools measured, code size and duration; what needs attention (failed tools, dynamic tools skipped because the app was down, truncated generated code); findings by the level each tool reported and by tool; most affected files and most frequent rules (each opens the filtered findings); metrics of complexity and duplication tools. +- **Findings**: filters by tool, level as reported, category, file and rule, plus search and sort. A row expands into everything the tool said: description, its suggested fix, code, location, rule documentation, CWE / OWASP, references, the vulnerable package and fixed versions, the HTTP request a scanner flagged, duplicates. **CSV** / **JSON** export the filtered findings. +- **Tools**: each tool in execution order with state, time, version, findings, warnings and errors; **Output** shows the command, exit code and raw output. +- **Re-run**: all tools, or only those that measured nothing. + +Levels are shown only as the tools printed them. A tool that rates nothing (ruff, flake8, pip-audit, …) gets no level; the normalized severity used by statistics is never displayed here. + +Reference: [Analysis pipeline](/docs/ANALYSIS_PIPELINE). + +### Analyzers (`/analyzers`) + +Choose the tools that measure your apps. Their findings feed analysis runs, experiments, reports and rankings. + +- **Coverage panel** ("What your generated apps contain"): technologies in your apps, and which ones no tool measures. +- **Shop**: the full catalog. + - Filters: category, technology, **Static** or **Dynamic** mode, install state (**Installed**, **Not installed**, **Updates**), and fit to your apps. + - Actions: **Install** a tool, or **Install N shown** for everything in the current view. +- **Installed**: your tools. + - **Configure & test** opens the tool's options, **Test** runs it on a sample, **Save** keeps the options. + - Also **Log**, **Update** or **Update all**, and **Uninstall**. + - Dynamic tools need the app's container running when the analysis starts. +- **Workspace**: your private analyzer container, with **Stop**, **Repair tools** and **Tear down**. + +Reference: [Analyzer guide](/docs/ANALYZER_GUIDE). + +### Runtime (`/runtime`) + +Lists the Docker containers of your generation jobs (staff see every user's). A dot shows whether the Docker daemon is online. + +- **List**: filter by status (**Pending**, **Building**, **Running**, **Stopped**, **Failed**, **Removed**). Each row has **Start**, **Stop**, **Restart** and **Remove**. **Remove** keeps the generated code. +- **Container detail** (`/runtime/`): the same actions, plus **Logs** and **Actions** (lifecycle history) tabs. + +Reference: [Execution](/docs/EXECUTION). + +## Insights + +### Rankings (`/rankings`) + +Puts empirical quality measured on this platform next to the metadata-based **MSS** decision aid. + +- **Columns**: **Rank**, **Model**, **Measured** (empirical score with its n), **MSS decision aid**, **Public benchmarks** and **Apps**. +- **Ranked by**: **MSS**, **Empirical**, **Composite**, **Benchmark**, **Cost**, **Accessibility** or **Adoption**. +- **Filters**: **Provider**, **Paid only** and **Has Benchmarks**. +- **Actions**: select up to 10 models and click **Compare**. **Export CSV** downloads the table. +- **Methodology** explains what is measured and what is opinion. **Sensitivity** shows how stable the ranking is under alternative severity weights (Kendall's τ). + +### Statistics (`/statistics`) + +Platform-wide analytics. + +- **Analysis Trends** and **Severity Distribution**. +- **Model Comparison**: apps, success %, smoke pass, empirical score, MSS and composite. +- **Tool Effectiveness**: scans, findings, average per scan and top rule. +- **Top Findings**, **Code Generation Stats** and **Recent Activity**. + +### Reports (`/reports`) + +Lists generated reports, with filters for **Type** and **Status**, **Sort**, and **Export** (**Reports CSV** or **Reports JSON**). + +**New report** (`/reports/create`): pick a type, configure it, then click **Generate report**. + +| Type | Configure | +| --- | --- | +| **Model Analysis** | Model | +| **Template Comparison** | Template | +| **Tool Analysis** | Tool name (blank aggregates all tools) | +| **Generation Analytics** / **Comprehensive** | Window (days) | +| **Experiment Report** | Experiment: leaderboard and paired comparisons against its baseline | +| **Study Report** | Optional experiment, random seed, bootstrap and AI-finding options. Produces paper-ready tables. | + +**Report detail** (`/reports/`) shows a **Summary**, **Details** and one card per data section. A completed report downloads as **Markdown**, **LaTeX** or **JSON**. + +Reference: [Study tables](/docs/STUDY_TABLES). + +## Help + +### Docs (`/docs`) + +The in-app viewer for these pages. The landing page groups documents by category and has a **Start here** link. + +- **Find a page**: press ⌘K (Ctrl+K) or use **Search docs…** to search every page. The sidebar has **Overview**, **Browse** and **Search**. +- **On a page**: **On this page** lists the headings, and **Copy link** copies a permalink. + +The REST API is documented in the [API reference](/docs/api-reference). + +## Admin + +### System (`/system`), staff only + +The System Monitor refreshes every 10 seconds. Other users see "Staff access required". + +- **Health panels**: + - **Host Metrics**: CPU, memory, uptime, load and disk. + - **Containers**. + - **Redis**. + - **Celery**: workers, active, scheduled and reserved tasks, and queue lengths. + - **Database**: connections, commits, deadlocks and the top tables by size. +- **Application Stats**: counts by status and in the last 24 hours. +- **Maintenance Actions**: **Clear Stuck Analysis Tasks**, **Clear Stuck Generation Jobs**, **Purge Orphan Containers** and **Clear All Caches**. Each one asks for confirmation. + +## Account + +### Settings (`/users/settings`) + +One page with a section menu and search. Each section has a deep link, `/users/settings#`. + +| Group | Section | What you do there | +| --- | --- | --- | +| Account | **Profile** (`#account`) | Display name. Email is read-only. | +| Account | **Security** (`#security`) | Change password | +| Preferences | **Appearance** (`#appearance`) | Theme and avatar color | +| Preferences | **Workspace** (`#workspace`) | **Collapsed sidebar**, **Compact tables** | +| Preferences | **Privacy & data** (`#privacy`) | What this browser stores. **Export**, **Import** or **Reset** your preferences. | +| Connections | **API keys** (`#credentials`) | Personal OpenRouter and Hugging Face keys | +| Connections | **LLM providers** (`#providers`) | Connect a cloud provider with a key. This creates an endpoint and imports its models. | +| Connections | **GitHub** (`#github`) | Personal access token for **Export to GitHub** (repository or Gist, optional SARIF upload) | +| Connections | **MCP server** (`#mcp`) | Server URL, a connection token and client snippets, so an AI assistant can drive LLM Lab | +| Connections | **Notifications** (`#notifications`) | **Add channel** for Slack, Discord, Teams, Telegram, ntfy or a signed webhook, with **Recent deliveries** | +| Connections | **Personal tokens** (`#tokens`) | Bearer tokens for scripts, CI and MCP clients, with **New token** and **Revoke** | + +Preferences are saved in this browser only. Keys and tokens are stored encrypted and shown only once. + +Reference: [Integrations](/docs/INTEGRATIONS). + +### Profile (`/users/`) + +Your profile shows account details (role, join date, last sign-in), connections, and **Your work**: applications, analyses, experiments and reports, plus recent activity. Profiles are visible only to their owner. + +### Sign-in pages + +The sign-in pages live under `/auth/`: + +- **Log In**, with **Stay logged in** and **Forgot your password?**. +- **Sign Up**. The first account becomes the administrator and does not need an email address. +- **Reset Password**. +- **Verify Your Email**. +- **Two-Factor Authentication**. + +In local development, verification and reset emails arrive in Mailpit. + +If something does not work as described, see [Troubleshooting](/docs/TROUBLESHOOTING). diff --git a/docs/api-reference.md b/docs/api-reference.md index 1a4d571a..6a2e6e40 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -1,62 +1,97 @@ # API reference -Authentication, conventions, and a router-by-router tour of the REST API at `/api`. +Authentication, conventions, and a router-by-router tour of the REST API at `/api/`. -The complete, always-current schema is the interactive OpenAPI UI at `/api/docs` (staff accounts only). This page covers what the schema can't tell you. +The complete, always-current schema is the interactive OpenAPI UI at `/api/docs` (and `/api/openapi.json`), both staff-only. The API currently has 255 operations. This page covers what the schema can't tell you. ## Authentication -Two interchangeable schemes on every endpoint (except where noted): +Two interchangeable schemes on most endpoints: -- **Bearer token** — create one under Settings → Personal Tokens (`POST /api/tokens/`), then send `Authorization: Bearer `. Token usage (including client IP) is tracked. -- **Session cookie** — what the frontend uses; sign-in flows are handled by allauth headless under `/_allauth/`. +- **Bearer token**: create one under **Settings → Personal tokens** (`/users/settings#tokens`), then send `Authorization: Bearer llml_…`. Each use records the time and client IP. +- **Session cookie**: what the frontend uses. Sign-in flows are handled by allauth headless under `/_allauth/`. ```bash curl -H "Authorization: Bearer $TOKEN" http://localhost:8001/api/models/ ``` +Exceptions: + +| Endpoints | Auth | +| --- | --- | +| `/api/tokens/*` | Session only: a token cannot mint or revoke tokens | +| `/api/docs/*` (viewer tree/page/search) | Session only | +| `/api/export/*.csv`, `*.json`, `*.sarif` | Session only (see [Export endpoints](#export-endpoints)) | +| `/api/system/*` | Staff only (session or Bearer token) | +| `/api/realtime/stream` | Session only | +| `/api/users/bootstrap-status/`, `/api/users/bootstrap-admin/` | None: first-run superuser creation, `409` once any user exists | +| `/api/relay/v1/*` | Endpoint-scoped relay token (`llr_…`), not a user token | +| `/api/mcp` | Bearer token, or `/api/mcp/t/` for `mcp`-scoped tokens; see [Integrations](/docs/INTEGRATIONS#1-mcp-server) | + +> [!WARNING] +> Token scopes (`mcp`, `read`) are enforced only by the MCP endpoint. On the REST API a token with `read` scope still has full read-write access as its user. + ## Conventions -- Errors are `{"detail": ""}` with a fitting status code. -- IDs are UUIDs; reports additionally have a shareable public ID with an `rpt_` prefix. -- List endpoints paginate with `page` / `per_page` query params and return the total alongside the items. +- Errors are `{"detail": ""}` with a fitting status code. Missing-OpenRouter-key errors (catalog sync/refresh, custom jobs) add a `remediation` field. +- Most IDs are UUIDs. Reports are addressed by their public ID (`rpt_…`) in `/api/reports/{report_id}/`. Endpoints, webhooks and presets use integer IDs, and catalog models are addressed by slug. +- List endpoints paginate with `page` / `per_page` query params and return the total alongside the items. The exact envelope varies by router. - The experiments API answers refusals with `400 {"detail", "code", "errors": [...]}`: `code` is machine-readable (e.g. `missing_api_key`, `digest_drift`, `not_draft`) and `errors` lists every blocking issue. -- Owner-scoped resources (experiments, their conditions and schedules, jobs, reports) answer 404 for other users, never 403. +- Owner-scoped resources (generation jobs, experiments and their conditions, schedules and post-runs, analysis runs, reports, endpoints, webhooks, runtime containers and their actions) answer 404 for other users, never 403. Staff see everything. +- Staff-only actions answer 403, for example catalog sync/import/refresh/delete, direct endpoints, cloud provider connections and executor settings. ## Routers +All routers are mounted in `config/api.py`. + | Router | What it covers | | --- | --- | -| `/users/`, `/tokens/`, `/credentials/` | Profile, API token CRUD, encrypted provider keys (OpenRouter) | -| `/models/` | Model catalog list/filter and OpenRouter sync — [Models reference](/docs/MODELS_REFERENCE) | -| `/generation/` | Jobs, batches, run presets, app specs, profiles, stacks, packages — [Generation process](/docs/GENERATION_PROCESS), [Template packages](/docs/TEMPLATE_PACKAGES) | -| `/generation/experiments/` | Experiments: one-call design with a condition cross-product, stateless preview, launch / top-up, pause / resume / cancel / clone, status, cell grid, results (app-blocked statistics), export, schedules and post-run — [Experiments](/docs/EXPERIMENTS#api-summary) | -| `/inference/` | LLM endpoints (OpenRouter, Ollama, OpenAI-compatible): CRUD, relay tokens, sessions, models, probes, metrics, install commands — [Local models](/docs/LOCAL_MODELS) | -| `/execution/` | Executor capacity and lanes, the pending-job queue with blocked reasons, staff-only slot settings and drain — [Execution](/docs/EXECUTION) | -| `/analysis/`, `/analyzers/` | Runs and findings; the tool catalog, workspace, and tool install/test — [Analyzer guide](/docs/ANALYZER_GUIDE) | -| `/statistics/` | Dashboard aggregates | -| `/rankings/` | Model scores and benchmarks, including a refresh action | -| `/reports/` | Saved report CRUD and generation | -| `/runtime/` | Containers for generated apps: build, start/stop, status | -| `/export/` | File downloads — see below | -| `/docs/`, `/system/` | This docs viewer's tree/page/search; staff-only system snapshots | +| `/users/` | First-run bootstrap, current user (`me/`, `me/activity/`), user list and detail (both scoped to yourself) | +| `/tokens/` | Personal API token list, create (raw token shown once), revoke | +| `/credentials/` | Encrypted per-user keys for `openrouter`, `huggingface`, `github`: status, set, delete, test | +| `/models/` | Catalog list/filter, stats, providers, comparison (up to 6), detail, related, insights, live provider endpoints. Staff-only sync, import, delete. JSON/CSV export. See [Models reference](/docs/MODELS_REFERENCE) | +| `/generation/` | Jobs (agent, scaffolding, custom, legacy copilot) with cancel / retry / replay / rerun / evaluate, artifacts, run records and export. Also batches, agent harnesses and config, run presets, blocks, profiles (bundles), stacks, app specs, packages and starters. See [Generation process](/docs/GENERATION_PROCESS), [Template packages](/docs/TEMPLATE_PACKAGES) | +| `/generation/experiments/` | Experiment design with a condition cross-product, stateless preview, launch / top-up, pause / resume / cancel / archive / clone, status, cell grid, results, batches, export, schedules (plus `run-now`) and post-run; `/generation/cron-preview/`. See [Experiments](/docs/EXPERIMENTS#api-summary) | +| `/inference/` | LLM endpoints (OpenRouter, Ollama, OpenAI-compatible; direct or relay): CRUD, refresh, test probes, unload, per-model settings, relay tokens (create/rotate/revoke), sessions, metrics, install commands. See [Local models](/docs/LOCAL_MODELS) | +| `/execution/` | Executor capacity and lanes, the pending-job queue with blocked reasons (filtered to what you may see), staff-only slot settings and drain. See [Execution](/docs/EXECUTION) | +| `/analysis/` | Analysis runs: create (one, or a batch), preview, list and stats, detail, cancel, re-run, bulk cancel/delete, tool raw output, findings and facets. See [Analysis pipeline](/docs/ANALYSIS_PIPELINE#runs-api) | +| `/analyzers/` | Tool catalog and catalog metadata, tool/technology coverage, your workspace (provision / start / stop / delete) and its installed tools (install, bulk install, update, update-all, config, uninstall, test). See [Analyzer guide](/docs/ANALYZER_GUIDE) | +| `/statistics/` | Dashboard, overview, severity, trends, models, tools, top findings, recent activity, code generation, analyzer health; study tables (`study/breakdowns`, `healing`, `tools`, `accounting`). See [Study tables](/docs/STUDY_TABLES) | +| `/rankings/` | Rankings, top models, status, severity-weight sensitivity, refresh, CSV export | +| `/reports/` | Report list, generate, detail, data, export, delete | +| `/runtime/` | Containers for generated apps: build for a job, smoke test, start / stop / restart / remove, logs, health, inspect, whitelisted diagnostic exec, async action status, Docker info. A container is yours if you built it or own its generation job; build and smoke take only your own jobs | +| `/export/` | File downloads and review samples, see below | +| `/docs/` | This docs viewer: tree, categories, search, page | +| `/system/` | Staff-only snapshots (host, containers, Redis, Celery, DB, app stats) and maintenance actions (clear stuck analysis/generation, purge orphan containers, clear caches) | +| `/integrations/` | Cloud provider presets (test key, connect), webhooks (event types, CRUD, test, deliveries), GitHub status and export, MCP client config. See [Integrations](/docs/INTEGRATIONS) | + +There is no `/automation/` router any more: schedules and post-runs live on experiments. ## Export endpoints -`GET /api/export/` endpoints return files: `findings.csv`, `findings.json`, `findings.sarif`, `generation-jobs.csv`, `generation-jobs.json`, `analysis-tasks.csv`, `analysis-tasks.json`, `reports.csv`, `reports.json`. Rows are filterable by query params and capped at `limit` (default 10 000, hard cap 50 000); large CSV exports stream. +`GET /api/export/` file endpoints: `findings.csv`, `findings.json`, `findings.sarif`, `generation-jobs.csv`, `generation-jobs.json`, `analysis-tasks.csv`, `analysis-tasks.json`, `reports.csv`, `reports.json`. Rows are scoped to your own objects (staff: all), filterable by query params and capped at `limit` (default 10 000, hard cap 50 000). Large CSV exports stream. > [!WARNING] -> Export endpoints authenticate via **session cookie only** — Bearer tokens do not work here. They opt out of the standard auth stack and check the session directly, so download them from a logged-in browser (or with the session cookie in `curl`). +> These file exports authenticate via **session cookie only**. Bearer tokens do not work here: the endpoints opt out of the standard auth stack and check the session directly. Download them from a logged-in browser, or with the session cookie in `curl`. The SARIF export makes findings importable into GitHub code scanning and most security tooling. +Two more export endpoints use the normal auth stack, so Bearer tokens work: + +- `GET /api/export/review-sample.zip?experiment_id=|batch_id=&size=30&stratify=model&seed=0&blind=true` draws a seeded, stratified sample of apps with their code, specs and blank label sheets for manual review. +- `POST /api/export/review-sample/score/` (multipart `file`) scores the filled-in ZIP against the automated verdicts. It reports precision with Wilson intervals, Cohen's kappa for the functional verdict and Spearman rho for requirement compliance. + ## Relay protocol -`/api/relay/v1/` (outside the ninja API) is the HTTPS protocol a relay on a GPU box speaks: `hello`, `inventory`, a long-poll `poll` that claims queued LLM requests, `requests//result` and `bye`. It authenticates with an endpoint-scoped relay token (`llr_…`), not a user token. The relay script and its install steps are in [Local models](/docs/LOCAL_MODELS). +`/api/relay/v1/` (outside the ninja API) is the HTTPS protocol a relay on a GPU box speaks: `hello`, `inventory`, a long-poll `poll` that claims queued LLM requests, `requests//result` and `bye`. The same prefix serves the relay's install files: `relay.py`, `Dockerfile` and `llmlab-relay.service`. Protocol calls authenticate with an endpoint-scoped relay token (`llr_…`), not a user token. Install steps are in [Local models](/docs/LOCAL_MODELS). + +## MCP endpoint + +`/api/mcp` (outside the ninja API) is a Model Context Protocol server that replays tool calls in-process against this API as the token's user. See [Integrations](/docs/INTEGRATIONS#1-mcp-server). ## Realtime stream -`GET /api/realtime/stream?channels=experiment:,capacity` is a server-sent-events endpoint living outside the ninja API (it streams). It is an async view: it checks every channel, closes its database connection, and then holds no worker thread or database connection while the stream is open. A `: heartbeat` comment goes out every 15 s. +`GET /api/realtime/stream?channels=experiment:,capacity` is a server-sent-events endpoint outside the ninja API (it streams). It is an async view: it checks every channel, closes its database connection, and then holds no worker thread or database connection while the stream is open. A `: heartbeat` comment goes out every 15 s. `channels` may be comma-separated or repeated. - `401` unauthenticated, `400` without channels or with more than 20, `403` when any channel is malformed, unknown or not yours. - Channels and who may subscribe: @@ -66,8 +101,8 @@ The SARIF export makes findings importable into GitHub code scanning and most se | `generation:` | the job's creator or staff | job status and phase | | `experiment:` | the experiment's creator or staff | `job` (every transition), `tick` (rollups, at most every 2 s), `state`, `post_run` | | `endpoint:` | anyone who may use the endpoint; non-owners are moved to `endpoint::public` (counts only, no IPs, hostnames or errors) | `endpoint`, `session` | -| `analysis:` | the analysis run's creator or staff | run status | -| `runtime:` | the container's creator or staff | container status | -| `capacity`, `dashboard` | any signed-in user (aggregates only) | `capacity`, dashboard updates | +| `analysis:` | the analysis run's creator or staff | `status` (run), `progress` (phase), `result` (a tool started / finished) | +| `runtime:` | the container's creator, the owner of its generation job, or staff | container status | +| `capacity` | any signed-in user (aggregates only) | `capacity` | The publishing side is described in [Background services](/docs/BACKGROUND_SERVICES). diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 1875394c..1dc21a09 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -1,11 +1,19 @@ # Deployment guide -Two ways to put LLM Lab on a server: the one-shot deploy script behind an existing reverse proxy, or the full production compose stack with Traefik. +Two ways to put LLM Lab on a server: the deploy script that runs the local stack behind an existing reverse proxy, or the production compose stack with Traefik. ## Choosing a path -- **`scripts/deploy.sh`** runs the *local* compose stack on a server and wires it into an existing Caddy reverse proxy. Quickest way to a working instance; apps are reached through the in-app path proxy rather than subdomains. -- **`docker-compose.production.yml`** is the full stack: hardened Django image, Traefik with TLS, nginx for media, and per-app subdomain routing. Use this for anything long-lived or multi-user. +| | `scripts/deploy.sh` (local stack) | `docker-compose.production.yml` | +| --- | --- | --- | +| Settings | `config.settings.local` (`DEBUG=True`, uvicorn `--reload`, Vite dev server) | `config.settings.production` (gunicorn + uvicorn workers, built SvelteKit) | +| Sandbox daemon | `apps-dind`: apps, agent sandboxes and analyzer workspaces are nested containers | Host Docker socket mounted into django, executor and Celery; no apps-dind | +| Edge | Existing Caddy or nginx proxies to port 8000 | Traefik on 80/443/5555 with Let's Encrypt | +| App URLs | `/apps//` path proxy, or subdomains through Django | `https://./` via Traefik | +| Local-model agent runs | Supported | Refused (no route from host-daemon gateways to the executor's LLM proxy) | +| Email | mailpit | Mailgun (django-anymail) when `MAILGUN_API_KEY` is set, otherwise SMTP from `EMAIL_*` | + +The deploy script is the quicker and better-isolated path today; the production stack is hardened at the HTTP layer but runs untrusted generated code on the host daemon. ## Path A: the deploy script @@ -13,50 +21,106 @@ Two ways to put LLM Lab on a server: the one-shot deploy script behind an existi bash <(curl -fsSL https://raw.githubusercontent.com/GrabowMar/AutoLLM_Code_Analyzer_rework/main/scripts/deploy.sh) ``` -Handles first-time installs and in-place updates (clone/pull, bootstrap, build, migrate, restart). Interactive by default; for CI set env vars: `DOMAIN` (required), `DEPLOY_DIR`, `REPO_URL`, `BRANCH`, `OPENROUTER_KEY`, `CADDY_CONTAINER` (auto-detected if empty), `NO_PROXY=1` to skip proxy wiring, `CI=1` for non-interactive mode. It reminds you to create a superuser afterwards. +From an existing clone, run `./scripts/deploy.sh` instead. + +Steps: check `git`, `python3`, `curl` and Docker Compose (falls back to `sudo docker`); clone or update the repo; run `scripts/bootstrap.py`; patch `.envs/.local/.django`; build and `up -d`; run `migrate`; wait for django on :8001 and the frontend on :8000; wire the reverse proxy; print a summary. + +| Env var | Default | Meaning | +| --- | --- | --- | +| `DOMAIN` | prompted | Public hostname (required in CI mode) | +| `DEPLOY_DIR` | `~/AutoLLM_Code_Analyzer_rework` | Install / update path | +| `REPO_URL` | the GitHub repo | Git remote | +| `BRANCH` | `main` | Branch to track | +| `OPENROUTER_KEY` | empty | Global OpenRouter key; also sets `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=True` | +| `CADDY_CONTAINER` | auto-detected | Name of a running Caddy container | +| `CADDY_HOST_IP` | Caddy network gateway | Host address Caddy proxies to | +| `NO_PROXY` | `0` | `1` skips reverse-proxy wiring | +| `CI` | empty | `1` or `true`: non-interactive | + +It writes `DJANGO_DOMAIN`, `FRONTEND_PUBLIC_ORIGIN=https://`, `DJANGO_ALLOWED_HOSTS`, `CORS_ALLOWED_ORIGINS` and `CSRF_TRUSTED_ORIGINS` into `.envs/.local/.django` and makes both env files mode 600. For the proxy it first looks for a Caddy container with a bind-mounted `/etc/caddy/Caddyfile`, appends ` { reverse_proxy :8000 }` and reloads Caddy; otherwise it writes `/etc/nginx/sites-available/autollm` (HTTP only; add TLS with certbot). + +> [!WARNING] +> An update runs `git reset --hard origin/` in `DEPLOY_DIR`, discarding local changes there. The local compose file publishes 8000, 8001, 8025 and 5555 on all interfaces, and Docker-published ports bypass host firewalls such as ufw; bind them to `127.0.0.1` with a compose override (`ports: !override`, as `.devcontainer/docker-compose.host.yml` does). Registration is open by default: set `DJANGO_ACCOUNT_ALLOW_REGISTRATION=False` once your accounts exist. + +The first sign-up on the empty database becomes the admin (see [Quickstart](/docs/QUICKSTART)); `createsuperuser` also works. ## Path B: production compose -Copy each template in `.envs/.production/` (`.django.example`, `.postgres.example`, `.frontend.example`, `.traefik.example`) to its real name and fill it in. The vars that matter most in `.django`: +Copy each template in `.envs/.production/` (`.django.example`, `.postgres.example`, `.frontend.example`, `.traefik.example`) to its name without `.example` and fill it in. Values that matter in `.django`: -- `DJANGO_DOMAIN`, `DJANGO_ALLOWED_HOSTS`, `FRONTEND_PUBLIC_ORIGIN` — your public identity; the frontend's `ORIGIN` (in `.frontend`) must match. -- `DJANGO_SECRET_KEY`, `DJANGO_ADMIN_URL` — generate both; the admin URL doubles as obscurity. -- `MAILGUN_API_KEY` / `MAILGUN_DOMAIN` — production email (verification, resets). -- `DJANGO_AWS_*` — optional S3 media storage. -- `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=False` — recommended: every user brings their own key ([Models reference](/docs/MODELS_REFERENCE)). -- `APPS_DOMAIN` — see below. +| Variable | Notes | +| --- | --- | +| `DJANGO_SECRET_KEY` | Required | +| `DJANGO_ADMIN_URL` | Admin path, default `admin/`. Traefik's django router derives its prefix from the same variable, so any value works | +| `SENTRY_DSN` | Optional; Sentry is initialised only when it is set | +| `DJANGO_DOMAIN`, `DJANGO_ALLOWED_HOSTS`, `FRONTEND_PUBLIC_ORIGIN` | Public identity; the frontend's `ORIGIN` (in `.frontend`) must match | +| `DJANGO_CORS_ALLOWED_ORIGINS`, `DJANGO_CSRF_TRUSTED_ORIGINS` | Comma-separated `https://` origins, normally your site; production names carry the `DJANGO_` prefix (the local ones do not) | +| `MAILGUN_API_KEY`, `MAILGUN_DOMAIN`, `MAILGUN_API_URL` | Mailgun through django-anymail when the key is set; domain defaults to `DJANGO_DOMAIN`, use `https://api.eu.mailgun.net/v3` for EU accounts | +| `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER`, `EMAIL_HOST_PASSWORD`, `EMAIL_USE_TLS`, `EMAIL_USE_SSL` | SMTP, used when `MAILGUN_API_KEY` is empty; defaults to `localhost:25` without auth | +| `DJANGO_DEFAULT_FROM_EMAIL`, `DJANGO_SERVER_EMAIL` | Sender addresses; default `LLM Eval Lab >` | +| `DJANGO_SECURE_SSL_REDIRECT` | Template sets `False` (Traefik already redirects HTTP); settings default `True` | +| `LETSENCRYPT_EMAIL` | Read by Traefik; defaults to `admin@` | +| `APPS_DOMAIN` | Base domain for app subdomains; defaults to `DJANGO_DOMAIN` | +| `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK=False` | Recommended: every user brings their own key ([Models reference](/docs/MODELS_REFERENCE)) | +| `REDIS_URL`, `CELERY_FLOWER_USER`, `CELERY_FLOWER_PASSWORD` | As locally | -Then: +`docker-compose.production.yml` sets `DJANGO_SETTINGS_MODULE=config.settings.production` on every Django-image service (django, executor, celeryworker, celerybeat, flower); do not override it, since `config/asgi.py`, `manage.py` and the Celery app fall back to local settings, which need `debug_toolbar` (absent from the production image). Media stays on the `production_django_media` volume (no S3 storage). In `.frontend`, keep `PORT=3000`: the image exposes it and Traefik routes to `frontend:3000`. + +Then build, migrate (production `/start` does not migrate; the executor waits up to 10 minutes for it) and start: ```bash docker compose -f docker-compose.production.yml build +docker compose -f docker-compose.production.yml run --rm django python manage.py migrate docker compose -f docker-compose.production.yml up -d docker compose -f docker-compose.production.yml run --rm django python manage.py createsuperuser ``` -Services: `django` (gunicorn+uvicorn), `postgres` (with a backups volume), `redis`, `celeryworker`, `celerybeat`, `flower`, `traefik` (ports 80/443/5555), `nginx` (serves Django media read-only), `frontend` (built SvelteKit, adapter-node). +Run `migrate` again after every deploy that adds migrations. + +| Service | Role | +| --- | --- | +| `django` | gunicorn with uvicorn workers on :5000; runs `collectstatic` on start | +| `executor` | `/start-executor`: waits for migrations, then `run_executor`; `restart: unless-stopped`; joins the internal `llmproxy` network | +| `postgres` | With a backups volume | +| `redis` | Broker, pub/sub, cache | +| `celeryworker`, `celerybeat`, `flower` | See [Background services](/docs/BACKGROUND_SERVICES) | +| `traefik` | Ports 80 (redirect), 443, 5555 (Flower); routes `/api/`, `/_allauth/`, `/static/` and the `DJANGO_ADMIN_URL` path to django, `/media/` to nginx, the rest to the frontend | +| `nginx` | Serves Django media read-only | +| `frontend` | Built SvelteKit (adapter-node) | + +## Sample-app routing -## Sample-app subdomain routing +How a browser reaches a generated app depends on which settings are active (`config/settings/base.py`): -In production each generated app is served at `https://./` — no host ports, no per-app port limit: +| Mode | Enabled by | App URL | +| --- | --- | --- | +| Traefik bridge | `TRAEFIK_DYNAMIC_DIR` + `DOCKER_APPS_NETWORK` (set by the production compose file) | `https://./` | +| Django subdomain proxy | `APPS_SUBDOMAIN_PROXY=True` + `APPS_DOMAIN` | `https://llm-./` | +| Django path proxy | `APPS_PROXY_PATH=True` (set by the local compose file) | `/apps//`, login required; only the app's owner or staff (others get 404) | -1. Set `APPS_DOMAIN` in `.django` (a dedicated subdomain like `apps.example.com` is recommended so the wildcard stays scoped) and create a DNS wildcard record `*.apps.example.com → your host`. -2. Fill `.traefik` with a DNS-01 provider and credentials (`TRAEFIK_DNS_PROVIDER=cloudflare` + `CF_DNS_API_TOKEN`, or your provider's equivalents). Traefik uses these to issue a single wildcard `*.APPS_DOMAIN` certificate; the file is loaded only by the traefik container so the DNS secret never reaches django. -3. Django writes a per-container route file into a shared volume; Traefik picks it up and routes the subdomain to the app container over the `llm_apps` network. +`APPS_SUBDOMAIN_PROXY` wins over the path proxy. The path proxy needs no DNS work, but most generated SPAs load root-absolute `/assets/...` and call `/api/...`, which then hit the platform instead of the app, so the page renders blank even though the app works. It also serves app JavaScript on the platform's origin. Prefer a subdomain mode for anything beyond quick checks. -If a route file goes stale (e.g. after manual container surgery), `python manage.py reconcile_app_routes` rewrites them all; `sync_container_status` reconciles DB state with Docker. Both are worth a cron entry. +### Traefik (production compose) -### Without Traefik: a single reverse proxy +1. Set `APPS_DOMAIN` in `.django`. A dedicated subdomain such as `apps.example.com` keeps the wildcard scoped. Create DNS `*.apps.example.com` pointing at the host. +2. Fill `.traefik` with a DNS-01 provider and credentials (`TRAEFIK_DNS_PROVIDER=cloudflare` + `CF_DNS_API_TOKEN`, or your provider's equivalents). Traefik refuses to start without `TRAEFIK_DNS_PROVIDER`. The file is loaded only by the traefik container, so the DNS secret never reaches django. +3. Traefik issues one wildcard `*.APPS_DOMAIN` certificate at startup. Django writes a route file per app container into the shared `production_traefik_dynamic` volume; Traefik watches it and routes the subdomain to the container over the `llm_apps` network. Apps publish no host ports. -The local compose stack can serve the same subdomain URLs through one existing edge proxy (Caddy, nginx) fronting Django: +If a route file goes stale (for example after manual container surgery), `python manage.py reconcile_app_routes` rewrites them all and `sync_container_status` reconciles container rows with Docker. Both are worth a cron entry. + +### Django subdomain proxy (any single edge proxy) + +The local stack can serve subdomain URLs through one existing edge proxy (Caddy, nginx) in front of the stack: 1. In `.envs/.local/.django` set `APPS_DOMAIN=` and `APPS_SUBDOMAIN_PROXY=True`, and add `.` to `DJANGO_ALLOWED_HOSTS`. -2. At the edge, route `*.` to Django's host port with a wildcard certificate. Wildcards need DNS-01 issuance, so use a Caddy build with your DNS provider's module (e.g. `xcaddy build --with github.com/caddy-dns/ovh`) or a certbot DNS plugin. More-specific site blocks (your main domain) keep winning over the wildcard. -3. Django's `AppSubdomainProxyMiddleware` picks up `llm-*.` requests and reverse-proxies them to the matching app container; every other subdomain falls through untouched. +2. At the edge, route `*.` to Django's port (8001 in the local stack, not the frontend's 8000) with a wildcard certificate. Wildcards need DNS-01 issuance: use a Caddy build with your DNS provider's module (for example `xcaddy build --with github.com/caddy-dns/ovh`) or a certbot DNS plugin. More specific site blocks (your main domain) keep winning over the wildcard. +3. `AppSubdomainProxyMiddleware` (right after `SecurityMiddleware`, before sessions and CSRF) picks up `llm-*.` requests and reverse-proxies them to the app's port on apps-dind (`APPS_UPSTREAM_HOST=apps-dind`). Other subdomains fall through untouched. + +## Local models in production -Apps keep their host ports in this mode — the middleware fronts them, so it also works for containers that are already running. +Relay endpoints and OpenRouter work in both stacks. If the instance sits behind proxies, set `TRUSTED_PROXY_CIDRS` so relay client IPs resolve correctly, and `LLMLAB_PUBLIC_URL` if the relay install commands should point somewhere other than the request host. See [Local models](/docs/LOCAL_MODELS) and [Execution engine](/docs/EXECUTION#production-compose). ## See also -- [Quickstart](/docs/QUICKSTART) — the local equivalent of all this. -- [Troubleshooting](/docs/TROUBLESHOOTING) — including the production-specific failure modes. +- [Quickstart](/docs/QUICKSTART): the local equivalent. +- [Troubleshooting](/docs/TROUBLESHOOTING): including production failure modes. diff --git a/docs/development-guide.md b/docs/development-guide.md index c0b4594a..6423d294 100644 --- a/docs/development-guide.md +++ b/docs/development-guide.md @@ -1,62 +1,138 @@ # Development guide -Day-to-day workflow: just targets, tests, environment files, and how the backend and frontend are laid out. +Day-to-day workflow: just targets, the devcontainer, tests and linting, environment variables, and how the backend and frontend are laid out. ## Toolchain -Python 3.13 with [uv](https://docs.astral.sh/uv/) (`pyproject.toml` + `uv.lock`), but you rarely need either on the host — every command runs inside Docker Compose. The `justfile` pins `COMPOSE_FILE=docker-compose.local.yml`, so `just` targets always hit the local stack. +Python 3.13 with [uv](https://docs.astral.sh/uv/) (`pyproject.toml` + `uv.lock`) for the backend, Node with npm for the frontend (`frontend/package.json`). You rarely need either on the host: every command runs inside Docker Compose. The `justfile` exports `COMPOSE_FILE=docker-compose.local.yml`, so `just` targets always hit the local stack. Plain `docker compose` commands need the same file (`-f docker-compose.local.yml` or `COMPOSE_FILE`); the compose project is named `llmlab`, so containers are `llmlab--1`. ## just targets | Target | What it does | | --- | --- | -| `just bootstrap` | Create `.envs/.local/.django` and `.postgres` from templates, generate secrets | +| `just bootstrap` | Create `.envs/.local/.django` and `.postgres` from the templates, generate credentials | | `just build [args]` | `docker compose build` | -| `just up` / `just down` | Start / stop the stack | -| `just prune [-v]` | Remove containers **and volumes** (full reset) | +| `just up` / `just down` | Start (`up -d --remove-orphans`) / stop the stack | +| `just prune [args]` | `docker compose down -v`: removes containers **and volumes** (database, apps-dind images, certs) | | `just logs [service]` | Follow container logs | -| `just test [args]` | `docker compose run --rm django pytest` — same command as CI | -| `just manage ` | Run any `manage.py` command (migrate, shell, seeds, …) | +| `just test [args]` | `docker compose run --rm django pytest`, the same command as CI | +| `just manage ` | Run any `manage.py` command in a one-off django container | | `just frontend-install` | `npm install` inside the frontend container | -| `just frontend-dev` | Run the frontend dev server in the foreground | -| `just frontend-build` | Production frontend build | +| `just frontend-dev` | Run the frontend service in the foreground | +| `just frontend-build` | Production frontend build (`npm run build`) | + +`docker compose exec django python manage.py ...` does **not** work: `DATABASE_URL` is built by the image's `/entrypoint` from the `POSTGRES_*` variables, and `exec` skips it. Use `just manage`, or prefix the command: `docker compose exec django /entrypoint python manage.py `. + +## Reload behaviour + +| Service | Picks up code changes | Notes | +| --- | --- | --- | +| `django` | `uvicorn --reload` | A reload kills threads running in it: interactive analysis runs, workspace provisioning, tool installs, report generation, container actions | +| `executor` | No | Restart it after backend edits; drain first so running jobs are not interrupted ([Execution engine](/docs/EXECUTION)). The UI flags it as running older code | +| `celeryworker`, `celerybeat`, `flower` | `watchfiles` restarts them on `.py` changes | | +| `frontend` | Vite HMR | | + +The scripts baked into the image (`/start`, `/start-celeryworker`, ...) change only after `just build`. The executor is the exception: locally it runs `compose/local/django/start-executor` from the bind-mounted repo. + +## Devcontainer + +`.devcontainer/devcontainer.json` attaches VS Code to the `django` service, layering `.devcontainer/docker-compose.devcontainer.yml` (a `dev-user` venv, caches, `DOCKER_HOST=tcp://apps-dind:2376` for the `docker` CLI) and a gitignored, host-specific `.devcontainer/docker-compose.host.yml` on top of the local compose file. In the devcontainer the django command is overridden: `post-attach.sh` starts `/entrypoint /start` in the background on attach and logs to `/tmp/django-start.log`, so the API (and the migrate step) only come up once an editor attaches. The git pre-commit hook uses the devcontainer's venv, so commit from inside it. ## Environment files -Local settings come from `.envs/.local/.django` (plus `.postgres` for DB credentials): +Each service reads `.envs/.local/.django` and `.envs/.local/.postgres` (`POSTGRES_HOST`, `POSTGRES_PORT`, `POSTGRES_DB`, `POSTGRES_USER`, `POSTGRES_PASSWORD`). Variables in the template: -- `DJANGO_DOMAIN` — used in `DEFAULT_FROM_EMAIL` and `ALLOWED_HOSTS` defaults. -- `REDIS_URL` — Celery broker/result backend and cache. -- `CELERY_FLOWER_USER` / `CELERY_FLOWER_PASSWORD` — Flower dashboard login (generated by bootstrap). -- `OPENROUTER_API_KEY` — optional global fallback key for users without a personal key. -- `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK` — set `False` to force every user to bring their own key. -- `FRONTEND_PUBLIC_ORIGIN` — the URL users open in the browser; used in verification/reset emails. -- `DJANGO_ALLOWED_HOSTS`, `CORS_ALLOWED_ORIGINS`, `CSRF_TRUSTED_ORIGINS` — set all three when serving from anything other than localhost. +| Variable | Purpose | +| --- | --- | +| `DJANGO_DOMAIN` | Default sender domain, OpenRouter `HTTP-Referer`, bootstrap admin email domain, `APPS_DOMAIN` default | +| `REDIS_URL` | Celery broker and result backend, realtime pub/sub, executor doorbell | +| `CELERY_FLOWER_USER` / `CELERY_FLOWER_PASSWORD` | Flower login (generated by bootstrap) | +| `OPENROUTER_API_KEY` | Optional global fallback key for users without a personal key | +| `OPENROUTER_ALLOW_GLOBAL_KEY_FALLBACK` | `False` forces every user to bring their own key (default `True`) | +| `FRONTEND_PUBLIC_ORIGIN` | The URL users open in the browser; used in verification and reset emails | +| `APPS_DOMAIN`, `APPS_SUBDOMAIN_PROXY`, `APPS_PUBLIC_ORIGIN` | How browsers reach generated apps ([Deployment guide](/docs/deployment-guide#sample-app-routing)) | +| `DJANGO_ALLOWED_HOSTS`, `CORS_ALLOWED_ORIGINS`, `CSRF_TRUSTED_ORIGINS` | Set all three when serving from anything other than localhost | + +Other settings read from the environment, with their defaults (`config/settings/base.py`, `local.py`): + +| Variable | Default | Purpose | +| --- | --- | --- | +| `ACCOUNT_EMAIL_VERIFICATION` | `optional` (local), `mandatory` (production) | allauth email verification | +| `DJANGO_ACCOUNT_ALLOW_REGISTRATION` | `True` | Allow self sign-up | +| `GENERATION_DEFAULT_STACK` | `flask-react` | Stack used when neither the request nor its profile names one | +| `AGENT_DEFAULT_HARNESS` | `opencode` | Harness for agent runs (`opencode`, `mini-swe-agent`, `aider`) | +| `AGENT_MAX_LLM_REQUESTS`, `AGENT_WALL_CLOCK_SECONDS`, `AGENT_MAX_TOTAL_TOKENS`, `AGENT_MAX_COST_USD` | `100`, `3600`, `30000000`, `5.0` | Per-run agent budgets (experiments and jobs may override) | +| `AGENT_MAX_CONCURRENT_RUNS` | `1` | Initial executor sandbox slots (editable at runtime by staff) | +| `AGENT_SANDBOX_MEM_LIMIT`, `AGENT_SANDBOX_NANO_CPUS` | `3g`, `2000000000` | Per-sandbox caps, frozen into job snapshots | +| `AGENT_AUTO_EVALUATE` | `True` | Grade every agent sample with the hidden conformance suite | +| `AGENT_NETWORK_POLICY`, `AGENT_SAMPLING_POLICY` | `packages`, `provider_default` | Sandbox egress and sampling defaults | +| `EXECUTOR_TICK_SECONDS`, `EXECUTOR_LEASE_SECONDS`, `EXECUTOR_HEARTBEAT_SECONDS` | `2.0`, `90`, `20` | Executor loop timing | +| `APPS_DIND_MEM_LIMIT`, `APPS_DIND_APP_RESERVE` | `4g`, `1g` | Memory rule for executor slots; keep the first equal to apps-dind's `mem_limit` | +| `ANALYSIS_MAX_WORKERS` | `4` | Analyzer tools run in parallel per analysis run | +| `LLM_LOCAL_AGENT_RUNS` | on when `APPS_DOCKER_HOST` is set | Allow agent runs on local (non-OpenRouter) endpoints | +| `INFERENCE_DIRECT_ALLOWED_TARGETS`, `INFERENCE_HOST_PUBLIC_IPS`, `INFERENCE_EXTRA_BLOCKED_CIDRS` | empty | Direct-endpoint allow and block lists ([Local models](/docs/LOCAL_MODELS)) | +| `LLMLAB_PUBLIC_URL` | request host | Base URL the relay install commands point at | +| `TRUSTED_PROXY_CIDRS` | empty | Proxies whose `X-Forwarded-For` hop is trusted for relay client IPs | +| `STACK_REQUIRE_APPROVAL` | `False` | Non-staff stacks need staff approval | +| `STACK_ALLOWED_PYTHON_IMAGES`, `STACK_ALLOWED_NODE_IMAGES` | `python:3.11-slim,python:3.12-slim`, `node:20-alpine,node:22-alpine` | Base-image allowlist for user stacks | + +The compose file sets the apps daemon wiring itself (`APPS_DOCKER_HOST=tcp://apps-dind:2376`, `APPS_DOCKER_CERT_PATH`, `APPS_UPSTREAM_HOST=apps-dind`, `APPS_PROXY_PATH=True`) and the executor's LLM proxy (`LLM_PROXY_BIND`, `LLM_PROXY_UPSTREAM`). It interpolates these from your shell or a root `.env` file: + +| Variable | Default | Used for | +| --- | --- | --- | +| `APPS_DIND_MEM_LIMIT`, `APPS_DIND_CPUS` | `4g`, `4` | apps-dind caps (also passed to django and executor) | +| `LLMPROXY_SUBNET`, `LLM_PROXY_ADDR`, `APPS_DIND_LLMPROXY_ADDR` | `172.30.99.0/29`, `172.30.99.2`, `172.30.99.3` | Internal `llmproxy` network; change together | +| `LLM_GATEWAY_EGRESS_SUBNET` | `10.213.0.0/24` | Nested network of local-model gateways inside apps-dind | +| `FRONTEND_ORIGIN` | `http://localhost:8000` | Passed to the frontend container as `ORIGIN` | +| `TRUSTED_PROXY_CIDRS` | empty | Passed through to django | ## Backend layout -Django 6 project; settings in `config/settings/{base,local,production,test}.py`, the API root in `config/api.py`. Fifteen apps under `backend/`: +Django 6 project with django-ninja. Settings live in `config/settings/{base,local,production,test}.py`, the API root and router mounts in `config/api.py`, plain Django views (SSE stream, relay protocol, MCP endpoint, `/apps/` proxy) in `config/urls.py`. Eighteen apps under `backend/`: | App | Responsibility | | --- | --- | -| `users`, `tokens`, `credentials` | Accounts (allauth headless + MFA), Bearer API tokens, encrypted per-user provider keys | -| `llm_models` | Model catalog synced from OpenRouter | -| `generation` | Generation jobs, templates, prompt assembly, OpenRouter client | -| `analysis` | Analyzer workspaces, tool catalog, runs, findings | -| `statistics`, `rankings`, `reports` | Aggregation over findings and jobs, model scoring, saved reports | -| `runtime` | Builds and runs generated apps as Docker containers, routing/proxy | -| `realtime` | Server-sent events stream backed by Redis pub/sub | -| `export` | CSV/JSON/SARIF downloads | -| `docs`, `system` | This docs viewer; staff-only system snapshots | -| `automation` | Pipelines, batches, schedules | +| `users`, `tokens`, `credentials` | Accounts (allauth headless + MFA, bootstrap admin), Bearer API tokens, encrypted per-user provider keys | +| `llm_models` | Model catalog synced from OpenRouter, models.dev / Hugging Face enrichment | +| `generation` | Generation jobs (agent, scaffolding, custom), experiments with schedules and post-runs, run presets, app specs, prompts, blocks, profiles, evaluation | +| `execution` | The executor process: leader lock, scheduler, lanes and slots, leases, reconciler | +| `inference` | LLM endpoints (OpenRouter, relay, direct), LLM request queue, the executor's internal LLM proxy, relay protocol | +| `analysis` | Analyzer workspaces, tool catalog, analysis runs, findings, AI review | +| `runtime` | Stacks, builds and runs generated apps as containers, app routing and proxying | +| `statistics`, `rankings`, `reports` | Aggregation and study tables, model scoring, saved and study reports | +| `integrations` | MCP server, cloud LLM providers, GitHub export, outbound webhooks | +| `realtime` | Server-sent events stream over Redis pub/sub | +| `export` | CSV/JSON/SARIF downloads, review samples | +| `docs`, `system` | This docs viewer; staff-only system snapshots and maintenance | +| `automation` | Legacy pipelines and batches; kept for existing data, its router is unmounted | + +Shared helpers live in `backend/common/` (exceptions, `dispatch_in_thread`). The relay client that connects a GPU box to an instance is in `relay/` at the repo root. + +Apps follow one convention: `models.py`, `api/{views,schema}.py`, `services/`, `tasks.py` (where there are Celery tasks), `tests/`. Views raise ninja's `HttpError`; services stay free of ninja imports and raise `backend.common.exceptions.*`, which an exception handler in `config/api.py` maps to HTTP responses. -Apps follow one convention: `models.py`, `api/{views,schema}.py`, `services/`, `tasks.py`, `tests/`. Views raise ninja's `HttpError`; services stay free of ninja imports and raise `backend.common.exceptions.*`, which an exception handler in `config/api.py` maps to HTTP responses. +Scoring constants live in one place: `backend/rankings/services/constants.py` defines `SEVERITY_WEIGHTS` (shared by rankings, statistics and the report generators) and the `SEVERITY_WEIGHT_SCHEMES` used by the sensitivity analysis. Change them there, nowhere else. The methodology is described in [Architecture](/docs/ARCHITECTURE). -Scoring constants live in one place: `backend/rankings/services/constants.py` defines `SEVERITY_WEIGHTS` (shared by rankings, statistics, and report generators) and the `SEVERITY_WEIGHT_SCHEMES` used by the sensitivity analysis — change them there, nowhere else. The methodology itself is described in [Architecture](/docs/ARCHITECTURE). +### Seed data and management commands + +Seeds sync after every `migrate` through `post_migrate` handlers; a seeding error is logged, never raised. Data files: `backend/analysis/data/tools/*.yaml` (tool catalog), `backend/generation/data/` (library), `backend/runtime/scaffolding/` (stacks). + +| Command | Purpose | +| --- | --- | +| `seed_analysis_tools`, `seed_generation_templates` | Re-run a seeder by hand | +| `lint_generation_data` | Validate the generation library data files | +| `enrich_models` | Attach models.dev / Hugging Face metadata to the catalog | +| `seed_benchmarks` | Load benchmark results from a JSON fixture | +| `run_executor`, `executor drain\|resume\|status`, `wait_for_migrations` | Executor process and control ([Execution engine](/docs/EXECUTION)) | +| `sync_container_status`, `reconcile_app_routes` | Reconcile container rows with Docker; rewrite Traefik route files | +| `export_study_tables`, `healing_ablation`, `study_report` | Study outputs ([Study tables](/docs/STUDY_TABLES)) | +| `export_review_sample`, `score_review_sample` | Human review sample export and scoring | +| `runautomationscheduler` | Legacy automation scheduler loop; nothing creates pipeline schedules any more | ## Frontend layout -SvelteKit with Svelte 5 runes, Tailwind 4, in `frontend/`. Routes live under `src/routes/(app)/` (one directory per feature: `generation`, `analysis`, `automation`, `rankings`, …) plus auth pages. API clients are in `src/lib/api/`. In the local stack the dev server on :8000 proxies API calls to django via `API_TARGET=http://django:8000` — always develop against http://localhost:8000, not :8001. +SvelteKit 2 with Svelte 5 runes, Tailwind 4 and adapter-node, in `frontend/`. App routes live under `src/routes/(app)/` (one directory per feature: `experiments`, `sample-generator`, `applications`, `analysis`, `analyzers`, `endpoints`, `models`, `templates`, `runtime`, `rankings`, `statistics`, `reports`, `docs`, `system`, `users`, ...), auth pages under `src/routes/(auth)/auth/`. Old URLs such as `/automation`, `/integrations` and `/settings` are redirects. API clients are in `src/lib/api/`, navigation in `src/lib/nav.ts`, settings sections in `src/lib/settings-sections.ts`. UI conventions are in `frontend/DESIGN_SYSTEM.md`. + +In the local stack the Vite dev server on :8000 proxies `/api`, `/_allauth`, `/admin`, `/media` and `/apps` to django via `API_TARGET=http://django:8000`. Always develop against http://localhost:8000, not :8001. ## Testing @@ -66,22 +142,34 @@ just test backend/analysis # one app just test -k workspace -x # any pytest args pass through ``` -pytest runs with `--ds=config.settings.test --reuse-db`. Add `--create-db` after changing seed data (e.g. the analyzer tool YAML), since the reused test DB keeps the old catalog. Shared fixtures live in `backend/conftest.py`; per-app factories in `backend//tests/factories.py`. +pytest runs with `--ds=config.settings.test --reuse-db --import-mode=importlib` (`pyproject.toml`). Add `--create-db` after changing seed data (for example the analyzer tool YAML), since the reused test DB keeps the old rows. Two suites running at once share the test database; give one its own with `docker compose run --rm -e POSTGRES_DB=llmlab_ django pytest`. Shared fixtures live in `backend/conftest.py`; factories in `backend//tests/factories.py` (analysis, automation, generation, llm_models, runtime, users). -Lint/format is ruff + mypy (backend), `svelte-check` (frontend), wired into `.pre-commit-config.yaml`. +Frontend unit tests use vitest: `npm test` in `frontend/`. Type checking is `npm run check` (`svelte-check`). The frontend container is capped at 2 GB and can be OOM-killed by `svelte-check` or `vite build`; run them with a larger heap on the host if that happens. + +## Linting and CI + +`.pre-commit-config.yaml` runs generic file hooks, django-upgrade (target 6.0), ruff check and ruff format, pyproject-fmt, djLint, prettier on `frontend/`, and gitleaks. It skips `docs/` and migrations. + +| Workflow | Jobs | +| --- | --- | +| `ci.yml` | pre-commit; mypy on `backend` (non-blocking); backend tests (bootstrap, `makemigrations --check`, migrate, pytest in the local compose stack); frontend (`npm ci`, `svelte-check` non-blocking, `npm test`, `npm run build`, Node 24) | +| `codeql.yml` | CodeQL for Python and JavaScript/TypeScript | +| `gitleaks.yml` | Secret scan of the full history | +| `trivy.yml` | Trivy filesystem and image scans (HIGH/CRITICAL) when compose files, Dockerfiles or dependency locks change, plus weekly | ## Working on these docs -The in-app viewer at `/docs` serves raw Markdown straight from the repo's `docs/` directory — no build step. Rules, enforced by how `backend/docs/services.py` and the client renderer work: +The in-app viewer at `/docs` serves raw Markdown straight from the repo's `docs/` directory; there is no build step. Rules, enforced by `backend/docs/services.py` and the client renderer: - Exactly one `# H1` as the first line; it becomes the page title. The first plain paragraph after it becomes the landing-card blurb. - No YAML frontmatter (it would render as content). No images (nothing serves them). -- Cross-link other pages as absolute `/docs/` paths — relative `.md` links don't resolve in the app. -- Mermaid fences and `> [!NOTE]` / `> [!WARNING]` alerts are supported; code fences highlight for the common languages (bash, python, ts, json, yaml, dockerfile, …). -- New top-level files land in the "Other" category unless you add their slug to `CATEGORY_MAP` in `backend/docs/services.py`. +- Cross-link other pages as absolute `/docs/` paths; relative `.md` links do not resolve in the app. +- Mermaid fences and `> [!NOTE]` / `> [!WARNING]` alerts are supported; code fences highlight for the common languages. +- New files land in the "Other" category unless you add their slug to `CATEGORY_MAP` in `backend/docs/services.py`. ## See also -- [Quickstart](/docs/QUICKSTART) — first-time setup. -- [Architecture](/docs/ARCHITECTURE) — the bigger picture. -- [API reference](/docs/api-reference) — routers, auth, conventions. +- [Quickstart](/docs/QUICKSTART): first-time setup. +- [Background services](/docs/BACKGROUND_SERVICES): which process runs what. +- [Architecture](/docs/ARCHITECTURE): the bigger picture. +- [API reference](/docs/api-reference): routers, auth, conventions. diff --git a/frontend/DESIGN_SYSTEM.md b/frontend/DESIGN_SYSTEM.md index 07c263d9..6b12d700 100644 --- a/frontend/DESIGN_SYSTEM.md +++ b/frontend/DESIGN_SYSTEM.md @@ -30,7 +30,7 @@ Tailwind v4 reads them via `@theme inline`. Dark stays the default mode. figure strokes, brand dot) — never buttons. `info` is sky so "running" never reads as the action color. - No gradients, glass or glow anywhere. The only decorative surfaces are the - `.lab-canvas` graph paper (app/auth layouts) and the dashboard `.hero-figure`. + `.lab-canvas` graph paper (app/auth layouts). - All text tokens were checked ≥ 4.5:1 on their surfaces (most ≥ 6:1). ### Chart palette (validated) @@ -60,19 +60,19 @@ Rules (see `src/lib/components/charts/theme.ts`): ## Typography - **Display (serif):** Source Serif 4 (`--font-display`, `font-display`) — `h1`, - `h2`, page titles, card titles, hero. Italic + primary for one emphasised phrase. + `h2`, page titles, card titles. Italic + primary for one emphasised phrase. - **UI/body:** IBM Plex Sans (`--font-sans`). `h3`–`h6` stay sans. - **Data (mono):** IBM Plex Mono (`--font-mono`, `font-mono`) — KPI values, big stat numbers, eyebrows, `.section-label`, `.kpi-label`, table headers, IDs. -- Hero scale: `.display-xl`. Page titles: `.page-header h1` (2xl/30px). -- KPI values: `.kpi-value` (3xl mono, tabular). Big numbers elsewhere use +- Page titles: `.page-header h1` (lg/24px). +- KPI values: `.kpi-value` (2xl mono, tabular). Big numbers elsewhere use `font-mono font-medium tabular-nums`, never the serif. - Eyebrow: `.page-eyebrow` (mono, uppercase, 0.14em tracking). ## Page anatomy (every page under `(app)/`) ```svelte -
+
{#snippet meta()}{/snippet} @@ -86,11 +86,20 @@ Rules (see `src/lib/components/charts/theme.ts`):
``` -- `PageHeader` renders the section eyebrow (`02 · Generate`, from `navSectionFor` in `nav.ts`) or the - `back` link, the serif `

`, and the masthead rule (hairline + short primary bar). +- `PageHeader` is a lean card (bordered `bg-card`, `px-4 py-2.5`, `text-lg` serif title): `[back] [leading] +Title [?] [meta] … [actions]`, then a one-line `description` (xs, muted). The optional `children` line + (stats chips / IDs) renders as a hairline-separated footer strip of the card; `leading` takes an avatar + or entity icon. No eyebrow: orientation (`Section › Parent › Page`) is the top bar breadcrumb, which ends + on the `

` PageHeader publishes (`$lib/stores/page-title.svelte`). `back` renders an icon button; + glossaries / how-it-works go in the `help` snippet (popover behind the `?`), never as extra paragraphs. +- Nothing sits between the header card and the content. Page-level tabs or a mode switch go in the + `nav` snippet (flush bottom strip; wrap the page in `Tabs.Root`); plain counts go in `children` as + `.header-stats` / `.header-stat` (`16 total`), not as a KPI row; one-line notices go right-aligned + in the `nav` strip or into `children`. Search/filter controls for a table are the top strip of that + table's card (`FilterBar embedded`, or a `.table-toolbar`), never a separate card above it. - `PageHeader` owns the `

` and the browser `` (`"{title} · LLM Lab"`) — no separate `<svelte:head><title>` and no hand-rolled `.page-header` markup. -- Vertical rhythm: `space-y-6` between page sections, `gap-4` inside grids, `gap-3` in KPI rows. +- Vertical rhythm: `space-y-4` between page sections, `gap-4` inside grids, `gap-3` in KPI rows. - Primary action is the last button in `actions`, `size="sm"`; secondary actions `variant="outline"`. ## Shared components (`src/lib/components/`) @@ -108,13 +117,33 @@ Rules (see `src/lib/components/charts/theme.ts`): | Clickable card | add `.card-lift` | hover effects on static cards | | Logo / wordmark | `BrandMark` (`compact`, `tagline`, `size="lg"`) | icon + hand-built "LLM Lab" spans | | KPI | `dashboard/KpiCard` or `.kpi-card` / `.kpi-label` / `.kpi-value` | one-off stat boxes | -| Small group label | `.section-label` | `text-[10px] uppercase …` variants | +| Heading that divides content | `SectionHeading` (`size="lg"` on the canvas, `size="sm"` for groups) | `border-t pt-3` + label, `border-b` under a title | +| Small group label | `.section-label` (a label over one field/value, no divider) | `text-[10px] uppercase …` variants | | Pick one of a list | `ui/select` (`Select.Simple` for plain lists, parts for rich rows) | raw `<select>` with copied class strings | | Boolean / multi-choice field | `ui/checkbox` (`Checkbox` in a `<label>`, or with `<Label for>`) | `<input type=checkbox>`, hand-drawn check boxes | | Mode switch (2–4 options) | `ui/segmented` (`Segmented`) | pill rows with `aria-pressed` and active classes | Navigation lives in `src/lib/nav.ts` (sidebar, mobile menu, breadcrumbs). Add routes there. +### Section headings (`SectionHeading`) + +One labelled divider for the whole app, in the sidebar's group motif: `[number] Title ───── [count | aside]`, +then an optional `lead` flush below. Styles are `.section-heading*` in `app.css`. + +- **`lg`** (default) — a section of the page on the canvas: mono number, serif `text-xl` title, ink rule. The + experiment page's numbered sections (`StudySection` wraps it). Sections sit `space-y-10` apart; heading → body + `space-y-3`. +- **`sm`** — mono caps label + lighter rule: a group of cards on the canvas (Settings' Account / Preferences / + Connections, Docs' "Start here") or a group inside a card (endpoint form, spec editor, "Key findings", "History"). + `count` puts a figure at the end of the rule ("API endpoints ─── 3"). +- One left edge: number, title, lead and the cards below all start at the same x — no hanging number column, no + indented lead. +- The rule is ink (`--foreground` at 30% / 22%), never `--border`: a border-coloured line vanishes into the + graph-paper grid. Don't add a `border-t`/`border-b` around a `SectionHeading`; the rule is the divider. +- A section's title and the title of the figure or card directly under it must differ ("4.1 Runs" → "Table D1. + Every run, in launch order"), never repeat. +- Card titles stay `Card.Title`/strip headers; nav rails (Library, Docs, Settings) keep their own group labels. + ### Form primitives (`src/lib/components/ui/`, bits-ui 2) - **Select** — `import * as Select from '$lib/components/ui/select'`. @@ -132,6 +161,25 @@ Navigation lives in `src/lib/nav.ts` (sidebar, mobile menu, breadcrumbs). Add ro - Number inputs bind numbers (`null` when empty): hand `<Input type="number">` the number itself, never `String(n)`, or Svelte rewrites partial input such as `-0` while the user types. +### Settings page (`/users/settings`) + +Laid out like the Generation Library (`/templates`): `PageHeader`, a bookmark rail card in the Library nav's +shape (`.section-label` groups, active = `bg-primary/10 text-primary` + left tick, a status dot where the +Library shows counts), and every section as one card — heading strip (serif title, `?` popover with the +description, `StatusBadge`, copy-link) with its rows flush below — in a single scrolling list. + +- The rail and the mobile sticky chips are bookmarks: they scroll to a section, follow the scroll position + (scroll-spy) and keep `#id` in the address bar; deep links re-align while sections load. +- Section bodies are built only from `users-settings/` primitives, never their own `Card`s: + - `SettingsBlock` — a shaded group strip (`title`, `description` string or snippet, `actions`) like the + Library's group rows, then its rows, then an optional `footer` action strip. + - `SettingsRow` — label + help left, control right (`stack` for long controls); inputs `sm:w-72`. + - Custom rows (tokens, providers, channels) and stray notes use `.settings-row`; `.settings-footer` is the + action strip, `.settings-link` the inline link style. + - `CredentialRow` — a stored provider key (status, test, replace, remove, inline form); `CopyField` — a + one-line copyable value. +- Snippets passed to a block (`footer`, `actions`) must be its direct children, not inside an `{#if}`. + ## Tables and lists (`src/lib/components/table/`) Import from `$lib/components/table`. Base `table`/`th`/`td` styles are global (`app.css`); don't restate @@ -190,22 +238,45 @@ mutated); `onchange` fires only on user edits; catalogs load once per page (`loa `{detail, remediation}`); map 422s onto fields with `apiFieldErrors(e)`; ignore `isAbortError(e)` from superseded requests. +### Sample-generator composer (`generator/composer/`) + +All three modes (agent, template scaffolding, single prompt) share one layout: + +- **Steps** are `ComposerSection`s. Required steps pass `done` (the number badge turns into a check) and + `meta` ("3 selected"); a step holding an unapplied value passes `invalid` (alert badge, red border). +- **Optional steps** (model settings, budget) are `collapsible`: folded to a one-line `summary` of what + applies right now ("Provider defaults v1 · no overrides · …") unless the draft already overrides + something. The body stays mounted so its fields keep validating, and an `invalid` step unfolds itself. +- **Rail** (≥ 56rem container): `ComposerSectionNav` (the experiments composer's section index) above the + Run summary. The summary opens with `RunFormula` — `2 app specs × 2 models × 1 harness × 3 trials = 12 +runs` — whose empty factors are dashed and link to their step; it replaces the old checklist (neutral, no + error before the user acts). Narrower, the summary follows the form and `StartBar` sticks to the bottom. + ## Experiments building blocks (`src/lib/components/experiments/`) Import from `$lib/components/experiments`. Endpoint pieces (`SlotMeter`, `EndpointChip`, …) come from `$lib/components/inference`. -| Component / module | Use for | -| ----------------------------- | ------------------------------------------------------------------------------------------------ | -| `ExperimentComposer` | the single-page design form (`/experiments/new`); sticky `PlanRail` on ≥ 56rem, `StartBar` below | -| `ExperimentRow` | one experiment in a list (progress, ETA, endpoints, inline lifecycle buttons) | -| `CapacityStrip` | one-line executor / lanes / endpoint capacity summary | -| `monitor/ConditionGrid` | condition × app × repeat squares (CSS grid, sticky label column, endpoint groups) | -| `monitor/EndpointLane` | one endpoint's live slots, loaded models, this user's active jobs, blocked reason | -| `Leaderboard`, `PairedDeltas` | results: ranked conditions with CIs; Δ vs the pre-registered baseline (forest plot) | -| `ComparabilityTable` | what each condition actually runs with; differing columns highlighted | -| `live.svelte.ts` | `ExperimentLive`: SSE + polling fallback; never hand-roll polling on experiment pages | -| `cells.ts`, `format.ts` | cell codes (`CELL_STATES` square styles), progress mapping, ETA / hours / Δ / p formats | +| Component / module | Use for | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| `ExperimentComposer` | the single-page design form (`/experiments/new`); sticky `PlanRail` on ≥ 56rem, `StartBar` below | +| `ComposerSectionNav` | the composer's section index above the Plan: current step, required done / to do, invalid flags | +| `ExperimentRow` | one experiment in a list (progress, ETA, endpoints, inline lifecycle buttons) | +| `CapacityStrip` | one-line executor / lanes / endpoint capacity summary | +| `monitor/ConditionGrid` | condition × app × repeat squares (CSS grid, sticky label column, endpoint groups); `framed={false}` inside a figure | +| `monitor/EndpointLane` | one endpoint's live slots, loaded models, this user's active jobs, blocked reason | +| `monitor/FailureSummary` | Abstract: failed runs grouped by error message, with a likely cause (`failureHint`) and a run link | +| `Leaderboard`, `PairedDeltas` | results: ranked conditions with CIs (table only); Δ vs the baseline as an ink forest plot | +| `study/StudyMasthead` | the experiment page header (replaces `PageHeader` there): accession id, hypothesis, spec strip | +| `study/StudySection` | a numbered section ("2.3 Conditions ───") with a lead: a `<section>` around a `SectionHeading` | +| `study/StudyFigure` | a numbered figure / table ("Figure 1", "Table M2") with title, controls and a caption | +| `study/PassRatePlot` | dot-and-whisker figure of pass rate with 95% CIs (stacked rows on phones) | +| `study/StudyRecord` | the Abstract's record panel (dates, design counts, provenance, copyable reference) | +| `study/findings.ts` | design shape, accession id and the Abstract's key findings / limitations (pure, unit-tested) | +| `study/results.svelte.ts` | `ResultsFeed`: one tab's `GET /results/` (abort-on-supersede, tick throttle) | +| `ComparabilityTable` | what each condition actually runs with; differing columns highlighted | +| `live.svelte.ts` | `ExperimentLive`: SSE + polling fallback; never hand-roll polling on experiment pages | +| `cells.ts`, `format.ts` | cell codes (`CELL_STATES` square styles), progress mapping, ETA / hours / Δ / p formats | - Scaling to ~30 models: everything is grouped by endpoint (picker, selection chips, plan rows, lanes, grid); condition labels are `model · harness`; compare against one baseline, never N² pairs. @@ -213,6 +284,62 @@ Import from `$lib/components/experiments`. Endpoint pieces (`SlotMeter`, `Endpoi for a model load) and always have an `aria-label`/`title`. - Statistics: never show a pooled pass rate across conditions; p-values only when results are not interim. +### Experiment page (`/experiments/[id]`) — the study record + +Laid out like a paper, not a dashboard (ui-ux-pro-max query `"scientific research experiment results academic +paper data" --design-system --density 7 --variance 3 --motion 2`, mapped onto this system's tokens and fonts): + +- **Masthead** (`StudyMasthead`): eyebrow `EXPERIMENT · AGENT · EXP-955A5BD5` (first UUID block as an accession + number), serif title, status + registered / launched / finished, the hypothesis as a serif-italic pull quote + with an ochre (`accent-2`) rule, then a spec strip — Design (`k conditions × n apps × r repeats = N cells`) · + Baseline · Outcome · Randomisation — then the data-collection strip (counts, ETA, `LiveStrip`) and the tabs. +- **Sections** (`?tab=`, numbered in the tab row): 1 Abstract (`summary`) · 2 Method (`method`) · 3 Results · + 4 Data (`data`) · 5 Operations (`operations`). Older names (`overview`, `design`, `runs`, `automation`) still + resolve (`study/tabs.ts`). Science first: execution limits, endpoints, activity and automation live in + Operations, never in Method. +- **Abstract**: description and a generated paragraph (design, baseline, outcome, estimator) in the display + serif (15px, `max-w-3xl`), a status sentence, numbered key findings and a "Limitations" list derived from the + pre-registered analysis (`keyFindings`); Figure 1 (pass rate), Figure 2 (data collection); the record panel + sticks on the right from `xl`. +- **Figures and tables** are numbered and captioned: Abstract Figures 1–2, Results Table 1 · Figure 3 · Table 2, + Method Tables M1–M3, Data Table D1. A caption says what the mark is, the estimator, n and the exclusions — + explanations go in the caption, not in paragraphs above the figure. +- **Plots** are ink on paper: one series in `chart-1`, the baseline as a hollow diamond plus a hairline through + every row, CI whiskers with end caps; forest-plot squares are filled when the 95% interval excludes 0 and hollow + when it includes it. Never red/green for "better/worse" — the axis carries the sign. Values are printed beside + the marks (tooltips only repeat them); below `sm` plots switch to stacked rows instead of scrolling sideways. +- **Results** opens with the analysis specification ("Pre-registered — common cells only · …"); any URL option + turns it into "Exploratory" with a way back to the pre-registered analysis. Server notes are lettered footnotes. +- Sections whose parts can be hidden number themselves from what is shown (no "5.1 → 5.3" gaps). + +## Analysis building blocks (`src/lib/components/analysis/`) + +Used by `/analysis`, `/analysis/create`, `/analysis/[id]` and the app page's Analyses card. + +| Component / module | Use for | +| ------------------------- | ----------------------------------------------------------------------------------------------------- | +| `LevelBadge` | a finding's level exactly as its tool printed it (`tool_severity`); nothing for tools that rate none | +| `RunStatusCell` | run status in tables: badge, and while active a bar + "3/12 tools · Semgrep" / queue / phase | +| `RunFindingsCell` | total findings (so far, while active) + up to three tool-reported levels | +| `RunLiveStrip` | an active run's progress: one segment per tool, running tool + clock, connection state | +| `ToolPipeline` | a run's tools in execution order: state, time, version, findings, warnings, error, Output/Findings | +| `ToolOutputDialog` | one tool's raw result (command, exit code, stdout, options) | +| `RunOverview` | coverage KPIs, attention notes, levels/tools bars, top files/rules (each opens filtered findings) | +| `FindingsExplorer` | facet rail + search + sort + expandable rows + CSV/JSON export; filters are bindable (URL-synced) | +| `FindingDetail` | everything a tool said about one finding, from `finding-data.ts` (standard + legacy keys merged) | +| `MetricsPanel` | metric tools' aggregates, generic over keys | +| `AppPicker`, `ToolPicker` | the creator's app multi-select (running-app filter) and tool selection with preview applicability | +| `live.svelte.ts` | `RunWatcher`: SSE on `analysis:<id>` channels + polling fallback + 1 s clock; never hand-roll polling | +| `format.ts`, `tools.ts` | status/phase/result-state labels, clocks, level ordering and tints; the tool catalog (loaded once) | + +- **Never display the normalized `severity`.** It is a heuristic for tools that rate nothing and exists only + for aggregate statistics. Show `tool_severity` with `LevelBadge`; count unrated findings apart ("Not rated by + tool"), never fold them into a level. `levelClasses` tints a tool's own word, it doesn't rate it. +- A skipped tool measured nothing unless its `skip_reason` is `not_applicable`: say why (`resultState`), never + show it as a clean "0 findings". +- Live pages show elapsed clocks from `started_at` and the run's `phase`; a long tool is legitimately quiet, so + the watcher keeps a slow poll even with SSE open. + ## Color & type rules - Components use tokens only: `primary`, `muted-foreground`, `success|warning|destructive|info` @@ -226,8 +353,11 @@ Import from `$lib/components/experiments`. Endpoint pieces (`SlotMeter`, `Endpoi - Swiss-minimal data surfaces: crisp 1px borders, `--shadow-card` hairline, no glass, glow or gradients on cards, tables or lists. Radius 0.3rem. -- `.lab-canvas` (graph paper) only on layout wrappers; `.hero-panel` + - `.hero-figure` only on the dashboard. +- `.lab-canvas` (graph paper) only on layout wrappers. No hero panels or + decorative figures: the dashboard opens with a plain `PageHeader` like every page. +- Density: catalogues and inventories (tools, containers, specs) are tables, not card + grids; zero-state panels use `EmptyState compact` or one muted line, never a + full-height placeholder. - KPI tiles: left accent tick via `--kpi-accent`; hover = border + faint tint, no lift. diff --git a/frontend/README.md b/frontend/README.md index 95fabbba..3a15420a 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -1,42 +1,58 @@ -# sv +# LLM Lab frontend -Everything you need to build a Svelte project, powered by [`sv`](https://github.com/sveltejs/cli). +The web UI for LLM Lab: SvelteKit 2 with Svelte 5 runes, TypeScript, Tailwind CSS 4 +and bits-ui (shadcn-svelte) components, built with `@sveltejs/adapter-node`. -## Creating a project +## Running it -If you're seeing this, you've probably already done this step. Congrats! +In the local stack the `frontend` compose service runs `npm run dev` on container +port 8080, published as <http://localhost:8000>. Always open the app there, not +Django's :8001: the frontend forwards `/api/`, `/_allauth/`, `/admin/`, `/media/` and +`/apps/` to Django (`API_TARGET`, `http://django:8000` in compose), so session +cookies and CSRF work on a single origin. -```sh -# create a new project -npx sv create my-app +```bash +just up # starts the frontend with the rest of the stack +just frontend-install # npm install inside the container (after package.json changes) +just frontend-dev # run the dev server in the foreground +just frontend-build # production build ``` -To recreate this project with the same configuration: +The forwarding is done twice: by Vite's dev-server `proxy` (`vite.config.ts`) and by +the `handle` hook in `src/hooks.server.ts`, which also serves the production +(adapter-node) build. The hook sets `X-Forwarded-Host` / `-Proto` / `-For`, and +strips `content-encoding` / `content-length` from upstream responses because Node's +`fetch` has already decompressed the body. -```sh -# recreate this project -npx sv@0.12.7 create --template minimal --types ts --no-install frontend -``` - -## Developing - -Once you've created a project and installed dependencies with `npm install` (or `pnpm install` or `yarn`), start a development server: - -```sh -npm run dev +## Checks -# or start the server and open the app in a new browser tab -npm run dev -- --open +```bash +npm run check # svelte-kit sync + svelte-check (types) +npm test # vitest (unit tests next to the code, *.test.ts) +npm run build # production build ``` -## Building - -To create a production version of your app: - -```sh -npm run build +CI runs all three on Node 24. `svelte-check` and `vite build` can exceed the frontend +container's 2 GB memory cap; run them on the host (or a host-side copy of +`frontend/`) if the container gets OOM-killed. Formatting is prettier, through the +repo's pre-commit hooks. + +## Layout + +```text +src/ + routes/(app)/ one directory per feature page (experiments, sample-generator, + analyzers, statistics, …); (auth)/ holds the sign-in flows + routes/{dashboard,generation,login,settings}/ redirects from old URLs + hooks.server.ts API/auth/admin/media/apps reverse proxy + lib/api/ typed API client, one module per backend router (client.ts, core.ts, sse.ts) + lib/components/ shared components (ui/ = bits-ui primitives) and per-feature folders + lib/docs/ the in-app docs renderer (marked + Shiki + Mermaid) + lib/stores/ auth, page title, preferences (Svelte 5 rune stores) + lib/nav.ts navigation: sidebar, mobile menu and breadcrumbs + app.css design tokens (light "paper" / dark "observatory" themes) ``` -You can preview the production build with `npm run preview`. - -> To deploy your app, you may need to install an [adapter](https://svelte.dev/docs/kit/adapters) for your target environment. +What each page does is described in the [user guide](../docs/USER_GUIDE.md). Visual +rules (tokens, typography, charts, component patterns) are in +[DESIGN_SYSTEM.md](DESIGN_SYSTEM.md); follow it for any UI change. diff --git a/frontend/src/app.css b/frontend/src/app.css index 2b767ea8..89bf0cd9 100644 --- a/frontend/src/app.css +++ b/frontend/src/app.css @@ -425,53 +425,166 @@ /* ─── Component Layer ─────────────────────────────────────────── */ @layer components { - /* ── Page header (compact, dense) ───────────────────────────── */ - /* Journal-style masthead: mono eyebrow, serif title, double rule. */ + /* ── Page header (lean card) ────────────────────────────────── */ + /* ┌[back] [leading] Title [?] [meta] [secondary…] [Primary]┐ + │ One-line description │ + ├ optional footer strip (stats chips, IDs) ─────────────────────────────┤ + A compact card like every other surface on the page: orientation lives in the + top bar breadcrumb, so the header only names the page and holds its actions. */ .page-header { - @apply relative flex flex-col gap-2 pb-4; - border-bottom: 1px solid var(--border); + @apply rounded-md border border-border bg-card; + box-shadow: var(--shadow-card); } - .page-header::after { - content: ""; - position: absolute; - left: 0; - bottom: -1px; - width: 3rem; - height: 2px; - background: var(--primary); + .page-header-main { + @apply flex flex-wrap items-center gap-x-3 gap-y-2 px-4 py-2.5; + } + .page-header-back { + @apply inline-flex size-7 shrink-0 cursor-pointer items-center justify-center rounded-md border border-border text-muted-foreground transition-colors duration-150 hover:bg-accent hover:text-foreground focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-ring; + } + .page-header-body { + @apply flex min-w-0 flex-[1_1_16rem] flex-col; + } + .page-header-titlebar { + @apply flex min-w-0 flex-wrap items-center gap-x-2 gap-y-1; } .page-header h1 { - @apply text-2xl font-semibold tracking-tight sm:text-[1.875rem] sm:leading-tight; + @apply min-w-0 break-words text-lg font-semibold leading-6 tracking-tight; font-family: var(--font-display); + text-wrap: balance; + } + .page-header-help-trigger { + @apply inline-flex size-6 shrink-0 cursor-pointer items-center justify-center rounded-md text-muted-foreground transition-colors duration-150 hover:bg-accent hover:text-foreground focus-visible:outline-2 focus-visible:outline-offset-1 focus-visible:outline-ring data-[state=open]:bg-accent data-[state=open]:text-foreground; + } + .page-header-help { + @apply z-50 w-[min(26rem,calc(100vw-1.5rem))] rounded-md border border-border bg-popover p-3 text-sm leading-relaxed text-popover-foreground; + box-shadow: + var(--shadow-card), + 0 8px 24px -8px rgb(0 0 0 / 0.18); + } + .page-header-desc { + @apply max-w-4xl text-xs leading-snug text-muted-foreground; + } + .page-header-extra { + @apply min-w-0 border-t border-border/60 px-4 py-1.5 text-xs; + } + .page-header-actions { + @apply ml-auto flex shrink-0 flex-wrap items-center gap-2 max-sm:ml-0 max-sm:w-full max-sm:shrink; + } + /* Tabs / mode switch along the bottom edge: the card border is the tab rule. */ + .page-header-nav { + @apply flex min-w-0 flex-wrap items-center gap-x-3 gap-y-1 border-t border-border/60 px-2; } - .page-header p { - @apply text-sm leading-relaxed text-muted-foreground; + .page-header-nav [role="tablist"] { + @apply border-b-0; + } + .page-header-nav [role="tab"] { + @apply mb-0 h-9; + } + .page-header-nav [role="radiogroup"] { + @apply my-1.5; + } + /* Inline counts in a header strip: "16 total · 0 running". */ + .header-stats { + @apply flex min-w-0 flex-wrap items-center gap-x-4 gap-y-1 text-xs text-muted-foreground; + } + .header-stat { + @apply inline-flex items-baseline gap-1 whitespace-nowrap; + } + .header-stat > b { + @apply font-mono text-sm font-medium text-foreground tabular-nums; } .page-eyebrow { - @apply inline-flex items-center gap-1.5 text-[11px] font-medium uppercase tracking-[0.14em] text-muted-foreground; + @apply inline-flex shrink-0 items-center gap-1.5 text-[11px] font-medium uppercase tracking-[0.14em] text-muted-foreground; font-family: var(--font-mono); } .back-link { - @apply inline-flex w-fit items-center gap-1 rounded text-xs font-medium text-muted-foreground transition-colors hover:text-foreground; + @apply inline-flex min-w-0 max-w-[14rem] shrink-0 items-center gap-1 rounded text-xs font-medium text-muted-foreground transition-colors duration-150 hover:text-foreground focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-ring; } /* Small uppercase label above a group of fields/values */ .section-label { - @apply text-[11px] font-medium uppercase tracking-[0.12em] text-muted-foreground; + @apply text-xs font-medium uppercase tracking-[0.08em] text-muted-foreground; + font-family: var(--font-mono); + } + + /* ── Section heading (SectionHeading.svelte) ────────────────── */ + /* The one labelled divider: [number] Title ─────── [count | aside], then an + optional lead flush below. The rule is ink, not `--border`: on the + graph-paper canvas a border-coloured line disappears into the grid. + `lg` = a page section on the canvas (serif); `sm` = a group label over cards + or between groups inside a card (mono caps, like the sidebar groups). */ + .section-heading { + @apply min-w-0 space-y-1; + } + .section-heading-line { + @apply flex min-w-0 flex-wrap items-center gap-x-3 gap-y-2; + } + .section-heading-num { + @apply shrink-0 text-[13px] font-medium tabular-nums text-muted-foreground; + font-family: var(--font-mono); + } + .section-heading-title { + @apply min-w-0 text-xl font-semibold leading-7 tracking-tight; + font-family: var(--font-display); + text-wrap: balance; + } + .section-heading-rule { + @apply h-px min-w-8 flex-1; + background-color: color-mix(in oklab, var(--foreground) 30%, transparent); + } + .section-heading-count { + @apply shrink-0 text-xs tabular-nums text-muted-foreground; font-family: var(--font-mono); } + .section-heading-aside { + @apply flex shrink-0 flex-wrap items-center gap-2; + } + .section-heading-aside:empty { + display: none; + } + .section-heading-lead { + @apply max-w-3xl text-sm leading-relaxed text-muted-foreground; + } + .section-heading[data-size="sm"] .section-heading-line { + @apply gap-x-2; + } + .section-heading[data-size="sm"] .section-heading-num { + @apply text-[11px] text-muted-foreground/70; + } + .section-heading[data-size="sm"] .section-heading-title { + @apply text-xs leading-5 font-medium uppercase tracking-[0.12em] text-muted-foreground; + font-family: var(--font-mono); + text-wrap: wrap; + } + .section-heading[data-size="sm"] .section-heading-rule { + background-color: color-mix(in oklab, var(--foreground) 22%, transparent); + } + .section-heading[data-size="sm"] .section-heading-lead { + @apply text-xs; + } + + /* Settings building blocks (users-settings/SettingsBlock, SettingsRow). */ + .settings-row { + @apply px-4 py-3.5; + } + .settings-footer { + @apply flex flex-wrap items-center justify-between gap-2 bg-muted/30 px-4 py-2.5; + } + .settings-link { + @apply text-primary underline-offset-2 hover:underline; + } /* ── Empty state ────────────────────────────────────────────── */ .empty-state { - @apply flex flex-col items-center justify-center py-10 text-center sm:py-16; + @apply flex flex-col items-center justify-center py-8 text-center sm:py-10; } .empty-state-icon { - @apply mb-4 flex h-14 w-14 items-center justify-center rounded-md border border-border bg-muted text-muted-foreground; + @apply mb-3 flex h-10 w-10 items-center justify-center rounded-md border border-border bg-muted text-muted-foreground; } /* ── KPI card (the workhorse for dashboards) ────────────────── */ .kpi-card { - @apply relative flex flex-col gap-1 overflow-hidden rounded-md border border-border bg-card px-4 py-3; + @apply relative flex flex-col gap-0.5 overflow-hidden rounded-md border border-border bg-card px-3.5 py-2.5; box-shadow: var(--shadow-card); transition: border-color 0.2s ease, @@ -482,7 +595,7 @@ content: ""; position: absolute; left: 0; - top: 0.75rem; + top: 0.625rem; width: 2px; height: 1.25rem; background: var(--kpi-accent, var(--primary)); @@ -494,11 +607,11 @@ background-color: color-mix(in oklab, var(--primary) 3%, var(--card)); } .kpi-label { - @apply text-[11px] font-medium uppercase tracking-[0.12em] text-muted-foreground; + @apply text-xs font-medium uppercase tracking-[0.08em] text-muted-foreground; font-family: var(--font-mono); } .kpi-value { - @apply text-3xl font-medium tabular-nums tracking-tight; + @apply text-2xl font-medium tabular-nums tracking-tight; font-family: var(--font-mono); } @@ -542,6 +655,34 @@ overflow: auto; max-height: calc(100dvh - 7rem); } + /* Fade the edge a horizontal scroller continues past (`scrollHint()`). Tables + only fade on phones, where scrollbars are overlays and otherwise invisible. */ + .scroll-hint[data-overflow-end], + .table-frame[data-overflow-end] { + --fade-end: 2.5rem; + } + .scroll-hint[data-overflow-start] { + --fade-start: 2.5rem; + } + .scroll-hint[data-overflow-start], + .scroll-hint[data-overflow-end] { + mask-image: linear-gradient( + to right, + transparent, + #000 var(--fade-start, 0px), + #000 calc(100% - var(--fade-end, 0px)), + transparent + ); + } + @media (max-width: 767px) { + .table-frame[data-overflow-end] { + mask-image: linear-gradient( + to right, + #000 calc(100% - var(--fade-end)), + transparent + ); + } + } .table-frame[data-fits] { overflow: visible; max-height: none; @@ -682,6 +823,10 @@ @apply rounded-md border border-border bg-card overflow-hidden; box-shadow: var(--shadow-card); } + /* Filter bar as the top strip of the table card it filters. */ + .fb-shell.fb-embedded { + @apply rounded-none border-0 border-b shadow-none; + } /* Filter chip — base (inactive) */ .fb-chip { @apply inline-flex h-7 cursor-pointer select-none items-center gap-1 rounded-md border border-border @@ -798,7 +943,7 @@ } .table-card-mobile tbody td::before { content: attr(data-label); - font-size: 0.7rem; + font-size: 0.75rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.05em; @@ -899,19 +1044,6 @@ background-position: -1px -1px; } - /* ── Hero panel (dashboard masthead) ───────────────────────── */ - .hero-panel { - @apply relative overflow-hidden rounded-md border border-border bg-card; - box-shadow: var(--shadow-card); - } - /* Faint plotted-curve figure on the right edge of the hero */ - .hero-figure { - @apply pointer-events-none absolute inset-y-0 right-0 hidden w-1/2 md:block; - color: var(--primary); - -webkit-mask-image: linear-gradient(90deg, transparent, #000 45%); - mask-image: linear-gradient(90deg, transparent, #000 45%); - } - /* ── Staggered entrance (set --stagger-i per section) ───────── */ @keyframes rise-in { from { @@ -952,10 +1084,4 @@ opacity: 0; } } - - /* ── Display heading (hero scale) ───────────────────────────── */ - .display-xl { - @apply text-3xl font-semibold tracking-tight sm:text-[2.5rem] sm:leading-[1.1]; - font-family: var(--font-display); - } } diff --git a/frontend/src/lib/api/analysis.ts b/frontend/src/lib/api/analysis.ts deleted file mode 100644 index 7105ddb4..00000000 --- a/frontend/src/lib/api/analysis.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { apiFetch } from "./core"; - -export interface AnalysisFinding { - id: number; - result_id: number; - severity: string; - category: string; - confidence: string; - title: string; - description: string; - suggestion: string; - file_path: string; - line_number: number | null; - column_number: number | null; - code_snippet: string; - rule_id: string; - tool_specific_data: Record<string, any>; - analyzer_name: string; - suppressed: boolean; - suppression_reason: string; - suppressed_by_email: string | null; - created_at: string; -} - -export interface AnalysisResult { - id: number; - task_id: string; - analyzer_type: string; - analyzer_name: string; - status: string; - raw_output: Record<string, any>; - summary: Record<string, any>; - error_message: string; - started_at: string | null; - completed_at: string | null; - duration_seconds: number | null; - findings_count: number; - finding_summary: Record<string, number>; - created_at: string; -} - -export interface AnalysisStats { - total_tasks: number; - completed_tasks: number; - failed_tasks: number; - running_tasks: number; - total_findings: number; - findings_by_severity: Record<string, number>; - findings_by_category: Record<string, number>; - most_common_issues: Array<{ title: string; count: number }>; -} - -export interface AnalysisTask { - id: string; - name: string; - status: string; - threshold_status: "not_configured" | "passed" | "exceeded"; - generation_job_id: string | null; - source_code: Record<string, string>; - configuration: Record<string, any>; - results_summary: Record<string, any>; - generation_job_name: string | null; - created_by_email: string; - results_count: number; - findings_count: number; - started_at: string | null; - completed_at: string | null; - duration_seconds: number | null; - error_message: string; - created_at: string; - updated_at: string; - target_url: string | null; - container_instance_id: string | null; - profile_id: number | null; -} - -export interface AnalysisTaskList { - id: string; - name: string; - status: string; - threshold_status: "not_configured" | "passed" | "exceeded"; - created_at: string; - updated_at: string; - generation_job_id: string | null; - created_by_email: string; - results_summary: Record<string, any>; - started_at: string | null; - completed_at: string | null; - duration_seconds: number | null; - target_url: string | null; - container_instance_id: string | null; - profile_id: number | null; -} - -export interface ConfigField { - name: string; - type: "string" | "number" | "boolean" | "select" | "multiselect"; - label: string; - description: string; - default: unknown; - options: Array<{ value: string; label: string }>; - required: boolean; - min: number | null; - max: number | null; - placeholder: string; -} - -export interface AnalyzerInfo { - name: string; - type: string; - display_name: string; - description: string; - available: boolean; - availability_message: string; - default_config: Record<string, any>; - config_schema: ConfigField[]; - supports_live_target: boolean; - supported_code_types: string[]; -} - -export interface AnalysisProfile { - id: number; - name: string; - description: string; - analyzers: string[]; - settings: Record<string, Record<string, unknown>>; - is_default: boolean; - created_by_email: string | null; - created_at: string; - updated_at: string; -} - -export interface PaginatedAnalysisTasks { - items: AnalysisTaskList[]; - total: number; - page: number; - per_page: number; - pages: number; -} - -export interface PaginatedFindings { - items: AnalysisFinding[]; - total: number; - page: number; - per_page: number; - pages: number; -} - -export async function cancelAnalysisTask( - taskId: string, -): Promise<{ success: boolean; status: string; message?: string }> { - const res = await apiFetch(`/analysis/tasks/${taskId}/cancel/`, { - method: "POST", - }); - return res.json(); -} - -export async function createAnalysisTask(data: { - name?: string; - generation_job_id?: string; - source_code?: Record<string, string>; - analyzers?: string[]; - settings?: Record<string, Record<string, unknown>>; - auto_start?: boolean; - live_target?: boolean; - profile_id?: number | null; - thresholds?: Record<string, number>; -}): Promise<AnalysisTask> { - const res = await apiFetch("/analysis/tasks/", { - method: "POST", - body: JSON.stringify(data), - }); - return res.json(); -} - -export async function deleteAnalysisTask( - taskId: string, -): Promise<{ success: boolean }> { - const res = await apiFetch(`/analysis/tasks/${taskId}/`, { - method: "DELETE", - }); - return res.json(); -} - -export async function getAnalysisFindings( - taskId: string, - params?: { - page?: number; - per_page?: number; - severity?: string; - category?: string; - analyzer?: string; - file_path?: string; - include_suppressed?: boolean; - }, -): Promise<PaginatedFindings> { - const searchParams = new URLSearchParams(); - if (params?.page) searchParams.set("page", String(params.page)); - if (params?.per_page) searchParams.set("per_page", String(params.per_page)); - if (params?.severity) searchParams.set("severity", params.severity); - if (params?.category) searchParams.set("category", params.category); - if (params?.analyzer) searchParams.set("analyzer", params.analyzer); - if (params?.file_path) searchParams.set("file_path", params.file_path); - if (params?.include_suppressed) - searchParams.set("include_suppressed", "true"); - const qs = searchParams.toString(); - const res = await apiFetch( - `/analysis/tasks/${taskId}/findings/${qs ? "?" + qs : ""}`, - ); - return res.json(); -} - -export async function suppressFinding( - taskId: string, - findingId: number, - reason: string = "", -): Promise<AnalysisFinding> { - const res = await apiFetch( - `/analysis/tasks/${taskId}/findings/${findingId}/suppress/`, - { method: "POST", body: JSON.stringify({ reason }) }, - ); - return res.json(); -} - -export async function unsuppressFinding( - taskId: string, - findingId: number, -): Promise<AnalysisFinding> { - const res = await apiFetch( - `/analysis/tasks/${taskId}/findings/${findingId}/unsuppress/`, - { method: "POST" }, - ); - return res.json(); -} - -export async function getAnalysisProfiles(): Promise<AnalysisProfile[]> { - const res = await apiFetch("/analysis/profiles/"); - return res.json(); -} - -export async function createAnalysisProfile(data: { - name: string; - description?: string; - analyzers: string[]; - settings?: Record<string, Record<string, unknown>>; - is_default?: boolean; -}): Promise<AnalysisProfile> { - const res = await apiFetch("/analysis/profiles/", { - method: "POST", - body: JSON.stringify(data), - }); - return res.json(); -} - -export async function updateAnalysisProfile( - id: number, - data: { - name: string; - description?: string; - analyzers: string[]; - settings?: Record<string, Record<string, unknown>>; - is_default?: boolean; - }, -): Promise<AnalysisProfile> { - const res = await apiFetch(`/analysis/profiles/${id}/`, { - method: "PUT", - body: JSON.stringify(data), - }); - return res.json(); -} - -export async function deleteAnalysisProfile( - id: number, -): Promise<{ success: boolean }> { - const res = await apiFetch(`/analysis/profiles/${id}/`, { - method: "DELETE", - }); - return res.json(); -} - -export async function getAnalysisResult( - taskId: string, - resultId: number, -): Promise<AnalysisResult> { - const res = await apiFetch(`/analysis/tasks/${taskId}/results/${resultId}/`); - return res.json(); -} - -export async function getAnalysisResults( - taskId: string, -): Promise<AnalysisResult[]> { - const res = await apiFetch(`/analysis/tasks/${taskId}/results/`); - return res.json(); -} - -export async function getAnalysisStats(): Promise<AnalysisStats> { - const res = await apiFetch("/analysis/stats/"); - return res.json(); -} - -export async function getAnalysisTask(taskId: string): Promise<AnalysisTask> { - const res = await apiFetch(`/analysis/tasks/${taskId}/`); - return res.json(); -} - -export async function getAnalysisTasks(params?: { - page?: number; - per_page?: number; - status?: string; - search?: string; - generation_job_id?: string; - sort_by?: string; - sort_dir?: string; -}): Promise<PaginatedAnalysisTasks> { - const searchParams = new URLSearchParams(); - if (params?.page) searchParams.set("page", String(params.page)); - if (params?.per_page) searchParams.set("per_page", String(params.per_page)); - if (params?.status) searchParams.set("status", params.status); - if (params?.search) searchParams.set("search", params.search); - if (params?.generation_job_id) - searchParams.set("generation_job_id", params.generation_job_id); - if (params?.sort_by) searchParams.set("sort_by", params.sort_by); - if (params?.sort_dir) searchParams.set("sort_dir", params.sort_dir); - const qs = searchParams.toString(); - const res = await apiFetch(`/analysis/tasks/${qs ? "?" + qs : ""}`); - return res.json(); -} - -export async function getAnalyzers(): Promise<AnalyzerInfo[]> { - const res = await apiFetch("/analysis/analyzers/"); - return res.json(); -} diff --git a/frontend/src/lib/api/analyzers.ts b/frontend/src/lib/api/analyzers.ts index 9f989a80..552ae672 100644 --- a/frontend/src/lib/api/analyzers.ts +++ b/frontend/src/lib/api/analyzers.ts @@ -20,6 +20,10 @@ export interface AnalyzerTool { category: string; kind: "container" | "ai"; target_language: string; + /** "dynamic" tools probe the app's running container (DAST); "static" ones read the source. */ + analysis_mode: "static" | "dynamic"; + /** Technology slugs the tool scans; labels come from `getCatalogMeta()`. */ + technologies: string[]; icon: string; version: string; docs_url: string; @@ -34,6 +38,33 @@ export interface AnalyzerTool { output_kind?: string; } +export interface CatalogEntry { + slug: string; + label: string; + description: string; + /** Technologies only: the `technology_groups` slug it belongs to. */ + group: string; +} + +/** Shop vocabulary from `data/tools/catalog.yaml`. */ +export interface CatalogMeta { + categories: CatalogEntry[]; + modes: CatalogEntry[]; + technology_groups: CatalogEntry[]; + technologies: CatalogEntry[]; +} + +/** How the caller's recent generated apps map onto the catalog. */ +export interface AppCoverage { + /** Generated apps with recognizable code (most recent `apps_considered`). */ + apps_total: number; + apps_considered: number; + /** Technology slug → number of apps containing it. */ + technologies: Record<string, number>; + /** Tool slug → number of apps containing something the tool scans. */ + tools: Record<string, number>; +} + export interface Workspace { id: string; status: "absent" | "provisioning" | "ready" | "stopped" | "error"; @@ -89,6 +120,17 @@ export async function getTool(slug: string): Promise<AnalyzerTool> { return res.json(); } +export async function getAppCoverage(): Promise<AppCoverage> { + const res = await apiFetch("/analyzers/coverage/"); + return res.json(); +} + +/** Categories, static/dynamic modes and the scanned-technology vocabulary. */ +export async function getCatalogMeta(): Promise<CatalogMeta> { + const res = await apiFetch("/analyzers/catalog/"); + return res.json(); +} + // --- Workspace -------------------------------------------------------------- export async function getWorkspace(): Promise<Workspace> { diff --git a/frontend/src/lib/api/client.ts b/frontend/src/lib/api/client.ts index 0f2c1a5b..6b82af59 100644 --- a/frontend/src/lib/api/client.ts +++ b/frontend/src/lib/api/client.ts @@ -2,7 +2,6 @@ // Existing route imports (`import { fooBar } from '$lib/api/client'`) keep // working unchanged. New code should import from the domain module directly. -export * from "./analysis"; export * from "./applications"; export * from "./credentials"; export * from "./generation"; diff --git a/frontend/src/lib/api/experiments.ts b/frontend/src/lib/api/experiments.ts index 46bc0141..e123ba29 100644 --- a/frontend/src/lib/api/experiments.ts +++ b/frontend/src/lib/api/experiments.ts @@ -133,8 +133,11 @@ export interface ExperimentComparabilityRow { /** "slug@v" or null. */ profile: string | null; param_overrides: Record<string, unknown>; - /** Local endpoints: where the condition runs. */ - endpoint?: { slug: string; kind: string; max_concurrency?: number } | null; + /** Where the condition runs: an endpoint slug ("openrouter" for the system endpoint). */ + endpoint?: + | string + | { slug: string; kind: string; max_concurrency?: number } + | null; /** Every problem, blocking or not. */ problems: string[]; /** The blocking subset of `problems`. */ diff --git a/frontend/src/lib/api/models.ts b/frontend/src/lib/api/models.ts index cca0ecc4..fb9e16f0 100644 --- a/frontend/src/lib/api/models.ts +++ b/frontend/src/lib/api/models.ts @@ -87,14 +87,23 @@ export interface SyncResult { upserted: number; } +/** + * Path-safe model identifier. Proxies decode `%2F`, so an OpenRouter id like + * `deepseek/deepseek-v4.1-flash` would split the route; the backend resolves + * the `provider_model` form back to the model. + */ +export function modelPathSegment(id: string): string { + return encodeURIComponent(id.replaceAll("/", "_")); +} + export async function deleteModel(slug: string): Promise<void> { - await apiFetch(`/models/detail/${encodeURIComponent(slug)}/`, { + await apiFetch(`/models/detail/${modelPathSegment(slug)}/`, { method: "DELETE", }); } export async function getModel(slug: string): Promise<LLMModelDetail> { - const res = await apiFetch(`/models/detail/${encodeURIComponent(slug)}/`); + const res = await apiFetch(`/models/detail/${modelPathSegment(slug)}/`); return res.json(); } @@ -164,7 +173,7 @@ export async function getRelatedModels( limit = 10, ): Promise<LLMModelSummary[]> { const res = await apiFetch( - `/models/detail/${encodeURIComponent(slug)}/related/?limit=${limit}`, + `/models/detail/${modelPathSegment(slug)}/related/?limit=${limit}`, ); return res.json(); } @@ -183,7 +192,7 @@ export async function refreshModelFromOpenRouter( slug: string, ): Promise<LLMModelDetail> { const res = await apiFetch( - `/models/detail/${encodeURIComponent(slug)}/refresh/`, + `/models/detail/${modelPathSegment(slug)}/refresh/`, { method: "POST" }, ); return res.json(); @@ -193,3 +202,192 @@ export async function syncModelsFromOpenRouter(): Promise<SyncResult> { const res = await apiFetch("/models/sync/", { method: "POST" }); return res.json(); } + +// ── Model page: platform measurements, catalog context, providers ───────── + +/** A proportion with its Wilson 95% interval (`rate`/`ci_*` null when n = 0). */ +export interface RateWithCI { + successes: number; + n: number; + rate: number | null; + ci_low: number | null; + ci_high: number | null; +} + +export type SeverityCounts = Record< + "critical" | "high" | "medium" | "low" | "info", + number +>; + +export interface RankPosition { + rank: number; + of: number; + score: number; +} + +export interface CatalogPercentile { + value: number | null; + /** Share of the catalog below this value (ties count half), 0–1. */ + percentile: number | null; + n: number; + median: number | null; +} + +export interface ModelInsights { + generation: { + jobs_total: number; + by_status: Record<string, number>; + by_mode: Record<string, number>; + completed: number; + success: RateWithCI; + functional: RateWithCI & { + mean_pass_rate: number | null; + by_status: Record<string, number>; + }; + resources: { + total_cost: number; + mean_cost: number | null; + cost_per_kloc: number | null; + mean_duration: number | null; + median_duration: number | null; + mean_prompt_tokens: number | null; + mean_completion_tokens: number | null; + mean_reasoning_tokens: number | null; + mean_total_tokens: number | null; + cache_hit_ratio: number | null; + mean_llm_calls: number | null; + mean_lines_of_code: number | null; + median_output_tps: number | null; + }; + repair_rate: number | null; + continuations_used: number; + engines: Record<string, number>; + providers: Record<string, number>; + terminations: Record<string, number>; + first_job_at: string | null; + last_job_at: string | null; + }; + templates: { + slug: string; + name: string; + jobs: number; + completed: number; + functional: RateWithCI; + }[]; + experiments: { + id: string; + name: string; + status: string; + jobs: number; + created_at: string; + }[]; + analysis: { + analyzed_jobs: number; + findings_static: SeverityCounts; + findings_ai: SeverityCounts; + by_tool: ({ tool: string; ai: boolean; total: number } & SeverityCounts)[]; + top_rules: { + tool: string; + rule_id: string; + title: string; + severity: string; + count: number; + }[]; + }; + ranking: { + scores: Record< + | "composite_score" + | "mss_score" + | "empirical_quality_score" + | "benchmark_score" + | "cost_efficiency_score" + | "accessibility_score" + | "adoption_score", + number | null + >; + positions: Record< + "composite" | "mss" | "empirical" | "benchmark", + RankPosition | null + >; + variance: { + n_jobs: number; + density_per_kloc_mean: number | null; + density_per_kloc_stdev: number | null; + smoke_pass_rate_stdev: number | null; + } | null; + local_loc: number; + n_trials: number; + models_exercised: number; + } | null; + catalog: { + input_price: CatalogPercentile; + output_price: CatalogPercentile; + context_window: CatalogPercentile; + max_output_tokens: CatalogPercentile; + artificial_analysis: Record< + "intelligence_index" | "coding_index" | "agentic_index", + CatalogPercentile + >; + catalog_size: number; + provider_models: number; + }; + benchmarks: { + benchmark: string; + score: number; + source: string; + fetched_at: string; + percentile: number | null; + n: number; + }[]; +} + +export interface ProviderEndpoint { + provider_name: string; + tag: string; + quantization: string; + context_length: number | null; + max_completion_tokens: number | null; + max_prompt_tokens: number | null; + input_price_per_million: number; + output_price_per_million: number; + cache_read_per_million: number | null; + discount: number; + /** OpenRouter endpoint status; 0 = OK, negative = degraded / down. */ + status: number | null; + uptime_last_5m: number | null; + uptime_last_30m: number | null; + uptime_last_1d: number | null; + /** p50 latency (ms) — OpenRouter returns a number or a percentile object. */ + latency_last_30m: number | Record<string, number> | null; + throughput_last_30m: number | Record<string, number> | null; + supported_parameters: string[]; + supports_implicit_caching: boolean; + native_tools: string[]; + service_tier: string | null; +} + +export interface ProviderEndpoints { + available: boolean; + fetched_at: string | null; + endpoints: ProviderEndpoint[]; +} + +/** Platform measurements (caller's jobs), ranking position and catalog percentiles for one model. */ +export async function getModelInsights(slug: string): Promise<ModelInsights> { + const res = await apiFetch( + `/models/detail/${modelPathSegment(slug)}/insights/`, + ); + return res.json(); +} + +/** Providers serving the model on OpenRouter (live; cached 10 min server-side). */ +export async function getModelEndpoints( + slug: string, + refresh = false, +): Promise<ProviderEndpoints> { + const q = refresh ? "?refresh=true" : ""; + const res = await apiFetch( + `/models/detail/${modelPathSegment(slug)}/endpoints/${q}`, + ); + return res.json(); +} diff --git a/frontend/src/lib/api/runs.ts b/frontend/src/lib/api/runs.ts index d51539ac..c7ef7803 100644 --- a/frontend/src/lib/api/runs.ts +++ b/frontend/src/lib/api/runs.ts @@ -1,47 +1,124 @@ -// Client for the container/tool-shop analysis API (`/analysis/runs/`). -// The legacy `analysis.ts` (tasks/profiles/analyzers) targets removed endpoints -// and is kept only for the not-yet-migrated applications/workflow views. +// Client for the analysis runs API (`/analysis/runs/`): runs of the analyzer +// workspace's tools over a generated app (or pasted code), their per-tool +// results and the normalized findings. import { apiFetch } from "./core"; +export type RunStatus = + | "pending" + | "running" + | "completed" + | "partial" + | "failed" + | "cancelled"; + +/** What a running run is doing between tools (`progress` events / `phase`). */ +export type RunPhase = + | "" + | "waiting_for_workspace" + | "preparing" + | "running_tools"; + +export type ToolResultStatus = + | "pending" + | "running" + | "completed" + | "failed" + | "skipped"; + +/** `summary.skip_reason` of a skipped tool result. Only `not_applicable` is a measurement. */ +export type SkipReason = + | "not_applicable" + | "app_not_running" + | "unavailable" + | "cancelled"; + +export interface RunTarget { + /** "job": a generated app; "code": pasted files; "deleted": the app was deleted since. */ + kind: "job" | "code" | "deleted"; + label: string; + job_id?: string | null; + job_status?: string; + mode?: string; + model_name?: string; + model_id?: string; + app_name?: string; + app_slug?: string; + stack_slug?: string; + experiment_id?: string | null; + file_count?: number | null; +} + +/** Live counters of an active (pending/running) run. */ +export interface RunProgress { + total: number; + done: number; + running: string[]; + findings: number; +} + export interface RunToolResult { id: string; tool_slug: string; category: string; - status: string; - // e.g. { severity_counts: { high: 2, ... }, total: 3 } + status: ToolResultStatus; + /** + * `finding_count`, `severity_counts` (normalized — don't display), `tool_version`, + * `config`, `warnings`, `skip_reason`. + */ summary: Record<string, any>; - // Aggregate tool metrics, e.g. radon's { average_complexity, rank_distribution } + /** Aggregate tool metrics, e.g. radon's { average_complexity, rank_distribution }. */ metrics: Record<string, any>; error_message: string; + started_at: string | null; + duration_seconds: number | null; } -export interface AnalysisRunDetail { +export interface ToolResultDetail extends RunToolResult { + /** `command`, `exit_code`, `stdout` (first 30k chars), `target_url` (dynamic tools). */ + raw_output: Record<string, any>; + tool_name: string; + tool_docs_url: string; + findings_count: number; +} + +/** + * Run summary. Finished runs: `total_findings`, `reported_severity_counts` + * (levels as the tools printed them, case-folded), `unrated_findings`, + * `tools_run` / `tools_completed` / `tools_failed`, `truncated_inputs`, + * `metrics_by_tool` (detail only). `severity_counts` is a normalization kept for + * aggregate statistics — never display it. + */ +export type RunSummary = Record<string, any>; + +export interface AnalysisRunListItem { id: string; name: string; - status: string; + status: RunStatus; tool_slugs: string[]; - // e.g. { total_findings: 5, severity_counts: { high: 2 }, tools_run: 3 } - summary: Record<string, any>; + summary: RunSummary; error_message: string; generation_job_id: string | null; + batch_id: string | null; + target: RunTarget; + created_at: string; started_at: string | null; completed_at: string | null; - created_at: string; - results: RunToolResult[]; + duration_seconds: number | null; + phase: RunPhase; + progress: RunProgress | null; } -export interface AnalysisRunListItem { - id: string; - name: string; - status: string; - tool_slugs: string[]; - summary: Record<string, any>; - created_at: string; +export interface AnalysisRunDetail extends AnalysisRunListItem { + results: RunToolResult[]; + source_files: { path: string; lines: number }[]; } export interface RunFinding { id: string; + /** Normalized level (a heuristic for tools that rate nothing): never display it. */ severity: string; + /** The level exactly as the tool printed it; "" when the tool rates nothing. */ + tool_severity: string; category: string; confidence: string; title: string; @@ -53,61 +130,192 @@ export interface RunFinding { code_snippet: string; rule_id: string; tool_slug: string; + /** Standardized extras (see `FindingDetail`): rule_name, docs_url, cwe, package, url, … */ + tool_specific_data: Record<string, any>; } -export interface PaginatedRuns { - items: AnalysisRunListItem[]; +export interface Paginated<T> { + items: T[]; total: number; page: number; per_page: number; pages: number; } -export interface PaginatedRunFindings { - items: RunFinding[]; +export type PaginatedRuns = Paginated<AnalysisRunListItem>; +export type PaginatedRunFindings = Paginated<RunFinding>; + +export interface FacetValue { + value: string; + count: number; + label?: string; +} + +export interface FindingFacets { total: number; - page: number; - per_page: number; - pages: number; + tools: FacetValue[]; + /** Tool-reported levels (case-folded), most severe first (`rank` 0 = most severe). */ + levels: { value: string; count: number; rank: number }[]; + /** Findings of tools that rate nothing. */ + unrated: number; + categories: FacetValue[]; + files: FacetValue[]; + files_total: number; + rules: FacetValue[]; + rules_total: number; } -export interface CreateRunPayload { - name?: string; - tool_slugs: string[]; - generation_job_id?: string | null; - source_code?: Record<string, string> | null; - auto_start?: boolean; +export interface RunStats { + total: number; + active: number; + by_status: Partial<Record<RunStatus, number>>; +} + +export interface RunPreviewTarget { + job_id: string | null; + label: string; + files: number; + lines: number; + app_running: boolean; +} + +export interface RunPreviewTool { + slug: string; + name: string; + known: boolean; + installed: boolean; + install_status: string; + mode: string; + kind: string; + /** Targets the tool would run on / skip. */ + runs: number; + not_applicable: number; + app_not_running: number; +} + +export interface RunPreview { + workspace_status: string; + targets: RunPreviewTarget[]; + tools: RunPreviewTool[]; + tool_runs: number; + warnings: string[]; } -export async function listRuns(params?: { +export interface ListRunsParams { page?: number; per_page?: number; + /** Comma-separated statuses; "active" = pending + running. */ status?: string; generation_job_id?: string; -}): Promise<PaginatedRuns> { + batch_id?: string; + tool_slug?: string; + search?: string; + sort?: + | "created_desc" + | "created_asc" + | "name_asc" + | "duration_desc" + | "findings_desc"; +} + +export interface FindingsParams { + page?: number; + per_page?: number; + tool_slug?: string; + /** A tool-reported level (case-insensitive) or "unrated". */ + tool_severity?: string; + category?: string; + file_path?: string; + rule_id?: string; + search?: string; + sort?: "severity" | "file" | "tool" | "rule"; +} + +interface RequestOptions { + signal?: AbortSignal; +} + +function query(params: object = {}): string { const sp = new URLSearchParams(); - if (params?.page) sp.set("page", String(params.page)); - if (params?.per_page) sp.set("per_page", String(params.per_page)); - if (params?.status) sp.set("status", params.status); - if (params?.generation_job_id) - sp.set("generation_job_id", params.generation_job_id); + for (const [key, value] of Object.entries(params)) { + if (value === undefined || value === null || value === "") continue; + sp.set(key, String(value)); + } const qs = sp.toString(); - const res = await apiFetch(`/analysis/runs/${qs ? "?" + qs : ""}`); + return qs ? `?${qs}` : ""; +} + +function post(path: string, body: unknown, options?: RequestOptions) { + return apiFetch(path, { + method: "POST", + body: JSON.stringify(body), + signal: options?.signal, + }); +} + +// --- Runs --------------------------------------------------------------------- + +export async function listRuns( + params?: ListRunsParams, + options?: RequestOptions, +): Promise<PaginatedRuns> { + const res = await apiFetch(`/analysis/runs/${query(params)}`, { + signal: options?.signal, + }); + return res.json(); +} + +export async function getRunStats(): Promise<RunStats> { + const res = await apiFetch("/analysis/runs/stats/"); return res.json(); } -export async function getRun(runId: string): Promise<AnalysisRunDetail> { - const res = await apiFetch(`/analysis/runs/${runId}/`); +export async function getRun( + runId: string, + options?: RequestOptions, +): Promise<AnalysisRunDetail> { + const res = await apiFetch(`/analysis/runs/${runId}/`, { + signal: options?.signal, + }); return res.json(); } +export interface CreateRunPayload { + name?: string; + tool_slugs: string[]; + generation_job_id?: string | null; + /** `{path: content}`; the file names decide which tools apply. */ + source_code?: Record<string, string> | null; + auto_start?: boolean; +} + export async function createRun( payload: CreateRunPayload, ): Promise<AnalysisRunDetail> { - const res = await apiFetch("/analysis/runs/", { - method: "POST", - body: JSON.stringify(payload), - }); + const res = await post("/analysis/runs/", payload); + return res.json(); +} + +/** One run per app, queued one after another. */ +export async function createBatch(payload: { + name?: string; + tool_slugs: string[]; + generation_job_ids: string[]; +}): Promise<{ batch_id: string; run_ids: string[] }> { + const res = await post("/analysis/runs/batch/", payload); + return res.json(); +} + +/** Where each tool would run (free; nothing is created). */ +export async function previewRuns( + payload: { + tool_slugs: string[]; + generation_job_ids?: string[]; + source_code?: Record<string, string> | null; + }, + options?: RequestOptions, +): Promise<RunPreview> { + const res = await post("/analysis/runs/preview/", payload, options); return res.json(); } @@ -118,27 +326,82 @@ export async function cancelRun(runId: string): Promise<AnalysisRunDetail> { return res.json(); } +/** A new run on the same code: all tools, `tool_slugs`, or only those that measured nothing. */ +export async function rerunRun( + runId: string, + payload: { + tool_slugs?: string[]; + only_unmeasured?: boolean; + name?: string; + } = {}, +): Promise<AnalysisRunDetail> { + const res = await post(`/analysis/runs/${runId}/rerun/`, payload); + return res.json(); +} + +/** 409 while the run is active (cancel it first). */ export async function deleteRun(runId: string): Promise<void> { await apiFetch(`/analysis/runs/${runId}/`, { method: "DELETE" }); } +/** Cancel or delete several runs; active runs are never deleted (`skipped`). */ +export async function bulkRunAction( + action: "cancel" | "delete", + ids: string[], +): Promise<{ affected: number; skipped: number }> { + const res = await post("/analysis/runs/bulk/", { action, ids }); + return res.json(); +} + +export async function getRunResult( + runId: string, + resultId: string, +): Promise<ToolResultDetail> { + const res = await apiFetch(`/analysis/runs/${runId}/results/${resultId}/`); + return res.json(); +} + +// --- Findings ------------------------------------------------------------------- + export async function getRunFindings( runId: string, - params?: { - page?: number; - per_page?: number; - severity?: string; - tool_slug?: string; - }, + params?: FindingsParams, + options?: RequestOptions, ): Promise<PaginatedRunFindings> { - const sp = new URLSearchParams(); - if (params?.page) sp.set("page", String(params.page)); - if (params?.per_page) sp.set("per_page", String(params.per_page)); - if (params?.severity) sp.set("severity", params.severity); - if (params?.tool_slug) sp.set("tool_slug", params.tool_slug); - const qs = sp.toString(); const res = await apiFetch( - `/analysis/runs/${runId}/findings/${qs ? "?" + qs : ""}`, + `/analysis/runs/${runId}/findings/${query(params)}`, + { signal: options?.signal }, ); return res.json(); } + +export async function getFindingFacets( + runId: string, + params?: { tool_slug?: string }, + options?: RequestOptions, +): Promise<FindingFacets> { + const res = await apiFetch( + `/analysis/runs/${runId}/findings/facets/${query(params)}`, + { signal: options?.signal }, + ); + return res.json(); +} + +/** Every finding matching *params* (pages of 200, at most `limit`), for export. */ +export async function getAllRunFindings( + runId: string, + params: Omit<FindingsParams, "page" | "per_page"> = {}, + limit = 5000, +): Promise<RunFinding[]> { + const all: RunFinding[] = []; + for (let page = 1; all.length < limit; page++) { + const data = await getRunFindings(runId, { + ...params, + page, + per_page: 200, + }); + all.push(...data.items); + if (page >= data.pages) break; + } + return all.slice(0, limit); +} diff --git a/frontend/src/lib/api/runtime.ts b/frontend/src/lib/api/runtime.ts index a082b8cb..06641c4a 100644 --- a/frontend/src/lib/api/runtime.ts +++ b/frontend/src/lib/api/runtime.ts @@ -153,6 +153,8 @@ export async function getContainers( per_page?: number; status?: ContainerStatus; job_id?: string; + /** OpenRouter model id: containers of that model's jobs. */ + model_id?: string; } = {}, ): Promise<PaginatedContainers> { const q = new URLSearchParams(); @@ -160,6 +162,7 @@ export async function getContainers( if (params.per_page) q.set("per_page", String(params.per_page)); if (params.status) q.set("status", params.status); if (params.job_id) q.set("job_id", params.job_id); + if (params.model_id) q.set("model_id", params.model_id); const qs = q.toString(); const res = await apiFetch(`/runtime/containers/${qs ? "?" + qs : ""}`); return res.json(); diff --git a/frontend/src/lib/api/sse.ts b/frontend/src/lib/api/sse.ts index af128cea..6a1ea3ce 100644 --- a/frontend/src/lib/api/sse.ts +++ b/frontend/src/lib/api/sse.ts @@ -2,7 +2,7 @@ * SSE subscription utility for realtime task progress. * * Usage: - * const cleanup = subscribe(['generation:42', 'dashboard'], (e) => { + * const cleanup = subscribe(['generation:42', 'capacity'], (e) => { * console.log(e.type, e.data); * }, { onState: (s) => (live = s === 'open') }); * // later… diff --git a/frontend/src/lib/api/users.ts b/frontend/src/lib/api/users.ts index 932987b8..01ac96b1 100644 --- a/frontend/src/lib/api/users.ts +++ b/frontend/src/lib/api/users.ts @@ -115,3 +115,33 @@ export async function verifyEmail(key: string): Promise<void> { if (isAllauthMutationSuccess(res, body)) return; throw body; } + +export type ActivityKind = "application" | "analysis" | "experiment" | "report"; + +export interface UserActivityItem { + kind: ActivityKind; + id: string; + title: string; + status: string; + created_at: string; + href: string; +} + +export interface UserActivity { + date_joined: string; + last_login: string | null; + counts: { + applications: number; + applications_completed: number; + analyses: number; + experiments: number; + reports: number; + }; + recent: UserActivityItem[]; +} + +/** Counts and most recent work of the signed-in user (profile page). */ +export async function getMyActivity(): Promise<UserActivity> { + const res = await apiFetch("/users/me/activity/"); + return res.json(); +} diff --git a/frontend/src/lib/components/AppFooter.svelte b/frontend/src/lib/components/AppFooter.svelte index 2afc3c3d..8f48108d 100644 --- a/frontend/src/lib/components/AppFooter.svelte +++ b/frontend/src/lib/components/AppFooter.svelte @@ -1,4 +1,22 @@ +<script lang="ts"> + const links = [ + { href: '/about', label: 'About' }, + { href: '/privacy', label: 'Privacy' }, + { href: '/docs', label: 'Docs' }, + ]; +</script> + <footer class="flex flex-wrap items-center justify-between gap-x-3 gap-y-1 border-t border-border/70 px-4 py-3 font-mono text-[11px] text-muted-foreground sm:px-5"> - <p>© {new Date().getFullYear()} LLM Eval Lab</p> + <div class="flex flex-wrap items-center gap-x-3 gap-y-1"> + <p>© {new Date().getFullYear()} LLM Lab</p> + <nav aria-label="Footer" class="flex items-center gap-3"> + {#each links as link (link.href)} + <a + href={link.href} + class="rounded-sm transition-colors duration-150 hover:text-foreground focus-visible:text-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring" + >{link.label}</a> + {/each} + </nav> + </div> <p class="hidden uppercase tracking-[0.12em] sm:block">Research platform for LLM code evaluation</p> </footer> diff --git a/frontend/src/lib/components/AppHeader.svelte b/frontend/src/lib/components/AppHeader.svelte index 5842ddd8..83473ee3 100644 --- a/frontend/src/lib/components/AppHeader.svelte +++ b/frontend/src/lib/components/AppHeader.svelte @@ -6,19 +6,20 @@ import User from '@lucide/svelte/icons/user'; import UserCog from '@lucide/svelte/icons/user-cog'; import Palette from '@lucide/svelte/icons/palette'; - import Cookie from '@lucide/svelte/icons/cookie'; + import Plug from '@lucide/svelte/icons/plug'; import Key from '@lucide/svelte/icons/key'; import LogOut from '@lucide/svelte/icons/log-out'; import X from '@lucide/svelte/icons/x'; + import ChevronRight from '@lucide/svelte/icons/chevron-right'; import SidebarNav from './SidebarNav.svelte'; import BrandMark from './BrandMark.svelte'; - import { routeLabels } from '$lib/nav'; + import { navSectionFor, routeLabels } from '$lib/nav'; + import { pageTitle } from '$lib/stores/page-title.svelte'; import { Separator } from '$lib/components/ui/separator'; import * as Sheet from '$lib/components/ui/sheet'; import ThemeToggle from '$lib/components/ThemeToggle.svelte'; import { getAuth } from '$lib/stores/auth.svelte'; import { getPreferences } from '$lib/stores/preferences.svelte'; - import { cn } from '$lib/utils'; interface AuthState { isAuthenticated: boolean; @@ -64,12 +65,17 @@ segments.push({ label: parts[parts.length - 1], href: pathname }); } + // Detail pages (/applications/<id>) end on the record's own name, published by PageHeader. + const last = segments[segments.length - 1]; + if (pageTitle.current && last.href !== pathname && last.label !== pageTitle.current) { + segments.push({ label: pageTitle.current, href: pathname }); + } + return segments; }); - // A lone breadcrumb repeats the page's <h1>: show it only once the title has scrolled away. - let scrollY = $state(0); - let breadcrumbVisible = $derived(breadcrumbSegments.length > 1 || scrollY > 120); + // Workflow section ("Generate") as a non-link root crumb, so the page itself needs no eyebrow. + let breadcrumbSection = $derived(navSectionFor(page.url?.pathname ?? '/')?.section.title ?? null); let avatarBgClass = $derived(avatarColorClasses[preferences.avatarColor] ?? 'bg-blue-500'); @@ -112,7 +118,7 @@ } </script> -<svelte:window onclick={handleWindowClick} onkeydown={handleKeydown} bind:scrollY /> +<svelte:window onclick={handleWindowClick} onkeydown={handleKeydown} /> <header class="fixed inset-x-0 top-0 z-30 flex h-12 items-center gap-3 border-b border-border bg-background/85 px-3 backdrop-blur-md transition-[padding] duration-200 ease-out motion-reduce:transition-none sm:px-4 md:gap-4 md:px-5 md:pl-[calc(var(--app-sidebar-offset,0px)+1.25rem)]" @@ -151,17 +157,16 @@ </a> {#if breadcrumbSegments.length > 0} - <nav - aria-label="Breadcrumb" - class={cn( - 'hidden min-w-0 transition-[opacity,visibility] duration-200 motion-reduce:transition-none md:block', - !breadcrumbVisible && 'invisible opacity-0' - )} - > - <ol class="flex items-center gap-1.5 text-sm text-muted-foreground"> + <nav aria-label="Breadcrumb" class="hidden min-w-0 md:block"> + <ol class="flex min-w-0 items-center gap-1.5 text-sm text-muted-foreground"> + {#if breadcrumbSection} + <li class="shrink-0">{breadcrumbSection}</li> + {/if} {#each breadcrumbSegments as segment, i (segment.href)} - {#if i > 0} - <li aria-hidden="true" class="font-mono text-muted-foreground/50">/</li> + {#if i > 0 || breadcrumbSection} + <li aria-hidden="true" class="shrink-0 text-muted-foreground/40"> + <ChevronRight class="size-3.5" /> + </li> {/if} <li class="min-w-0 truncate"> {#if i === breadcrumbSegments.length - 1} @@ -244,7 +249,7 @@ <div role="separator" class="my-1 h-px bg-border/60"></div> <p class="px-2 pt-0.5 pb-1 text-[11px] uppercase tracking-[0.14em] text-muted-foreground/70"> - Preferences + Settings </p> <a href="/users/settings#appearance" @@ -256,22 +261,22 @@ Appearance </a> <a - href="/users/settings#privacy" + href="/users/settings#credentials" class="flex cursor-pointer items-center gap-2 rounded px-2.5 py-1.5 text-sm hover:bg-accent hover:text-accent-foreground transition-colors" role="menuitem" onclick={closeDropdown} > - <Cookie class="h-3.5 w-3.5" /> - Privacy & cookies + <Key class="h-3.5 w-3.5" /> + API keys </a> <a - href="/users/settings#credentials" + href="/users/settings#mcp" class="flex cursor-pointer items-center gap-2 rounded px-2.5 py-1.5 text-sm hover:bg-accent hover:text-accent-foreground transition-colors" role="menuitem" onclick={closeDropdown} > - <Key class="h-3.5 w-3.5" /> - API credentials + <Plug class="h-3.5 w-3.5" /> + Integrations </a> <div role="separator" class="my-1 h-px bg-border/60"></div> diff --git a/frontend/src/lib/components/EmptyState.svelte b/frontend/src/lib/components/EmptyState.svelte index b6acde3e..70363c1c 100644 --- a/frontend/src/lib/components/EmptyState.svelte +++ b/frontend/src/lib/components/EmptyState.svelte @@ -17,7 +17,7 @@ let { title, description, icon: Icon = Inbox, action, compact = false, class: className }: Props = $props(); </script> -<div class={cn('empty-state', compact && 'py-6 sm:py-8', className)}> +<div class={cn('empty-state', compact && 'py-4 sm:py-5', className)}> <div class="empty-state-icon" aria-hidden="true"><Icon class="h-5 w-5" /></div> <p class="text-sm font-medium">{title}</p> {#if description} diff --git a/frontend/src/lib/components/FilterBar.svelte b/frontend/src/lib/components/FilterBar.svelte index bd55b1f2..b57cd3cd 100644 --- a/frontend/src/lib/components/FilterBar.svelte +++ b/frontend/src/lib/components/FilterBar.svelte @@ -39,6 +39,11 @@ actions?: Snippet; /** Additional classes on the outer wrapper */ class?: string; + /** + * Render as the top strip of the table card it filters (no own border/shadow). + * The results/pagination row is dropped: the table toolbar and footer carry it. + */ + embedded?: boolean; } let { @@ -54,6 +59,7 @@ filters, actions, class: cls = '', + embedded = false, }: Props = $props(); let filtersExpanded = $state(false); @@ -87,7 +93,7 @@ }} /> -<div class="fb-shell {cls}" role="search" aria-label="Filter and search"> +<div class="fb-shell {embedded ? 'fb-embedded' : ''} {cls}" role="search" aria-label="Filter and search"> <!-- ── Row 1: Search + filter toggle + actions ─────────────── --> <div class="flex items-center gap-2 p-2"> <!-- Search --> @@ -194,11 +200,13 @@ {/if} <!-- ── Row 4: Results + pagination bar ───────────────────────── --> - <PaginationBar - {resultsText} - {page} - {pages} - {onGoToPage} - class="border-t border-border" - /> + {#if !embedded} + <PaginationBar + {resultsText} + {page} + {pages} + {onGoToPage} + class="border-t border-border" + /> + {/if} </div> diff --git a/frontend/src/lib/components/PageHeader.svelte b/frontend/src/lib/components/PageHeader.svelte index e1192815..23be5a4a 100644 --- a/frontend/src/lib/components/PageHeader.svelte +++ b/frontend/src/lib/components/PageHeader.svelte @@ -1,13 +1,14 @@ <script lang="ts"> import type { Snippet } from 'svelte'; + import { Popover } from 'bits-ui'; import ArrowLeft from '@lucide/svelte/icons/arrow-left'; - import { page } from '$app/state'; - import { navSectionFor } from '$lib/nav'; + import CircleHelp from '@lucide/svelte/icons/circle-help'; + import { pageTitle } from '$lib/stores/page-title.svelte'; interface Props { title: string; description?: string; - /** Parent page link shown above the title on detail/create pages. */ + /** Parent page; rendered as a back button before the title on detail/create pages. */ back?: { href: string; label: string }; /** Browser tab title; defaults to `title`. */ documentTitle?: string; @@ -15,13 +16,27 @@ meta?: Snippet; /** Right-aligned page actions (primary action last). */ actions?: Snippet; - /** Extra content under the description (chips, stats line). */ + /** Reference text (glossary, how-it-works) in a popover behind a `?` next to the title. */ + help?: Snippet; + /** A short line under the description (stats chips, IDs); rendered as the card's footer strip. */ children?: Snippet; + /** Visual before the title (avatar, entity icon). */ + leading?: Snippet; + /** + * Page-level navigation or mode switch (a `Tabs.List`, a `Segmented`), flush along the card's + * bottom edge. Wrap the page in `Tabs.Root` so the list and its panels share state. + */ + nav?: Snippet; } - let { title, description, back, documentTitle, meta, actions, children }: Props = $props(); + let { title, description, back, documentTitle, meta, actions, help, children, leading, nav }: Props = $props(); - const owner = $derived(navSectionFor(page.url?.pathname ?? '/')); + $effect(() => { + pageTitle.current = title; + return () => { + if (pageTitle.current === title) pageTitle.current = ''; + }; + }); </script> <svelte:head> @@ -29,31 +44,42 @@ </svelte:head> <header class="page-header"> - {#if back} - <a href={back.href} class="back-link"> - <ArrowLeft class="h-3.5 w-3.5" aria-hidden="true" /> - {back.label} - </a> - {:else if owner} - <p class="page-eyebrow"> - <span class="tabular-nums text-primary">{String(owner.index).padStart(2, '0')}</span> - <span aria-hidden="true">·</span> - {owner.section.title} - </p> - {/if} - <div class="flex flex-col gap-3 sm:flex-row sm:items-start sm:justify-between"> - <div class="min-w-0 space-y-1"> - <div class="flex min-w-0 flex-wrap items-center gap-2"> - <h1 class="min-w-0 break-words">{title}</h1> + <div class="page-header-main"> + {#if back} + <a href={back.href} class="page-header-back" aria-label="Back to {back.label}" title="Back to {back.label}"> + <ArrowLeft class="size-4" aria-hidden="true" /> + </a> + {/if} + {@render leading?.()} + <div class="page-header-body"> + <div class="page-header-titlebar"> + <h1>{title}</h1> + {#if help} + <Popover.Root> + <Popover.Trigger class="page-header-help-trigger" aria-label="About this page"> + <CircleHelp class="size-4" aria-hidden="true" /> + </Popover.Trigger> + <Popover.Portal> + <Popover.Content side="bottom" align="start" sideOffset={6} collisionPadding={12} class="page-header-help"> + {@render help()} + </Popover.Content> + </Popover.Portal> + </Popover.Root> + {/if} {@render meta?.()} </div> {#if description} - <p class="max-w-3xl">{description}</p> + <p class="page-header-desc">{description}</p> {/if} - {@render children?.()} </div> {#if actions} - <div class="flex shrink-0 flex-wrap items-center gap-2">{@render actions()}</div> + <div class="page-header-actions">{@render actions()}</div> {/if} </div> + {#if children} + <div class="page-header-extra">{@render children()}</div> + {/if} + {#if nav} + <div class="page-header-nav">{@render nav()}</div> + {/if} </header> diff --git a/frontend/src/lib/components/SectionHeading.svelte b/frontend/src/lib/components/SectionHeading.svelte new file mode 100644 index 00000000..b35aebed --- /dev/null +++ b/frontend/src/lib/components/SectionHeading.svelte @@ -0,0 +1,47 @@ +<script lang="ts"> + /** + * SectionHeading — the one labelled divider of the app: an optional mono + * number, the title, then an ink rule running to the right edge (or to the + * count / aside), and an optional lead flush below. + * + * 2.1 Design ──────────────────────────────── [aside] + * One sentence on what the section holds. + * + * Sizes: + * - `lg` — a section of the page on the canvas (serif title), e.g. the + * experiment page's "2.1 Design" (`StudySection` wraps it). + * - `sm` — a group label over a run of cards, or between groups inside a + * card (mono caps, the sidebar's group motif), e.g. Settings' "Account". + * + * Props: `id` (the heading's id, for `aria-labelledby`), `title`, `number`, + * `size` ('lg'), `level` (2), `count` (trailing figure, "4 pages"), `lead`, + * `aside` snippet (controls at the end of the line), `class`. + */ + import type { Snippet } from 'svelte'; + import { cn } from '$lib/utils'; + + interface Props { + id?: string; + title: string; + number?: string | number; + size?: 'lg' | 'sm'; + level?: 2 | 3 | 4; + count?: string | number; + lead?: string; + aside?: Snippet; + class?: string; + } + + let { id, title, number, size = 'lg', level = 2, count, lead, aside, class: className }: Props = $props(); +</script> + +<div class={cn('section-heading', className)} data-size={size}> + <div class="section-heading-line"> + {#if number != null}<span class="section-heading-num" aria-hidden="true">{number}</span>{/if} + <svelte:element this={`h${level}`} {id} class="section-heading-title">{title}</svelte:element> + <span class="section-heading-rule" aria-hidden="true"></span> + {#if count != null}<span class="section-heading-count">{count}</span>{/if} + {#if aside}<div class="section-heading-aside">{@render aside()}</div>{/if} + </div> + {#if lead}<p class="section-heading-lead">{lead}</p>{/if} +</div> diff --git a/frontend/src/lib/components/analysis/AnalysisTargetForm.svelte b/frontend/src/lib/components/analysis/AnalysisTargetForm.svelte deleted file mode 100644 index e1687f47..00000000 --- a/frontend/src/lib/components/analysis/AnalysisTargetForm.svelte +++ /dev/null @@ -1,183 +0,0 @@ -<script lang="ts"> -import * as Card from '$lib/components/ui/card'; -import { Button } from '$lib/components/ui/button'; -import Search from '@lucide/svelte/icons/search'; -import FileCode from '@lucide/svelte/icons/file-code'; -import ClipboardPaste from '@lucide/svelte/icons/clipboard-paste'; -import LoadingState from '$lib/components/LoadingState.svelte'; -import ErrorAlert from '$lib/components/ErrorAlert.svelte'; -import EmptyState from '$lib/components/EmptyState.svelte'; -import type { GenerationJobList } from '$lib/api/client'; -import { tableFrame, shortModelName, modelNamePrefix } from '$lib/components/table'; - -type SourceMode = 'job' | 'paste'; - -interface Props { - sourceMode: SourceMode; - jobsLoading: boolean; - jobsError: string; - jobs: GenerationJobList[]; - filteredJobs: GenerationJobList[]; - selectedJobId: string | null; - jobSearch: string; - pasteBackend: string; - pasteFrontend: string; - onSourceModeChange: (mode: SourceMode) => void; - onSelectJob: (id: string) => void; - onRetryLoadJobs: () => void; -} - -let { - sourceMode, - jobsLoading, - jobsError, - jobs, - filteredJobs, - selectedJobId, - jobSearch = $bindable(), - pasteBackend = $bindable(), - pasteFrontend = $bindable(), - onSourceModeChange, - onSelectJob, - onRetryLoadJobs, -}: Props = $props(); -</script> - -<Card.Root> - <Card.Header> - <Card.Title>Select Source</Card.Title> - <Card.Description>Choose a completed generation job or paste code directly.</Card.Description> - </Card.Header> - <Card.Content> - <div class="mb-4 flex gap-2"> - <Button - variant={sourceMode === 'job' ? 'default' : 'outline'} - size="sm" - aria-pressed={sourceMode === 'job'} - onclick={() => onSourceModeChange('job')} - > - <FileCode class="mr-1.5 h-3.5 w-3.5" /> - Generation Job - </Button> - <Button - variant={sourceMode === 'paste' ? 'default' : 'outline'} - size="sm" - aria-pressed={sourceMode === 'paste'} - onclick={() => onSourceModeChange('paste')} - > - <ClipboardPaste class="mr-1.5 h-3.5 w-3.5" /> - Paste Code - </Button> - </div> - - {#if sourceMode === 'job'} - <div class="mb-4"> - <div class="relative max-w-sm"> - <Search class="absolute left-3 top-1/2 h-4 w-4 -translate-y-1/2 text-muted-foreground" /> - <input - type="text" - placeholder="Search jobs..." - aria-label="Search generation jobs" - class="h-9 w-full rounded-md border border-input bg-background pl-9 pr-3 text-sm" - bind:value={jobSearch} - /> - </div> - </div> - - {#if jobsLoading} - <LoadingState label="loading jobs" /> - {:else if jobsError} - <ErrorAlert message={jobsError} onretry={onRetryLoadJobs} /> - {:else if filteredJobs.length === 0} - <EmptyState - compact - icon={FileCode} - title={jobs.length === 0 ? 'No completed generation jobs found' : 'No jobs match your search'} - description={jobs.length === 0 ? 'Generate an app first, or paste code directly.' : undefined} - > - {#snippet action()} - {#if jobs.length === 0} - <Button variant="outline" size="sm" onclick={() => onSourceModeChange('paste')}> - <ClipboardPaste class="mr-1.5 h-3.5 w-3.5" /> - Paste code instead - </Button> - {:else} - <Button variant="outline" size="sm" onclick={() => (jobSearch = '')}>Clear search</Button> - {/if} - {/snippet} - </EmptyState> - {:else} - <div class="table-frame rounded-md border border-border" {@attach tableFrame}> - <table> - <thead> - <tr> - <th>Job ID</th> - <th>Model</th> - <th>Template</th> - <th>Completed</th> - <th><span class="sr-only">Select</span></th> - </tr> - </thead> - <tbody> - {#each filteredJobs as job} - {@const selected = selectedJobId === job.id} - {@const provider = modelNamePrefix(job.model_name) || job.model_id_str?.split('/')[0] || ''} - <tr data-selected={selected}> - <td class="whitespace-nowrap font-mono text-xs">{job.id.slice(0, 8)}…</td> - <td class="max-w-0 w-[35%]"> - {#if job.model_name} - <div class="min-w-0" title={job.model_name}> - <span class="block truncate">{shortModelName(job.model_name)}</span> - {#if provider}<span class="cell-sub block truncate text-[11px] text-muted-foreground">{provider}</span>{/if} - </div> - {:else} - <span class="text-muted-foreground">—</span> - {/if} - </td> - <td> - {#if job.template_name}{job.template_name}{:else}<span class="text-muted-foreground">—</span>{/if} - </td> - <td class="whitespace-nowrap text-xs text-muted-foreground"> - {job.completed_at ? new Date(job.completed_at).toLocaleDateString() : '—'} - </td> - <td class="w-px whitespace-nowrap text-right"> - <Button - variant={selected ? 'default' : 'ghost'} - size="sm" - class="h-7 px-2 text-xs" - aria-pressed={selected} - onclick={() => onSelectJob(job.id)} - > - {selected ? 'Selected' : 'Select'} - </Button> - </td> - </tr> - {/each} - </tbody> - </table> - </div> - {/if} - {:else} - <div class="space-y-4"> - <div> - <label for="paste-backend" class="mb-1 block text-sm font-medium">Backend Code</label> - <textarea - id="paste-backend" - class="h-40 w-full rounded-md border border-input bg-background p-3 font-mono text-sm" - placeholder="Paste backend code here…" - bind:value={pasteBackend} - ></textarea> - </div> - <div> - <label for="paste-frontend" class="mb-1 block text-sm font-medium">Frontend Code</label> - <textarea - id="paste-frontend" - class="h-40 w-full rounded-md border border-input bg-background p-3 font-mono text-sm" - placeholder="Paste frontend code here…" - bind:value={pasteFrontend} - ></textarea> - </div> - </div> - {/if} - </Card.Content> -</Card.Root> diff --git a/frontend/src/lib/components/analysis/AppPicker.svelte b/frontend/src/lib/components/analysis/AppPicker.svelte new file mode 100644 index 00000000..859dfadf --- /dev/null +++ b/frontend/src/lib/components/analysis/AppPicker.svelte @@ -0,0 +1,280 @@ +<script lang="ts" module> + /** A generated app picked for analysis (what the summary and chips need). */ + export interface PickedApp { + id: string; + app: string; + model: string; + stack: string; + created_at: string | null; + } + + export const MAX_APPS = 100; + + export function toPicked(j: { + id: string; + template_name?: string | null; + model_name?: string | null; + stack_slug?: string | null; + created_at?: string | null; + }): PickedApp { + return { + id: j.id, + app: j.template_name ?? '', + model: j.model_name ?? '', + stack: j.stack_slug ?? '', + created_at: j.created_at ?? null, + }; + } +</script> + +<script lang="ts"> + /** + * AppPicker — choose completed generated apps to analyze: server-side search + * (app, model, stack, ID), "app running" filter (dynamic tools need it), + * paging, select page / clear, and removable chips for the selection (kept + * across pages and searches). Each row shows whether the app's container is + * running. + * + * Props: `selected` (bindable `PickedApp[]`, replaced on change), `running` + * (bindable output: job ids whose container runs, for the summary). + */ + import { onMount, untrack } from 'svelte'; + import { Input } from '$lib/components/ui/input'; + import { Checkbox } from '$lib/components/ui/checkbox'; + import { Segmented } from '$lib/components/ui/segmented'; + import { Button } from '$lib/components/ui/button'; + import Search from '@lucide/svelte/icons/search'; + import X from '@lucide/svelte/icons/x'; + import AppWindow from '@lucide/svelte/icons/app-window'; + import EmptyState from '$lib/components/EmptyState.svelte'; + import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import LoadingState from '$lib/components/LoadingState.svelte'; + import PaginationBar from '$lib/components/PaginationBar.svelte'; + import { modelNamePrefix, shortModelName, tableFrame } from '$lib/components/table'; + import { formatApiError, isAbortError } from '$lib/api/core'; + import { getGenerationJobs, type GenerationJobList } from '$lib/api/generation'; + import { getContainers } from '$lib/api/runtime'; + import { formatDate } from '$lib/utils/formatters'; + import { cn } from '$lib/utils'; + + interface Props { + selected: PickedApp[]; + running?: Set<string>; + } + + let { selected = $bindable(), running = $bindable(new Set()) }: Props = $props(); + + const PER_PAGE = 10; + let items = $state<GenerationJobList[]>([]); + let total = $state(0); + let pages = $state(1); + let pageNo = $state(1); + let loading = $state(true); + let error = $state(''); + let search = $state(''); + let searchText = $state(''); + let scope = $state<'all' | 'running'>('all'); + let ctl: AbortController | null = null; + + const selectedIds = $derived(new Set(selected.map((a) => a.id))); + + async function load() { + ctl?.abort(); + const mine = new AbortController(); + ctl = mine; + loading = true; + try { + const data = await getGenerationJobs( + { + status: 'completed', + page: pageNo, + per_page: PER_PAGE, + search: search || undefined, + container_status: scope === 'running' ? 'running' : undefined, + sort_by: 'created_desc', + }, + { signal: mine.signal } + ); + items = data.items; + total = data.total; + pages = data.pages; + pageNo = data.page; + error = ''; + } catch (e) { + if (isAbortError(e) && mine.signal.aborted) return; + error = formatApiError(e, 'Could not load generated apps.'); + } finally { + if (ctl === mine) loading = false; + } + } + + async function loadRunning() { + try { + const res = await getContainers({ status: 'running', per_page: 100 }); + running = new Set((res.containers ?? []).map((c) => c.generation_job_id).filter((id): id is string => !!id)); + } catch { + // Only a hint: rows just don't show the running dot. + } + } + + const queryKey = $derived(`${pageNo}|${search}|${scope}`); + $effect(() => { + void queryKey; + untrack(() => void load()); + }); + onMount(() => void loadRunning()); + + let timer: ReturnType<typeof setTimeout> | undefined; + function onSearch(value: string) { + searchText = value; + clearTimeout(timer); + timer = setTimeout(() => { + search = value.trim(); + pageNo = 1; + }, 300); + } + + function toggle(job: GenerationJobList, on: boolean) { + if (on && !selectedIds.has(job.id) && selected.length < MAX_APPS) selected = [...selected, toPicked(job)]; + else if (!on) selected = selected.filter((a) => a.id !== job.id); + } + + const pageAllSelected = $derived(items.length > 0 && items.every((j) => selectedIds.has(j.id))); + const pageSomeSelected = $derived(items.some((j) => selectedIds.has(j.id)) && !pageAllSelected); + + function togglePage(on: boolean) { + if (on) { + const add = items.filter((j) => !selectedIds.has(j.id)).map(toPicked); + selected = [...selected, ...add].slice(0, MAX_APPS); + } else { + const ids = new Set(items.map((j) => j.id)); + selected = selected.filter((a) => !ids.has(a.id)); + } + } + + let showAllChips = $state(false); + const chips = $derived(showAllChips ? selected : selected.slice(0, 12)); +</script> + +<div class="space-y-3"> + <div class="flex flex-wrap items-center gap-2"> + <div class="relative min-w-52 flex-1"> + <Search class="pointer-events-none absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted-foreground" aria-hidden="true" /> + <Input + type="search" + value={searchText} + oninput={(e) => onSearch((e.currentTarget as HTMLInputElement).value)} + placeholder="Search app, model, stack or job ID…" + aria-label="Search generated apps" + class="h-9 pl-8" + /> + </div> + <Segmented + size="sm" + aria-label="Which apps" + value={scope} + onValueChange={(v) => { + scope = v as typeof scope; + pageNo = 1; + }} + items={[ + { value: 'all', label: 'All completed' }, + { value: 'running', label: 'App running', title: 'Apps whose container runs now: dynamic tools (ZAP, Nuclei, Wapiti) can scan them' }, + ]} + /> + </div> + + {#if selected.length > 0} + <div class="flex flex-wrap items-center gap-1.5" aria-label="Selected apps"> + {#each chips as app (app.id)} + <span class="inline-flex max-w-full items-center gap-1 rounded-full border border-primary/30 bg-primary/10 py-0.5 pr-1 pl-2 text-xs text-primary"> + <span class="truncate" title="{app.app || app.stack} · {app.model}">{app.app || app.stack || app.id.slice(0, 8)} · {shortModelName(app.model)}</span> + <button type="button" class="cursor-pointer rounded-full p-0.5 hover:bg-primary/15" aria-label="Remove {app.app || app.id}" onclick={() => (selected = selected.filter((a) => a.id !== app.id))}> + <X class="size-3" /> + </button> + </span> + {/each} + {#if selected.length > 12} + <button type="button" class="cursor-pointer text-xs text-muted-foreground hover:text-foreground" onclick={() => (showAllChips = !showAllChips)}> + {showAllChips ? 'Show fewer' : `+${selected.length - 12} more`} + </button> + {/if} + <button type="button" class="cursor-pointer text-xs text-muted-foreground hover:text-foreground" onclick={() => (selected = [])}>Clear</button> + </div> + {/if} + + {#if error} + <ErrorAlert message={error} onretry={load} /> + {:else if loading && items.length === 0} + <LoadingState label="loading generated apps" /> + {:else if items.length === 0} + <EmptyState + compact + icon={AppWindow} + title={search || scope === 'running' ? 'No apps match' : 'No completed apps yet'} + description={search || scope === 'running' ? undefined : 'Generate an app first, or paste code instead.'} + > + {#snippet action()} + {#if search || scope === 'running'} + <Button variant="outline" size="sm" onclick={() => { search = ''; searchText = ''; scope = 'all'; pageNo = 1; }}>Show all apps</Button> + {:else} + <Button variant="outline" size="sm" href="/sample-generator">Generate an app</Button> + {/if} + {/snippet} + </EmptyState> + {:else} + <div class="overflow-clip rounded-md border border-border"> + <div class={cn('table-frame transition-opacity', loading && 'opacity-60')} {@attach tableFrame}> + <table> + <thead> + <tr> + <th class="w-8"><Checkbox checked={pageAllSelected} indeterminate={pageSomeSelected} onCheckedChange={(v) => togglePage(!!v)} aria-label="Select all apps on this page" /></th> + <th>App</th> + <th>Model</th> + <th>Mode</th> + <th>Created</th> + <th title="Whether the app's container runs now (needed by dynamic tools)">App</th> + </tr> + </thead> + <tbody> + {#each items as job (job.id)} + {@const on = selectedIds.has(job.id)} + {@const provider = modelNamePrefix(job.model_name)} + <!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_noninteractive_element_interactions (the checkbox is the keyboard path) --> + <tr class="row-link" data-selected={on} onclick={(e) => { if (!(e.target as HTMLElement).closest('button, a')) toggle(job, !on); }}> + <td class="w-8"><Checkbox checked={on} onCheckedChange={(v) => toggle(job, !!v)} aria-label="Select {job.template_name ?? job.id}" disabled={!on && selected.length >= MAX_APPS} /></td> + <td class="max-w-0 w-[30%]"> + <span class="block truncate font-medium">{job.template_name ?? job.stack_slug ?? '—'}</span> + <span class="cell-sub block truncate font-mono text-[11px] text-muted-foreground">{job.stack_slug ?? ''} · {job.id.slice(0, 8)}</span> + </td> + <td class="max-w-0 w-[30%]"> + <span class="block truncate" title={job.model_name ?? ''}>{shortModelName(job.model_name) || '—'}</span> + {#if provider}<span class="cell-sub block truncate text-[11px] text-muted-foreground">{provider}</span>{/if} + </td> + <td class="text-xs capitalize text-muted-foreground">{job.mode}</td> + <td class="whitespace-nowrap text-xs text-muted-foreground">{formatDate(job.completed_at ?? job.created_at)}</td> + <td class="whitespace-nowrap text-xs"> + {#if running.has(job.id)} + <span class="inline-flex items-center gap-1 text-success-fg"><span class="status-dot status-dot-live h-1.5 w-1.5" aria-hidden="true"></span>running</span> + {:else} + <span class="text-muted-foreground">stopped</span> + {/if} + </td> + </tr> + {/each} + </tbody> + </table> + </div> + <PaginationBar + resultsText="{total} completed app{total === 1 ? '' : 's'}{selected.length ? ` · ${selected.length} selected` : ''}" + page={pageNo} + {pages} + onGoToPage={(p) => (pageNo = p)} + class="border-t border-border" + /> + </div> + {#if selected.length >= MAX_APPS} + <p class="text-xs text-warning-fg">A batch analyzes at most {MAX_APPS} apps.</p> + {/if} + {/if} +</div> diff --git a/frontend/src/lib/components/analysis/CodeFilesEditor.svelte b/frontend/src/lib/components/analysis/CodeFilesEditor.svelte new file mode 100644 index 00000000..87675038 --- /dev/null +++ b/frontend/src/lib/components/analysis/CodeFilesEditor.svelte @@ -0,0 +1,89 @@ +<script lang="ts" module> + export interface CodeFile { + key: number; + path: string; + content: string; + } + + let nextKey = 1; + export function newFile(path = '', content = ''): CodeFile { + return { key: nextKey++, path, content }; + } + + /** `{path: content}` of the non-empty files (what the API takes). */ + export function toSourceCode(files: CodeFile[]): Record<string, string> { + const out: Record<string, string> = {}; + for (const f of files) { + const path = f.path.trim().replace(/^\/+/, ''); + if (path && f.content.trim()) out[path] = f.content; + } + return out; + } +</script> + +<script lang="ts"> + /** + * CodeFilesEditor — pasted code as named files. File names matter: they + * decide which tools apply (pip-audit reads requirements.txt, hadolint a + * Dockerfile, ESLint *.js / *.jsx …). + * + * Props: `files` (bindable, replaced on change). + */ + import { Input } from '$lib/components/ui/input'; + import { Button } from '$lib/components/ui/button'; + import Plus from '@lucide/svelte/icons/plus'; + import Trash2 from '@lucide/svelte/icons/trash-2'; + + let { files = $bindable() }: { files: CodeFile[] } = $props(); + + const duplicates = $derived.by(() => { + const seen = new Map<string, number>(); + for (const f of files) { + const p = f.path.trim(); + if (p) seen.set(p, (seen.get(p) ?? 0) + 1); + } + return new Set([...seen].filter(([, n]) => n > 1).map(([p]) => p)); + }); + + function update(key: number, patch: Partial<CodeFile>) { + files = files.map((f) => (f.key === key ? { ...f, ...patch } : f)); + } +</script> + +<div class="space-y-3"> + {#each files as file, i (file.key)} + {@const dup = duplicates.has(file.path.trim())} + <div class="overflow-hidden rounded-md border border-border"> + <div class="flex items-center gap-2 border-b border-border bg-muted/40 px-2 py-1.5"> + <Input + value={file.path} + oninput={(e) => update(file.key, { path: (e.currentTarget as HTMLInputElement).value })} + placeholder="path/to/file.py" + aria-label="File {i + 1} path" + aria-invalid={dup || (!file.path.trim() && !!file.content.trim())} + class="h-8 flex-1 font-mono text-xs" + /> + <span class="font-mono text-[11px] text-muted-foreground tabular-nums">{file.content ? file.content.split('\n').length : 0} lines</span> + <Button variant="ghost" size="sm" class="size-8 p-0" disabled={files.length === 1} onclick={() => (files = files.filter((f) => f.key !== file.key))} aria-label="Remove file {file.path || i + 1}"> + <Trash2 class="size-4" /> + </Button> + </div> + <textarea + value={file.content} + oninput={(e) => update(file.key, { content: (e.currentTarget as HTMLTextAreaElement).value })} + class="block h-48 w-full resize-y bg-background p-3 font-mono text-xs leading-5 focus-visible:outline-none" + placeholder="Paste the file's content…" + spellcheck="false" + aria-label="File {i + 1} content" + ></textarea> + {#if dup}<p class="border-t border-border px-3 py-1 text-xs text-danger-fg">Two files share this path.</p>{/if} + </div> + {/each} + <div class="flex flex-wrap items-center gap-3"> + <Button variant="outline" size="sm" onclick={() => (files = [...files, newFile()])}><Plus />Add file</Button> + <p class="text-xs text-muted-foreground"> + File names decide which tools apply — e.g. <span class="font-mono">requirements.txt</span> for pip-audit, + <span class="font-mono">Dockerfile</span> for hadolint, <span class="font-mono">*.jsx</span> for ESLint. + </p> + </div> +</div> diff --git a/frontend/src/lib/components/analysis/FindingDetail.svelte b/frontend/src/lib/components/analysis/FindingDetail.svelte new file mode 100644 index 00000000..7ff6d431 --- /dev/null +++ b/frontend/src/lib/components/analysis/FindingDetail.svelte @@ -0,0 +1,251 @@ +<script lang="ts"> + /** + * FindingDetail — everything a tool said about one finding: what and why + * (description), how to fix it (the tool's suggestion / autofix), where (code, + * location, duplicate), the rule (name, docs, CWE / OWASP / WSTG, references), + * and the domain facts parsers extract — vulnerable package and fixed versions, + * the HTTP request a scanner flagged, complexity — plus any other tool data. + * Only what the tool reported is shown; nothing is rated here. + * + * Props: `finding`, `toolName`. + */ + import type { RunFinding } from '$lib/api/runs'; + import ExternalLink from '@lucide/svelte/icons/external-link'; + import Copy from '@lucide/svelte/icons/copy'; + import { toast } from 'svelte-sonner'; + import LevelBadge from './LevelBadge.svelte'; + import { + advisoryUrl, + complexityInfo, + cweUrl, + dependencyInfo, + duplicateInfo, + httpInfo, + isPrivateUrl, + locationText, + otherData, + ruleInfo, + safeUrl, + } from './finding-data'; + + interface Props { + finding: RunFinding; + toolName: string; + } + + let { finding: f, toolName }: Props = $props(); + + const rule = $derived(ruleInfo(f)); + const dep = $derived(dependencyInfo(f)); + const http = $derived(httpInfo(f)); + const dup = $derived(duplicateInfo(f)); + const cx = $derived(complexityInfo(f)); + const other = $derived(otherData(f)); + const location = $derived(locationText(f)); + const description = $derived(f.description && f.description.trim() !== f.title.trim() ? f.description.trim() : ''); + // Some tools put only a link in the description. + const descriptionUrl = $derived(safeUrl(description)); + const isDependency = $derived(!!dep && (dep.ids.length > 0 || dep.installed !== '' || dep.fixedIn.length > 0)); + const hasMain = $derived(!!(descriptionUrl || description || f.suggestion || rule.fix || f.code_snippet || dup || http)); + + async function copy(text: string, what: string) { + try { + await navigator.clipboard.writeText(text); + toast.success(`${what} copied`); + } catch { + toast.error('Copy failed'); + } + } +</script> + +{#snippet link(href: string, text: string)} + <a {href} target="_blank" rel="noopener noreferrer" class="inline-flex items-center gap-1 text-primary underline-offset-2 hover:underline"> + {text}<ExternalLink class="size-3 shrink-0" aria-hidden="true" /> + </a> +{/snippet} + +{#snippet chipLink(id: string, href: string | null)} + {#if href} + <a {href} target="_blank" rel="noopener noreferrer" class="rounded border border-border bg-muted px-1.5 py-px font-mono text-[11px] text-foreground hover:border-primary/50 hover:text-primary">{id}</a> + {:else} + <span class="rounded border border-border bg-muted px-1.5 py-px font-mono text-[11px]">{id}</span> + {/if} +{/snippet} + +<div class={hasMain ? 'grid gap-x-6 gap-y-4 text-sm @3xl:grid-cols-[minmax(0,1fr)_minmax(16rem,22rem)]' : 'text-sm'}> + <!-- What / how to fix / where --> + <div class={hasMain ? 'min-w-0 space-y-3' : 'hidden'}> + {#if descriptionUrl} + <p>{@render link(descriptionUrl, 'Advisory')}</p> + {:else if description} + <p class="whitespace-pre-line text-foreground/90">{description}</p> + {/if} + + {#if f.suggestion || rule.fix} + <div class="rounded-md border border-success/30 bg-success/5 px-3 py-2"> + <p class="section-label mb-1">Suggested fix (from the tool)</p> + {#if f.suggestion}<p class="whitespace-pre-line">{f.suggestion}</p>{/if} + {#if rule.fix}<pre class="mt-1 overflow-x-auto rounded bg-muted px-2 py-1 font-mono text-xs">{rule.fix}</pre>{/if} + </div> + {/if} + + {#if f.code_snippet} + <div> + <p class="section-label mb-1">Code{location ? ` · ${location}` : ''}</p> + <pre class="max-h-64 overflow-auto rounded-md border border-border bg-muted/50 px-3 py-2 font-mono text-xs leading-5">{f.code_snippet}</pre> + </div> + {/if} + + {#if dup} + <p class="text-xs text-muted-foreground"> + Same code also in <span class="font-mono text-foreground">{dup.file}{dup.startLine != null ? `:${dup.startLine}` : ''}{dup.endLine != null ? `–${dup.endLine}` : ''}</span> + {#if dup.lines != null}· {dup.lines} lines{/if} + {#if dup.tokens != null}· {dup.tokens} tokens{/if} + </p> + {/if} + + {#if http} + <div class="space-y-1.5 rounded-md border border-border px-3 py-2"> + <p class="section-label">Request the scanner flagged</p> + {#if http.url} + <p class="break-all font-mono text-xs"> + {#if http.method}<span class="mr-1.5 font-semibold">{http.method}</span>{/if}{http.url} + {#if !isPrivateUrl(http.url) && safeUrl(http.url)}<span class="ml-2 font-sans">{@render link(safeUrl(http.url)!, 'Open')}</span>{/if} + </p> + {/if} + <dl class="grid grid-cols-[auto_minmax(0,1fr)] gap-x-3 gap-y-1 text-xs"> + {#if http.param}<dt class="text-muted-foreground">Parameter</dt><dd class="font-mono">{http.param}</dd>{/if} + {#if http.attack}<dt class="text-muted-foreground">Attack</dt><dd class="break-all font-mono">{http.attack}</dd>{/if} + {#if http.evidence}<dt class="text-muted-foreground">Evidence</dt><dd class="break-all font-mono">{http.evidence}</dd>{/if} + {#if http.matcher}<dt class="text-muted-foreground">Matched</dt><dd class="font-mono">{http.matcher}</dd>{/if} + {#if http.instances != null}<dt class="text-muted-foreground">Instances</dt><dd class="font-mono tabular-nums">{http.instances}</dd>{/if} + </dl> + {#if http.otherUrls.length > 0} + <details class="text-xs"> + <summary class="cursor-pointer text-muted-foreground hover:text-foreground">{http.otherUrls.length} more URL{http.otherUrls.length === 1 ? '' : 's'}</summary> + <ul class="mt-1 max-h-40 space-y-0.5 overflow-auto font-mono"> + {#each http.otherUrls as u (u)}<li class="break-all">{u}</li>{/each} + </ul> + </details> + {/if} + {#if http.otherInfo}<p class="whitespace-pre-line text-xs text-muted-foreground">{http.otherInfo}</p>{/if} + {#if http.curl} + <div class="flex items-start gap-2"> + <pre class="min-w-0 flex-1 overflow-x-auto rounded bg-muted px-2 py-1 font-mono text-xs">{http.curl}</pre> + <button type="button" class="cursor-pointer rounded p-1 text-muted-foreground hover:bg-muted hover:text-foreground" title="Copy command" aria-label="Copy reproduction command" onclick={() => copy(http.curl, 'Command')}> + <Copy class="size-3.5" /> + </button> + </div> + {/if} + </div> + {/if} + </div> + + <!-- Facts --> + <dl class="grid h-fit min-w-0 max-w-2xl grid-cols-[7rem_minmax(0,1fr)] gap-x-3 gap-y-1.5 text-xs"> + <dt class="text-muted-foreground">Tool</dt> + <dd>{toolName}</dd> + + <dt class="text-muted-foreground">Level</dt> + <dd><LevelBadge label={f.tool_severity} fallback="Not rated by this tool" /></dd> + + {#if rule.toolConfidence} + <dt class="text-muted-foreground">Confidence</dt> + <dd class="font-mono">{rule.toolConfidence}</dd> + {/if} + + {#if f.rule_id || rule.name} + <dt class="text-muted-foreground">Rule</dt> + <dd class="min-w-0 space-y-0.5"> + {#if f.rule_id}<span class="block break-all font-mono">{f.rule_id}{#if rule.messageId && rule.messageId !== f.rule_id}<span class="text-muted-foreground"> ({rule.messageId})</span>{/if}</span>{/if} + {#if rule.name && rule.name !== f.rule_id}<span class="block text-muted-foreground">{rule.name}</span>{/if} + {#if rule.docsUrl}{@render link(rule.docsUrl, 'Rule documentation')}{/if} + </dd> + {/if} + + {#if location && !f.code_snippet} + <dt class="text-muted-foreground">Location</dt> + <dd class="break-all font-mono">{location}</dd> + {/if} + + {#if f.category} + <dt class="text-muted-foreground">Category</dt> + <dd>{f.category}</dd> + {/if} + + {#if rule.cwe.length || rule.owasp.length || rule.wstg.length} + <dt class="text-muted-foreground">Weakness</dt> + <dd class="flex flex-wrap gap-1"> + {#each rule.cwe as id (id)}{@render chipLink(id, cweUrl(id))}{/each} + {#each rule.owasp as id (id)}{@render chipLink(id, null)}{/each} + {#each rule.wstg as id (id)}{@render chipLink(id, null)}{/each} + </dd> + {/if} + + {#if rule.likelihood || rule.impact} + <dt class="text-muted-foreground">Likelihood / impact</dt> + <dd class="font-mono">{rule.likelihood || '—'} / {rule.impact || '—'}</dd> + {/if} + + {#if rule.secretType} + <dt class="text-muted-foreground">Secret type</dt> + <dd>{rule.secretType}{#if rule.entropy != null}<span class="text-muted-foreground"> · entropy {rule.entropy.toFixed(2)}</span>{/if}</dd> + {/if} + + {#if dep} + <dt class="text-muted-foreground">{isDependency ? 'Package' : 'Module'}</dt> + <dd class="font-mono"> + {dep.package}{#if dep.installed}<span class="text-muted-foreground"> {dep.installed}</span>{/if} + {#if dep.ecosystem}<span class="ml-1 font-sans text-muted-foreground">({dep.ecosystem})</span>{/if} + {#if dep.direct != null}<span class="ml-1 font-sans text-muted-foreground">· {dep.direct ? 'direct' : 'transitive'}</span>{/if} + </dd> + {#if dep.fixedIn.length > 0} + <dt class="text-muted-foreground">Fixed in</dt> + <dd class="font-mono text-success-fg">{dep.fixedIn.join(', ')}</dd> + {:else if dep.ids.length > 0} + <dt class="text-muted-foreground">Fixed in</dt> + <dd class="text-muted-foreground">No fixed version reported</dd> + {/if} + {#if dep.cvssScore != null || dep.cvssVector} + <dt class="text-muted-foreground">CVSS</dt> + <dd class="break-all font-mono">{dep.cvssScore ?? ''}{#if dep.cvssVector}<span class="block text-muted-foreground">{dep.cvssVector}</span>{/if}</dd> + {/if} + {#if dep.manifest} + <dt class="text-muted-foreground">Manifest</dt> + <dd class="break-all font-mono">{dep.manifest}</dd> + {/if} + {#if dep.ids.length > 0} + <dt class="text-muted-foreground">Advisories</dt> + <dd class="flex flex-wrap gap-1"> + {#each dep.ids as id (id)}{@render chipLink(id, advisoryUrl(id))}{/each} + </dd> + {/if} + {/if} + + {#if cx} + {#if cx.fn}<dt class="text-muted-foreground">Function</dt><dd class="break-all font-mono">{cx.fn}</dd>{/if} + {#if cx.complexity != null}<dt class="text-muted-foreground">Complexity</dt><dd class="font-mono tabular-nums">{cx.complexity}{#if cx.rank}<span class="text-muted-foreground"> · rank {cx.rank}</span>{/if}</dd>{/if} + {#if cx.nloc != null}<dt class="text-muted-foreground">Lines of code</dt><dd class="font-mono tabular-nums">{cx.nloc}</dd>{/if} + {/if} + + {#if rule.tags.length > 0} + <dt class="text-muted-foreground">Tags</dt> + <dd class="flex flex-wrap gap-1">{#each rule.tags as t (t)}<span class="rounded bg-muted px-1.5 py-px text-[11px]">{t}</span>{/each}</dd> + {/if} + + {#if rule.references.length > 0} + <dt class="text-muted-foreground">References</dt> + <dd class="min-w-0 space-y-0.5"> + {#each rule.references.slice(0, 8) as url (url)} + <a href={url} target="_blank" rel="noopener noreferrer" class="block truncate text-primary underline-offset-2 hover:underline" title={url}>{url.replace(/^https?:\/\//, '')}</a> + {/each} + {#if rule.references.length > 8}<span class="text-muted-foreground">+{rule.references.length - 8} more</span>{/if} + </dd> + {/if} + + {#each other as [key, value] (key)} + <dt class="capitalize text-muted-foreground">{key}</dt> + <dd class="break-all font-mono">{value}</dd> + {/each} + </dl> +</div> diff --git a/frontend/src/lib/components/analysis/FindingsExplorer.svelte b/frontend/src/lib/components/analysis/FindingsExplorer.svelte new file mode 100644 index 00000000..1d43c300 --- /dev/null +++ b/frontend/src/lib/components/analysis/FindingsExplorer.svelte @@ -0,0 +1,527 @@ +<script lang="ts" module> + export interface FindingFilters { + tool: string; + /** A tool-reported level (case-folded) or "unrated". */ + level: string; + category: string; + file: string; + rule: string; + search: string; + sort: 'severity' | 'file' | 'tool' | 'rule'; + } + + export const EMPTY_FILTERS: FindingFilters = { + tool: '', + level: '', + category: '', + file: '', + rule: '', + search: '', + sort: 'severity', + }; +</script> + +<script lang="ts"> + /** + * FindingsExplorer — a run's findings: facet rail (tool, level as the tool + * reported it, category, file, rule — with counts), search, sort, a table + * whose rows expand into `FindingDetail`, pagination and CSV/JSON export of + * the current filter. + * + * Props: `runId`, `tools` (slug → catalog tool, for names/icons), `filters` + * (bindable; the page keeps it in the URL), `refreshKey` (bump to refetch, e.g. + * when a tool finishes during a live run), `live` (a run still adding findings). + */ + import { untrack } from 'svelte'; + import * as Card from '$lib/components/ui/card'; + import { Button } from '$lib/components/ui/button'; + import { Input } from '$lib/components/ui/input'; + import * as Select from '$lib/components/ui/select'; + import Search from '@lucide/svelte/icons/search'; + import X from '@lucide/svelte/icons/x'; + import Download from '@lucide/svelte/icons/download'; + import ChevronRight from '@lucide/svelte/icons/chevron-right'; + import SearchX from '@lucide/svelte/icons/search-x'; + import CircleCheck from '@lucide/svelte/icons/circle-check'; + import SlidersHorizontal from '@lucide/svelte/icons/sliders-horizontal'; + import LoaderCircle from '@lucide/svelte/icons/loader-circle'; + import { toast } from 'svelte-sonner'; + import EmptyState from '$lib/components/EmptyState.svelte'; + import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import LoadingState from '$lib/components/LoadingState.svelte'; + import PaginationBar from '$lib/components/PaginationBar.svelte'; + import ToolIcon from '$lib/components/analyzers/ToolIcon.svelte'; + import { TableToolbar, tableFrame } from '$lib/components/table'; + import { formatApiError, isAbortError } from '$lib/api/core'; + import { + getAllRunFindings, + getFindingFacets, + getRunFindings, + type FindingFacets, + type FindingsParams, + type RunFinding, + } from '$lib/api/runs'; + import type { AnalyzerTool } from '$lib/api/analyzers'; + import { cn } from '$lib/utils'; + import LevelBadge from './LevelBadge.svelte'; + import FindingDetail from './FindingDetail.svelte'; + import { locationText } from './finding-data'; + import { downloadText, toCsv } from './format'; + + interface Props { + runId: string; + tools: Map<string, AnalyzerTool>; + filters: FindingFilters; + refreshKey?: number; + live?: boolean; + } + + let { runId, tools, filters = $bindable(), refreshKey = 0, live = false }: Props = $props(); + + const PER_PAGE = 50; + let page = $state(1); + let findings = $state<RunFinding[]>([]); + let total = $state(0); + let pages = $state(1); + let loading = $state(true); + let error = $state(''); + let facets = $state<FindingFacets | null>(null); + let expanded = $state(new Set<string>()); + let searchText = $state(untrack(() => filters.search)); + let exporting = $state(false); + let showFacetsMobile = $state(false); + + let findingsCtl: AbortController | null = null; + let facetsCtl: AbortController | null = null; + + function toolName(slug: string): string { + return tools.get(slug)?.name ?? slug; + } + + function params(): FindingsParams { + return { + tool_slug: filters.tool, + tool_severity: filters.level, + category: filters.category, + file_path: filters.file, + rule_id: filters.rule, + search: filters.search, + sort: filters.sort, + }; + } + + async function loadFindings(quiet = false) { + findingsCtl?.abort(); + const ctl = new AbortController(); + findingsCtl = ctl; + if (!quiet) loading = true; + try { + const data = await getRunFindings(runId, { ...params(), page, per_page: PER_PAGE }, { signal: ctl.signal }); + findings = data.items; + total = data.total; + pages = data.pages; + page = data.page; + error = ''; + } catch (e) { + if (isAbortError(e) && ctl.signal.aborted) return; + error = formatApiError(e, 'Could not load findings.'); + } finally { + if (findingsCtl === ctl) loading = false; + } + } + + async function loadFacets() { + facetsCtl?.abort(); + const ctl = new AbortController(); + facetsCtl = ctl; + try { + facets = await getFindingFacets(runId, { tool_slug: filters.tool }, { signal: ctl.signal }); + } catch (e) { + if (isAbortError(e) && ctl.signal.aborted) return; + // Facets are a convenience: the table still works without them. + } + } + + // Filters (and the page) drive the table; the tool filter and refreshKey drive the facets. + const filterKey = $derived(JSON.stringify(params())); + let lastFilterKey = ''; + $effect(() => { + const key = filterKey; + void runId; + untrack(() => { + if (key !== lastFilterKey) { + lastFilterKey = key; + page = 1; + expanded = new Set(); + } + void loadFindings(); + }); + }); + $effect(() => { + void filters.tool; + void runId; + untrack(() => void loadFacets()); + }); + // Live runs: refresh quietly (keep the current page and expanded rows). + let lastRefresh = untrack(() => refreshKey); + $effect(() => { + const key = refreshKey; + untrack(() => { + if (key === lastRefresh) return; + lastRefresh = key; + void loadFindings(true); + void loadFacets(); + }); + }); + + // Debounced search box → filters.search. + let searchTimer: ReturnType<typeof setTimeout> | undefined; + function onSearchInput(value: string) { + searchText = value; + clearTimeout(searchTimer); + searchTimer = setTimeout(() => { + if (filters.search !== value.trim()) filters = { ...filters, search: value.trim() }; + }, 300); + } + $effect(() => { + // External changes (Overview links, clear all) update the box. + const s = filters.search; + untrack(() => { + if (searchText.trim() !== s) searchText = s; + }); + }); + + function setFilter<K extends keyof FindingFilters>(key: K, value: FindingFilters[K]) { + const next = { ...filters, [key]: filters[key] === value ? '' : value }; + // Levels, files and rules are counted within the selected tool. + if (key === 'tool') { + next.level = ''; + next.file = ''; + next.rule = ''; + next.category = ''; + } + filters = next; + } + + function clearAll() { + filters = { ...EMPTY_FILTERS, sort: filters.sort }; + } + + function goToPage(p: number) { + if (p < 1 || p > pages) return; + page = p; + expanded = new Set(); + void loadFindings(); + } + + function toggle(id: string) { + const next = new Set(expanded); + if (next.has(id)) next.delete(id); + else next.add(id); + expanded = next; + } + + const activeChips = $derived( + [ + filters.tool && { key: 'tool' as const, label: `Tool: ${toolName(filters.tool)}` }, + filters.level && { + key: 'level' as const, + label: filters.level === 'unrated' ? 'Not rated by tool' : `Level: ${filters.level}`, + }, + filters.category && { key: 'category' as const, label: `Category: ${filters.category}` }, + filters.file && { key: 'file' as const, label: `File: ${filters.file}` }, + filters.rule && { key: 'rule' as const, label: `Rule: ${filters.rule}` }, + filters.search && { key: 'search' as const, label: `“${filters.search}”` }, + ].filter(Boolean) as { key: keyof FindingFilters; label: string }[] + ); + + const resultsText = $derived( + total === 0 ? '' : `${(page - 1) * PER_PAGE + 1}–${Math.min(page * PER_PAGE, total)} of ${total}` + ); + + const sortItems = [ + { value: 'severity', label: 'Level (as reported)' }, + { value: 'file', label: 'File' }, + { value: 'tool', label: 'Tool' }, + { value: 'rule', label: 'Rule' }, + ]; + + async function exportAs(format: 'csv' | 'json') { + exporting = true; + try { + const rows = await getAllRunFindings(runId, params()); + const stamp = runId.slice(0, 8); + if (format === 'json') { + // The normalized severity is a heuristic for unrated tools: export what the tools said. + const data = rows.map(({ severity: _severity, ...rest }) => rest); + downloadText(`findings-${stamp}.json`, JSON.stringify(data, null, 2), 'application/json'); + } else { + const header = ['tool', 'level_reported', 'rule', 'title', 'file', 'line', 'column', 'category', 'description', 'suggestion', 'cwe', 'package', 'installed_version', 'fixed_versions', 'docs_url']; + const body = rows.map((f) => { + const d = f.tool_specific_data ?? {}; + const fixed = d.fixed_versions ?? d.fix_versions ?? d.fixed_version ?? ''; + return [ + f.tool_slug, + f.tool_severity, + f.rule_id, + f.title, + f.file_path, + f.line_number, + f.column_number, + f.category, + f.description, + f.suggestion, + Array.isArray(d.cwe) ? d.cwe.join(' ') : (d.cwe ?? ''), + d.package ?? '', + d.installed_version ?? d.version ?? '', + Array.isArray(fixed) ? fixed.join(' ') : fixed, + d.docs_url ?? '', + ]; + }); + downloadText(`findings-${stamp}.csv`, toCsv([header, ...body]), 'text/csv'); + } + toast.success(`Exported ${rows.length} finding${rows.length === 1 ? '' : 's'}`); + } catch (e) { + toast.error(formatApiError(e, 'Export failed.')); + } finally { + exporting = false; + } + } +</script> + +{#snippet facetButton(active: boolean, onclick: () => void, count: number, children: import('svelte').Snippet, title = '')} + <button + type="button" + class={cn( + 'flex w-full cursor-pointer items-center gap-2 rounded px-2 py-1 text-left text-xs transition-colors duration-150 focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:outline-none', + active ? 'bg-primary/10 font-medium text-primary' : 'text-foreground/90 hover:bg-muted' + )} + aria-pressed={active} + {title} + {onclick} + > + <span class="min-w-0 flex-1">{@render children()}</span> + <span class="shrink-0 font-mono text-[11px] tabular-nums text-muted-foreground">{count}</span> + </button> +{/snippet} + +{#snippet facetRail()} + {#if facets} + <div class="space-y-4"> + <div> + <p class="section-label mb-1 px-2">Tool</p> + {#each facets.tools as t (t.value)} + {#snippet label()} + <span class="flex min-w-0 items-center gap-1.5"> + <ToolIcon tool={tools.get(t.value) ?? { slug: t.value }} size="xs" /> + <span class="truncate">{toolName(t.value)}</span> + </span> + {/snippet} + {@render facetButton(filters.tool === t.value, () => setFilter('tool', t.value), t.count, label)} + {/each} + </div> + + {#if facets.levels.length > 0 || facets.unrated > 0} + <div> + <p class="section-label mb-1 px-2" title="Levels exactly as the tools printed them">Level (as reported)</p> + {#each facets.levels as l (l.value)} + {#snippet label()}<LevelBadge label={l.value} />{/snippet} + {@render facetButton(filters.level === l.value, () => setFilter('level', l.value), l.count, label)} + {/each} + {#if facets.unrated > 0} + {#snippet unratedLabel()}<span class="text-muted-foreground">Not rated by tool</span>{/snippet} + {@render facetButton(filters.level === 'unrated', () => setFilter('level', 'unrated'), facets.unrated, unratedLabel, 'Findings of tools that print no severity')} + {/if} + </div> + {/if} + + {#if facets.categories.length > 1} + <div> + <p class="section-label mb-1 px-2">Category</p> + {#each facets.categories as c (c.value)} + {#snippet label()}<span class="truncate capitalize">{c.value.replaceAll('_', ' ')}</span>{/snippet} + {@render facetButton(filters.category === c.value, () => setFilter('category', c.value), c.count, label)} + {/each} + </div> + {/if} + + {#if facets.files.length > 0} + <div> + <p class="section-label mb-1 px-2">File</p> + <div class="max-h-64 overflow-y-auto"> + {#each facets.files as file (file.value)} + {#snippet label()}<span class="block truncate font-mono" dir="rtl" title={file.value}>‎{file.value}</span>{/snippet} + {@render facetButton(filters.file === file.value, () => setFilter('file', file.value), file.count, label, file.value)} + {/each} + </div> + {#if facets.files_total > facets.files.length} + <p class="px-2 pt-1 text-[11px] text-muted-foreground">+{facets.files_total - facets.files.length} more files — search to find them</p> + {/if} + </div> + {/if} + + {#if facets.rules.length > 0} + <div> + <p class="section-label mb-1 px-2">Rule</p> + <div class="max-h-64 overflow-y-auto"> + {#each facets.rules as r (r.value)} + {#snippet label()} + <span class="block truncate font-mono" title={r.label ? `${r.value} — ${r.label}` : r.value}>{r.value}</span> + {/snippet} + {@render facetButton(filters.rule === r.value, () => setFilter('rule', r.value), r.count, label)} + {/each} + </div> + {#if facets.rules_total > facets.rules.length} + <p class="px-2 pt-1 text-[11px] text-muted-foreground">+{facets.rules_total - facets.rules.length} more rules</p> + {/if} + </div> + {/if} + </div> + {:else} + <LoadingState label="loading filters" class="py-6" /> + {/if} +{/snippet} + +<div class="grid items-start gap-4 lg:grid-cols-[15rem_minmax(0,1fr)]"> + <aside class="hidden lg:sticky lg:top-[4.5rem] lg:block lg:max-h-[calc(100dvh-5.5rem)] lg:overflow-y-auto lg:overscroll-contain" aria-label="Filter findings"> + <Card.Root class="py-3"> + <Card.Content class="px-2">{@render facetRail()}</Card.Content> + </Card.Root> + </aside> + + <Card.Root class="min-w-0 gap-0 overflow-clip py-0"> + <div class="flex flex-wrap items-center gap-2 border-b border-border px-3 py-2"> + <div class="relative min-w-48 flex-1"> + <Search class="pointer-events-none absolute top-1/2 left-2.5 size-3.5 -translate-y-1/2 text-muted-foreground" aria-hidden="true" /> + <Input + value={searchText} + oninput={(e) => onSearchInput((e.currentTarget as HTMLInputElement).value)} + placeholder="Search title, file, rule, code…" + aria-label="Search findings" + class="h-8 pl-8 text-sm" + /> + </div> + <Button variant="outline" size="sm" class="lg:hidden" aria-expanded={showFacetsMobile} onclick={() => (showFacetsMobile = !showFacetsMobile)}> + <SlidersHorizontal class="size-3.5" />Filters + </Button> + <div class="w-44"> + <Select.Simple + bind:value={() => filters.sort, (v) => (filters = { ...filters, sort: (v || 'severity') as FindingFilters['sort'] })} + items={sortItems} + size="sm" + aria-label="Sort findings" + /> + </div> + <Button variant="outline" size="sm" disabled={exporting || total === 0} onclick={() => exportAs('csv')} title="Download the filtered findings as CSV"> + {#if exporting}<LoaderCircle class="size-3.5 animate-spin motion-reduce:animate-none" />{:else}<Download class="size-3.5" />{/if}CSV + </Button> + <Button variant="outline" size="sm" disabled={exporting || total === 0} onclick={() => exportAs('json')} title="Download the filtered findings as JSON">JSON</Button> + </div> + + {#if showFacetsMobile} + <div class="border-b border-border p-2 lg:hidden">{@render facetRail()}</div> + {/if} + + {#if activeChips.length > 0} + <div class="flex flex-wrap items-center gap-1.5 border-b border-border px-3 py-1.5"> + {#each activeChips as chip (chip.key)} + <button + type="button" + class="inline-flex max-w-full cursor-pointer items-center gap-1 rounded-full border border-primary/30 bg-primary/10 px-2 py-0.5 text-xs text-primary hover:bg-primary/15" + onclick={() => (filters = { ...filters, [chip.key]: '' })} + aria-label="Remove filter {chip.label}" + > + <span class="truncate">{chip.label}</span><X class="size-3 shrink-0" /> + </button> + {/each} + <button type="button" class="cursor-pointer text-xs text-muted-foreground hover:text-foreground" onclick={clearAll}>Clear all</button> + </div> + {/if} + + <TableToolbar> + {#snippet summary()} + {#if resultsText}{resultsText} findings · click a row for details{:else if !loading}No findings{/if} + {#if live}<span class="text-info-fg"> · more arrive as tools finish</span>{/if} + {/snippet} + </TableToolbar> + + {#if error} + <div class="p-3"><ErrorAlert message={error} onretry={() => loadFindings()} /></div> + {:else if loading && findings.length === 0} + <LoadingState label="loading findings" /> + {:else if findings.length === 0} + {#if activeChips.length > 0} + <EmptyState compact icon={SearchX} title="No findings match these filters"> + {#snippet action()}<Button variant="outline" size="sm" onclick={clearAll}>Clear filters</Button>{/snippet} + </EmptyState> + {:else if live} + <EmptyState compact title="No findings yet" description="Findings appear here as each tool finishes." /> + {:else} + <EmptyState compact icon={CircleCheck} title="No findings" description="The tools that ran reported nothing for this code. Check the Tools tab for tools that failed or were skipped." /> + {/if} + {:else} + <div class={cn('table-frame transition-opacity', loading && 'opacity-60')} {@attach tableFrame}> + <table> + <thead> + <tr> + <th class="w-8"><span class="sr-only">Expand</span></th> + <th>Finding</th> + <th>Location</th> + <th>Tool</th> + </tr> + </thead> + <tbody> + {#each findings as f (f.id)} + {@const open = expanded.has(f.id)} + <!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_noninteractive_element_interactions (the expand button is the keyboard path) --> + <tr class="row-link align-top" data-selected={open} onclick={() => toggle(f.id)}> + <td class="w-8 pr-0"> + <button + type="button" + class="cursor-pointer rounded p-0.5 text-muted-foreground hover:text-foreground focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:outline-none" + aria-expanded={open} + aria-controls="finding-{f.id}" + aria-label="{open ? 'Hide' : 'Show'} details of {f.title}" + onclick={(e) => { e.stopPropagation(); toggle(f.id); }} + > + <ChevronRight class={cn('size-4 transition-transform duration-150', open && 'rotate-90')} /> + </button> + </td> + <td class="max-w-0 w-[62%] min-w-[16rem]"> + <div class="flex items-start gap-2"> + <LevelBadge label={f.tool_severity} class="mt-0.5" /> + <div class="min-w-0"> + <p class="font-medium leading-5 [overflow-wrap:anywhere]">{f.title}</p> + {#if f.rule_id}<p class="cell-sub truncate font-mono text-[11px] text-muted-foreground">{f.rule_id}</p>{/if} + </div> + </div> + </td> + <td class="max-w-0 w-[22%] font-mono text-xs text-muted-foreground"> + {#if f.file_path}<span class="block truncate" title={locationText(f)} dir="rtl">‎{locationText(f)}</span>{:else}—{/if} + </td> + <td class="whitespace-nowrap text-xs"> + <span class="inline-flex items-center gap-1.5"> + <ToolIcon tool={tools.get(f.tool_slug) ?? { slug: f.tool_slug }} size="xs" /> + {toolName(f.tool_slug)} + </span> + </td> + </tr> + {#if open} + <tr class="bg-muted/20" id="finding-{f.id}"> + <td></td> + <td colspan="3" class="py-3"> + <!-- width 0 / min-width 100%: long code lines scroll inside instead of widening the table --> + <div class="@container w-0 min-w-full"> + <FindingDetail finding={f} toolName={toolName(f.tool_slug)} /> + </div> + </td> + </tr> + {/if} + {/each} + </tbody> + </table> + </div> + <PaginationBar page={page} pages={pages} onGoToPage={goToPage} class="border-t border-border" scrollToCard /> + {/if} + </Card.Root> +</div> diff --git a/frontend/src/lib/components/analysis/LevelBadge.svelte b/frontend/src/lib/components/analysis/LevelBadge.svelte new file mode 100644 index 00000000..8220f544 --- /dev/null +++ b/frontend/src/lib/components/analysis/LevelBadge.svelte @@ -0,0 +1,35 @@ +<script lang="ts"> + /** + * LevelBadge — a finding's level exactly as its tool printed it ("HIGH", + * "error", "convention"). Tools that rate nothing get no badge (or the + * `fallback` text): the normalized severity is never shown. + * + * Props: `label` (`tool_severity`), `fallback` (shown when empty, muted), `class`. + */ + import { cn } from '$lib/utils'; + import { levelClasses } from './format'; + + interface Props { + label: string | null | undefined; + fallback?: string; + class?: string; + } + + let { label, fallback = '', class: className }: Props = $props(); + const text = $derived((label ?? '').trim()); +</script> + +{#if text} + <span + class={cn( + 'inline-flex shrink-0 items-center whitespace-nowrap rounded border px-1.5 py-px font-mono text-[11px] leading-4', + levelClasses(text), + className + )} + title="Level reported by the tool" + > + {text} + </span> +{:else if fallback} + <span class={cn('text-[11px] text-muted-foreground', className)} title="This tool doesn't rate its findings">{fallback}</span> +{/if} diff --git a/frontend/src/lib/components/analysis/MetricsPanel.svelte b/frontend/src/lib/components/analysis/MetricsPanel.svelte new file mode 100644 index 00000000..320f0919 --- /dev/null +++ b/frontend/src/lib/components/analysis/MetricsPanel.svelte @@ -0,0 +1,132 @@ +<script lang="ts"> + /** + * MetricsPanel — aggregate metrics of metric-shaped tools (complexity, + * duplication, …), one block per tool: scalar values as figures, count maps + * (rank distributions) as a compact row, lists (top functions, clones) as a + * short table. Generic over keys; known keys get readable labels. + * + * Props: `metrics` (tool slug → metrics), `tools` (slug → catalog tool). + */ + import type { AnalyzerTool } from '$lib/api/analyzers'; + import ToolIcon from '$lib/components/analyzers/ToolIcon.svelte'; + + interface Props { + metrics: Record<string, Record<string, any>>; + tools: Map<string, AnalyzerTool>; + } + + let { metrics, tools }: Props = $props(); + + const LABELS: Record<string, string> = { + duplication_percentage: 'Duplicated lines %', + token_duplication_percentage: 'Duplicated tokens %', + duplicated_lines: 'Duplicated lines', + duplicated_tokens: 'Duplicated tokens', + // jscpd counts a file once per language in it (JSX + JS + CSS…). + total_lines: 'Lines (per language)', + total_sources: 'Sources (per language)', + clones: 'Clones', + average_complexity: 'Avg complexity', + max_complexity: 'Max complexity', + total_blocks: 'Blocks', + rank_distribution: 'Complexity ranks', + top_functions: 'Most complex functions', + avg_ccn: 'Avg CCN', + max_ccn: 'Max CCN', + max_ccn_function: 'Most complex', + functions: 'Functions', + total_nloc: 'Lines of code (NLOC)', + functions_over_15: 'Functions with CCN > 15', + }; + + function label(key: string): string { + return LABELS[key] ?? key.replaceAll('_', ' ').replace(/^./, (c) => c.toUpperCase()); + } + + function fmt(value: number): string { + return Number.isInteger(value) ? value.toLocaleString() : value.toFixed(2); + } + + function isCountMap(v: unknown): v is Record<string, number> { + return !!v && typeof v === 'object' && !Array.isArray(v) && Object.values(v).every((n) => typeof n === 'number'); + } + + function isRowList(v: unknown): v is Record<string, unknown>[] { + return Array.isArray(v) && v.length > 0 && v.every((r) => r && typeof r === 'object' && !Array.isArray(r)); + } + + function columns(rows: Record<string, unknown>[]): string[] { + const keys = [...new Set(rows.flatMap((r) => Object.keys(r)))]; + // Merge file + line into one location column. + return keys.filter((k) => k !== 'line' && k !== 'type').slice(0, 5); + } + + function cell(row: Record<string, unknown>, key: string): string { + const v = row[key]; + if (key === 'file' && typeof v === 'string') { + const path = v.replace(/^\/work\//, ''); + return row.line != null ? `${path}:${row.line}` : path; + } + if (typeof v === 'number') return fmt(v); + return v == null ? '—' : String(v); + } + + const blocks = $derived( + Object.entries(metrics) + .filter(([, m]) => m && Object.keys(m).length > 0) + .map(([slug, m]) => { + const entries = Object.entries(m); + return { + slug, + scalars: entries.filter(([, v]) => typeof v === 'number' || (typeof v === 'string' && v.length < 80)) as [string, number | string][], + maps: entries.filter(([, v]) => isCountMap(v)) as [string, Record<string, number>][], + lists: entries.filter(([, v]) => isRowList(v)) as [string, Record<string, unknown>[]][], + }; + }) + ); +</script> + +<div class="grid items-start gap-4 @3xl:grid-cols-2"> + {#each blocks as block (block.slug)} + {@const tool = tools.get(block.slug)} + <section class="min-w-0 rounded-md border border-border p-3" aria-label="{tool?.name ?? block.slug} metrics"> + <h3 class="mb-2 flex items-center gap-2 text-sm font-medium"> + <ToolIcon tool={tool ?? { slug: block.slug }} size="xs" />{tool?.name ?? block.slug} + </h3> + {#if block.scalars.length > 0} + <dl class="grid grid-cols-2 gap-x-4 gap-y-2 sm:grid-cols-3"> + {#each block.scalars as [key, value] (key)} + <div class="min-w-0"> + <dt class="truncate text-[11px] text-muted-foreground" title={label(key)}>{label(key)}</dt> + <dd class="truncate font-mono text-sm font-medium tabular-nums" title={String(value)}>{typeof value === 'number' ? fmt(value) : value}</dd> + </div> + {/each} + </dl> + {/if} + {#each block.maps as [key, map] (key)} + <div class="mt-3"> + <p class="text-[11px] text-muted-foreground">{label(key)}</p> + <p class="flex flex-wrap gap-x-3 font-mono text-xs tabular-nums"> + {#each Object.entries(map) as [k, n] (k)}<span><span class="text-muted-foreground">{k}</span> {n}</span>{/each} + </p> + </div> + {/each} + {#each block.lists as [key, rows] (key)} + {@const cols = columns(rows)} + <div class="mt-3"> + <p class="mb-1 text-[11px] text-muted-foreground">{label(key)}{rows.length > 5 ? ` (top 5 of ${rows.length})` : ''}</p> + <div class="overflow-x-auto"> + <table class="text-xs"> + <thead><tr>{#each cols as c (c)}<th class={typeof rows[0][c] === 'number' ? 'num' : ''}>{c.replaceAll('_', ' ')}</th>{/each}</tr></thead> + <tbody> + {#each rows.slice(0, 5) as row, i (i)} + <tr>{#each cols as c (c)}<td class={typeof row[c] === 'number' ? 'num' : 'font-mono'}>{cell(row, c)}</td>{/each}</tr> + {/each} + </tbody> + </table> + </div> + </div> + {/each} + </section> + {/each} +</div> diff --git a/frontend/src/lib/components/analysis/RunFindingsCell.svelte b/frontend/src/lib/components/analysis/RunFindingsCell.svelte new file mode 100644 index 00000000..55c91500 --- /dev/null +++ b/frontend/src/lib/components/analysis/RunFindingsCell.svelte @@ -0,0 +1,32 @@ +<script lang="ts"> + /** + * RunFindingsCell — a run's findings for tables: the total (so far, while + * active) and up to three levels exactly as the tools reported them; findings + * of tools that rate nothing are left out of the chips, never rated here. + * + * Props: `run` (list item). + */ + import type { AnalysisRunListItem } from '$lib/api/runs'; + import LevelBadge from './LevelBadge.svelte'; + import { isActive, reportedLevels } from './format'; + + let { run }: { run: AnalysisRunListItem } = $props(); + + const active = $derived(isActive(run.status)); + const total = $derived( + active ? (run.progress?.findings ?? 0) : (run.summary?.total_findings as number | undefined) + ); + const levels = $derived(reportedLevels(run.summary).slice(0, 3)); +</script> + +{#if total == null} + <span class="text-muted-foreground">—</span> +{:else} + <div class="flex flex-wrap items-center justify-end gap-1.5"> + <span class="font-mono tabular-nums">{total}</span> + {#if active}<span class="text-[11px] text-muted-foreground">so far</span>{/if} + {#each levels as [label, n] (label)} + <span class="cell-sub inline-flex items-center gap-0.5"><LevelBadge {label} /><span class="font-mono text-[11px] text-muted-foreground tabular-nums">{n}</span></span> + {/each} + </div> +{/if} diff --git a/frontend/src/lib/components/analysis/RunLiveStrip.svelte b/frontend/src/lib/components/analysis/RunLiveStrip.svelte new file mode 100644 index 00000000..e58a601f --- /dev/null +++ b/frontend/src/lib/components/analysis/RunLiveStrip.svelte @@ -0,0 +1,79 @@ +<script lang="ts"> + /** + * RunLiveStrip — an active run's progress: one segment per tool in execution + * order (filled when done, tinted by outcome, pulsing while running), what the + * run is doing right now (queued, waiting for the workspace, preparing, which + * tool and for how long), counts so far and the live-connection state. + * + * Props: `run`, `results` (by slug), `tools` (names), `now` (ms), `connection`. + */ + import type { AnalyzerTool } from '$lib/api/analyzers'; + import type { AnalysisRunDetail, RunToolResult } from '$lib/api/runs'; + import type { Tone } from '$lib/constants/colors'; + import { toneFillClasses } from '$lib/constants/colors'; + import { cn } from '$lib/utils'; + import type { WatchConnection } from './live.svelte'; + import { elapsedSince, formatClock, phaseLabel, resultState } from './format'; + + interface Props { + run: AnalysisRunDetail; + results: Map<string, RunToolResult>; + tools: Map<string, AnalyzerTool>; + now: number; + connection: WatchConnection; + } + + let { run, results, tools, now, connection }: Props = $props(); + + const segments = $derived( + run.tool_slugs.map((slug) => ({ slug, result: results.get(slug), state: resultState(results.get(slug)) })) + ); + const done = $derived(segments.filter((s) => s.result && !s.state.live && s.result.status !== 'pending').length); + const running = $derived(segments.find((s) => s.state.live)); + const findings = $derived(run.progress?.findings ?? 0); + + const headline = $derived.by(() => { + if (run.status === 'pending') return 'Queued — runs of your workspace execute one at a time'; + if (running) return `Running ${tools.get(running.slug)?.name ?? running.slug}`; + if (run.phase && run.phase !== 'running_tools') return phaseLabel(run.phase); + return done === run.tool_slugs.length ? 'Finishing up' : 'Starting the next tool'; + }); + + const connectionText = $derived( + connection === 'open' + ? { text: 'Live', cls: 'text-success-fg' } + : connection === 'reconnecting' + ? { text: 'Reconnecting — polling every 3 s', cls: 'text-warning-fg' } + : { text: 'Connecting…', cls: 'text-muted-foreground' } + ); + + function fill(tone: Tone, live: boolean, has: boolean): string { + if (!has) return 'bg-transparent'; + if (live) return 'bg-info animate-pulse motion-reduce:animate-none'; + return toneFillClasses[tone]; + } +</script> + +<section class="rounded-md border border-info/30 bg-card px-4 py-3 shadow-[var(--shadow-card)]" aria-label="Run progress"> + <div class="flex flex-wrap items-baseline justify-between gap-x-4 gap-y-1"> + <p class="text-sm font-medium" aria-live="polite"> + {headline}{#if running?.result?.started_at}<span class="ml-2 font-mono text-info-fg tabular-nums">{formatClock(elapsedSince(running.result.started_at, now))}</span>{/if} + </p> + <p class="flex flex-wrap items-center gap-x-3 text-xs text-muted-foreground"> + <span><b class="font-mono font-medium text-foreground tabular-nums">{done}</b>/{run.tool_slugs.length} tools</span> + <span><b class="font-mono font-medium text-foreground tabular-nums">{findings}</b> findings so far</span> + {#if run.started_at}<span>elapsed <span class="font-mono tabular-nums">{formatClock(elapsedSince(run.started_at, now))}</span></span>{/if} + <span class={cn('inline-flex items-center gap-1', connectionText.cls)}> + <span class={cn('status-dot h-1.5 w-1.5', connection === 'open' && 'status-dot-live')} aria-hidden="true"></span>{connectionText.text} + </span> + </p> + </div> + <div class="mt-2 flex h-2 gap-0.5" role="progressbar" aria-valuemin={0} aria-valuemax={run.tool_slugs.length} aria-valuenow={done} aria-label="Tools finished"> + {#each segments as s (s.slug)} + <span + class={cn('h-full min-w-1 flex-1 rounded-[1px] border', s.result && s.result.status !== 'pending' ? 'border-transparent' : 'border-border', fill(s.state.tone, s.state.live, !!s.result && s.result.status !== 'pending'))} + title="{tools.get(s.slug)?.name ?? s.slug}: {s.state.label}" + ></span> + {/each} + </div> +</section> diff --git a/frontend/src/lib/components/analysis/RunOverview.svelte b/frontend/src/lib/components/analysis/RunOverview.svelte new file mode 100644 index 00000000..c47b9912 --- /dev/null +++ b/frontend/src/lib/components/analysis/RunOverview.svelte @@ -0,0 +1,260 @@ +<script lang="ts"> + /** + * RunOverview — a run at a glance: what it covered (tools measured / failed / + * skipped, code size, time), what needs attention (failed tools, dynamic tools + * skipped because the app was down, truncated generated code), findings by the + * level each tool reported and by tool, the most affected files and rules, and + * the metric tools' figures. Every count links into the Findings tab. + * + * Props: `run`, `tools`, `results` (by slug), `facets` (null while loading), + * `onfilter(partial filters)`, `ontab(tab)`, `onrerununmeasured()`, `rerunning`. + */ + import type { AnalyzerTool } from '$lib/api/analyzers'; + import type { AnalysisRunDetail, FindingFacets, RunToolResult } from '$lib/api/runs'; + import * as Card from '$lib/components/ui/card'; + import { Button } from '$lib/components/ui/button'; + import TriangleAlert from '@lucide/svelte/icons/triangle-alert'; + import CircleX from '@lucide/svelte/icons/circle-x'; + import Globe from '@lucide/svelte/icons/globe'; + import RotateCcw from '@lucide/svelte/icons/rotate-ccw'; + import LoaderCircle from '@lucide/svelte/icons/loader-circle'; + import ToolIcon from '$lib/components/analyzers/ToolIcon.svelte'; + import LoadingState from '$lib/components/LoadingState.svelte'; + import LevelBadge from './LevelBadge.svelte'; + import MetricsPanel from './MetricsPanel.svelte'; + import type { FindingFilters } from './FindingsExplorer.svelte'; + import { formatSpan, isActive, resultFindingCount, resultState } from './format'; + + interface Props { + run: AnalysisRunDetail; + tools: Map<string, AnalyzerTool>; + results: Map<string, RunToolResult>; + facets: FindingFacets | null; + onfilter: (filters: Partial<FindingFilters>) => void; + ontab: (tab: string) => void; + onrerununmeasured: () => void; + rerunning?: boolean; + } + + let { run, tools, results, facets, onfilter, ontab, onrerununmeasured, rerunning = false }: Props = $props(); + + const states = $derived(run.tool_slugs.map((slug) => ({ slug, result: results.get(slug), state: resultState(results.get(slug)) }))); + const measured = $derived(states.filter((s) => s.state.measured).length); + const failed = $derived(states.filter((s) => s.result?.status === 'failed')); + const appDown = $derived(states.filter((s) => s.result?.summary?.skip_reason === 'app_not_running')); + const notApplicable = $derived(states.filter((s) => s.result?.summary?.skip_reason === 'not_applicable')); + const unmeasured = $derived(states.filter((s) => s.result && !s.state.measured && !s.state.live)); + const active = $derived(isActive(run.status)); + + const total = $derived(facets?.total ?? (run.summary?.total_findings as number | undefined) ?? 0); + const levelMax = $derived(Math.max(1, ...(facets?.levels.map((l) => l.count) ?? [1]), facets?.unrated ?? 0)); + const byTool = $derived( + states + .map((s) => ({ slug: s.slug, count: resultFindingCount(s.result) })) + .filter((t) => t.count > 0) + .sort((a, b) => b.count - a.count) + ); + const toolMax = $derived(Math.max(1, ...byTool.map((t) => t.count))); + const codeLines = $derived(run.source_files.reduce((n, f) => n + f.lines, 0)); + const truncated = $derived((run.summary?.truncated_inputs ?? []) as string[]); + const metrics = $derived((run.summary?.metrics_by_tool ?? {}) as Record<string, Record<string, any>>); + const hasMetrics = $derived(Object.values(metrics).some((m) => m && Object.keys(m).length > 0)); + + function toolName(slug: string) { + return tools.get(slug)?.name ?? slug; + } +</script> + +<div class="space-y-4"> + {#if truncated.length > 0 || failed.length > 0 || appDown.length > 0} + <div class="space-y-2"> + {#if truncated.length > 0} + <div class="flex items-start gap-2 rounded-md border border-warning/30 bg-warning/10 px-3 py-2 text-sm text-warning-fg" role="note"> + <TriangleAlert class="mt-0.5 size-4 shrink-0" /> + <p>The generated {truncated.join(' and ')} code was cut off at the model's token limit: these findings describe an incomplete app.</p> + </div> + {/if} + {#if failed.length > 0} + <div class="flex flex-wrap items-start gap-2 rounded-md border border-destructive/30 bg-destructive/10 px-3 py-2 text-sm text-danger-fg" role="note"> + <CircleX class="mt-0.5 size-4 shrink-0" /> + <p class="min-w-0 flex-1"> + {failed.length} tool{failed.length === 1 ? '' : 's'} failed and measured nothing: {failed.map((f) => toolName(f.slug)).join(', ')}. + Their absence is not a clean result. + </p> + <Button variant="outline" size="sm" class="h-7 text-xs" onclick={() => ontab('tools')}>See why</Button> + </div> + {/if} + {#if appDown.length > 0} + <div class="flex flex-wrap items-start gap-2 rounded-md border border-warning/30 bg-warning/10 px-3 py-2 text-sm text-warning-fg" role="note"> + <Globe class="mt-0.5 size-4 shrink-0" /> + <p class="min-w-0 flex-1"> + {appDown.length} dynamic tool{appDown.length === 1 ? ' was' : 's were'} skipped because the app wasn't running ({appDown.map((f) => toolName(f.slug)).join(', ')}). + {#if run.target.job_id}Start it on the <a class="underline underline-offset-2" href="/applications/{run.target.job_id}">app page</a>, then re-run them.{/if} + </p> + </div> + {/if} + </div> + {/if} + + <div class="grid grid-cols-2 gap-3 md:grid-cols-4"> + <button type="button" class="kpi-card cursor-pointer text-left hover:border-primary/50 focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:outline-none" style="--kpi-accent: var(--chart-1)" onclick={() => onfilter({})}> + <span class="kpi-label">Findings{active ? ' so far' : ''}</span> + <span class="kpi-value">{total}</span> + <span class="text-[11px] text-muted-foreground">{facets?.unrated ? `${facets.unrated} not rated by their tool` : 'open the list'}</span> + </button> + <button type="button" class="kpi-card cursor-pointer text-left hover:border-primary/50 focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:outline-none" style="--kpi-accent: var(--chart-2)" onclick={() => ontab('tools')}> + <span class="kpi-label">Tools measured</span> + <span class="kpi-value">{measured}<span class="text-base text-muted-foreground">/{run.tool_slugs.length}</span></span> + <span class="text-[11px] text-muted-foreground"> + {#if failed.length}{failed.length} failed{:else if notApplicable.length}{notApplicable.length} had nothing to read{:else}all ran{/if} + </span> + </button> + <div class="kpi-card" style="--kpi-accent: var(--chart-3)"> + <span class="kpi-label">Code</span> + {#if run.source_files.length > 0} + <span class="kpi-value">{run.source_files.length}<span class="text-base text-muted-foreground"> files</span></span> + <span class="truncate text-[11px] text-muted-foreground">{codeLines.toLocaleString()} lines{run.target.stack_slug ? ` · ${run.target.stack_slug}` : ''}</span> + {:else} + <span class="kpi-value text-base">{run.target.kind === 'deleted' ? 'App deleted' : '—'}</span> + <span class="truncate text-[11px] text-muted-foreground">{run.target.stack_slug ?? ''}</span> + {/if} + </div> + <div class="kpi-card" style="--kpi-accent: var(--chart-5)"> + <span class="kpi-label">Duration</span> + <span class="kpi-value">{formatSpan(run.duration_seconds)}</span> + <span class="text-[11px] text-muted-foreground">{run.tool_slugs.length} tool{run.tool_slugs.length === 1 ? '' : 's'} in sequence</span> + </div> + </div> + + <div class="grid items-start gap-4 lg:grid-cols-2"> + <Card.Root class="gap-3"> + <Card.Header> + <Card.Title class="text-base">Levels as the tools reported them</Card.Title> + <Card.Description class="text-xs">Each tool's own words; tools that print no level are counted apart, never rated for them.</Card.Description> + </Card.Header> + <Card.Content> + {#if !facets} + <LoadingState label="loading levels" class="py-4" /> + {:else if facets.levels.length === 0 && facets.unrated === 0} + <p class="text-sm text-muted-foreground">No findings.</p> + {:else} + {#if facets.levels.length === 0 && run.summary?.reported_severity_counts === undefined && !active} + <p class="mb-2 text-xs text-muted-foreground">This run was recorded before tools' own levels were stored; re-run it to see them.</p> + {/if} + <ul class="space-y-1.5"> + {#each facets.levels as l (l.value)} + <li> + <button type="button" class="group grid w-full cursor-pointer grid-cols-[7rem_minmax(0,1fr)_3rem] items-center gap-2 text-left" onclick={() => onfilter({ level: l.value })}> + <LevelBadge label={l.value} class="justify-self-start" /> + <span class="h-2 rounded-sm bg-muted"><span class="block h-2 rounded-sm bg-foreground/50 transition-colors group-hover:bg-primary" style="width: {(l.count / levelMax) * 100}%"></span></span> + <span class="text-right font-mono text-xs tabular-nums">{l.count}</span> + </button> + </li> + {/each} + {#if facets.unrated > 0} + <li> + <button type="button" class="group grid w-full cursor-pointer grid-cols-[7rem_minmax(0,1fr)_3rem] items-center gap-2 text-left" onclick={() => onfilter({ level: 'unrated' })}> + <span class="text-xs text-muted-foreground">Not rated</span> + <span class="h-2 rounded-sm bg-muted"><span class="block h-2 rounded-sm bg-muted-foreground/40 transition-colors group-hover:bg-primary" style="width: {(facets.unrated / levelMax) * 100}%"></span></span> + <span class="text-right font-mono text-xs tabular-nums">{facets.unrated}</span> + </button> + </li> + {/if} + </ul> + {/if} + </Card.Content> + </Card.Root> + + <Card.Root class="gap-3"> + <Card.Header> + <Card.Title class="text-base">Findings by tool</Card.Title> + <Card.Description class="text-xs">Tools overlap (e.g. three dependency auditors report the same CVE); compare tools, don't sum them.</Card.Description> + </Card.Header> + <Card.Content> + {#if byTool.length === 0} + <p class="text-sm text-muted-foreground">{active ? 'No findings yet.' : 'No tool reported findings.'}</p> + {:else} + <ul class="space-y-1.5"> + {#each byTool as t (t.slug)} + <li> + <button type="button" class="group grid w-full cursor-pointer grid-cols-[9rem_minmax(0,1fr)_3rem] items-center gap-2 text-left" onclick={() => onfilter({ tool: t.slug })}> + <span class="flex min-w-0 items-center gap-1.5 text-xs"><ToolIcon tool={tools.get(t.slug) ?? { slug: t.slug }} size="xs" /><span class="truncate">{toolName(t.slug)}</span></span> + <span class="h-2 rounded-sm bg-muted"><span class="block h-2 rounded-sm bg-[var(--chart-1)] transition-opacity group-hover:opacity-80" style="width: {(t.count / toolMax) * 100}%"></span></span> + <span class="text-right font-mono text-xs tabular-nums">{t.count}</span> + </button> + </li> + {/each} + </ul> + {/if} + </Card.Content> + </Card.Root> + + {#if facets && facets.files.length > 0} + <Card.Root class="gap-0 overflow-clip py-0"> + <Card.Header class="border-b border-border py-3"> + <Card.Title class="text-base">Most affected files</Card.Title> + </Card.Header> + <Card.Content class="p-0"> + <ul class="divide-y divide-border/60"> + {#each facets.files.slice(0, 8) as file (file.value)} + <li> + <button type="button" class="flex w-full cursor-pointer items-center gap-3 px-4 py-1.5 text-left text-xs hover:bg-muted/50" onclick={() => onfilter({ file: file.value })}> + <span class="min-w-0 flex-1 truncate font-mono" title={file.value}>{file.value}</span> + <span class="font-mono tabular-nums text-muted-foreground">{file.count}</span> + </button> + </li> + {/each} + </ul> + </Card.Content> + </Card.Root> + {/if} + + {#if facets && facets.rules.length > 0} + <Card.Root class="gap-0 overflow-clip py-0"> + <Card.Header class="border-b border-border py-3"> + <Card.Title class="text-base">Most frequent rules</Card.Title> + </Card.Header> + <Card.Content class="p-0"> + <ul class="divide-y divide-border/60"> + {#each facets.rules.slice(0, 8) as rule (rule.value)} + <li> + <button type="button" class="flex w-full cursor-pointer items-center gap-3 px-4 py-1.5 text-left text-xs hover:bg-muted/50" onclick={() => onfilter({ rule: rule.value })}> + <span class="min-w-0 flex-1"> + <span class="block truncate font-mono">{rule.value}</span> + {#if rule.label}<span class="block truncate text-muted-foreground">{rule.label}</span>{/if} + </span> + <span class="font-mono tabular-nums text-muted-foreground">{rule.count}</span> + </button> + </li> + {/each} + </ul> + </Card.Content> + </Card.Root> + {/if} + </div> + + {#if hasMetrics} + <Card.Root class="gap-3"> + <Card.Header> + <Card.Title class="text-base">Metrics</Card.Title> + <Card.Description class="text-xs">Aggregate figures from complexity and duplication tools.</Card.Description> + </Card.Header> + <Card.Content class="@container"> + <MetricsPanel {metrics} {tools} /> + </Card.Content> + </Card.Root> + {/if} + + {#if unmeasured.length > 0 && !active} + <div class="flex flex-wrap items-center gap-3 rounded-md border border-border bg-card px-4 py-3 text-sm"> + <p class="min-w-0 flex-1 text-muted-foreground"> + {unmeasured.length} tool{unmeasured.length === 1 ? '' : 's'} measured nothing ({unmeasured.map((u) => u.state.label.toLowerCase()).filter((v, i, a) => a.indexOf(v) === i).join(', ')}). + Re-running only those keeps this run's other results. + </p> + <Button size="sm" variant="outline" disabled={rerunning} onclick={onrerununmeasured}> + {#if rerunning}<LoaderCircle class="size-3.5 animate-spin motion-reduce:animate-none" />{:else}<RotateCcw class="size-3.5" />{/if} + Re-run {unmeasured.length} tool{unmeasured.length === 1 ? '' : 's'} + </Button> + </div> + {/if} +</div> diff --git a/frontend/src/lib/components/analysis/RunStatusCell.svelte b/frontend/src/lib/components/analysis/RunStatusCell.svelte new file mode 100644 index 00000000..b54a864e --- /dev/null +++ b/frontend/src/lib/components/analysis/RunStatusCell.svelte @@ -0,0 +1,45 @@ +<script lang="ts"> + /** + * RunStatusCell — a run's status for tables: the badge, and while active a + * thin progress bar with "3/12 tools · Semgrep" (or the phase / queue state). + * Finished runs with failed tools say how many. + * + * Props: `run` (list item), `toolName(slug)`. + */ + import StatusBadge from '$lib/components/StatusBadge.svelte'; + import type { AnalysisRunListItem } from '$lib/api/runs'; + import { RUN_STATUS_HELP, isActive, runStatusLabel } from './format'; + + interface Props { + run: AnalysisRunListItem; + toolName?: (slug: string) => string; + } + + let { run, toolName = (s: string) => s }: Props = $props(); + + const p = $derived(run.progress); + const total = $derived(p?.total || run.tool_slugs.length); + const pct = $derived(p && total ? Math.round((p.done / total) * 100) : 0); + const note = $derived.by(() => { + if (run.status === 'pending') return 'waiting for its turn'; + if (p?.running.length) return toolName(p.running[0]); + if (run.phase === 'waiting_for_workspace') return 'waiting for workspace'; + if (run.phase === 'preparing') return 'preparing'; + return ''; + }); + const failedTools = $derived((run.summary?.tools_failed as number | undefined) ?? 0); +</script> + +<div class="min-w-[9rem] space-y-1" title={RUN_STATUS_HELP[run.status]}> + <StatusBadge status={run.status} label={runStatusLabel(run.status)} /> + {#if isActive(run.status)} + <div class="h-1 w-full max-w-[10rem] overflow-hidden rounded-full bg-muted" role="progressbar" aria-valuemin={0} aria-valuemax={total} aria-valuenow={p?.done ?? 0} aria-label="Tools finished"> + <div class="h-full rounded-full bg-info transition-[width] duration-300" style="width: {pct}%"></div> + </div> + <p class="truncate text-[11px] text-muted-foreground"> + <span class="font-mono tabular-nums">{p?.done ?? 0}/{total}</span> tools{note ? ` · ${note}` : ''} + </p> + {:else if failedTools > 0 && run.status !== 'failed'} + <p class="cell-sub text-[11px] text-danger-fg">{failedTools} tool{failedTools === 1 ? '' : 's'} failed</p> + {/if} +</div> diff --git a/frontend/src/lib/components/analysis/ToolOutputDialog.svelte b/frontend/src/lib/components/analysis/ToolOutputDialog.svelte new file mode 100644 index 00000000..e9fc7ed4 --- /dev/null +++ b/frontend/src/lib/components/analysis/ToolOutputDialog.svelte @@ -0,0 +1,143 @@ +<script lang="ts"> + /** + * ToolOutputDialog — one tool's raw result for debugging: the exact command, + * exit code, target URL (dynamic tools), version, options, warnings, error and + * the stored stdout (first 30k characters). + * + * Props: `open` (bindable), `runId`, `result` (the tool result to show). + */ + import * as Dialog from '$lib/components/ui/dialog'; + import { Button } from '$lib/components/ui/button'; + import Copy from '@lucide/svelte/icons/copy'; + import ExternalLink from '@lucide/svelte/icons/external-link'; + import { toast } from 'svelte-sonner'; + import LoadingState from '$lib/components/LoadingState.svelte'; + import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import { formatApiError } from '$lib/api/core'; + import { getRunResult, type RunToolResult, type ToolResultDetail } from '$lib/api/runs'; + import { formatSpan, resultState } from './format'; + import { safeUrl } from './finding-data'; + + interface Props { + open: boolean; + runId: string; + result: RunToolResult | null; + } + + let { open = $bindable(), runId, result }: Props = $props(); + + let detail = $state<ToolResultDetail | null>(null); + let loading = $state(false); + let error = $state(''); + let loadedFor = ''; + + async function load(id: string) { + loading = true; + error = ''; + try { + detail = await getRunResult(runId, id); + loadedFor = id; + } catch (e) { + error = formatApiError(e, 'Could not load the tool output.'); + } finally { + loading = false; + } + } + + $effect(() => { + if (open && result && result.id !== loadedFor) { + detail = null; + void load(result.id); + } + }); + + const raw = $derived(detail?.raw_output ?? {}); + const config = $derived(Object.entries((detail?.summary?.config ?? {}) as Record<string, unknown>)); + const stdout = $derived(typeof raw.stdout === 'string' ? raw.stdout : ''); + + async function copy(text: string) { + try { + await navigator.clipboard.writeText(text); + toast.success('Copied'); + } catch { + toast.error('Copy failed'); + } + } +</script> + +<Dialog.Root bind:open> + <Dialog.Content class="flex max-h-[90dvh] w-[calc(100vw-2rem)] max-w-3xl flex-col gap-3 p-5"> + <Dialog.Header> + <Dialog.Title class="font-display text-lg">{detail?.tool_name ?? result?.tool_slug ?? 'Tool'} output</Dialog.Title> + <Dialog.Description class="text-xs text-muted-foreground"> + {#if result}{resultState(result).label}{#if result.duration_seconds != null} · {formatSpan(result.duration_seconds)}{/if}{/if} + {#if detail} · {detail.findings_count} finding{detail.findings_count === 1 ? '' : 's'} stored{/if} + </Dialog.Description> + </Dialog.Header> + + {#if loading} + <LoadingState label="loading output" /> + {:else if error} + <ErrorAlert message={error} onretry={() => result && load(result.id)} /> + {:else if detail} + <div class="min-h-0 flex-1 space-y-3 overflow-y-auto pr-1 text-sm"> + <dl class="grid grid-cols-[7rem_minmax(0,1fr)] gap-x-3 gap-y-1.5 text-xs"> + {#if raw.command} + <dt class="text-muted-foreground">Command</dt> + <dd class="flex items-start gap-1.5"> + <code class="min-w-0 flex-1 break-all rounded bg-muted px-1.5 py-0.5 font-mono">{raw.command}</code> + <button type="button" class="cursor-pointer rounded p-0.5 text-muted-foreground hover:text-foreground" aria-label="Copy command" onclick={() => copy(String(raw.command))}><Copy class="size-3.5" /></button> + </dd> + {/if} + {#if raw.exit_code !== undefined} + <dt class="text-muted-foreground">Exit code</dt> + <dd class="font-mono">{raw.exit_code ?? '—'}</dd> + {/if} + {#if raw.target_url} + <dt class="text-muted-foreground">Target</dt> + <dd class="break-all font-mono">{raw.target_url}</dd> + {/if} + {#if detail.summary?.tool_version} + <dt class="text-muted-foreground">Version</dt> + <dd class="font-mono">{detail.summary.tool_version}</dd> + {/if} + {#if config.length > 0} + <dt class="text-muted-foreground">Options</dt> + <dd class="flex flex-wrap gap-1"> + {#each config as [k, v] (k)} + <span class="rounded bg-muted px-1.5 py-px font-mono text-[11px]">{k}={Array.isArray(v) ? v.join(',') : String(v)}</span> + {/each} + </dd> + {/if} + {#if detail.tool_docs_url && safeUrl(detail.tool_docs_url)} + <dt class="text-muted-foreground">Docs</dt> + <dd> + <a href={safeUrl(detail.tool_docs_url)} target="_blank" rel="noopener noreferrer" class="inline-flex items-center gap-1 text-primary hover:underline">{detail.tool_docs_url.replace(/^https?:\/\//, '')}<ExternalLink class="size-3" /></a> + </dd> + {/if} + </dl> + + {#if detail.error_message} + <ErrorAlert title={detail.status === 'failed' ? 'Why it failed' : 'Note'} message={detail.error_message} /> + {/if} + {#if (detail.summary?.warnings ?? []).length > 0} + <div class="rounded-md border border-warning/30 bg-warning/10 px-3 py-2 text-xs text-warning-fg"> + {#each detail.summary.warnings as w, i (i)}<p class="[overflow-wrap:anywhere]">{w}</p>{/each} + </div> + {/if} + + <div> + <div class="mb-1 flex items-center justify-between"> + <p class="section-label">Output{stdout.length >= 30000 ? ' (first 30,000 characters)' : ''}</p> + {#if stdout}<Button variant="ghost" size="sm" class="h-6 px-2 text-xs" onclick={() => copy(stdout)}><Copy class="size-3" />Copy</Button>{/if} + </div> + {#if stdout} + <pre class="max-h-[50dvh] overflow-auto rounded-md border border-border bg-muted/50 p-3 font-mono text-[11px] leading-4 whitespace-pre-wrap [overflow-wrap:anywhere]">{stdout}</pre> + {:else} + <p class="text-xs text-muted-foreground">No output was stored{detail.status === 'skipped' ? ' (the tool did not run)' : ''}.</p> + {/if} + </div> + </div> + {/if} + </Dialog.Content> +</Dialog.Root> diff --git a/frontend/src/lib/components/analysis/ToolPicker.svelte b/frontend/src/lib/components/analysis/ToolPicker.svelte new file mode 100644 index 00000000..c74a31ac --- /dev/null +++ b/frontend/src/lib/components/analysis/ToolPicker.svelte @@ -0,0 +1,175 @@ +<script lang="ts"> + /** + * ToolPicker — choose the workspace tools a run uses, grouped by category, + * with quick sets (all installed, static only, security & dependencies) and + * per-group toggles. Each row says what the tool reads and, from the + * preview, where it would actually run ("3 of 4 apps", "nothing to read", + * "app not running"). Tools that aren't installed are listed but disabled + * (install them in Analyzers); dynamic tools are disabled for pasted code. + * + * Props: `tools` (catalog), `selected` (bindable slugs, replaced on change), + * `preview` (null while unknown), `targets` (number of targets), `codeMode`. + */ + import type { AnalyzerTool } from '$lib/api/analyzers'; + import type { RunPreview } from '$lib/api/runs'; + import { Button } from '$lib/components/ui/button'; + import Check from '@lucide/svelte/icons/check'; + import Minus from '@lucide/svelte/icons/minus'; + import Globe from '@lucide/svelte/icons/globe'; + import Sparkles from '@lucide/svelte/icons/sparkles'; + import ExternalLink from '@lucide/svelte/icons/external-link'; + import ToolIcon from '$lib/components/analyzers/ToolIcon.svelte'; + import { cn } from '$lib/utils'; + import { categoryLabel, groupByCategory } from './tools'; + + interface Props { + tools: AnalyzerTool[]; + selected: string[]; + preview: RunPreview | null; + targets: number; + codeMode: boolean; + } + + let { tools, selected = $bindable(), preview, targets, codeMode }: Props = $props(); + + const selectedSet = $derived(new Set(selected)); + const previewBySlug = $derived(new Map((preview?.tools ?? []).map((t) => [t.slug, t]))); + + function usable(t: AnalyzerTool): boolean { + return t.installed && t.is_enabled && !(codeMode && t.analysis_mode === 'dynamic'); + } + + function why(t: AnalyzerTool): string { + if (!t.installed) return t.install_status === 'failed' ? 'Install failed' : 'Not installed'; + if (!t.is_enabled) return 'Disabled'; + if (codeMode && t.analysis_mode === 'dynamic') return 'Needs a running app'; + return ''; + } + + const groups = $derived(groupByCategory(tools)); + const usableTools = $derived(tools.filter(usable)); + + function set(slugs: string[]) { + // Keep catalog order (the run executes tools in the order given). + const want = new Set(slugs); + selected = tools.filter((t) => want.has(t.slug) && usable(t)).map((t) => t.slug); + } + + function toggle(t: AnalyzerTool) { + if (!usable(t)) return; + set(selectedSet.has(t.slug) ? selected.filter((s) => s !== t.slug) : [...selected, t.slug]); + } + + function toggleGroup(items: AnalyzerTool[]) { + const usableItems = items.filter(usable); + const all = usableItems.every((t) => selectedSet.has(t.slug)); + const slugs = new Set(usableItems.map((t) => t.slug)); + set(all ? selected.filter((s) => !slugs.has(s)) : [...selected, ...slugs]); + } + + const presets = $derived([ + { label: 'All installed', slugs: usableTools.filter((t) => t.kind !== 'ai').map((t) => t.slug), title: 'Every installed tool except the AI reviewer (it costs tokens)' }, + { label: 'Static only', slugs: usableTools.filter((t) => t.analysis_mode !== 'dynamic' && t.kind !== 'ai').map((t) => t.slug), title: 'Tools that read the source; no running app needed' }, + { label: 'Security & deps', slugs: usableTools.filter((t) => ['security', 'dependencies', 'secrets'].includes(t.category)).map((t) => t.slug), title: 'Security scanners, dependency auditors and secret detectors' }, + ]); + + function where(t: AnalyzerTool): { text: string; tone: string } | null { + const p = previewBySlug.get(t.slug); + if (!p || targets === 0) return null; + const unit = codeMode ? '' : ` of ${targets} app${targets === 1 ? '' : 's'}`; + if (p.runs === targets) { + const text = codeMode ? 'Will run' : targets === 1 ? 'Will run on this app' : `Runs on all ${targets} apps`; + return { text, tone: 'text-success-fg' }; + } + if (p.runs === 0 && p.app_not_running > 0) return { text: 'App not running — would be skipped', tone: 'text-warning-fg' }; + if (p.runs === 0) return { text: 'Nothing to read — would be skipped', tone: 'text-muted-foreground' }; + const parts = [`Runs on ${p.runs}${unit}`]; + if (p.not_applicable) parts.push(`${p.not_applicable} nothing to read`); + if (p.app_not_running) parts.push(`${p.app_not_running} not running`); + return { text: parts.join(' · '), tone: 'text-foreground/80' }; + } +</script> + +<div class="space-y-3"> + <div class="flex flex-wrap items-center gap-1.5"> + <span class="mr-1 text-xs text-muted-foreground">Quick sets:</span> + {#each presets as p (p.label)} + <Button variant="outline" size="sm" class="h-7 text-xs" title={p.title} disabled={p.slugs.length === 0} onclick={() => set(p.slugs)}> + {p.label}<span class="font-mono text-muted-foreground">{p.slugs.length}</span> + </Button> + {/each} + <Button variant="ghost" size="sm" class="h-7 text-xs" disabled={selected.length === 0} onclick={() => set([])}>Clear</Button> + <a href="/analyzers" target="_blank" rel="noopener" class="ml-auto inline-flex items-center gap-1 text-xs text-primary hover:underline"> + Install more tools<ExternalLink class="size-3" /> + </a> + </div> + + <div class="grid items-start gap-3 @2xl:grid-cols-2"> + {#each groups as group (group.category)} + {@const groupUsable = group.items.filter(usable)} + {@const groupSelected = groupUsable.filter((t) => selectedSet.has(t.slug)).length} + <section class="min-w-0 overflow-hidden rounded-md border border-border" aria-label={categoryLabel(group.category)}> + <header class="flex items-center gap-2 border-b border-border bg-muted/40 px-3 py-1.5"> + <h3 class="flex-1 text-xs font-medium">{categoryLabel(group.category)}</h3> + <span class="font-mono text-[11px] text-muted-foreground tabular-nums">{groupSelected}/{groupUsable.length}</span> + {#if groupUsable.length > 0} + <button type="button" class="cursor-pointer text-[11px] text-primary hover:underline" onclick={() => toggleGroup(group.items)}> + {groupSelected === groupUsable.length ? 'None' : 'All'} + </button> + {/if} + </header> + <ul> + {#each group.items as tool (tool.slug)} + {@const on = selectedSet.has(tool.slug)} + {@const ok = usable(tool)} + {@const reason = why(tool)} + {@const w = on ? where(tool) : null} + <li class="border-b border-border/60 last:border-b-0"> + <button + type="button" + role="checkbox" + aria-checked={on} + aria-disabled={!ok} + aria-label="{tool.name}{reason ? ` (${reason.toLowerCase()})` : ''}" + class={cn( + 'flex w-full items-start gap-2.5 px-3 py-2 text-left transition-colors duration-150 focus-visible:ring-2 focus-visible:ring-ring/40 focus-visible:outline-none focus-visible:ring-inset', + ok ? 'cursor-pointer hover:bg-muted/50' : 'cursor-not-allowed opacity-60', + on && 'bg-primary/5' + )} + onclick={() => toggle(tool)} + > + <span + class={cn( + 'mt-1 flex size-4 shrink-0 items-center justify-center rounded-[4px] border transition-colors', + on ? 'border-primary bg-primary text-primary-foreground' : 'border-input bg-background' + )} + aria-hidden="true" + > + {#if on}<Check class="size-3" />{/if} + </span> + <ToolIcon {tool} size="sm" /> + <span class="min-w-0 flex-1"> + <span class="flex flex-wrap items-center gap-1.5 text-sm font-medium"> + {tool.name} + {#if tool.analysis_mode === 'dynamic'} + <span class="inline-flex items-center gap-0.5 rounded border border-border px-1 text-[11px] font-normal text-muted-foreground"><Globe class="size-3" />Dynamic</span> + {/if} + {#if tool.kind === 'ai'} + <span class="inline-flex items-center gap-0.5 rounded border border-border px-1 text-[11px] font-normal text-muted-foreground"><Sparkles class="size-3" />AI · costs tokens</span> + {/if} + {#if reason}<span class="text-[11px] font-normal text-warning-fg">{reason}</span>{/if} + </span> + <span class="line-clamp-1 text-xs text-muted-foreground" title={tool.description}>{tool.description}</span> + {#if w}<span class={cn('block text-[11px]', w.tone)}>{w.text}</span>{/if} + </span> + </button> + </li> + {/each} + </ul> + </section> + {/each} + </div> + {#if selected.length === 0} + <p class="flex items-center gap-1.5 text-xs text-muted-foreground"><Minus class="size-3" />No tools selected.</p> + {/if} +</div> diff --git a/frontend/src/lib/components/analysis/ToolPipeline.svelte b/frontend/src/lib/components/analysis/ToolPipeline.svelte new file mode 100644 index 00000000..6a2822f4 --- /dev/null +++ b/frontend/src/lib/components/analysis/ToolPipeline.svelte @@ -0,0 +1,134 @@ +<script lang="ts"> + /** + * ToolPipeline — every tool of a run in execution order: state (queued, + * running with a live clock, completed, failed, skipped and why), time, + * findings with the levels the tool reported, version, warnings and the + * error. Rows link to the tool's findings and its raw output. + * + * Props: `tools` (slug → catalog tool), `slugs` (the run's tool order), + * `results` (by slug), `now` (ms, ticks while live), `onfindings(slug)`, + * `onoutput(result)`. + */ + import type { AnalyzerTool } from '$lib/api/analyzers'; + import type { RunToolResult } from '$lib/api/runs'; + import ToolIcon from '$lib/components/analyzers/ToolIcon.svelte'; + import { Button } from '$lib/components/ui/button'; + import Globe from '@lucide/svelte/icons/globe'; + import Sparkles from '@lucide/svelte/icons/sparkles'; + import TriangleAlert from '@lucide/svelte/icons/triangle-alert'; + import FileText from '@lucide/svelte/icons/file-text'; + import ListFilter from '@lucide/svelte/icons/list-filter'; + import { cn } from '$lib/utils'; + import { toneClasses } from '$lib/constants/colors'; + import LevelBadge from './LevelBadge.svelte'; + import { elapsedSince, formatClock, formatSpan, resultFindingCount, resultState, reportedLevels } from './format'; + + interface Props { + tools: Map<string, AnalyzerTool>; + slugs: string[]; + results: Map<string, RunToolResult>; + now: number; + onfindings?: (slug: string) => void; + onoutput?: (result: RunToolResult) => void; + } + + let { tools, slugs, results, now, onfindings, onoutput }: Props = $props(); + + let openErrors = $state(new Set<string>()); + + function toggleError(slug: string) { + const next = new Set(openErrors); + if (next.has(slug)) next.delete(slug); + else next.add(slug); + openErrors = next; + } +</script> + +<ul class="divide-y divide-border/60" aria-label="Tools of this run"> + {#each slugs as slug, i (slug)} + {@const r = results.get(slug)} + {@const tool = tools.get(slug)} + {@const st = resultState(r)} + {@const count = resultFindingCount(r)} + {@const levels = reportedLevels(r?.summary)} + {@const warnings = (r?.summary?.warnings ?? []) as string[]} + {@const version = (r?.summary?.tool_version ?? '') as string} + {@const hasOutput = !!r && (r.status === 'completed' || r.status === 'failed')} + <li class={cn('px-3 py-2.5 transition-colors duration-150', st.live && 'bg-info/5')}> + <div class="flex flex-wrap items-center gap-x-3 gap-y-1.5 md:grid md:grid-cols-[1.25rem_1.75rem_minmax(0,1fr)_8rem_4rem_minmax(8rem,13rem)_10rem]"> + <span class="w-5 shrink-0 text-right font-mono text-[11px] text-muted-foreground tabular-nums">{i + 1}</span> + <ToolIcon tool={tool ?? { slug }} size="sm" /> + <div class="min-w-[9rem] flex-1"> + <p class="flex flex-wrap items-center gap-1.5 text-sm font-medium"> + {tool?.name ?? slug} + {#if tool?.analysis_mode === 'dynamic'} + <span class="inline-flex items-center gap-0.5 rounded border border-border px-1 text-[11px] font-normal text-muted-foreground" title="Probes the running app"><Globe class="size-3" />Dynamic</span> + {/if} + {#if tool?.kind === 'ai'} + <span class="inline-flex items-center gap-0.5 rounded border border-border px-1 text-[11px] font-normal text-muted-foreground" title="An LLM reviewer: findings are opinions, not measurements"><Sparkles class="size-3" />AI</span> + {/if} + </p> + {#if version}<p class="cell-sub truncate font-mono text-[11px] text-muted-foreground" title={version}>{version}</p>{/if} + </div> + + <span class={cn('inline-flex shrink-0 items-center gap-1.5 justify-self-start rounded-full border px-2 py-0.5 text-[11px] font-medium', toneClasses[st.tone])}> + <span class={cn('status-dot h-1.5 w-1.5', st.live && 'status-dot-live')} aria-hidden="true"></span> + {st.label} + </span> + + <span class="w-16 shrink-0 text-right font-mono text-xs text-muted-foreground tabular-nums" title={r?.started_at ? `Started ${new Date(r.started_at).toLocaleTimeString()}` : undefined}> + {#if st.live} + <span class="text-info-fg">{formatClock(elapsedSince(r?.started_at, now))}</span> + {:else if r?.duration_seconds != null} + {formatSpan(r.duration_seconds)} + {:else} + — + {/if} + </span> + + <div class="flex min-w-0 flex-wrap items-center gap-1 md:justify-end"> + {#if r && (st.measured || count > 0)} + <span class="mr-1 font-mono text-sm tabular-nums">{count}</span> + <span class="mr-1 text-xs text-muted-foreground">finding{count === 1 ? '' : 's'}</span> + {#each levels.slice(0, 3) as [label, n] (label)} + <span class="inline-flex items-center gap-0.5"><LevelBadge {label} /><span class="font-mono text-[11px] text-muted-foreground tabular-nums">{n}</span></span> + {/each} + {/if} + </div> + + <div class="ml-auto flex shrink-0 items-center justify-end gap-1 md:ml-0"> + {#if count > 0 && onfindings} + <Button variant="ghost" size="sm" class="h-7 px-2 text-xs" onclick={() => onfindings(slug)} title="Show this tool's findings"> + <ListFilter class="size-3.5" />Findings + </Button> + {/if} + {#if hasOutput && onoutput} + <Button variant="ghost" size="sm" class="h-7 px-2 text-xs" onclick={() => onoutput(r!)} title="Command, exit code and raw output"> + <FileText class="size-3.5" />Output + </Button> + {/if} + </div> + </div> + + {#if r?.error_message && r.status !== 'running'} + {@const long = r.error_message.length > 160} + <div class={cn('mt-1.5 ml-8 text-xs', r.status === 'failed' ? 'text-danger-fg' : 'text-muted-foreground')}> + <p class={cn('whitespace-pre-line [overflow-wrap:anywhere]', long && !openErrors.has(slug) && 'line-clamp-2')}>{r.error_message}</p> + {#if long} + <button type="button" class="cursor-pointer text-muted-foreground underline-offset-2 hover:text-foreground hover:underline" onclick={() => toggleError(slug)}> + {openErrors.has(slug) ? 'Show less' : 'Show all'} + </button> + {/if} + </div> + {/if} + {#if warnings.length > 0} + <div class="mt-1.5 ml-8 flex items-start gap-1.5 text-xs text-warning-fg"> + <TriangleAlert class="mt-0.5 size-3.5 shrink-0" aria-hidden="true" /> + <div class="min-w-0 space-y-0.5"> + {#each warnings as w, wi (wi)}<p class="[overflow-wrap:anywhere]">{w}</p>{/each} + </div> + </div> + {/if} + </li> + {/each} +</ul> diff --git a/frontend/src/lib/components/analysis/analysis.test.ts b/frontend/src/lib/components/analysis/analysis.test.ts new file mode 100644 index 00000000..7d3a6730 --- /dev/null +++ b/frontend/src/lib/components/analysis/analysis.test.ts @@ -0,0 +1,274 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { RunFinding, RunToolResult } from "$lib/api/runs"; +import type { SseEvent } from "$lib/api/sse"; +import { RunWatcher } from "./live.svelte"; +import { + formatClock, + levelClasses, + levelRank, + reportedLevels, + resultState, + toCsv, +} from "./format"; +import { + advisoryUrl, + dependencyInfo, + httpInfo, + otherData, + safeUrl, +} from "./finding-data"; + +function result(over: Partial<RunToolResult> = {}): RunToolResult { + return { + id: "r1", + tool_slug: "bandit", + category: "security", + status: "completed", + summary: {}, + metrics: {}, + error_message: "", + started_at: null, + duration_seconds: 1, + ...over, + }; +} + +function finding( + data: Record<string, unknown>, + over: Partial<RunFinding> = {}, +): RunFinding { + return { + id: "f1", + severity: "high", + tool_severity: "", + category: "dependency", + confidence: "medium", + title: "t", + description: "", + suggestion: "", + file_path: "requirements.txt", + line_number: null, + column_number: null, + code_snippet: "", + rule_id: "", + tool_slug: "pip-audit", + tool_specific_data: data, + ...over, + }; +} + +describe("resultState", () => { + it("counts a not-applicable skip as measured, other skips and failures as not", () => { + expect( + resultState( + result({ + status: "skipped", + summary: { skip_reason: "not_applicable" }, + }), + ).measured, + ).toBe(true); + expect( + resultState( + result({ + status: "skipped", + summary: { skip_reason: "app_not_running" }, + }), + ).measured, + ).toBe(false); + expect(resultState(result({ status: "failed" })).measured).toBe(false); + expect(resultState(result()).measured).toBe(true); + }); + + it("treats a tool without a result as queued and a running one as live", () => { + expect(resultState(undefined).label).toBe("Queued"); + expect(resultState(result({ status: "running" })).live).toBe(true); + }); +}); + +describe("tool-reported levels", () => { + it("orders the tools' own words most severe first and ignores unrated", () => { + const summary = { + reported_severity_counts: { + convention: 9, + warning: 2, + critical: 1, + "8.6": 1, + }, + }; + expect(reportedLevels(summary).map(([label]) => label)).toEqual([ + "critical", + "warning", + "convention", + "8.6", + ]); + expect(reportedLevels({})).toEqual([]); + }); + + it("tints words case-insensitively and leaves unknown labels neutral", () => { + expect(levelRank("HIGH")).toBe(levelRank("high")); + expect(levelClasses("CRITICAL")).toContain("bg-destructive"); + expect(levelClasses("3")).toContain("text-muted-foreground"); + }); +}); + +describe("formatting", () => { + it("formats clocks", () => { + expect(formatClock(5)).toBe("0:05"); + expect(formatClock(3725)).toBe("1:02:05"); + }); + + it("quotes CSV cells that need it", () => { + expect(toCsv([["a", 'say "hi"', "x,y", null]])).toBe( + 'a,"say ""hi""","x,y",', + ); + }); +}); + +describe("finding data", () => { + it("merges standardized and legacy dependency keys", () => { + const dep = dependencyInfo( + finding( + { + package: "flask", + version: "3.0.0", + fix_versions: ["3.1.3"], + aliases: ["CVE-2026-1", "GHSA-68rp-wp8r-4726"], + cvss: 0, + }, + { rule_id: "PYSEC-2026-2151" }, + ), + )!; + expect(dep.installed).toBe("3.0.0"); + expect(dep.fixedIn).toEqual(["3.1.3"]); + expect(dep.ids).toEqual([ + "PYSEC-2026-2151", + "CVE-2026-1", + "GHSA-68rp-wp8r-4726", + ]); + // npm audit's 0 means "not scored". + expect(dep.cvssScore).toBeNull(); + }); + + it("links advisories by id family and only http(s) URLs", () => { + expect(advisoryUrl("CVE-2026-1")).toBe( + "https://nvd.nist.gov/vuln/detail/CVE-2026-1", + ); + expect(advisoryUrl("GHSA-68rp-wp8r-4726")).toBe( + "https://github.com/advisories/GHSA-68rp-wp8r-4726", + ); + expect(advisoryUrl("PYSEC-2026-2151")).toBe( + "https://osv.dev/vulnerability/PYSEC-2026-2151", + ); + expect(safeUrl("javascript:alert(1)")).toBeNull(); + expect(safeUrl("https://example.org/a")).toBe("https://example.org/a"); + }); + + it("shows evidence once when it repeats the code snippet", () => { + const info = httpInfo( + finding( + { url: "http://172.17.0.3:8000/", evidence: "Werkzeug/3.0.1" }, + { code_snippet: "Werkzeug/3.0.1" }, + ), + )!; + expect(info.evidence).toBe(""); + }); + + it("hides dedup keys and secret hashes from other data", () => { + const rest = otherData( + finding({ + issue_key: "x", + hashed_secret: "abc", + ecosystem: "PyPI", + sbom: { a: 1 }, + }), + ); + expect(rest).toEqual([["sbom", '{"a":1}']]); + }); +}); + +describe("RunWatcher", () => { + let now = 0; + let onState: ((s: "open" | "reconnecting" | "closed") => void) | undefined; + const subscribed: string[][] = []; + const unsubscribe = vi.fn(); + + beforeEach(() => { + vi.useFakeTimers(); + now = 0; + subscribed.length = 0; + unsubscribe.mockReset(); + }); + afterEach(() => vi.useRealTimers()); + + function make(active: { value: boolean }, channels: { value: string[] }) { + const poll = vi.fn(async () => {}); + const events: SseEvent[] = []; + const watcher = new RunWatcher({ + channels: () => channels.value, + active: () => active.value, + onEvent: (e) => events.push(e), + poll, + fastPollMs: 3_000, + slowPollMs: 10_000, + deps: { + subscribe: (chs, _onEvent, opts) => { + subscribed.push(chs); + onState = opts.onState; + return unsubscribe; + }, + now: () => now, + hidden: () => false, + }, + }); + return { watcher, poll, events }; + } + + async function advance(ms: number) { + for (let t = 0; t < ms; t += 1_000) { + now += 1_000; + await vi.advanceTimersByTimeAsync(1_000); + } + } + + it("polls fast while SSE is not open and slowly once it is", async () => { + const active = { value: true }; + const { watcher, poll } = make(active, { value: ["analysis:a"] }); + watcher.start(); + expect(subscribed).toEqual([["analysis:a"]]); + await advance(3_000); + expect(poll).toHaveBeenCalledTimes(1); + onState?.("open"); + poll.mockClear(); + await advance(9_000); + expect(poll).not.toHaveBeenCalled(); + await advance(2_000); + expect(poll).toHaveBeenCalledTimes(1); + watcher.destroy(); + }); + + it("stops polling and disconnects once nothing is active", async () => { + const active = { value: true }; + const { watcher, poll } = make(active, { value: ["analysis:a"] }); + watcher.start(); + active.value = false; + watcher.sync(); + expect(unsubscribe).toHaveBeenCalledTimes(1); + expect(watcher.connection).toBe("closed"); + await advance(30_000); + expect(poll).not.toHaveBeenCalled(); + watcher.destroy(); + }); + + it("reconnects only when the channel set changes", () => { + const channels = { value: ["analysis:a"] }; + const { watcher } = make({ value: true }, channels); + watcher.start(); + watcher.sync(); + expect(subscribed).toHaveLength(1); + channels.value = ["analysis:a", "analysis:b"]; + watcher.sync(); + expect(subscribed).toHaveLength(2); + expect(unsubscribe).toHaveBeenCalledTimes(1); + watcher.destroy(); + }); +}); diff --git a/frontend/src/lib/components/analysis/finding-data.ts b/frontend/src/lib/components/analysis/finding-data.ts new file mode 100644 index 00000000..a037ca4b --- /dev/null +++ b/frontend/src/lib/components/analysis/finding-data.ts @@ -0,0 +1,306 @@ +/** + * Reading a finding's `tool_specific_data`: parsers store standardized keys + * (rule_name, docs_url, references, cwe, owasp, package, installed_version, + * fixed_versions, vuln_id, aliases, cvss_score, url, method, param, evidence, + * duplicate_of, complexity, …) next to older tool-specific ones (trivy + * `fixed_version`, osv `fix_versions`, jscpd `second_file`, …). These helpers + * merge both so the detail view renders whichever a parser wrote. + */ +import type { RunFinding } from "$lib/api/runs"; + +type Data = Record<string, any>; + +/** + * Keys never shown: internal deduplication identifiers, and detect-secrets' + * SHA-1 of the secret (reversible for weak secrets). + */ +const HIDDEN = new Set(["issue_key", "hashed_secret"]); + +/** Keys the structured sections render (everything else goes to "Other data"). */ +export const KNOWN_KEYS = new Set([ + "rule_name", + "test_name", + "message_id", + "likelihood", + "impact", + "rank", + "entropy", + "docs_url", + "references", + "cwe", + "owasp", + "wstg", + "tags", + "end_line", + "end_column", + "fix", + "tool_confidence", + "package", + "version", + "installed_version", + "fixed_version", + "fix_versions", + "fixed_versions", + "vuln_id", + "aliases", + "cvss", + "cvss_score", + "cvss_vector", + "ecosystem", + "manifest", + "direct", + "url", + "urls", + "method", + "param", + "evidence", + "attack", + "instances", + "other_info", + "curl", + "matcher", + "secret_type", + "complexity", + "function", + "nloc", + "duplicate_of", + "second_file", + "second_start", + "lines", + "tokens", + "module", +]); + +/** Only http(s) URLs become links (tool output is untrusted). */ +export function safeUrl(value: unknown): string | null { + if (typeof value !== "string") return null; + try { + const u = new URL(value.trim()); + return u.protocol === "http:" || u.protocol === "https:" ? u.href : null; + } catch { + return null; + } +} + +/** Target URLs of dynamic tools point at the app's private container address: text, not links. */ +export function isPrivateUrl(value: string): boolean { + try { + const host = new URL(value).hostname; + return ( + /^(10|127)\./.test(host) || + /^172\.(1[6-9]|2\d|3[01])\./.test(host) || + /^192\.168\./.test(host) || + host === "localhost" || + !host.includes(".") + ); + } catch { + return true; + } +} + +function strings(value: unknown): string[] { + if (Array.isArray(value)) + return value.filter((v) => typeof v === "string" && v.trim()).map(String); + if (typeof value === "string" && value.trim()) return [value]; + return []; +} + +function num(value: unknown): number | null { + const n = typeof value === "string" ? Number.parseFloat(value) : value; + return typeof n === "number" && Number.isFinite(n) ? n : null; +} + +export function cweUrl(cwe: string): string | null { + const m = /CWE-?(\d+)/i.exec(cwe); + return m ? `https://cwe.mitre.org/data/definitions/${m[1]}.html` : null; +} + +/** Advisory page for a vulnerability id (CVE → NVD, GHSA → GitHub, others → OSV). */ +export function advisoryUrl(id: string): string | null { + const v = id.trim(); + if (/^CVE-\d{4}-\d+$/i.test(v)) + return `https://nvd.nist.gov/vuln/detail/${v.toUpperCase()}`; + if (/^GHSA(-[a-z0-9]{4}){3}$/i.test(v)) + return `https://github.com/advisories/${v}`; + if (/^(PYSEC|RUSTSEC|GO|OSV|MAL|GSD|DSA|USN|RHSA|ALSA)-/i.test(v)) { + return `https://osv.dev/vulnerability/${v}`; + } + return null; +} + +export interface DependencyInfo { + package: string; + installed: string; + fixedIn: string[]; + ecosystem: string; + manifest: string; + direct: boolean | null; + ids: string[]; + cvssScore: number | null; + cvssVector: string; +} + +export function dependencyInfo(f: RunFinding): DependencyInfo | null { + const d: Data = f.tool_specific_data ?? {}; + const pkg = d.package ?? d.module; + if (!pkg) return null; + const cvssRaw = d.cvss_score ?? d.cvss; + const vector = + typeof d.cvss_vector === "string" + ? d.cvss_vector + : typeof d.cvss === "string" && d.cvss.startsWith("CVSS:") + ? d.cvss + : ""; + const score = num(cvssRaw); + const ids = [ + ...new Set([ + ...strings(d.vuln_id), + ...(f.rule_id && advisoryUrl(f.rule_id) ? [f.rule_id] : []), + ...strings(d.aliases), + ]), + ]; + return { + package: String(pkg), + installed: String(d.installed_version ?? d.version ?? ""), + fixedIn: [ + ...new Set([ + ...strings(d.fixed_versions), + ...strings(d.fix_versions), + ...strings(d.fixed_version), + ]), + ], + ecosystem: String(d.ecosystem ?? ""), + manifest: String(d.manifest ?? ""), + direct: typeof d.direct === "boolean" ? d.direct : null, + ids, + // A 0 score means "not scored" in npm audit: not a measurement. + cvssScore: score && score > 0 ? score : null, + cvssVector: vector, + }; +} + +export interface HttpInfo { + method: string; + url: string; + param: string; + attack: string; + evidence: string; + instances: number | null; + otherUrls: string[]; + otherInfo: string; + curl: string; + matcher: string; +} + +export function httpInfo(f: RunFinding): HttpInfo | null { + const d: Data = f.tool_specific_data ?? {}; + if (!d.url && !d.curl && !d.method) return null; + const url = String(d.url ?? ""); + const evidence = String(d.evidence ?? ""); + return { + method: String(d.method ?? ""), + url, + param: String(d.param ?? ""), + attack: String(d.attack ?? ""), + // ZAP/nuclei also store the evidence as the code snippet: show it once. + evidence: evidence.trim() === (f.code_snippet ?? "").trim() ? "" : evidence, + instances: num(d.instances), + otherUrls: strings(d.urls).filter((u) => u !== url), + otherInfo: String(d.other_info ?? ""), + curl: String(d.curl ?? ""), + matcher: String(d.matcher ?? ""), + }; +} + +export interface DuplicateInfo { + file: string; + startLine: number | null; + endLine: number | null; + lines: number | null; + tokens: number | null; +} + +export function duplicateInfo(f: RunFinding): DuplicateInfo | null { + const d: Data = f.tool_specific_data ?? {}; + const dup = (d.duplicate_of ?? {}) as Data; + const file = String(dup.file ?? d.second_file ?? ""); + if (!file) return null; + return { + file, + startLine: num(dup.start_line ?? d.second_start), + endLine: num(dup.end_line), + lines: num(d.lines), + tokens: num(d.tokens), + }; +} + +/** Rule-level facts: human name, docs link, references, CWE/OWASP/WSTG, tags. */ +export function ruleInfo(f: RunFinding) { + const d: Data = f.tool_specific_data ?? {}; + const cwe = strings(d.cwe); + return { + name: String(d.rule_name ?? d.test_name ?? ""), + docsUrl: safeUrl(d.docs_url), + references: [ + ...new Set( + strings(d.references) + .map(safeUrl) + .filter((u): u is string => !!u), + ), + ], + cwe, + owasp: strings(d.owasp), + wstg: strings(d.wstg), + tags: strings(d.tags), + messageId: String(d.message_id ?? ""), + likelihood: String(d.likelihood ?? ""), + impact: String(d.impact ?? ""), + toolConfidence: String(d.tool_confidence ?? ""), + fix: String(d.fix ?? ""), + secretType: String(d.secret_type ?? ""), + entropy: num(d.entropy), + }; +} + +export function complexityInfo(f: RunFinding) { + const d: Data = f.tool_specific_data ?? {}; + const complexity = num(d.complexity); + if (complexity == null && !d.function && num(d.nloc) == null) return null; + return { + complexity, + fn: String(d.function ?? ""), + nloc: num(d.nloc), + rank: String(d.rank ?? ""), + }; +} + +/** `file:line:col–end` for display. */ +export function locationText(f: RunFinding): string { + if (!f.file_path) return ""; + const d: Data = f.tool_specific_data ?? {}; + let text = f.file_path; + if (f.line_number != null) { + text += `:${f.line_number}`; + if (f.column_number != null) text += `:${f.column_number}`; + const end = num(d.end_line); + if (end != null && end !== f.line_number) text += `–${end}`; + } + return text; +} + +/** Leftover tool data the structured sections don't cover, as `[key, display]`. */ +export function otherData(f: RunFinding): [string, string][] { + return Object.entries(f.tool_specific_data ?? {}) + .filter( + ([k, v]) => + !KNOWN_KEYS.has(k) && + !HIDDEN.has(k) && + v !== null && + v !== "" && + !(Array.isArray(v) && v.length === 0), + ) + .map(([k, v]) => [ + k.replaceAll("_", " "), + typeof v === "object" ? JSON.stringify(v) : String(v), + ]); +} diff --git a/frontend/src/lib/components/analysis/format.ts b/frontend/src/lib/components/analysis/format.ts new file mode 100644 index 00000000..ee19d4e5 --- /dev/null +++ b/frontend/src/lib/components/analysis/format.ts @@ -0,0 +1,273 @@ +/** + * Display helpers for analysis runs: statuses, phases, tool-result states, + * elapsed time and tool-reported levels. + * + * Levels: a finding shows only the level its tool printed (`tool_severity`). + * The normalized `severity` is a heuristic for tools that rate nothing and is + * never displayed; `levelTone` colors a tool's own word, it doesn't rate it. + */ +import type { Tone } from "$lib/constants/colors"; +import { toneClasses } from "$lib/constants/colors"; +import type { + RunPhase, + RunStatus, + RunTarget, + RunToolResult, + SkipReason, +} from "$lib/api/runs"; +import { shortModelName } from "$lib/components/table"; + +export const ACTIVE_STATUSES: ReadonlySet<string> = new Set([ + "pending", + "running", +]); + +export function isActive(status: string | null | undefined): boolean { + return ACTIVE_STATUSES.has(status ?? ""); +} + +export const RUN_STATUS_LABELS: Record<RunStatus, string> = { + pending: "Queued", + running: "Running", + completed: "Completed", + partial: "Partial", + failed: "Failed", + cancelled: "Cancelled", +}; + +/** What each run status means, for filter chips and badges' titles. */ +export const RUN_STATUS_HELP: Record<RunStatus, string> = { + pending: "Waiting for its turn: runs of one workspace execute one at a time.", + running: "Tools are running.", + completed: "Every tool measured the code (or had nothing to read).", + partial: "Some tools failed; the others' findings stand.", + failed: "No tool produced a result.", + cancelled: "Stopped before every tool ran.", +}; + +export function runStatusLabel(status: string): string { + return RUN_STATUS_LABELS[status as RunStatus] ?? status; +} + +export const PHASE_LABELS: Record<Exclude<RunPhase, "">, string> = { + waiting_for_workspace: + "Waiting for the workspace — another analysis of yours is running", + preparing: "Preparing the workspace and copying the code in", + running_tools: "Running tools", +}; + +export function phaseLabel(phase: string | null | undefined): string { + return PHASE_LABELS[phase as Exclude<RunPhase, "">] ?? ""; +} + +export interface ResultState { + label: string; + tone: Tone; + /** Whether the result says something about the code (completed, or nothing to read). */ + measured: boolean; + /** Work in flight. */ + live: boolean; +} + +const SKIP_STATES: Record<SkipReason, ResultState> = { + not_applicable: { + label: "Not applicable", + tone: "neutral", + measured: true, + live: false, + }, + app_not_running: { + label: "App not running", + tone: "warning", + measured: false, + live: false, + }, + unavailable: { + label: "Unavailable", + tone: "warning", + measured: false, + live: false, + }, + cancelled: { + label: "Cancelled", + tone: "neutral", + measured: false, + live: false, + }, +}; + +/** State of one tool in a run; `result` is undefined while the tool is still queued. */ +export function resultState(result: RunToolResult | undefined): ResultState { + if (!result || result.status === "pending") { + return { label: "Queued", tone: "neutral", measured: false, live: false }; + } + switch (result.status) { + case "running": + return { label: "Running", tone: "info", measured: false, live: true }; + case "completed": + return { + label: "Completed", + tone: "success", + measured: true, + live: false, + }; + case "failed": + return { label: "Failed", tone: "danger", measured: false, live: false }; + default: { + const reason = (result.summary?.skip_reason ?? "") as SkipReason; + return ( + SKIP_STATES[reason] ?? { + label: "Skipped", + tone: "neutral", + measured: false, + live: false, + } + ); + } + } +} + +/** Findings a tool result stored (failed results can keep some). */ +export function resultFindingCount(result: RunToolResult | undefined): number { + const n = result?.summary?.finding_count; + return typeof n === "number" ? n : 0; +} + +/** Seconds since `iso` at `nowMs` (0 when unknown or in the future). */ +export function elapsedSince( + iso: string | null | undefined, + nowMs: number, +): number { + if (!iso) return 0; + return Math.max(0, (nowMs - new Date(iso).getTime()) / 1000); +} + +/** Clock-style elapsed time: "0:42", "12:03", "1:02:03". */ +export function formatClock(seconds: number | null | undefined): string { + if (seconds == null || !Number.isFinite(seconds)) return "—"; + const s = Math.floor(seconds); + const h = Math.floor(s / 3600); + const m = Math.floor((s % 3600) / 60); + const sec = String(s % 60).padStart(2, "0"); + return h > 0 ? `${h}:${String(m).padStart(2, "0")}:${sec}` : `${m}:${sec}`; +} + +/** Compact duration: "850ms", "42s", "3m 12s", "1h 04m". */ +export function formatSpan(seconds: number | null | undefined): string { + if (seconds == null || !Number.isFinite(seconds)) return "—"; + if (seconds < 1) return `${Math.round(seconds * 1000)}ms`; + if (seconds < 60) return `${Math.round(seconds)}s`; + if (seconds < 3600) { + return `${Math.floor(seconds / 60)}m ${String(Math.round(seconds % 60)).padStart(2, "0")}s`; + } + return `${Math.floor(seconds / 3600)}h ${String(Math.floor((seconds % 3600) / 60)).padStart(2, "0")}m`; +} + +// --- Tool-reported levels ------------------------------------------------------- + +// Common level words across tools, most severe first: used to order a run's +// reported levels and to tint a tool's own word. Not a rating of the finding. +const LEVEL_WORDS: [string, number, Tone | "critical"][] = [ + ["critical", 0, "critical"], + ["blocker", 0, "critical"], + ["fatal", 1, "danger"], + ["error", 1, "danger"], + ["high", 1, "danger"], + ["severe", 1, "danger"], + ["major", 2, "warning"], + ["medium", 2, "warning"], + ["moderate", 2, "warning"], + ["warning", 2, "warning"], + ["warn", 2, "warning"], + ["low", 3, "info"], + ["minor", 3, "info"], + ["refactor", 4, "neutral"], + ["convention", 4, "neutral"], + ["style", 4, "neutral"], + ["note", 4, "neutral"], + ["info", 5, "neutral"], + ["informational", 5, "neutral"], + ["unknown", 6, "neutral"], +]; +const LEVEL_INDEX = new Map( + LEVEL_WORDS.map(([w, rank, tone]) => [w, { rank, tone }]), +); + +export function levelRank(label: string): number { + return LEVEL_INDEX.get(label.trim().toLowerCase())?.rank ?? 7; +} + +/** Badge classes for a tool's own level word (critical is solid, like the severity palette). */ +export function levelClasses(label: string): string { + const tone = LEVEL_INDEX.get(label.trim().toLowerCase())?.tone ?? "neutral"; + if (tone === "critical") { + return "border-transparent bg-destructive text-destructive-foreground"; + } + return toneClasses[tone]; +} + +/** `summary.reported_severity_counts` of a finished run, most severe first. */ +export function reportedLevels( + summary: Record<string, any> | null | undefined, +): [string, number][] { + const counts = (summary?.reported_severity_counts ?? {}) as Record< + string, + number + >; + return Object.entries(counts) + .filter(([, n]) => n > 0) + .sort((a, b) => levelRank(a[0]) - levelRank(b[0]) || b[1] - a[1]); +} + +// --- Targets ------------------------------------------------------------------------ + +/** Two-line identity of what a run analyzed: app (or pasted code) and model · stack. */ +export function targetLines(target: RunTarget | null | undefined): { + title: string; + sub: string; +} { + if (!target) return { title: "—", sub: "" }; + if (target.kind !== "job") return { title: target.label, sub: "" }; + const title = target.app_name || target.stack_slug || "Generated app"; + const sub = [ + shortModelName(target.model_name), + target.app_name ? target.stack_slug : "", + ] + .filter(Boolean) + .join(" · "); + return { title, sub }; +} + +export function runDisplayName(run: { name: string; id: string }): string { + return run.name || `Analysis ${run.id.slice(0, 8)}`; +} + +/** Download *text* as a file named *filename*. */ +export function downloadText( + filename: string, + text: string, + type = "text/plain", +): void { + const url = URL.createObjectURL(new Blob([text], { type })); + const a = document.createElement("a"); + a.href = url; + a.download = filename; + document.body.append(a); + a.click(); + a.remove(); + setTimeout(() => URL.revokeObjectURL(url), 1000); +} + +/** RFC 4180 CSV of *rows* (first row = header). */ +export function toCsv(rows: (string | number | null | undefined)[][]): string { + return rows + .map((row) => + row + .map((cell) => { + const s = cell == null ? "" : String(cell); + return /[",\n\r]/.test(s) ? `"${s.replaceAll('"', '""')}"` : s; + }) + .join(","), + ) + .join("\r\n"); +} diff --git a/frontend/src/lib/components/analysis/live.svelte.ts b/frontend/src/lib/components/analysis/live.svelte.ts new file mode 100644 index 00000000..239de992 --- /dev/null +++ b/frontend/src/lib/components/analysis/live.svelte.ts @@ -0,0 +1,156 @@ +/** + * RunWatcher — live updates for analysis pages: one SSE connection to the + * `analysis:<id>` channels of the runs on screen, plus polling. + * + * - `channels()` is re-read on `sync()`; the stream reconnects only when the + * set changes (at most 20 channels, the server's limit). + * - While `active()`: `poll()` runs every `slowPollMs` as a safety net even with + * SSE open (a long tool is legitimately quiet: the server heartbeat is an SSE + * comment), and every `fastPollMs` while SSE is connecting/reconnecting. + * A hidden tab doesn't poll; becoming visible polls at once. + * - `now` ticks every second while active (elapsed clocks). + * + * Dependencies are injectable for tests. + */ +import { + subscribe as sseSubscribe, + type SseEvent, + type SseState, +} from "$lib/api/sse"; + +export type WatchConnection = + | "idle" + | "connecting" + | "open" + | "reconnecting" + | "closed"; + +export interface RunWatcherOptions { + channels: () => string[]; + active: () => boolean; + onEvent: (event: SseEvent) => void; + poll: () => Promise<void> | void; + fastPollMs?: number; + slowPollMs?: number; + deps?: Partial<WatcherDeps>; +} + +export interface WatcherDeps { + subscribe: ( + channels: string[], + onEvent: (e: SseEvent) => void, + options: { onState?: (s: SseState) => void }, + ) => () => void; + now: () => number; + hidden: () => boolean; +} + +const MAX_CHANNELS = 20; + +export class RunWatcher { + connection = $state<WatchConnection>("idle"); + now = $state(Date.now()); + + private opts: Required<Omit<RunWatcherOptions, "deps">>; + private deps: WatcherDeps; + private unsubscribe: (() => void) | null = null; + private channelsKey = ""; + private ticker: ReturnType<typeof setInterval> | null = null; + private lastPoll = 0; + private polling = false; + private destroyed = false; + private onVisibility = () => { + if (!this.deps.hidden()) void this.pollNow(); + }; + + constructor(options: RunWatcherOptions) { + this.opts = { + fastPollMs: 3_000, + slowPollMs: 10_000, + ...options, + } as Required<Omit<RunWatcherOptions, "deps">>; + this.deps = { + subscribe: sseSubscribe, + now: () => Date.now(), + hidden: () => typeof document !== "undefined" && document.hidden, + ...options.deps, + }; + } + + start(): void { + this.destroyed = false; + this.lastPoll = this.deps.now(); + if (typeof document !== "undefined") { + document.addEventListener("visibilitychange", this.onVisibility); + } + this.sync(); + } + + /** Re-read `channels()` / `active()`: (re)connect, start or stop the ticker. */ + sync(): void { + if (this.destroyed) return; + const active = this.opts.active(); + const channels = active ? this.opts.channels().slice(0, MAX_CHANNELS) : []; + const key = channels.join(","); + if (key !== this.channelsKey) { + this.unsubscribe?.(); + this.unsubscribe = null; + this.channelsKey = key; + if (channels.length > 0) { + this.connection = "connecting"; + this.unsubscribe = this.deps.subscribe( + channels, + (e) => this.opts.onEvent(e), + { + onState: (s) => { + this.connection = s; + }, + }, + ); + } else { + this.connection = "closed"; + } + } + if (active && this.ticker === null) { + this.ticker = setInterval(() => this.tick(), 1_000); + } else if (!active && this.ticker !== null) { + clearInterval(this.ticker); + this.ticker = null; + } + } + + /** Poll now (manual refresh, visibility change); skipped while one is in flight. */ + async pollNow(): Promise<void> { + if (this.polling || this.destroyed) return; + this.polling = true; + this.lastPoll = this.deps.now(); + try { + await this.opts.poll(); + } finally { + this.polling = false; + this.sync(); + } + } + + destroy(): void { + this.destroyed = true; + this.unsubscribe?.(); + this.unsubscribe = null; + this.channelsKey = ""; + if (this.ticker !== null) clearInterval(this.ticker); + this.ticker = null; + if (typeof document !== "undefined") { + document.removeEventListener("visibilitychange", this.onVisibility); + } + this.connection = "closed"; + } + + private tick(): void { + const now = this.deps.now(); + this.now = now; + if (!this.opts.active() || this.deps.hidden()) return; + const every = + this.connection === "open" ? this.opts.slowPollMs : this.opts.fastPollMs; + if (now - this.lastPoll >= every) void this.pollNow(); + } +} diff --git a/frontend/src/lib/components/analysis/tools.ts b/frontend/src/lib/components/analysis/tools.ts new file mode 100644 index 00000000..c712ee9a --- /dev/null +++ b/frontend/src/lib/components/analysis/tools.ts @@ -0,0 +1,62 @@ +/** + * The analyzer tool catalog (names, icons, modes, install state) for analysis + * pages, loaded once per page load. `loadTools(true)` refetches (after the user + * installed tools in another tab). + */ +import { getToolCatalog, type AnalyzerTool } from "$lib/api/analyzers"; + +let cache: Promise<AnalyzerTool[]> | null = null; + +export function loadTools(force = false): Promise<AnalyzerTool[]> { + if (!cache || force) { + cache = getToolCatalog().catch((error) => { + cache = null; + throw error; + }); + } + return cache; +} + +export function toolIndex(tools: AnalyzerTool[]): Map<string, AnalyzerTool> { + return new Map(tools.map((t) => [t.slug, t])); +} + +/** Category display order and labels (unknown categories follow, never hidden). */ +export const CATEGORY_ORDER = [ + "security", + "dependencies", + "secrets", + "lint", + "performance", + "ai", +]; + +export const CATEGORY_LABELS: Record<string, string> = { + security: "Security", + dependencies: "Dependencies", + secrets: "Secrets", + lint: "Lint & quality", + performance: "Performance & complexity", + ai: "AI review", +}; + +export function categoryLabel(category: string): string { + return ( + CATEGORY_LABELS[category] ?? + category.charAt(0).toUpperCase() + category.slice(1) + ); +} + +export function groupByCategory<T extends { category: string }>( + items: T[], +): { category: string; items: T[] }[] { + const extra = [...new Set(items.map((t) => t.category))].filter( + (c) => !CATEGORY_ORDER.includes(c), + ); + return [...CATEGORY_ORDER, ...extra] + .map((category) => ({ + category, + items: items.filter((t) => t.category === category), + })) + .filter((g) => g.items.length > 0); +} diff --git a/frontend/src/lib/components/analyzers/CoveragePanel.svelte b/frontend/src/lib/components/analyzers/CoveragePanel.svelte new file mode 100644 index 00000000..60deefd0 --- /dev/null +++ b/frontend/src/lib/components/analyzers/CoveragePanel.svelte @@ -0,0 +1,122 @@ +<script lang="ts"> + /** + * CoveragePanel — what the user's generated apps contain, and whether the + * installed analyzers measure it. One row per technology found in the apps + * (most common first): share of apps, the installed tools that scan it + * specifically, and a gap warning when none do. A tool that matches nothing + * the models wrote reports "0 findings", which reads as "clean" in model + * comparisons — this panel is where that shows up before a study. + * + * Props: + * - `coverage`, `meta`, `catalog`, `installedSlugs` (tools installed and ready). + * - `selected` (technology slug or ''), `onselect(slug)`: rows filter the shop table. + */ + import * as Card from '$lib/components/ui/card'; + import StatusBadge from '$lib/components/StatusBadge.svelte'; + import { cn } from '$lib/utils'; + import type { AnalyzerTool, AppCoverage, CatalogMeta } from '$lib/api/analyzers'; + + interface Props { + coverage: AppCoverage; + meta: CatalogMeta | null; + catalog: AnalyzerTool[]; + installedSlugs: Set<string>; + selected: string; + onselect: (slug: string) => void; + } + + let { coverage, meta, catalog, installedSlugs, selected, onselect }: Props = $props(); + + const labels = $derived(new Map((meta?.technologies ?? []).map((t) => [t.slug, t.label]))); + const groupOf = $derived(new Map((meta?.technologies ?? []).map((t) => [t.slug, t.group]))); + + // "Any source" tools (secret scanners, typo checkers…) apply everywhere and + // would make every row look covered; rows count technology-specific tools. + const rows = $derived( + Object.entries(coverage.technologies) + .filter(([slug]) => slug !== 'generic') + .sort((a, b) => b[1] - a[1] || (labels.get(a[0]) ?? a[0]).localeCompare(labels.get(b[0]) ?? b[0])) + .map(([slug, apps]) => { + const tools = catalog.filter((t) => t.technologies?.includes(slug)); + const installed = tools.filter((t) => installedSlugs.has(t.slug)); + return { + slug, + apps, + label: labels.get(slug) ?? slug, + runtime: groupOf.get(slug) === 'runtime', + installed, + available: tools.length - installed.length, + total: tools.length, + }; + }) + ); + + const gaps = $derived(rows.filter((r) => r.installed.length === 0)); + + function pct(n: number): number { + return coverage.apps_total ? Math.round((n / coverage.apps_total) * 100) : 0; + } + + function names(tools: AnalyzerTool[], max = 3): string { + const shown = tools.slice(0, max).map((t) => t.name); + return tools.length > max ? `${shown.join(', ')} +${tools.length - max}` : shown.join(', '); + } +</script> + +<Card.Root class="gap-0 overflow-clip py-0"> + <Card.Header class="border-b border-border px-4 py-3"> + <Card.Title class="text-sm">What your generated apps contain</Card.Title> + <Card.Description class="text-xs"> + {#if coverage.apps_total === 0} + No generated apps yet. Once models have written some, this shows which tools can measure their code. + {:else} + Across your {coverage.apps_total} most recent generated app{coverage.apps_total === 1 ? '' : 's'}, and which installed tools measure each part. + {#if gaps.length > 0} + <span class="text-warning-fg">{gaps.length} {gaps.length === 1 ? 'technology is' : 'technologies are'} not measured by any installed tool.</span> + {/if} + Select a row to list its tools. + {/if} + </Card.Description> + </Card.Header> + {#if rows.length > 0} + <!-- Two columns on wide screens; each cell keeps a hairline under it. --> + <ul class="grid xl:grid-cols-2" aria-label="Technologies in your generated apps"> + {#each rows as row (row.slug)} + <li class="border-b border-border xl:odd:border-r"> + <button + type="button" + class={cn( + 'grid h-full w-full cursor-pointer grid-cols-[minmax(7rem,9rem)_1fr] items-center gap-x-4 gap-y-1 px-4 py-2 text-left text-xs transition-colors hover:bg-muted/40 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-inset focus-visible:ring-ring/40 sm:grid-cols-[9rem_10rem_1fr]', + selected === row.slug && 'bg-primary/5' + )} + aria-pressed={selected === row.slug} + onclick={() => onselect(selected === row.slug ? '' : row.slug)} + > + <span class="font-medium text-foreground">{row.label}</span> + <span class="flex items-center gap-2" title="{row.apps} of {coverage.apps_total} apps"> + <span class="h-1.5 w-14 overflow-hidden rounded-full bg-muted" aria-hidden="true"> + <span class="block h-full rounded-full bg-primary/70" style:width="{pct(row.apps)}%"></span> + </span> + <span class="whitespace-nowrap tabular-nums text-muted-foreground">{row.apps}/{coverage.apps_total} apps</span> + </span> + <span class="col-span-2 flex min-w-0 flex-wrap items-center gap-x-2 gap-y-1 sm:col-span-1"> + {#if row.installed.length > 0} + <StatusBadge status="measured" tone="success" label="{row.installed.length} installed" /> + <span class="truncate text-muted-foreground">{names(row.installed)}</span> + {:else if row.total > 0} + <StatusBadge status="gap" tone="warning" label="Not measured" /> + <span class="text-muted-foreground">{row.available} tool{row.available === 1 ? '' : 's'} in the shop</span> + {:else} + <StatusBadge status="gap" tone="warning" label="No tool" /> + <span class="text-muted-foreground">Only language-agnostic tools read it</span> + {/if} + {#if row.runtime && row.installed.length > 0} + <span class="text-muted-foreground">· needs the app running</span> + {/if} + </span> + </button> + </li> + {/each} + </ul> + {/if} +</Card.Root> diff --git a/frontend/src/lib/components/analyzers/ToolConfigDialog.svelte b/frontend/src/lib/components/analyzers/ToolConfigDialog.svelte index f84f8ed1..31009eb0 100644 --- a/frontend/src/lib/components/analyzers/ToolConfigDialog.svelte +++ b/frontend/src/lib/components/analyzers/ToolConfigDialog.svelte @@ -2,7 +2,6 @@ import * as Dialog from '$lib/components/ui/dialog'; import * as Select from '$lib/components/ui/select'; import { Button } from '$lib/components/ui/button'; - import { Badge } from '$lib/components/ui/badge'; import { Input } from '$lib/components/ui/input'; import { Label } from '$lib/components/ui/label'; import { Switch } from '$lib/components/ui/switch'; @@ -14,7 +13,7 @@ import CheckCircle2 from '@lucide/svelte/icons/check-circle-2'; import XCircle from '@lucide/svelte/icons/circle-x'; import ExternalLink from '@lucide/svelte/icons/external-link'; - import { severityColors } from '$lib/constants/analysis'; + import LevelBadge from '$lib/components/analysis/LevelBadge.svelte'; import { numberInputValue } from '$lib/components/generator/llm-params'; import { testTool, @@ -219,9 +218,7 @@ <ul class="mt-1 space-y-1"> {#each testResult.findings.slice(0, 5) as f, i (i)} <li class="flex items-center gap-2 text-xs"> - <Badge class={severityColors[(f.severity as string) ?? 'info'] ?? ''}> - {f.severity} - </Badge> + <LevelBadge label={f.tool_severity as string} /> <span class="truncate">{f.title}</span> </li> {/each} diff --git a/frontend/src/lib/components/analyzers/ToolIcon.svelte b/frontend/src/lib/components/analyzers/ToolIcon.svelte index b1e27ffd..4071866f 100644 --- a/frontend/src/lib/components/analyzers/ToolIcon.svelte +++ b/frontend/src/lib/components/analyzers/ToolIcon.svelte @@ -6,10 +6,13 @@ import Boxes from '@lucide/svelte/icons/boxes'; import Braces from '@lucide/svelte/icons/braces'; import CheckCheck from '@lucide/svelte/icons/check-check'; + import Bug from '@lucide/svelte/icons/bug'; import Container from '@lucide/svelte/icons/container'; import Copy from '@lucide/svelte/icons/copy'; import FileCode from '@lucide/svelte/icons/file-code'; import Gauge from '@lucide/svelte/icons/gauge'; + import Globe from '@lucide/svelte/icons/globe'; + import History from '@lucide/svelte/icons/history'; import Key from '@lucide/svelte/icons/key'; import KeyRound from '@lucide/svelte/icons/key-round'; import ListChecks from '@lucide/svelte/icons/list-checks'; @@ -20,8 +23,10 @@ import Palette from '@lucide/svelte/icons/palette'; import Puzzle from '@lucide/svelte/icons/puzzle'; import Radar from '@lucide/svelte/icons/radar'; + import ScanSearch from '@lucide/svelte/icons/scan-search'; import ShieldAlert from '@lucide/svelte/icons/shield-alert'; import ShieldCheck from '@lucide/svelte/icons/shield-check'; + import ShieldHalf from '@lucide/svelte/icons/shield-half'; import Skull from '@lucide/svelte/icons/skull'; import Snowflake from '@lucide/svelte/icons/snowflake'; import Sparkles from '@lucide/svelte/icons/sparkles'; @@ -39,11 +44,14 @@ bot: Bot, boxes: Boxes, braces: Braces, + bug: Bug, 'check-check': CheckCheck, container: Container, copy: Copy, 'file-code': FileCode, gauge: Gauge, + globe: Globe, + history: History, key: Key, 'key-round': KeyRound, 'package-check': PackageCheck, @@ -51,8 +59,10 @@ 'package-x': PackageX, palette: Palette, radar: Radar, + 'scan-search': ScanSearch, 'shield-alert': ShieldAlert, 'shield-check': ShieldCheck, + 'shield-half': ShieldHalf, skull: Skull, snowflake: Snowflake, sparkles: Sparkles, @@ -78,7 +88,7 @@ interface Props { tool: { slug: string; icon?: string; category?: string; name?: string }; - size?: 'sm' | 'md' | 'lg'; + size?: 'xs' | 'sm' | 'md' | 'lg'; class?: string; } @@ -87,8 +97,8 @@ const logo = $derived(toolLogos[tool.slug]); const IconComponent = $derived(icons[tool.icon ?? ''] ?? categoryIcons[tool.category ?? ''] ?? Puzzle); - const tile = { sm: 'size-7 rounded', md: 'size-9 rounded-md', lg: 'size-11 rounded-md' }; - const glyph = { sm: 'size-4', md: 'size-5', lg: 'size-6' }; + const tile = { xs: 'size-5 rounded-sm', sm: 'size-7 rounded', md: 'size-9 rounded-md', lg: 'size-11 rounded-md' }; + const glyph = { xs: 'size-3.5', sm: 'size-4', md: 'size-5', lg: 'size-6' }; </script> {#if logo} diff --git a/frontend/src/lib/components/applications/ApplicationAnalyses.svelte b/frontend/src/lib/components/applications/ApplicationAnalyses.svelte index 6ba7bf9f..73f2d103 100644 --- a/frontend/src/lib/components/applications/ApplicationAnalyses.svelte +++ b/frontend/src/lib/components/applications/ApplicationAnalyses.svelte @@ -1,70 +1,89 @@ <script lang="ts"> -import { onMount } from 'svelte'; -import * as Card from '$lib/components/ui/card'; -import StatusBadge from '$lib/components/StatusBadge.svelte'; -import LoadingState from '$lib/components/LoadingState.svelte'; -import EmptyState from '$lib/components/EmptyState.svelte'; -import ErrorAlert from '$lib/components/ErrorAlert.svelte'; -import { Button } from '$lib/components/ui/button'; -import ChartBar from '@lucide/svelte/icons/chart-bar'; -import RefreshCw from '@lucide/svelte/icons/refresh-cw'; -import { listRuns, type AnalysisRunListItem } from '$lib/api/runs'; -import ChevronRight from '@lucide/svelte/icons/chevron-right'; -import { rowLink } from '$lib/components/table'; + /** + * ApplicationAnalyses — the analysis runs of one generated app (newest first, + * live while any is active) with links to analyze it again and to the full + * list scoped to this app. + * + * Props: `jobId`. + */ + import { onDestroy, onMount } from 'svelte'; + import * as Card from '$lib/components/ui/card'; + import { Button } from '$lib/components/ui/button'; + import LoadingState from '$lib/components/LoadingState.svelte'; + import EmptyState from '$lib/components/EmptyState.svelte'; + import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import Microscope from '@lucide/svelte/icons/microscope'; + import Plus from '@lucide/svelte/icons/plus'; + import ChevronRight from '@lucide/svelte/icons/chevron-right'; + import { formatApiError } from '$lib/api/core'; + import { listRuns, type AnalysisRunListItem } from '$lib/api/runs'; + import { rowLink } from '$lib/components/table'; + import RunStatusCell from '$lib/components/analysis/RunStatusCell.svelte'; + import RunFindingsCell from '$lib/components/analysis/RunFindingsCell.svelte'; + import { RunWatcher } from '$lib/components/analysis/live.svelte'; + import { formatSpan, isActive, runDisplayName } from '$lib/components/analysis/format'; + import { formatDate } from '$lib/utils/formatters'; -interface Props { jobId: string } -let { jobId }: Props = $props(); - -let tasks = $state<AnalysisRunListItem[]>([]); -let loading = $state(true); -let error = $state(''); - -async function refresh() { - loading = true; - error = ''; - try { - const res = await listRuns({ generation_job_id: jobId, per_page: 50 }); - tasks = res.items; - } catch (e) { - error = (e as Error).message ?? 'Failed to load'; - } finally { - loading = false; + interface Props { + jobId: string; } -} + let { jobId }: Props = $props(); -onMount(refresh); + let runs = $state<AnalysisRunListItem[]>([]); + let total = $state(0); + let loading = $state(true); + let error = $state(''); + async function refresh() { + try { + const res = await listRuns({ generation_job_id: jobId, per_page: 10 }); + runs = res.items; + total = res.total; + error = ''; + } catch (e) { + error = formatApiError(e, 'Could not load analyses.'); + } finally { + loading = false; + watcher.sync(); + } + } -const summary = $derived.by(() => { - const total = tasks.length; - const done = tasks.filter(t => t.status === 'completed').length; - const running = tasks.filter(t => t.status === 'running').length; - const failed = tasks.filter(t => t.status === 'failed').length; - return { total, done, running, failed }; -}); + const activeIds = $derived(runs.filter((r) => isActive(r.status)).map((r) => r.id)); + let timer: ReturnType<typeof setTimeout> | undefined; + const watcher = new RunWatcher({ + channels: () => activeIds.map((id) => `analysis:${id}`), + active: () => activeIds.length > 0, + onEvent: () => { + clearTimeout(timer); + timer = setTimeout(() => void watcher.pollNow(), 500); + }, + poll: refresh, + }); -function findingsOf(t: AnalysisRunListItem): number | string { - const rs = t.summary || {}; - if (typeof rs.total_findings === 'number') return rs.total_findings; - if (typeof rs.findings === 'number') return rs.findings; - return '—'; -} + onMount(() => { + watcher.start(); + void refresh(); + }); + onDestroy(() => { + watcher.destroy(); + clearTimeout(timer); + }); </script> -<section id="analyses" class="space-y-4"> +<section id="analyses"> <Card.Root class="gap-0 overflow-clip py-0"> <Card.Header class="flex flex-row flex-wrap items-start justify-between gap-3 border-b border-border py-3"> <div class="space-y-1.5"> <Card.Title>Analyses</Card.Title> <Card.Description> - {summary.total} total · {summary.done} done · {summary.running} running · {summary.failed} failed + {#if total > 0}{total} run{total === 1 ? '' : 's'}{#if activeIds.length} · <span class="text-info-fg">{activeIds.length} active</span>{/if}{:else}Static and dynamic analysis of this app's code.{/if} </Card.Description> </div> <div class="flex flex-wrap items-center gap-2"> - <Button size="sm" variant="outline" onclick={refresh}> - <RefreshCw class="h-3.5 w-3.5" /> Refresh - </Button> - <Button size="sm" href="/analysis/create?generation_job_id={jobId}">New Analysis</Button> + {#if total > runs.length} + <Button size="sm" variant="outline" href="/analysis?job={jobId}">All {total}</Button> + {/if} + <Button size="sm" href="/analysis/create?generation_job_id={jobId}"><Plus />Analyze</Button> </div> </Card.Header> <Card.Content class="p-0"> @@ -72,12 +91,10 @@ function findingsOf(t: AnalysisRunListItem): number | string { <LoadingState label="loading analyses" class="py-8" /> {:else if error} <div class="p-4"><ErrorAlert message={error} onretry={refresh} /></div> - {:else if tasks.length === 0} + {:else if runs.length === 0} <div class="p-4"> - <EmptyState compact icon={ChartBar} title="No analyses yet" description="Run an analysis to scan this application for issues."> - {#snippet action()} - <Button size="sm" href="/analysis/create?generation_job_id={jobId}">New Analysis</Button> - {/snippet} + <EmptyState compact icon={Microscope} title="Not analyzed yet" description="Run your workspace's tools on this app's code (and on the running app, for dynamic tools)."> + {#snippet action()}<Button size="sm" href="/analysis/create?generation_job_id={jobId}"><Plus />Analyze</Button>{/snippet} </EmptyState> </div> {:else} @@ -85,31 +102,29 @@ function findingsOf(t: AnalysisRunListItem): number | string { <table> <thead> <tr> - <th>Task</th> + <th>Run</th> <th>Status</th> <th class="num">Findings</th> + <th class="num">Duration</th> <th>Created</th> - <th><span class="sr-only">Actions</span></th> + <th><span class="sr-only">Open</span></th> </tr> </thead> <tbody> - {#each tasks as t (t.id)} - {@const href = `/analysis/${t.id}`} - {@const findings = findingsOf(t)} + {#each runs as run (run.id)} + {@const href = `/analysis/${run.id}`} <!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_noninteractive_element_interactions (the name link is the keyboard path) --> - <tr class="row-link" onclick={rowLink(href)}> - <td class="max-w-0 w-[45%]"> - <a {href} class="block truncate font-medium hover:text-primary" title={t.name}>{t.name}</a> - </td> - <td class="whitespace-nowrap"><StatusBadge status={t.status} /></td> - <td class="num"> - {#if findings === '—'}<span class="text-muted-foreground">—</span>{:else}{findings}{/if} + <tr class="row-link align-top" onclick={rowLink(href)}> + <td class="max-w-0 w-[34%]"> + <a {href} class="block truncate font-medium hover:text-primary" title={runDisplayName(run)}>{runDisplayName(run)}</a> + <span class="cell-sub block font-mono text-[11px] text-muted-foreground">{run.tool_slugs.length} tools</span> </td> - <td class="whitespace-nowrap text-xs text-muted-foreground">{new Date(t.created_at).toLocaleString()}</td> - <td class="w-px whitespace-nowrap text-right"> - <Button size="sm" variant="ghost" class="size-8 p-0 text-muted-foreground hover:text-foreground" {href} title="View analysis" aria-label="View analysis"> - <ChevronRight class="size-4" /> - </Button> + <td><RunStatusCell {run} /></td> + <td class="num"><RunFindingsCell {run} /></td> + <td class="num text-muted-foreground">{isActive(run.status) ? '—' : formatSpan(run.duration_seconds)}</td> + <td class="whitespace-nowrap text-xs text-muted-foreground">{formatDate(run.created_at)}</td> + <td class="w-px text-right"> + <Button size="sm" variant="ghost" class="size-8 p-0 text-muted-foreground hover:text-foreground" {href} aria-label="Open {runDisplayName(run)}"><ChevronRight class="size-4" /></Button> </td> </tr> {/each} diff --git a/frontend/src/lib/components/applications/ApplicationHeader.svelte b/frontend/src/lib/components/applications/ApplicationHeader.svelte index 18ba204f..31aef637 100644 --- a/frontend/src/lib/components/applications/ApplicationHeader.svelte +++ b/frontend/src/lib/components/applications/ApplicationHeader.svelte @@ -1,7 +1,7 @@ <script lang="ts"> +import type { Snippet } from 'svelte'; import { Badge } from '$lib/components/ui/badge'; import { Button } from '$lib/components/ui/button'; -import { Alert, AlertDescription } from '$lib/components/ui/alert'; import PageHeader from '$lib/components/PageHeader.svelte'; import StatusBadge from '$lib/components/StatusBadge.svelte'; import Copy from '@lucide/svelte/icons/copy'; @@ -35,6 +35,8 @@ interface Props { onDelete: () => void; /** Opens the GitHub export dialog; the button is hidden without it. */ onExportGithub?: () => void; + /** Section tabs, rendered in the header's bottom strip (the page owns `Tabs.Root`). */ + nav?: Snippet; } let { @@ -49,6 +51,7 @@ let { onDownloadCode, onDelete, onExportGithub, + nav, }: Props = $props(); // Agent runs sample at the provider default unless a temperature was set explicitly. @@ -57,12 +60,12 @@ const temperature = $derived( ); const kpis = $derived([ - { label: 'Duration', value: fmtDur(job.duration_seconds), sub: job.started_at ? 'wall clock' : 'not started' }, - { label: 'Tokens', value: fmt(totalTokens, 0), sub: 'prompt + completion' }, - { label: 'Cost', value: fmtCost(totalCost), sub: 'USD' }, - { label: 'Lines of code', value: fmt(codeFootprint?.totalLines ?? 0, 0), sub: codeFootprint ? `${codeFootprint.fileCount} file${codeFootprint.fileCount === 1 ? '' : 's'}` : 'no code' }, - { label: 'Temperature', value: temperature, sub: 'sampling' }, - { label: 'Max tokens', value: fmt(job.max_tokens, 0), sub: 'per response' }, + { label: 'duration', value: fmtDur(job.duration_seconds), title: job.started_at ? 'Wall clock' : 'Not started' }, + { label: 'tokens', value: fmt(totalTokens, 0), title: 'Prompt + completion' }, + { label: 'cost', value: fmtCost(totalCost), title: 'USD' }, + { label: 'LOC', value: fmt(codeFootprint?.totalLines ?? 0, 0), title: codeFootprint ? `Lines of code in ${codeFootprint.fileCount} file${codeFootprint.fileCount === 1 ? '' : 's'}` : 'No code' }, + { label: 'temp', value: temperature, title: 'Sampling temperature' }, + { label: 'max tokens', value: fmt(job.max_tokens, 0), title: 'Max tokens per response' }, ]); </script> @@ -70,6 +73,7 @@ const kpis = $derived([ title={jobDescription(job)} documentTitle="{jobDescription(job)} · {job.model_name ?? 'Unknown model'}" back={{ href: '/applications', label: 'Applications' }} + {nav} > {#snippet meta()} <StatusBadge status={jobDisplayStatus(job)} tones={jobPhaseTones} /> @@ -139,28 +143,19 @@ const kpis = $derived([ {job.id.substring(0, 8)}<Copy class="size-3 opacity-60 group-hover/id:opacity-100" aria-hidden="true" /> <span class="sr-only">Copy job ID</span> </button> + <span aria-hidden="true" class="opacity-50">·</span> + <span class="header-stats"> + {#each kpis as kpi (kpi.label)} + <span class="header-stat" title={kpi.title}><b>{kpi.value}</b>{kpi.label}</span> + {/each} + </span> </div> + {#if job.status === 'failed'} + <p class="mt-1 flex min-w-0 items-center gap-1.5 text-danger-fg" role="alert"> + <AlertTriangle class="size-3.5 shrink-0" aria-hidden="true" /> + <span class="font-medium">Generation failed:</span> + <span class="min-w-0 truncate" title={job.error_message || undefined}>{job.error_message || 'Unknown error'}</span> + <a href="/applications/{job.id}/failure" class="shrink-0 underline hover:text-foreground">View details</a> + </p> + {/if} </PageHeader> - -{#if job.status === 'failed'} - <Alert variant="destructive"> - <AlertTriangle /> - <div class="flex flex-wrap items-center justify-between gap-2"> - <div class="min-w-0"> - <p class="font-medium">Generation failed</p> - <AlertDescription class="truncate">{job.error_message || 'Unknown error'}</AlertDescription> - </div> - <Button variant="outline" size="sm" href="/applications/{job.id}/failure">View Details</Button> - </div> - </Alert> -{/if} - -<div class="grid grid-cols-2 gap-3 sm:grid-cols-3 lg:grid-cols-6"> - {#each kpis as kpi (kpi.label)} - <div class="kpi-card"> - <span class="kpi-label">{kpi.label}</span> - <span class="kpi-value text-xl">{kpi.value}</span> - <span class="text-[11px] text-muted-foreground">{kpi.sub}</span> - </div> - {/each} -</div> diff --git a/frontend/src/lib/components/applications/JobCards.svelte b/frontend/src/lib/components/applications/JobCards.svelte index 4f595c60..46c29b28 100644 --- a/frontend/src/lib/components/applications/JobCards.svelte +++ b/frontend/src/lib/components/applications/JobCards.svelte @@ -56,7 +56,8 @@ }: Props = $props(); </script> -<div class="data-list"> +<!-- Inside the page's table card: the card supplies the border. --> +<div class="data-list rounded-none border-0 shadow-none"> {#each jobs as job (job.id)} {@const container = containersByJobId[job.id]} {@const isBusy = containerBusy[job.id]} @@ -85,7 +86,7 @@ {#if container}<StatusBadge status={container.status} tones={containerTones} />{/if} <span class="font-mono tabular-nums text-muted-foreground">{formatDuration(job.duration_seconds)}</span> {#if job.cost_usd != null && job.cost_usd > 0}<span class="font-mono tabular-nums text-muted-foreground">${job.cost_usd.toFixed(job.cost_usd < 0.1 ? 3 : 2)}</span>{/if} - {#if job.pass_rate != null}<span class="font-mono tabular-nums text-muted-foreground" title="Smoke-test pass rate">smoke {(job.pass_rate * 100).toFixed(0)}%</span>{/if} + {#if job.pass_rate != null}<span class="font-mono tabular-nums text-muted-foreground" title="Share of checks passed: the conformance suite when the run was graded, otherwise the runtime smoke test">pass {(job.pass_rate * 100).toFixed(0)}%</span>{/if} </div> {#if job.error_message} <p class="flex items-center gap-1 pl-11 text-[11px] text-danger-fg" title={job.error_message}> diff --git a/frontend/src/lib/components/applications/JobProvenance.svelte b/frontend/src/lib/components/applications/JobProvenance.svelte index fa58b78b..626dceae 100644 --- a/frontend/src/lib/components/applications/JobProvenance.svelte +++ b/frontend/src/lib/components/applications/JobProvenance.svelte @@ -14,6 +14,7 @@ } from '$lib/api/generation'; import { harnessOptionChips, summarizeBudget, summarizeLLMParams } from '$lib/components/generator'; import { batchHref } from '$lib/components/generator/batches'; + import SectionHeading from '$lib/components/SectionHeading.svelte'; import { copyText } from './utils'; import Copy from '@lucide/svelte/icons/copy'; import LoaderCircle from '@lucide/svelte/icons/loader-circle'; @@ -200,8 +201,8 @@ </div> </dl> - <div class="space-y-2 border-t border-border pt-3"> - <p class="section-label">Reproduce</p> + <div class="space-y-2 pt-1"> + <SectionHeading size="sm" level={3} title="Reproduce" /> <div class="grid gap-2 sm:grid-cols-3"> <div class="space-y-1"> <Button variant="outline" size="sm" class="w-full sm:w-auto" disabled={!canClone || !!busy} onclick={onReplay}> diff --git a/frontend/src/lib/components/applications/JobsTable.svelte b/frontend/src/lib/components/applications/JobsTable.svelte index 6c07d3b1..af848070 100644 --- a/frontend/src/lib/components/applications/JobsTable.svelte +++ b/frontend/src/lib/components/applications/JobsTable.svelte @@ -1,5 +1,4 @@ <script lang="ts"> - import * as Card from '$lib/components/ui/card'; import StatusBadge from '$lib/components/StatusBadge.svelte'; import { jobDisplayStatus, jobPhaseTones } from '$lib/components/generator/batches'; import { containerTones } from '$lib/constants/colors'; @@ -9,7 +8,6 @@ jobDescription, timeAgo, } from './utils'; - import type { Snippet } from 'svelte'; import type { GenerationJobList } from '$lib/api/client'; import type { ContainerInstance } from '$lib/api/runtime'; import Eye from '@lucide/svelte/icons/eye'; @@ -29,7 +27,7 @@ import Layers from '@lucide/svelte/icons/layers'; import ChevronRight from '@lucide/svelte/icons/chevron-right'; import ProviderIcon from '$lib/components/ProviderIcon.svelte'; - import { SortableTh, TableToolbar, rowLink, tableFrame, shortModelName, modelNamePrefix } from '$lib/components/table'; + import { SortableTh, rowLink, tableFrame, shortModelName, modelNamePrefix } from '$lib/components/table'; interface Props { jobs: GenerationJobList[]; @@ -46,8 +44,6 @@ onDelete: (jobId: string) => void; onRetry: (jobId: string) => void; onCopyId: (jobId: string) => void; - /** Card footer under the rows (pagination). */ - footer?: Snippet; } let { jobs, @@ -64,179 +60,169 @@ onDelete, onRetry, onCopyId, - footer, }: Props = $props(); </script> -<Card.Root class="gap-0 overflow-clip py-0"> - <Card.Content class="p-0"> - <TableToolbar> - {#snippet summary()} - {jobs.length} on this page · click a row to open - {/snippet} - </TableToolbar> - <div class="table-frame" {@attach tableFrame}> - <table> - <thead> - <tr> - <SortableTh label="Application" field="created_at" {sortBy} {sortDir} onSort={onToggleSort} help="Sorted by creation time" /> - <SortableTh label="Model" field="model_name" {sortBy} {sortDir} onSort={onToggleSort} ascOnly /> - <th>Status</th> - <th>App</th> - <SortableTh label="Duration" field="duration_seconds" {sortBy} {sortDir} onSort={onToggleSort} align="right" /> - <th class="text-right"><span class="sr-only">Actions</span></th> - </tr> - </thead> - <tbody> - {#each jobs as job (job.id)} - {@const container = containersByJobId[job.id]} - {@const isBusy = containerBusy[job.id]} - {@const href = `/applications/${job.id}`} - {@const ModeIcon = job.mode === 'custom' ? Pencil : job.mode === 'scaffolding' ? Layers : Bot} - <!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_noninteractive_element_interactions (the name link is the keyboard path) --> - <tr class="row-link group" onclick={rowLink(href)}> - <!-- Application --> - <td class="max-w-0 w-[30%]"> - <div class="flex items-center gap-2.5"> - <span class="flex size-7 shrink-0 items-center justify-center rounded-md border border-border bg-muted/50 text-muted-foreground transition-colors group-hover:text-primary" title={job.mode}> - <ModeIcon class="size-3.5" aria-hidden="true" /> - </span> - <div class="min-w-0"> - <a {href} class="block truncate font-medium hover:text-primary" title={jobDescription(job)}>{jobDescription(job)}</a> - <p class="cell-sub truncate text-[11px] text-muted-foreground"> - <span class="capitalize">{job.mode}</span> - {#if job.harness}<span class="mx-1 opacity-50">·</span><span class="font-mono" title="Harness">{job.harness}</span>{/if} - <span class="mx-1 opacity-50">·</span><time class="font-mono tabular-nums" datetime={job.created_at} title={new Date(job.created_at).toLocaleString()}>{timeAgo(job.created_at)}</time> - </p> - </div> +<!-- Rendered inside the page's table card, under its filter strip and toolbar. --> +<div class="table-frame" {@attach tableFrame}> + <table> + <thead> + <tr> + <SortableTh label="Application" field="created_at" {sortBy} {sortDir} onSort={onToggleSort} help="Sorted by creation time" /> + <SortableTh label="Model" field="model_name" {sortBy} {sortDir} onSort={onToggleSort} ascOnly /> + <th>Status</th> + <th>App</th> + <SortableTh label="Duration" field="duration_seconds" {sortBy} {sortDir} onSort={onToggleSort} align="right" /> + <th class="text-right"><span class="sr-only">Actions</span></th> + </tr> + </thead> + <tbody> + {#each jobs as job (job.id)} + {@const container = containersByJobId[job.id]} + {@const isBusy = containerBusy[job.id]} + {@const href = `/applications/${job.id}`} + {@const ModeIcon = job.mode === 'custom' ? Pencil : job.mode === 'scaffolding' ? Layers : Bot} + <!-- svelte-ignore a11y_click_events_have_key_events, a11y_no_noninteractive_element_interactions (the name link is the keyboard path) --> + <tr class="row-link group" onclick={rowLink(href)}> + <!-- Application --> + <td class="max-w-0 w-[30%]"> + <div class="flex items-center gap-2.5"> + <span class="flex size-7 shrink-0 items-center justify-center rounded-md border border-border bg-muted/50 text-muted-foreground transition-colors group-hover:text-primary" title={job.mode}> + <ModeIcon class="size-3.5" aria-hidden="true" /> + </span> + <div class="min-w-0"> + <a {href} class="block truncate font-medium hover:text-primary" title={jobDescription(job)}>{jobDescription(job)}</a> + <p class="cell-sub truncate text-[11px] text-muted-foreground"> + <span class="capitalize">{job.mode}</span> + {#if job.harness}<span class="mx-1 opacity-50">·</span><span class="font-mono" title="Harness">{job.harness}</span>{/if} + <span class="mx-1 opacity-50">·</span><time class="font-mono tabular-nums" datetime={job.created_at} title={new Date(job.created_at).toLocaleString()}>{timeAgo(job.created_at)}</time> + </p> + </div> + </div> + </td> + + <!-- Model --> + <td class="max-w-0 w-[22%]"> + {#if job.model_name} + {@const provider = modelNamePrefix(job.model_name) || job.model_id_str?.split('/')[0] || ''} + <div class="flex items-center gap-2" title={job.model_name}> + {#if job.model_id_str}<ProviderIcon provider={job.model_id_str.split('/')[0]} size={14} class="shrink-0" />{/if} + <div class="min-w-0"> + <span class="block truncate">{shortModelName(job.model_name)}</span> + {#if provider}<span class="cell-sub block truncate text-[11px] text-muted-foreground">{provider}</span>{/if} </div> - </td> + </div> + {:else} + <span class="text-muted-foreground">—</span> + {/if} + </td> - <!-- Model --> - <td class="max-w-0 w-[22%]"> - {#if job.model_name} - {@const provider = modelNamePrefix(job.model_name) || job.model_id_str?.split('/')[0] || ''} - <div class="flex items-center gap-2" title={job.model_name}> - {#if job.model_id_str}<ProviderIcon provider={job.model_id_str.split('/')[0]} size={14} class="shrink-0" />{/if} - <div class="min-w-0"> - <span class="block truncate">{shortModelName(job.model_name)}</span> - {#if provider}<span class="cell-sub block truncate text-[11px] text-muted-foreground">{provider}</span>{/if} - </div> - </div> - {:else} - <span class="text-muted-foreground">—</span> - {/if} - </td> + <!-- Status --> + <td class="whitespace-nowrap"> + <StatusBadge status={jobDisplayStatus(job)} tones={jobPhaseTones} /> + {#if job.error_message} + <p class="cell-sub mt-1 flex max-w-[200px] items-center gap-1 text-[11px] text-danger-fg" title={job.error_message}> + <AlertTriangle class="size-3 shrink-0" aria-hidden="true" /> + <span class="truncate">{job.error_message}</span> + </p> + {:else if job.pass_rate != null} + <p class="cell-sub mt-1 font-mono text-[11px] text-muted-foreground tabular-nums" title="Share of checks passed: the conformance suite when the run was graded, otherwise the runtime smoke test">pass {(job.pass_rate * 100).toFixed(0)}%</p> + {/if} + </td> - <!-- Status --> - <td class="whitespace-nowrap"> - <StatusBadge status={jobDisplayStatus(job)} tones={jobPhaseTones} /> - {#if job.error_message} - <p class="cell-sub mt-1 flex max-w-[200px] items-center gap-1 text-[11px] text-danger-fg" title={job.error_message}> - <AlertTriangle class="size-3 shrink-0" aria-hidden="true" /> - <span class="truncate">{job.error_message}</span> - </p> - {:else if job.pass_rate != null} - <p class="cell-sub mt-1 font-mono text-[11px] text-muted-foreground tabular-nums" title="Smoke-test pass rate">smoke {(job.pass_rate * 100).toFixed(0)}%</p> - {/if} - </td> - - <!-- App / container: state + the one next step --> - <td class="whitespace-nowrap"> - {#if container} - <div class="flex items-center gap-2"> - <StatusBadge status={container.status} tones={containerTones} /> - {#if container.status === 'running' && container.app_url} - <Button - id="access-app-btn-{job.id}" - variant="ghost" - size="sm" - href={container.app_url} - target="_blank" - rel="noopener noreferrer" - class="h-7 gap-1 px-2 text-xs text-primary hover:text-primary" - title="Open the app on port {container.app_port}" - > - Open<ExternalLink class="size-3" /> - </Button> - {:else if container.status === 'stopped' || container.status === 'failed'} - <Button - id="start-container-btn-{job.id}" - variant="ghost" - size="sm" - class="h-7 gap-1 px-2 text-xs" - disabled={isBusy != null} - title={container.status === 'failed' ? container.last_error || 'Container crashed' : undefined} - onclick={() => onStart(job.id, container.id)} - > - {#if isBusy === 'start'}<LoaderCircle class="size-3 animate-spin" />Starting…{:else}<Play class="size-3" />{container.status === 'failed' ? 'Restart' : 'Start'}{/if} - </Button> - {/if} - </div> - {:else if job.status === 'completed'} + <!-- App / container: state + the one next step --> + <td class="whitespace-nowrap"> + {#if container} + <div class="flex items-center gap-2"> + <StatusBadge status={container.status} tones={containerTones} /> + {#if container.status === 'running' && container.app_url} + <Button + id="access-app-btn-{job.id}" + variant="ghost" + size="sm" + href={container.app_url} + target="_blank" + rel="noopener noreferrer" + class="h-7 gap-1 px-2 text-xs text-primary hover:text-primary" + title="Open the app on port {container.app_port}" + > + Open<ExternalLink class="size-3" /> + </Button> + {:else if container.status === 'stopped' || container.status === 'failed'} <Button - id="provision-app-btn-{job.id}" + id="start-container-btn-{job.id}" variant="ghost" size="sm" - class="-ml-2 h-7 gap-1.5 px-2 text-xs text-muted-foreground hover:text-primary" + class="h-7 gap-1 px-2 text-xs" disabled={isBusy != null} - title="Build and start a container for this app" - onclick={() => onBuild(job.id)} + title={container.status === 'failed' ? container.last_error || 'Container crashed' : undefined} + onclick={() => onStart(job.id, container.id)} > - {#if isBusy === 'build'}<LoaderCircle class="size-3.5 animate-spin" />Building…{:else}<Hammer class="size-3.5" />Provision{/if} + {#if isBusy === 'start'}<LoaderCircle class="size-3 animate-spin" />Starting…{:else}<Play class="size-3" />{container.status === 'failed' ? 'Restart' : 'Start'}{/if} </Button> - {:else} - <span class="text-muted-foreground">—</span> {/if} - </td> + </div> + {:else if job.status === 'completed'} + <Button + id="provision-app-btn-{job.id}" + variant="ghost" + size="sm" + class="-ml-2 h-7 gap-1.5 px-2 text-xs text-muted-foreground hover:text-primary" + disabled={isBusy != null} + title="Build and start a container for this app" + onclick={() => onBuild(job.id)} + > + {#if isBusy === 'build'}<LoaderCircle class="size-3.5 animate-spin" />Building…{:else}<Hammer class="size-3.5" />Provision{/if} + </Button> + {:else} + <span class="text-muted-foreground">—</span> + {/if} + </td> - <!-- Duration --> - <td class="num"> - <span class="text-xs">{formatDuration(job.duration_seconds)}</span> - {#if job.cost_usd != null && job.cost_usd > 0} - <span class="cell-sub block text-[11px] text-muted-foreground">${job.cost_usd.toFixed(job.cost_usd < 0.1 ? 3 : 2)}</span> - {/if} - </td> + <!-- Duration --> + <td class="num"> + <span class="text-xs">{formatDuration(job.duration_seconds)}</span> + {#if job.cost_usd != null && job.cost_usd > 0} + <span class="cell-sub block text-[11px] text-muted-foreground">${job.cost_usd.toFixed(job.cost_usd < 0.1 ? 3 : 2)}</span> + {/if} + </td> - <!-- Actions: view stays visible, the rest appear on hover/focus --> - <td class="w-px whitespace-nowrap"> - <div class="flex items-center justify-end gap-0.5"> - <div class="row-actions-more flex items-center gap-0.5"> - {#if container?.status === 'running'} - {@render iconAction(`action-stop-${job.id}`, 'Stop container', Square, () => onStop(job.id, container.id), isBusy === 'stop', isBusy != null)} - {/if} - {#if container && container.status !== 'building' && container.status !== 'pending'} - {@render iconAction(`action-remove-container-${job.id}`, 'Remove container (keeps the code)', Server, () => onRemoveContainer(job.id, container.id), isBusy === 'remove', isBusy != null, true)} - {/if} - {#if job.status === 'failed' || job.status === 'cancelled'} - {@render iconAction(`action-retry-${job.id}`, 'Retry job', RotateCcw, () => onRetry(job.id))} - {/if} - {#if job.status === 'pending' || job.status === 'running'} - {@render iconAction(`action-cancel-${job.id}`, 'Cancel job', Ban, () => onCancel(job.id))} - {/if} - {@render iconAction(`action-copy-${job.id}`, 'Copy job ID', Copy, () => onCopyId(job.id))} - {#if job.status !== 'running'} - {@render iconAction(`action-delete-${job.id}`, 'Delete job', Trash2, () => onDelete(job.id), false, false, true)} - {/if} - </div> - {#if job.status === 'failed'} - <Button id="action-failure-{job.id}" variant="ghost" size="sm" class="size-8 p-0 text-danger-fg hover:bg-destructive/10" href="/applications/{job.id}/failure" title="Failure logs" aria-label="Failure logs"> - <AlertTriangle class="size-4" /> - </Button> - {/if} - <Button id="action-view-{job.id}" variant="ghost" size="sm" class="size-8 p-0 text-muted-foreground hover:text-foreground" {href} title="View details" aria-label="View details"> - <ChevronRight class="size-4" /> - </Button> - </div> - </td> - </tr> - {/each} - </tbody> - </table> - </div> - {@render footer?.()} - </Card.Content> -</Card.Root> + <!-- Actions: view stays visible, the rest appear on hover/focus --> + <td class="w-px whitespace-nowrap"> + <div class="flex items-center justify-end gap-0.5"> + <div class="row-actions-more flex items-center gap-0.5"> + {#if container?.status === 'running'} + {@render iconAction(`action-stop-${job.id}`, 'Stop container', Square, () => onStop(job.id, container.id), isBusy === 'stop', isBusy != null)} + {/if} + {#if container && container.status !== 'building' && container.status !== 'pending'} + {@render iconAction(`action-remove-container-${job.id}`, 'Remove container (keeps the code)', Server, () => onRemoveContainer(job.id, container.id), isBusy === 'remove', isBusy != null, true)} + {/if} + {#if job.status === 'failed' || job.status === 'cancelled'} + {@render iconAction(`action-retry-${job.id}`, 'Retry job', RotateCcw, () => onRetry(job.id))} + {/if} + {#if job.status === 'pending' || job.status === 'running'} + {@render iconAction(`action-cancel-${job.id}`, 'Cancel job', Ban, () => onCancel(job.id))} + {/if} + {@render iconAction(`action-copy-${job.id}`, 'Copy job ID', Copy, () => onCopyId(job.id))} + {#if job.status !== 'running'} + {@render iconAction(`action-delete-${job.id}`, 'Delete job', Trash2, () => onDelete(job.id), false, false, true)} + {/if} + </div> + {#if job.status === 'failed'} + <Button id="action-failure-{job.id}" variant="ghost" size="sm" class="size-8 p-0 text-danger-fg hover:bg-destructive/10" href="/applications/{job.id}/failure" title="Failure logs" aria-label="Failure logs"> + <AlertTriangle class="size-4" /> + </Button> + {/if} + <Button id="action-view-{job.id}" variant="ghost" size="sm" class="size-8 p-0 text-muted-foreground hover:text-foreground" {href} title="View details" aria-label="View details"> + <ChevronRight class="size-4" /> + </Button> + </div> + </td> + </tr> + {/each} + </tbody> + </table> +</div> {#snippet iconAction(id: string, label: string, Icon: typeof Eye, onclick: () => void, busy = false, disabled = false, danger = false)} <Button diff --git a/frontend/src/lib/components/dashboard/ActivityFeed.svelte b/frontend/src/lib/components/dashboard/ActivityFeed.svelte index 7128e093..340a1691 100644 --- a/frontend/src/lib/components/dashboard/ActivityFeed.svelte +++ b/frontend/src/lib/components/dashboard/ActivityFeed.svelte @@ -1,8 +1,6 @@ <script lang="ts"> import * as Card from '$lib/components/ui/card'; import { Button } from '$lib/components/ui/button'; - import { fly } from 'svelte/transition'; - import { prefersReducedMotion } from 'svelte/motion'; import Activity from '@lucide/svelte/icons/activity'; import AppWindow from '@lucide/svelte/icons/app-window'; import Clock from '@lucide/svelte/icons/clock'; @@ -25,6 +23,11 @@ if (status === 'running') return 'text-info'; return 'text-muted-foreground'; } + + // Titles already start with the kind ("Generation agent job — …"); the status is the dot. + function label(event: RecentActivityItem): string { + return event.title.replace(/^(Generation|Analysis)\s+/i, ''); + } </script> <Card.Root> @@ -48,41 +51,21 @@ </Card.Header> <Card.Content> {#if items.length > 0} - <!-- Timeline rail --> - <div class="relative ml-1.5 space-y-4 border-l border-border pl-4"> - {#each items as event, i (event.kind + event.id)} + <ul class="-mx-1 text-xs"> + {#each items as event (event.kind + event.id)} {@const EvtIcon = event.kind === 'analysis' ? Activity : AppWindow} - <div - class="relative" - in:fly={{ - y: 8, - duration: prefersReducedMotion.current ? 0 : 250, - delay: prefersReducedMotion.current ? 0 : Math.min(i, 8) * 35, - }} - > + <li class="flex items-center gap-2 rounded px-1 py-1" title="{event.title} ({event.status})"> <span - class="absolute -left-[1.4rem] top-1 flex h-3 w-3 items-center justify-center rounded-full bg-card" - aria-hidden="true" - > - <span - class="status-dot h-2 w-2 {dotColor(event.status)} {event.status === 'running' - ? 'status-dot-live' - : ''}" - ></span> - </span> - <div class="flex items-start gap-2"> - <EvtIcon class="mt-0.5 h-3.5 w-3.5 shrink-0 {dotColor(event.status)}" /> - <div class="min-w-0 flex-1"> - <p class="break-words text-xs leading-relaxed"> - <span class="capitalize">{event.kind}</span>: {event.title} - <span class="text-muted-foreground">({event.status})</span> - </p> - <p class="mt-0.5 text-[11px] text-muted-foreground">{timeAgo(event.created_at)}</p> - </div> - </div> - </div> + class="status-dot h-1.5 w-1.5 {dotColor(event.status)} {event.status === 'running' ? 'status-dot-live' : ''}" + role="img" + aria-label={event.status} + ></span> + <EvtIcon class="h-3.5 w-3.5 shrink-0 text-muted-foreground" aria-label={event.kind} /> + <span class="min-w-0 flex-1 truncate">{label(event)}</span> + <span class="shrink-0 font-mono text-[11px] text-muted-foreground tabular-nums">{timeAgo(event.created_at)}</span> + </li> {/each} - </div> + </ul> {:else} <EmptyState compact icon={Activity} title="No recent activity yet" /> {/if} diff --git a/frontend/src/lib/components/dashboard/AnalyzerHealthGrid.svelte b/frontend/src/lib/components/dashboard/AnalyzerHealthGrid.svelte index d36968dc..0eca2b61 100644 --- a/frontend/src/lib/components/dashboard/AnalyzerHealthGrid.svelte +++ b/frontend/src/lib/components/dashboard/AnalyzerHealthGrid.svelte @@ -48,38 +48,27 @@ </Card.Header> <Card.Content> {#if health} - <div class="grid grid-cols-2 gap-2 sm:grid-cols-3 xl:grid-cols-4"> + <!-- One line per analyzer: a status dot, the name and its type. --> + <ul class="grid grid-cols-2 gap-x-4 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-6"> {#each health.analyzers as a (a.name)} - <div - class="flex flex-col gap-1.5 rounded-md border border-border bg-surface-2/50 px-3 py-2.5" + <li + class="flex min-w-0 items-center gap-2 border-b border-border/50 py-1 text-xs" + title="{a.display_name} · {a.type} · {a.message}" > - <div class="flex items-center justify-between gap-2"> - <span class="truncate text-sm font-medium">{a.display_name}</span> - <span - class="status-dot h-2 w-2 {a.available - ? 'text-success' - : 'text-destructive'}" - role="img" - aria-label={a.available ? 'online' : 'offline'} - ></span> - </div> - <div class="flex items-center justify-between gap-2"> - <span class="mono-tag uppercase">{a.type}</span> - <span - class="truncate text-[11px] {a.available - ? 'text-success-fg' - : 'text-muted-foreground'}" - > - {a.message} - </span> - </div> - </div> + <span + class="status-dot h-1.5 w-1.5 shrink-0 {a.available ? 'text-success' : 'text-destructive'}" + role="img" + aria-label={a.available ? 'online' : 'offline'} + ></span> + <span class="min-w-0 flex-1 truncate font-medium">{a.display_name}</span> + <span class="shrink-0 font-mono text-[10px] uppercase tracking-wide text-muted-foreground">{a.type}</span> + </li> {/each} - </div> + </ul> {:else} - <div class="grid grid-cols-2 gap-2 sm:grid-cols-3 xl:grid-cols-4" aria-hidden="true"> - {#each Array(4) as _, i (i)} - <div class="skeleton h-[4.25rem]"></div> + <div class="grid grid-cols-2 gap-x-4 gap-y-1 sm:grid-cols-3 lg:grid-cols-4 xl:grid-cols-6" aria-hidden="true"> + {#each Array(6) as _, i (i)} + <div class="skeleton h-5"></div> {/each} </div> {/if} diff --git a/frontend/src/lib/components/dashboard/HeroPanel.svelte b/frontend/src/lib/components/dashboard/HeroPanel.svelte index 611c4f56..9dc06364 100644 --- a/frontend/src/lib/components/dashboard/HeroPanel.svelte +++ b/frontend/src/lib/components/dashboard/HeroPanel.svelte @@ -1,102 +1,123 @@ <script lang="ts"> + import { onMount, type Component, type Snippet } from 'svelte'; + import { DropdownMenu } from 'bits-ui'; + import { goto } from '$app/navigation'; import { Button } from '$lib/components/ui/button'; + import PageHeader from '$lib/components/PageHeader.svelte'; import Plus from '@lucide/svelte/icons/plus'; import Wand2 from '@lucide/svelte/icons/wand-2'; import RefreshCw from '@lucide/svelte/icons/refresh-cw'; import Loader2 from '@lucide/svelte/icons/loader-2'; - import { greeting, timeAgo } from '$lib/utils/dashboard'; + import Wrench from '@lucide/svelte/icons/wrench'; + import CloudDownload from '@lucide/svelte/icons/cloud-download'; + import Eraser from '@lucide/svelte/icons/eraser'; + import Activity from '@lucide/svelte/icons/activity'; + import Cpu from '@lucide/svelte/icons/cpu'; + import { timeAgo } from '$lib/utils/dashboard'; + import { getMe } from '$lib/api/users'; interface Props { - userName: string | null; isRefreshing: boolean; lastRefreshed: Date | null; onRefresh: () => void; + /** Running maintenance action ('sync' | 'caches' | 'stuck'), or null. */ + pendingAction?: string | null; + onSyncModels?: () => void; + onClearCaches?: () => void; + onClearStuck?: () => void; + /** Inline counts for the header's footer strip. */ + stats?: Snippet; } - let { userName, isRefreshing, lastRefreshed, onRefresh }: Props = $props(); + let { + isRefreshing, + lastRefreshed, + onRefresh, + pendingAction = null, + onSyncModels, + onClearCaches, + onClearStuck, + stats, + }: Props = $props(); - const today = new Date().toLocaleDateString(undefined, { - weekday: 'long', - month: 'long', - day: 'numeric', - year: 'numeric', + // Every maintenance action is staff-only on the API, so non-staff never see the menu + // (same check as the sidebar's staffOnly sections). + let isStaff = $state(false); + onMount(async () => { + try { + isStaff = (await getMe()).is_staff ?? false; + } catch { + isStaff = false; + } }); -</script> - -<section class="hero-panel" aria-labelledby="hero-title"> - <!-- Decorative figure: two fitted curves with a confidence band on a log-like grid --> - <svg class="hero-figure" viewBox="0 0 480 200" preserveAspectRatio="xMaxYMid slice" aria-hidden="true"> - <g stroke="currentColor" stroke-opacity="0.10" stroke-width="1"> - {#each [40, 80, 120, 160] as y (y)}<line x1="0" x2="480" y1={y} y2={y} />{/each} - {#each [60, 140, 220, 300, 380, 460] as x (x)}<line y1="0" y2="200" x1={x} x2={x} />{/each} - </g> - <path - d="M0 170 C 80 160, 140 120, 220 100 S 360 60, 480 40 L 480 70 C 360 92, 300 118, 220 130 S 80 178, 0 190 Z" - fill="currentColor" - fill-opacity="0.08" - /> - <path d="M0 180 C 80 168, 140 126, 220 115 S 360 76, 480 55" fill="none" stroke="currentColor" stroke-opacity="0.55" stroke-width="2" /> - <path - d="M0 186 C 90 182, 160 160, 240 150 S 380 120, 480 110" - fill="none" - stroke="var(--accent-2)" - stroke-opacity="0.6" - stroke-width="1.5" - stroke-dasharray="5 4" - /> - <g fill="var(--card)" stroke="currentColor" stroke-opacity="0.7" stroke-width="1.5"> - {#each [[60, 170], [140, 135], [220, 114], [300, 94], [380, 74], [460, 58]] as [cx, cy] (cx)} - <circle {cx} {cy} r="3.5" /> - {/each} - </g> - </svg> - <div class="relative flex flex-col gap-5 p-5 sm:p-7"> - <p class="page-eyebrow"> - <span class="text-primary">Lab notebook</span> - <span aria-hidden="true">·</span> - <span class="normal-case tracking-normal">{today}</span> - </p> + const itemClass = + 'flex cursor-pointer select-none items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-none transition-colors duration-150 data-[highlighted]:bg-surface-2 data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:size-3.5 [&_svg]:shrink-0 [&_svg]:text-muted-foreground'; +</script> - <div class="max-w-2xl space-y-2"> - <h1 id="hero-title" class="display-xl"> - {greeting()}{#if userName}, <span class="italic text-primary">{userName}</span>{/if}. - </h1> - <p class="max-w-xl text-sm leading-relaxed text-muted-foreground sm:text-[15px]"> - Benchmark LLM-generated applications end to end — generation, static and dynamic analysis, - and comparative statistics across models and harnesses. - </p> - </div> +{#snippet actionIcon(key: string, Icon: Component)} + {#if pendingAction === key}<Loader2 class="animate-spin motion-reduce:animate-none" />{:else}<Icon />{/if} +{/snippet} - <div class="flex flex-wrap items-center gap-2"> - <Button href="/sample-generator" size="sm" class="gap-1.5"> - <Wand2 class="h-3.5 w-3.5" /> - Generate app - </Button> - <Button href="/analysis/create" size="sm" variant="outline" class="gap-1.5 bg-card"> - <Plus class="h-3.5 w-3.5" /> - New analysis - </Button> - <Button - onclick={onRefresh} - size="icon" - variant="ghost" - class="h-8 w-8" - disabled={isRefreshing} - aria-label="Refresh dashboard" - > - {#if isRefreshing} - <Loader2 class="h-4 w-4 animate-spin" /> - {:else} - <RefreshCw class="h-4 w-4" /> - {/if} - </Button> - {#if lastRefreshed} - <span class="ml-1 flex items-center gap-1.5 font-mono text-[11px] text-muted-foreground"> - <span class="status-dot status-dot-live h-1.5 w-1.5 text-success"></span> - live · updated {timeAgo(lastRefreshed.toISOString())} - </span> +<!-- Plain page header: the dashboard is an instrument panel, not a landing page. --> +<PageHeader title="Dashboard" description="Generation, analysis and evaluation activity across the lab." children={stats}> + {#snippet meta()} + {#if lastRefreshed} + <span class="flex items-center gap-1.5 font-mono text-[11px] text-muted-foreground"> + <span class="status-dot status-dot-live h-1.5 w-1.5 text-success"></span> + live · {timeAgo(lastRefreshed.toISOString())} + </span> + {/if} + {/snippet} + {#snippet actions()} + <Button onclick={onRefresh} size="sm" variant="outline" disabled={isRefreshing} aria-label="Refresh dashboard" title="Refresh dashboard"> + {#if isRefreshing} + <Loader2 class="h-3.5 w-3.5 animate-spin" /> + {:else} + <RefreshCw class="h-3.5 w-3.5" /> {/if} - </div> - </div> -</section> + </Button> + <!-- Maintenance shortcuts (formerly the Quick Actions card); staff only. --> + {#if isStaff} + <DropdownMenu.Root> + <DropdownMenu.Trigger> + {#snippet child({ props })} + <Button {...props} size="sm" variant="outline" aria-label="Maintenance actions" title="Maintenance"> + {#if pendingAction}<Loader2 class="h-3.5 w-3.5 animate-spin" />{:else}<Wrench class="h-3.5 w-3.5" />{/if} + </Button> + {/snippet} + </DropdownMenu.Trigger> + <DropdownMenu.Portal> + <DropdownMenu.Content + align="end" + sideOffset={6} + collisionPadding={12} + class="z-50 min-w-48 rounded-md border border-border bg-popover p-1 text-popover-foreground shadow-md" + > + <DropdownMenu.Item class={itemClass} disabled={pendingAction !== null} onSelect={() => onSyncModels?.()}> + {@render actionIcon('sync', CloudDownload)}Sync models + </DropdownMenu.Item> + <DropdownMenu.Item class={itemClass} disabled={pendingAction !== null} onSelect={() => onClearCaches?.()}> + {@render actionIcon('caches', Eraser)}Clear caches + </DropdownMenu.Item> + <DropdownMenu.Item class={itemClass} disabled={pendingAction !== null} onSelect={() => onClearStuck?.()}> + {@render actionIcon('stuck', Activity)}Clear stuck jobs + </DropdownMenu.Item> + <DropdownMenu.Separator class="-mx-1 my-1 h-px bg-border" /> + <DropdownMenu.Item class={itemClass} onSelect={() => goto('/system')}> + <Cpu />Open system panel + </DropdownMenu.Item> + </DropdownMenu.Content> + </DropdownMenu.Portal> + </DropdownMenu.Root> + {/if} + <Button href="/analysis/create" size="sm" variant="outline" class="gap-1.5"> + <Plus class="h-3.5 w-3.5" /> + New analysis + </Button> + <Button href="/sample-generator" size="sm" class="gap-1.5"> + <Wand2 class="h-3.5 w-3.5" /> + Generate app + </Button> + {/snippet} +</PageHeader> diff --git a/frontend/src/lib/components/dashboard/QuickActions.svelte b/frontend/src/lib/components/dashboard/QuickActions.svelte deleted file mode 100644 index 77ec92c7..00000000 --- a/frontend/src/lib/components/dashboard/QuickActions.svelte +++ /dev/null @@ -1,90 +0,0 @@ -<script lang="ts"> - import * as Card from '$lib/components/ui/card'; - import { Button } from '$lib/components/ui/button'; - import Cpu from '@lucide/svelte/icons/cpu'; - import RefreshCw from '@lucide/svelte/icons/refresh-cw'; - import CloudDownload from '@lucide/svelte/icons/cloud-download'; - import Eraser from '@lucide/svelte/icons/eraser'; - import Activity from '@lucide/svelte/icons/activity'; - import Loader2 from '@lucide/svelte/icons/loader-2'; - - interface Props { - isRefreshing: boolean; - pendingAction: string | null; - onRefresh: () => void; - onSyncModels: () => void; - onClearCaches: () => void; - onClearStuck: () => void; - } - - let { isRefreshing, pendingAction, onRefresh, onSyncModels, onClearCaches, onClearStuck }: Props = - $props(); -</script> - -<Card.Root> - <Card.Header> - <div class="flex items-center gap-2"> - <Cpu class="h-4 w-4 text-muted-foreground" /> - <div> - <Card.Title>Quick Actions</Card.Title> - <Card.Description>Maintenance shortcuts</Card.Description> - </div> - </div> - </Card.Header> - <Card.Content> - <div class="grid grid-cols-2 gap-2"> - <Button - variant="outline" - size="sm" - class="justify-start" - onclick={onRefresh} - disabled={isRefreshing} - > - {#if isRefreshing}<Loader2 - class="mr-2 h-3.5 w-3.5 animate-spin" - />{:else}<RefreshCw class="mr-2 h-3.5 w-3.5 text-[color:var(--accent-2)]" />{/if} - Refresh - </Button> - <Button - variant="outline" - size="sm" - class="justify-start" - onclick={onSyncModels} - disabled={pendingAction !== null} - > - {#if pendingAction === 'sync'}<Loader2 - class="mr-2 h-3.5 w-3.5 animate-spin" - />{:else}<CloudDownload class="mr-2 h-3.5 w-3.5 text-primary" />{/if} - Sync Models - </Button> - <Button - variant="outline" - size="sm" - class="justify-start" - onclick={onClearCaches} - disabled={pendingAction !== null} - > - {#if pendingAction === 'caches'}<Loader2 - class="mr-2 h-3.5 w-3.5 animate-spin" - />{:else}<Eraser class="mr-2 h-3.5 w-3.5 text-warning" />{/if} - Clear Caches - </Button> - <Button - variant="outline" - size="sm" - class="justify-start" - onclick={onClearStuck} - disabled={pendingAction !== null} - > - {#if pendingAction === 'stuck'}<Loader2 - class="mr-2 h-3.5 w-3.5 animate-spin" - />{:else}<Activity class="mr-2 h-3.5 w-3.5 text-success" />{/if} - Clear Stuck - </Button> - <Button variant="outline" size="sm" class="col-span-2 justify-start" href="/system"> - <Cpu class="mr-2 h-3.5 w-3.5 text-muted-foreground" /> - Open System Panel - </Button> - </div> - </Card.Content> -</Card.Root> diff --git a/frontend/src/lib/components/dashboard/SeverityPanel.svelte b/frontend/src/lib/components/dashboard/SeverityPanel.svelte index 014b21c2..5bfa4fd7 100644 --- a/frontend/src/lib/components/dashboard/SeverityPanel.svelte +++ b/frontend/src/lib/components/dashboard/SeverityPanel.svelte @@ -16,12 +16,10 @@ <Card.Root class="h-full"> <Card.Header> - <div class="flex items-center gap-2"> - <ShieldAlert class="h-4 w-4 text-muted-foreground" /> - <div> - <Card.Title>Finding Severity</Card.Title> - <Card.Description>Distribution across all analyses</Card.Description> - </div> + <div class="flex min-w-0 items-center gap-2"> + <ShieldAlert class="h-4 w-4 shrink-0 text-muted-foreground" /> + <Card.Title>Finding Severity</Card.Title> + <span class="truncate text-xs text-muted-foreground">all analyses</span> </div> </Card.Header> <Card.Content> @@ -30,7 +28,7 @@ {:else if severity && severity.total > 0} <SeverityDonut {severity} /> {:else} - <ChartEmpty message="No findings recorded yet." /> + <ChartEmpty height={56} message="No findings recorded yet." /> {/if} </Card.Content> </Card.Root> diff --git a/frontend/src/lib/components/dashboard/TrendPanel.svelte b/frontend/src/lib/components/dashboard/TrendPanel.svelte index 5aaa9b15..39a2bb28 100644 --- a/frontend/src/lib/components/dashboard/TrendPanel.svelte +++ b/frontend/src/lib/components/dashboard/TrendPanel.svelte @@ -16,20 +16,15 @@ <Card.Root class="h-full"> <Card.Header> - <div class="flex items-center justify-between"> - <div class="flex items-center gap-2"> - <TrendingUp class="h-4 w-4 text-muted-foreground" /> - <div> - <Card.Title>Analysis Activity</Card.Title> - <Card.Description>Last {trends?.days ?? 30} days</Card.Description> - </div> - </div> - <div class="text-right"> - <div class="text-xl font-medium font-mono"> - {trends?.total ?? 0} - </div> - <p class="text-[11px] text-muted-foreground">total runs</p> + <div class="flex items-center justify-between gap-2"> + <div class="flex min-w-0 items-center gap-2"> + <TrendingUp class="h-4 w-4 shrink-0 text-muted-foreground" /> + <Card.Title>Analysis Activity</Card.Title> + <span class="truncate text-xs text-muted-foreground">last {trends?.days ?? 30} days</span> </div> + <span class="flex shrink-0 items-baseline gap-1 text-[11px] text-muted-foreground"> + <span class="font-mono text-base font-medium text-foreground tabular-nums">{trends?.total ?? 0}</span>runs + </span> </div> </Card.Header> <Card.Content> @@ -38,7 +33,7 @@ {:else if trends && trends.series.length > 1 && trends.total > 0} <TrendAreaChart points={trends.series} /> {:else} - <ChartEmpty message={trends ? `No analyses in the last ${trends.days} days.` : undefined} /> + <ChartEmpty height={56} message={trends ? `No analyses in the last ${trends.days} days.` : undefined} /> {/if} </Card.Content> </Card.Root> diff --git a/frontend/src/lib/components/docs/DocsNav.svelte b/frontend/src/lib/components/docs/DocsNav.svelte new file mode 100644 index 00000000..e8759a83 --- /dev/null +++ b/frontend/src/lib/components/docs/DocsNav.svelte @@ -0,0 +1,73 @@ +<script lang="ts"> + /** + * DocsNav — the docs pages grouped by category, in the same rail card as the + * Generation Library's nav: section labels, a primary bar on the open page. + * Rendered in the layout's sticky column from `lg`, and in a sheet below it. + */ + import type { DocNode } from '$lib/api/system'; + import { metaFor } from '$lib/docs/categories'; + import House from '@lucide/svelte/icons/house'; + + interface Props { + groups: Array<{ category: string; items: DocNode[] }>; + currentSlug: string; + onNavigate?: () => void; + /** Drop the card surface (inside the mobile sheet). */ + bare?: boolean; + } + let { groups, currentSlug, onNavigate, bare = false }: Props = $props(); +</script> + +{#snippet link(href: string, label: string, active: boolean)} + <li class="relative"> + {#if active}<span class="absolute inset-y-1 left-0 w-0.5 rounded-full bg-primary" aria-hidden="true"></span>{/if} + <a + {href} + aria-current={active ? 'page' : undefined} + title={label} + onclick={onNavigate} + class="flex cursor-pointer items-center gap-2 rounded-md px-2 py-1.5 text-sm transition-colors duration-150 {active + ? 'bg-primary/10 font-medium text-primary' + : 'text-muted-foreground hover:bg-muted hover:text-foreground'}" + > + <span class="truncate">{label}</span> + </a> + </li> +{/snippet} + +<nav + aria-label="Documentation" + class="space-y-4 {bare ? '' : 'rounded-md border border-border bg-card p-2 pt-3 shadow-[var(--shadow-card)]'}" +> + <ul> + <li class="relative"> + {#if currentSlug === ''}<span class="absolute inset-y-1 left-0 w-0.5 rounded-full bg-primary" aria-hidden="true"></span>{/if} + <a + href="/docs" + aria-current={currentSlug === '' ? 'page' : undefined} + onclick={onNavigate} + class="flex cursor-pointer items-center gap-2 rounded-md px-2 py-1.5 text-sm transition-colors duration-150 {currentSlug === '' + ? 'bg-primary/10 font-medium text-primary' + : 'text-muted-foreground hover:bg-muted hover:text-foreground'}" + > + <House class="size-4 shrink-0" aria-hidden="true" /> + Overview + </a> + </li> + </ul> + {#each groups as g (g.category)} + {@const meta = metaFor(g.category)} + <div class="space-y-1 border-t border-border/60 pt-3"> + <p class="section-label flex items-center gap-1.5 px-2"> + <meta.icon class="size-3.5 {meta.accent}" aria-hidden="true" /> + <span class="flex-1 truncate">{g.category}</span> + <span class="font-mono tabular-nums">{g.items.length}</span> + </p> + <ul class="space-y-0.5"> + {#each g.items as item (item.slug)} + {@render link(`/docs/${item.slug}`, item.title, currentSlug === item.slug)} + {/each} + </ul> + </div> + {/each} +</nav> diff --git a/frontend/src/lib/components/experiments/CapacityStrip.svelte b/frontend/src/lib/components/experiments/CapacityStrip.svelte index 6c577830..44f1c896 100644 --- a/frontend/src/lib/components/experiments/CapacityStrip.svelte +++ b/frontend/src/lib/components/experiments/CapacityStrip.svelte @@ -2,7 +2,7 @@ /** * CapacityStrip — one line of where runs can execute right now: executor * state, lane usage (sandboxes · eval · LLM jobs) and one chip per visible - * endpoint, plus capacity warnings. Other users' models appear as "+N". + * endpoint, plus capacity warnings behind a disclosure chip. Other users' models appear as "+N". * * Props: `capacity` (`GET /execution/capacity/`, patched by SSE `capacity`). */ @@ -74,15 +74,24 @@ {/each} </span> {/if} - <!-- Warnings are sentences: their own wrapped row, never truncated pills. --> + <!-- Warnings are sentences: a chip that discloses them, so the strip stays one line. --> {#if warnings.length > 0} - <ul class="basis-full space-y-1 border-t border-border pt-2 text-warning-fg"> - {#each warnings as warning (warning)} - <li class="flex items-start gap-1.5"> - <TriangleAlert class="mt-px size-3.5 shrink-0" aria-hidden="true" /> - <span>{warning}</span> - </li> - {/each} - </ul> + <details class="open:basis-full"> + <summary + class={cn('inline-flex cursor-pointer list-none items-center gap-1.5 rounded-full border px-2 py-0.5 text-[11px] font-medium', toneClasses.warning)} + title={warnings.join('\n')} + > + <TriangleAlert class="size-3 shrink-0" aria-hidden="true" /> + {warnings.length} capacity warning{warnings.length === 1 ? '' : 's'} + </summary> + <ul class="mt-1.5 space-y-1 text-warning-fg"> + {#each warnings as warning (warning)} + <li class="flex items-start gap-1.5"> + <TriangleAlert class="mt-px size-3.5 shrink-0" aria-hidden="true" /> + <span>{warning}</span> + </li> + {/each} + </ul> + </details> {/if} </div> diff --git a/frontend/src/lib/components/experiments/ComparabilityTable.svelte b/frontend/src/lib/components/experiments/ComparabilityTable.svelte index 9fdbc2bd..d65c0e7e 100644 --- a/frontend/src/lib/components/experiments/ComparabilityTable.svelte +++ b/frontend/src/lib/components/experiments/ComparabilityTable.svelte @@ -13,6 +13,7 @@ * - `rowLabel(row)`: override the condition label. */ import { cn } from '$lib/utils'; + import { scrollHint } from '$lib/utils/scroll-hint'; import { toneClasses } from '$lib/constants/colors'; import type { ExperimentComparability, ExperimentComparabilityRow } from '$lib/api/experiments'; @@ -46,6 +47,12 @@ } const rows = $derived(comparability.conditions as CompRow[]); + + /** The API sends a slug; older payloads sent `{slug, kind}`. */ + function endpointSlug(row: CompRow): string { + const ep = row.endpoint; + return (typeof ep === 'string' ? ep : ep?.slug) || 'openrouter'; + } const differsOn = $derived(new Set(comparability.differs_on)); const columns = $derived.by((): CompColumn[] => { @@ -56,7 +63,7 @@ id: 'endpoint', label: 'Endpoint', keys: ['endpoint', 'max_concurrency'], - cell: (r) => (r.endpoint ? r.endpoint.slug : 'openrouter'), + cell: endpointSlug, }); } list.push( @@ -86,7 +93,7 @@ cell: (r) => (r.output_cap != null ? r.output_cap.toLocaleString() : '—'), }); } - if (has('context_window') && (differsOn.has('context_window') || rows.some((r) => r.endpoint))) { + if (has('context_window') && (differsOn.has('context_window') || rows.some((r) => endpointSlug(r) !== 'openrouter'))) { list.push({ id: 'ctx', label: 'Context', @@ -152,7 +159,7 @@ {/if} {/if} <!-- Plain scroll frame (no fit attachment): this table also renders inside a Sheet. --> - <div class="table-frame rounded-md border border-border bg-card"> + <div class="table-frame rounded-md border border-border bg-card" {@attach scrollHint()}> <table class="min-w-[40rem]"> <thead> <tr> @@ -169,7 +176,7 @@ {#each rows as row (row.condition_id)} <tr class="group"> <th scope="row" class="col-sticky top-auto z-[1] min-w-44 max-w-56 border-b-border/60 bg-card group-hover:bg-[color-mix(in_oklab,var(--primary)_5%,var(--card))] text-xs font-medium normal-case tracking-normal text-foreground backdrop-blur-none"> - <span class="block truncate" title={label(row)}>{label(row)}</span> + <span class="line-clamp-2 break-words" title={label(row)}>{label(row)}</span> </th> {#each columns as column (column.id)} <td class={cn('font-mono text-xs', NUMERIC.has(column.id) && 'num', differs(column) && 'bg-warning/5')}> diff --git a/frontend/src/lib/components/experiments/ComposerSectionNav.svelte b/frontend/src/lib/components/experiments/ComposerSectionNav.svelte new file mode 100644 index 00000000..a0e799f0 --- /dev/null +++ b/frontend/src/lib/components/experiments/ComposerSectionNav.svelte @@ -0,0 +1,131 @@ +<script lang="ts" module> + export interface ComposerNavSection { + /** Anchor of the `ComposerSection`. */ + id: string; + step: number; + label: string; + /** Required sections: whether they are complete. `undefined` = optional. */ + done?: boolean; + /** Holds a value that wasn't applied (or a server field error). */ + invalid?: boolean; + } +</script> + +<script lang="ts"> + /** + * ComposerSectionNav — the composer's section index in the sticky rail: + * every numbered step as a link, the one in view highlighted, required + * steps marked done / to do and steps with invalid fields flagged, so + * "Fix the highlighted fields" says where. + * + * Props: `sections`. + */ + import { onMount } from 'svelte'; + import { cn } from '$lib/utils'; + import Circle from '@lucide/svelte/icons/circle'; + import CircleAlert from '@lucide/svelte/icons/circle-alert'; + import CircleCheck from '@lucide/svelte/icons/circle-check'; + + let { sections }: { sections: ComposerNavSection[] } = $props(); + + let current = $state(''); + // After a click the clicked section stays current until the user scrolls by hand + // (near the bottom the scroll position alone would name a later section). + let pinned = false; + + /** Below the sticky app bar: a section counts as current once its top passes this line. */ + const TRIGGER_PX = 120; + + function update() { + if (pinned) return; + const els = sections.map((s) => document.getElementById(s.id)).filter((el): el is HTMLElement => !!el); + if (els.length === 0) return; + // At the bottom the last sections can never reach the line: the last one wins. + const atBottom = window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - 4; + if (atBottom) { + current = els[els.length - 1].id; + return; + } + let next = els[0].id; + for (const el of els) { + if (el.getBoundingClientRect().top <= TRIGGER_PX) next = el.id; + else break; + } + current = next; + } + + onMount(() => { + let frame = 0; + const onScroll = () => { + cancelAnimationFrame(frame); + frame = requestAnimationFrame(update); + }; + const unpin = () => (pinned = false); + const unpinKeys = (e: KeyboardEvent) => { + if (['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End', ' '].includes(e.key)) pinned = false; + }; + update(); + window.addEventListener('scroll', onScroll, { passive: true }); + window.addEventListener('resize', onScroll, { passive: true }); + window.addEventListener('wheel', unpin, { passive: true }); + window.addEventListener('touchmove', unpin, { passive: true }); + window.addEventListener('keydown', unpinKeys); + return () => { + cancelAnimationFrame(frame); + window.removeEventListener('scroll', onScroll); + window.removeEventListener('resize', onScroll); + window.removeEventListener('wheel', unpin); + window.removeEventListener('touchmove', unpin); + window.removeEventListener('keydown', unpinKeys); + }; + }); + + function jump(event: MouseEvent, id: string) { + const el = document.getElementById(id); + if (!el) return; + event.preventDefault(); + el.scrollIntoView({ block: 'start' }); + // Move keyboard focus with the view (WCAG 2.4.3): the section heading. + const heading = document.getElementById(`${id}-title`); + if (heading) { + heading.setAttribute('tabindex', '-1'); + heading.focus({ preventScroll: true }); + } + history.replaceState(history.state, '', `#${id}`); + current = id; + pinned = true; + } +</script> + +<nav aria-label="Form sections" class="rounded-md border border-border bg-card p-2 shadow-[var(--shadow-card)]"> + <ol class="grid grid-cols-2 gap-x-1 gap-y-0.5"> + {#each sections as s (s.id)} + {@const active = current === s.id} + <li class="min-w-0"> + <a + href="#{s.id}" + onclick={(e) => jump(e, s.id)} + aria-current={active ? 'location' : undefined} + class={cn( + 'flex min-h-7 min-w-0 items-center gap-1.5 rounded px-1.5 py-1 text-xs transition-colors duration-150', + 'focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/40', + active ? 'bg-primary/10 text-foreground' : 'text-muted-foreground hover:bg-muted hover:text-foreground' + )} + > + <span class={cn('w-4 shrink-0 text-right font-mono text-[11px] tabular-nums', active && 'text-primary')}>{s.step}</span> + <span class="min-w-0 flex-1 truncate">{s.label}</span> + {#if s.invalid} + <CircleAlert class="size-3.5 shrink-0 text-danger-fg" aria-hidden="true" /> + <span class="sr-only">(has a field to fix)</span> + {:else if s.done === true} + <CircleCheck class="size-3.5 shrink-0 text-success-fg" aria-hidden="true" /> + <span class="sr-only">(done)</span> + {:else if s.done === false} + <Circle class="size-3.5 shrink-0 opacity-60" aria-hidden="true" /> + <span class="sr-only">(required, to do)</span> + {/if} + </a> + </li> + {/each} + </ol> +</nav> diff --git a/frontend/src/lib/components/experiments/ExperimentComposer.svelte b/frontend/src/lib/components/experiments/ExperimentComposer.svelte index 1ca5662e..29014ceb 100644 --- a/frontend/src/lib/components/experiments/ExperimentComposer.svelte +++ b/frontend/src/lib/components/experiments/ExperimentComposer.svelte @@ -93,6 +93,7 @@ import Trash2 from '@lucide/svelte/icons/trash-2'; import TriangleAlert from '@lucide/svelte/icons/triangle-alert'; import AutomationSettings from './AutomationSettings.svelte'; + import ComposerSectionNav, { type ComposerNavSection } from './ComposerSectionNav.svelte'; import ExecutionSettings from './ExecutionSettings.svelte'; import PlanRail from './PlanRail.svelte'; import SelectionSummary from './SelectionSummary.svelte'; @@ -558,6 +559,37 @@ let fieldErrors = $state<Record<string, string>>({}); let confirmOpen = $state(false); + // ── section index (rail) ──────────────────────────────────────────── + const navSections = $derived<ComposerNavSection[]>([ + { id: 'composer-basics', step: 1, label: 'Basics', done: checklist[0].done, invalid: !!(fieldErrors.name || fieldErrors.slug) }, + { id: 'composer-method', step: 2, label: 'Method' }, + { id: 'composer-models', step: 3, label: 'Models', done: checklist[1].done }, + { id: 'composer-conditions', step: 4, label: isAgent ? 'Harness presets' : 'Prompt profiles', done: checklist[2].done }, + { id: 'composer-apps', step: 5, label: 'Apps', done: checklist[3].done }, + { + id: 'composer-replication', + step: 6, + label: 'Replication', + invalid: repeatsInvalid || seedInvalid || repeats == null || !!(fieldErrors.repeats || fieldErrors.base_seed), + }, + { id: 'composer-sampling', step: 7, label: 'Sampling & budget', invalid: llmInvalid || (isAgent && budgetInvalid) }, + { id: 'composer-ablations', step: 8, label: 'Ablations' }, + { id: 'composer-execution', step: 9, label: 'Execution', invalid: executionInvalid }, + { id: 'composer-automation', step: 10, label: 'Automation', invalid: scheduleInvalid }, + ]); + + /** The section that clears `blockedReason`, for its link in the Plan. */ + const blockedHref = $derived.by(() => { + if (!name.trim() || !slug) return '#composer-basics'; + if (modelIds.size === 0) return '#composer-models'; + if (axisIds.length === 0) return '#composer-conditions'; + if (appIds.size === 0) return '#composer-apps'; + if (!stackSlug) return '#composer-method'; + if (localInvalid) return `#${navSections.find((x) => x.invalid)?.id ?? 'composer-basics'}`; + if (needsConsent) return '#composer-models'; + return ''; + }); + function finalBody(): ExperimentCreateV2 { return JSON.parse(JSON.stringify(body)) as ExperimentCreateV2; } @@ -925,6 +957,7 @@ bind:invalid={llmInvalid} hideProviderRouting={selectedModels.length > 0 && !anyOpenRouter} localModels={selectedModels.some((m) => isLocal(m))} + hint="Blank fields inherit the value shown in grey. What each model actually receives is in the Plan's comparability check." /> {#if isAgent} <div class="space-y-3 border-t border-border/60 pt-4"> @@ -1020,6 +1053,9 @@ class="min-w-0 @4xl/composer:sticky @4xl/composer:top-[4.5rem] @4xl/composer:max-h-[calc(100dvh-5.5rem)] @4xl/composer:overflow-y-auto @4xl/composer:overscroll-contain" aria-label="Experiment plan" > + <div class="mb-3 hidden @4xl/composer:block"> + <ComposerSectionNav sections={navSections} /> + </div> <PlanRail {checklist} ready={designReady} @@ -1037,6 +1073,7 @@ {canLaunch} {canSave} {blockedReason} + {blockedHref} {submitting} {submitError} saveLabel={composerMode === 'edit' ? 'Save changes' : 'Save draft'} diff --git a/frontend/src/lib/components/experiments/ExperimentDesignSummary.svelte b/frontend/src/lib/components/experiments/ExperimentDesignSummary.svelte deleted file mode 100644 index d439fe21..00000000 --- a/frontend/src/lib/components/experiments/ExperimentDesignSummary.svelte +++ /dev/null @@ -1,100 +0,0 @@ -<script lang="ts"> - /** - * ExperimentDesignSummary — the experiment-wide design as a read-only - * definition list: method, stack, apps, repeats, seed, presets and - * overrides, execution plan and post-run steps. Extracted from the old - * experiment page. - * - * Props: - * - `experiment`: the detail. - * - `appName(id)`: app spec display name. - * - `policyEffective`: per-endpoint effective policy once launched. - */ - import { cn } from '$lib/utils'; - import { summarizeBudget, summarizeLLMParams } from '$lib/components/generator'; - import type { ExperimentDetail } from '$lib/api/types/experiments'; - import { effectivePolicyLabel, policyLabel } from './format'; - - interface Props { - experiment: ExperimentDetail; - appName: (id: number) => string; - policyEffective?: Record<string, string> | string | null; - } - - let { experiment, appName, policyEffective = null }: Props = $props(); - - const isAgent = $derived(experiment.mode === 'agent'); - - function presetText(ref: { name: string; version: number } | null | undefined, fallback: string): string { - return ref ? `${ref.name} v${ref.version}` : fallback; - } - - const priorityText = (p: number) => (p > 0 ? `High (${p})` : p < 0 ? `Low (${p})` : 'Normal'); - - const items = $derived.by(() => { - const e = experiment; - const list: { label: string; value: string; mono?: boolean; wide?: boolean }[] = [ - { label: 'Method', value: isAgent ? 'Agent' : 'Template scaffolding' }, - { label: 'Stack', value: e.stack_slug || 'each condition profile’s stack', mono: !!e.stack_slug }, - { label: 'Repeats', value: `${e.repeats}× per cell` }, - { - label: 'Base seed', - value: e.base_seed != null ? String(e.base_seed) : 'random per run (no common random numbers)', - mono: e.base_seed != null, - }, - { - label: 'Apps', - value: e.app_requirement_ids.length ? e.app_requirement_ids.map(appName).join(', ') : 'none', - wide: true, - }, - { - label: 'Sampling preset', - value: presetText(e.sampling_preset_ref, isAgent ? 'Provider defaults (platform default)' : 'none (template mode defaults)'), - }, - { - label: 'Experiment overrides', - value: summarizeLLMParams(e.llm_defaults) || 'none', - mono: !!summarizeLLMParams(e.llm_defaults), - }, - ]; - if (isAgent) { - list.push( - { label: 'Budget preset', value: presetText(e.budget_preset_ref, 'Platform defaults') }, - { label: 'Budget overrides', value: summarizeBudget(e.agent_config) || 'none' } - ); - } - list.push( - { - label: 'Scheduling', - value: e.status === 'draft' ? policyLabel(e.scheduling_policy) : effectivePolicyLabel(policyEffective, e.scheduling_policy), - }, - { label: 'Priority', value: priorityText(e.priority) }, - { label: 'Max concurrent runs', value: e.max_concurrency != null ? String(e.max_concurrency) : 'Unlimited' }, - { label: 'Infra retries', value: String(e.infra_retries) }, - { - label: 'Model preload', - value: - (e.execution_options?.warmup ?? true ? 'On (Ollama endpoints)' : 'Off') + - (e.execution_options?.max_resident_models != null ? ` · max ${e.execution_options.max_resident_models} resident` : ''), - } - ); - const post = e.post_run ?? {}; - const steps = [ - !isAgent && post.evaluate ? 'evaluate' : '', - post.analysis?.enabled ? `analysis (${post.analysis.tool_slugs?.join(', ') || 'default tools'})` : '', - post.report?.enabled ? 'report' : '', - ].filter(Boolean); - list.push({ label: 'After the run', value: steps.length ? steps.join(' · ') : 'nothing' }); - if (e.cloned_from) list.push({ label: 'Cloned from', value: e.cloned_from.name }); - return list; - }); -</script> - -<dl class="grid gap-x-6 gap-y-3 text-sm sm:grid-cols-2 lg:grid-cols-3"> - {#each items as item (item.label)} - <div class={cn('min-w-0', item.wide && 'sm:col-span-2 lg:col-span-3')}> - <dt class="section-label">{item.label}</dt> - <dd class={cn('break-words', item.mono && 'font-mono text-xs')}>{item.value}</dd> - </div> - {/each} -</dl> diff --git a/frontend/src/lib/components/experiments/ExperimentRow.svelte b/frontend/src/lib/components/experiments/ExperimentRow.svelte index d4878b67..eed459ec 100644 --- a/frontend/src/lib/components/experiments/ExperimentRow.svelte +++ b/frontend/src/lib/components/experiments/ExperimentRow.svelte @@ -47,6 +47,8 @@ let { experiment: exp, busy = false, onaction }: Props = $props(); const done = $derived(finishedCells(exp.progress)); + const failed = $derived(exp.progress?.failed ?? 0); + const cancelled = $derived(exp.progress?.cancelled ?? 0); const terminal = $derived(isTerminal(exp.status)); const launched = $derived(exp.status !== 'draft'); const design = $derived( @@ -78,6 +80,8 @@ <BatchProgress counts={toBatchCounts(exp.progress)} total={exp.total_cells} /> <p class="flex flex-wrap gap-x-3 text-[11px] text-muted-foreground tabular-nums"> <span><span class="font-mono text-foreground">{done}/{exp.total_cells}</span> done</span> + {#if failed > 0}<span class="text-danger-fg"><span class="font-mono">{failed}</span> failed</span>{/if} + {#if cancelled > 0}<span><span class="font-mono">{cancelled}</span> cancelled</span>{/if} {#if !terminal && exp.eta_seconds != null}<span>ETA {formatEta(exp.eta_seconds)}</span>{/if} {#if exp.cost_usd > 0}<span class="font-mono">{formatUsd(exp.cost_usd)}</span>{/if} </p> @@ -116,7 +120,7 @@ <Button variant="ghost" size="sm" - class="size-8 p-0 text-muted-foreground hover:text-foreground" + class="size-9 p-0 text-muted-foreground hover:text-foreground lg:size-8" href="/experiments/new?from={exp.id}" title="Clone: new draft with this design" aria-label="Clone: new draft with this design"><Copy class="size-4" /></Button @@ -124,11 +128,11 @@ {/if} </div> {#if exp.status === 'running'} - <Button variant="ghost" size="sm" class="h-7 gap-1 px-2 text-xs" disabled={busy} onclick={() => onaction('pause')}><Pause class="size-3" />Pause</Button> + <Button variant="ghost" size="sm" class="h-9 gap-1 px-3 text-xs lg:h-7 lg:px-2" disabled={busy} onclick={() => onaction('pause')}><Pause class="size-3" />Pause</Button> {:else if exp.status === 'paused'} - <Button variant="ghost" size="sm" class="h-7 gap-1 px-2 text-xs" disabled={busy} onclick={() => onaction('resume')}><Play class="size-3" />Resume</Button> + <Button variant="ghost" size="sm" class="h-9 gap-1 px-3 text-xs lg:h-7 lg:px-2" disabled={busy} onclick={() => onaction('resume')}><Play class="size-3" />Resume</Button> {:else} - <Button variant="ghost" size="sm" class="h-7 gap-1 px-2 text-xs" href="/experiments/new?from={exp.id}" title="New draft with this design"><Copy class="size-3" />Clone</Button> + <Button variant="ghost" size="sm" class="h-9 gap-1 px-3 text-xs lg:h-7 lg:px-2" href="/experiments/new?from={exp.id}" title="New draft with this design"><Copy class="size-3" />Clone</Button> {/if} </div> </div> @@ -138,7 +142,7 @@ <Button variant="ghost" size="sm" - class="size-8 p-0 text-muted-foreground {danger ? 'hover:bg-destructive/10 hover:text-danger-fg' : 'hover:text-foreground'}" + class="size-9 p-0 text-muted-foreground lg:size-8 {danger ? 'hover:bg-destructive/10 hover:text-danger-fg' : 'hover:text-foreground'}" title={label} aria-label={label} disabled={busy} diff --git a/frontend/src/lib/components/experiments/Leaderboard.svelte b/frontend/src/lib/components/experiments/Leaderboard.svelte index 7506b833..d1613db9 100644 --- a/frontend/src/lib/components/experiments/Leaderboard.svelte +++ b/frontend/src/lib/components/experiments/Leaderboard.svelte @@ -3,20 +3,20 @@ * Leaderboard — conditions ranked by mean pass rate (app-level means, * cluster-bootstrap 95% CI), with the paired Δ against the baseline, what * is missing and why, timing (only comparable within one endpoint), tokens, - * cost, and validity / termination mixes as small stacked bars. + * cost, and validity / termination mixes as small stacked bars. Renders the + * table only: the page's StudyFigure frames and captions it. * * Props: * - `results`: `GET /results/`. * - `sort` (bindable): 'rank' | 'delta' | 'time' (time sorts within each endpoint). */ import { cn } from '$lib/utils'; - import * as Card from '$lib/components/ui/card'; import { tableFrame } from '$lib/components/table'; import { statusTone, type Tone } from '$lib/constants/colors'; import { formatUsd } from '$lib/components/generator/batches/batch-utils'; import { formatTokens } from '$lib/utils/formatters'; import type { ExperimentResults, LeaderboardRow } from '$lib/api/types/experiments'; - import { formatDeltaPp, formatPercent } from './format'; + import { MIN_APPS_FOR_CI, formatDeltaPp, formatPercent } from './format'; interface Props { results: ExperimentResults; @@ -105,109 +105,113 @@ {/if} {/snippet} -<Card.Root class="gap-0 overflow-clip py-0"> - <Card.Content class="p-0"> - <div class="table-frame" {@attach tableFrame}> - <table class="min-w-[56rem]"> - <thead> - <tr> - <th scope="col" class="num w-10">#</th> - <th scope="col" class="col-sticky">Condition</th> - <th scope="col" class="num" title="Apps × repeats per app">Apps × R</th> - <th scope="col" class="num" title="Cells that passed every check / graded cells">k/n</th> - <th scope="col" class="whitespace-nowrap" title="Mean of app-level pass rates, with a cluster-bootstrap 95% interval over apps">Pass rate (95% CI)</th> - <th scope="col" class="num" title="Paired mean difference vs the baseline (app-level)">Δ vs baseline</th> - <th scope="col" class="num" title="Cells without a usable grade">Missing</th> - <th scope="col" class="num" title="Median run time; only comparable within one endpoint">Median time</th> - <th scope="col" class="num">Tokens</th> - <th scope="col" class="num">Cost</th> - <th scope="col">Validity</th> - <th scope="col">Termination</th> - </tr> - </thead> - <tbody> - {#each rows as row (row.condition_id)} - {@const p = paired.get(row.condition_id)} - {@const isBaseline = row.condition_id === baselineId} - {@const missing = missingTotal(row)} - <tr class={cn('group', isBaseline && 'bg-primary/5')}> - <td class="num text-xs text-muted-foreground">{row.rank}</td> - <th - scope="row" - class={cn( - 'col-sticky top-auto z-[1] min-w-52 max-w-72 border-b-border/60 text-left font-sans text-sm font-normal normal-case tracking-normal text-foreground backdrop-blur-none', - isBaseline ? 'bg-[color-mix(in_oklab,var(--primary)_5%,var(--card))]' : 'bg-card group-hover:bg-[color-mix(in_oklab,var(--primary)_5%,var(--card))]' - )} - > - <span class="flex min-w-0 items-center gap-1.5"> - <span class="truncate font-medium" title={row.label}>{row.label}</span> - {#if isBaseline} - <span class="shrink-0 rounded-full border border-primary/40 bg-primary/10 px-1.5 text-[11px] text-primary">baseline</span> - {/if} - {#if row.mixed_fingerprints} - <span - class="shrink-0 rounded-full border border-warning/30 bg-warning/10 px-1.5 text-[11px] text-warning-fg" - title="Runs span {row.mixed_fingerprints.count} configuration fingerprints (majority {formatPercent(row.mixed_fingerprints.majority_share)}). Filter with “Majority fingerprint only”." - >mixed fingerprints</span> - {/if} - </span> - <span class="cell-sub block truncate font-mono text-[11px] text-muted-foreground">{row.model.endpoint_slug ?? 'openrouter'} · {row.harness}</span> - </th> - <td class="num text-xs">{row.n_apps}×{row.repeats_per_app}</td> - <td class="num text-xs">{row.k_passed}/{row.n_graded}</td> - <td> - <span class="flex items-center gap-2"> - <span class="w-10 text-right font-mono text-xs font-semibold tabular-nums">{formatPercent(row.pass_rate_mean)}</span> - <span class="relative h-2 w-24 rounded-full bg-muted" aria-hidden="true"> - {#if row.pass_rate_ci95} - <span - class="absolute inset-y-0 rounded-full bg-foreground/25" - style="left: {row.pass_rate_ci95[0] * 100}%; width: {Math.max(1, (row.pass_rate_ci95[1] - row.pass_rate_ci95[0]) * 100)}%" - ></span> - {/if} - {#if row.pass_rate_mean != null} - <span class="absolute inset-y-[-2px] w-0.5 rounded bg-foreground" style="left: calc({row.pass_rate_mean * 100}% - 1px)"></span> - {/if} - </span> - {#if row.pass_rate_ci95} - <span class="whitespace-nowrap font-mono text-[11px] tabular-nums text-muted-foreground"> - {formatPercent(row.pass_rate_ci95[0])}–{formatPercent(row.pass_rate_ci95[1])} - </span> - {/if} - </span> - </td> - <td class="num text-xs"> +<div class="table-frame" {@attach tableFrame}> + <table class="min-w-[56rem]"> + <thead> + <tr> + <th scope="col" class="num w-10">#</th> + <th scope="col" class="col-sticky">Condition</th> + <th scope="col" class="num" title="Apps × repeats per app">Apps × R</th> + <th scope="col" class="num" title="Cells that passed every check / graded cells">k/n</th> + <th scope="col" class="whitespace-nowrap" title="Mean of app-level pass rates, with a cluster-bootstrap 95% interval over apps">Pass rate (95% CI)</th> + <th scope="col" class="num" title="Paired mean difference vs the baseline (app-level)">Δ vs baseline</th> + <th scope="col" class="num" title="Cells without a usable grade">Missing</th> + <th scope="col" class="num" title="Median run time; only comparable within one endpoint">Median time</th> + <th scope="col" class="num">Tokens</th> + <th scope="col" class="num">Cost</th> + <th scope="col">Validity</th> + <th scope="col">Termination</th> + </tr> + </thead> + <tbody> + {#each rows as row (row.condition_id)} + {@const p = paired.get(row.condition_id)} + {@const isBaseline = row.condition_id === baselineId} + {@const missing = missingTotal(row)} + {@const ci = row.n_apps >= MIN_APPS_FOR_CI ? row.pass_rate_ci95 : null} + <tr class={cn('group', isBaseline && 'bg-primary/5')}> + <td class="num text-xs text-muted-foreground">{row.rank}</td> + <th + scope="row" + class={cn( + 'col-sticky top-auto z-[1] min-w-52 max-w-72 border-b-border/60 text-left font-sans text-sm font-normal normal-case tracking-normal text-foreground backdrop-blur-none', + isBaseline ? 'bg-[color-mix(in_oklab,var(--primary)_5%,var(--card))]' : 'bg-card group-hover:bg-[color-mix(in_oklab,var(--primary)_5%,var(--card))]' + )} + > + <span class="line-clamp-2 break-words font-medium" title={row.label}>{row.label}</span> + {#if isBaseline || row.mixed_fingerprints} + <span class="mt-0.5 flex flex-wrap gap-1"> {#if isBaseline} - <span class="text-muted-foreground" title="This is the baseline">—</span> - {:else if p?.mean_delta != null} - <span class={p.mean_delta > 0 ? 'text-success-fg' : p.mean_delta < 0 ? 'text-danger-fg' : ''}>{formatDeltaPp(p.mean_delta)}</span> - {:else} - <span class="text-muted-foreground">—</span> + <span class="rounded-full border border-primary/40 bg-primary/10 px-1.5 text-xs text-primary">baseline</span> + {/if} + {#if row.mixed_fingerprints} + <span + class="rounded-full border border-warning/30 bg-warning/10 px-1.5 text-xs text-warning-fg" + title="Runs span {row.mixed_fingerprints.count} configuration fingerprints (majority {formatPercent(row.mixed_fingerprints.majority_share)}). Filter with “Majority fingerprint only”." + >mixed fingerprints</span> + {/if} + </span> + {/if} + <span class="cell-sub block truncate font-mono text-xs text-muted-foreground">{row.model.endpoint_slug ?? 'openrouter'} · {row.harness}</span> + </th> + <td class="num text-xs">{row.n_apps}×{row.repeats_per_app}</td> + <td class="num text-xs">{row.k_passed}/{row.n_graded}</td> + <td> + <span class="flex items-center gap-2"> + <span class="w-10 text-right font-mono text-xs font-semibold tabular-nums">{formatPercent(row.pass_rate_mean)}</span> + <span class="relative h-2 w-24 rounded-full bg-muted" aria-hidden="true"> + {#if ci} + <span + class="absolute inset-y-0 rounded-full bg-foreground/25" + style="left: {ci[0] * 100}%; width: {Math.max(1, (ci[1] - ci[0]) * 100)}%" + ></span> {/if} - </td> - <td class="num text-xs" title={missingTitle(row)}> - <span class={missing > 0 ? 'text-warning-fg' : 'text-muted-foreground'}>{missing}</span> - </td> - <td class="num text-xs" title={row.timing_endpoint_slug ? `Measured on ${row.timing_endpoint_slug}` : ''}> - {#if row.median_seconds != null}{Math.round(row.median_seconds)} s{:else}<span class="text-muted-foreground">—</span>{/if} - {#if row.timing_endpoint_slug}<span class="cell-sub block text-[11px] text-muted-foreground">{row.timing_endpoint_slug}</span>{/if} - </td> - <td class="num text-xs">{#if row.tokens_total}{formatTokens(row.tokens_total)}{:else}<span class="text-muted-foreground">—</span>{/if}</td> - <td class="num text-xs">{#if row.cost_usd > 0}{formatUsd(row.cost_usd)}{:else}<span class="text-muted-foreground">—</span>{/if}</td> - <td> - {@render stacked( - mix( - Object.entries(row.validity) as [string, number][], - (k) => VALIDITY_TONES[k] ?? 'neutral' - ), - 'Validity' - )} - </td> - <td>{@render stacked(mix(Object.entries(row.terminations), terminationTone), 'Termination')}</td> - </tr> - {/each} - </tbody> - </table> - </div> - </Card.Content> -</Card.Root> + {#if row.pass_rate_mean != null} + <span class="absolute inset-y-[-2px] w-0.5 rounded bg-foreground" style="left: calc({row.pass_rate_mean * 100}% - 1px)"></span> + {/if} + </span> + {#if ci} + <span class="whitespace-nowrap font-mono text-xs tabular-nums text-muted-foreground"> + {formatPercent(ci[0])}–{formatPercent(ci[1])} + </span> + {:else if row.pass_rate_mean != null} + <span + class="whitespace-nowrap text-xs text-muted-foreground" + title="An interval over {row.n_apps} app{row.n_apps === 1 ? '' : 's'} would overstate the certainty; it needs at least {MIN_APPS_FOR_CI} apps." + >no CI (n={row.n_apps})</span> + {/if} + </span> + </td> + <td class="num text-xs"> + {#if isBaseline} + <span class="text-muted-foreground" title="This is the baseline">—</span> + {:else if p?.mean_delta != null} + <span class="text-foreground">{formatDeltaPp(p.mean_delta)}</span> + {:else} + <span class="text-muted-foreground">—</span> + {/if} + </td> + <td class="num text-xs" title={missingTitle(row)}> + <span class={missing > 0 ? 'text-warning-fg' : 'text-muted-foreground'}>{missing}</span> + </td> + <td class="num text-xs" title={row.timing_endpoint_slug ? `Measured on ${row.timing_endpoint_slug}` : ''}> + {#if row.median_seconds != null}{Math.round(row.median_seconds)} s{:else}<span class="text-muted-foreground">—</span>{/if} + {#if row.timing_endpoint_slug}<span class="cell-sub block text-xs text-muted-foreground">{row.timing_endpoint_slug}</span>{/if} + </td> + <td class="num text-xs">{#if row.tokens_total}{formatTokens(row.tokens_total)}{:else}<span class="text-muted-foreground">—</span>{/if}</td> + <td class="num text-xs">{#if row.cost_usd > 0}{formatUsd(row.cost_usd)}{:else}<span class="text-muted-foreground">—</span>{/if}</td> + <td> + {@render stacked( + mix( + Object.entries(row.validity) as [string, number][], + (k) => VALIDITY_TONES[k] ?? 'neutral' + ), + 'Validity' + )} + </td> + <td>{@render stacked(mix(Object.entries(row.terminations), terminationTone), 'Termination')}</td> + </tr> + {/each} + </tbody> + </table> +</div> diff --git a/frontend/src/lib/components/experiments/PairedDeltas.svelte b/frontend/src/lib/components/experiments/PairedDeltas.svelte index c1048eec..8252adbd 100644 --- a/frontend/src/lib/components/experiments/PairedDeltas.svelte +++ b/frontend/src/lib/components/experiments/PairedDeltas.svelte @@ -1,19 +1,21 @@ <script lang="ts"> /** * PairedDeltas — each condition against the baseline as a forest plot: - * mean app-level Δ with its paired cluster-bootstrap 95% CI around a zero - * line, wins / losses / ties over apps and the Holm-adjusted p (hidden - * while the experiment is interim). + * the mean app-level Δ (square) with its paired cluster-bootstrap 95% CI + * (whiskers) around a zero line, wins / losses / ties over apps, n and the + * Holm-adjusted p (hidden while the experiment is interim). Ink marks only: + * a filled square's interval excludes 0, a hollow one's includes it — the + * sign is read from the axis, never from a red/green hue. Renders the plot + * table only; the page's StudyFigure supplies the title and caption. * * Props: * - `results`: `GET /results/`. * - `labelOf(conditionId)`: display label. */ import { cn } from '$lib/utils'; - import * as Card from '$lib/components/ui/card'; import { tableFrame } from '$lib/components/table'; - import type { ExperimentResults } from '$lib/api/types/experiments'; - import { formatDeltaPp, formatP } from './format'; + import type { ExperimentResults, PairedRow } from '$lib/api/types/experiments'; + import { MIN_APPS_FOR_CI, formatDeltaPp, formatP } from './format'; interface Props { results: ExperimentResults; @@ -22,8 +24,11 @@ let { results, labelOf }: Props = $props(); + const baselineId = $derived(results.paired.baseline_condition_id); const rows = $derived( - [...results.paired.rows].sort((a, b) => (b.mean_delta ?? -Infinity) - (a.mean_delta ?? -Infinity)) + [...results.paired.rows] + .filter((r) => r.condition_id !== baselineId) + .sort((a, b) => (b.mean_delta ?? -Infinity) - (a.mean_delta ?? -Infinity)) ); // Symmetric axis around 0, at least ±10 pp, rounded up to 10 pp. const extent = $derived.by(() => { @@ -33,79 +38,144 @@ } return Math.min(1, Math.ceil(max * 10) / 10); }); - const pos = (v: number) => ((v + extent) / (2 * extent)) * 100; + const ticks = $derived([-extent, -extent / 2, 0, extent / 2, extent]); + const pos = (v: number) => `${((Math.max(-extent, Math.min(extent, v)) + extent) / (2 * extent)) * 100}%`; + const ciOf = (row: PairedRow) => (row.n_pairs >= MIN_APPS_FOR_CI ? row.ci95 : null); + const excludesZero = (ci: [number, number] | null) => !!ci && (ci[0] > 0 || ci[1] < 0); + + function describe(row: PairedRow): string { + const ci = ciOf(row); + if (row.mean_delta == null) return `${labelOf(row.condition_id)}: no paired apps`; + return `${labelOf(row.condition_id)}: Δ ${formatDeltaPp(row.mean_delta)}${ + ci ? `, 95% CI ${formatDeltaPp(ci[0])} to ${formatDeltaPp(ci[1])}` : ', no interval' + }; ${row.wins} wins, ${row.losses} losses, ${row.ties} ties over ${row.n_pairs} apps`; + } </script> -<div class="space-y-2"> - <Card.Root class="gap-0 overflow-clip py-0"> - <Card.Content class="p-0"> - <div class="table-frame" {@attach tableFrame}> - <table class="min-w-[40rem]"> - <thead> - <tr> - <th scope="col">Condition vs baseline</th> - <th scope="col" class="w-[40%]"> - <span class="flex justify-between normal-case tracking-normal"> - <span>{formatDeltaPp(-extent, 0)}</span><span>0</span><span>{formatDeltaPp(extent, 0)}</span> - </span> - </th> - <th scope="col" class="num">Δ</th> - <th scope="col" class="num" title="Apps where the condition beat / lost to / tied the baseline">W/L/T</th> - <th scope="col" class="num" title="Apps with a mean for both conditions">n</th> - <th scope="col" class="num" title="Wilcoxon signed-rank, Holm-adjusted across conditions">p (Holm)</th> - </tr> - </thead> - <tbody> - {#each rows as row (row.condition_id)} - {@const d = row.mean_delta} - <tr> - <td class="max-w-64"><span class="block truncate font-medium" title={labelOf(row.condition_id)}>{labelOf(row.condition_id)}</span></td> - <td> - <span - class="relative block h-4" - role="img" - aria-label={d != null - ? `Δ ${formatDeltaPp(d)}${row.ci95 ? `, 95% CI ${formatDeltaPp(row.ci95[0])} to ${formatDeltaPp(row.ci95[1])}` : ''}` - : 'No paired apps'} - > - <span class="absolute inset-y-0 left-1/2 w-px bg-border"></span> - {#if row.ci95} - <span - class="absolute top-1/2 h-0.5 -translate-y-1/2 bg-foreground/40" - style="left: {pos(row.ci95[0])}%; width: {Math.max(0.5, pos(row.ci95[1]) - pos(row.ci95[0]))}%" - ></span> - {/if} - {#if d != null} - <span - class={cn( - 'absolute top-1/2 size-2.5 -translate-x-1/2 -translate-y-1/2 rounded-full border-2 border-card', - d > 0 ? 'bg-success' : d < 0 ? 'bg-destructive' : 'bg-muted-foreground' - )} - style="left: {pos(d)}%" - ></span> - {/if} - </span> - </td> - <td class="num text-xs">{#if d != null}{formatDeltaPp(d)}{:else}<span class="text-muted-foreground">—</span>{/if}</td> - <td class="num text-xs">{row.wins}/{row.losses}/{row.ties}</td> - <td class="num text-xs">{row.n_pairs}</td> - <td class="num text-xs"> - {#if results.interim} - <span class="text-muted-foreground" title="Hidden while conditions are at unequal progress">—</span> - {:else} - {#if row.p_holm == null}<span class="text-muted-foreground">—</span>{:else}<span class={row.p_holm < 0.05 ? 'font-semibold text-foreground' : ''}>{formatP(row.p_holm)}</span>{/if} - {/if} - </td> - </tr> - {:else} - <tr><td colspan="6" class="py-6 text-center text-muted-foreground">No other condition to compare.</td></tr> - {/each} - </tbody> - </table> - </div> - </Card.Content> - </Card.Root> - <p class="text-[11px] text-muted-foreground"> - Paired by app (repeats averaged per app); n = number of apps. Whiskers: paired cluster-bootstrap 95% interval. +{#snippet axis()} + <span class="relative block h-4 font-mono text-[11px] normal-case tracking-normal text-muted-foreground" aria-hidden="true"> + {#each ticks as t, i (i)} + <span + class={cn('absolute top-0 whitespace-nowrap', i === 0 ? '' : i === ticks.length - 1 ? '-translate-x-full' : '-translate-x-1/2')} + style="left: {pos(t)}" + >{t === 0 ? '0' : formatDeltaPp(t, 0).replace(' pp', '')}</span> + {/each} + </span> +{/snippet} + +{#snippet marks(row: PairedRow)} + {@const d = row.mean_delta} + {@const ci = ciOf(row)} + <span class="relative block h-8" role="img" aria-label={describe(row)} title={describe(row)}> + {#each ticks as t, i (i)} + <span class={cn('absolute inset-y-0 w-px', t === 0 ? 'bg-foreground/50' : 'bg-border/70')} style="left: {pos(t)}" aria-hidden="true"></span> + {/each} + {#if ci} + <span + class="absolute top-1/2 h-px -translate-y-1/2 bg-foreground/70" + style="left: {pos(ci[0])}; width: calc({pos(ci[1])} - {pos(ci[0])})" + aria-hidden="true" + ></span> + {#each [ci[0], ci[1]] as end, i (i)} + <span class="absolute top-1/2 h-2.5 w-px -translate-y-1/2 bg-foreground/70" style="left: {pos(end)}" aria-hidden="true"></span> + {/each} + {/if} + {#if d != null} + <span + class={cn( + 'absolute top-1/2 size-2.5 -translate-x-1/2 -translate-y-1/2 border-2 border-foreground ring-2 ring-card', + excludesZero(ci) ? 'bg-foreground' : 'bg-card' + )} + style="left: {pos(d)}" + aria-hidden="true" + ></span> + {/if} + </span> +{/snippet} + +{#snippet pValue(row: PairedRow)} + {#if results.interim} + <span class="text-muted-foreground" title="Hidden while conditions are at unequal progress">—</span> + {:else if row.p_holm == null} + <span class="text-muted-foreground">—</span> + {:else} + <span class={row.p_holm < 0.05 ? 'font-semibold text-foreground' : ''}>{formatP(row.p_holm)}</span> + {/if} +{/snippet} + +{#if rows.length === 0} + <p class="px-4 py-3 text-sm text-muted-foreground"> + {results.leaderboard.length <= 1 + ? 'Only one condition — nothing to compare against the baseline.' + : baselineId == null + ? 'No baseline is set, so there is nothing to pair against.' + : 'No condition has apps graded alongside the baseline yet.'} </p> -</div> +{:else} + <!-- Phones: one stacked row per condition (label and Δ, the plot, then the counts). --> + <div class="sm:hidden"> + <div class="border-b border-border bg-muted/40 px-4 py-1.5">{@render axis()}</div> + <ol class="divide-y divide-border/60"> + {#each rows as row (row.condition_id)} + {@const ci = ciOf(row)} + <li class="px-4 py-2"> + <div class="flex items-baseline gap-2 text-sm"> + <span class="min-w-0 flex-1 break-words font-medium">{labelOf(row.condition_id)}</span> + <span class="shrink-0 font-mono text-xs font-semibold">{row.mean_delta != null ? formatDeltaPp(row.mean_delta) : '—'}</span> + </div> + {@render marks(row)} + <p class="flex flex-wrap gap-x-3 font-mono text-xs text-muted-foreground"> + {#if ci}<span>[{formatDeltaPp(ci[0])}, {formatDeltaPp(ci[1])}]</span>{/if} + <span>W/L/T {row.wins}/{row.losses}/{row.ties}</span> + <span>n {row.n_pairs}</span> + <span>p {@render pValue(row)}</span> + </p> + </li> + {/each} + </ol> + </div> + + <div class="table-frame max-sm:hidden" {@attach tableFrame}> + <table class="min-w-[44rem]"> + <thead> + <tr> + <th scope="col" class="col-sticky">Condition</th> + <th scope="col" class="w-[42%] min-w-60"> + {@render axis()} + <span class="sr-only">Difference from the baseline, percentage points</span> + </th> + <th scope="col" class="num">Δ [95% CI]</th> + <th scope="col" class="num" title="Apps where the condition beat / lost to / tied the baseline">W/L/T</th> + <th scope="col" class="num" title="Apps with a mean for both conditions">n</th> + <th scope="col" class="num" title="Wilcoxon signed-rank, Holm-adjusted across conditions">p (Holm)</th> + </tr> + </thead> + <tbody> + {#each rows as row (row.condition_id)} + {@const d = row.mean_delta} + {@const ci = ciOf(row)} + <tr> + <th + scope="row" + class="col-sticky top-auto z-[1] min-w-44 max-w-64 bg-card text-left font-sans text-sm font-normal normal-case tracking-normal text-foreground backdrop-blur-none" + > + <span class="line-clamp-2 break-words font-medium" title={labelOf(row.condition_id)}>{labelOf(row.condition_id)}</span> + </th> + <td class="py-0">{@render marks(row)}</td> + <td class="num text-xs"> + {#if d != null} + <span class="font-semibold text-foreground">{formatDeltaPp(d)}</span> + {#if ci}<span class="block text-muted-foreground">[{formatDeltaPp(ci[0])}, {formatDeltaPp(ci[1])}]</span>{/if} + {:else} + <span class="text-muted-foreground">—</span> + {/if} + </td> + <td class="num text-xs">{row.wins}/{row.losses}/{row.ties}</td> + <td class="num text-xs">{row.n_pairs}</td> + <td class="num text-xs">{@render pValue(row)}</td> + </tr> + {/each} + </tbody> + </table> + </div> +{/if} diff --git a/frontend/src/lib/components/experiments/PlanRail.svelte b/frontend/src/lib/components/experiments/PlanRail.svelte index fa8bd7ec..401b7fb7 100644 --- a/frontend/src/lib/components/experiments/PlanRail.svelte +++ b/frontend/src/lib/components/experiments/PlanRail.svelte @@ -10,7 +10,7 @@ * Props: `checklist`, `ready`, `preview` (latest successful, maybe `stale`), * `loading`, `error` + `onretry`, `localInvalid`, client-side tallies * (`conditions`, `apps`, `repeats`), `issues` + `actions`, submit state - * (`canLaunch`, `canSave`, `blockedReason`, `submitting`, `submitError`, + * (`canLaunch`, `canSave`, `blockedReason` + `blockedHref`, `submitting`, `submitError`, * `onsave`, `onlaunch`, `saveLabel`), `experimentStack`. */ import { cn } from '$lib/utils'; @@ -50,6 +50,8 @@ canLaunch: boolean; canSave: boolean; blockedReason: string; + /** Anchor of the section that clears `blockedReason`. */ + blockedHref?: string; submitting: 'save' | 'launch' | null; submitError: string; saveLabel?: string; @@ -75,6 +77,7 @@ canLaunch, canSave, blockedReason, + blockedHref = '', submitting, submitError, saveLabel = 'Save draft', @@ -218,7 +221,13 @@ {/if} <div class="hidden space-y-2 border-t border-border pt-3 @4xl/composer:block"> {#if blockedReason && !canLaunch} - <p class="text-[11px] text-muted-foreground">{blockedReason}</p> + <p class="text-[11px] text-muted-foreground"> + {#if blockedHref} + <a href={blockedHref} class="underline decoration-dotted underline-offset-2 hover:text-foreground">{blockedReason}</a> + {:else} + {blockedReason} + {/if} + </p> {/if} <div class="flex gap-2"> <Button variant="outline" size="sm" class="flex-1" disabled={!canSave || submitting !== null} onclick={onsave}> diff --git a/frontend/src/lib/components/experiments/PostRunCard.svelte b/frontend/src/lib/components/experiments/PostRunCard.svelte index 49231511..5d8ce440 100644 --- a/frontend/src/lib/components/experiments/PostRunCard.svelte +++ b/frontend/src/lib/components/experiments/PostRunCard.svelte @@ -15,6 +15,7 @@ import { Button } from '$lib/components/ui/button'; import StatusBadge from '$lib/components/StatusBadge.svelte'; import EmptyState from '$lib/components/EmptyState.svelte'; + import SectionHeading from '$lib/components/SectionHeading.svelte'; import { putPostRun, runPostRun, type PostRunConfig, type PostRunSpec } from '$lib/api/experiments'; import { formatApiError } from '$lib/api/core'; import { formatDateTime } from '$lib/utils/formatters'; @@ -111,8 +112,8 @@ {/if} <PostRunEditor bind:value {mode} disabled={readonly} idPrefix="post-run-card" /> - <div class="space-y-2 border-t border-border/60 pt-4"> - <p class="section-label">History</p> + <div class="space-y-2 pt-2"> + <SectionHeading size="sm" level={3} title="History" /> {#if data.runs.length === 0} <EmptyState compact icon={History} title="No post-runs yet" /> {:else} @@ -135,7 +136,7 @@ <td data-label="Trigger">{TRIGGERS[run.trigger] ?? run.trigger}</td> <td data-label="Status"> <StatusBadge status={run.status} /> - {#if run.error}<span class="cell-sub mt-0.5 block max-w-60 truncate text-[11px] text-danger-fg" title={run.error}>{run.error}</span>{/if} + {#if run.error}<span class="cell-sub mt-0.5 block max-w-60 truncate text-xs text-danger-fg" title={run.error}>{run.error}</span>{/if} </td> <td data-label="Analyses" class="num text-xs max-md:text-left"> {#if p.analysis_total} diff --git a/frontend/src/lib/components/experiments/PostRunEditor.svelte b/frontend/src/lib/components/experiments/PostRunEditor.svelte index 3c0aa257..73dea058 100644 --- a/frontend/src/lib/components/experiments/PostRunEditor.svelte +++ b/frontend/src/lib/components/experiments/PostRunEditor.svelte @@ -59,7 +59,14 @@ setAnalysis({ tool_slugs: [...set] }); } - const usable = (t: AnalyzerTool) => t.kind === 'ai' || t.installed; + // Dynamic scanners need the app running, and evaluation tears apps down. + const isDynamic = (t: AnalyzerTool) => t.analysis_mode === 'dynamic'; + const usable = (t: AnalyzerTool) => !isDynamic(t) && (t.kind === 'ai' || t.installed); + + function unusableReason(t: AnalyzerTool): string { + if (isDynamic(t)) return "Dynamic scanner: needs the app running, which post-run analysis doesn't do yet"; + return 'Not installed in your analyzer workspace'; + } </script> <div class="space-y-4"> @@ -74,7 +81,7 @@ /> <div class="min-w-0 space-y-0.5"> <label for="{idPrefix}-evaluate" class="text-sm font-medium">Evaluate every run</label> - <p id="{idPrefix}-evaluate-note" class="text-[11px] text-muted-foreground"> + <p id="{idPrefix}-evaluate-note" class="text-xs text-muted-foreground"> Builds the app and runs its hidden conformance suite, so Results can rank conditions. Without it runs are ungraded. </p> @@ -93,7 +100,7 @@ /> <div class="min-w-0 space-y-0.5"> <label for="{idPrefix}-analysis" class="text-sm font-medium">Static analysis when the experiment completes</label> - <p id="{idPrefix}-analysis-note" class="text-[11px] text-muted-foreground"> + <p id="{idPrefix}-analysis-note" class="text-xs text-muted-foreground"> Runs the chosen analyzers over the finished runs, one at a time, after every cell is done. </p> </div> @@ -105,23 +112,25 @@ {:else if !tools} <p class="loading-mono">loading analyzers</p> {:else if tools.length === 0} - <p class="text-[11px] text-muted-foreground">No analyzers are enabled. <a href="/analyzers" class="underline">Set them up</a>.</p> + <p class="text-xs text-muted-foreground">No analyzers are enabled. <a href="/analyzers" class="underline">Set them up</a>.</p> {:else} <fieldset class="grid gap-1.5 sm:grid-cols-2" aria-label="Analyzers"> {#each tools as tool (tool.slug)} - <label class="flex items-center gap-2 text-xs" title={usable(tool) ? tool.description : 'Not installed in your analyzer workspace'}> + <label class="flex items-center gap-2 text-xs" title={usable(tool) ? tool.description : unusableReason(tool)}> <Checkbox checked={value.analysis.tool_slugs.includes(tool.slug)} disabled={disabled || !usable(tool)} onCheckedChange={(v) => toggleTool(tool.slug, v === true)} /> <span class="truncate">{tool.name}</span> - <span class="text-[11px] text-muted-foreground">{tool.category}{usable(tool) ? '' : ' · not installed'}</span> + <span class="text-xs text-muted-foreground" + >{tool.category}{isDynamic(tool) ? ' · needs running app' : usable(tool) ? '' : ' · not installed'}</span + > </label> {/each} </fieldset> {#if value.analysis.tool_slugs.length === 0} - <p class="text-[11px] text-warning-fg">Pick at least one analyzer.</p> + <p class="text-xs text-warning-fg">Pick at least one analyzer.</p> {/if} {/if} <label class="flex items-center gap-2 text-xs"> @@ -147,7 +156,7 @@ /> <div class="min-w-0 space-y-0.5"> <label for="{idPrefix}-report" class="text-sm font-medium">Experiment report</label> - <p id="{idPrefix}-report-note" class="text-[11px] text-muted-foreground"> + <p id="{idPrefix}-report-note" class="text-xs text-muted-foreground"> Generated after analysis: leaderboard and paired comparisons against the baseline. </p> </div> @@ -164,6 +173,6 @@ {/if} </div> {#if mode === 'agent'} - <p class="text-[11px] text-muted-foreground">Agent runs are evaluated as they finish; no separate step is needed.</p> + <p class="text-xs text-muted-foreground">Agent runs are evaluated as they finish; no separate step is needed.</p> {/if} </div> diff --git a/frontend/src/lib/components/experiments/ScheduleEditor.svelte b/frontend/src/lib/components/experiments/ScheduleEditor.svelte index b40bcf44..89610a75 100644 --- a/frontend/src/lib/components/experiments/ScheduleEditor.svelte +++ b/frontend/src/lib/components/experiments/ScheduleEditor.svelte @@ -139,7 +139,7 @@ {disabled} onValueChange={(v) => patch({ action: v as ScheduleAction })} /> - <p class="text-[11px] text-muted-foreground">{SCHEDULE_ACTION_DESCRIPTIONS[value.action]}</p> + <p class="text-xs text-muted-foreground">{SCHEDULE_ACTION_DESCRIPTIONS[value.action]}</p> </div> {#if value.action === 'add_replicates'} <NumberField @@ -163,7 +163,7 @@ {disabled} oninput={(e: Event) => patch({ params: { ...(value.params ?? {}), name_template: (e.currentTarget as HTMLInputElement).value } })} /> - <p class="text-[11px] text-muted-foreground">Placeholders: <span class="font-mono">{'{name}'}</span>, <span class="font-mono">{'{date}'}</span>.</p> + <p class="text-xs text-muted-foreground">Placeholders: <span class="font-mono">{'{name}'}</span>, <span class="font-mono">{'{date}'}</span>.</p> </div> {/if} </div> @@ -198,7 +198,7 @@ {disabled} oninput={(e: Event) => patch({ cron: (e.currentTarget as HTMLInputElement).value })} /> - <p id="{idPrefix}-cron-note" class="text-[11px] {invalid ? 'text-danger-fg' : 'text-muted-foreground'}"> + <p id="{idPrefix}-cron-note" class="text-xs {invalid ? 'text-danger-fg' : 'text-muted-foreground'}"> {#if !shapeOk} Five fields: minute hour day-of-month month day-of-week. {:else if current && !current.valid} @@ -220,7 +220,7 @@ {#if previewError} <p class="text-danger-fg">{previewError}</p> {:else if current?.valid} - <ul class="mt-1 grid gap-0.5 font-mono text-[11px] tabular-nums sm:grid-cols-2"> + <ul class="mt-1 grid gap-0.5 font-mono text-xs tabular-nums sm:grid-cols-2"> {#each current.next as run (run)}<li>{when(run)}</li>{/each} </ul> {:else if shapeOk && !current} diff --git a/frontend/src/lib/components/experiments/SchedulesCard.svelte b/frontend/src/lib/components/experiments/SchedulesCard.svelte index 75797140..8227f26f 100644 --- a/frontend/src/lib/components/experiments/SchedulesCard.svelte +++ b/frontend/src/lib/components/experiments/SchedulesCard.svelte @@ -190,17 +190,17 @@ </td> <td data-label="When"> <span title="{s.cron} ({s.timezone})">{describeCron(s.cron)}</span> - <span class="cell-sub block text-[11px] text-muted-foreground">{s.timezone}{s.skip_if_active ? ' · skips while running' : ''}</span> + <span class="cell-sub block text-xs text-muted-foreground">{s.timezone}{s.skip_if_active ? ' · skips while running' : ''}</span> </td> <td data-label="Next run" class="whitespace-nowrap tabular-nums" title={s.next_run_at ? new Date(s.next_run_at).toLocaleString() : ''}> {#if s.enabled}{timeUntil(s.next_run_at)}{:else}<span class="text-muted-foreground">paused</span>{/if} </td> <td data-label="Last result"> {#if result} - <span class="inline-block rounded-full border px-2 py-0.5 text-[11px] {toneClasses[result.tone]}">{result.text}</span> - <span class="cell-sub mt-0.5 block text-[11px] text-muted-foreground">{timeAgo(s.last_run_at)}</span> + <span class="inline-block rounded-full border px-2 py-0.5 text-xs {toneClasses[result.tone]}">{result.text}</span> + <span class="cell-sub mt-0.5 block text-xs text-muted-foreground">{timeAgo(s.last_run_at)}</span> {#if s.last_result.experiment_id && s.last_result.experiment_id !== experimentId} - <a href="/experiments/{s.last_result.experiment_id}" class="text-[11px] text-success-fg hover:underline">Open the clone</a> + <a href="/experiments/{s.last_result.experiment_id}" class="text-xs text-success-fg hover:underline">Open the clone</a> {/if} {:else} <span class="text-muted-foreground">never ran</span> diff --git a/frontend/src/lib/components/experiments/format.ts b/frontend/src/lib/components/experiments/format.ts index 52f501ec..55f2ea79 100644 --- a/frontend/src/lib/components/experiments/format.ts +++ b/frontend/src/lib/components/experiments/format.ts @@ -230,6 +230,13 @@ export function formatHours(hours: number | null | undefined): string { return `≈ ${Math.round(hours)} h`; } +/** + * Fewest apps for which a cluster-bootstrap interval over apps is shown: with + * one or two apps the resampled means barely vary, so the "interval" would + * look far more certain than the data is. + */ +export const MIN_APPS_FOR_CI = 3; + /** 0..1 → "87%"; null → "—". */ export function formatPercent( value: number | null | undefined, diff --git a/frontend/src/lib/components/experiments/index.ts b/frontend/src/lib/components/experiments/index.ts index 3bf7a689..81bba856 100644 --- a/frontend/src/lib/components/experiments/index.ts +++ b/frontend/src/lib/components/experiments/index.ts @@ -11,7 +11,6 @@ export { default as PostRunEditor } from "./PostRunEditor.svelte"; export { default as CapacityStrip } from "./CapacityStrip.svelte"; export { default as ExperimentRow } from "./ExperimentRow.svelte"; export { default as ComparabilityTable } from "./ComparabilityTable.svelte"; -export { default as ExperimentDesignSummary } from "./ExperimentDesignSummary.svelte"; export { default as Leaderboard } from "./Leaderboard.svelte"; export { default as PairedDeltas } from "./PairedDeltas.svelte"; export { default as SchedulesCard } from "./SchedulesCard.svelte"; diff --git a/frontend/src/lib/components/experiments/monitor/ActivityFeed.svelte b/frontend/src/lib/components/experiments/monitor/ActivityFeed.svelte index a2d71f8a..69b36a8a 100644 --- a/frontend/src/lib/components/experiments/monitor/ActivityFeed.svelte +++ b/frontend/src/lib/components/experiments/monitor/ActivityFeed.svelte @@ -56,13 +56,13 @@ <ol class="max-h-80 divide-y divide-border/60 overflow-y-auto rounded-md border border-border" aria-live="polite"> {#each visible as item (item.key)} <li class="flex min-w-0 items-center gap-2 px-2.5 py-1.5 text-xs"> - <span class="w-16 shrink-0 font-mono text-[11px] tabular-nums text-muted-foreground">{time(item.ts)}</span> + <span class="w-16 shrink-0 font-mono text-xs tabular-nums text-muted-foreground">{time(item.ts)}</span> <span class={cn('size-2.5 shrink-0 rounded-[2px]', CELL_STATES[item.code].square, 'animate-none')} aria-hidden="true"></span> <a href="/applications/{item.jobId}" class="min-w-0 flex-1 truncate hover:underline"> <span class="font-medium">{conditionLabel(item.conditionId)}</span> <span class="text-muted-foreground"> · {appName(item.appId)}{item.repeat != null ? ` · #${item.repeat + 1}` : ''}</span> </a> - <span class="shrink-0 text-[11px] text-muted-foreground"> + <span class="shrink-0 text-xs text-muted-foreground"> {CELL_STATES[item.code].label}{item.attempt > 0 ? ` · attempt ${item.attempt + 1}` : ''} </span> </li> diff --git a/frontend/src/lib/components/experiments/monitor/ConditionGrid.svelte b/frontend/src/lib/components/experiments/monitor/ConditionGrid.svelte index 74c5c9d5..94971963 100644 --- a/frontend/src/lib/components/experiments/monitor/ConditionGrid.svelte +++ b/frontend/src/lib/components/experiments/monitor/ConditionGrid.svelte @@ -11,8 +11,10 @@ * - `grid`: the live cell grid (`ExperimentLive.grid`). * - `groupBy`: 'endpoint' | 'none'. `sort`: 'sequence' | 'label' | 'progress' | 'pass'. * - `baselineId`: condition marked "baseline". + * - `framed` (default true): own border; false inside a frame that has one (a StudyFigure). */ import { cn } from '$lib/utils'; + import { scrollHint } from '$lib/utils/scroll-hint'; import BatchProgress from '$lib/components/generator/batches/BatchProgress.svelte'; import ChevronRight from '@lucide/svelte/icons/chevron-right'; import { @@ -33,9 +35,10 @@ groupBy?: GridGroupBy; sort?: GridSort; baselineId?: number | null; + framed?: boolean; } - let { grid, groupBy = 'endpoint', sort = 'sequence', baselineId = null }: Props = $props(); + let { grid, groupBy = 'endpoint', sort = 'sequence', baselineId = null, framed = true }: Props = $props(); let collapsed = $state(new Set<string>()); @@ -116,7 +119,7 @@ > <span class="min-w-0 truncate font-medium" title={condition.label}>{condition.label}</span> {#if condition.id === baselineId} - <span class="shrink-0 rounded-full border border-primary/40 bg-primary/10 px-1.5 text-[11px] text-success-fg">baseline</span> + <span class="shrink-0 rounded-full border border-primary/40 bg-primary/10 px-1.5 text-xs text-primary">baseline</span> {/if} </div> {#each grid.apps as app, ai (app.id)} @@ -146,7 +149,7 @@ </div> {/each} <div role="cell" class="flex min-w-0 flex-col justify-center gap-1 border-b border-border/50 py-1 pl-2"> - <span class="flex items-baseline gap-1.5 font-mono text-[11px] tabular-nums"> + <span class="flex items-baseline gap-1.5 font-mono text-xs tabular-nums"> {#if summary.graded > 0} <span class="font-semibold text-foreground" title="Passed every check / graded">{summary.passed}/{summary.graded}</span> <span class="text-muted-foreground">{formatPercent(summary.passRate)}</span> @@ -159,28 +162,30 @@ </div> {/snippet} -<div class="space-y-3"> - <div class="relative overflow-x-auto rounded-md border border-border" role="region" aria-label="Condition by app grid"> +<div class={cn('space-y-3', !framed && 'pb-3')}> + <div class={cn('relative overflow-x-auto', framed && 'rounded-md border border-border')} role="region" aria-label="Condition by app grid" {@attach scrollHint()}> + <!-- Phones: two columns (label, progress) that fit the width; no sideways scroll. --> <div - class="grid w-max min-w-full items-stretch px-2 max-md:!grid-cols-[minmax(0,1fr)_auto]" + class="grid w-max min-w-full items-stretch px-2 max-md:!w-full max-md:!grid-cols-[minmax(0,1fr)_auto]" style="grid-template-columns: {gridTemplate}" role="table" aria-label="Runs per condition and app; one square per repeat" > <div class="contents" role="row"> - <div role="columnheader" class="sticky left-0 z-20 border-b border-border bg-card py-1.5 pr-2 section-label"> + <div role="columnheader" class="sticky left-0 z-20 flex items-end border-b border-border bg-card py-1.5 pr-2 section-label"> Condition </div> {#each grid.apps as app (app.id)} + <!-- App columns are only as wide as their squares: the name runs vertically. --> <div role="columnheader" - class="hidden min-w-0 border-b border-border px-1 py-1.5 text-[11px] text-muted-foreground md:block" + class="hidden min-w-0 items-end justify-center border-b border-border px-1 py-1.5 text-xs text-muted-foreground md:flex" title={app.name} > - <span class="block truncate">{app.slug || app.name}</span> + <span class="block max-h-28 truncate [writing-mode:vertical-rl] rotate-180">{app.name}</span> </div> {/each} - <div role="columnheader" class="border-b border-border py-1.5 pl-2 section-label">Done</div> + <div role="columnheader" class="flex items-end border-b border-border py-1.5 pl-2 section-label">Done</div> </div> {#each groups as group (group.key)} @@ -219,7 +224,7 @@ </div> {#if legend.length > 0} - <ul class="hidden flex-wrap gap-x-4 gap-y-1 text-[11px] text-muted-foreground md:flex" aria-label="Square legend"> + <ul class={cn('hidden flex-wrap gap-x-4 gap-y-1 text-xs text-muted-foreground md:flex', !framed && 'px-4')} aria-label="Square legend"> {#each legend as code (code)} <li class="inline-flex items-center gap-1.5"> <span class={cn('inline-block size-2.5 rounded-[2px]', CELL_STATES[code].square, 'animate-none')} aria-hidden="true"></span> diff --git a/frontend/src/lib/components/experiments/monitor/EndpointLane.svelte b/frontend/src/lib/components/experiments/monitor/EndpointLane.svelte index b06eaf0c..62f800ac 100644 --- a/frontend/src/lib/components/experiments/monitor/EndpointLane.svelte +++ b/frontend/src/lib/components/experiments/monitor/EndpointLane.svelte @@ -49,14 +49,14 @@ const SHOWN_JOBS = 6; </script> -<Card.Root class="gap-3 py-4" aria-labelledby="lane-{experimentId}-{endpoint.id}"> +<Card.Root class="min-w-0 gap-3 py-4" aria-labelledby="lane-{experimentId}-{endpoint.id}"> <Card.Header class="gap-1 px-4"> <div class="flex items-start justify-between gap-2"> <div class="min-w-0"> <Card.Title class="flex items-center gap-1.5 text-sm"> <span id="lane-{experimentId}-{endpoint.id}" class="truncate font-mono">{endpoint.name || endpoint.slug}</span> </Card.Title> - <p class="text-[11px] text-muted-foreground"> + <p class="text-xs text-muted-foreground"> {[kindLabel(endpoint.kind), isOpenRouter ? '' : endpoint.connection, endpoint.sessions > 1 ? `${endpoint.sessions} relays` : ''] .filter(Boolean) .join(' · ')} @@ -83,7 +83,7 @@ /> </div> {#if endpoint.jobs.held_by_others > 0} - <p class="text-[11px] text-muted-foreground tabular-nums"> + <p class="text-xs text-muted-foreground tabular-nums"> {endpoint.jobs.held_by_others} held by other experiments (hatched) </p> {/if} @@ -96,7 +96,7 @@ {#each endpoint.loaded as model (model.name)} <span class={cn( - 'inline-flex items-center gap-1 rounded-full border px-2 py-0.5 font-mono text-[11px]', + 'inline-flex items-center gap-1 rounded-full border px-2 py-0.5 font-mono text-xs', model.cpu_spill ? toneClasses.warning : 'border-border', ownModels.has(model.name) && 'font-semibold' )} @@ -112,11 +112,11 @@ </span> {:else} {#if endpoint.loaded_others === 0} - <span class="text-[11px] text-muted-foreground">No model loaded</span> + <span class="text-xs text-muted-foreground">No model loaded</span> {/if} {/each} {#if endpoint.loaded_others > 0} - <span class="rounded-full border border-border px-2 py-0.5 text-[11px] text-muted-foreground" title="Loaded for other users"> + <span class="rounded-full border border-border px-2 py-0.5 text-xs text-muted-foreground" title="Loaded for other users"> +{endpoint.loaded_others} </span> {/if} @@ -127,7 +127,7 @@ <div class="space-y-1"> <p class="section-label">Running for this experiment · {activeJobs.length}</p> {#if activeJobs.length === 0} - <p class="text-[11px] text-muted-foreground">Nothing running here right now.</p> + <p class="text-xs text-muted-foreground">Nothing running here right now.</p> {:else} <ul class="space-y-0.5"> {#each activeJobs.slice(0, SHOWN_JOBS) as job (job.jobId)} @@ -135,17 +135,17 @@ <a href="/applications/{job.jobId}" class="min-w-0 flex-1 truncate hover:underline" title="{job.label} · {job.app} · repeat {job.repeat + 1}"> {job.label} <span class="text-muted-foreground">· {job.app} · #{job.repeat + 1}</span> </a> - <span class="shrink-0 text-[11px] text-muted-foreground">{job.phase}</span> + <span class="shrink-0 text-xs text-muted-foreground">{job.phase}</span> </li> {/each} </ul> {#if activeJobs.length > SHOWN_JOBS} - <p class="text-[11px] text-muted-foreground">+{activeJobs.length - SHOWN_JOBS} more</p> + <p class="text-xs text-muted-foreground">+{activeJobs.length - SHOWN_JOBS} more</p> {/if} {/if} </div> - <dl class="grid grid-cols-3 gap-2 border-t border-border/60 pt-2 text-[11px]"> + <dl class="grid grid-cols-3 gap-2 border-t border-border/60 pt-2 text-xs"> <div> <dt class="text-muted-foreground">Queued here</dt> <dd class="font-mono tabular-nums">{endpoint.queued_jobs_this_experiment}</dd> @@ -161,12 +161,12 @@ </dl> {#if endpoint.circuit === 'open'} - <p class="rounded-md border px-2 py-1 text-[11px] {toneClasses.danger}"> + <p class="rounded-md border px-2 py-1 text-xs {toneClasses.danger}"> Paused after repeated errors; the executor retries shortly. </p> {/if} {#if endpoint.blocked_reason && endpoint.queued_jobs_this_experiment > 0} - <p class="rounded-md border px-2 py-1 text-[11px] {toneClasses.warning}"> + <p class="rounded-md border px-2 py-1 text-xs {toneClasses.warning}"> Queued runs: {blockedReasonText(endpoint.blocked_reason)} </p> {/if} @@ -174,7 +174,7 @@ <div> <button type="button" - class="flex cursor-pointer items-center gap-1 text-[11px] font-medium text-danger-fg" + class="flex cursor-pointer items-center gap-1 text-xs font-medium text-danger-fg" aria-expanded={errorOpen} onclick={() => (errorOpen = !errorOpen)} > @@ -182,7 +182,7 @@ Last error </button> {#if errorOpen} - <p class="mt-1 break-words font-mono text-[11px] text-muted-foreground">{endpoint.last_error}</p> + <p class="mt-1 break-words font-mono text-xs text-muted-foreground">{endpoint.last_error}</p> {/if} </div> {/if} diff --git a/frontend/src/lib/components/experiments/monitor/FailureSummary.svelte b/frontend/src/lib/components/experiments/monitor/FailureSummary.svelte new file mode 100644 index 00000000..5429b0af --- /dev/null +++ b/frontend/src/lib/components/experiments/monitor/FailureSummary.svelte @@ -0,0 +1,120 @@ +<script lang="ts" module> + /** A likely cause and next step for common failure messages (first match wins). */ + const HINTS: { test: RegExp; hint: string }[] = [ + { + test: /\b429\b|rate.?limit/i, + hint: 'The provider is rate-limiting this model. Free (“:free”) models share one upstream pool: retry later, or clone the experiment with a paid variant.', + }, + { test: /\b402\b|insufficient (credit|funds)|out of credit/i, hint: 'The OpenRouter account is out of credit.' }, + { test: /\b40[13]\b|api key|unauthori[sz]ed|forbidden/i, hint: 'The provider rejected the API key: check Settings → API keys.' }, + { test: /timed? ?out|timeout|wall.?clock/i, hint: 'The run hit a time limit. If it repeats, raise the wall clock in the budget preset.' }, + ]; + + export function failureHint(message: string): string | null { + return HINTS.find((h) => h.test.test(message))?.hint ?? null; + } +</script> + +<script lang="ts"> + /** + * FailureSummary — why this experiment's runs failed: current failed + * attempts grouped by error message (most frequent first), each with a + * likely cause when the message is a known one and a link to one example run. + * Renders nothing while no run has failed. + * + * Props: + * - `experimentId`. + * - `failed`: failed + cancelled cell count from the live status; a change refetches. + */ + import { untrack } from 'svelte'; + import * as Card from '$lib/components/ui/card'; + import { Button } from '$lib/components/ui/button'; + import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import { formatApiError } from '$lib/api/core'; + import { getGenerationJobs, type GenerationJobList } from '$lib/api/generation'; + import { plural } from '../format'; + + interface Props { + experimentId: string; + failed: number; + } + + let { experimentId, failed }: Props = $props(); + + const MAX_GROUPS = 5; + + let jobs = $state.raw<GenerationJobList[] | null>(null); + let total = $state(0); + let error = $state(''); + let seq = 0; + + async function load() { + const mine = ++seq; + try { + const res = await getGenerationJobs({ experiment_id: experimentId, status: 'failed', current: true, per_page: 100, sort_by: 'sequence' }); + if (mine !== seq) return; + jobs = res.items; + total = res.total; + error = ''; + } catch (e) { + if (mine === seq) error = formatApiError(e, 'Could not load the failed runs.'); + } + } + + $effect(() => { + void experimentId; + if (failed <= 0) return; + untrack(() => void load()); + }); + + const groups = $derived.by(() => { + const byMessage = new Map<string, { message: string; count: number; exampleId: string }>(); + for (const job of jobs ?? []) { + const message = (job.error_message || 'No error message recorded').replace(/\s+/g, ' ').trim(); + const group = byMessage.get(message); + if (group) group.count += 1; + else byMessage.set(message, { message, count: 1, exampleId: job.id }); + } + return [...byMessage.values()].sort((a, b) => b.count - a.count); + }); +</script> + +{#if failed > 0 && (error || groups.length > 0)} + <Card.Root class="gap-0 py-0"> + <Card.Header class="flex-row items-center justify-between gap-2 border-b border-border py-3"> + <div class="space-y-1"> + <Card.Title>Why runs failed</Card.Title> + <Card.Description> + {plural(total, 'failed run')}{groups.length > 1 ? `, ${plural(groups.length, 'distinct error')}` : ''}. Failed cells can be re-run with their original seeds. + </Card.Description> + </div> + <Button variant="ghost" size="sm" href="?tab=runs">Open runs</Button> + </Card.Header> + <Card.Content class="p-0"> + {#if error} + <div class="p-3"><ErrorAlert message={error} onretry={load} /></div> + {:else} + <ul class="divide-y divide-border/60"> + {#each groups.slice(0, MAX_GROUPS) as group (group.message)} + {@const hint = failureHint(group.message)} + <li class="flex min-w-0 items-start gap-3 px-4 py-2.5 text-sm"> + <span class="mt-px w-10 shrink-0 text-right font-mono tabular-nums text-danger-fg" title={plural(group.count, 'run')}>{group.count}×</span> + <div class="min-w-0 flex-1 space-y-0.5"> + <p class="line-clamp-2 break-words font-mono text-xs text-foreground" title={group.message}>{group.message}</p> + {#if hint}<p class="text-xs text-muted-foreground">{hint}</p>{/if} + </div> + <a class="shrink-0 text-xs text-muted-foreground underline-offset-2 hover:text-foreground hover:underline" href="/applications/{group.exampleId}"> + Example run + </a> + </li> + {/each} + </ul> + {#if groups.length > MAX_GROUPS} + <p class="border-t border-border/60 px-4 py-2 text-xs text-muted-foreground"> + {plural(groups.length - MAX_GROUPS, 'more error')} — see Runs. + </p> + {/if} + {/if} + </Card.Content> + </Card.Root> +{/if} diff --git a/frontend/src/lib/components/experiments/monitor/LiveStrip.svelte b/frontend/src/lib/components/experiments/monitor/LiveStrip.svelte new file mode 100644 index 00000000..e87ff034 --- /dev/null +++ b/frontend/src/lib/components/experiments/monitor/LiveStrip.svelte @@ -0,0 +1,66 @@ +<script lang="ts"> + /** + * LiveStrip — overall progress, slot lanes and the live-connection state, + * one block for the experiment page header strip (shown on every tab). + * Once `finished`, only the bar stays (with its legend when the outcome is + * mixed): idle lanes and "not live" add nothing to a completed run. + */ + import { cn } from '$lib/utils'; + import BatchProgress from '$lib/components/generator/batches/BatchProgress.svelte'; + import type { ExperimentLive } from '../live.svelte'; + import { toBatchCounts } from '../format'; + + interface Props { + live: ExperimentLive; + finished?: boolean; + } + + let { live, finished = false }: Props = $props(); + + const status = $derived(live.status); + + const connectionText = $derived( + live.connection === 'open' && !live.polling + ? { text: 'Live', tone: 'text-success-fg' } + : live.connection === 'closed' + ? { text: 'Finished — not live', tone: 'text-muted-foreground' } + : live.polling + ? { text: 'Polling every 5 s', tone: 'text-warning-fg' } + : { text: 'Connecting…', tone: 'text-muted-foreground' } + ); + + const batchCounts = $derived(status ? toBatchCounts(status.counts) : null); + const mixedOutcome = $derived( + batchCounts ? Object.values(batchCounts).filter((n) => typeof n === 'number' && n > 0).length > 1 : false + ); + + const lanes = $derived( + status + ? [ + { label: 'Sandboxes', ...status.lanes.sandbox }, + { label: 'Eval', ...status.lanes.eval }, + { label: 'LLM jobs', ...status.lanes.llm }, + { label: 'Post', ...status.lanes.post }, + ] + : [] + ); +</script> + +{#if status} + <div class="space-y-1"> + <BatchProgress counts={batchCounts!} total={status.cells.total} legend={!finished || mixedOutcome} /> + {#if !finished} + <div class="flex flex-wrap items-center justify-between gap-2 text-xs"> + <p class="flex flex-wrap gap-x-3 font-mono tabular-nums text-muted-foreground"> + {#each lanes as lane (lane.label)} + <span><span class="font-sans">{lane.label}</span> <span class={cn(lane.total > 0 && lane.used >= lane.total ? 'text-warning-fg' : 'text-foreground')}>{lane.used}/{lane.total}</span></span> + {/each} + </p> + <p class={cn('inline-flex items-center gap-1.5', connectionText.tone)} role="status"> + <span class={cn('status-dot h-1.5 w-1.5', live.connection === 'open' && !live.polling && 'status-dot-live')} aria-hidden="true"></span> + {connectionText.text} + </p> + </div> + {/if} + </div> +{/if} diff --git a/frontend/src/lib/components/experiments/study/PassRatePlot.svelte b/frontend/src/lib/components/experiments/study/PassRatePlot.svelte new file mode 100644 index 00000000..92b9d14e --- /dev/null +++ b/frontend/src/lib/components/experiments/study/PassRatePlot.svelte @@ -0,0 +1,168 @@ +<script lang="ts"> + /** + * PassRatePlot — dot-and-whisker figure: every condition's mean app-level + * pass rate with its cluster-bootstrap 95% interval (shown from + * MIN_APPS_FOR_CI apps), ranked, on one 0–100 % axis. A single series, so + * one mark colour (chart slot 1) and no legend; the baseline is a hollow + * diamond and a hairline through every row at its mean. Every value is + * printed beside its mark, so the native tooltip only repeats them. + * + * Props: + * - `results`: `GET /results/`. + * - `limit`: show only the first N ranks (the Abstract), with a "+N more" row + * linking to `moreHref`. + * Phones get a stacked list (label and estimate over a full-width plot row). + */ + import { cn } from '$lib/utils'; + import { tableFrame } from '$lib/components/table'; + import type { ExperimentResults, LeaderboardRow } from '$lib/api/types/experiments'; + import { MIN_APPS_FOR_CI, formatPercent } from '../format'; + + interface Props { + results: ExperimentResults; + limit?: number; + moreHref?: string; + } + + let { results, limit, moreHref }: Props = $props(); + + const TICKS = [0, 0.25, 0.5, 0.75, 1]; + + const ranked = $derived([...results.leaderboard].sort((a, b) => a.rank - b.rank)); + const rows = $derived(limit ? ranked.slice(0, limit) : ranked); + const hidden = $derived(ranked.length - rows.length); + const baselineId = $derived(results.baseline_condition_id); + const baselineMean = $derived(ranked.find((r) => r.condition_id === baselineId)?.pass_rate_mean ?? null); + + const pct = (v: number) => `${Math.min(100, Math.max(0, v * 100))}%`; + const ciOf = (row: LeaderboardRow) => (row.n_apps >= MIN_APPS_FOR_CI ? row.pass_rate_ci95 : null); + + function describe(row: LeaderboardRow): string { + if (row.pass_rate_mean == null) return `${row.label}: not graded yet`; + const ci = ciOf(row); + return `${row.label}: ${formatPercent(row.pass_rate_mean, 1)}${ + ci ? `, 95% CI ${formatPercent(ci[0], 1)} to ${formatPercent(ci[1], 1)}` : ', no interval' + }; ${row.k_passed}/${row.n_graded} cells passed over ${row.n_apps} app${row.n_apps === 1 ? '' : 's'}`; + } +</script> + +{#snippet axis()} + <span class="relative block h-4 font-mono text-[11px] normal-case tracking-normal text-muted-foreground" aria-hidden="true"> + {#each TICKS as t (t)} + <span class={cn('absolute top-0', t === 0 ? '' : t === 1 ? '-translate-x-full' : '-translate-x-1/2')} style="left: {pct(t)}">{t * 100}%</span> + {/each} + </span> +{/snippet} + +{#snippet marks(row: LeaderboardRow)} + {@const isBaseline = row.condition_id === baselineId} + {@const ci = ciOf(row)} + {@const mean = row.pass_rate_mean} + <span class="relative block h-8" role="img" aria-label={describe(row)} title={describe(row)}> + {#each TICKS as t (t)} + <span class="absolute inset-y-0 w-px bg-border/70" style="left: {pct(t)}" aria-hidden="true"></span> + {/each} + {#if baselineMean != null} + <span class="absolute inset-y-0 w-px bg-foreground/35" style="left: {pct(baselineMean)}" aria-hidden="true"></span> + {/if} + {#if ci} + <span + class="absolute top-1/2 h-px -translate-y-1/2 bg-foreground/60" + style="left: {pct(ci[0])}; width: {pct(Math.max(0.002, ci[1] - ci[0]))}" + aria-hidden="true" + ></span> + {#each [ci[0], ci[1]] as end, i (i)} + <span class="absolute top-1/2 h-2.5 w-px -translate-y-1/2 bg-foreground/60" style="left: {pct(end)}" aria-hidden="true"></span> + {/each} + {/if} + {#if mean != null} + <span + class={cn( + 'absolute top-1/2 size-2.5 -translate-x-1/2 -translate-y-1/2 ring-2 ring-card', + isBaseline ? 'rotate-45 border-2 border-foreground bg-card' : 'rounded-full bg-chart-1' + )} + style="left: {pct(mean)}" + aria-hidden="true" + ></span> + {/if} + </span> +{/snippet} + +{#snippet estimate(row: LeaderboardRow)} + {@const ci = ciOf(row)} + {#if row.pass_rate_mean == null} + <span class="text-muted-foreground">not graded</span> + {:else} + <span class="font-semibold text-foreground">{formatPercent(row.pass_rate_mean, 1)}</span> + {#if ci} + <span class="text-muted-foreground">[{formatPercent(ci[0])}, {formatPercent(ci[1])}]</span> + {:else} + <span class="text-muted-foreground" title="An interval needs at least {MIN_APPS_FOR_CI} apps">[n={row.n_apps}]</span> + {/if} + {/if} +{/snippet} + +<!-- Phones: one stacked row per condition (label and estimate, then the plot at full width). --> +<div class="sm:hidden"> + <div class="border-b border-border bg-muted/40 px-4 py-1.5">{@render axis()}</div> + <ol class="divide-y divide-border/60"> + {#each rows as row (row.condition_id)} + <li class="px-4 pt-2"> + <div class="flex items-baseline gap-2 text-sm"> + <span class="font-mono text-xs text-muted-foreground">{row.rank}</span> + <span class="min-w-0 flex-1 break-words font-medium">{row.label}</span> + <span class="shrink-0 font-mono text-xs">{@render estimate(row)}</span> + </div> + {#if row.condition_id === baselineId}<p class="pl-4 font-mono text-xs text-muted-foreground">baseline</p>{/if} + {@render marks(row)} + </li> + {/each} + </ol> + {#if hidden > 0} + <p class="border-t border-border/60 px-4 py-2 text-xs text-muted-foreground"> + +{hidden} more condition{hidden === 1 ? '' : 's'}{#if moreHref} — <a class="settings-link" href={moreHref}>see Table 1</a>{/if} + </p> + {/if} +</div> + +<div class="table-frame max-sm:hidden" {@attach tableFrame}> + <table class="min-w-[38rem]"> + <thead> + <tr> + <th scope="col" class="w-8 text-right">#</th> + <th scope="col" class="col-sticky">Condition</th> + <th scope="col" class="w-[48%] min-w-60"> + {@render axis()} + <span class="sr-only">Mean pass rate with 95% interval</span> + </th> + <th scope="col" class="num" title="Mean of app-level pass rates [cluster-bootstrap 95% CI]">Mean [95% CI]</th> + <th scope="col" class="num" title="Cells that passed every check / graded cells">k/n</th> + </tr> + </thead> + <tbody> + {#each rows as row (row.condition_id)} + <tr> + <td class="text-right font-mono text-xs tabular-nums text-muted-foreground">{row.rank}</td> + <th + scope="row" + class="col-sticky top-auto z-[1] min-w-44 max-w-64 bg-card text-left font-sans text-sm font-normal normal-case tracking-normal text-foreground backdrop-blur-none" + > + <span class="line-clamp-2 break-words font-medium" title={row.label}>{row.label}</span> + {#if row.condition_id === baselineId}<span class="block font-mono text-xs text-muted-foreground">baseline</span>{/if} + </th> + <td class="py-0">{@render marks(row)}</td> + <td class="num text-xs">{@render estimate(row)}</td> + <td class="num text-xs text-muted-foreground">{row.k_passed}/{row.n_graded}</td> + </tr> + {/each} + {#if hidden > 0} + <tr> + <td></td> + <td colspan="4" class="text-xs text-muted-foreground"> + +{hidden} more condition{hidden === 1 ? '' : 's'}{#if moreHref} — <a class="settings-link" href={moreHref}>see Table 1</a>{/if} + </td> + </tr> + {/if} + </tbody> + </table> +</div> diff --git a/frontend/src/lib/components/experiments/study/StudyFigure.svelte b/frontend/src/lib/components/experiments/study/StudyFigure.svelte new file mode 100644 index 00000000..462a477e --- /dev/null +++ b/frontend/src/lib/components/experiments/study/StudyFigure.svelte @@ -0,0 +1,52 @@ +<script lang="ts"> + /** + * StudyFigure — a numbered figure or table of the experiment page: + * "FIGURE 1" eyebrow and a serif title above the body, controls on the + * right, and a muted caption below (estimator, n, what the marks mean) — + * the way a paper prints them. + * + * Props: + * - `id`: anchor; `kind`: 'figure' | 'table'; `number` (1, 2, "M1" …). + * - `title`: one line saying what is plotted. + * - `flush`: no body padding (flush tables, which bring their own frame). + * - `controls`, `caption`, `children` snippets. + */ + import type { Snippet } from 'svelte'; + import { cn } from '$lib/utils'; + + interface Props { + id: string; + kind?: 'figure' | 'table'; + number: number | string; + title: string; + flush?: boolean; + class?: string; + controls?: Snippet; + caption?: Snippet; + children: Snippet; + } + + let { id, kind = 'figure', number, title, flush = false, class: className, controls, caption, children }: Props = $props(); +</script> + +<figure + {id} + aria-labelledby="{id}-title" + class={cn('min-w-0 scroll-mt-16 overflow-clip rounded-md border border-border bg-card shadow-[var(--shadow-card)]', className)} +> + <div class="flex flex-wrap items-end justify-between gap-x-4 gap-y-2 border-b border-border/60 px-4 py-2.5"> + <div class="min-w-0 space-y-0.5"> + <p class="section-label">{kind === 'table' ? 'Table' : 'Figure'} {number}</p> + <h3 id="{id}-title" class="font-display text-[17px] font-semibold leading-snug tracking-tight">{title}</h3> + </div> + {#if controls}<div class="flex flex-wrap items-center gap-x-4 gap-y-2">{@render controls()}</div>{/if} + </div> + <div class={cn('min-w-0', !flush && 'p-4')}> + {@render children()} + </div> + {#if caption} + <figcaption class="border-t border-border/60 px-4 py-2.5 text-xs leading-relaxed text-muted-foreground"> + {@render caption()} + </figcaption> + {/if} +</figure> diff --git a/frontend/src/lib/components/experiments/study/StudyMasthead.svelte b/frontend/src/lib/components/experiments/study/StudyMasthead.svelte new file mode 100644 index 00000000..0b5f8a16 --- /dev/null +++ b/frontend/src/lib/components/experiments/study/StudyMasthead.svelte @@ -0,0 +1,159 @@ +<script lang="ts"> + /** + * StudyMasthead — the experiment page's header, set like the first page of + * a paper: accession id and method as the eyebrow, the title in the display + * serif, status and dates, the hypothesis as a pull quote, a spec strip + * (design formula · baseline · outcome · randomisation), then the page's + * progress strip and section tabs. Owns the page `<h1>` and `<title>` (it + * stands in for PageHeader on this page). + * + * Props: + * - `experiment`: the detail; `status`: live status (null for drafts). + * - `policyEffective`: the launched run order per endpoint. + * - `actions`, `progress`, `nav` snippets. + */ + import type { Snippet } from 'svelte'; + import ArrowLeft from '@lucide/svelte/icons/arrow-left'; + import FlaskConical from '@lucide/svelte/icons/flask-conical'; + import StatusBadge from '$lib/components/StatusBadge.svelte'; + import { pageTitle } from '$lib/stores/page-title.svelte'; + import type { ExperimentDetail, ExperimentStatusV2Payload } from '$lib/api/types/experiments'; + import { timeAgo } from '$lib/utils/dashboard'; + import { MODE_LABELS, conditionName, effectivePolicyLabel, experimentTones, isTerminal, plural, policyLabel } from '../format'; + import { designShape, shortExperimentId } from './findings'; + + interface Props { + experiment: ExperimentDetail; + status?: ExperimentStatusV2Payload | null; + policyEffective?: Record<string, string> | string | null; + actions?: Snippet; + progress?: Snippet; + nav?: Snippet; + } + + let { experiment, status = null, policyEffective = null, actions, progress, nav }: Props = $props(); + + $effect(() => { + const title = experiment.name; + pageTitle.current = title; + return () => { + if (pageTitle.current === title) pageTitle.current = ''; + }; + }); + + const shape = $derived(designShape(experiment)); + const lifecycle = $derived(status?.status ?? experiment.status); + const baseline = $derived(experiment.conditions.find((c) => c.id === experiment.baseline_condition_id) ?? null); + const dateFmt = new Intl.DateTimeFormat(undefined, { day: 'numeric', month: 'short', year: 'numeric' }); + + const outcome = $derived( + experiment.mode === 'agent' + ? { value: 'Pass rate', detail: 'hidden conformance suite, app-level mean' } + : experiment.post_run?.evaluate + ? { value: 'Pass rate', detail: 'evaluation checks, app-level mean' } + : { value: 'Pass rate', detail: 'needs Evaluate (Operations → Automation)' } + ); + const order = $derived( + experiment.status === 'draft' + ? policyLabel(experiment.scheduling_policy) + : effectivePolicyLabel(policyEffective, experiment.scheduling_policy) + ); +</script> + +<svelte:head> + <title>{experiment.name} · LLM Lab + + +
+
+ + +
+

+

+

+ {experiment.name} +

+

+ + Registered {dateFmt.format(new Date(experiment.created_at))} + {#if experiment.launched_at} + + launched {timeAgo(experiment.launched_at)} + {/if} + {#if experiment.completed_at && isTerminal(lifecycle)} + + finished {timeAgo(experiment.completed_at)} + {/if} + {#if experiment.cloned_from} + + clone of {experiment.cloned_from.name} + {/if} +

+
+ {#if actions} +
{@render actions()}
+ {/if} +
+ + {#if experiment.hypothesis} +
+

Hypothesis

+
+ {experiment.hypothesis} +
+
+ {/if} + +
+
+
Design
+
+ {shape.conditions} condition{shape.conditions === 1 ? '' : 's'} + × + {shape.apps} app{shape.apps === 1 ? '' : 's'} + × + {shape.repeats} repeat{shape.repeats === 1 ? '' : 's'} + = + {shape.cells.toLocaleString()} cells +
+
+
+
Baseline
+
+ {#if baseline}{conditionName(baseline)}{:else}None set{/if} +
+
+
+
Outcome
+
+ {outcome.value} · {outcome.detail} +
+
+
+
Randomisation
+
+ {#if experiment.base_seed != null} + seed {experiment.base_seed} + {:else} + random seeds + {/if} + · {order} +
+
+
+ + {#if progress} +
{@render progress()}
+ {/if} + {#if nav} +
{@render nav()}
+ {/if} +
diff --git a/frontend/src/lib/components/experiments/study/StudyRecord.svelte b/frontend/src/lib/components/experiments/study/StudyRecord.svelte new file mode 100644 index 00000000..7cd62328 --- /dev/null +++ b/frontend/src/lib/components/experiments/study/StudyRecord.svelte @@ -0,0 +1,135 @@ + + + diff --git a/frontend/src/lib/components/experiments/study/StudySection.svelte b/frontend/src/lib/components/experiments/study/StudySection.svelte new file mode 100644 index 00000000..de7256c8 --- /dev/null +++ b/frontend/src/lib/components/experiments/study/StudySection.svelte @@ -0,0 +1,32 @@ + + +
+ + {@render children()} +
diff --git a/frontend/src/lib/components/experiments/study/findings.test.ts b/frontend/src/lib/components/experiments/study/findings.test.ts new file mode 100644 index 00000000..b8ce9adf --- /dev/null +++ b/frontend/src/lib/components/experiments/study/findings.test.ts @@ -0,0 +1,254 @@ +import { describe, expect, it } from "vitest"; +import type { + ExperimentConditionV2, + ExperimentResults, + LeaderboardRow, + PairedRow, +} from "$lib/api/types/experiments"; +import { + designShape, + keyFindings, + shortExperimentId, + type Finding, +} from "./findings"; +import { resolveExperimentTab } from "./tabs"; + +function condition( + id: number, + model: number, + preset: number, + overrides: Record = {}, +): ExperimentConditionV2 { + return { + id, + label: `c${id}`, + profile: null, + model, + harness: "opencode", + harness_preset: preset, + model_name: `m${model}`, + bundle_slug: null, + bundle_version: null, + param_overrides: overrides, + created_at: "", + }; +} + +function row( + id: number, + rank: number, + mean: number | null, + extra: Partial = {}, +): LeaderboardRow { + return { + rank, + condition_id: id, + label: `Condition ${id}`, + model: { id, name: `m${id}`, endpoint_slug: null }, + harness: "opencode", + n_apps: 8, + repeats_per_app: 3, + n_cells: 24, + n_graded: mean == null ? 0 : 24, + k_passed: 0, + n_missing: { + failed: 0, + ungraded: 0, + excluded_invalid: 0, + excluded_truncated: 0, + not_common: 0, + }, + pass_rate_mean: mean, + pass_rate_ci95: mean == null ? null : [mean - 0.1, mean + 0.1], + cost_usd: 0, + tokens_total: 0, + median_seconds: null, + timing_endpoint_slug: null, + terminations: {}, + validity: { ok: 0, warn: 0, invalid: 0 }, + mixed_fingerprints: null, + ...extra, + }; +} + +function paired(id: number, delta: number, pHolm: number | null): PairedRow { + return { + condition_id: id, + n_pairs: 8, + mean_delta: delta, + ci95: [delta - 0.05, delta + 0.05], + wins: 4, + losses: 2, + ties: 2, + sign_test_p: null, + wilcoxon_p: null, + p_holm: pHolm, + }; +} + +function results( + leaderboard: LeaderboardRow[], + rows: PairedRow[], + extra: Partial = {}, +): ExperimentResults { + return { + metric: "pass_rate", + exclude_invalid: false, + exclude_truncated: true, + common_cells: true, + fingerprint_filter: "all", + baseline_condition_id: 1, + baseline_source: "preregistered", + interim: false, + balanced: true, + leaderboard, + matrix: { apps: [], cells: [] }, + paired: { + baseline_condition_id: 1, + pairing: "app_blocked", + n_apps: 8, + rows, + }, + notes: [], + generated_at: "", + ...extra, + }; +} + +const text = (f: Finding) => + f.parts.map((p) => (typeof p === "string" ? p : p.strong)).join(""); +const labelOf = (id: number) => `Condition ${id}`; + +describe("designShape", () => { + it("counts models × variants, ablations apart, and the cells", () => { + const shape = designShape({ + conditions: [ + condition(1, 10, 3), + condition(2, 10, 5), + condition(3, 11, 3), + condition(4, 11, 5), + condition(5, 10, 3, { temperature: 0.2 }), + ], + app_requirement_ids: [1, 2, 3], + repeats: 2, + total_cells: 0, + }); + expect(shape).toEqual({ + conditions: 5, + models: 2, + variants: 2, + ablations: 1, + apps: 3, + repeats: 2, + cells: 30, + }); + }); +}); + +describe("shortExperimentId", () => { + it("uses the first UUID block as an accession number", () => { + expect(shortExperimentId("955a5bd5-031f-4518-a6bc-afe09b0e98c3")).toBe( + "EXP-955A5BD5", + ); + }); +}); + +describe("keyFindings", () => { + it("is empty while nothing is graded", () => { + expect(keyFindings(results([row(1, 1, null)], []), labelOf)).toEqual([]); + }); + + it("names the leader and the significant effect against the baseline", () => { + const findings = keyFindings( + results( + [row(2, 1, 0.8), row(1, 2, 0.6), row(3, 3, 0.4)], + [paired(2, 0.2, 0.01), paired(3, -0.2, 0.2)], + ), + labelOf, + ); + expect(findings.map((f) => f.kind)).toEqual(["leader", "effect"]); + expect(text(findings[0])).toContain( + "Highest mean pass rate: Condition 2, 80%", + ); + expect(text(findings[0])).toContain("Lowest: Condition 3, 40%"); + expect(text(findings[1])).toContain( + "1 of 2 conditions differ from the baseline Condition 1", + ); + expect(text(findings[1])).toContain("+20.0 pp"); + }); + + it("reports a null result with the largest observed difference", () => { + const findings = keyFindings( + results([row(2, 1, 0.65), row(1, 2, 0.6)], [paired(2, 0.05, 0.4)]), + labelOf, + ); + expect(findings[1].kind).toBe("null"); + expect(text(findings[1])).toContain( + "No condition differs from the baseline", + ); + expect(text(findings[1])).toContain("p = 0.400"); + }); + + it("withholds tests while interim", () => { + const findings = keyFindings( + results([row(2, 1, 0.65), row(1, 2, 0.6)], [paired(2, 0.05, null)], { + interim: true, + }), + labelOf, + ); + expect(findings[1].kind).toBe("interim"); + expect(text(findings[1])).not.toContain("p ="); + }); + + it("calls out ties for first place", () => { + const findings = keyFindings( + results([row(1, 1, 0.5), row(2, 1, 0.5)], []), + labelOf, + ); + expect(text(findings[0])).toContain( + "2 conditions share the highest mean pass rate", + ); + }); + + it("lists missing grades, mixed fingerprints and too few apps as limitations", () => { + const findings = keyFindings( + results( + [ + row(1, 1, 1, { + n_apps: 1, + n_cells: 4, + n_missing: { + failed: 1, + ungraded: 0, + excluded_invalid: 0, + excluded_truncated: 1, + not_common: 0, + }, + mixed_fingerprints: { count: 2, majority_share: 0.75 }, + }), + ], + [], + ), + labelOf, + ); + const caveats = findings.filter((f) => f.kind === "caveat").map(text); + expect(caveats).toHaveLength(3); + expect(caveats[0]).toContain("2 of 4 cells"); + expect(caveats[1]).toContain( + "1 condition mixes configuration fingerprints", + ); + expect(caveats[2]).toContain("With 1 app per condition"); + }); +}); + +describe("resolveExperimentTab", () => { + it("accepts the sections and the names older links used", () => { + expect(resolveExperimentTab("results")).toBe("results"); + expect(resolveExperimentTab("overview")).toBe("summary"); + expect(resolveExperimentTab("design")).toBe("method"); + expect(resolveExperimentTab("runs")).toBe("data"); + expect(resolveExperimentTab("automation")).toBe("operations"); + expect(resolveExperimentTab("bogus")).toBeNull(); + expect(resolveExperimentTab(null)).toBeNull(); + }); +}); diff --git a/frontend/src/lib/components/experiments/study/findings.ts b/frontend/src/lib/components/experiments/study/findings.ts new file mode 100644 index 00000000..a4b88595 --- /dev/null +++ b/frontend/src/lib/components/experiments/study/findings.ts @@ -0,0 +1,305 @@ +/** + * The experiment page's research summary, derived from the detail and the + * `GET /results/` statistics: the design as factor counts, the short + * identifier, and the "key findings" bullets of the Abstract (leader, effects + * against the baseline, caveats). Pure functions — the Svelte side only + * renders the parts. + */ +import type { + ExperimentDetail, + ExperimentResults, + LeaderboardRow, + PairedRow, +} from "$lib/api/types/experiments"; +import { + MIN_APPS_FOR_CI, + conditionName, + formatDeltaPp, + formatP, + formatPercent, + plural, +} from "../format"; + +/** Significance level the page reports against (Holm-adjusted p). */ +export const ALPHA = 0.05; + +/** A finding's text: plain strings with emphasised (bold) parts. */ +export type FindingPart = string | { strong: string }; + +export interface Finding { + kind: "leader" | "effect" | "null" | "interim" | "caveat"; + parts: FindingPart[]; +} + +/** The design as factor counts: conditions = models × variants (+ ablations). */ +export interface DesignShape { + conditions: number; + models: number; + /** Harness presets (agent) or prompt profiles (scaffolding) across the crossed conditions. */ + variants: number; + /** Conditions with parameter overrides (ablations). */ + ablations: number; + apps: number; + repeats: number; + cells: number; +} + +type DesignInput = Pick< + ExperimentDetail, + "conditions" | "app_requirement_ids" | "repeats" | "total_cells" +>; + +function hasOverrides(params: object | null | undefined): boolean { + return !!params && Object.keys(params).length > 0; +} + +/** The variant a condition runs: its harness preset / harness, or its profile. */ +export function variantKey(c: DesignInput["conditions"][number]): string { + if (c.harness_preset != null) return `preset:${c.harness_preset}`; + if (c.harness) return `harness:${c.harness}`; + if (c.profile != null) return `profile:${c.profile}`; + return "default"; +} + +export function designShape(e: DesignInput): DesignShape { + const crossed = e.conditions.filter((c) => !hasOverrides(c.param_overrides)); + const apps = e.app_requirement_ids.length; + const conditions = e.conditions.length; + return { + conditions, + models: new Set(e.conditions.map((c) => c.model)).size, + variants: new Set((crossed.length ? crossed : e.conditions).map(variantKey)) + .size, + ablations: conditions - crossed.length, + apps, + repeats: e.repeats, + cells: Math.max(e.total_cells ?? 0, conditions * apps * e.repeats), + }; +} + +/** "EXP-955A5BD5": the first block of the UUID, as a short accession number. */ +export function shortExperimentId(id: string): string { + return `EXP-${id.split("-")[0].toUpperCase()}`; +} + +function ciText(row: LeaderboardRow): string { + const ci = row.n_apps >= MIN_APPS_FOR_CI ? row.pass_rate_ci95 : null; + if (ci) return ` (95% CI ${formatPercent(ci[0])}–${formatPercent(ci[1])})`; + return ` (${plural(row.n_apps, "app")} × ${plural(row.repeats_per_app, "repeat")}, no CI)`; +} + +function deltaCiText(row: PairedRow): string { + const ci = row.n_pairs >= MIN_APPS_FOR_CI ? row.ci95 : null; + return ci + ? `95% CI ${formatDeltaPp(ci[0])} to ${formatDeltaPp(ci[1])}` + : `${plural(row.n_pairs, "paired app")}`; +} + +function missingTotal(row: LeaderboardRow): number { + const m = row.n_missing; + return ( + m.failed + + m.ungraded + + m.excluded_invalid + + m.excluded_truncated + + m.not_common + ); +} + +/** Highest (and, with ≥ 3 conditions, lowest) mean pass rate. */ +function leaderFindings(rows: LeaderboardRow[]): Finding[] { + const scored = rows + .filter((r) => r.pass_rate_mean != null) + .sort((a, b) => a.rank - b.rank); + if (scored.length === 0) return []; + const top = scored[0]; + const best = top.pass_rate_mean as number; + if (scored.length === 1) { + return [ + { + kind: "leader", + parts: [ + { strong: top.label }, + ": mean pass rate ", + { strong: formatPercent(best) }, + `${ciText(top)}.`, + ], + }, + ]; + } + const tied = scored.filter( + (r) => Math.abs((r.pass_rate_mean as number) - best) < 1e-9, + ); + const parts: FindingPart[] = + tied.length > 1 + ? [ + `${tied.length} conditions share the highest mean pass rate, `, + { strong: formatPercent(best) }, + `: ${tied.map((r) => r.label).join(", ")}.`, + ] + : [ + "Highest mean pass rate: ", + { strong: top.label }, + ", ", + { strong: formatPercent(best) }, + `${ciText(top)}.`, + ]; + const last = scored[scored.length - 1]; + if (scored.length >= 3 && tied.length < scored.length) { + parts.push( + ` Lowest: ${last.label}, ${formatPercent(last.pass_rate_mean)}.`, + ); + } + return [{ kind: "leader", parts }]; +} + +/** Differences against the baseline (paired by app, Holm-adjusted). */ +function effectFindings( + results: ExperimentResults, + labelOf: (id: number) => string, +): Finding[] { + const baselineId = results.paired.baseline_condition_id; + const rows = results.paired.rows.filter( + (r) => r.condition_id !== baselineId && r.mean_delta != null, + ); + if (baselineId == null || rows.length === 0) return []; + const baseline = labelOf(baselineId); + const largest = [...rows].sort( + (a, b) => + Math.abs(b.mean_delta as number) - Math.abs(a.mean_delta as number), + )[0]; + const largestParts = (prefix: string, withP: boolean): FindingPart[] => [ + prefix, + { strong: labelOf(largest.condition_id) }, + " ", + { strong: formatDeltaPp(largest.mean_delta) }, + ` (${deltaCiText(largest)}${withP && largest.p_holm != null ? `, p = ${formatP(largest.p_holm)}` : ""}).`, + ]; + + if (results.interim) { + return [ + { + kind: "interim", + parts: [ + "Interim: conditions are at unequal progress, so tests against ", + { strong: baseline }, + " are withheld. ", + ...largestParts("Largest observed difference so far: ", false), + ], + }, + ]; + } + if (rows.every((r) => r.p_holm == null)) { + return [ + { + kind: "null", + parts: [ + "Too few apps graded alongside ", + { strong: baseline }, + " to test differences. ", + ...largestParts("Largest observed difference: ", false), + ], + }, + ]; + } + const significant = rows.filter((r) => r.p_holm != null && r.p_holm < ALPHA); + if (significant.length === 0) { + return [ + { + kind: "null", + parts: [ + "No condition differs from the baseline ", + { strong: baseline }, + ` at α = ${ALPHA} (Holm). `, + ...largestParts("Largest observed difference: ", true), + ], + }, + ]; + } + const strongest = [...significant].sort( + (a, b) => + Math.abs(b.mean_delta as number) - Math.abs(a.mean_delta as number), + )[0]; + return [ + { + kind: "effect", + parts: [ + `${significant.length} of ${plural(rows.length, "condition")} differ from the baseline `, + { strong: baseline }, + ` (Holm-adjusted p < ${ALPHA}). Largest: `, + { strong: labelOf(strongest.condition_id) }, + " ", + { strong: formatDeltaPp(strongest.mean_delta) }, + ` (${deltaCiText(strongest)}, p = ${formatP(strongest.p_holm)}).`, + ], + }, + ]; +} + +/** What limits the estimates: missing grades, mixed fingerprints, too few apps. */ +function caveatFindings(results: ExperimentResults): Finding[] { + const rows = results.leaderboard; + const out: Finding[] = []; + const cells = rows.reduce((sum, r) => sum + r.n_cells, 0); + const missing = rows.reduce((sum, r) => sum + missingTotal(r), 0); + if (missing > 0) { + out.push({ + kind: "caveat", + parts: [ + { strong: `${missing} of ${plural(cells, "cell")}` }, + " have no usable grade (failed, ungraded, excluded or outside the common cells) and are left out of the estimates.", + ], + }); + } + const mixed = rows.filter((r) => r.mixed_fingerprints).length; + if (mixed > 0) { + out.push({ + kind: "caveat", + parts: [ + { strong: plural(mixed, "condition") }, + ` ${mixed === 1 ? "mixes" : "mix"} configuration fingerprints: a model or harness changed while it ran.`, + ], + }); + } + const maxApps = Math.max(0, ...rows.map((r) => r.n_apps)); + if (rows.length > 0 && maxApps < MIN_APPS_FOR_CI) { + out.push({ + kind: "caveat", + parts: [ + `With ${plural(maxApps, "app")} per condition, confidence intervals are not shown (they need at least ${MIN_APPS_FOR_CI} apps).`, + ], + }); + } + if (results.baseline_source === "post_hoc") { + out.push({ + kind: "caveat", + parts: [ + "The baseline was chosen after the fact: comparisons are exploratory.", + ], + }); + } + return out; +} + +/** + * Key findings for the Abstract: the leader, the effects against the + * baseline, then caveats. Empty while nothing is graded. + */ +export function keyFindings( + results: ExperimentResults, + labelOf: (id: number) => string, +): Finding[] { + if (!results.leaderboard.some((r) => r.n_graded > 0)) return []; + return [ + ...leaderFindings(results.leaderboard), + ...effectFindings(results, labelOf), + ...caveatFindings(results), + ]; +} + +/** Condition id → display name (the detail's labels). */ +export function conditionLabels( + e: Pick, +): Map { + return new Map(e.conditions.map((c) => [c.id, conditionName(c)])); +} diff --git a/frontend/src/lib/components/experiments/study/results.svelte.ts b/frontend/src/lib/components/experiments/study/results.svelte.ts new file mode 100644 index 00000000..45faa82b --- /dev/null +++ b/frontend/src/lib/components/experiments/study/results.svelte.ts @@ -0,0 +1,69 @@ +/** + * ResultsFeed — one experiment's `GET /results/` for a tab: the latest + * request wins (earlier ones are aborted), failures keep the previous + * results on screen, and `tick()` refetches at most every `tickMs` while the + * experiment runs. The Abstract (default analysis) and Results (the URL's + * analysis) each own one. + */ +import { + getResults, + type ExperimentResults, + type ResultsQuery, +} from "$lib/api/experiments"; +import { formatApiError, isAbortError } from "$lib/api/core"; + +export class ResultsFeed { + results = $state.raw(null); + loading = $state(true); + error = $state(""); + + private experimentId: string; + private tickMs: number; + private controller: AbortController | null = null; + private lastFetch = 0; + private query: ResultsQuery = {}; + + constructor(experimentId: string, tickMs = 30_000) { + this.experimentId = experimentId; + this.tickMs = tickMs; + } + + async load(query: ResultsQuery = this.query): Promise { + this.query = query; + this.controller?.abort(); + const mine = new AbortController(); + this.controller = mine; + this.lastFetch = Date.now(); + try { + const next = await getResults(this.experimentId, query, { + signal: mine.signal, + }); + if (mine.signal.aborted) return; + this.results = next; + this.error = ""; + } catch (e) { + if (isAbortError(e) || mine.signal.aborted) return; + this.error = formatApiError(e, "Could not compute the results."); + } finally { + if (!mine.signal.aborted) this.loading = false; + } + } + + /** A live tick: refetch when the last fetch is older than `tickMs`. */ + tick(): void { + if (!this.results) return; + if (Date.now() - this.lastFetch >= this.tickMs) void this.load(); + } + + destroy(): void { + this.controller?.abort(); + } +} + +/** The pre-registered analysis: what the Results tab shows with no URL options. */ +export const DEFAULT_RESULTS_QUERY: ResultsQuery = { + common_cells: true, + exclude_invalid: false, + exclude_truncated: true, + fingerprint: "all", +}; diff --git a/frontend/src/lib/components/experiments/study/tabs.ts b/frontend/src/lib/components/experiments/study/tabs.ts new file mode 100644 index 00000000..4bdea3ac --- /dev/null +++ b/frontend/src/lib/components/experiments/study/tabs.ts @@ -0,0 +1,36 @@ +/** + * The experiment page's sections (`?tab=`), in paper order, and the names + * older links used before the study layout (Overview, Design, Runs, + * Automation), so bookmarks and links from other pages keep landing. + */ +export type ExperimentTab = + | "summary" + | "method" + | "results" + | "data" + | "operations"; + +export const EXPERIMENT_TABS: { value: ExperimentTab; label: string }[] = [ + { value: "summary", label: "Abstract" }, + { value: "method", label: "Method" }, + { value: "results", label: "Results" }, + { value: "data", label: "Data" }, + { value: "operations", label: "Operations" }, +]; + +const ALIASES: Record = { + abstract: "summary", + overview: "summary", + design: "method", + runs: "data", + automation: "operations", +}; + +/** `?tab=` → a section; unknown values → null (the page's default). */ +export function resolveExperimentTab( + raw: string | null | undefined, +): ExperimentTab | null { + if (!raw) return null; + if (EXPERIMENT_TABS.some((t) => t.value === raw)) return raw as ExperimentTab; + return ALIASES[raw] ?? null; +} diff --git a/frontend/src/lib/components/experiments/tabs/AutomationTab.svelte b/frontend/src/lib/components/experiments/tabs/AutomationTab.svelte deleted file mode 100644 index 113eac26..00000000 --- a/frontend/src/lib/components/experiments/tabs/AutomationTab.svelte +++ /dev/null @@ -1,92 +0,0 @@ - - -
- {#if error} - - {/if} - - {#if postRun} - - {:else if !error} - - {/if} -
diff --git a/frontend/src/lib/components/experiments/tabs/DataTab.svelte b/frontend/src/lib/components/experiments/tabs/DataTab.svelte new file mode 100644 index 00000000..cd23b661 --- /dev/null +++ b/frontend/src/lib/components/experiments/tabs/DataTab.svelte @@ -0,0 +1,354 @@ + + +
+ + + {#snippet controls()} + + {#if experiment.status !== 'draft' && experiment.status !== 'archived' && failedCells > 0} + + {/if} + {/snippet} +
+
+
+ + setFilter(() => (conditionFilter = v))} /> +
+
+ + setFilter(() => (endpointFilter = v))} /> +
+
+ + setFilter(() => (appFilter = v))} /> +
+
+ + setFilter(() => (statusFilter = v))} /> +
+
+
+ + +
+
+
+ {#if error} + + {/if} + {#if loading && !data} + + {:else if data && data.items.length === 0} + + {:else if data} + + (pageNum = p)} + /> + {/if} +
+ {#snippet caption()} + Table D1. + {currentOnly ? 'Current attempt of each cell' : 'Every attempt'}{retriedOnly ? ', retries only' : ''}. Trial = repeat; seed = the seed the + run used ({experiment.base_seed != null ? 'shared across conditions per repeat' : 'drawn per run'}). + {/snippet} + + + + + {#if batches === null} + + {:else if batches.length === 0} +

Not launched yet.

+ {:else} +
+ + + + + + + + + + + + {#each batches as batch (batch.id)} + {@const batchCount = batchCounts(batch)} + {@const kind = (batch as GenerationBatchSummary & { kind?: string }).kind ?? ''} + + + + + + + + {/each} + +
BatchKindQueuedProgressFinished
{batch.name}{kind ? (KIND_LABELS[kind] ?? kind) : '—'}{formatDateTime(batch.created_at)}{finishedCount(batchCount)}/{batch.total_jobs}
+
+ {/if} + + + +
+ + +
+

{plural(experiment.total_cells, 'cell')} in the design; the statistics in §3 can also be written up as a report from the Results tab.

+ +
+ + diff --git a/frontend/src/lib/components/experiments/tabs/DesignTab.svelte b/frontend/src/lib/components/experiments/tabs/DesignTab.svelte deleted file mode 100644 index 02d94f0f..00000000 --- a/frontend/src/lib/components/experiments/tabs/DesignTab.svelte +++ /dev/null @@ -1,291 +0,0 @@ - - -
- - -
- Design - Shared by every condition; a condition only varies what its row below says. -
- {#if isDraft} - - {/if} - - - - {#if experiment.hypothesis} -

“{experiment.hypothesis}”

- {/if} - - - - - - Conditions ({experiment.conditions.length}) - Grouped by endpoint. The same weights on two endpoints are two conditions. - - - {#each groups as [key, conditions] (key)} -
-

{key === 'openrouter' ? 'OpenRouter' : key} · {conditions.length}

-
    - {#each conditions as c (c.id)} - {@const overrides = summarizeLLMParams(c.param_overrides)} -
  • - {conditionName(c)} - {#if c.id === experiment.baseline_condition_id} - baseline - {/if} - {#if overrides} - ablation - {overrides} - {/if} - - {#if c.served_model_name}{c.served_model_name}{/if} - {#if c.digest} - - {shortDigest(c.digest)}{#if c.pinned_digest && c.pinned_digest !== c.digest} ≠ {shortDigest(c.pinned_digest)}{/if} - - {/if} - {#if numCtx.get(c.model) != null}ctx {formatCtx(numCtx.get(c.model))}{/if} - {#if c.bundle_slug}{c.bundle_slug}@v{c.bundle_version}{/if} - -
    • - {/each} -
-
- {/each} - - - - - - Comparability - What each condition actually runs with; highlighted columns differ between conditions. - - - {#if previewError} - - {:else if previewLoading && !preview} - - {:else if preview?.comparability} - - {:else} -

No comparability data for this experiment.

- {/if} - - - - - - Execution - - {#if isDraft} - Set in the composer. The run order is frozen at launch. - {:else} - The run order is frozen. Priority, limits and preloading can change while it runs; repeats can only grow. - {/if} - - - - {#if preview?.schedule} -
-
-
Order per endpoint
-
- {Object.entries(preview.schedule.policy_effective_by_endpoint).map(([s, p]) => `${s}: ${policyLabel(p)}`).join(' · ') || - policyLabel(experiment.scheduling_policy)} -
-
-
-
Order seed
-
{preview.schedule.order_seed ?? '—'}
-
-
-
Resident models (K)
-
- {preview.endpoints.filter((e) => e.resident_limit != null).map((e) => `${e.slug}: ${e.resident_limit}`).join(' · ') || '—'} -
-
-
- {/if} - {#if !isDraft && !isTerminal(experiment.status)} -
-
- - -
- - - -
- -
- -
- {/if} - - -
diff --git a/frontend/src/lib/components/experiments/tabs/MethodTab.svelte b/frontend/src/lib/components/experiments/tabs/MethodTab.svelte new file mode 100644 index 00000000..814f5bc2 --- /dev/null +++ b/frontend/src/lib/components/experiments/tabs/MethodTab.svelte @@ -0,0 +1,335 @@ + + +{#snippet definitionList(items: { label: string; value: string; mono?: boolean }[])} +
+ {#each items as item (item.label)} +
+
{item.label}
+
{item.value}
+
+ {/each} +
+{/snippet} + +
+ + {#snippet aside()} + {#if isDraft}{/if} + {/snippet} + +
+ + + + + + + + + + + {#each factors as factor (factor.name)} + + + + + + + {/each} + +
FactorRoleLevelsn
{factor.name}{factor.role} + {factor.levels.join(', ')} + {#if factor.note}{factor.note}{/if} + {factor.count}
+
+ {#snippet caption()} + Table M1. + Conditions are the crossed levels of the manipulated factors{shape.ablations ? ' plus the ablations' : ''}. The pre-registered baseline is + {#if experiment.baseline_condition_id != null} + {conditionName(experiment.conditions.find((c) => c.id === experiment.baseline_condition_id) ?? { id: experiment.baseline_condition_id, label: '' })}; every other condition is compared with it, paired by app. + {:else} + not set. + {/if} + {/snippet} + + + + + +
+ + + + + + + + + + + + + + {#each conditions as c (c.id)} + {@const isBaseline = c.id === experiment.baseline_condition_id} + {@const overrides = summarizeLLMParams(c.param_overrides)} + + + + + + + + + + {/each} + +
ConditionModelEndpoint{isAgent ? 'Harness preset' : 'Profile'}OverridesDigestContext
+ + {conditionName(c)} + {#if isBaseline}baseline{/if} + {#if overrides}ablation{/if} + + + {modelLabel(c)} + {#if c.served_model_name}{c.served_model_name}{/if} + {endpointLabel(c)}{variantLabel(c)}{#if overrides}{overrides}{:else}—{/if} + {#if c.digest} + {shortDigest(c.digest)}{#if c.pinned_digest && c.pinned_digest !== c.digest} ≠ {shortDigest(c.pinned_digest)}{/if} + {:else} + — + {/if} + {formatCtx(numCtx.get(c.model))}
+
+ {#snippet caption()} + Table M2. + {plural(experiment.conditions.length, 'condition')} on {plural(new Set(experiment.conditions.map(endpointLabel)).size, 'endpoint')}. Digest: the + first 8 characters of the weights’ sha256 (local endpoints); context: the effective num_ctx. + {/snippet} + + + + + {@render definitionList(constants)} + + + + {#if previewError} + + {:else if previewLoading && !preview} + + {:else if preview?.comparability} + + + + {:else} +

No comparability data for this experiment.

+ {/if} + + + + {@render definitionList(procedure)} + +
diff --git a/frontend/src/lib/components/experiments/tabs/OperationsTab.svelte b/frontend/src/lib/components/experiments/tabs/OperationsTab.svelte new file mode 100644 index 00000000..56ba9a3f --- /dev/null +++ b/frontend/src/lib/components/experiments/tabs/OperationsTab.svelte @@ -0,0 +1,327 @@ + + +
+ {#if live} + + {#if !status} + {#if live.loading} + + {:else} + void live?.resync()} /> + {/if} + {:else} +
+ {#each lanes as lane, i (lane.label)} +
0 && 'border-border/60 sm:border-l', i % 2 === 1 && 'max-sm:border-l', i > 1 && 'max-sm:border-t')} title={lane.hint}> +
{lane.label}
+
+ 0 && lane.used >= lane.total ? 'text-warning-fg' : 'text-foreground'}>{lane.used}/{lane.total} +
+
+ {/each} +
+ {#if status.endpoints.length > 0} +
2 && 'md:grid-cols-2 xl:grid-cols-3')}> + {#each status.endpoints as endpoint (endpoint.id)} + + {/each} +
+ {/if} + {/if} + + + {#if showActivity} + +
+ (id != null ? (conditionLabel.get(id) ?? `Condition #${id}`) : '—')} + {appName} + /> +
+ + {/if} + {/if} + + + {#if editable} +
+
+
+ + +
+ + + +
+
+ + +
+
+ {:else} +
+ {#each readonlyLimits as item, i (item.label)} +
0 && 'border-border/60 sm:border-l', i % 2 === 1 && 'max-sm:border-l', i > 1 && 'max-sm:border-t')}> +
{item.label}
+
{item.value}
+
+ {/each} +
+ {/if} + + + + {#if automationError} + + {/if} + + {#if postRun} + + {:else if !automationError} + + {/if} + +
diff --git a/frontend/src/lib/components/experiments/tabs/OverviewTab.svelte b/frontend/src/lib/components/experiments/tabs/OverviewTab.svelte deleted file mode 100644 index 457a3c00..00000000 --- a/frontend/src/lib/components/experiments/tabs/OverviewTab.svelte +++ /dev/null @@ -1,219 +0,0 @@ - - -{#if !status} - {#if live.loading} - - {:else} - - {#snippet action()} - - {/snippet} - - {/if} -{:else} -
-
-
- Done - {status.cells.done}/{status.cells.total} - - {status.cells.in_flight} in flight{status.cells.failed_final ? ` · ${status.cells.failed_final} failed` : ''}{status.cells.retried - ? ` · ${status.cells.retried} retried` - : ''} - -
-
- Throughput - {status.throughput.per_hour_ewma.toFixed(1)} - cells/h · {status.throughput.completed_last_15m} in 15 min -
-
- ETA - {formatEta(status.eta_seconds)} - slowest endpoint + evaluation -
-
- Tokens/s - {tokensPerS ? tokensPerS.toFixed(0) : '—'} - all endpoints, last 5 min -
-
- Cost - {costUsd > 0 ? formatUsd(costUsd) : '—'} - OpenRouter only -
-
- Bottleneck - {bottleneck?.label ?? 'None'} - {bottleneck?.title || 'Nothing is waiting on capacity.'} -
-
- -
- -
-

- {#each lanes as lane (lane.label)} - {lane.label} 0 && lane.used >= lane.total ? 'text-warning-fg' : 'text-foreground')}>{lane.used}/{lane.total} - {/each} -

-

- - {connectionText.text} -

-
-
- - {#if status.endpoints.length > 0} -
-

Endpoints

-
- {#each status.endpoints as endpoint (endpoint.id)} - - {/each} -
-
- {/if} - - - -
- Cells - One square per run (condition × app × repeat). Open a square for its run. -
-
- (groupBy = v as GridGroupBy)} /> -
- - (sort = v as GridSort)} /> -
-
- - - {#if grid && grid.conditions.length > 0} - - {:else} - - {/if} - - - - - - Activity - The latest run transitions while this page is open. - - - (id != null ? (conditionLabel.get(id) ?? `Condition #${id}`) : '—')} - {appName} - /> - - -
-{/if} diff --git a/frontend/src/lib/components/experiments/tabs/ResultsTab.svelte b/frontend/src/lib/components/experiments/tabs/ResultsTab.svelte index ea7ea68c..ba3b472b 100644 --- a/frontend/src/lib/components/experiments/tabs/ResultsTab.svelte +++ b/frontend/src/lib/components/experiments/tabs/ResultsTab.svelte @@ -1,20 +1,20 @@ -
- - +
+
+
+
+

Analysis

+

+ {#if customised || results?.baseline_source === 'post_hoc'} + + — differs from the pre-registered analysis: + {:else} + Pre-registered + — + {/if} + {specification}. +

+
+
+ {#if customised} + + {/if} + +
+
+
setParam('baseline', v === String(experiment.baseline_condition_id) ? null : v)} /> - {#if results?.baseline_source === 'post_hoc'} -

Post hoc — exploratory. The pre-registered baseline is the default.

- {/if}
-
+
{#each toggles as t (t.key)} -
- - - +
+
- {#if error} - + {#if feed.error} + feed.load()} /> {/if} - {#if loading && !results} + {#if feed.loading && !results} {:else if results} {#if results.interim} - - - Interim: conditions are at unequal progress — p-values are hidden until the experiment finishes. - +

+ Interim + Conditions are at unequal progress: estimates can still move and p-values are withheld until the experiment finishes. +

{/if} + {#if onlyOutsideCommon} {:else} -
-

Leaderboard

- -
+ + + + {#snippet caption()} + Table 1. + Pass rate: mean of app-level pass rates, with a cluster-bootstrap 95% interval over apps (from {MIN_APPS_FOR_CI} apps; bar: interval, tick: + mean). Δ: paired difference from the baseline. Missing: cells without a usable grade (hover for why). Median time compares only + within one endpoint. Validity and termination: share of runs per outcome. + {/snippet} + + -
-

Against the baseline

- conditionLabel.get(id) ?? `Condition #${id}`} /> -
+ + + + {#snippet caption()} + Figure 3. + Mean app-level difference in pass rate from the baseline (square), in percentage points, with the paired cluster-bootstrap 95% + interval (whiskers; from {MIN_APPS_FOR_CI} paired apps). Filled: the interval excludes 0; hollow: it includes 0. W/L/T: apps won, lost + and tied; n: apps graded for both. p: Wilcoxon signed-rank, Holm-adjusted across conditions{results.interim ? ' (withheld: interim)' : ''}. + {/snippet} + + {#if matrix} -
-
-

Per app

- -
- -
+ + + {#snippet controls()} + + {/snippet} +
+ +
+ {#snippet caption()} + Table 2. + Each cell aggregates the repeats of one condition on one app: k/n cells passed, mean pass rate, one dot per repeat. Open a cell for its runs. + {/snippet} + + {/if} {/if} - {#if results.notes.length > 0} -
    - {#each results.notes as note (note)} -
    • - {/each} -
+ {#if notes.length > 0} +
+ +
    + {#each notes as note, i (note)} +
  1. + {String.fromCharCode(97 + (i % 26))} + {note} +
    2. + {/each} +
+
{/if} {/if}
diff --git a/frontend/src/lib/components/experiments/tabs/RunsTab.svelte b/frontend/src/lib/components/experiments/tabs/RunsTab.svelte deleted file mode 100644 index 4a2d821e..00000000 --- a/frontend/src/lib/components/experiments/tabs/RunsTab.svelte +++ /dev/null @@ -1,300 +0,0 @@ - - -
- - -
- Runs - Every run in launch order. Open one for its logs, code, evaluation and retry options. -
-
- - {#if experiment.status !== 'draft' && experiment.status !== 'archived'} - - {/if} -
- - -
-
- - setFilter(() => (conditionFilter = v))} /> -
-
- - setFilter(() => (endpointFilter = v))} /> -
-
- - setFilter(() => (appFilter = v))} /> -
-
- - setFilter(() => (statusFilter = v))} /> -
-
-
- - -
- - {#if error} - - {/if} - {#if loading && !data} - - {:else if data && data.items.length === 0} - - {:else if data} - - (pageNum = p)} - /> - {/if} - - - - - - Launch history - Every launch, top-up and scheduled run as a batch. - - - {#if batches === null} - - {:else if batches.length === 0} -

Not launched yet.

- {:else} -
    - {#each batches as batch (batch.id)} - {@const counts = batchCounts(batch)} - {@const kind = (batch as GenerationBatchSummary & { kind?: string }).kind ?? ''} -
  • - {batch.name} - {#if kind}{KIND_LABELS[kind] ?? kind}{/if} - {formatDateTime(batch.created_at)} - - {finishedCount(counts)}/{batch.total_jobs} -
    • - {/each} -
- {/if} - - -
- - diff --git a/frontend/src/lib/components/experiments/tabs/SummaryTab.svelte b/frontend/src/lib/components/experiments/tabs/SummaryTab.svelte new file mode 100644 index 00000000..c70ae3c2 --- /dev/null +++ b/frontend/src/lib/components/experiments/tabs/SummaryTab.svelte @@ -0,0 +1,288 @@ + + +{#snippet findingText(parts: import('../study/findings').FindingPart[])} + {#each parts as part, i (i)}{#if typeof part === 'string'}{part}{:else}{part.strong}{/if}{/each} +{/snippet} + + +
+
+
+
+ {#if experiment.description} +

{experiment.description}

+ {/if} +

+ This {experiment.mode === 'agent' ? 'agent' : 'template-scaffolding'} experiment compares + {plural(shape.conditions, 'condition')} + {#if shape.conditions > 1} + ({plural(shape.models, 'model')} × {plural(shape.variants, variantNoun)}{shape.ablations ? `, plus ${plural(shape.ablations, 'ablation')}` : ''}) + {/if} + on {plural(shape.apps, 'app spec')}, each run {plural(shape.repeats, 'time')} + ({plural(shape.cells, 'cell')}). + {#if baseline && shape.conditions > 1} + Every condition is compared with the pre-registered baseline, {conditionName(baseline)}, paired by app, + {:else} + Runs are scored + {/if} + on the share of {experiment.mode === 'agent' ? 'hidden conformance checks' : 'evaluation checks'} a generated app passes, averaged per app + with cluster-bootstrap 95% intervals. +

+

+ {#if isDraft} + Not launched. The design can still change; launching freezes it together with the baseline and the seeds. + {:else if experiment.status === 'running'} + Data collection in progress: {done.toLocaleString()} of {plural(shape.cells, 'cell')} finished{failedFinal ? `, ${failedFinal} failed` : ''}{status?.eta_seconds != null ? `, about ${formatEta(status.eta_seconds)} left` : ''}. Figures are interim. + {:else if experiment.status === 'paused'} + Data collection paused ({pauseReasonLabel(experiment.pause_reason) || 'paused'}) at {done.toLocaleString()} of {plural(shape.cells, 'cell')}. + {:else if experiment.status === 'cancelled'} + Cancelled at {done.toLocaleString()} of {plural(shape.cells, 'cell')}{failedFinal ? ` (${failedFinal} failed or cancelled)` : ''}. + {:else} + Data collection finished: {done.toLocaleString()} of {plural(shape.cells, 'cell')}{failedFinal ? `, ${failedFinal} failed` : ', none failed'}{(costUsd ?? 0) > 0 ? `, ${formatUsd(costUsd ?? 0)} spent` : ''}. + {/if} +

+ + {#if !isDraft} +
+ + {#if feed.loading && !results} + + {:else if feed.error && !results} + feed.load(DEFAULT_RESULTS_QUERY)} /> + {:else if claims.length > 0} +
    + {#each claims as finding, i (i)} +
  1. + {i + 1}. + {@render findingText(finding.parts)} +
    2. + {/each} +
+ {#if caveats.length > 0} +
+

Limitations

+
    + {#each caveats as finding, i (i)} +
  • + + {@render findingText(finding.parts)} +
    • + {/each} +
+
+ {/if} + {:else if onlyOutsideCommon} +

+ Runs are graded, but no app is graded for every condition yet, so the paired comparison has nothing to stand on. + Compare all graded cells. +

+ {:else} +

+ {experiment.mode === 'scaffolding' && !experiment.post_run?.evaluate + ? 'Nothing is graded: turn on Evaluate under Operations → Automation.' + : 'Nothing graded yet — findings appear as runs are evaluated.'} +

+ {/if} +
+ {/if} +
+
+ + {#if graded && results} + + + {#snippet caption()} + Figure 1. + Mean of app-level pass rates per condition (dot), ranked, with the cluster-bootstrap 95% interval over apps (whiskers; shown from 3 apps). + {#if baseline}The hollow diamond and the vertical hairline mark the baseline.{/if} + k/n: cells that passed every check / graded cells. Common cells only, truncated runs excluded{results.interim ? '; interim' : ''}. + Full results + {/snippet} + + {:else if !isDraft && !feed.loading && !feed.error && !onlyOutsideCommon && experiment.status !== 'running'} + + {/if} + + {#if live} + + + {#snippet controls()} + (groupBy = v as GridGroupBy)} /> +
+ + (sort = v as GridSort)} /> +
+ {/snippet} + {#if live.grid && live.grid.conditions.length > 0} + + {:else if live.loading} + + {:else} + + {/if} + {#snippet caption()} + Figure 2. + Every cell of the design: one row per condition, one column per app, one square per repeat. A solid square has finished; a + ring is waiting (dashed: for a model to load). Open a square for its run, code and evaluation. + {/snippet} + + {/if} +
+ +
+ + + {#if isDraft} +
+
+

Before launch

+

Launch estimate

+
+
+ {#if previewError} + + {:else if !preview} + + {:else} +
+
Cells
+
{preview.total_jobs.toLocaleString()}
+
Wall clock
+
{formatHours(preview.estimate.total_hours)}
+
Spend
+
+ {preview.cost.openrouter_upper_bound_usd > 0 ? `≤ ${formatUsd(preview.cost.openrouter_upper_bound_usd)}` : '$0 (local)'} +
+
Endpoints
+
+ {preview.endpoints.map((e) => `${e.name || e.slug}${e.online ? '' : ' (offline)'}`).join(', ') || '—'} +
+
+ {#if blocking.length > 0} +
    + {#each blocking as issue (issue.code + issue.detail)}
  • {issue.detail}
    • {/each} +
+ {/if} + {/if} +

+ A launch freezes the conditions, the baseline, the apps and the seeds, so the comparison stays as registered. Clone it to change any of them later. +

+
+
+ {/if} +
+
+ diff --git a/frontend/src/lib/components/generator/StackSelect.svelte b/frontend/src/lib/components/generator/StackSelect.svelte index 204ea20a..e12a31e5 100644 --- a/frontend/src/lib/components/generator/StackSelect.svelte +++ b/frontend/src/lib/components/generator/StackSelect.svelte @@ -94,7 +94,7 @@ {#snippet stackLine(stack: Stack)} {stack.name || stack.slug} - {stack.slug} + {#if stack.name && stack.name !== stack.slug}{stack.slug}{/if} {#if !stack.has_frontend} backend only {/if} diff --git a/frontend/src/lib/components/generator/batches/BatchJobsTable.svelte b/frontend/src/lib/components/generator/batches/BatchJobsTable.svelte index 7ef74748..621417ee 100644 --- a/frontend/src/lib/components/generator/batches/BatchJobsTable.svelte +++ b/frontend/src/lib/components/generator/batches/BatchJobsTable.svelte @@ -121,8 +121,11 @@ {#if job.model_name} - {shortModelName(job.model_name)} - {#if provider}{provider}{/if} + + + {shortModelName(job.model_name)} + {#if provider}{provider}{/if} + {:else} — {/if} @@ -165,10 +168,10 @@ {#if job.cost_usd != null}{formatUsd(job.cost_usd)}{:else}—{/if} - + {#if job.duration_seconds != null}{formatDuration(job.duration_seconds)}{:else}—{/if} - + {#if job.experiment_seed != null}{job.experiment_seed}{:else}—{/if} diff --git a/frontend/src/lib/components/generator/batches/ResultsMatrix.svelte b/frontend/src/lib/components/generator/batches/ResultsMatrix.svelte index b016855e..7184f646 100644 --- a/frontend/src/lib/components/generator/batches/ResultsMatrix.svelte +++ b/frontend/src/lib/components/generator/batches/ResultsMatrix.svelte @@ -167,13 +167,13 @@ {#snippet axisLabel(axis: Axis, header: boolean)} {#if axis.kind === 'task'} - {axis.item.label} + {axis.item.label} {#if axis.item.slug === null} free-form {/if} {:else} {@const col = axis.item} - {shortModelName(col.model)} + {shortModelName(col.model)} {#if col.variant} {col.variant} {/if} diff --git a/frontend/src/lib/components/generator/catalog.ts b/frontend/src/lib/components/generator/catalog.ts index d754e691..755858bc 100644 --- a/frontend/src/lib/components/generator/catalog.ts +++ b/frontend/src/lib/components/generator/catalog.ts @@ -307,6 +307,10 @@ export function filterByEndpointScope( * vendor as before. Models inside a group are sorted by name. Inputs are not * mutated. */ +/** Group of OpenRouter "~vendor/model-latest" aliases (they point at whatever version is newest). */ +export const ALIAS_GROUP_KEY = "provider:~aliases"; +export const ALIAS_GROUP_LABEL = "Latest aliases (version can change)"; + export function groupModels( models: LLMModelSummary[], groupBy: ModelGroupBy, @@ -315,7 +319,13 @@ export function groupModels( const map = new Map(); for (const m of models) { const local = groupBy === "endpoint" && isLocalModel(m) && m.endpoint; - const key = local ? `endpoint:${m.endpoint!.id}` : `provider:${m.provider}`; + // OpenRouter's floating "~vendor/…-latest" aliases share one group. + const alias = !local && m.provider.startsWith("~"); + const key = local + ? `endpoint:${m.endpoint!.id}` + : alias + ? ALIAS_GROUP_KEY + : `provider:${m.provider}`; let group = map.get(key); if (!group) { group = local @@ -330,10 +340,10 @@ export function groupModels( } : { key, - label: m.provider, + label: alias ? ALIAS_GROUP_LABEL : m.provider, type: "provider", endpoint: null, - provider: m.provider, + provider: alias ? "openrouter" : m.provider, models: [], selectedCount: 0, }; @@ -348,6 +358,9 @@ export function groupModels( } return groups.sort((a, b) => { if (a.type !== b.type) return a.type === "endpoint" ? -1 : 1; + // Aliases last: their weights can change, so they are a deliberate pick. + if ((a.key === ALIAS_GROUP_KEY) !== (b.key === ALIAS_GROUP_KEY)) + return a.key === ALIAS_GROUP_KEY ? 1 : -1; return a.label.localeCompare(b.label); }); } diff --git a/frontend/src/lib/components/generator/composer/AgentComposer.svelte b/frontend/src/lib/components/generator/composer/AgentComposer.svelte index 4b1401c4..1bfe21a5 100644 --- a/frontend/src/lib/components/generator/composer/AgentComposer.svelte +++ b/frontend/src/lib/components/generator/composer/AgentComposer.svelte @@ -2,7 +2,9 @@ /** * AgentComposer — the primary mode: tasks × models × harness presets × * trials, each run by an agent harness in a sandbox. Sections on the left - * and the sticky Run summary (server preview) on the right once the composer + * (the three choices first; the optional settings folded to a one-line + * summary until opened or invalid) and the sticky rail on the right — the + * section index and the Run summary (server preview) — once the composer * is ≥ 56rem wide (a container query: the sidebar state matters, not just the * viewport); narrower, the summary follows the form and a StartBar sticks * to the bottom. @@ -37,6 +39,7 @@ import { Textarea } from '$lib/components/ui/textarea'; import ConfirmDialog from '$lib/components/ConfirmDialog.svelte'; import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import ComposerSectionNav, { type ComposerNavSection } from '$lib/components/experiments/ComposerSectionNav.svelte'; import LoadingState from '$lib/components/LoadingState.svelte'; import AppSpecPicker from '../AppSpecPicker.svelte'; import ModelPicker from '../ModelPicker.svelte'; @@ -65,7 +68,9 @@ type PresetPins, type TaskKind, } from './draft'; - import { presetLookupIssue, previewIssues, type ChecklistItem, type IssueAction, type SummaryIssue } from './issues'; + import { presetLookupIssue, previewIssues, type IssueAction, type SummaryIssue } from './issues'; + import type { RunFactor } from './RunFormula.svelte'; + import { llmOverrideCount, overridesLabel, presetLabel } from './section-summary'; import { trackPresets } from './preset-lookup.svelte'; import Dices from '@lucide/svelte/icons/dices'; import Save from '@lucide/svelte/icons/save'; @@ -101,6 +106,11 @@ let trialsInvalid = $state(false); let seedInvalid = $state(false); let saveBudgetOpen = $state(false); + // Optional steps start folded unless the draft already overrides something. + let samplingOpen = $state( + Object.keys(cleanLLMParams(init.llmParams)).length > 0 || init.agentConfig.sampling_policy != null + ); + let budgetOpen = $state(Object.keys(budgetOnly(init.agentConfig)).length > 0); function budgetOnly(config: AgentConfigOverrides): AgentConfigOverrides { const out: Record = {}; @@ -394,23 +404,55 @@ if (currentPreview?.warnings.some((w) => w.code === 'missing_api_key')) previewNonce += 1; } - // ── checklist ─────────────────────────────────────────────────────── - const checklist = $derived([ - { - label: taskKind === 'freeform' ? 'Describe the free-form task' : 'Pick at least one app spec', - done: taskReady, - detail: taskKind === 'freeform' ? 'free-form (not graded)' : `${appIds.size} selected`, - href: '#composer-task', - }, - { label: 'Pick at least one model', done: modelIds.size > 0, detail: `${modelIds.size} selected`, href: '#composer-models' }, - { - label: 'Choose harness presets', - done: harnessPresetIds.length > 0, - detail: `${harnessPresetIds.length} selected`, - href: '#composer-harnesses', - }, + // ── run formula + section index ───────────────────────────────── + const fresh = $derived(ready && currentPreview ? currentPreview : null); + const factors = $derived([ + taskKind === 'freeform' + ? { label: 'tasks', singular: 'task', count: descriptionText ? 1 : 0, href: '#composer-task' } + : { label: 'app specs', singular: 'app spec', count: fresh?.task_count ?? appIds.size, href: '#composer-task' }, + { label: 'models', singular: 'model', count: fresh?.model_count ?? modelIds.size, href: '#composer-models' }, + { label: 'harnesses', singular: 'harness', count: fresh?.harness_count ?? harnessPresetIds.length, href: '#composer-harnesses' }, + { label: 'trials', singular: 'trial', count: trialsInvalid ? null : trials }, + ]); + const formulaTotal = $derived( + fresh?.run_count ?? + (factors.some((f) => f.count == null) ? null : factors.reduce((n, f) => n * (f.count ?? 0), 1)) + ); + + const samplingInvalid = $derived(llmInvalid || Object.keys(llmErrors).length > 0); + const budgetSectionInvalid = $derived( + budgetInvalid || + trialsInvalid || + seedInvalid || + trials == null || + Object.keys(budgetErrors).length > 0 || + !!fieldErrors.trials || + !!fieldErrors.base_seed + ); + const navSections = $derived([ + { id: 'composer-task', step: 1, label: 'Task', done: taskReady }, + { id: 'composer-models', step: 2, label: 'Models', done: modelIds.size > 0 }, + { id: 'composer-harnesses', step: 3, label: 'Harnesses', done: harnessPresetIds.length > 0 }, + { id: 'composer-sampling', step: 4, label: 'Model settings', invalid: samplingInvalid }, + { id: 'composer-budget', step: 5, label: 'Budget & repeats', invalid: budgetSectionInvalid }, ]); + const samplingSummary = $derived( + [ + presetLabel(samplingPreset, 'No preset'), + overridesLabel(llmOverrideCount(llmParams) + (samplingPolicy ? 1 : 0)), + `policy: ${(samplingPolicy ?? inheritedPolicy?.value) === 'harness_native' ? 'harness-native' : 'provider defaults'}`, + ].join(' · ') + ); + const budgetSummary = $derived( + [ + presetLabel(budgetPreset, 'Platform defaults'), + overridesLabel(Object.keys(budgetOnly(budgetOverrides)).length), + trials != null ? `${trials} trial${trials === 1 ? '' : 's'} per cell` : 'trials not set', + baseSeed != null ? `base seed ${baseSeed}` : 'random seeds', + ].join(' · ') + ); + // ── draft persistence ─────────────────────────────────────────────── $effect(() => { saveDraft('agent', { @@ -444,6 +486,8 @@ step={1} id="composer-task" title="Task" + done={taskReady} + meta={taskKind === 'specs' && appIds.size > 0 ? `${appIds.size} selected` : undefined} description="What every run builds. App specs carry a hidden conformance suite, so their runs are graded." > 0} + meta={modelIds.size > 0 ? `${modelIds.size} selected` : undefined} description={requireTools ? 'A selected harness needs native tool calling, so models without it are hidden.' : 'Every selected model runs every task with every harness preset.'} @@ -505,6 +551,8 @@ step={3} id="composer-harnesses" title="Harnesses" + done={harnessPresetIds.length > 0} + meta={harnessPresetIds.length > 0 ? `${harnessPresetIds.length} selected` : undefined} description="The comparison axis: each selected harness preset runs every model on every task." > {#if config} @@ -520,6 +568,10 @@ step={4} id="composer-sampling" title="Model settings" + collapsible + bind:open={samplingOpen} + summary={samplingSummary} + invalid={samplingInvalid} description="Sampling settings apply identically to every model; per-model support is shown in the Run summary." > @@ -612,8 +668,12 @@ class="min-w-0 @4xl/composer:sticky @4xl/composer:top-[4.5rem] @4xl/composer:max-h-[calc(100dvh-5.5rem)] @4xl/composer:overflow-y-auto @4xl/composer:overscroll-contain" aria-label="Run summary" > + /** - * AgentPreviewPanel — the agent composer's Run summary. Before a task and a - * model are chosen it is a neutral checklist; afterwards it renders the - * server preview (`POST /jobs/agent/preview/`): tally, cost bound, resolved + * AgentPreviewPanel — the agent composer's Run summary: the run formula + * (tasks × models × harness presets × trials, empty factors link to their + * step) always on top; once the form is complete, the server preview (`POST /jobs/agent/preview/`): tally, cost bound, resolved * settings, per-model effects and issues, plus the Start button (two-column layout only; * the StartBar carries it otherwise). * - * Props: `checklist`, `ready`, `preview` (latest successful one, possibly + * Props: `factors` + `total` (RunFormula), `ready`, `preview` (latest successful one, possibly * `stale` for the current form), `loading`, `error` + `onretry`, `localInvalid`, * `baseSeed`, `harnessSchemas` (harness → option schema, for chips), * `issues` + `actions`, Start state (`canStart`, `blockedReason`, `submitting`, @@ -22,9 +22,9 @@ import LoadingState from '$lib/components/LoadingState.svelte'; import { harnessOptionChips, summarizeBudget } from '../presets.svelte'; import EffectiveParams from './EffectiveParams.svelte'; - import SummaryChecklist from './SummaryChecklist.svelte'; + import RunFormula, { type RunFactor } from './RunFormula.svelte'; import SummaryIssues from './SummaryIssues.svelte'; - import { type ChecklistItem, type IssueAction, type SummaryIssue } from './issues'; + import { type IssueAction, type SummaryIssue } from './issues'; import Check from '@lucide/svelte/icons/check'; import CircleCheck from '@lucide/svelte/icons/circle-check'; import LoaderCircle from '@lucide/svelte/icons/loader-circle'; @@ -33,7 +33,8 @@ import X from '@lucide/svelte/icons/x'; interface Props { - checklist: ChecklistItem[]; + factors: RunFactor[]; + total: number | null; ready: boolean; preview: AgentPreview | null; stale: boolean; @@ -54,7 +55,8 @@ } let { - checklist, + factors, + total, ready, preview, stale, @@ -103,10 +105,6 @@ return `$${n.toLocaleString(undefined, { minimumFractionDigits: 2, maximumFractionDigits: 2 })}`; } - function plural(n: number, word: string): string { - return `${n} ${word}${n === 1 ? '' : 's'}`; - } - const chip = 'inline-flex items-center rounded-full border px-1.5 text-[11px] font-medium leading-4'; @@ -124,12 +122,11 @@ {/if} + {#if !ready}

- Choose what to run. Every run is resolved here — models, harnesses, settings and costs — before anything - starts. + Every run is resolved here — models, harnesses, settings and costs — before anything starts.

- {:else} {#if localInvalid}

@@ -143,20 +140,11 @@ {#if !error && !localInvalid}{/if} {:else}

-
-

- {preview.run_count} - run{preview.run_count === 1 ? '' : 's'} -

-

- {plural(preview.task_count, 'task')} × {plural(preview.model_count, 'model')} × {plural( - preview.harness_count, - 'harness preset' - )} × {plural(preview.trials, 'trial')} -

-

- Max cost ≤ {usd(preview.max_cost_usd_total)} - ({usd(preview.agent_config.max_cost_usd)} cap per run) +

+

+ Max cost + ≤ {usd(preview.max_cost_usd_total)} + {usd(preview.agent_config.max_cost_usd)} cap per run

{preview.graded diff --git a/frontend/src/lib/components/generator/composer/ComposerSection.svelte b/frontend/src/lib/components/generator/composer/ComposerSection.svelte index 149fc318..fa6e7d30 100644 --- a/frontend/src/lib/components/generator/composer/ComposerSection.svelte +++ b/frontend/src/lib/components/generator/composer/ComposerSection.svelte @@ -2,9 +2,22 @@ /** * ComposerSection — one numbered step of the composer form (a Card with a * step badge, heading, one-line description and optional header actions). + * + * The badge carries the step's state: a check once a required step is + * `done`, an alert while it holds an `invalid` value. `meta` is a short + * right-aligned status ("3 selected"). + * + * `collapsible` steps (optional settings) start folded when `open` is + * false and show `summary` — what applies right now — instead of their + * body. The body stays mounted (its fields keep validating) and an + * invalid step unfolds itself. */ import type { Snippet } from 'svelte'; + import { cn } from '$lib/utils'; import * as Card from '$lib/components/ui/card'; + import Check from '@lucide/svelte/icons/check'; + import ChevronDown from '@lucide/svelte/icons/chevron-down'; + import CircleAlert from '@lucide/svelte/icons/circle-alert'; interface Props { step: number; @@ -12,31 +25,98 @@ /** Section anchor; the heading id is `${id}-title`. */ id: string; description?: string; + /** Required steps: whether they are complete. */ + done?: boolean; + invalid?: boolean; + meta?: string; + collapsible?: boolean; + open?: boolean; + /** One line shown while folded. */ + summary?: string; actions?: Snippet; children: Snippet; } - let { step, title, id, description, actions, children }: Props = $props(); + let { + step, + title, + id, + description, + done = false, + invalid = false, + meta, + collapsible = false, + open = $bindable(true), + summary, + actions, + children, + }: Props = $props(); + + $effect(() => { + if (invalid && collapsible) open = true; + }); + + const folded = $derived(collapsible && !open);

- - + + + {#if invalid}{:else if done}{:else}{step}{/if} +
-

{title}

 - {#if description} + +

+ {#if collapsible} + + {:else} + {title} + {/if} + {invalid ? '(has a field to fix)' : done ? '(done)' : ''} +

+ + {#if folded && summary} +

{summary}

+ {:else if description} {description} {/if}
- {#if actions} -
{@render actions()}
+ {#if meta || actions} +
+ {#if meta} + {meta} + {/if} + {@render actions?.()} +
{/if} - + diff --git a/frontend/src/lib/components/generator/composer/RunFormula.svelte b/frontend/src/lib/components/generator/composer/RunFormula.svelte new file mode 100644 index 00000000..b0016ab5 --- /dev/null +++ b/frontend/src/lib/components/generator/composer/RunFormula.svelte @@ -0,0 +1,80 @@ + + + + +
+

{spoken}

+ + {#if missing.length > 0} +

+ Still to choose: + {#each missing as f, i (f.label)}{#if i > 0}{i === missing.length - 1 ? ' and ' : ', '}{/if}{#if f.href}{f.label}{:else}{f.label}{/if}{/each}. +

+ {/if} +
diff --git a/frontend/src/lib/components/generator/composer/SamplingSettings.svelte b/frontend/src/lib/components/generator/composer/SamplingSettings.svelte index 4ca8faea..19d5d780 100644 --- a/frontend/src/lib/components/generator/composer/SamplingSettings.svelte +++ b/frontend/src/lib/components/generator/composer/SamplingSettings.svelte @@ -68,6 +68,8 @@ disabled?: boolean; hideProviderRouting?: boolean; localModels?: boolean; + /** Replaces the mode's inheritance note (e.g. when the section already says it). */ + hint?: string; onchange?: () => void; } @@ -85,6 +87,7 @@ disabled = false, hideProviderRouting = false, localModels = false, + hint: hintProp, onchange, }: Props = $props(); @@ -149,7 +152,7 @@ custom: 'Blank fields inherit the value shown in grey: the mode default (temperature 0.3, output cap 32,000), then the sampling preset.', }; - const hint = $derived(HINTS[mode]); + const hint = $derived(hintProp ?? HINTS[mode]); function onSaved(saved: RunPreset) { presetId = saved.id; diff --git a/frontend/src/lib/components/generator/composer/ScaffoldingComposer.svelte b/frontend/src/lib/components/generator/composer/ScaffoldingComposer.svelte index 7a2a95f8..16aaae74 100644 --- a/frontend/src/lib/components/generator/composer/ScaffoldingComposer.svelte +++ b/frontend/src/lib/components/generator/composer/ScaffoldingComposer.svelte @@ -27,6 +27,7 @@ import { lookupBatchProfileVersion } from './prepare'; import ConfirmDialog from '$lib/components/ConfirmDialog.svelte'; import ErrorAlert from '$lib/components/ErrorAlert.svelte'; + import ComposerSectionNav, { type ComposerNavSection } from '$lib/components/experiments/ComposerSectionNav.svelte'; import LoadingState from '$lib/components/LoadingState.svelte'; import AppSpecPicker from '../AppSpecPicker.svelte'; import ModelPicker from '../ModelPicker.svelte'; @@ -48,7 +49,9 @@ type PresetPins, type ScaffoldingDraft, } from './draft'; - import { presetLookupIssue, type ChecklistItem, type IssueAction, type SummaryIssue } from './issues'; + import { presetLookupIssue, type IssueAction, type SummaryIssue } from './issues'; + import type { RunFactor } from './RunFormula.svelte'; + import { llmOverrideCount, overridesLabel, presetLabel } from './section-summary'; import { trackLLMPreview } from './llm-preview.svelte'; import { trackPresets } from './preset-lookup.svelte'; @@ -78,6 +81,8 @@ let trials = $state(init.trials); let llmInvalid = $state(false); let trialsInvalid = $state(false); + // Optional step: folded unless the draft already overrides something. + let samplingOpen = $state(llmOverrideCount(init.llmParams) > 0); // ── supporting data ───────────────────────────────────────────────── let defaultStack = $state(undefined); @@ -320,11 +325,22 @@ ) ); - const checklist = $derived([ - { label: 'Pick at least one app spec', done: appIds.size > 0, detail: `${appIds.size} selected`, href: '#scaffolding-task' }, - { label: 'Choose a stack', done: !!stackSlug, detail: stackSlug, href: '#scaffolding-task' }, - { label: 'Pick at least one model', done: modelIds.size > 0, detail: `${modelIds.size} selected`, href: '#scaffolding-models' }, + const factors = $derived([ + { label: 'app specs', singular: 'app spec', count: appIds.size, href: '#scaffolding-task' }, + { label: 'models', singular: 'model', count: modelIds.size, href: '#scaffolding-models' }, + { label: 'trials', singular: 'trial', count: trialsInvalid ? null : trials, href: '#scaffolding-repeats' }, ]); + const samplingInvalid = $derived(llmInvalid || Object.keys(llmErrors).length > 0); + const trialsSectionInvalid = $derived(trialsInvalid || trials == null || !!serverFieldErrors.trials); + const navSections = $derived([ + { id: 'scaffolding-task', step: 1, label: 'Task & prompts', done: appIds.size > 0 && !!stackSlug }, + { id: 'scaffolding-models', step: 2, label: 'Models', done: modelIds.size > 0 }, + { id: 'scaffolding-sampling', step: 3, label: 'Model settings', invalid: samplingInvalid }, + { id: 'scaffolding-repeats', step: 4, label: 'Repeats', invalid: trialsSectionInvalid }, + ]); + const samplingSummary = $derived( + [presetLabel(samplingPreset, 'Mode default (temperature 0.3)'), overridesLabel(llmOverrideCount(llmParams))].join(' · ') + ); $effect(() => { saveDraft('scaffolding', { @@ -347,6 +363,8 @@ step={1} id="scaffolding-task" title="Task & prompts" + done={appIds.size > 0 && !!stackSlug} + meta={appIds.size > 0 ? `${appIds.size} selected` : undefined} description="App specs to build, the stack template, and the prompt profile that fills it." > @@ -378,6 +396,8 @@ step={2} id="scaffolding-models" title="Models" + done={modelIds.size > 0} + meta={modelIds.size > 0 ? `${modelIds.size} selected` : undefined} description="Every selected model scaffolds every app spec." > @@ -387,6 +407,10 @@ step={3} id="scaffolding-sampling" title="Model settings" + collapsible + bind:open={samplingOpen} + summary={samplingSummary} + invalid={samplingInvalid} description="Sampling settings apply identically to every model." > - + + -
-

- {runCount} - run{runCount === 1 ? '' : 's'} -

-

- {appIds.size} app spec{appIds.size === 1 ? '' : 's'} × {modelIds.size} model{modelIds.size === 1 ? '' : 's'} × - {trials ?? '—'} trial{trials === 1 ? '' : 's'} -

-
+
+
Stack
{stackSlug || '—'}
Profile
diff --git a/frontend/src/lib/components/generator/composer/SimpleSummaryPanel.svelte b/frontend/src/lib/components/generator/composer/SimpleSummaryPanel.svelte index 046186c1..6456d4a4 100644 --- a/frontend/src/lib/components/generator/composer/SimpleSummaryPanel.svelte +++ b/frontend/src/lib/components/generator/composer/SimpleSummaryPanel.svelte @@ -1,11 +1,11 @@ - - -
- - - {#if capacity} - - - {executorState.label} - - {/if} -
- Shared execution slots for every user's runs. - - + + +
+ + {#if !capacity} - + {:else} + + + {executorState.label} + {#if executor?.draining} -

+ {#if lanesIdle && executor.draining_remaining.jobs === 0 && executor.draining_remaining.post_runs === 0} Drained: safe to restart the executor. {:else} - Draining: {executor.draining_remaining.jobs} runs and {executor.draining_remaining.post_runs} post-run steps - still finishing. No new runs start. + {executor.draining_remaining.jobs} runs and {executor.draining_remaining.post_runs} post-run steps still finishing {/if} -

+ {:else if executor?.code_stale} -

- The executor runs older code than the web app. Drain it and restart it to pick up the change. -

+ + Drain and restart to pick up new code + {:else if !executor?.online} -

No executor heartbeat: queued runs wait until it is back.

+ Queued runs wait until it is back {/if} -
+
{#each lanes as lane (lane.key)} - + {/each}
- {#each capacity.limits.warnings as warning (warning)} -

-

- {/each} + {#if isStaff && settings} + + {/if} - {#if isStaff} -
- {#if settingsError} -

{settingsError}

- {:else if !settings} - - {:else} -
- {#each slotFields as field (field.key)} - draft[field.key], (v) => (draft = { ...draft, [field.key]: v })} - bind:invalid={() => !!invalid[field.key], (v) => { - if (!!invalid[field.key] !== v) invalid = { ...invalid, [field.key]: v }; - }} - /> - {/each} -
-

- {#if capacity.limits.max_sandbox_slots_allowed > 0} - apps-dind memory {capacity.limits.apps_dind_mem || 'unknown'} allows at most - {capacity.limits.max_sandbox_slots_allowed} sandbox slots (3 GB per sandbox plus gateway, evaluation and - analyzer headroom). - {:else} - apps-dind memory {capacity.limits.apps_dind_mem || 'unknown'} fits one sandbox only when evaluation - is idle: 1 sandbox slot runs, and sandbox runs and evaluations take turns (3 GB per sandbox plus - gateway, evaluation and analyzer headroom). Raise the apps-dind memory to run them side by side. - {/if} -

- {#if ruleError} - + + {#if capacity.limits.warnings.length > 0} + {@const warnings = capacity.limits.warnings} +
+ + +
    + {#each warnings as warning (warning)} +
  • +
    • + {/each} +
+
+ {/if} + {/if} +
+ + {#if capacity && isStaff} +
+ + Slot settings + +
+ {#if settingsError} +

{settingsError}

+ {:else if !settings} + + {:else} +
+ {#each slotFields as field (field.key)} + draft[field.key], (v) => (draft = { ...draft, [field.key]: v })} + bind:invalid={() => !!invalid[field.key], (v) => { + if (!!invalid[field.key] !== v) invalid = { ...invalid, [field.key]: v }; + }} + /> + {/each} +
+
+ + {#if capacity.limits.max_sandbox_slots_allowed > 0} +

+ apps-dind memory {capacity.limits.apps_dind_mem || 'unknown'}: at most {capacity.limits.max_sandbox_slots_allowed} sandbox slots +

+ {:else} +

+ apps-dind memory {capacity.limits.apps_dind_mem || 'unknown'}: one sandbox, taking turns with evaluation +

{/if} -
- -
- {#if dirty} - - {/if} - -
+ {/if} + +
+
+ {#if ruleError} + {/if} -
- {/if} - {/if} - + {/if} +
+ + {/if} diff --git a/frontend/src/lib/components/inference/EndpointForm.svelte b/frontend/src/lib/components/inference/EndpointForm.svelte index de757525..abf53e7f 100644 --- a/frontend/src/lib/components/inference/EndpointForm.svelte +++ b/frontend/src/lib/components/inference/EndpointForm.svelte @@ -31,6 +31,7 @@ import * as Select from '$lib/components/ui/select'; import Switch from '$lib/components/ui/switch/switch.svelte'; import NumberField from '$lib/components/generator/composer/NumberField.svelte'; + import SectionHeading from '$lib/components/SectionHeading.svelte'; import { endpointSlugFromName, slugError, SLUG_MAX_LENGTH } from './endpoint-display'; import LoaderCircle from '@lucide/svelte/icons/loader-circle'; import Lock from '@lucide/svelte/icons/lock'; @@ -208,7 +209,7 @@
{#if showConnection && !systemRow}
-

Server and connection

+ {#if mode === 'create'}
@@ -282,7 +283,7 @@ {#if showDetails} {#if !systemRow}
-

Name

+
@@ -327,7 +328,7 @@ {/if}
-

Capacity

+
-

Requests

+
-

Sharing

+
visibility, (v) => (visibility = v as 'private' | 'shared')} items={visibilityItems} /> diff --git a/frontend/src/lib/components/inference/tests/catalog.test.ts b/frontend/src/lib/components/inference/tests/catalog.test.ts index 83d3408c..59c8ac70 100644 --- a/frontend/src/lib/components/inference/tests/catalog.test.ts +++ b/frontend/src/lib/components/inference/tests/catalog.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import type { LLMModelSummary } from "$lib/api/models"; import { + ALIAS_GROUP_LABEL, filterByEndpointScope, groupModels, isLocalModel, @@ -110,6 +111,24 @@ describe("groupModels", () => { ]); }); + it("collects ~vendor aliases into one group after the vendors", () => { + const alias1 = model({ + provider: "~anthropic", + model_name: "Claude Opus Latest", + }); + const alias2 = model({ provider: "~openai", model_name: "GPT Latest" }); + const groups = groupModels([alias1, a, alias2, b], "endpoint"); + expect(groups.map((g) => g.label)).toEqual([ + "anthropic", + "openai", + ALIAS_GROUP_LABEL, + ]); + expect(groups[2].models.map((m) => m.model_name)).toEqual([ + "Claude Opus Latest", + "GPT Latest", + ]); + }); + it("puts local endpoints first, by name, then OpenRouter vendors", () => { const groups = groupModels(all, "endpoint", new Set([l1.id, l3.id, a.id])); expect(groups.map((g) => [g.type, g.label])).toEqual([ diff --git a/frontend/src/lib/components/integrations/GitHubExportDialog.svelte b/frontend/src/lib/components/integrations/GitHubExportDialog.svelte index 32807571..7ea90c63 100644 --- a/frontend/src/lib/components/integrations/GitHubExportDialog.svelte +++ b/frontend/src/lib/components/integrations/GitHubExportDialog.svelte @@ -6,7 +6,7 @@ * Props: `open` (bindable), `applicationId` (GenerationJob UUID), * `defaultName` (suggested repository name; slugified). * Checks `/integrations/github/status/` on open and points to the - * Integrations page when no working token is stored. + * Settings → GitHub when no working token is stored. */ import * as Dialog from '$lib/components/ui/dialog'; import { Button } from '$lib/components/ui/button'; @@ -138,7 +138,7 @@ ? status.message || 'GitHub rejected the stored token.' : 'No GitHub token is stored yet.'}

- +
{/if} diff --git a/frontend/src/lib/components/integrations/GitHubSection.svelte b/frontend/src/lib/components/integrations/GitHubSection.svelte index 43ee22e8..91df72f9 100644 --- a/frontend/src/lib/components/integrations/GitHubSection.svelte +++ b/frontend/src/lib/components/integrations/GitHubSection.svelte @@ -5,22 +5,31 @@ * (GitHubExportDialog). */ import { onMount } from 'svelte'; - import * as Card from '$lib/components/ui/card'; + import SettingsBlock from '$lib/components/users-settings/SettingsBlock.svelte'; import { Button } from '$lib/components/ui/button'; import ErrorAlert from '$lib/components/ErrorAlert.svelte'; - import LoadingState from '$lib/components/LoadingState.svelte'; import StatusBadge from '$lib/components/StatusBadge.svelte'; - import CredentialCard from '$lib/components/users-settings/CredentialCard.svelte'; + import CredentialRow from '$lib/components/users-settings/CredentialRow.svelte'; + import { useSectionStatus } from '$lib/components/users-settings/section-status.svelte'; import { formatApiError } from '$lib/api/core'; import { getGitHubStatus, type GitHubStatus } from '$lib/api/integrations'; - import GitBranch from '@lucide/svelte/icons/git-branch'; import RotateCw from '@lucide/svelte/icons/rotate-cw'; import ExternalLink from '@lucide/svelte/icons/external-link'; + import ChevronDown from '@lucide/svelte/icons/chevron-down'; + import GitBranch from '@lucide/svelte/icons/git-branch'; let status = $state(null); let loading = $state(true); let error = $state(''); + const report = useSectionStatus('github'); + $effect(() => { + if (!status) return report(null); + if (status.connected) report({ tone: 'success', label: 'connected' }); + else if (status.configured) report({ tone: 'danger', label: 'rejected' }); + else report(null); + }); + async function load() { loading = status === null; error = ''; @@ -36,96 +45,94 @@ onMount(load); -
- - - - GitHub account - - - - Export generated applications to a new repository (one commit, optionally with the stack scaffold and a - SARIF upload of the analysis findings to code scanning) or to a Gist. Open an application and choose - Export to GitHub. - - - - {#if error} - - {:else if loading} - - {:else if status} -
- {#if status.avatar_url} - - {/if} -
-

- {#if status.connected && status.login} - - {status.name || status.login} - {#if status.name}@{status.login}{/if} - {/if} + + {#snippet description()} + Exports are one commit, optionally with the stack scaffold and a SARIF upload to code scanning. Start one from + an application with Export to GitHub. + {/snippet} + +

+ {#if error} +
+ {:else if loading} +
+ {:else if status} +
+ {#if status.avatar_url} + + {:else} + + + {/if} +
+

+ {#if status.connected && status.login} + + {status.name || status.login} + {#if status.name}@{status.login}{/if} + {:else} + {status.configured ? 'Token rejected' : 'Not connected'} + {/if} + {#if status.configured} -

- {#if status.message}

{status.message}

{/if} + {/if} +

+

{#if status.connected} -

- {status.token_type ? `${status.token_type} token` : 'Token'} - {#if status.scopes.length} - · scopes - {#each status.scopes as s (s)}{s}{/each} - {/if} -

+ {status.token_type ? `${status.token_type} token` : 'Token'} + {#if status.scopes.length} + · {#each status.scopes as s (s)}{s}{/each} + {/if} + {:else} + {status.message || 'Add a personal access token below.'} {/if} -
+

- {/if} -
-

Token permissions

-

- Classic token: repo (create and push repositories), - gist, security_events (SARIF upload). -

-

- Fine-grained token: repository access to all repositories with - Administration and Contents set to - read and write (add Code scanning alerts write for SARIF). Fine-grained - tokens cannot create Gists. -

-

- - Create a token on GitHub -

- - + {/if} + +
- -
+ +
+ + Which permissions does the token need? + +
+

+ Classic token: repo (create and + push repositories), gist, security_events (SARIF upload). +

+

+ Fine-grained token: access to all repositories with + Administration and Contents set to read and write (add Code scanning alerts write for SARIF). Fine-grained + tokens cannot create Gists. +

+
+
+ diff --git a/frontend/src/lib/components/integrations/McpSection.svelte b/frontend/src/lib/components/integrations/McpSection.svelte index b6356ce8..76dac2c3 100644 --- a/frontend/src/lib/components/integrations/McpSection.svelte +++ b/frontend/src/lib/components/integrations/McpSection.svelte @@ -8,12 +8,13 @@ */ import { onMount } from 'svelte'; import { toast } from 'svelte-sonner'; - import * as Card from '$lib/components/ui/card'; + import SettingsBlock from '$lib/components/users-settings/SettingsBlock.svelte'; + import SettingsRow from '$lib/components/users-settings/SettingsRow.svelte'; + import CopyField from '$lib/components/users-settings/CopyField.svelte'; import * as Select from '$lib/components/ui/select'; import { Button } from '$lib/components/ui/button'; import { Badge } from '$lib/components/ui/badge'; import { Input } from '$lib/components/ui/input'; - import { Label } from '$lib/components/ui/label'; import { Segmented } from '$lib/components/ui/segmented'; import ErrorAlert from '$lib/components/ErrorAlert.svelte'; import LoadingState from '$lib/components/LoadingState.svelte'; @@ -21,7 +22,6 @@ import { formatApiError } from '$lib/api/core'; import { createApiToken } from '$lib/api/tokens'; import { fillToken, getMcpConfig, type McpConfig } from '$lib/api/integrations'; - import Bot from '@lucide/svelte/icons/bot'; import KeyRound from '@lucide/svelte/icons/key-round'; import LoaderCircle from '@lucide/svelte/icons/loader-circle'; import TriangleAlert from '@lucide/svelte/icons/triangle-alert'; @@ -90,181 +90,145 @@ onMount(load); -
- - - MCP server - - Let an AI assistant (Claude, ChatGPT, Cursor, VS Code…) drive LLM Lab over the Model Context Protocol: - browse models and results, start generations and analyses, and read reports. - - - - {#if error} - - {:else if loading} - - {:else if config} -
-

Server URL

- +{#snippet tokenFooter()} + Shown once after you create it. + +{/snippet} + +{#if error} + +{:else if loading} + +{:else if config} + + + + +
+ {#each [{ k: 'Transport', v: 'Streamable HTTP' }, { k: 'Protocol', v: config.protocol_versions.join(', ') }, { k: 'Tools', v: String(config.tool_count) }, { k: 'Prompts', v: String(config.prompts.length) }] as fact (fact.k)} +
+

{fact.k}

+

{fact.v}

-

- Streamable HTTP transport - Protocol {config.protocol_versions.join(', ')} - {config.tool_count} tools - {config.prompts.length} prompts + {/each} +

+ + + + {#snippet description()} + Clients sign in with a personal token. Tokens embedded in a URL (claude.ai, ChatGPT) need the + mcp scope; read hides every tool that changes data. + {/snippet} + {#if token} +
+

+

- {/if} - - + +
+ + {tokenPrefix}… · revoke it any time under + Personal tokens. + + +
+
+ {:else} + + + + + + + {/if} + - {#if config} - - - Connection token - - Clients authenticate with a personal API token. The mcp scope is required - for tokens embedded in a URL (claude.ai and ChatGPT connectors); adding - read hides every tool that changes data. - - - - {#if token} -
-

-

- -
- - Token {tokenPrefix}… · revoke it any time in - Settings → Personal Tokens. - - -
-
- {:else} -
-
- - -
-
- Access - -
-
-
+ + {#snippet description()} + {#if token} + The snippet includes your new token. + {:else} + Replace <TOKEN> with a connection token, or create one above to fill it in. + {/if} + {/snippet} + {#if config.clients.length === 0} +
No client snippets available.
+ {:else} + +
+ + {#if client} +
+

+ {client.auth === 'url' ? 'Token in the URL' : 'Token in an Authorization header'} + + Goes in {client.location} + {#if client.docs_url} + + Docs + {/if} +

+ + {#if client.auth === 'url'}

- Scopes: {access === 'read' ? 'mcp, read' : 'mcp'} + The token travels in the URL: use a dedicated mcp token and revoke it if + the URL leaks.

- -
- {/if} - - - - - - Set up a client - - {#if token} - Snippets include your new token. - {:else} - Replace <TOKEN> with a connection token, or create one above to fill - it in. - {/if} - - - - {#if config.clients.length === 0} -

No client snippets available.

- {:else} -
- - -
- {#if client} -
-

- {client.auth === 'url' ? 'Token in URL' : 'Header auth'} - Where: {client.location} - {#if client.docs_url} - - Docs - {/if} -

- - {#if client.auth === 'url'} -

- The token travels in the URL: use a dedicated token with the mcp - scope and revoke it if the URL leaks. -

- {/if} -
{/if} - {/if} - - +
+ {/if} + {/if} + - - -
- - - Tools ({config.tools.length} · {readTools} read-only) - - -
    - {#each config.tools as tool (tool.name)} -
  • - {tool.name} - {tool.read_only ? 'read-only' : 'write'} -

    {tool.description}

    -
    • - {/each} -
-
- {#if config.prompts.length > 0} -
- - - Prompts ({config.prompts.length}) - - -
    - {#each config.prompts as prompt (prompt)} -
  • {prompt}
    • - {/each} -
-
- {/if} - - - {/if} -
+ +
+ + Tools · {config.tools.length}, {readTools} read-only + +
    + {#each config.tools as tool (tool.name)} +
  • +

    + {tool.name} + {tool.read_only ? 'read-only' : 'write'} +

    +

    {tool.description}

    +
    • + {/each} +
+
+ {#if config.prompts.length > 0} +
+ + Prompts · {config.prompts.length} + +
    + {#each config.prompts as prompt (prompt)} +
  • {prompt}
    • + {/each} +
+
+ {/if} + +{/if} diff --git a/frontend/src/lib/components/integrations/ProvidersSection.svelte b/frontend/src/lib/components/integrations/ProvidersSection.svelte index 807a080d..437ad49a 100644 --- a/frontend/src/lib/components/integrations/ProvidersSection.svelte +++ b/frontend/src/lib/components/integrations/ProvidersSection.svelte @@ -1,19 +1,19 @@ -
-

- Connect a cloud provider with its API key: LLM Lab creates an endpoint and imports its models, ready for - generation and experiments. -

+{#if error} + load()} /> +{:else if loading} + +{:else if providers.length === 0} + +{:else} + {#if connectedProviders.length} + + {@render rows(connectedProviders)} + + {/if} + + {@render rows(availableProviders)} + +{/if} - {#if error} - load()} /> - {:else if loading} - - {:else if providers.length === 0} - - {:else} -
- {#each providers as p (p.id)} - - - - {p.name} - {#if p.connected_endpoints.length > 0} - - {/if} - - {p.description} - - -

{p.base_url}

- {#if p.connected_endpoints.length > 0} -
    - {#each p.connected_endpoints as ep (ep.id)} -
  • - {ep.name} - {ep.model_count} models - {#if !ep.enabled}{/if} - {#if ep.last_error} -

    {ep.last_error}

    - {/if} -
    • - {/each} -
- {/if} -
-
- {#if p.keys_url} - - Get a key +{#snippet rows(list: ProviderPreset[])} + {#each list as p (p.id)} + {@const isConnected = p.connected_endpoints.length > 0} +
+ +
+

{p.name}

+

{p.description}

+ {#if isConnected} +
    + {#each p.connected_endpoints as ep (ep.id)} +
  • + + {ep.name} + {ep.model_count} models + {#if !ep.enabled}· disabled{/if} + {#if ep.last_error} + {ep.last_error} {/if} - {#if p.docs_url} - - Docs - {/if} -
- {#if p.can_connect} - - {:else} - - - {/if} -
- - - {/each} + + {/each} + + {/if} +

+ {#if p.keys_url} + + Get a key + {/if} + {#if p.docs_url} + + Docs + {/if} +

+
+
+ {#if p.can_connect} + + {:else} + + + {/if} +
- {/if} -
+ {/each} +{/snippet} diff --git a/frontend/src/lib/components/integrations/WebhooksSection.svelte b/frontend/src/lib/components/integrations/WebhooksSection.svelte index 92c89e1c..4f52b6f1 100644 --- a/frontend/src/lib/components/integrations/WebhooksSection.svelte +++ b/frontend/src/lib/components/integrations/WebhooksSection.svelte @@ -5,9 +5,8 @@ */ import { onMount } from 'svelte'; import { toast } from 'svelte-sonner'; - import * as Card from '$lib/components/ui/card'; + import SettingsBlock from '$lib/components/users-settings/SettingsBlock.svelte'; import { Button } from '$lib/components/ui/button'; - import { Badge } from '$lib/components/ui/badge'; import { Switch } from '$lib/components/ui/switch'; import ErrorAlert from '$lib/components/ErrorAlert.svelte'; import LoadingState from '$lib/components/LoadingState.svelte'; @@ -28,6 +27,7 @@ import WebhookFormDialog from './WebhookFormDialog.svelte'; import WebhookDeliveries from './WebhookDeliveries.svelte'; import { webhookKind } from './webhook-kinds'; + import { useSectionStatus } from '$lib/components/users-settings/section-status.svelte'; import Bell from '@lucide/svelte/icons/bell'; import Plus from '@lucide/svelte/icons/plus'; import Send from '@lucide/svelte/icons/send'; @@ -38,6 +38,13 @@ const MAX_WEBHOOKS = 25; + const report = useSectionStatus('notifications'); + $effect(() => { + if (loading || error) return; + const on = subs.filter((s) => s.enabled).length; + report(on ? { tone: 'success', label: `${on} on` } : null); + }); + let subs = $state([]); let eventTypes = $state([]); let loading = $state(true); @@ -134,117 +141,115 @@ onMount(load); -
-
-

- Get a message in Slack, Discord, Teams, ntfy or Telegram, or a signed JSON POST to your own service, when your - generations, analyses, experiments or reports finish. Targets and secrets are stored encrypted and never shown - again. -

+ + {#snippet actions()} -
+ {/snippet} {#if error} - +
{:else if loading} {:else if subs.length === 0} - - {#snippet action()}{/snippet} - + {:else} -
- {#each subs as sub (sub.id)} - {@const kindInfo = webhookKind(sub.kind)} - -
- setEnabled(sub, v)} - class="mt-0.5" - /> -
-

- {sub.name} - {kindInfo?.label ?? sub.kind} - {#if !sub.enabled} - - {:else if sub.last_error && (!sub.last_success_at || (sub.last_error_at ?? '') > sub.last_success_at)} - - {:else if sub.last_success_at} - - {/if} -

-

- {#if sub.kind === 'telegram'} - chat {sub.telegram_chat_id} - {:else if sub.url_hint} - {sub.url_hint} - {/if} - {#if sub.has_secret}secret {sub.secret_hint}{/if} - last success {formatAgo(sub.last_success_at)} -

- {#if sub.last_error} -

- {formatAgo(sub.last_error_at)}: {sub.last_error} -

+ {#each subs as sub (sub.id)} + {@const kindInfo = webhookKind(sub.kind)} +
+
+ setEnabled(sub, v)} + class="mt-0.5" + /> +
+

+ {sub.name} + {kindInfo?.label ?? sub.kind} + {#if !sub.enabled} + + {:else if sub.last_error && (!sub.last_success_at || (sub.last_error_at ?? '') > sub.last_success_at)} + + {:else if sub.last_success_at} + + {/if} +

+

+ {#if sub.kind === 'telegram'} + chat {sub.telegram_chat_id} + + {:else if sub.url_hint} + {sub.url_hint} + {/if} -

- {#if sub.events.length === 0} - No events selected: only tests are sent. - {:else} - {#each sub.events as ev (ev)}{ev}{/each} - {/if} - {#if sub.include_experiment_jobs}+ experiment jobs{/if} + {#if sub.has_secret}secret {sub.secret_hint}{/if} + last success {formatAgo(sub.last_success_at)} +

+ {#if sub.last_error} +

+ {formatAgo(sub.last_error_at)}: {sub.last_error}

-
-
- - - - -
+ {/if} +

+ {#if sub.events.length === 0} + No events selected: only tests are sent. + {:else} + {#each sub.events as ev (ev)}{ev}{/each} + {/if} + {#if sub.include_experiment_jobs}+ experiment jobs{/if} +

+
+
+ + + + +
+
+ {#if expandedId === sub.id} +
+

Recent deliveries

+
- {#if expandedId === sub.id} -
-

Recent deliveries

- -
- {/if} - - {/each} -
- {#if subs.length >= MAX_WEBHOOKS} -

At most {MAX_WEBHOOKS} notifications per account.

- {/if} + {/if} +
+ {/each} {/if} -
+ +{#if subs.length >= MAX_WEBHOOKS} +

At most {MAX_WEBHOOKS} channels per account.

+{/if} diff --git a/frontend/src/lib/components/integrations/index.ts b/frontend/src/lib/components/integrations/index.ts index 202ff8d9..518937c7 100644 --- a/frontend/src/lib/components/integrations/index.ts +++ b/frontend/src/lib/components/integrations/index.ts @@ -1,4 +1,4 @@ -// Integrations page sections and the GitHub export dialog (application detail page). +// Settings → Connections sections and the GitHub export dialog (application detail page). export { default as McpSection } from "./McpSection.svelte"; export { default as ProvidersSection } from "./ProvidersSection.svelte"; export { default as ProviderConnectDialog } from "./ProviderConnectDialog.svelte"; diff --git a/frontend/src/lib/components/models/ModelBenchmarksTab.svelte b/frontend/src/lib/components/models/ModelBenchmarksTab.svelte new file mode 100644 index 00000000..1f12b635 --- /dev/null +++ b/frontend/src/lib/components/models/ModelBenchmarksTab.svelte @@ -0,0 +1,173 @@ + + +{#if !hasAny} + +{:else} +
+ {#if aa} + + + + Artificial Analysis + + + Composite indices across their evaluation suites, via OpenRouter; meters place the score among catalog models that have one + + + {#each Object.entries(AA_LABELS) as [key, label] (key)} + {@const value = aa[key as keyof typeof AA_LABELS]} + {@const cat = aaCatalog?.[key as keyof typeof AA_LABELS]} + + {/each} + + + {/if} + + {#if arenaGroups.length} + + + + Design Arena + + + Pairwise human votes on generated UIs, apps and agent builds — Elo, leaderboard rank and win rate per category + + + {#each arenaGroups as group (group.name)} +
+ + + + + + + + + + + {#each group.rows as e (e.category)} + + + + + + + {/each} + +
{group.name} arenaEloRankWin rate
{categoryLabel(e.category)}{e.elo}#{e.rank}{e.win_rate.toFixed(1)}%
+
+ {/each} + + + {/if} + + {#if local.length} + + + Seeded benchmarks + Rows feeding the rankings' benchmark score (manage.py seed_benchmarks) + + +
+ + + + {#each local as b (b.benchmark)} + + + + + + + + {/each} + +
BenchmarkScorePercentileSourceUpdated
{b.benchmark}{b.score}{b.percentile != null ? `p${Math.round(b.percentile * 100)}` : '—'}of {b.n}{b.source}{b.fetched_at.slice(0, 10)}
+
+ + + {/if} +
+{/if} diff --git a/frontend/src/lib/components/models/ModelCapabilitiesTab.svelte b/frontend/src/lib/components/models/ModelCapabilitiesTab.svelte new file mode 100644 index 00000000..23c5eb6c --- /dev/null +++ b/frontend/src/lib/components/models/ModelCapabilitiesTab.svelte @@ -0,0 +1,217 @@ + + +
+
+ + Features + +
    + {#each features as f (f.label)} +
  • + {#if f.on}
    • + {/each} +
+
+
Modalities
+
+ {#each inputMods.length ? inputMods : ['text'] as mod (mod)} + {mod} + {/each} + {#each outputMods.length ? outputMods : ['text'] as mod (mod)} + {mod} + {/each} + {#if arch.modality}{arch.modality}{/if} +
+
+ {#if voices.length} +
+
Voices
+

{voices.join(', ')}

+
+ {/if} + + + + + Reasoning + + {#if !reasoning && !mdReasoning.length} +

No reasoning controls published.

+ {:else} + {#if reasoning} +
+
Always on
+
{reasoning.mandatory ? 'Yes — cannot be disabled' : 'No — optional'}
+ {#if reasoning.default_effort} +
Default effort
+
{reasoning.default_effort}
+ {/if} + {#if reasoning.supported_efforts?.length} +
Effort levels
+
+ {#each reasoning.supported_efforts as e (e)} + {e} + {/each} +
+ {/if} +
+ {/if} + {#each mdReasoning as opt, i (i)} +

+ models.dev: {opt.type} + {#if opt.values?.length}{opt.values.join(' / ')}{/if} + {#if opt.min != null}min {formatNumber(opt.min)}{/if}{#if opt.max != null}, max {formatNumber(opt.max)}{/if} +

+ {/each} + {/if} + + +
+ + + + Request parameters {params.length} + What OpenRouter forwards for this model; anything else is dropped. Hover for what a parameter does. + + + {#if params.length} +
+ {#each [...params].sort() as p (p)} + {p} + {/each} +
+ {:else} +

Not published.

+ {/if} + + + +
+ + Limits + +
+
Context window
{formatNumber(model.context_window)}
+
Max output
{model.max_output_tokens ? formatNumber(model.max_output_tokens) : '—'}
+ {#if md?.limit?.input}
Max input
{formatNumber(md.limit.input)}
{/if} + {#if limits.prompt_tokens}
Per request · prompt
{formatNumber(limits.prompt_tokens)}
{/if} + {#if limits.completion_tokens}
Per request · completion
{formatNumber(limits.completion_tokens)}
{/if} + {#if model.endpoint && ollama?.context_length}
Native context
{formatNumber(Number(ollama.context_length))}
{/if} +
+ + + + + Default parameters + + {#if defaults.length} +
+ {#each defaults as [k, v] (k)} +
{k}
{String(v)}
+ {/each} +
+ {:else} +

Provider defaults (none published).

+ {/if} + + + + {#if ollama?.capabilities} + + Ollama capabilities +

{(ollama.capabilities as string[]).join(', ')}

 + + {/if} +
+
diff --git a/frontend/src/lib/components/models/ModelComparisonChart.svelte b/frontend/src/lib/components/models/ModelComparisonChart.svelte index 1d5ae7bb..68adad34 100644 --- a/frontend/src/lib/components/models/ModelComparisonChart.svelte +++ b/frontend/src/lib/components/models/ModelComparisonChart.svelte @@ -1,92 +1,62 @@ - +{/if} diff --git a/frontend/src/lib/components/models/ModelDataTab.svelte b/frontend/src/lib/components/models/ModelDataTab.svelte new file mode 100644 index 00000000..1d8394e5 --- /dev/null +++ b/frontend/src/lib/components/models/ModelDataTab.svelte @@ -0,0 +1,78 @@ + + +
+
+ + +
+ + {#each groups as g (g.id)} + {@const entries = Object.entries(g.data).sort(([a], [b]) => a.localeCompare(b))} + + + + {g.title} + {entries.length} keys + {g.source} + + + +
+ + + {#each entries as [key, value] (key)} + + + + + {/each} + +
{key} + {#if isEmpty(value)} + — + {:else if typeof value === 'object'} + 
{JSON.stringify(value, null, 2)}
+ {:else} + {String(value)} + {/if} +
+
+ + + {/each} +
diff --git a/frontend/src/lib/components/models/ModelHeader.svelte b/frontend/src/lib/components/models/ModelHeader.svelte index f6a7d12c..701f3461 100644 --- a/frontend/src/lib/components/models/ModelHeader.svelte +++ b/frontend/src/lib/components/models/ModelHeader.svelte @@ -10,8 +10,9 @@ import Download from '@lucide/svelte/icons/download'; import ProviderIcon from '$lib/components/ProviderIcon.svelte'; import PageHeader from '$lib/components/PageHeader.svelte'; + import type { Snippet } from 'svelte'; import type { LLMModelDetail } from '$lib/api/client'; - import { formatPrice, formatTokens, copyToClipboard } from './helpers'; + import { formatPrice, formatTokens, copyToClipboard, getCaps, getHuggingFace, getModelsDev, isoDay, fmtParams } from './helpers'; import { shortModelName, modelNamePrefix } from '$lib/components/table'; interface Props { @@ -20,10 +21,13 @@ efficiency: { grade: string; color: string }; refreshing: boolean; applicationCount?: number; - onRefresh: () => void; + /** Staff-only metadata refresh; the button is hidden when omitted. */ + onRefresh?: () => void; + /** Section tabs for the header's nav strip. */ + nav?: Snippet; } - let { model, slug, efficiency, refreshing, applicationCount, onRefresh }: Props = $props(); + let { model, slug, efficiency, refreshing, applicationCount, onRefresh, nav: navSnippet }: Props = $props(); function exportJson() { const blob = new Blob([JSON.stringify(model, null, 2)], { type: 'application/json' }); @@ -40,27 +44,37 @@ const lastUpdated = $derived(model.updated_at ? new Date(model.updated_at).toISOString().slice(0, 10) : null); const providerLabel = $derived(modelNamePrefix(model.model_name) || model.provider); - const kpis = $derived<{ label: string; value: string | number; sub?: string; title?: string; class?: string }[]>([ - { label: 'Applications', value: applicationCount ?? '—', sub: 'generated here' }, - { label: 'Context', value: model.context_window_display, sub: 'tokens in' }, - { label: 'Max output', value: model.max_output_tokens ? formatTokens(model.max_output_tokens) : '—', sub: 'tokens out' }, - { - label: 'Price $/1M', - value: model.is_free ? 'Free' : `${formatPrice(model.input_price_per_million)} / ${formatPrice(model.output_price_per_million)}`, - sub: model.is_free ? 'no token cost' : 'input / output', - class: model.is_free ? 'text-success-fg' : '', - }, - { - label: 'Value', - value: efficiency.grade, - sub: `cost efficiency ${(model.cost_efficiency * 100).toFixed(0)}/100`, - title: 'Capability per dollar, A+ best', - class: efficiency.color, - }, - ]); + const md = $derived(getModelsDev(model)); + const hf = $derived(getHuggingFace(model)); + const released = $derived(md?.release_date || isoDay(getCaps(model).created)); + const knowledge = $derived(md?.knowledge || isoDay(getCaps(model).knowledge_cutoff)); + + interface HeaderStat { + label: string; + value: string | number; + title?: string; + } + + const stats = $derived( + ( + [ + { label: 'context', value: model.context_window_display }, + model.max_output_tokens ? { label: 'max out', value: formatTokens(model.max_output_tokens) } : null, + { + label: model.is_free ? 'no token cost' : '$/1M in · out', + value: model.is_free ? 'Free' : `${formatPrice(model.input_price_per_million)} · ${formatPrice(model.output_price_per_million)}`, + }, + hf?.parameters ? { label: 'params', value: fmtParams(hf.parameters) } : null, + released ? { label: 'released', value: released } : null, + knowledge ? { label: 'knowledge', value: knowledge } : null, + { label: 'value', value: efficiency.grade, title: `Cost efficiency ${(model.cost_efficiency * 100).toFixed(0)}/100 — context per dollar, A+ best` }, + { label: applicationCount === 1 ? 'app here' : 'apps here', value: applicationCount ?? '—' }, + ] as (HeaderStat | null)[] + ).filter((s): s is HeaderStat => s !== null) + ); - + {#snippet meta()} {#if model.is_free} @@ -89,6 +103,11 @@ metadata from {/if}
+

+ {#each stats as s (s.label)} + {s.value} {s.label} + {/each} +

{#snippet actions()}
- + {#if onRefresh} + + {/if}
{/snippet} - - -
- {#each kpis as kpi (kpi.label)} -
- {kpi.label} - {kpi.value} - {#if kpi.sub}{kpi.sub}{/if} -
- {/each} -
diff --git a/frontend/src/lib/components/models/ModelOverviewTab.svelte b/frontend/src/lib/components/models/ModelOverviewTab.svelte new file mode 100644 index 00000000..94f65469 --- /dev/null +++ b/frontend/src/lib/components/models/ModelOverviewTab.svelte @@ -0,0 +1,281 @@ + + +
+ {#if expiration} +
+
+ {/if} + +
+
+ + About + +

+ {#if model.description}{stripMarkdown(model.description)}{:else}{model.model_name} by {model.provider}.{/if} +

+ {#if aliasTarget} +

Alias of {aliasTarget} — requests are routed to that model.

+ {/if} +
+ {#if !model.is_local} + + OpenRouter + {/if} + {#if md} + + models.dev + {/if} + {#if hfId} + + Hugging Face + {/if} + {#each hf?.arxiv ?? [] as paper (paper)} + + arXiv:{paper} + {/each} +
+ {#if hf} +

+ {compactNumber(hf.downloads)} downloads / 30 d + {compactNumber(hf.likes)} likes + {#if hf.spaces_count}{compactNumber(hf.spaces_count)} Spaces{/if} + {#if hf.gated}Gated{/if} +

+ {/if} + + + + + + Model card + OpenRouter catalog{md ? ', models.dev' : ''}{hf ? ', Hugging Face Hub' : ''} + + +
+ {#each facts as fact (fact.label)} +
+
{fact.label}
+
+ {#if fact.href} + {fact.value} + {:else}{fact.value}{/if} +
+
+ {/each} +
+ + +
+ +
+ + + On this platform + Your generations with this model + + + {#if insightsError} + + {:else if !insights || !gen} + + {:else if gen.jobs_total === 0} +

Not generated with yet. Run it on a template to measure build success, conformance and code findings.

+ + {:else} +
+
+
Apps
+
{gen.jobs_total}
+
{gen.completed} completed
+
+
+
Build success
+
{fmtRate(gen.success.rate)}
+
{fmtCI(gen.success.ci_low, gen.success.ci_high) || '—'} · n={gen.success.n}
+
+
+
Conformance
+
{fmtRate(gen.functional.rate)}
+
{gen.functional.n ? `${fmtCI(gen.functional.ci_low, gen.functional.ci_high)} · n=${gen.functional.n}` : 'not evaluated'}
+
+
+
Rank
+
{pos?.empirical ? `#${pos.empirical.rank}` : '—'}
+
{pos?.empirical ? `of ${pos.empirical.of} measured models` : 'no measured quality yet'}
+
+
+ + {/if} + + + + + + Catalog position + Against {cat?.catalog_size ?? '…'} OpenRouter models; the centre tick is the median + + + {#if !cat} + {#if !insightsError}{/if} + {:else} + {#if !model.is_free} + + + {/if} + + {#if model.max_output_tokens} + + {/if} + {#each [['coding_index', 'AA coding index'], ['intelligence_index', 'AA intelligence index']] as const as [key, label] (key)} + {@const e = cat.artificial_analysis[key]} + {#if e.value != null} + + {/if} + {/each} +

{cat.provider_models} models from {model.provider} in the catalog.

+ {/if} + + +
+
+ + +
diff --git a/frontend/src/lib/components/models/ModelPricingTab.svelte b/frontend/src/lib/components/models/ModelPricingTab.svelte new file mode 100644 index 00000000..485bf486 --- /dev/null +++ b/frontend/src/lib/components/models/ModelPricingTab.svelte @@ -0,0 +1,263 @@ + + +
+
+ + + List prices + OpenRouter, cheapest routing; providers below may differ + + +
+ + + + {#each priceRows as r (r.key)} + + + + + + {/each} + {#each extraKeys as k (k)} + + {/each} + {#if parseFloat(pricing.discount ?? '0') > 0} + + {/if} + +
TypePriceUnit
{r.label}{perUnit(r.value, r.unit)}{UNIT_LABEL[r.unit]}
{k}${pricing[k]}raw
Discount{fmtRate(parseFloat(pricing.discount))}
+
+ {#if tiers.length} +
+
Long-context tiers (models.dev)
+
    + {#each tiers as t, i (i)} +
  • + above {t.tier?.size ? formatTokens(t.tier.size) : '?'} {t.tier?.type ?? ''}: + {fmtUsd(t.input)} in · {fmtUsd(t.output)} out{#if t.cache_read != null} · {fmtUsd(t.cache_read)} cache read{/if} +
    • + {/each} +
+
+ {/if} + + + + + + Cost calculator + + {#if measured?.mean_prompt_tokens}Seeded with your measured mean per app ({compactNumber(measured.mean_prompt_tokens)} in, {compactNumber(measured.mean_completion_tokens)} out).{:else}At list price.{/if} + + + +
+ + + +
+
+
+ {fmtUsd(calc.inCost)} input + {fmtUsd(calc.outCost)} output +
+
+
Estimate
+
{calc.total === 0 ? 'Free' : fmtUsd(calc.total)}
+
+
+ {#if measured?.mean_cost != null} +

Billed mean per app here: {fmtUsd(measured.mean_cost)} (actual provider prices and cache use).

+ {/if} + + +
+ + {#if !model.is_local} + + +
+ Providers {endpoints?.endpoints.length ?? ''} + + Who serves this model through OpenRouter — live uptime, p50 latency and throughput over the last 30 min + {#if endpoints?.fetched_at}· fetched {new Date(endpoints.fetched_at).toLocaleTimeString()}{/if} + +
+ + + + {#if epError} +
loadEndpoints(true)} />
+ {:else if !endpoints} + + {:else if !endpoints.available} +

OpenRouter's endpoint list is unreachable right now.

+ {:else if endpoints.endpoints.length === 0} +

No provider currently serves this model.

+ {:else} +
+ + + + + + + + + + + + + + + + + {#each endpoints.endpoints as ep (ep.tag || ep.provider_name)} + {@const lat = p50(ep.latency_last_30m)} + {@const tps = p50(ep.throughput_last_30m)} + + + + + + + + + + + + + {/each} + +
ProviderQuant.ContextMax out$ in / outCache readUptime 30mUptime 1dLatencySpeed
+ {ep.provider_name} + {#if ep.status != null && ep.status < 0}degraded{/if} + {ep.tag} + {ep.quantization || '—'}{ep.context_length ? formatTokens(ep.context_length) : '—'}{ep.max_completion_tokens ? formatTokens(ep.max_completion_tokens) : '—'}{formatPrice(ep.input_price_per_million)} / {formatPrice(ep.output_price_per_million)}{ep.cache_read_per_million != null ? formatPrice(ep.cache_read_per_million) : '—'}{uptime(ep.uptime_last_30m)}{uptime(ep.uptime_last_1d)}{lat != null ? `${Math.round(lat)} ms` : '—'}{tps != null ? `${Math.round(tps)} tok/s` : '—'}
+
+ {/if} + + + {/if} +
diff --git a/frontend/src/lib/components/models/ModelResultsTab.svelte b/frontend/src/lib/components/models/ModelResultsTab.svelte new file mode 100644 index 00000000..75d022ba --- /dev/null +++ b/frontend/src/lib/components/models/ModelResultsTab.svelte @@ -0,0 +1,342 @@ + + +{#snippet breakdown(title: string, data: Record, unit: string)} + {@const entries = Object.entries(data)} + {@const max = Math.max(1, ...entries.map(([, n]) => n))} +
+
{title}
+ {#if entries.length === 0} +

—

+ {:else} +
    + {#each entries as [name, n] (name)} +
  • +
    + {name} + {n} +
    + +
    • + {/each} +
+ {/if} +
+{/snippet} + +{#if error} + +{:else if !insights || !gen || !res} + +{:else if gen.jobs_total === 0} + + {#snippet action()} +
+ + +
+ {/snippet} + +{:else} +
+
+ {#each kpis as kpi (kpi.label)} +
+ {kpi.label} + {kpi.value} + {#if kpi.sub}{kpi.sub}{/if} +
+ {/each} +
+

+ Rates show Wilson 95% intervals. Your jobs only{gen.first_job_at ? `, ${gen.first_job_at.slice(0, 10)} – ${gen.last_job_at?.slice(0, 10)}` : ''}. +

+ +
+ + + Ranking + + Same scoring as Rankings; 0–100. + {#if ranking}{ranking.models_exercised} model{ranking.models_exercised === 1 ? '' : 's'} measured.{/if} + + + + {#if !ranking} +

Not in the rankings.

+ {:else} +
+ + + + + + {#each scoreRows as row (row.key)} + + + + + + {/each} + +
ScoreValuePosition
+ {row.label} + {#if row.help}{row.help}{/if} + + {#if row.pos}#{row.pos.rank} / {row.pos.of}{:else}—{/if} +
+
+

+ {ranking.n_trials} trials + {compactNumber(ranking.local_loc)} LOC + {#if density != null} + + {density.toFixed(1)}{#if ranking.variance?.density_per_kloc_stdev != null}± {ranking.variance.density_per_kloc_stdev.toFixed(1)}{/if} findings/KLOC + + {/if} + {#if ranking.variance?.smoke_pass_rate_stdev != null} + {ranking.variance.smoke_pass_rate_stdev.toFixed(2)} pass-rate sd + {/if} +

+ {/if} + + + + + Resource useMeans over completed apps + +
+ {#each resourceFacts as f (f.label)} +
+
{f.label}
+
{f.value}
+
+ {/each} +
+
+ {@render breakdown('Harness', gen.engines, 'apps')} + {@render breakdown('Served by', gen.providers, 'LLM calls')} + {@render breakdown('Termination', gen.terminations, 'apps')} +
+ + +
+ + + + Code findings + + Latest analysis run of each app · {analysis?.analyzed_jobs ?? 0} of {gen.jobs_total} apps analyzed + + + + {#if !analysis || analysis.analyzed_jobs === 0} +
+ + {#snippet action()}{/snippet} + +
+ {:else} +
+ {#each [['Static tools', analysis.findings_static, totalStatic], ['AI reviewer', analysis.findings_ai, totalAi]] as const as [label, counts, total] (label)} +
+
{label} · {total}
+
+ {#each SEVERITIES as sev (sev)} + {counts[sev]} {sev} + {/each} +
+
+ {/each} +
+
+
+ + + + {#each analysis.by_tool as t (t.tool)} + + + + + + + + + {/each} + +
ToolTotalCritHighMedLow
{t.tool}{#if t.ai}AI{/if}{t.total}{t.critical}{t.high}{t.medium}{t.low}
+
+
+ + + + {#each analysis.top_rules as r (`${r.tool}:${r.rule_id}:${r.severity}`)} + + + + + + {:else} + + {/each} + +
Most frequent ruleSeverityCount
+ {r.title} + {r.tool} · {r.rule_id} + {r.severity}{r.count}
No rule ids reported.
+
+
+ {/if} + + + +
+ + + By app spec + Conformance = the hidden API suite passed + + + {#if insights.templates.length === 0} +

No app-spec runs (custom prompts only).

+ {:else} +
+ + + + {#each insights.templates as t (t.slug)} + + + + + + + {/each} + +
App specAppsCompletedConformance
{t.name}{t.slug}{t.jobs}{t.completed} + {#if t.functional.n} + {fmtRate(t.functional.rate)} + {fmtCI(t.functional.ci_low, t.functional.ci_high)} · n={t.functional.n} + {:else}—{/if} +
+
+ {/if} + + + + + + Experiments + Designed experiments this model ran in + + + {#if insights.experiments.length === 0} +

Not part of an experiment yet.

+ {:else} + + {/if} + + +
+
+{/if} diff --git a/frontend/src/lib/components/models/ModelStatsTabs.svelte b/frontend/src/lib/components/models/ModelStatsTabs.svelte deleted file mode 100644 index 94f6f49c..00000000 --- a/frontend/src/lib/components/models/ModelStatsTabs.svelte +++ /dev/null @@ -1,609 +0,0 @@ - - - -
-
- {#each sections as section} - - {/each} -
-
- - -
-

Overview

- -
- - - About - -

- {#if model.description}{stripMarkdown(model.description)}{:else}{model.model_name} by {model.provider}. A large language model available via OpenRouter.{/if} -

-
- - - {#if hfId} - - Hugging Face - {/if} -
- - - - - - Identity - -
-
Model ID
-
{model.model_id}
-
Provider
-
{model.provider}
- {#if hfId} -
Weights
-
{hfId}
- {/if} - {#if inputMods.length || outputMods.length} -
Modalities
-
- {inputMods.join(', ') || 'text'} - → - {outputMods.join(', ') || 'text'} -
- {/if} -
Access
-
- {#if model.is_free}Free{/if} - {#if (meta.openrouter_top_provider as Record)?.is_moderated} - Moderated - {:else} - Unmoderated - {/if} -
-
Metadata
-
{new Date(model.updated_at).toLocaleDateString()}
-
- - -
-
- - -
-

Capabilities

- - - - -
-
- - Skills -
-
- {#each getCoreSkills(model) as skill} - - - {skill.name} - - {/each} -
-
- - - -
- -
-
- - Architecture -
-
-
- Modality: - {arch.modality || meta.architecture_modality || '—'} -
-
- Tokenizer: - {arch.tokenizer || meta.architecture_tokenizer || '—'} -
-
- {#if inputMods.length || outputMods.length} -
- {#each inputMods as mod} - {mod} - {/each} - {#each outputMods as mod} - {mod} - {/each} -
- {/if} -
- - -
-
- - Supported Parameters -
- {#if supportedParams.length} -
- {#each supportedParams.slice(0, 8) as param} - {param} - {/each} - {#if supportedParams.length > 8} - +{supportedParams.length - 8} more - {/if} -
- {:else} - Not published - {/if} -
-
- - - -
- -
-
- - Per-Request Limits -
- {#if Object.keys(perRequestLimits).length} -
- {#if perRequestLimits.prompt_tokens} -
- Prompt: - {formatNumber(perRequestLimits.prompt_tokens)} -
- {/if} - {#if perRequestLimits.completion_tokens} -
- Completion: - {formatNumber(perRequestLimits.completion_tokens)} -
- {/if} -
- {:else} - Not specified - {/if} -
- - - {#if Object.keys(capMatrix).length} - {@const sortedEntries = Object.entries(capMatrix).sort(([a], [b]) => a.localeCompare(b))} - {@const visibleEntries = capMatrixExpanded ? sortedEntries : sortedEntries.slice(0, 8)} -
-
- - Capability Matrix -
-
- {#each visibleEntries as [key, value]} - - {#if value}{:else}{/if} - {key.replace(/_/g, ' ')} - - {/each} - {#if !capMatrixExpanded && sortedEntries.length > 8} - - {:else if capMatrixExpanded && sortedEntries.length > 8} - - {/if} -
-
- {/if} -
- - - {#if instructType || Object.keys(defaultParams).length} - -
- {#if instructType} -
-
- - Instruct Type -
- {instructType} -
- {/if} - {#if Object.keys(defaultParams).length} -
-
- - Default Parameters -
-
- {#each Object.entries(defaultParams) as [key, value]} -
- {key} - {String(value)} -
- {/each} -
-
- {/if} -
- {/if} - - -
- - -
-

Pricing

- -
- - Input - -
{formatPrice(model.input_price_per_million)}
-
per 1M tokens
- - - - Output - -
{formatPrice(model.output_price_per_million)}
-
per 1M tokens
- - - - Efficiency - -
{efficiency.grade}
-
score {model.cost_efficiency.toFixed(2)}
- - - - Tier - -
{model.is_free ? 'Free' : 'Paid'}
-
{model.is_free ? 'No cost' : 'Usage-based'}
- - -
- - - - Pricing Breakdown - -
- - - - - - - - - - - - - - - - - - - - - - - {#each getAllPricingTiers(model) as tier} - - - - - - - {/each} - -
TypePer 1M tokensPer 1K tokensPer token
Input{formatPrice(model.input_price_per_million)}{formatPrice(model.input_price_per_million / 1000)}{model.input_price_per_token === 0 ? 'Free' : `$${model.input_price_per_token.toExponential(2)}`}
Output{formatPrice(model.output_price_per_million)}{formatPrice(model.output_price_per_million / 1000)}{model.output_price_per_token === 0 ? 'Free' : `$${model.output_price_per_token.toExponential(2)}`}
{tier.label}{formatPrice(tier.perMillionTokens)}{formatPrice(tier.perMillionTokens / 1000)}${(tier.perMillionTokens / 1_000_000).toExponential(2)}
-
- - - - - - - Cost Calculator - -
-
- - -
${calcInputCost.toFixed(4)}
-
-
- - -
${calcOutputCost.toFixed(4)}
-
-
-
Estimated Cost
-
{calcTotalCost === 0 ? 'Free' : `$${calcTotalCost.toFixed(4)}`}
-
{calcInputTokens}K in + {calcOutputTokens}K out
-
-
-
- {#each [{ label: '1K', v: 1000 }, { label: '10K', v: 10000 }, { label: '100K', v: 100000 }, { label: '1M', v: 1000000 }] as preset} - - {/each} -
- - -
- - - - - - - - -
-

Metadata

- - - - -
-
- ID: - {model.model_id} -
-
- - - - -
- - {#if Object.keys(caps).length} - - - - - OpenRouter Metadata - {Object.keys(caps).length} keys - - - -
- - - {#each Object.entries(caps).filter(([, v]) => v !== null && v !== '') as [key, value]} - - - - - {/each} - -
{key}{formatMetaValue(value)}
-
- - - {/if} - - - {#if Object.keys(meta).length} - - - - - Enriched Dataset - {Object.keys(meta).length} keys - - - -
- - - {#each Object.entries(meta).filter(([, v]) => v !== null && v !== '') as [key, value]} - - - - - {/each} - -
{key}{formatMetaValue(value)}
-
- - - {/if} -
- - - - - - {#if rawJsonOpen} -
- - 
{JSON.stringify(caps, null, 2)}
-
- {/if} - - -
diff --git a/frontend/src/lib/components/models/ModelUsageSection.svelte b/frontend/src/lib/components/models/ModelUsageSection.svelte index 98a1bfb8..0c1d2e66 100644 --- a/frontend/src/lib/components/models/ModelUsageSection.svelte +++ b/frontend/src/lib/components/models/ModelUsageSection.svelte @@ -1,7 +1,6 @@ + +
+
+ {label} + {value} +
+ {#if pct != null} + + {/if} + {#if hint}

{hint}

{/if} +
diff --git a/frontend/src/lib/components/models/helpers.ts b/frontend/src/lib/components/models/helpers.ts index 8d21bc4f..a889cf69 100644 --- a/frontend/src/lib/components/models/helpers.ts +++ b/frontend/src/lib/components/models/helpers.ts @@ -1,24 +1,5 @@ import type { LLMModelDetail } from "$lib/api/client"; import { toast } from "svelte-sonner"; -import MessageSquare from "@lucide/svelte/icons/message-square"; -import Eye from "@lucide/svelte/icons/eye"; -import Wrench from "@lucide/svelte/icons/wrench"; -import Radio from "@lucide/svelte/icons/radio"; -import Braces from "@lucide/svelte/icons/braces"; -import Mic from "@lucide/svelte/icons/mic"; -import Box from "@lucide/svelte/icons/box"; -import Zap from "@lucide/svelte/icons/zap"; - -export interface SkillBadge { - name: string; - icon: typeof Zap; - active: boolean; -} - -export interface PricingTier { - label: string; - perMillionTokens: number; -} export { formatPrice, formatTokens } from "$lib/utils/formatters"; @@ -54,12 +35,6 @@ export function getArchitecture(m: LLMModelDetail): Record { return (caps.architecture || {}) as Record; } -export function getHuggingFaceId(m: LLMModelDetail): string | null { - const caps = getCaps(m); - const hfId = caps.hugging_face_id as string | undefined; - return hfId || null; -} - export function getSupportedParameters(m: LLMModelDetail): string[] { const caps = getCaps(m); const meta = getMeta(m); @@ -82,94 +57,6 @@ export function getDefaultParameters( return (caps.default_parameters || {}) as Record; } -export function getCoreSkills(m: LLMModelDetail): SkillBadge[] { - const caps = getCaps(m); - const arch = getArchitecture(m); - const inputMods = (arch.input_modalities || []) as string[]; - const outputMods = (arch.output_modalities || []) as string[]; - return [ - { name: "Text/Chat", icon: MessageSquare, active: true }, - { - name: "Vision", - icon: Eye, - active: m.supports_vision || inputMods.includes("image"), - }, - { name: "Functions", icon: Wrench, active: m.supports_function_calling }, - { name: "Streaming", icon: Radio, active: m.supports_streaming }, - { name: "JSON Mode", icon: Braces, active: m.supports_json_mode }, - { - name: "Audio", - icon: Mic, - active: inputMods.includes("audio") || outputMods.includes("audio"), - }, - { - name: "Multimodal", - icon: Box, - active: inputMods.length > 1 || (caps.multimodal as boolean) || false, - }, - ]; -} - -export function getCapabilityMatrix( - m: LLMModelDetail, -): Record { - const caps = getCaps(m); - const result: Record = {}; - for (const [key, value] of Object.entries(caps)) { - if (typeof value === "boolean") { - result[key] = value; - } - } - result["function_calling"] = m.supports_function_calling; - result["vision"] = m.supports_vision; - result["streaming"] = m.supports_streaming; - result["json_mode"] = m.supports_json_mode; - return result; -} - -export function getAllPricingTiers(m: LLMModelDetail): PricingTier[] { - const meta = getMeta(m); - const caps = getCaps(m); - const pricing = (meta.openrouter_pricing || - (caps.pricing as Record) || - {}) as Record; - const tiers: PricingTier[] = []; - const tierMap: Record = { - input_cache_read: "Cache Read", - input_cache_write: "Cache Write", - web_search: "Web Search", - internal_reasoning: "Internal Reasoning", - request: "Per Request", - image: "Per Image", - }; - for (const [key, label] of Object.entries(tierMap)) { - const raw = pricing[key]; - if (raw) { - const val = parseFloat(raw); - if (val > 0) tiers.push({ label, perMillionTokens: val * 1_000_000 }); - } - } - return tiers; -} - -export function formatMetaValue(value: unknown): string { - if (value === null || value === undefined) return "—"; - if (typeof value === "object") { - if (Array.isArray(value)) { - if (value.length === 0) return "—"; - if (value.length > 3) - return `${value.slice(0, 3).join(", ")} (+${value.length - 3})`; - return value.join(", "); - } - return `${Object.keys(value).length} keys`; - } - if (typeof value === "number") { - return value > 1000 ? formatNumber(value) : String(value); - } - const sv = String(value); - return sv.length > 80 ? sv.slice(0, 77) + "..." : sv; -} - export async function copyToClipboard(text: string): Promise { await navigator.clipboard.writeText(text); toast.success("Copied to clipboard"); @@ -199,3 +86,162 @@ export function stripMarkdown(text: string): string { .replace(/#{1,6}\s/g, "") .replace(/`([^`]+)`/g, "$1"); } + +// ── Enrichment payloads (see backend/llm_models/services/enrichment.py) ── + +export interface ModelsDevCost { + input?: number; + output?: number; + cache_read?: number; + cache_write?: number; +} + +/** models.dev record stored in `metadata.models_dev` (all fields optional). */ +export interface ModelsDevInfo { + id?: string; + family?: string; + knowledge?: string; + release_date?: string; + last_updated?: string; + open_weights?: boolean; + reasoning?: boolean; + reasoning_options?: { + type: string; + values?: string[]; + min?: number; + max?: number; + }[]; + tool_call?: boolean; + structured_output?: boolean; + temperature?: boolean; + attachment?: boolean; + modalities?: { input?: string[]; output?: string[] }; + limit?: { context?: number; output?: number; input?: number }; + cost?: ModelsDevCost & { + tiers?: (ModelsDevCost & { tier?: { type?: string; size?: number } })[]; + }; + status?: string; + fetched_at?: string; +} + +/** Hugging Face Hub record stored in `metadata.huggingface`. */ +export interface HuggingFaceInfo { + id: string; + author: string; + license: string; + license_name: string; + pipeline_tag: string; + library_name: string; + downloads: number; + downloads_all_time: number | null; + likes: number; + parameters: number | null; + parameters_by_dtype: Record; + architectures: string[]; + model_type: string; + base_models: string[]; + languages: string[]; + arxiv: string[]; + gated: boolean; + created_at: string | null; + last_modified: string | null; + spaces_count: number; + fetched_at: string; +} + +/** OpenRouter `reasoning` block of the catalog payload. */ +export interface ReasoningInfo { + mandatory?: boolean; + default_effort?: string; + supported_efforts?: string[]; +} + +export interface DesignArenaEntry { + arena: string; + category: string; + elo: number; + rank: number; + win_rate: number; +} + +export interface ArtificialAnalysisIndices { + intelligence_index?: number; + coding_index?: number; + agentic_index?: number; +} + +export function getModelsDev(m: LLMModelDetail): ModelsDevInfo | null { + return (getMeta(m).models_dev as ModelsDevInfo | undefined) ?? null; +} + +export function getHuggingFace(m: LLMModelDetail): HuggingFaceInfo | null { + return (getMeta(m).huggingface as HuggingFaceInfo | undefined) ?? null; +} + +export function getReasoning(m: LLMModelDetail): ReasoningInfo | null { + return (getCaps(m).reasoning as ReasoningInfo | undefined) ?? null; +} + +export function getDesignArena(m: LLMModelDetail): DesignArenaEntry[] { + const b = (getCaps(m).benchmarks || {}) as Record; + return Array.isArray(b.design_arena) + ? (b.design_arena as DesignArenaEntry[]) + : []; +} + +export function getArtificialAnalysis( + m: LLMModelDetail, +): ArtificialAnalysisIndices | null { + const b = (getCaps(m).benchmarks || {}) as Record; + return ( + (b.artificial_analysis as ArtificialAnalysisIndices | undefined) ?? null + ); +} + +/** Unix seconds (OpenRouter `created`) or ISO date → "2026-02-17". */ +export function isoDay(value: unknown): string | null { + if (typeof value === "number" && value > 0) + return new Date(value * 1000).toISOString().slice(0, 10); + if (typeof value === "string" && value) return value.slice(0, 10); + return null; +} + +/** 0–1 rate → "83%" ("—" when null). */ +export function fmtRate(rate: number | null | undefined, digits = 0): string { + return rate == null ? "—" : `${(rate * 100).toFixed(digits)}%`; +} + +/** Wilson interval → "55–95%" ("" when unknown). */ +export function fmtCI( + low: number | null | undefined, + high: number | null | undefined, +): string { + if (low == null || high == null) return ""; + return `${Math.round(low * 100)}–${Math.round(high * 100)}%`; +} + +/** Compact count → "1.4M" / "4.5K" / "12". */ +export function compactNumber(n: number | null | undefined): string { + if (n == null) return "—"; + return new Intl.NumberFormat(undefined, { + notation: "compact", + maximumFractionDigits: 1, + }).format(n); +} + +/** USD amount with precision that scales down for small sums. */ +export function fmtUsd(n: number | null | undefined): string { + if (n == null) return "—"; + if (n === 0) return "$0"; + if (n < 0.01) return `$${n.toFixed(4)}`; + if (n < 100) return `$${n.toFixed(2)}`; + return `$${Math.round(n).toLocaleString()}`; +} + +/** Parameter count → "685B" / "7.2B" / "350M". */ +export function fmtParams(n: number | null | undefined): string { + if (!n) return "—"; + if (n >= 1e12) return `${(n / 1e12).toFixed(1)}T`; + if (n >= 1e9) return `${(n / 1e9).toFixed(n >= 1e11 ? 0 : 1)}B`; + return `${Math.round(n / 1e6)}M`; +} diff --git a/frontend/src/lib/components/table/index.ts b/frontend/src/lib/components/table/index.ts index 6c33c37f..c4254692 100644 --- a/frontend/src/lib/components/table/index.ts +++ b/frontend/src/lib/components/table/index.ts @@ -1,4 +1,5 @@ import { goto } from "$app/navigation"; +import { scrollHint } from "$lib/utils/scroll-hint"; export { default as SortableTh } from "./SortableTh.svelte"; export { default as TableToolbar } from "./TableToolbar.svelte"; @@ -8,9 +9,11 @@ export { default as ScoreCell } from "./ScoreCell.svelte"; * Attachment for `div.table-frame` (`{@attach tableFrame}`). While the table fits * the frame it sets `data-fits`: the page does the scrolling and the header sticks * under the app bar. A wider table keeps the frame as its own bounded scroll box so - * the horizontal scrollbar stays on screen. + * the horizontal scrollbar stays on screen. It also gets `scrollHint()`: an edge + * fade on phones and a tab stop while it scrolls sideways. */ export function tableFrame(node: HTMLElement) { + const detachHint = scrollHint()(node); const update = () => { const table = node.querySelector("table"); node.toggleAttribute( @@ -23,7 +26,10 @@ export function tableFrame(node: HTMLElement) { const table = node.querySelector("table"); if (table) observer.observe(table); update(); - return () => observer.disconnect(); + return () => { + observer.disconnect(); + detachHint(); + }; } const INTERACTIVE = diff --git a/frontend/src/lib/components/templates/BlocksSection.svelte b/frontend/src/lib/components/templates/BlocksSection.svelte index 4b2285c1..5d28fad8 100644 --- a/frontend/src/lib/components/templates/BlocksSection.svelte +++ b/frontend/src/lib/components/templates/BlocksSection.svelte @@ -61,7 +61,6 @@ onCloseEditor={close} > {#snippet toolbar()} - {/snippet} diff --git a/frontend/src/lib/components/templates/LibraryNav.svelte b/frontend/src/lib/components/templates/LibraryNav.svelte index d67eefea..f2b24ce0 100644 --- a/frontend/src/lib/components/templates/LibraryNav.svelte +++ b/frontend/src/lib/components/templates/LibraryNav.svelte @@ -45,15 +45,16 @@