diff --git a/.github/workflows/deploy-dashboard.yml b/.github/workflows/deploy-dashboard.yml new file mode 100644 index 000000000..5abb3d7bc --- /dev/null +++ b/.github/workflows/deploy-dashboard.yml @@ -0,0 +1,35 @@ +name: Deploy dashboard + +on: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write + +concurrency: + group: pages + cancel-in-progress: true + +jobs: + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + steps: + - name: Configure GitHub Pages + uses: actions/configure-pages@v5 + + - name: Upload dashboard site + uses: actions/upload-pages-artifact@v3 + with: + path: dashboard + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/README.md b/README.md index a3b51e7fc..a337dd7a1 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,31 @@ Or jump in instantly: --- +## Interactive Dashboard + +The repository-backed [`dashboard/`](./dashboard/) site reads scenario definitions and seeded data from the repository at request time. It can execute the real Plan-Execute, Claude, OpenAI, and Deep agent runners over the project's MCP servers, display their returned trajectories/tool outputs, and call the repository's `LLMJudgeScorer`. Run results and saved history include an interactive evidence-linked replay of the scenario, recorded agent notes, tool inputs and exact returned outputs, final answer, and evaluation. The replay preserves the recorded sequence and does not imply that an agent relied on a particular tool result. The data cards are derived from checked-in sample manifests and files; they are not a live plant feed. The linked Engineering Lab provides policy-filtered MCP workbenches, runner comparison, bounded local benchmark-profile batches, evaluation summaries, and persisted run history. + +### Run locally + +Install the project dependencies and configure one of the repository-supported model providers plus any needed CouchDB/MCP services. The server reads `.env` at the repository root; keep credentials server-side and never commit them. For example, the selected model and its provider credentials must be configured as described in [INSTRUCTIONS.md](./INSTRUCTIONS.md) and the `src/llm/` backend documentation. + +```bash +uv sync +uv run python dashboard/server.py +``` + +Then open . Without model credentials, you can still browse repository scenarios, local sample data, and dynamically discovered MCP tool definitions, but live model runs and LLM-As-Judge scoring report the missing configuration. Live runs use actual model/MCP services and may incur provider costs. The dashboard forces `AOB_READONLY=1` for spawned WO and FMSR MCP servers; WO write tools and persistent FMSR edits are also excluded from the Engineering Lab tool policy. + +### Dashboard architecture and deployment + +- `dashboard/server.py` reads scenario files, scenario manifests, seeded telemetry/FMEA/work-order/asset records, discovers tools from repository MCP servers, and exposes the localhost JSON API. It instantiates repository agent runners and the `LLMJudgeScorer`. +- `dashboard/index.html`, `dashboard/engineering.html`, the shared `dashboard/styles.css` and `dashboard/replay.js`, and the two page-specific JavaScript files provide the responsive dashboard and linked Engineering Lab. Tool responses and agent results come from repository calls, not synthesized browser fixtures. +- Data summaries are re-read from checked-in files for each API request. Agent runners persist trajectories under `traces/dashboard/`; the Engineering Lab reads these records as local run history. `AGENT_TRAJECTORY_DIR` can override this location, but history is only exposed for directories under the repository's `traces/` folder. +- Benchmark profile definitions are read from the repository YAML files. The dashboard batch endpoint runs at most 20 profile scenarios whose definitions are available in the local checkout and does not reset/load CouchDB. Remaining profile IDs are reported as skipped. The repository's full benchmark CLI has data-reset behavior and is intentionally not launched by this dashboard. +- [`.github/workflows/deploy-dashboard.yml`](./.github/workflows/deploy-dashboard.yml) publishes both static pages to GitHub Pages on pushes to `main` and on manual dispatch. GitHub Pages cannot run the Python API or MCP servers, so Pages provides only the static preview and reports that its repository API is unavailable. For full functionality, run the dashboard server in the same trusted environment as the repository and its data services. The API is unauthenticated and must not be exposed to the public internet. + +--- + ## What is AssetOpsBench? AssetOpsBench is a **unified framework for developing, orchestrating, and evaluating domain-specific AI agents** in industrial asset operations and maintenance. It provides reproducible scenarios, agent tooling, and evaluation pipelines for multi-step workflows in simulated industrial environments. diff --git a/dashboard/app.js b/dashboard/app.js new file mode 100644 index 000000000..faae10f08 --- /dev/null +++ b/dashboard/app.js @@ -0,0 +1,420 @@ +const state = { + scenarios: [], + results: [], + search: "", + filters: { category: new Set(), type: new Set(), asset: new Set(), agent: new Set() }, + expanded: new Set(), + apiAvailable: false +}; + +const byId = (id) => document.getElementById(id); +const escapeHTML = (value) => String(value ?? "").replace(/[&<>"']/g, (character) => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'" +})[character]); +const numberFormat = new Intl.NumberFormat(); + +async function api(path, options = {}) { + const response = await fetch(`${window.ASSETOPSBENCH_API_BASE || ""}${path}`, { + ...options, + headers: { "Content-Type": "application/json", ...options.headers } + }); + const payload = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(payload.error || `Dashboard API returned ${response.status}.`); + return payload; +} + +function setBackendStatus(label, ready = false) { + const status = byId("backend-status"); + status.innerHTML = ` ${escapeHTML(label)}`; + status.classList.toggle("backend-ready", ready); +} + +function uniqueValues(field) { + return [...new Set(state.scenarios.flatMap((scenario) => { + const value = scenario[field]; + return Array.isArray(value) ? value : value ? [value] : []; + }))].sort((a, b) => a.localeCompare(b)); +} + +function renderFilters(id, values, key) { + const container = byId(id); + container.innerHTML = values.map((value) => ``).join(""); + const empty = byId(`${key}-filter-empty`); + if (empty) empty.hidden = values.length > 0; +} + +function renderScenarioOptions() { + const options = state.scenarios.map((scenario) => ``).join(""); + byId("scenario-run-select").innerHTML = options || ''; + const judgeScenarios = state.scenarios.filter((scenario) => scenario.characteristic_form); + byId("evaluation-scenario").innerHTML = judgeScenarios.length + ? judgeScenarios.map((scenario) => ``).join("") + : ''; + byId("run-scenario").disabled = !state.apiAvailable || !state.scenarios.length; + byId("repo-scenario-count").textContent = numberFormat.format(state.scenarios.length); +} + +function renderScenarios() { + const query = state.search.trim().toLowerCase(); + const filtered = state.scenarios.filter((scenario) => { + const searchable = [scenario.title, scenario.text, scenario.category, scenario.type, scenario.source, ...(scenario.assets || []), ...(scenario.agents || [])].join(" ").toLowerCase(); + return searchable.includes(query) + && (!state.filters.category.size || state.filters.category.has(scenario.category)) + && (!state.filters.type.size || state.filters.type.has(scenario.type)) + && (!state.filters.asset.size || (scenario.assets || []).some((asset) => state.filters.asset.has(asset))) + && (!state.filters.agent.size || (scenario.agents || []).some((agent) => state.filters.agent.has(agent))); + }); + if (byId("scenario-sort").value === "title") filtered.sort((a, b) => a.title.localeCompare(b.title)); + + byId("scenario-list").innerHTML = filtered.length ? filtered.map((scenario) => { + const expanded = state.expanded.has(scenario.id); + const tags = [ + scenario.category && `${escapeHTML(scenario.category)}`, + scenario.type && `${escapeHTML(scenario.type)}`, + ...(scenario.agents || []).map((agent) => `${escapeHTML(agent)}`) + ].filter(Boolean).join(""); + return `
+ +
+
REPOSITORY TASK

${escapeHTML(scenario.text)}

SOURCE FILE

${escapeHTML(scenario.source)}

${scenario.agents?.length ? `DECLARED AGENTS
${scenario.agents.map((agent) => `${escapeHTML(agent)}`).join("")}
` : ""}
+ +
+
`; + }).join("") : `
${state.scenarios.length ? "No repository scenarios match those filters." : "No scenario definitions were found in src/scenarios or src/couchdb/scenarios_data."}
`; + byId("scenario-count").textContent = `${filtered.length} of ${state.scenarios.length} repository scenarios`; + byId("scenario-summary").textContent = `Loaded ${state.scenarios.length} definitions directly from repository files · showing ${filtered.length}`; +} + +function refreshScenarioView() { + renderFilters("category-filters", uniqueValues("category"), "category"); + renderFilters("type-filters", uniqueValues("type"), "type"); + renderFilters("asset-filters", uniqueValues("assets"), "asset"); + renderFilters("agent-filters", uniqueValues("agents"), "agent"); + renderScenarios(); +} + +function renderTelemetryChart(values) { + const chart = byId("telemetry-chart"); + if (!values?.length) { + chart.innerHTML = ''; + chart.setAttribute("aria-label", "No numeric sensor readings were found in the checked-in telemetry files."); + return; + } + const min = Math.min(...values); + const max = Math.max(...values); + const range = max - min || 1; + const points = values.map((value, index) => { + const x = values.length === 1 ? 240 : index * 480 / (values.length - 1); + const y = 90 - ((value - min) / range) * 76; + return `${x.toFixed(1)} ${y.toFixed(1)}`; + }); + chart.innerHTML = ``; + chart.setAttribute("aria-label", `Sensor series read from repository data, with ${values.length} sampled values ranging from ${min.toPrecision(4)} to ${max.toPrecision(4)}.`); +} + +function renderOverview(data) { + byId("telemetry-count").textContent = numberFormat.format(data.telemetry.observations); + byId("telemetry-caption").textContent = `numeric sensor observations · ${numberFormat.format(data.telemetry.records)} records`; + byId("telemetry-source").textContent = data.telemetry.sources.map((path) => path.split("/").pop()).join(", ") || "No source files"; + renderTelemetryChart(data.telemetry.series); + + byId("fmea-count").textContent = numberFormat.format(data.fmea.failure_modes); + byId("fmea-caption").textContent = `failure modes in ${numberFormat.format(data.fmea.records)} source records`; + byId("fmea-names").textContent = data.fmea.names.join(" · ") || "No failure modes are present in the referenced local sample."; + byId("fmea-source").textContent = data.fmea.sources.map((path) => path.split("/").pop()).join(", ") || "No source files"; + + byId("workorder-count").textContent = numberFormat.format(data.workorders.records); + const statusEntries = Object.entries(data.workorders.statuses); + byId("workorder-statuses").innerHTML = statusEntries.length + ? statusEntries.map(([status, count], index) => `
${numberFormat.format(count)}${escapeHTML(status)}
`).join("") + : "No records in the repository work-order sample."; + byId("workorder-source").textContent = data.workorders.sources.map((path) => path.split("/").pop()).join(", ") || "No source files"; + + const rate = data.kpis.completion_rate; + byId("kpi-count").textContent = rate === null ? "—" : `${rate}%`; + byId("kpi-caption").textContent = `completed work orders · ${data.kpis.completed} of ${data.kpis.total}`; + byId("kpi-ring").style.background = rate === null + ? "conic-gradient(#26332f 0 100%)" + : `conic-gradient(var(--teal) 0 ${rate}%,#26332f ${rate}% 100%)`; + byId("kpi-ring-label").innerHTML = rate === null ? "—" : `${rate}%`; + byId("kpi-breakdown").innerHTML = [ + `
Repository assets${numberFormat.format(data.assets.records)}
`, + `
Open / awaiting approval${numberFormat.format(data.kpis.open)}
`, + `
Separate alerts datasetNot present
` + ].join(""); + byId("data-source-note").textContent = `${data.source}. Values reflect checked-in files only; this local seed does not represent live plant telemetry.${data.alerts.message}`; +} + +function jsonText(value) { + if (value === null || value === undefined) return "No output recorded."; + return typeof value === "string" ? value : JSON.stringify(value, null, 2); +} + +function renderAgentTrajectory(result) { + const plan = result.plan?.steps?.length + ? `
Generated execution plan · ${result.plan.steps.length} steps
${escapeHTML(jsonText(result.plan))}
` + : ""; + const trajectory = result.trajectory; + let body = ""; + if (Array.isArray(trajectory)) { + body = trajectory.map((step) => `
${escapeHTML(step.server || "Agent")} · ${escapeHTML(step.tool || "step")}${step.error ? "FAILED" : "RETURNED"} +

${escapeHTML(step.task || "")}

${step.tool_args && Object.keys(step.tool_args).length ? `TOOL ARGUMENTS
${escapeHTML(jsonText(step.tool_args))}
` : ""}RETRIEVED DATA / TOOL OUTPUT
${escapeHTML(step.error || step.response || "No output recorded.")}
`).join(""); + } else if (trajectory?.turns) { + body = trajectory.turns.map((turn) => `
Agent turn ${Number(turn.index) + 1}${(turn.tool_calls || []).length} tool calls
+ ${turn.text ? `

${escapeHTML(turn.text)}

` : ""} + ${(turn.tool_calls || []).map((call) => `
TOOL · ${escapeHTML(call.name)}
${escapeHTML(jsonText(call.input))}
TOOL OUTPUT
${escapeHTML(jsonText(call.output))}
`).join("")} +
`).join(""); + } else { + body = `
Raw runner trajectory
${escapeHTML(jsonText(trajectory))}
`; + } + return `${plan}${body || '

The runner returned no structured tool trajectory.

'}`; +} + +function renderRunResult(result) { + const evaluation = result.evaluation; + const evaluationMarkup = evaluation?.error + ? `
LLM-As-Judge could not score this run

${escapeHTML(evaluation.error)}

` + : `
Repository LLM-As-Judge · ${evaluation?.score === undefined ? "no score" : `${Math.round(evaluation.score * 100)}%`}

${escapeHTML(evaluation?.rationale || "See the rubric details below.")}

Rubric result
${escapeHTML(jsonText(evaluation?.details || evaluation))}
`; + byId("execution-output").innerHTML = `
+
REAL RUN · ${escapeHTML(result.runner)} · ${escapeHTML(result.model)}

${escapeHTML(result.scenario_title)}

${escapeHTML(result.answer || "The runner returned no final answer.")}

+ ${evaluationMarkup} + ${window.AssetOpsReplay.render(result)} +
Agent trajectory and tool calls${renderAgentTrajectory(result)}
+
`; +} + +function renderLeaderboard(results) { + byId("leaderboard-empty").hidden = results.length > 0; + byId("leaderboard-body").innerHTML = results.map((result, index) => { + const evaluation = result.evaluation; + const score = evaluation?.score === undefined ? "—" : `${Math.round(evaluation.score * 100)}%`; + return `
${String(index + 1).padStart(2, "0")}${escapeHTML(result.scenario_id)}${escapeHTML(result.scenario_title)}${escapeHTML(result.question)}
${escapeHTML(result.runner)}${escapeHTML(result.model)}${score}${evaluation?.passed ? "Passed" : evaluation?.error ? "Scoring error" : "Review needed"}`; + }).join(""); +} + +async function loadResults() { + const response = await api("/api/results"); + state.results = response.results; + renderLeaderboard(state.results); +} + +async function refreshRepository() { + const button = byId("refresh-repo"); + button.disabled = true; + const original = button.textContent; + button.textContent = "Refreshing…"; + try { + const [scenarioResponse, overview] = await Promise.all([api("/api/scenarios"), api("/api/overview")]); + state.scenarios = scenarioResponse.scenarios; + renderOverview(overview); + renderScenarioOptions(); + refreshScenarioView(); + byId("run-message").textContent = "Repository scenarios and data reloaded from disk."; + } catch (error) { + byId("run-message").textContent = `Repository refresh failed: ${error.message}`; + } finally { + button.disabled = false; + button.textContent = original; + } +} + +async function runScenario(scenarioId) { + if (!scenarioId) return; + const button = byId("run-scenario"); + const runMessage = byId("run-message"); + button.disabled = true; + button.innerHTML = '◌ Running repository agent…'; + byId("console-state").classList.add("running"); + byId("console-state").innerHTML = " RUNNING"; + byId("console-message").textContent = "Calling the selected repository agent runner…"; + byId("execution-output").innerHTML = '

Agent execution is in progress. This calls configured model and MCP services and may take a few minutes.

'; + runMessage.textContent = "Running the real AssetOpsBench pipeline…"; + try { + const result = await api("/api/run", { + method: "POST", + body: JSON.stringify({ scenario_id: scenarioId, runner: byId("runner-select").value }) + }); + renderRunResult(result); + await loadResults(); + runMessage.textContent = "Repository pipeline completed."; + byId("console-message").textContent = "Real agent trajectory and evaluation returned."; + byId("console-state").innerHTML = " COMPLETE"; + } catch (error) { + byId("execution-output").innerHTML = `
${escapeHTML(error.message)}
`; + runMessage.textContent = error.message; + byId("console-message").textContent = "The repository run did not complete."; + byId("console-state").innerHTML = " ERROR"; + } finally { + byId("console-state").classList.remove("running"); + button.disabled = !state.apiAvailable || !state.scenarios.length; + button.innerHTML = '▶ Run scenario ↗'; + } +} + +async function evaluateLog(file) { + const note = byId("judge-note"); + const selectedScenario = byId("evaluation-scenario").value; + if (!selectedScenario) throw new Error("Choose a repository scenario with an evaluation reference first."); + const contents = await file.text(); + let record; + if (/\.jsonl$/i.test(file.name)) { + const records = contents.split(/\r?\n/).filter((line) => line.trim()).map((line) => JSON.parse(line)); + record = records.find((item) => String(item.scenario_id || "") === selectedScenario) || (records.length === 1 ? records[0] : null); + } else { + record = JSON.parse(contents); + } + if (!record || typeof record !== "object" || Array.isArray(record)) throw new Error("No matching trajectory record found in the file."); + note.innerHTML = '◌Scoring with the repository LLM-As-Judge…
The configured model is reviewing the selected scenario and trajectory.'; + const result = await api("/api/evaluate-log", { + method: "POST", + body: JSON.stringify({ scenario_id: selectedScenario, record }) + }); + note.classList.add("file-selected"); + note.innerHTML = `${result.passed ? "✓" : "!"}Repository LLM-As-Judge · ${result.score === undefined ? "no score" : `${Math.round(result.score * 100)}%`}
${escapeHTML(result.rationale || JSON.stringify(result.details || {}))}`; +} + +byId("scenario-search").addEventListener("input", (event) => { + state.search = event.target.value; + renderScenarios(); +}); +document.querySelector(".filter-panel").addEventListener("click", (event) => { + const button = event.target.closest("[data-filter]"); + if (!button) return; + const selected = state.filters[button.dataset.filter]; + selected.has(button.dataset.value) ? selected.delete(button.dataset.value) : selected.add(button.dataset.value); + refreshScenarioView(); +}); +byId("scenario-list").addEventListener("click", (event) => { + const runButton = event.target.closest("[data-run-id]"); + if (runButton) { + event.stopPropagation(); + byId("scenario-run-select").value = runButton.dataset.runId; + byId("runner-select").focus(); + byId("demo").scrollIntoView({ behavior: "smooth", block: "start" }); + runScenario(runButton.dataset.runId); + return; + } + const toggle = event.target.closest("[data-scenario]"); + if (!toggle) return; + const id = toggle.dataset.scenario; + state.expanded.has(id) ? state.expanded.delete(id) : state.expanded.add(id); + renderScenarios(); +}); +byId("scenario-sort").addEventListener("change", renderScenarios); +byId("refresh-repo").addEventListener("click", refreshRepository); +byId("clear-filters").addEventListener("click", () => { + state.search = ""; + Object.values(state.filters).forEach((set) => set.clear()); + byId("scenario-search").value = ""; + refreshScenarioView(); +}); +byId("teach-toggle").checked = localStorage.getItem("aob-teach-mode") === "on"; +document.body.classList.toggle("teach-mode", byId("teach-toggle").checked); +byId("teach-toggle").addEventListener("change", (event) => { + document.body.classList.toggle("teach-mode", event.target.checked); + localStorage.setItem("aob-teach-mode", event.target.checked ? "on" : "off"); +}); + +document.querySelectorAll(".paradigm-tab").forEach((tab) => { + tab.addEventListener("click", () => { + document.querySelectorAll(".paradigm-tab").forEach((other) => { + const active = other === tab; + other.classList.toggle("active", active); + other.setAttribute("aria-selected", String(active)); + const panel = byId(other.getAttribute("aria-controls")); + panel.hidden = !active; + panel.classList.toggle("active", active); + }); + }); +}); + +document.querySelectorAll(".agent-node").forEach((node) => { + const setActive = (active) => { + node.classList.toggle("is-active", active); + document.querySelector(`[data-agent-card="${node.dataset.agent}"]`)?.classList.toggle("is-active", active); + }; + node.addEventListener("mouseenter", () => setActive(true)); + node.addEventListener("mouseleave", () => setActive(false)); + node.addEventListener("focus", () => setActive(true)); + node.addEventListener("blur", () => setActive(false)); +}); + +byId("run-scenario").addEventListener("click", () => runScenario(byId("scenario-run-select").value)); +byId("scenario-run-select").addEventListener("change", (event) => { byId("run-scenario").disabled = !event.target.value || !state.apiAvailable; }); + +const fileInput = byId("log-upload"); +const fileDrop = byId("file-drop"); +const judgeNote = byId("judge-note"); +async function showFile(file) { + if (!file) return; + if (!/\.(json|jsonl)$/i.test(file.name)) { + judgeNote.textContent = "Unsupported file type. Choose a JSON or JSONL trajectory log."; + fileInput.value = ""; + return; + } + try { + await evaluateLog(file); + } catch (error) { + judgeNote.textContent = `Evaluation failed: ${error.message}`; + } +} +fileInput.addEventListener("change", () => showFile(fileInput.files[0])); +fileDrop.addEventListener("keydown", (event) => { + if (event.key === "Enter" || event.key === " ") { event.preventDefault(); fileInput.click(); } +}); +["dragenter", "dragover"].forEach((eventName) => fileDrop.addEventListener(eventName, (event) => { + event.preventDefault(); + fileDrop.classList.add("drag-over"); +})); +["dragleave", "drop"].forEach((eventName) => fileDrop.addEventListener(eventName, (event) => { + event.preventDefault(); + fileDrop.classList.remove("drag-over"); +})); +fileDrop.addEventListener("drop", (event) => showFile(event.dataTransfer.files[0])); + +async function initializeDashboard() { + try { + const status = await api("/api/status"); + state.apiAvailable = true; + setBackendStatus("REPOSITORY API", true); + byId("run-message").textContent = status.ready + ? `Ready · ${status.model} · live model/MCP calls may use external services` + : `Run setup needed · configure ${status.missing_environment.join(", ") || status.agent_import_error || "repository dependencies"}`; + document.querySelectorAll(".paradigm-tab").length && (byId("runner-select").innerHTML = status.runners.map((runner) => ``).join("")); + const [scenarioResponse, overview] = await Promise.all([api("/api/scenarios"), api("/api/overview")]); + state.scenarios = scenarioResponse.scenarios; + renderOverview(overview); + renderScenarioOptions(); + refreshScenarioView(); + await loadResults(); + byId("console-message").textContent = `Loaded ${state.scenarios.length} repository scenario files; data values read from checked-in samples.`; + } catch (error) { + state.apiAvailable = false; + setBackendStatus("API OFFLINE"); + renderScenarioOptions(); + byId("scenario-list").innerHTML = `
Cannot read repository files because the dashboard API is unavailable. Start it with uv run python dashboard/server.py. ${escapeHTML(error.message)}
`; + byId("scenario-summary").textContent = "Repository API unavailable"; + byId("run-message").textContent = error.message; + byId("console-message").textContent = "Start the repository-backed dashboard server."; + } +} + +if ("IntersectionObserver" in window) { + const revealObserver = new IntersectionObserver((entries) => { + entries.forEach((entry) => { + if (entry.isIntersecting) { + entry.target.classList.add("is-visible"); + revealObserver.unobserve(entry.target); + } + }); + }, { threshold: 0.08 }); + document.querySelectorAll(".section-block, .demo-section").forEach((section) => { + section.classList.add("reveal-section"); + revealObserver.observe(section); + }); +} + +initializeDashboard(); diff --git a/dashboard/engineering.html b/dashboard/engineering.html new file mode 100644 index 000000000..34264380b --- /dev/null +++ b/dashboard/engineering.html @@ -0,0 +1,124 @@ + + + + + + + Engineering Lab · AssetOpsBench + + + + + + +
+ + + AssetOpsBench + + +
+ + GitHub +
+
+
+
+
REPOSITORY-CONNECTED ENGINEERING WORKSPACE
+

Go beyond the dashboard.

+

Inspect the real MCP server tools, compare repository agent runners on the same scenario, launch a bounded set of locally available benchmark scenarios, and revisit persisted trajectories.

+
Open a tool workbench How to use this lab← Main dashboard
+
Local execution: This page calls the repository's local Python API and MCP servers. Model runs may incur provider costs. Work-order writes and persistent FMSR edits are not enabled.
+
Connecting to the local repository API…
+
+ +
+
01 / REAL MCP TOOLS

Tool workbench.

Tool names and parameter contracts are discovered from the repository's running MCP server modules. Results are returned by the selected tool, not generated by the page.

+
+
+ + +

Choose a domain to discover the tools available in its MCP server.

+ + +

Use repository-relative data file paths when a tool requests one. MCP server or data-service errors are shown in the result panel.

+
+
+
LIVE TOOL RESPONSE

Waiting for a tool call

MCP · REPOSITORY
+ 
Select a workbench and tool to get started. The tool's response will appear here.
+
+
+
+ +
+
02 / ORCHESTRATION DEBUGGER

Compare agent runners.

Run the same repository scenario through multiple actual runner implementations, then compare returned answers, trajectories, and judge scores.

+
+
+ + +
+
Runners to compare (choose 2–4) + + + + +
+
Model credentials are required for live runs.
+

Run a comparison to inspect the real responses and evaluation results side by side.

+
+
+ +
+
03 / BENCHMARK PROFILES

Launch a local scenario batch.

Profiles are read from the project's benchmark YAML files. The dashboard runs only scenarios whose definitions are present in this checkout, without resetting or loading CouchDB.

+
+
+ + +
+
Loading profiles from the repository…
+ +
One batch is limited to 20 live scenario runs.
+
+
+
Profile coverage note: benchmark profiles can reference scenarios from the external benchmark dataset and loader. Those are listed as unavailable in this local scenario-file view and are not run here. The repository's full benchmark CLI may reset and reload CouchDB; this dashboard does not invoke that operation.
+
+ +
+
04 / OBSERVABILITY

Run & trajectory history.

Agent runners persist their real trajectory records under traces/dashboard/. History is read locally and includes the answer, runner, model, scenario, and execution trace.

+
Loading saved runs…
+

Evaluation summary will use completed runs that include a repository judge score.

+

Loading persisted dashboard runs…

+
+ +
+
FIELD GUIDE

How to use the Engineering Lab.

Each panel is a direct window into repository components; connect the local services you need before running them.

+
+
01

Start locally

From the repository root, run uv run python dashboard/server.py. Keep the API on localhost; it is not authenticated for public hosting.

+
02

Explore tools safely

Select a real MCP server and inspect its discovered tool signatures. Calls use repository tools directly; WO tools are read-only and FMSR edits are excluded.

+
03

Compare and evaluate

Pick a scenario and two to four runners. Agent outputs use real model/MCP calls; LLM-As-Judge runs only when the scenario includes a reference answer.

+
04

Understand batch limits

Profiles come from the benchmark YAMLs, but only locally discovered scenarios are run here. A batch is capped at 20 scenarios and does not reset CouchDB.

+
+

Configure model-provider credentials and any required data services in the server environment. Never put secrets in browser code or commit them. Runs can incur API costs; tool, dependency, and data-service errors are displayed as returned or raised by the repository.

+
+
+ + + diff --git a/dashboard/engineering.js b/dashboard/engineering.js new file mode 100644 index 000000000..467478e62 --- /dev/null +++ b/dashboard/engineering.js @@ -0,0 +1,350 @@ +const $ = (id) => document.getElementById(id); +const API_ROOT = "/api"; +let statusInfo = null; +let toolsByName = new Map(); +let profiles = []; + +async function api(path, options = {}) { + const response = await fetch(`${API_ROOT}${path}`, { + ...options, + headers: { "Content-Type": "application/json", ...(options.headers || {}) }, + }); + const result = await response.json(); + if (!response.ok) throw new Error(result.error || `Request failed (${response.status})`); + return result; +} + +function showError(target, error) { + target.textContent = error instanceof Error ? error.message : String(error); + target.classList.add("is-error"); +} + +function setTeachMode(enabled) { + document.body.classList.toggle("teach-mode", enabled); + localStorage.setItem("aob-teach-mode", enabled ? "on" : "off"); +} + +function renderRun(run, compact = false) { + const article = document.createElement("article"); + article.className = compact ? "run-card compact-run-card" : "run-card"; + const heading = document.createElement("div"); + heading.className = "run-card-heading"; + const title = document.createElement("h3"); + title.textContent = run.scenario_title || `Scenario ${run.scenario_id || "unknown"}`; + const meta = document.createElement("span"); + meta.textContent = `${run.runner || "Repository run"} · ${run.model || "model unavailable"}`; + heading.append(title, meta); + article.append(heading); + if (run.evaluation) { + const evaluation = document.createElement("p"); + evaluation.className = "run-evaluation"; + const score = run.evaluation.score; + evaluation.textContent = score === undefined + ? `LLM judge: ${run.evaluation.error || run.evaluation.rationale || "No score returned"}` + : `LLM judge score: ${score}${run.evaluation.passed === true ? " · passed" : run.evaluation.passed === false ? " · not passed" : ""}`; + article.append(evaluation); + } + const answer = document.createElement("p"); + answer.className = "run-answer"; + answer.textContent = run.answer || "No answer was persisted for this record."; + article.append(answer); + const replay = document.createElement("div"); + replay.innerHTML = window.AssetOpsReplay.render(run); + article.append(replay.firstElementChild); + const details = document.createElement("details"); + const summary = document.createElement("summary"); + summary.textContent = "Inspect raw run record"; + const pre = document.createElement("pre"); + pre.className = "lab-output history-output"; + pre.textContent = JSON.stringify({ + question: run.question, + plan: run.plan, + trajectory: run.trajectory, + evaluation: run.evaluation, + run_id: run.run_id, + }, null, 2); + details.append(summary, pre); + article.append(details); + return article; +} + +function renderRunCollection(container, runs, failures = []) { + container.replaceChildren(); + if (!runs.length && !failures.length) { + const empty = document.createElement("p"); + empty.className = "execution-empty"; + empty.textContent = "No local scenarios were run."; + container.append(empty); + return; + } + if (failures.length) { + const errors = document.createElement("div"); + errors.className = "lab-warning"; + errors.textContent = failures.map((item) => `${item.runner || item.scenario_id}: ${item.error}`).join(" · "); + container.append(errors); + } + runs.forEach((run) => container.append(renderRun(run, true))); +} + +async function loadTools(serverName) { + const select = $("tool-name"); + select.disabled = true; + $("call-tool").disabled = true; + $("tool-description").textContent = "Discovering tool definitions from the repository MCP server…"; + $("tool-result").textContent = "Waiting for the repository MCP server response…"; + $("tool-result").classList.remove("is-error"); + try { + const response = await api(`/tools?server=${encodeURIComponent(serverName)}`); + toolsByName = new Map(response.tools.map((tool) => [tool.name, tool])); + select.replaceChildren(); + response.tools.forEach((tool) => { + const option = document.createElement("option"); + option.value = tool.name; + option.textContent = tool.name; + select.append(option); + }); + select.disabled = response.tools.length === 0; + $("call-tool").disabled = response.tools.length === 0; + if (response.tools.length) updateToolDescription(); + else $("tool-description").textContent = "No tools are enabled by the dashboard's read-only tool policy."; + } catch (error) { + select.replaceChildren(new Option("Unable to load tools", "")); + $("tool-description").textContent = "The MCP server could not be reached or initialized."; + showError($("tool-result"), error); + } +} + +function updateToolDescription() { + const tool = toolsByName.get($("tool-name").value); + if (!tool) return; + $("tool-description").textContent = tool.description || "No description is provided by this repository tool."; + const required = tool.parameters.filter((parameter) => parameter.required); + $("tool-arguments").value = required.length + ? JSON.stringify(Object.fromEntries(required.map(({ name, type }) => [name, type === "integer" || type === "number" ? 0 : type === "boolean" ? false : type === "object" || type === "array" ? (type === "array" ? [] : {}) : ""])), null, 2) + : "{}"; +} + +async function loadScenarios() { + try { + const { scenarios } = await api("/scenarios"); + for (const id of ["compare-scenario"]) { + const select = $(id); + select.replaceChildren(); + scenarios.forEach((scenario) => { + const option = document.createElement("option"); + option.value = scenario.id; + option.textContent = `${scenario.id} · ${scenario.title}`; + select.append(option); + }); + } + } catch (error) { + $("compare-scenario").replaceChildren(new Option("Scenario API unavailable", "")); + $("compare-status").textContent = error.message; + } +} + +function selectedRunners() { + return [...document.querySelectorAll(".runner-options input:checked")].map((input) => input.value); +} + +async function loadProfiles() { + const select = $("benchmark-profile"); + try { + const response = await api("/benchmark/profiles"); + profiles = response.profiles; + select.replaceChildren(); + profiles.forEach((profile) => select.append(new Option(profile.id, profile.id))); + updateBenchmarkDetail(); + } catch (error) { + select.replaceChildren(new Option("Benchmark profiles unavailable", "")); + showError($("benchmark-detail"), error); + } +} + +function updateBenchmarkDetail() { + const profile = profiles.find((item) => item.id === $("benchmark-profile").value); + const detail = $("benchmark-detail"); + if (!profile) return; + detail.classList.remove("is-error"); + const localMatches = profile.locally_available.length; + detail.textContent = `${profile.id.toUpperCase()} · ${profile.scenario_count} IDs in ${profile.source} · ${localMatches} locally runnable scenario${localMatches === 1 ? "" : "s"} · ${profile.scenario_count - localMatches} require the external benchmark scenario loader.`; + $("run-benchmark").disabled = !statusInfo?.ready || localMatches === 0 || localMatches > 20; + $("benchmark-status").textContent = !statusInfo?.ready + ? `Configure credentials for ${statusInfo?.missing_environment?.join(", ") || "the selected model"} before running.` + : localMatches > 20 + ? "This profile exceeds the dashboard's 20-scenario batch limit." + : localMatches === 0 + ? "No profile scenario IDs match scenario definitions in this checkout." + : `Runs ${localMatches} actual scenario${localMatches === 1 ? "" : "s"}; model and judge calls may incur costs.`; +} + +async function refreshHistory() { + const count = $("history-count"); + const container = $("run-history-list"); + count.textContent = "Loading saved runs…"; + try { + const response = await api("/history"); + container.replaceChildren(); + response.runs.forEach((run) => container.append(renderRun(run))); + renderEvaluationSummary(response.runs); + count.textContent = `${response.runs.length} saved repository run${response.runs.length === 1 ? "" : "s"}`; + if (!response.runs.length) { + const empty = document.createElement("p"); + empty.className = "execution-empty"; + empty.textContent = "No persisted dashboard trajectories yet. Run an agent scenario to create one."; + container.append(empty); + } + } catch (error) { + count.textContent = "History unavailable"; + container.replaceChildren(); + showError(container, error); + } +} + +function renderEvaluationSummary(runs) { + const container = $("evaluation-summary"); + container.replaceChildren(); + const scored = runs.filter((run) => typeof run.evaluation?.score === "number"); + const mean = scored.length + ? scored.reduce((total, run) => total + run.evaluation.score, 0) / scored.length + : null; + const metrics = [ + ["Saved trajectories", String(runs.length)], + ["Repository-judge scores", String(scored.length)], + ["Mean score", mean === null ? "—" : `${Math.round(mean * 100)}%`], + ]; + metrics.forEach(([label, value]) => { + const card = document.createElement("div"); + card.className = "evaluation-metric"; + const name = document.createElement("span"); + name.textContent = label; + const number = document.createElement("strong"); + number.textContent = value; + card.append(name, number); + container.append(card); + }); + const byRunner = new Map(); + scored.forEach((run) => { + const group = byRunner.get(run.runner) || []; + group.push(run.evaluation.score); + byRunner.set(run.runner, group); + }); + if (byRunner.size) { + const comparison = document.createElement("div"); + comparison.className = "runner-score-summary"; + [...byRunner.entries()].sort(([left], [right]) => left.localeCompare(right)).forEach(([runner, scores]) => { + const item = document.createElement("span"); + const average = scores.reduce((sum, score) => sum + score, 0) / scores.length; + item.textContent = `${runner}: ${Math.round(average * 100)}% mean · ${scores.length} scored`; + comparison.append(item); + }); + container.append(comparison); + } +} + +$("teach-toggle").checked = localStorage.getItem("aob-teach-mode") === "on"; +setTeachMode($("teach-toggle").checked); +$("teach-toggle").addEventListener("change", (event) => setTeachMode(event.target.checked)); +$("tool-server").addEventListener("change", (event) => loadTools(event.target.value)); +$("tool-name").addEventListener("change", updateToolDescription); +$("call-tool").addEventListener("click", async () => { + const output = $("tool-result"); + $("call-tool").disabled = true; + $("tool-result-title").textContent = `${$("tool-name").value} · calling…`; + output.classList.remove("is-error"); + try { + const args = JSON.parse($("tool-arguments").value || "{}"); + if (!args || typeof args !== "object" || Array.isArray(args)) throw new Error("Tool arguments must be a JSON object."); + const result = await api("/tool-call", { + method: "POST", + body: JSON.stringify({ server: $("tool-server").value, tool: $("tool-name").value, arguments: args }), + }); + $("tool-result-title").textContent = `${result.server} / ${result.tool}`; + output.textContent = typeof result.output === "string" ? result.output : JSON.stringify(result.output, null, 2); + } catch (error) { + $("tool-result-title").textContent = "Tool call failed"; + showError(output, error); + } finally { + $("call-tool").disabled = !$("tool-name").value; + } +}); +$("compare-runners").addEventListener("click", async () => { + const button = $("compare-runners"); + const runners = selectedRunners(); + button.disabled = true; + $("compare-status").textContent = "Executing selected repository agent runners…"; + try { + const model = $("compare-model").value.trim(); + const result = await api("/compare", { + method: "POST", + body: JSON.stringify({ scenario_id: $("compare-scenario").value, runners, ...(model ? { model } : {}) }), + }); + renderRunCollection($("compare-results"), result.runs, result.failures); + $("compare-status").textContent = `${result.runs.length} runner${result.runs.length === 1 ? "" : "s"} completed; ${result.failures.length} failed.`; + refreshHistory(); + } catch (error) { + $("compare-status").textContent = error.message; + showError($("compare-results"), error); + } finally { + button.disabled = !statusInfo?.ready || selectedRunners().length < 2 || selectedRunners().length > 4; + } +}); +document.querySelectorAll(".runner-options input").forEach((input) => input.addEventListener("change", () => { + const count = selectedRunners().length; + $("compare-runners").disabled = !statusInfo?.ready || count < 2 || count > 4 || !$("compare-scenario").value; +})); +$("compare-scenario").addEventListener("change", () => { + $("compare-runners").disabled = !statusInfo?.ready || selectedRunners().length < 2 || !$("compare-scenario").value; +}); +$("benchmark-profile").addEventListener("change", updateBenchmarkDetail); +$("run-benchmark").addEventListener("click", async () => { + const button = $("run-benchmark"); + button.disabled = true; + $("benchmark-status").textContent = "Running repository scenarios sequentially; this may take time…"; + $("benchmark-results").replaceChildren(); + try { + const model = $("benchmark-model").value.trim(); + const result = await api("/benchmark/run", { + method: "POST", + body: JSON.stringify({ + profile: $("benchmark-profile").value, + runner: $("benchmark-runner").value, + ...(model ? { model } : {}), + }), + }); + renderRunCollection($("benchmark-results"), result.runs, result.failures); + if (result.skipped_scenario_ids.length) { + const skipped = document.createElement("p"); + skipped.className = "lab-help"; + skipped.textContent = `Not run (not present as local scenario definitions): ${result.skipped_scenario_ids.join(", ")}`; + $("benchmark-results").append(skipped); + } + $("benchmark-status").textContent = `${result.runs.length} completed, ${result.failures.length} failed; no data reset performed.`; + refreshHistory(); + } catch (error) { + $("benchmark-status").textContent = error.message; + } finally { + updateBenchmarkDetail(); + } +}); +$("refresh-history").addEventListener("click", refreshHistory); + +async function initialize() { + try { + statusInfo = await api("/status"); + $("lab-connection").textContent = statusInfo.ready + ? `Connected · ${statusInfo.scenarios} repository scenarios · model ${statusInfo.model}` + : `Repository API connected · ${statusInfo.scenarios} scenarios · missing credentials: ${statusInfo.missing_environment.join(", ")}`; + $("lab-connection").classList.toggle("is-warning", !statusInfo.ready); + $("compare-runners").disabled = !statusInfo.ready; + } catch (error) { + $("lab-connection").textContent = `Local repository API unavailable: ${error.message}. Start it with uv run python dashboard/server.py.`; + $("lab-connection").classList.add("is-warning"); + } + await Promise.all([loadTools($("tool-server").value), loadScenarios(), loadProfiles(), refreshHistory()]); + if (statusInfo) { + $("compare-runners").disabled = !statusInfo.ready || selectedRunners().length < 2 || !$("compare-scenario").value; + } +} + +initialize(); diff --git a/dashboard/index.html b/dashboard/index.html new file mode 100644 index 000000000..daff1065a --- /dev/null +++ b/dashboard/index.html @@ -0,0 +1,212 @@ + + + + + + + + AssetOpsBench — Industrial AI, made understandable + + + + + + +
+ + + AssetOpsBench + + +
+ + GitHub +
+
+ +
+
+
+
INDUSTRIAL AI, WITHOUT THE BLACK BOX
+

Make every
asset smarter.

+

A benchmark for building and evaluating AI agents that help teams understand equipment, predict failures, and coordinate maintenance.

+
+ Explore the live demo + Browse scenarios +
+
+
—local scenario files
+
5domain agents
+
9asset classes
+
+
+
+
FIG. 02 / AGENT INTERACTIONS CONNECTING
+ +
One question. The right experts. A grounded answer.
+
+
+
IoT

Finds the asset and reads its sensors.

↗
+
TSFM

Finds patterns in data over time.

↗
+
FMSR

Connects symptoms to possible failures.

↗
+
WO

Turns a finding into a work order.

↗
+
+
+ +
+
+
01 / SCENARIO EXPLORER

Real work. Measurable outcomes.

Explore benchmark tasks across industrial assets, from simple lookups to multi-agent maintenance workflows.

+ 0 scenarios +
+
+ +
+
Loading scenario files…
+
+

Scenario text is loaded from checked-in repository files. The full scenario catalogue may also be available through the configured benchmark data services.

+
+
+
+ +
+
+
02 / DATA MODALITIES

One operations picture.
Many useful signals.

Agents combine sensor telemetry, failure analysis, work-order history, and operational KPIs to produce actionable recommendations.

+ REPOSITORY DATA +
+
+
+
⌁SENSOR TELEMETRY iLOCAL FILES
+
—loading repository series…
+ +
First sampleRepository telemetryLatest sample
+
+
+
⌬FMEA RECORDS i● curated
+
—loading repository records…
+
Reading the checked-in failure-mode records…
+
FMRepository FMEA records
+
+
+
▤WORK ORDERS iSEEDED FILES
+
—checked-in work orders
+
Loading repository statuses…
+
WORepository work-order records
+
+
+
◷WORK-ORDER KPI iDERIVED
+
—derived from checked-in work orders
+
—
Loading repository-derived counts…
+
+
+

Values are read from checked-in repository files, not live plant systems. Separate alert records are not present in the local scenario manifests.

+
+ +
+
+
03 / AGENT ARCHITECTURE

Two ways to get to the answer.

Compare two orchestration patterns used to coordinate domain tools and agent reasoning.

+
+
+
+ + +
+
+
↗
PATTERN A

Call in the right specialist.

A lead agent delegates focused subtasks to domain agents and synthesizes their results.

✓Great forFlexible, mixed-domain questions
−Trade-offMore delegation to coordinate
+
✳CoordinatorUnderstands the request
IoTTSFMWO
↖   insights returned
✓ One grounded answer
+
+ +
+ +
+
+
04 / LEADERBOARD & EVALUATION

Good answers. Verified results.

Review completed repository agent runs and their actual LLM-As-Judge outputs.

+ REAL RUNS ONLY +
+
+
+
Completed agent runsCurrent server session
LLM-As-Judge uses the repository scorer.
+
RUN / SCENARIOORCHESTRATORMODELJUDGE SCORERESULT
+
No real scenario runs have been completed in this server session.
+
+
+
⇧OPTIONAL
+

Bring your agent logs

+

Drop a real repository trajectory JSON/JSONL and score it with AssetOpsBench’s LLM-as-Judge.

+ + +
✳Repository evaluation
The actual AssetOpsBench LLMJudgeScorer will assess the selected scenario and trajectory.
+
+
+
+ +
+
05 / LIVE REPOSITORY RUN

Run the actual agents.

Select a scenario file and call the repository's Plan-Execute or Claude agent runner over its configured MCP servers. The response includes the returned trajectory and repository LLM-As-Judge score.

◉ Uses configured models and MCP services; external calls may incur provider costs
+
+
ASSETOPS / REPOSITORY EXECUTION WAITING
+
+ + +
Waiting for the repository API…
+

Select a repository scenario and run it to inspect the real answer, tool calls, retrieved outputs, and evaluation.

+
+
Connected to local dashboard server when available.REPO MODULES · MCP
+
+
+ +
+
START HERE

How to use the dashboard.

Everything shown here comes from the checkout's scenario files, sample data, agent modules, and evaluation code.

+
+
01

Explore repository data

Browse the discovered scenarios and inspect the checked-in telemetry, FMEA, and work-order data summaries.

+
02

Run an agent

Choose a scenario and an orchestration runner. The local API calls repository MCP tools and returns the actual answer and trajectory.

+
03

Review the evidence

Expand the result to inspect tool activity and retrieved outputs. The repository's LLM judge is used only when a reference answer exists.

+
+

For engineers: Open the Engineering Lab for direct MCP tool workbenches, agent comparison, benchmark profiles, evaluation, and saved run history.

Open Engineering Lab
+

Run uv run python dashboard/server.py from the repository root for the full local experience. Live model calls require credentials and may incur provider costs. The dashboard API is intended for localhost only.

+
+
+ + + diff --git a/dashboard/replay.js b/dashboard/replay.js new file mode 100644 index 000000000..df3d6ba27 --- /dev/null +++ b/dashboard/replay.js @@ -0,0 +1,195 @@ +(() => { + const escapeHTML = (value) => String(value ?? "").replace(/[&<>"']/g, (character) => ({ + "&": "&", "<": "<", ">": ">", '"': """, "'": "'" + })[character]); + const jsonText = (value) => typeof value === "string" ? value : JSON.stringify(value, null, 2) ?? "No output recorded."; + + function eventMarkup(event, index) { + const evidence = event.kind === "evidence"; + const detailMarkup = evidence + ? `
Inspect tool input and exact returned output + ${event.input !== undefined ? `TOOL INPUT · ${escapeHTML(event.tool)}
${escapeHTML(jsonText(event.input))}
` : ""} + TOOL RESPONSE · ${escapeHTML(event.tool)}
${escapeHTML(event.output)}
+
` + : event.detail + ? `
${escapeHTML(event.detailLabel || "Inspect details")}
${escapeHTML(jsonText(event.detail))}
` + : ""; + return `
  • + +
    + + ${event.description ? `

    ${escapeHTML(event.description)}

    ` : ""} + ${detailMarkup} +
    +
    • `; + } + + function eventsFor(run) { + const events = [{ + kind: "scenario", + label: "SCENARIO", + title: run.question || `Scenario ${run.scenario_id || ""}`, + description: run.scenario_id ? `Repository scenario ${run.scenario_id}` : "Scenario prompt", + }]; + if (run.plan) { + events.push({ + kind: "plan", + label: "EXECUTION PLAN", + title: "Plan returned by the runner", + detailLabel: "RECORDED PLAN", + detail: run.plan, + }); + } + + const trajectory = run.trajectory; + let evidenceCount = 0; + if (Array.isArray(trajectory)) { + trajectory.forEach((step, index) => { + if (step.task) { + events.push({ kind: "note", label: `AGENT NOTE · STEP ${index + 1}`, title: step.task }); + } + const tool = [step.server, step.tool].filter(Boolean).join(" · ") || `Agent step ${index + 1}`; + events.push({ + kind: "evidence", + label: step.error ? "TOOL ERROR" : "RETRIEVED EVIDENCE", + title: tool, + tool, + input: step.tool_args, + output: jsonText(step.error || step.response || "No tool response was recorded."), + }); + evidenceCount += 1; + }); + } else if (Array.isArray(trajectory?.turns)) { + trajectory.turns.forEach((turn, index) => { + if (typeof turn.text === "string" && turn.text.trim()) { + events.push({ + kind: "note", + label: `AGENT NOTE · TURN ${Number(turn.index ?? index) + 1}`, + title: turn.text, + }); + } + (Array.isArray(turn.tool_calls) ? turn.tool_calls : []).forEach((call) => { + const tool = call.name || "Unnamed tool"; + events.push({ + kind: "evidence", + label: call.error ? "TOOL ERROR" : "RETRIEVED EVIDENCE", + title: tool, + tool, + input: call.input, + output: call.error || (call.output === undefined || call.output === null + ? "No tool response was recorded." + : jsonText(call.output)), + }); + evidenceCount += 1; + }); + }); + } else if (trajectory !== undefined && trajectory !== null) { + events.push({ + kind: "note", + label: "RECORDED TRAJECTORY", + title: "This runner returned an unstructured trajectory", + detailLabel: "RAW TRAJECTORY", + detail: trajectory, + }); + } + + events.push({ + kind: "answer", + label: "FINAL RECOMMENDATION", + title: run.answer || "The runner returned no final answer.", + }); + if (run.evaluation) { + const evaluation = run.evaluation; + const score = typeof evaluation.score === "number" ? ` · ${Math.round(evaluation.score * 100)}%` : ""; + events.push({ + kind: "evaluation", + label: "REPOSITORY EVALUATION", + title: `${evaluation.passed === true ? "Passed" : evaluation.passed === false ? "Needs review" : "Evaluation returned"}${score}`, + description: evaluation.rationale || evaluation.error || "", + detail: evaluation.details, + detailLabel: "EVALUATION DETAILS", + }); + } + return { events, evidenceCount }; + } + + function render(run) { + const { events, evidenceCount } = eventsFor(run); + const runLabel = run.run_id ? `RUN ${run.run_id}` : `${run.runner || "AGENT"} · ${run.model || "MODEL"}`; + return `
    +
    EVIDENCE-LINKED REPLAY

    Trace the run, step by step.

    ${evidenceCount} tool response${evidenceCount === 1 ? "" : "s"}
    +

    Ordered record of the prompt, agent notes, tool inputs and returned outputs, recommendation, and evaluation. Tool outputs are shown as returned; their presence does not prove the agent relied on them.

    +
    + + + + Step 1 of ${events.length} · ${escapeHTML(events[0].title)} +
    +
      ${events.map(eventMarkup).join("")}
    +
    ${escapeHTML(runLabel)}${run.created_at ? ` · ${escapeHTML(run.created_at)}` : ""}
    +
    `; + } + + const timers = new WeakMap(); + function selectStep(panel, index) { + const steps = [...panel.querySelectorAll("[data-replay-step]")]; + if (!steps.length) return; + const selected = Math.max(0, Math.min(index, steps.length - 1)); + steps.forEach((step, position) => { + const active = position === selected; + step.classList.toggle("is-current", active); + const button = step.querySelector("[data-replay-select]"); + if (active) button.setAttribute("aria-current", "step"); + else button.removeAttribute("aria-current"); + }); + const step = steps[selected]; + const status = panel.querySelector("[data-replay-status]"); + status.textContent = `Step ${selected + 1} of ${steps.length} · ${step.querySelector(".replay-step-title").textContent}`; + const details = step.querySelector("details"); + if (details) details.open = true; + } + + function stop(panel) { + const timer = timers.get(panel); + if (timer) clearInterval(timer); + timers.delete(panel); + const play = panel.querySelector('[data-replay-action="play"]'); + if (play) play.textContent = "▶ Play replay"; + } + + document.addEventListener("click", (event) => { + const panel = event.target.closest(".evidence-replay"); + if (!panel) return; + const select = event.target.closest("[data-replay-select]"); + const action = event.target.closest("[data-replay-action]"); + if (select) { + stop(panel); + selectStep(panel, Number(select.dataset.replaySelect)); + return; + } + if (!action) return; + const steps = [...panel.querySelectorAll("[data-replay-step]")]; + const current = Math.max(0, steps.findIndex((step) => step.classList.contains("is-current"))); + if (action.dataset.replayAction === "prev" || action.dataset.replayAction === "next") { + stop(panel); + selectStep(panel, current + (action.dataset.replayAction === "prev" ? -1 : 1)); + } else if (timers.has(panel)) { + stop(panel); + } else { + action.textContent = "Ⅱ Pause replay"; + timers.set(panel, setInterval(() => { + const active = [...panel.querySelectorAll("[data-replay-step]")].findIndex((step) => step.classList.contains("is-current")); + if (active >= steps.length - 1) { + stop(panel); + } else { + selectStep(panel, active + 1); + } + }, 1500)); + } + }); + + document.addEventListener("DOMContentLoaded", () => { + document.querySelectorAll(".evidence-replay").forEach((panel) => selectStep(panel, 0)); + }); + window.AssetOpsReplay = { render }; +})(); diff --git a/dashboard/server.py b/dashboard/server.py new file mode 100644 index 000000000..35c49ca70 --- /dev/null +++ b/dashboard/server.py @@ -0,0 +1,712 @@ +"""Local web/API server that exposes the repository's real AssetOpsBench modules.""" + +from __future__ import annotations + +import asyncio +import csv +import dataclasses +import json +import logging +import math +import os +import sys +import threading +import uuid +from datetime import datetime, timezone +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, urlparse + +ROOT = Path(__file__).resolve().parent.parent +SOURCE = ROOT / "src" +DASHBOARD = Path(__file__).resolve().parent +SCENARIO_ROOTS = (SOURCE / "scenarios", SOURCE / "couchdb" / "scenarios_data") +DEFAULT_MODEL = "litellm_proxy/aws/claude-opus-4-6" +MAX_REQUEST_BYTES = 2_000_000 +sys.path.insert(0, str(SOURCE)) + +logging.basicConfig(level=os.environ.get("DASHBOARD_LOG_LEVEL", "INFO")) +LOG = logging.getLogger("assetopsbench.dashboard") + +RUNS: list[dict[str, Any]] = [] +RUNS_LOCK = threading.Lock() +TOOL_ALLOWLIST = { + "iot": None, + "tsfm": { + "list_tasks", "profile_series", "characterize_series", "data_quality", + "list_features", "list_models", "search_models", "find_models", + "describe_candidates", "describe_models", "count_models", "list_domains", + "get_model_lineage", "resolve_model", "hf_model_stats", "count_features", + "describe_features", "extract_features", "select_features", + "search_feature_catalog", "get_feature", "get_feature_lineage", + "recipe_template", "run_recipe", "run_tabular_recipe", "evaluate", + "get_result", "list_results", "get_run", "list_runs", + }, + "fmsr": {"get_failure_modes", "generate_failure_modes"}, + "wo": { + "list_workorders", "get_workorder", "get_workorder_tasks", + "get_workorder_costs", "get_workorder_actuals_vs_planned", + "get_workorder_kpis", "get_schedule_calendar", + "get_my_assigned_workorders", "get_failure_codes", + }, + "vibration": None, +} +MAX_TOOL_RESULT_CHARS = 1_000_000 + + +def _read_json(path: Path) -> Any: + with path.open(encoding="utf-8") as source: + return json.load(source) + + +def _scenario_from_record(record: dict[str, Any], source_path: Path) -> dict[str, Any] | None: + text = record.get("text") or record.get("question") + if not text or record.get("id") is None: + return None + raw_assets = record.get("assets", record.get("asset", [])) + if isinstance(raw_assets, str): + raw_assets = [raw_assets] + if not isinstance(raw_assets, list): + raw_assets = [] + return { + "id": str(record["id"]), + "title": str(record.get("title") or text.splitlines()[0][:110]), + "text": str(text), + "category": str(record.get("category") or "Repository scenario"), + "type": str(record.get("type") or ""), + "agents": [str(agent) for agent in record.get("agents", [])], + "assets": [str(asset) for asset in raw_assets if asset], + "characteristic_form": record.get("characteristic_form") or record.get("expected_answer") or "", + "source": source_path.relative_to(ROOT).as_posix(), + } + + +def load_scenarios() -> list[dict[str, Any]]: + """Read scenario definitions from their checked-in JSON and question files.""" + found: dict[str, dict[str, Any]] = {} + for root in SCENARIO_ROOTS: + if not root.is_dir(): + continue + for path in sorted(root.rglob("*.json")): + if any(part.startswith(".") for part in path.relative_to(root).parts): + continue + try: + content = _read_json(path) + except (OSError, json.JSONDecodeError) as exc: + LOG.warning("Skipping unreadable scenario candidate %s: %s", path, exc) + continue + records = content if isinstance(content, list) else [content] + for record in records: + if not isinstance(record, dict): + continue + scenario = _scenario_from_record(record, path) + if scenario: + found[scenario["id"]] = scenario + + for question_path in sorted(root.glob("scenario_*/question.txt")): + folder = question_path.parent + manifest_path = folder / "manifest.json" + try: + question = question_path.read_text(encoding="utf-8").strip() + manifest = _read_json(manifest_path) if manifest_path.is_file() else {} + groundtruth_path = folder / "groundtruth.txt" + groundtruth = groundtruth_path.read_text(encoding="utf-8").strip() if groundtruth_path.is_file() else "" + except (OSError, json.JSONDecodeError) as exc: + LOG.warning("Skipping incomplete scenario %s: %s", folder, exc) + continue + if not question: + continue + scenario_id = folder.name.removeprefix("scenario_") + found.setdefault(scenario_id, { + "id": scenario_id, + "title": question.splitlines()[0][:110], + "text": question, + "category": "CouchDB scenario", + "type": "Repository scenario", + "agents": [], + "assets": [], + "characteristic_form": groundtruth, + "manifest": manifest, + "source": question_path.relative_to(ROOT).as_posix(), + }) + return sorted(found.values(), key=lambda scenario: (scenario["category"].lower(), scenario["id"])) + + +def _manifest_files() -> list[tuple[Path, dict[str, Any]]]: + output = [] + data_root = SOURCE / "couchdb" / "scenarios_data" + for manifest_path in sorted(data_root.glob("*/manifest.json")): + try: + manifest = _read_json(manifest_path) + except (OSError, json.JSONDecodeError) as exc: + LOG.warning("Skipping unreadable data manifest %s: %s", manifest_path, exc) + continue + output.append((manifest_path.parent, manifest)) + return output + + +def _data_path(base: Path, relative: str) -> Path | None: + allowed_root = (SOURCE / "couchdb" / "scenarios_data").resolve() + for root in (base, base.parent): + candidate = (root / relative).resolve() + if candidate.is_relative_to(allowed_root) and candidate.is_file(): + return candidate + return None + + +def _iter_manifest_paths(manifest: dict[str, Any], key: str) -> list[str]: + value = manifest.get(key, []) + return [value] if isinstance(value, str) else [item for item in value if isinstance(item, str)] + + +def _json_rows(path: Path) -> list[dict[str, Any]]: + value = _read_json(path) + records = value if isinstance(value, list) else [value] + return [record for record in records if isinstance(record, dict)] + + +def load_data_overview() -> dict[str, Any]: + """Read the repository's seeded data and derive truthful dashboard metrics.""" + manifests = _manifest_files() + workorder_paths: set[Path] = set() + iot_paths: set[Path] = set() + failure_paths: set[Path] = set() + missing_sources: list[str] = [] + for base, manifest in manifests: + for field, paths in (("workorder", workorder_paths), ("iot", iot_paths), ("failure_mode", failure_paths)): + for value in _iter_manifest_paths(manifest, field): + path = _data_path(base, value) + if path: + paths.add(path) + else: + missing_sources.append(f"{base.relative_to(ROOT) / value}") + if missing_sources: + raise FileNotFoundError(f"Scenario manifests reference missing repository data: {', '.join(missing_sources)}") + if not iot_paths or not workorder_paths or not failure_paths: + missing = [ + name for name, files in ( + ("IoT telemetry", iot_paths), + ("work-order", workorder_paths), + ("FMEA", failure_paths), + ) if not files + ] + raise FileNotFoundError(f"No repository sample source found for: {', '.join(missing)}") + + readings = 0 + chart_values: list[float] = [] + telemetry_records = 0 + for path in sorted(iot_paths): + rows = _json_rows(path) + telemetry_records += len(rows) + for row in rows: + measurements = [ + value for key, value in row.items() + if key not in {"asset_id", "assetnum", "timestamp", "siteid"} + and isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + ] + readings += len(measurements) + if not chart_values and measurements: + chart_values.append(float(measurements[0])) + elif len(chart_values) < 48 and measurements: + chart_values.append(float(measurements[0])) + + failure_records = 0 + failure_modes = 0 + for path in sorted(failure_paths): + for row in _json_rows(path): + failure_records += 1 + modes = row.get("failure_modes") + failure_modes += len(modes) if isinstance(modes, list) else 1 + + workorders: list[dict[str, str]] = [] + for path in sorted(workorder_paths): + with path.open(encoding="utf-8", newline="") as source: + workorders.extend(dict(row) for row in csv.DictReader(source)) + statuses: dict[str, int] = {} + for order in workorders: + status = (order.get("status") or "Unknown").strip().upper() + statuses[status] = statuses.get(status, 0) + 1 + completed = sum(count for status, count in statuses.items() if status in {"COMP", "CLOSE", "CLOSED"}) + completion_rate = round(100 * completed / len(workorders), 1) if workorders else None + + asset_paths = { + path for base, manifest in manifests + for value in _iter_manifest_paths(manifest, "asset") + if (path := _data_path(base, value)) + } + assets = sum(len(_json_rows(path)) for path in asset_paths) + return { + "telemetry": { + "records": telemetry_records, + "observations": readings, + "series": chart_values, + "sources": [path.relative_to(ROOT).as_posix() for path in sorted(iot_paths)], + }, + "fmea": { + "records": failure_records, + "failure_modes": failure_modes, + "names": [ + name + for path in sorted(failure_paths) + for row in _json_rows(path) + for name in row.get("failure_modes", []) + if isinstance(name, str) + ], + "sources": [path.relative_to(ROOT).as_posix() for path in sorted(failure_paths)], + }, + "workorders": { + "records": len(workorders), + "statuses": statuses, + "sources": [path.relative_to(ROOT).as_posix() for path in sorted(workorder_paths)], + }, + "kpis": { + "completed": completed, + "total": len(workorders), + "completion_rate": completion_rate, + "open": sum(count for status, count in statuses.items() if status in {"WAPPR", "APPR"}), + "sources": [path.relative_to(ROOT).as_posix() for path in sorted(workorder_paths)], + }, + "assets": {"records": assets}, + "alerts": {"available": False, "message": "No separate alert collection is included in the checked-in sample manifests."}, + "source": "Checked-in scenario manifests and sample data files", + } + + +def _required_environment(model: str) -> list[str]: + if model.startswith("watsonx/"): + names = ("WATSONX_APIKEY", "WATSONX_PROJECT_ID") + elif model.startswith("tokenrouter/"): + names = ("TOKENROUTER_BASE_URL", "TOKENROUTER_API_KEY") + else: + names = ("LITELLM_BASE_URL", "LITELLM_API_KEY") + return [name for name in names if not os.environ.get(name)] + + +def get_status() -> dict[str, Any]: + model = os.environ.get("ASSETOPSBENCH_DASHBOARD_MODEL", DEFAULT_MODEL) + judge_model = os.environ.get("ASSETOPSBENCH_DASHBOARD_JUDGE_MODEL", model) + missing = sorted(set(_required_environment(model) + _required_environment(judge_model))) + try: + from agent.runner import DEFAULT_SERVER_PATHS + servers = sorted(DEFAULT_SERVER_PATHS) + except ImportError as exc: + servers = [] + import_error = str(exc) + else: + import_error = None + return { + "repo_root": str(ROOT), + "model": model, + "judge_model": judge_model, + "missing_environment": missing, + "runners": [ + {"id": "plan-execute", "label": "Plan-Execute"}, + {"id": "claude-agent", "label": "Claude Agent"}, + {"id": "openai-agent", "label": "OpenAI Agent"}, + {"id": "deep-agent", "label": "Deep Agent"}, + ], + "servers": servers, + "agent_modules_available": import_error is None, + "agent_import_error": import_error, + "scenarios": len(load_scenarios()), + "ready": not missing and import_error is None, + } + + +def _plain(value: Any) -> Any: + if dataclasses.is_dataclass(value): + return dataclasses.asdict(value) + if hasattr(value, "model_dump"): + return value.model_dump(mode="json") + if isinstance(value, Path): + return str(value) + return str(value) + + +def _trajectory_as_text(value: Any) -> str: + return json.dumps(value, default=_plain, ensure_ascii=False, indent=2) + + +def _run_real_agent(question: str, runner_name: str, model: str) -> tuple[str, Any, str]: + from llm import make_backend + + os.environ["AOB_READONLY"] = "1" + os.environ.setdefault("FMSR_MODEL_ID", model) + if runner_name == "plan-execute": + from agent.plan_execute.runner import PlanExecuteRunner + + runner = PlanExecuteRunner(llm=make_backend(model)) + result = asyncio.run(runner.run(question)) + return result.answer, result, model + if runner_name == "claude-agent": + from agent.claude_agent.runner import ClaudeAgentRunner + + runner = ClaudeAgentRunner(model=model) + result = asyncio.run(runner.run(question)) + return result.answer, result, model + if runner_name == "openai-agent": + from agent.openai_agent.runner import OpenAIAgentRunner + + runner = OpenAIAgentRunner(model=model) + result = asyncio.run(runner.run(question)) + return result.answer, result, model + if runner_name == "deep-agent": + from agent.deep_agent.runner import DeepAgentRunner + + runner = DeepAgentRunner(model=model) + result = asyncio.run(runner.run(question)) + return result.answer, result, model + raise ValueError("runner must be one of: plan-execute, claude-agent, openai-agent, deep-agent.") + + +def evaluate_result(scenario: dict[str, Any], answer: str, trajectory: Any, model: str) -> dict[str, Any]: + from evaluation.models import Scenario + from evaluation.scorers.llm_judge import LLMJudgeScorer + from llm import make_backend + + expected = scenario.get("characteristic_form", "") + evaluation_scenario = Scenario.from_raw({ + "id": scenario["id"], + "text": scenario["text"], + "category": scenario.get("category", ""), + "type": scenario.get("type", ""), + "characteristic_form": expected, + }) + if not expected: + return { + "scorer": "llm_judge", + "passed": False, + "error": "This repository scenario has no characteristic_form or expected_answer for LLM-As-Judge evaluation.", + } + judge_model = os.environ.get("ASSETOPSBENCH_DASHBOARD_JUDGE_MODEL", model) + missing = _required_environment(judge_model) + if missing: + return {"scorer": "llm_judge", "passed": False, "error": f"Missing judge-model environment: {', '.join(missing)}"} + score = LLMJudgeScorer(make_backend(judge_model))(evaluation_scenario, answer, _trajectory_as_text(trajectory)) + result = score.model_dump(mode="json") + result["model"] = judge_model + if score.rationale.startswith(("judge backend error:", "judge returned unparseable JSON")): + result.pop("score", None) + result["error"] = score.rationale + return result + + +def run_scenario(scenario_id: str, runner_name: str, model: str | None = None) -> dict[str, Any]: + scenario = next((item for item in load_scenarios() if item["id"] == scenario_id), None) + if scenario is None: + raise LookupError(f"Scenario '{scenario_id}' is not present in the repository scenario files.") + selected_model = (model or os.environ.get("ASSETOPSBENCH_DASHBOARD_MODEL", DEFAULT_MODEL)).strip() + if not selected_model or len(selected_model) > 200: + raise ValueError("A valid model ID is required.") + missing = _required_environment(selected_model) + if missing: + raise RuntimeError(f"Configure the model credentials in the server environment: {', '.join(missing)}") + + run_id = uuid.uuid4().hex + from observability.runspan import set_run_context + + set_run_context(run_id=run_id, scenario_id=scenario_id) + answer, result, used_model = _run_real_agent(scenario["text"], runner_name, selected_model) + trajectory = result.trajectory if hasattr(result, "trajectory") else None + serialized = _plain(trajectory) if dataclasses.is_dataclass(trajectory) else ( + [_plain(step) for step in trajectory] if isinstance(trajectory, list) else trajectory + ) + try: + score = evaluate_result(scenario, answer, serialized, used_model) + except Exception as exc: + LOG.exception("Evaluation failed for dashboard run, scenario=%s", scenario_id) + score = {"scorer": "llm_judge", "passed": False, "error": str(exc)} + run = { + "run_id": run_id, + "scenario_id": scenario["id"], + "scenario_title": scenario["title"], + "created_at": datetime.now(timezone.utc).isoformat(), + "runner": runner_name, + "model": used_model, + "question": scenario["text"], + "answer": answer, + "trajectory": serialized, + "plan": _plain(result.plan) if hasattr(result, "plan") else None, + "evaluation": score, + "is_real_run": True, + } + with RUNS_LOCK: + RUNS.insert(0, run) + del RUNS[20:] + return run + + +def evaluate_uploaded_log(scenario_id: str, record: dict[str, Any]) -> dict[str, Any]: + scenario = next((item for item in load_scenarios() if item["id"] == scenario_id), None) + if scenario is None: + raise LookupError(f"Scenario '{scenario_id}' is not present in the repository scenario files.") + model = str(record.get("model") or os.environ.get("ASSETOPSBENCH_DASHBOARD_MODEL", DEFAULT_MODEL)) + missing = _required_environment(model) + if missing: + raise RuntimeError(f"Configure the model credentials in the server environment: {', '.join(missing)}") + answer = record.get("answer") + if not isinstance(answer, str) or not answer.strip(): + raise ValueError("The evaluation log must contain an 'answer' string.") + trajectory = record.get("trajectory") + if trajectory is None: + raise ValueError("The evaluation log must contain a 'trajectory'.") + return evaluate_result(scenario, answer, trajectory, model) + + +def discover_tools(server_name: str) -> list[dict[str, Any]]: + from agent.runner import DEFAULT_SERVER_PATHS + from agent.plan_execute.executor import _list_tools + + if server_name not in TOOL_ALLOWLIST or server_name not in DEFAULT_SERVER_PATHS: + raise ValueError(f"Unknown or unsupported read-only MCP server: {server_name}") + os.environ["AOB_READONLY"] = "1" + tools = asyncio.run(_list_tools(DEFAULT_SERVER_PATHS[server_name])) + allowed = TOOL_ALLOWLIST[server_name] + return [ + tool for tool in tools + if allowed is None or tool["name"] in allowed + ] + + +def call_repository_tool(server_name: str, tool_name: str, arguments: dict[str, Any]) -> dict[str, Any]: + from agent.runner import DEFAULT_SERVER_PATHS + from agent.plan_execute.executor import _call_tool + + if server_name not in TOOL_ALLOWLIST or server_name not in DEFAULT_SERVER_PATHS: + raise ValueError(f"Unknown or unsupported read-only MCP server: {server_name}") + allowed = TOOL_ALLOWLIST[server_name] + if allowed is not None and tool_name not in allowed: + raise ValueError(f"Tool '{tool_name}' is not enabled in the dashboard's read-only tool policy.") + available = {tool["name"] for tool in discover_tools(server_name)} + if tool_name not in available: + raise LookupError(f"Tool '{tool_name}' is not registered by the repository's {server_name} MCP server.") + os.environ["AOB_READONLY"] = "1" + output = asyncio.run(_call_tool(DEFAULT_SERVER_PATHS[server_name], tool_name, arguments)) + return {"server": server_name, "tool": tool_name, "arguments": arguments, "output": output[:MAX_TOOL_RESULT_CHARS]} + + +def benchmark_profiles() -> list[dict[str, Any]]: + from benchmark.scenario_suite_runner import SCENARIO_PROFILE_PATHS, scenario_ids_for_selector + + available = {scenario["id"] for scenario in load_scenarios()} + profiles = [] + for name, path in SCENARIO_PROFILE_PATHS.items(): + ids = scenario_ids_for_selector(name) + profiles.append({ + "id": name, + "source": path.relative_to(ROOT).as_posix(), + "scenario_count": len(ids), + "scenario_ids": ids, + "locally_available": [scenario_id for scenario_id in ids if scenario_id in available], + "requires_data_reset": True, + }) + return profiles + + +def run_comparison(scenario_id: str, runners: list[str], model: str | None = None) -> dict[str, Any]: + selected = list(dict.fromkeys(runners)) + if len(selected) < 2 or len(selected) > 4: + raise ValueError("Select between two and four repository agent runners to compare.") + completed = [] + failures = [] + for runner in selected: + try: + completed.append(run_scenario(scenario_id, runner, model)) + except Exception as exc: + LOG.exception("Agent comparison run failed: runner=%s scenario=%s", runner, scenario_id) + failures.append({"runner": runner, "error": str(exc)}) + if not completed and failures: + raise RuntimeError("; ".join(f"{item['runner']}: {item['error']}" for item in failures)) + return {"runs": completed, "failures": failures} + + +def run_local_profile(profile_name: str, runner_name: str, model: str | None = None) -> dict[str, Any]: + profiles = {profile["id"]: profile for profile in benchmark_profiles()} + if profile_name not in profiles: + raise ValueError("Choose a repository benchmark profile: mini, lite, open, or all.") + profile = profiles[profile_name] + available_ids = profile["locally_available"] + if not available_ids: + raise ValueError( + f"The {profile_name} profile has no scenario definitions in the local repository checkout. " + "The remaining profile scenarios require the benchmark's dataset and scenario loader." + ) + if len(available_ids) > 20: + raise ValueError( + f"The {profile_name} profile matches {len(available_ids)} local scenarios. " + "This dashboard limits one launch to 20 scenario runs; choose a smaller profile." + ) + results = [] + failures = [] + for scenario_id in available_ids: + try: + results.append(run_scenario(scenario_id, runner_name, model)) + except Exception as exc: + LOG.exception("Benchmark scenario failed: profile=%s scenario=%s", profile_name, scenario_id) + failures.append({"scenario_id": scenario_id, "error": str(exc)}) + return { + "profile": profile_name, + "source": profile["source"], + "runner": runner_name, + "runs": results, + "failures": failures, + "skipped_scenario_ids": [item for item in profile["scenario_ids"] if item not in available_ids], + "data_reset_performed": False, + } + + +def load_run_history() -> list[dict[str, Any]]: + history_dir = Path(os.environ.get("AGENT_TRAJECTORY_DIR", str(ROOT / "traces" / "dashboard"))).resolve() + if not history_dir.is_relative_to(ROOT / "traces") or not history_dir.is_dir(): + return [] + persisted_runs = [] + for path in sorted(history_dir.glob("*.json"), key=lambda candidate: candidate.stat().st_mtime, reverse=True): + try: + record = _read_json(path) + if isinstance(record, dict) and record.get("run_id"): + persisted_runs.append(record) + except (OSError, json.JSONDecodeError) as exc: + LOG.warning("Skipping unreadable saved run %s: %s", path, exc) + with RUNS_LOCK: + current_runs = [dict(record) for record in RUNS] + known_ids = {record.get("run_id") for record in current_runs} + runs = current_runs + [record for record in persisted_runs if record.get("run_id") not in known_ids] + return runs[:100] + + +class DashboardHandler(SimpleHTTPRequestHandler): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, directory=str(DASHBOARD), **kwargs) + + def do_GET(self) -> None: + parsed = urlparse(self.path) + route = parsed.path + try: + if route == "/api/status": + self._send_json(200, get_status()) + elif route == "/api/scenarios": + self._send_json(200, {"scenarios": load_scenarios()}) + elif route == "/api/overview": + self._send_json(200, load_data_overview()) + elif route == "/api/results": + with RUNS_LOCK: + self._send_json(200, {"results": list(RUNS)}) + elif route == "/api/tools": + server_name = parse_qs(parsed.query).get("server", [""])[0] + self._send_json(200, {"server": server_name, "tools": discover_tools(server_name)}) + elif route == "/api/benchmark/profiles": + self._send_json(200, {"profiles": benchmark_profiles()}) + elif route == "/api/history": + self._send_json(200, {"runs": load_run_history()}) + else: + super().do_GET() + except Exception as exc: + LOG.exception("GET %s failed", route) + self._send_json(500, {"error": str(exc)}) + + def do_POST(self) -> None: + route = urlparse(self.path).path + if route not in {"/api/run", "/api/evaluate-log", "/api/tool-call", "/api/compare", "/api/benchmark/run"}: + self._send_json(404, {"error": "Unknown API endpoint."}) + return + try: + content_length = int(self.headers.get("Content-Length", "0")) + if content_length <= 0 or content_length > MAX_REQUEST_BYTES: + raise ValueError(f"Request body must be between 1 and {MAX_REQUEST_BYTES} bytes.") + payload = json.loads(self.rfile.read(content_length)) + if not isinstance(payload, dict): + raise ValueError("The request body must be a JSON object.") + if route == "/api/run": + scenario_id = str(payload.get("scenario_id") or "").strip() + if not scenario_id or len(scenario_id) > 100: + raise ValueError("A valid repository scenario_id is required.") + runner = str(payload.get("runner") or "plan-execute") + model = payload.get("model") + if model is not None and not isinstance(model, str): + raise ValueError("model must be a string.") + result = run_scenario(scenario_id, runner, model) + elif route == "/api/evaluate-log": + scenario_id = str(payload.get("scenario_id") or "").strip() + if not scenario_id or len(scenario_id) > 100: + raise ValueError("A valid repository scenario_id is required.") + record = payload.get("record") + if not isinstance(record, dict): + raise ValueError("record must be a JSON object.") + result = evaluate_uploaded_log(scenario_id, record) + elif route == "/api/tool-call": + server_name = payload.get("server") + tool_name = payload.get("tool") + arguments = payload.get("arguments", {}) + if not isinstance(server_name, str) or not isinstance(tool_name, str): + raise ValueError("server and tool must be strings.") + if not isinstance(arguments, dict): + raise ValueError("arguments must be a JSON object.") + result = call_repository_tool(server_name, tool_name, arguments) + elif route == "/api/compare": + scenario_id = str(payload.get("scenario_id") or "").strip() + runners = payload.get("runners") + model = payload.get("model") + if not scenario_id or len(scenario_id) > 100: + raise ValueError("A valid repository scenario_id is required.") + if not isinstance(runners, list) or any(not isinstance(item, str) for item in runners): + raise ValueError("runners must be a list of agent runner IDs.") + if model is not None and not isinstance(model, str): + raise ValueError("model must be a string.") + result = run_comparison(scenario_id, runners, model) + else: + profile_name = payload.get("profile") + runner_name = payload.get("runner") + model = payload.get("model") + if not isinstance(profile_name, str) or not isinstance(runner_name, str): + raise ValueError("profile and runner must be strings.") + if model is not None and not isinstance(model, str): + raise ValueError("model must be a string.") + result = run_local_profile(profile_name, runner_name, model) + self._send_json(200, result) + except (json.JSONDecodeError, ValueError, LookupError, RuntimeError) as exc: + self._send_json(400, {"error": str(exc)}) + except Exception as exc: + LOG.exception("POST %s failed", route) + self._send_json(500, {"error": f"AssetOpsBench execution failed: {exc}"}) + + def _send_json(self, status: int, payload: Any) -> None: + body = json.dumps(payload, default=_plain, ensure_ascii=False).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(body))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: Any) -> None: + LOG.info("%s - %s", self.address_string(), format % args) + + +def main() -> None: + try: + from dotenv import load_dotenv + + load_dotenv(ROOT / ".env") + except ImportError: + LOG.warning("python-dotenv is unavailable; using the current process environment.") + os.chdir(ROOT) + os.environ.setdefault("AGENT_TRAJECTORY_DIR", str(ROOT / "traces" / "dashboard")) + host = os.environ.get("ASSETOPSBENCH_DASHBOARD_HOST", "127.0.0.1") + port = int(os.environ.get("ASSETOPSBENCH_DASHBOARD_PORT", "8000")) + server = ThreadingHTTPServer((host, port), DashboardHandler) + LOG.info("AssetOpsBench dashboard: http://%s:%s", host, port) + LOG.info("Repository root: %s", ROOT) + LOG.info("Live agent calls require configured model credentials and available MCP data services.") + try: + server.serve_forever() + except KeyboardInterrupt: + LOG.info("Stopping dashboard server.") + finally: + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/dashboard/styles.css b/dashboard/styles.css new file mode 100644 index 000000000..99d56eb9f --- /dev/null +++ b/dashboard/styles.css @@ -0,0 +1,331 @@ +:root { + color-scheme: dark; + --bg: #0a0d12; + --panel: #10151c; + --panel-raised: #131a22; + --panel-soft: #0e1319; + --line: #202a35; + --line-bright: #303d4b; + --text: #f0f4f6; + --muted: #8a96a4; + --faint: #5b6877; + --teal: #55d7b5; + --blue: #6ea9ff; + --violet: #b39aff; + --amber: #f3bb65; + --rose: #f17b91; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + font-synthesis: none; + text-rendering: optimizeLegibility; +} + +* { box-sizing: border-box; } +html { scroll-behavior: smooth; scroll-padding-top: 86px; } +body { margin: 0; min-width: 320px; background: var(--bg); color: var(--text); font-size: 14px; } +a { color: inherit; text-decoration: none; } +button, input, select { font: inherit; } +button, a, input, select { -webkit-tap-highlight-color: transparent; } +button:focus-visible, a:focus-visible, input:focus-visible, select:focus-visible, [tabindex="0"]:focus-visible { outline: 2px solid var(--teal); outline-offset: 3px; } +::selection { color: #07110f; background: var(--teal); } +.skip-link { position: absolute; top: -60px; left: 20px; z-index: 20; padding: 10px 15px; color: var(--bg); background: var(--teal); border-radius: 8px; } +.skip-link:focus { top: 10px; } +.topbar { position: sticky; z-index: 10; top: 0; height: 68px; display: flex; align-items: center; justify-content: space-between; padding: 0 max(calc((100vw - 1160px) / 2), 5.5vw); border-bottom: 1px solid rgba(255,255,255,.055); background: rgba(10,13,18,.89); backdrop-filter: blur(16px); } +.brand { display: inline-flex; align-items: center; gap: 9px; font-size: 14px; font-weight: 720; letter-spacing: -.035em; white-space: nowrap; } +.brand-mark { display: grid; width: 27px; height: 27px; place-items: center; border: 1px solid #31443f; border-radius: 8px; color: var(--teal); background: #13201e; font-size: 16px; font-weight: 800; } +.brand-mark span { color: #a4f3df; } +.brand-muted { color: #83909d; font-weight: 500; } +.main-nav { display: flex; height: 100%; align-items: center; gap: 30px; margin-left: 12px; } +.main-nav a { color: #a5afb9; font-size: 12px; transition: color .2s; } +.main-nav a:hover, .github-link:hover { color: var(--teal); } +.top-actions { display: flex; align-items: center; gap: 22px; } +.teach-switch { display: flex; align-items: center; gap: 8px; color: #bbc4cc; font-size: 11px; cursor: pointer; } +.teach-icon { color: var(--teal); font-size: 14px; } +.teach-switch input { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); } +.switch-track { width: 30px; height: 17px; padding: 2px; border: 1px solid #35414d; border-radius: 10px; background: #161c24; transition: background .2s,border-color .2s; } +.switch-track span { display: block; width: 11px; height: 11px; border-radius: 50%; background: #7e8a96; transition: transform .2s,background .2s; } +.teach-switch input:checked + .switch-track { border-color: #45ad94; background: #245a4d; } +.teach-switch input:checked + .switch-track span { transform: translateX(13px); background: #b6ffea; } +.teach-switch input:focus-visible + .switch-track { outline: 2px solid var(--teal); outline-offset: 3px; } +.github-link { color: #c5cdd4; font-size: 11px; font-weight: 600; transition: color .2s; } +.github-link span { color: var(--teal); } +.section-shell { width: min(1080px, 89%); margin-right: auto; margin-left: auto; } +.hero { display: grid; grid-template-columns: .91fr 1.09fr; column-gap: 58px; row-gap: 30px; padding-top: 77px; padding-bottom: 80px; } +.hero-copy { align-self: center; padding-bottom: 10px; } +.eyebrow { display: flex; align-items: center; gap: 8px; color: #8795a4; font-size: 9px; font-weight: 730; letter-spacing: .16em; } +.live-dot { position: relative; display: inline-block; width: 6px; height: 6px; flex: 0 0 6px; border-radius: 50%; background: var(--teal); box-shadow: 0 0 9px rgba(85,215,181,.45); } +.hero h1 { margin: 22px 0 18px; font-size: clamp(48px, 5.6vw, 68px); line-height: .99; letter-spacing: -.075em; font-weight: 650; } +.accent-text { color: var(--teal); } +.hero-lede { max-width: 435px; margin: 0; color: #9ba6b1; font-size: 14px; line-height: 1.8; } +.hero-buttons { display: flex; flex-wrap: wrap; gap: 11px; margin-top: 27px; } +.button { display: inline-flex; min-height: 41px; align-items: center; justify-content: center; gap: 12px; padding: 0 16px; border: 1px solid transparent; border-radius: 7px; font-size: 11px; font-weight: 650; transition: transform .2s, background .2s, border-color .2s, box-shadow .2s; cursor: pointer; } +.button:hover { transform: translateY(-2px); } +.button-primary { color: #07110f; background: var(--teal); box-shadow: 0 5px 22px rgba(46,190,153,.08); } +.button-primary:hover { background: #7ce9cc; box-shadow: 0 7px 25px rgba(46,190,153,.2); } +.button-quiet { border-color: #2a3540; color: #bbc5ce; background: #10151b; } +.button-quiet:hover { border-color: #465560; background: #151d24; } +.hero-stats { display: flex; gap: 36px; margin-top: 43px; padding-top: 21px; border-top: 1px solid #202832; } +.hero-stats div { display: grid; gap: 4px; } +.hero-stats strong { font-size: 22px; font-weight: 600; letter-spacing: -.05em; } +.hero-stats strong span { color: var(--teal); } +.hero-stats small { color: #75818e; font-size: 9px; } +.architecture-card { min-width: 0; padding: 16px 18px 12px; border: 1px solid #242e38; border-radius: 12px; background: radial-gradient(ellipse at 50% 48%,rgba(28,48,50,.26),transparent 62%), linear-gradient(145deg,#111820,#0e1319); box-shadow: 0 18px 60px rgba(0,0,0,.16); } +.card-topline { display: flex; align-items: center; justify-content: space-between; color: #80909f; font-size: 8px; font-weight: 700; letter-spacing: .12em; } +.topline-divider { padding: 0 5px; color: #46525f; } +.status-pill { display: inline-flex; align-items: center; gap: 6px; padding: 5px 7px; border: 1px solid #293b38; border-radius: 10px; color: #9edac8; font-size: 7px; letter-spacing: .08em; background: rgba(35,73,63,.14); } +.status-pill .live-dot { width: 4px; height: 4px; flex-basis: 4px; } +.hero-diagram { position: relative; height: 308px; overflow: hidden; } +.hero-diagram::before { position: absolute; inset: 15% 25%; border: 1px dashed rgba(131,160,158,.1); border-radius: 50%; content: ""; } +.diagram-lines { position: absolute; inset: 0; width: 100%; height: 100%; } +.flow-path { fill: none; stroke: url(#flow-gradient); stroke-width: 1; stroke-dasharray: 4 5; opacity: .42; } +.flow-particle { fill: #6af0d0; filter: drop-shadow(0 0 4px #4be4be); } +.particle-two { fill: #b5a0ff; filter: drop-shadow(0 0 4px #8f71ff); } +.agent-node { position: absolute; z-index: 1; left: var(--x); top: var(--y); display: flex; width: 105px; min-height: 46px; flex-direction: column; align-items: flex-start; justify-content: center; padding: 7px 10px; border: 1px solid #293541; border-radius: 7px; color: #f0f3f5; background: rgba(15,21,28,.96); text-align: left; font-size: 10px; font-weight: 750; cursor: pointer; transition: transform .2s,border-color .2s,box-shadow .2s; } +.agent-node:hover, .agent-node:focus-visible, .agent-node.is-active { z-index: 2; transform: scale(1.04); border-color: var(--node-color); box-shadow: 0 0 20px color-mix(in srgb,var(--node-color) 20%,transparent); } +.agent-node span { margin-top: 3px; color: #84909b; font-size: 8px; font-weight: 450; } +.agent-iot { --node-color: var(--teal); }.agent-tsfm { --node-color: var(--blue); }.agent-fmsr { --node-color: var(--violet); }.agent-wo { --node-color: var(--amber); }.agent-human { --node-color: #d1d9df; } +.orchestrator-node { position: absolute; top: 37%; left: 41%; display: flex; width: 96px; height: 96px; flex-direction: column; align-items: center; justify-content: center; border: 1px solid #2e5e55; border-radius: 50%; background: radial-gradient(circle at 50% 43%,#1d3935,#111c1d 72%); box-shadow: 0 0 35px rgba(76,207,173,.1),inset 0 0 24px rgba(70,193,161,.07); text-align: center; } +.orchestrator-icon { margin-bottom: 5px; color: var(--teal); font-size: 16px; } +.orchestrator-node strong { font-size: 10px; line-height: 1.25; } +.orchestrator-node small { margin-top: 4px; color: #879791; font-size: 7px; } +.diagram-callout { position: absolute; bottom: 1px; left: 50%; transform: translateX(-50%); padding: 6px 10px; border: 1px solid #29343c; border-radius: 20px; color: #aab4bd; background: #11171d; font-size: 8px; white-space: nowrap; } +.diagram-callout span { margin-right: 5px; color: var(--teal); font-size: 7px; } +.diagram-caption { display: flex; align-items: center; justify-content: space-between; padding: 12px 2px 1px; border-top: 1px solid #222d36; color: #8b98a5; font-size: 9px; } +.caption-arrow { color: var(--teal); font-size: 12px; } +.agent-legend { display: grid; grid-column: 1 / -1; grid-template-columns: repeat(4,1fr); gap: 10px; } +.legend-card { display: flex; min-width: 0; align-items: flex-start; gap: 10px; padding: 12px 12px 11px; border: 1px solid #202a33; border-radius: 8px; background: #0e1319; transition: border-color .2s,background .2s,transform .2s,box-shadow .2s; } +.legend-card.is-active { transform: translateY(-2px); border-color: #3d7768; background: linear-gradient(120deg,#13221f,#10151b); box-shadow: 0 6px 20px rgba(63,199,162,.07); } +.agent-swatch { width: 7px; height: 7px; flex: 0 0 7px; margin-top: 4px; border-radius: 2px; } +.iot-swatch { background: var(--teal); }.tsfm-swatch { background: var(--blue); }.fmsr-swatch { background: var(--violet); }.wo-swatch { background: var(--amber); } +.legend-card strong { font-size: 10px; }.legend-card p { margin: 4px 0 0; color: #7e8b98; font-size: 8px; line-height: 1.5; } +.legend-arrow { margin-left: auto; color: #61707e; font-size: 10px; } +.section-block { padding-top: 68px; padding-bottom: 15px; } +.section-heading { display: flex; align-items: flex-end; justify-content: space-between; gap: 20px; margin-bottom: 25px; } +.section-heading h2, .demo-copy h2 { margin: 10px 0 8px; font-size: clamp(27px,3.25vw,38px); line-height: 1.12; letter-spacing: -.055em; font-weight: 620; } +.muted-heading { color: #7d8895; font-weight: 450; } +.section-heading p { max-width: 620px; margin: 0; color: #8995a1; font-size: 11px; line-height: 1.7; } +.count-pill, .sample-badge { padding: 7px 10px; border: 1px solid #26333c; border-radius: 20px; color: #96a4b0; background: #11171d; font-size: 8px; white-space: nowrap; } +.scenario-layout { display: grid; grid-template-columns: 212px minmax(0,1fr); gap: 17px; } +.filter-panel, .scenario-toolbar, .scenario-card, .data-card, .paradigm-card, .leaderboard-card, .upload-card { border: 1px solid #202a34; border-radius: 9px; background: var(--panel); } +.filter-panel { align-self: start; padding: 15px 13px; } +.filter-heading { display: flex; align-items: center; justify-content: space-between; margin-bottom: 13px; } +.filter-heading strong { font-size: 10px; }.text-button { padding: 0; border: 0; color: #8895a1; background: none; font-size: 8px; cursor: pointer; }.text-button:hover { color: var(--teal); } +.search-wrap { display: flex; height: 32px; align-items: center; gap: 7px; padding: 0 9px; border: 1px solid #26313b; border-radius: 5px; color: #8c9aa7; background: #0c1116; } +.search-wrap span { font-size: 18px; }.search-wrap input { width: 100%; border: 0; outline: 0; color: #dce3e8; background: transparent; font-size: 9px; }.search-wrap input::placeholder { color: #606c78; } +.filter-group { display: grid; gap: 8px; margin: 18px 0 0; padding: 0; border: 0; } +.filter-group legend { margin-bottom: 2px; color: #86929e; font-size: 8px; font-weight: 650; } +.chip-list { display: flex; flex-wrap: wrap; gap: 5px; } +.filter-empty { color: #6f7b87; font-size: 7px; line-height: 1.45; } +.filter-chip { padding: 5px 7px; border: 1px solid #29323b; border-radius: 4px; color: #9ba5af; background: #121820; font-size: 8px; cursor: pointer; transition: background .15s,border-color .15s,color .15s; } +.filter-chip:hover { border-color: #53616c; color: #e6ebee; }.filter-chip[aria-pressed="true"] { border-color: #388c76; color: #a7efda; background: #153029; } +.scenario-main { min-width: 0; } +.scenario-toolbar { display: flex; align-items: center; justify-content: space-between; min-height: 42px; margin-bottom: 9px; padding: 0 12px; color: #8e9aa6; font-size: 9px; } +.scenario-toolbar-actions { display: flex; align-items: center; gap: 13px; }.scenario-toolbar-actions .text-button { font-size: 8px; } +.scenario-toolbar label { display: flex; align-items: center; gap: 6px; } +select { padding: 5px 22px 5px 8px; border: 1px solid #2a343e; border-radius: 4px; color: #c8d1d8; background: #11171d; font-size: 8px; } +.scenario-list { display: grid; gap: 8px; } +.scenario-card { overflow: hidden; transition: transform .18s,border-color .18s,box-shadow .18s; } +.scenario-card:hover { transform: translateY(-1px); border-color: #35424e; box-shadow: 0 7px 22px rgba(0,0,0,.14); } +.scenario-toggle { display: grid; width: 100%; grid-template-columns: minmax(0,1fr) auto auto; align-items: center; gap: 12px; padding: 13px 14px; border: 0; color: inherit; background: transparent; text-align: left; cursor: pointer; } +.scenario-title-area { min-width: 0; }.scenario-title-area strong { display: block; overflow: hidden; color: #e1e6ea; font-size: 10px; font-weight: 600; text-overflow: ellipsis; white-space: nowrap; }.scenario-tags { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 7px; } +.scenario-tag { padding: 3px 5px; border-radius: 3px; color: #9cabb6; background: #1b242d; font-size: 7px; }.agent-tag { color: #9de2cf; background: #153027; }.scenario-type { color: #b7a4f5; background: #251f37; }.scenario-chevron { color: #84909b; transition: transform .2s; }.scenario-card.expanded .scenario-chevron { transform: rotate(180deg); } +.complexity { color: #93a0ac; font-size: 8px; white-space: nowrap; }.complexity-dots { margin-left: 5px; color: var(--amber); letter-spacing: 1px; } +.scenario-details { display: grid; grid-template-rows: 0fr; transition: grid-template-rows .28s ease; }.scenario-card.expanded .scenario-details { grid-template-rows: 1fr; }.scenario-details-inner { min-height: 0; overflow: hidden; } +.scenario-detail-content { padding: 0 14px 13px; border-top: 1px solid #222c35; }.scenario-detail-grid { display: grid; grid-template-columns: minmax(0,1fr) auto; gap: 12px; padding-top: 12px; }.detail-label-spaced { margin-top: 10px; } +.detail-label { display: block; margin-bottom: 5px; color: #76838e; font-size: 7px; font-weight: 700; letter-spacing: .1em; }.detail-text { max-width: 650px; margin: 0; color: #b5bec6; font-size: 9px; line-height: 1.65; }.agent-pills { display: flex; flex-wrap: wrap; gap: 4px; } +.agent-pill { padding: 4px 6px; border: 1px solid #2b393c; border-radius: 4px; color: #a5e5d3; background: #13231f; font-size: 7px; white-space: nowrap; } +.scenario-explain-button { margin-top: 11px; padding: 0; border: 0; color: #94d7c2; background: none; font-size: 8px; cursor: pointer; }.scenario-explain-button:hover { color: var(--teal); }.simple-explain[hidden] { display: none; } +.simple-explain { display: flex; align-items: flex-start; gap: 8px; margin-top: 12px; padding: 9px 10px; border: 1px solid #3a3a2b; border-radius: 5px; color: #ded9b8; background: #211f17; font-size: 8px; line-height: 1.5; }.simple-explain span { color: var(--amber); } +.scenario-empty { padding: 25px 15px; border: 1px dashed #2e3943; border-radius: 7px; color: #8d9aa6; text-align: center; font-size: 10px; } +.sample-note, .data-disclaimer { margin: 10px 0 0; color: #6f7c89; font-size: 8px; }.sample-note a { color: #84cbb7; }.sample-note a:hover { text-decoration: underline; } +.data-grid { display: grid; grid-template-columns: repeat(2,minmax(0,1fr)); gap: 11px; } +.data-card { min-width: 0; padding: 15px 16px 12px; transition: transform .2s,border-color .2s,box-shadow .2s; }.data-card:hover { transform: translateY(-2px); border-color: #34414c; box-shadow: 0 10px 28px rgba(0,0,0,.14); } +.data-card-head { display: flex; align-items: center; gap: 8px; }.data-icon { display: grid; width: 24px; height: 24px; place-items: center; border: 1px solid currentColor; border-radius: 6px; font-size: 13px; opacity: .9; }.icon-teal { color: var(--teal); background: #14241f; }.icon-violet { color: var(--violet); background: #211c30; }.icon-blue { color: var(--blue); background: #172237; }.icon-amber { color: var(--amber); background: #2a2418; } +.data-label { color: #a0abb5; font-size: 8px; font-weight: 700; letter-spacing: .09em; }.info-dot { display: inline-grid; width: 13px; height: 13px; place-items: center; margin-left: 3px; border: 1px solid #46515b; border-radius: 50%; color: #95a2ad; font-size: 8px; cursor: help; vertical-align: middle; } +.info-dot:hover, .info-dot:focus-visible { border-color: var(--teal); color: var(--teal); } +.info-dot[data-tooltip] { position: relative; }.info-dot[data-tooltip]::after { position: absolute; z-index: 5; bottom: calc(100% + 8px); left: 50%; width: max-content; max-width: 220px; padding: 8px 10px; transform: translate(-50%,3px); border: 1px solid #35434c; border-radius: 6px; color: #d5dedf; background: #192128; box-shadow: 0 8px 20px rgba(0,0,0,.28); content: attr(data-tooltip); font-size: 9px; font-weight: 450; line-height: 1.45; letter-spacing: 0; opacity: 0; pointer-events: none; transition: opacity .16s,transform .16s; white-space: normal; }.info-dot[data-tooltip]:hover::after,.info-dot[data-tooltip]:focus-visible::after { transform: translate(-50%,0); opacity: 1; } +.trend { margin-left: auto; font-size: 8px; white-space: nowrap; }.trend.up { color: #6cd7b5; }.trend.neutral { color: #a99aeb; } +.metric-line { display: flex; align-items: baseline; gap: 9px; margin-top: 14px; }.metric-line strong { font-size: 29px; font-weight: 600; letter-spacing: -.06em; }.metric-line strong span { color: #a3b1bc; font-size: 16px; }.metric-line > span { color: #788591; font-size: 9px; } +.chart-line { display: block; width: 100%; height: 72px; margin-top: 8px; overflow: visible; }.chart-area { fill: url(#area-fill); }.chart-stroke { fill: none; stroke: var(--teal); stroke-width: 2; stroke-linecap: round; stroke-dasharray: 750; stroke-dashoffset: 750; animation: draw-chart 1.8s .15s ease forwards; filter: drop-shadow(0 0 4px rgba(85,215,181,.25)); } +.chart-axis { display: flex; justify-content: space-between; color: #65717e; font-size: 7px; } +@keyframes draw-chart { to { stroke-dashoffset: 0; } } +.fmea-bars { display: grid; gap: 10px; margin-top: 16px; }.bar-row { display: grid; grid-template-columns: 60px 1fr 21px; align-items: center; gap: 8px; color: #8996a1; font-size: 8px; }.bar-row strong { color: #cbd2d8; font-size: 8px; text-align: right; }.bar-track { height: 5px; overflow: hidden; border-radius: 4px; background: #222b33; }.bar-track i { display: block; width: var(--bar); height: 100%; border-radius: 4px; background: var(--bar-color); animation: grow-bar .9s ease both; transform-origin: left; } +@keyframes grow-bar { from { transform: scaleX(.05); } to { transform: scaleX(1); } } +.card-footer { display: flex; align-items: center; gap: 7px; margin-top: 14px; padding-top: 10px; border-top: 1px solid #232c34; color: #7e8a96; font-size: 8px; }.mini-avatar { display: grid; width: 19px; height: 19px; place-items: center; border-radius: 5px; font-size: 7px; font-weight: 700; }.violet-avatar { color: var(--violet); background: #29223b; }.blue-avatar { color: var(--blue); background: #1c2a41; }.footer-link { margin-left: auto; color: #8a99a6; } +.workorder-summary { display: grid; grid-template-columns: repeat(3,1fr); gap: 6px; margin-top: 16px; }.workorder-summary > div { display: grid; grid-template-columns: 7px auto; align-items: center; column-gap: 6px; }.workorder-summary strong { font-size: 14px; font-weight: 600; }.workorder-summary small { grid-column: 2; margin-top: 3px; color: #75818d; font-size: 7px; }.workorder-summary > span { color: #7e8a96; font-size: 8px; }.state-dot { width: 6px; height: 6px; border-radius: 50%; }.state-open { background: var(--amber); }.state-progress { background: var(--blue); }.state-done { background: var(--teal); } +.fmea-names { min-height: 44px; margin-top: 13px; padding: 11px 9px; border: 1px solid #2b2b38; border-radius: 5px; color: #c0b4e6; background: #17151f; font-size: 9px; line-height: 1.6; } +.kpi-visual { display: flex; align-items: center; gap: 20px; margin: 9px 0 3px; }.kpi-ring { display: grid; width: 88px; height: 88px; flex: 0 0 88px; place-items: center; border-radius: 50%; background: conic-gradient(var(--teal) 0 96.8%,#26332f 96.8% 100%); position: relative; }.kpi-ring::before { position: absolute; inset: 6px; border-radius: 50%; background: #10151c; content: ""; }.kpi-ring > span { position: relative; font-size: 16px; font-weight: 650; }.kpi-ring small { color: #9caab5; font-size: 9px; }.kpi-breakdown { display: grid; width: 100%; gap: 8px; }.kpi-breakdown div { display: flex; justify-content: space-between; gap: 8px; color: #83909c; font-size: 8px; }.kpi-breakdown strong { color: #c3cdd3; font-size: 8px; } +.data-disclaimer { text-align: right; } +.paradigm-card { overflow: hidden; }.paradigm-tabs { display: flex; border-bottom: 1px solid #26313b; }.paradigm-tab { display: flex; align-items: center; gap: 9px; padding: 14px 18px; border: 0; border-bottom: 2px solid transparent; color: #828f9b; background: transparent; font-size: 10px; cursor: pointer; }.paradigm-tab.active { border-bottom-color: var(--teal); color: #e8efef; background: linear-gradient(180deg,rgba(73,188,156,.07),transparent); }.tab-meta { color: #64727e; font-size: 7px; letter-spacing: .08em; }.paradigm-tab.active .tab-meta { color: #83c7b2; } +.paradigm-panel { display: grid; grid-template-columns: .95fr 1.05fr; gap: 35px; padding: 25px 27px; animation: panel-in .25s ease; }.paradigm-panel[hidden] { display: none; } +@keyframes panel-in { from { opacity: .35; transform: translateX(7px); } to { opacity: 1; transform: translateX(0); } } +.paradigm-explain { align-self: center; }.pattern-icon { display: grid; width: 30px; height: 30px; place-items: center; margin-bottom: 15px; border: 1px solid #2e5148; border-radius: 7px; color: var(--teal); background: #14221f; font-size: 17px; }.pattern-plan { border-color: #39324f; color: var(--violet); background: #1e1b2a; }.paradigm-explain h3 { margin: 7px 0; font-size: 17px; font-weight: 590; letter-spacing: -.03em; }.paradigm-explain p { max-width: 360px; margin: 0; color: #8996a2; font-size: 10px; line-height: 1.7; }.pro-con { display: grid; gap: 8px; margin-top: 16px; }.pro-con div { display: grid; grid-template-columns: 13px 68px 1fr; align-items: center; gap: 7px; color: #87939f; font-size: 8px; }.pro-con strong { color: #bfc9d0; font-weight: 600; }.check-icon { color: var(--teal); }.minus-icon { display: grid; width: 13px; height: 13px; place-items: center; border-radius: 50%; color: var(--amber); background: #312817; } +.flow-visual { position: relative; display: flex; min-height: 226px; flex-direction: column; align-items: center; justify-content: center; padding: 19px; border: 1px solid #26323b; border-radius: 8px; background: radial-gradient(ellipse at 50% 27%,rgba(44,73,67,.14),transparent 55%),#0d1217; }.flow-lead { display: grid; grid-template-columns: 25px auto; align-items: center; gap: 8px; min-width: 170px; padding: 9px 11px; border: 1px solid #31574e; border-radius: 6px; background: #14211e; }.flow-lead > span { display: grid; width: 23px; height: 23px; place-items: center; border-radius: 5px; color: var(--teal); background: #1d3b33; }.flow-lead strong { font-size: 9px; }.flow-lead small { grid-column: 2; color: #80948d; font-size: 7px; }.flow-fan { display: flex; width: 70%; height: 25px; justify-content: space-between; border-top: 1px dashed #3d665d; margin-top: 9px; }.flow-fan i { width: 1px; height: 25px; border-left: 1px dashed #3d665d; }.flow-specialists { display: flex; gap: 8px; }.flow-chip { padding: 6px 12px; border: 1px solid; border-radius: 5px; font-size: 8px; animation: chip-pulse 3s infinite; }.flow-chip:nth-child(2) { animation-delay: .5s; }.flow-chip:nth-child(3) { animation-delay: 1s; }.iot-chip { border-color: #356b5c; color: var(--teal); background: #14241f; }.tsfm-chip { border-color: #354e70; color: var(--blue); background: #172134; }.wo-chip { border-color: #6b5735; color: var(--amber); background: #251f16; } +@keyframes chip-pulse { 0%,65%,100% { box-shadow: 0 0 0 transparent; } 12% { box-shadow: 0 0 13px rgba(85,215,181,.18); } } +.flow-return { margin-top: 8px; color: #75868a; font-size: 8px; }.flow-result { display: flex; align-items: center; gap: 7px; margin-top: 10px; padding: 7px 10px; border: 1px solid #293a36; border-radius: 5px; color: #b7dacf; background: #121e1b; font-size: 8px; }.flow-result span { color: var(--teal); } +.plan-flow { align-items: stretch; gap: 7px; padding-left: 40px; }.plan-line { position: absolute; top: 47px; bottom: 70px; left: 27px; width: 1px; border-left: 1px dashed #6b5a95; }.plan-line i { position: absolute; left: -3px; width: 5px; height: 5px; border-radius: 50%; background: var(--violet); box-shadow: 0 0 8px var(--violet); animation: plan-dot 2.7s infinite; }.plan-line i:nth-child(2) { top: 50%; animation-delay: .7s; }.plan-line i:nth-child(3) { bottom: 0; animation-delay: 1.4s; } +@keyframes plan-dot { 0%,100% { opacity: .3; } 50% { opacity: 1; } } +.plan-step { display: grid; grid-template-columns: 23px 1fr auto; align-items: center; gap: 9px; padding: 8px 10px; border: 1px solid #292b37; border-radius: 5px; background: #12151d; }.plan-step > b { color: #a995ed; font-size: 8px; }.plan-step strong { display: block; color: #d3d9df; font-size: 9px; }.plan-step small { color: #74818d; font-size: 7px; }.step-done { color: var(--teal); font-size: 11px; }.step-wait { color: var(--amber); }.plan-flow .flow-result { align-self: center; } +.evaluation-layout { display: grid; grid-template-columns: minmax(0,1.8fr) minmax(250px,.8fr); gap: 12px; }.leaderboard-card { overflow: hidden; }.table-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; padding: 13px 15px; border-bottom: 1px solid #25303a; }.table-heading > div { display: flex; align-items: center; gap: 7px; }.table-heading strong { font-size: 10px; }.table-heading small { color: #77838f; font-size: 8px; }.table-heading label { display: flex; align-items: center; gap: 7px; color: #788591; font-size: 8px; }.table-scroll { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; text-align: left; white-space: nowrap; } th { padding: 10px 11px; color: #74818d; background: #0e1319; font-size: 7px; font-weight: 650; letter-spacing: .07em; } th:first-child,td:first-child { padding-left: 15px; } td { padding: 11px; border-top: 1px solid #202933; color: #aab4bd; font-size: 8px; } tbody tr { transition: background .17s,opacity .2s,transform .2s; animation: row-in .25s ease both; } tbody tr:hover { background: #151c23; } +@keyframes row-in { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } } +.model-cell { display: flex; align-items: center; gap: 8px; }.rank-num { width: 16px; color: #788490; font-size: 8px; }.model-icon { display: grid; width: 23px; height: 23px; place-items: center; border: 1px solid #36443f; border-radius: 5px; color: #9fe5d2; background: #182723; font-size: 8px; }.model-cell strong { display: block; color: #d5dce1; font-size: 8px; font-weight: 600; }.model-cell small { display: block; margin-top: 3px; color: #77838e; font-size: 7px; }.score-cell { color: #dfe8e6; font-weight: 650; }.metric-value { display: inline-block; min-width: 30px; color: #c8d3db; }.mini-meter { display: inline-block; width: 45px; height: 3px; margin-left: 5px; overflow: hidden; border-radius: 3px; background: #29333c; vertical-align: middle; }.mini-meter i { display: block; height: 100%; border-radius: inherit; background: var(--teal); }.table-footnote { padding: 9px 14px; border-top: 1px solid #222c35; color: #6f7c88; font-size: 7px; } +.upload-card { padding: 15px; }.upload-top { display: flex; align-items: center; justify-content: space-between; }.upload-icon { color: var(--teal); font-size: 18px; }.upload-tag { padding: 4px 6px; border: 1px solid #32433e; border-radius: 4px; color: #90baa9; font-size: 7px; letter-spacing: .08em; }.upload-card h3 { margin: 13px 0 6px; font-size: 12px; font-weight: 600; }.upload-card > p { margin: 0; color: #87939e; font-size: 8px; line-height: 1.6; }.file-drop { display: flex; min-height: 89px; flex-direction: column; align-items: center; justify-content: center; gap: 4px; margin-top: 13px; border: 1px dashed #3b4a52; border-radius: 6px; background: #0d1318; text-align: center; cursor: pointer; transition: border-color .2s,background .2s,box-shadow .2s; }.file-drop:hover,.file-drop.drag-over { border-color: var(--teal); background: #12211e; box-shadow: 0 0 20px rgba(85,215,181,.08); }.file-drop input { position: absolute; width: 1px; height: 1px; opacity: 0; }.file-drop-icon { color: var(--teal); font-size: 18px; }.file-drop strong { color: #cbd4da; font-size: 8px; }.file-drop small { color: #73808c; font-size: 7px; }.judge-note { display: flex; gap: 8px; margin-top: 11px; padding: 9px; border: 1px solid #2c3438; border-radius: 5px; color: #85929c; background: #13181b; font-size: 7px; line-height: 1.55; }.judge-note > span:first-child { color: var(--amber); }.judge-note strong { color: #bfc9cf; font-size: 7px; }.judge-note.file-selected { border-color: #386b5d; background: #13201c; } +.evaluation-scenario-label,.run-field { display: grid; gap: 6px; margin-top: 11px; color: #8996a2; font-size: 8px; }.evaluation-scenario-label select,.run-field select { width: 100%; min-height: 30px; font-size: 9px; } +.backend-ready { border-color: #315a4d; color: #97d8c4; }.status-pill:not(.backend-ready) { color: #d5b477; }.status-pill:not(.backend-ready) .live-dot { background: var(--amber); } +.run-controls { display: flex; align-items: center; gap: 12px; margin: 12px 0; }.run-controls > span { color: #87939e; font-size: 8px; line-height: 1.5; }.run-button:disabled { opacity: .5; cursor: not-allowed; transform: none; } +.execution-output { max-height: 550px; overflow-y: auto; padding: 10px; border: 1px solid #273239; border-radius: 6px; background: #0b1013; }.execution-empty { margin: 0; color: #7c898f; font-size: 9px; line-height: 1.6; }.execution-error { margin: 0; color: #ffc0b3; border-color: #654039; background: #271a18; }.answer-block { padding: 10px; border-left: 2px solid var(--teal); background: #121b1b; }.answer-block h3 { margin: 5px 0; font-size: 11px; }.answer-block p,.judge-result p { margin: 6px 0 0; color: #b6c2c2; font-size: 9px; line-height: 1.6; white-space: pre-wrap; } +.judge-result { margin-top: 8px; padding: 9px; border: 1px solid #355748; border-radius: 5px; color: #b7e2d1; background: #13201b; font-size: 8px; }.judge-result strong { font-size: 9px; }.judge-result p { color: #91a99d; font-size: 8px; }.judge-error { border-color: #61443b; color: #f1b7a9; background: #211917; }.trajectory-group,.trajectory-step { margin-top: 8px; border: 1px solid #29343a; border-radius: 5px; background: #11171b; }.trajectory-group > summary,.trajectory-step > summary { display: flex; justify-content: space-between; gap: 9px; padding: 9px; color: #b6c3c8; font-size: 8px; cursor: pointer; }.trajectory-content { padding: 0 9px 9px; }.trajectory-content p { color: #a4b1b5; font-size: 8px; line-height: 1.5; }.trajectory-group pre,.trajectory-step pre,.judge-result pre { max-height: 210px; overflow: auto; margin: 5px 0 8px; padding: 8px; border: 1px solid #202a30; border-radius: 4px; color: #aab9bd; background: #0a0e11; font: 8px/1.55 ui-monospace,SFMono-Regular,Consolas,monospace; white-space: pre-wrap; overflow-wrap: anywhere; }.tool-call { margin: 7px 0; padding-top: 8px; border-top: 1px solid #263139; } +.demo-section { display: grid; grid-template-columns: .7fr 1.3fr; align-items: center; gap: 50px; margin-top: 75px; margin-bottom: 65px; padding: 30px; border: 1px solid #29363a; border-radius: 12px; background: radial-gradient(ellipse at 80% 30%,rgba(35,91,75,.16),transparent 48%),linear-gradient(130deg,#10161b,#10171a); } +.demo-copy h2 { margin-top: 15px; font-size: clamp(31px,4vw,46px); }.demo-copy > p { max-width: 320px; color: #8997a0; font-size: 10px; line-height: 1.8; }.run-button { margin-top: 9px; }.play-icon { font-size: 8px; }.button-arrow { margin-left: 5px; }.demo-safety { display: flex; align-items: center; gap: 6px; margin-top: 14px; color: #72827f; font-size: 8px; }.demo-safety span { color: var(--teal); } +.console-card { min-width: 0; overflow: hidden; border: 1px solid #2c3c40; border-radius: 8px; background: #0c1114; box-shadow: 0 13px 32px rgba(0,0,0,.22); }.console-bar { display: flex; height: 34px; align-items: center; justify-content: space-between; padding: 0 12px; border-bottom: 1px solid #273136; color: #7a898f; background: #13191d; font-size: 7px; font-weight: 700; letter-spacing: .1em; }.console-dots { display: flex; gap: 4px; }.console-dots i { width: 6px; height: 6px; border-radius: 50%; background: #576167; }.console-dots i:first-child { background: #ee766c; }.console-dots i:nth-child(2) { background: #e8bd69; }.console-dots i:nth-child(3) { background: #69c79a; }.console-ready { display: flex; align-items: center; gap: 5px; color: #93a19e; font-size: 6px; }.console-ready i { width: 5px; height: 5px; border-radius: 50%; background: #66736f; }.console-ready.running { color: var(--teal); }.console-ready.running i { background: var(--teal); animation: blink 1s infinite; } +@keyframes blink { 50% { opacity: .35; } } +.console-content { padding: 14px 15px 12px; }.console-request { padding: 9px 10px; border-left: 2px solid #55b49a; background: #131d1d; }.console-request > span { color: #6da893; font-size: 6px; font-weight: 700; letter-spacing: .13em; }.console-request p { margin: 4px 0 0; color: #d1d9da; font-size: 9px; }.timeline { display: grid; gap: 0; margin-top: 9px; }.timeline-step { display: grid; grid-template-columns: 25px 1fr auto; align-items: center; gap: 9px; min-height: 42px; padding: 6px 9px; border-bottom: 1px solid #1b2429; border-radius: 4px; opacity: .56; transition: opacity .3s,background .3s,box-shadow .3s; }.timeline-step:last-child { border-bottom: 0; }.timeline-step.active { opacity: 1; background: #14201e; box-shadow: inset 2px 0 var(--step-color); }.timeline-step.completed { opacity: .94; }.timeline-icon { display: grid; width: 22px; height: 22px; place-items: center; border: 1px solid currentColor; border-radius: 5px; background: #161b20; font-size: 11px; }.iot-timeline { color: var(--teal); }.tsfm-timeline { color: var(--blue); }.fmsr-timeline { color: var(--violet); }.wo-timeline { color: var(--amber); }.timeline-step > div > strong { color: #c5cfd2; font-size: 8px; }.timeline-step > div > strong small { margin-left: 6px; color: #74828a; font-size: 6px; letter-spacing: .08em; }.timeline-step p { margin: 4px 0 0; color: #889691; font-size: 8px; line-height: 1.4; }.step-state { color: #54625f; font-size: 11px; }.timeline-step.completed .step-state { color: var(--teal); }.timeline-step.active .step-state { color: var(--teal); animation: blink 1s infinite; }.demo-result { display: flex; align-items: flex-start; gap: 9px; margin-top: 9px; padding: 10px; border: 1px solid #37584d; border-radius: 5px; background: #14201c; animation: result-in .4s ease both; }.demo-result[hidden] { display: none; }.result-check { display: grid; width: 18px; height: 18px; flex: 0 0 18px; place-items: center; border-radius: 50%; color: #08120f; background: var(--teal); font-size: 10px; }.demo-result strong { color: #b9e6d6; font-size: 8px; }.demo-result p { margin: 4px 0 0; color: #a4b4ad; font-size: 8px; line-height: 1.5; }.result-tag { margin-left: auto; padding: 3px 5px; border: 1px solid #426656; border-radius: 3px; color: #a1c7b2; font-size: 6px; white-space: nowrap; }.console-foot { display: flex; justify-content: space-between; padding: 9px 13px; border-top: 1px solid #222d31; color: #6d7c7f; background: #101619; font-size: 7px; }.console-cursor { display: inline-block; width: 5px; height: 9px; margin-right: 4px; background: var(--teal); vertical-align: middle; animation: blink 1s infinite; } +@keyframes result-in { from { opacity: 0; transform: translateY(5px); } to { opacity: 1; transform: translateY(0); } } +.site-footer { display: flex; min-height: 68px; align-items: center; justify-content: space-between; gap: 20px; padding: 0 max(calc((100vw - 1080px) / 2), 5.5vw); border-top: 1px solid #202932; color: #77848f; font-size: 8px; }.site-footer .brand { color: #b6c0c8; font-size: 11px; }.site-footer .brand-mark { width: 22px; height: 22px; font-size: 13px; }.site-footer > a:last-child { color: #9acdbd; }.site-footer > a:last-child:hover { color: var(--teal); } +.reveal-section { opacity: 0; transform: translateY(12px); transition: opacity .55s ease,transform .55s ease; }.reveal-section.is-visible { opacity: 1; transform: translateY(0); } +body.teach-mode [data-dense] { display: none; }body.teach-mode [data-simple] { display: block; }[data-simple] { display: none; } + +@media (min-width: 1200px) { .hero { padding-top: 88px; } } +@media (max-width: 820px) { + .topbar { padding: 0 5.5%; }.main-nav { gap: 16px; }.main-nav a { font-size: 10px; }.top-actions { gap: 14px; } + .hero { grid-template-columns: 1fr 1fr; column-gap: 25px; padding-top: 60px; }.hero h1 { font-size: clamp(46px,7.3vw,59px); }.hero-stats { gap: 22px; }.hero-diagram { height: 275px; }.agent-node { width: 88px; }.orchestrator-node { width: 84px; height: 84px; left: 42%; top: 37%; } + .section-shell { width: 90%; }.scenario-layout { grid-template-columns: 185px minmax(0,1fr); gap: 10px; }.scenario-toggle { gap: 7px; padding-right: 10px; padding-left: 10px; }.scenario-title-area strong { font-size: 9px; }.evaluation-layout { grid-template-columns: 1fr; }.upload-card { display: grid; grid-template-columns: auto 1fr 1.2fr; align-items: center; column-gap: 14px; }.upload-top { grid-row: 1 / span 2; }.upload-card h3 { margin: 0; }.upload-card > p { grid-column: 2; }.file-drop { grid-column: 3; grid-row: 1 / span 2; margin: 0; }.judge-note { grid-column: 1 / -1; } + .demo-section { gap: 25px; padding: 24px; } +} +@media (max-width: 620px) { + html { scroll-padding-top: 66px; }.topbar { height: 60px; padding: 0 5%; }.main-nav { position: fixed; right: 0; bottom: 0; left: 0; z-index: 10; height: 48px; justify-content: space-around; gap: 0; margin: 0; border-top: 1px solid #252f39; background: rgba(12,16,21,.96); backdrop-filter: blur(12px); }.main-nav a { padding: 13px 5px; font-size: 9px; }.top-actions { gap: 12px; }.teach-switch { gap: 5px; font-size: 9px; }.github-link { font-size: 9px; } + .section-shell { width: 90%; }.hero { grid-template-columns: 1fr; gap: 24px; padding-top: 50px; padding-bottom: 40px; }.hero h1 { margin-top: 17px; font-size: clamp(54px,13vw,70px); }.hero-lede { max-width: 500px; font-size: 13px; }.hero-stats { gap: 0; justify-content: space-between; margin-top: 28px; }.hero-stats strong { font-size: 20px; }.architecture-card { padding: 13px; }.hero-diagram { height: 285px; }.agent-node { width: 90px; }.orchestrator-node { left: 41%; }.agent-legend { grid-template-columns: repeat(2,1fr); gap: 7px; }.legend-card { padding: 10px; }.legend-card p { font-size: 8px; } + .section-block { padding-top: 43px; }.section-heading { align-items: flex-start; margin-bottom: 18px; }.section-heading h2 { font-size: 29px; }.section-heading p { font-size: 10px; }.section-heading > .sample-badge,.section-heading > .count-pill { align-self: flex-start; margin-top: 2px; } + .scenario-layout { grid-template-columns: 1fr; }.filter-panel { padding: 12px; }.filter-heading { margin-bottom: 9px; }.filter-group { display: inline-grid; width: calc(33.333% - 4px); vertical-align: top; margin-top: 11px; }.chip-list { max-height: 61px; overflow-y: auto; }.scenario-toolbar { flex-wrap: wrap; gap: 8px; margin-top: 8px; padding: 8px; }.scenario-toolbar-actions { flex-wrap: wrap; justify-content: flex-end; }.scenario-toggle { grid-template-columns: minmax(0,1fr) auto; }.complexity { display: none; }.scenario-title-area strong { font-size: 9px; } + .data-grid { grid-template-columns: 1fr; }.data-card { padding: 14px; }.metric-line strong { font-size: 27px; }.chart-line { height: 66px; }.data-disclaimer { text-align: left; } + .paradigm-tabs { overflow-x: auto; }.paradigm-tab { flex: 1; justify-content: center; gap: 5px; padding: 12px 8px; white-space: nowrap; font-size: 9px; }.tab-meta { font-size: 6px; }.paradigm-panel { grid-template-columns: 1fr; gap: 18px; padding: 19px; }.paradigm-explain h3 { font-size: 16px; }.flow-visual { min-height: 207px; } + .evaluation-layout { gap: 9px; }.table-heading { align-items: flex-start; padding: 12px 10px; }.table-heading > div { flex-wrap: wrap; }.table-heading small { width: 100%; margin-left: 14px; }.table-heading label { flex-direction: column; align-items: flex-start; gap: 4px; font-size: 7px; }.table-heading select { max-width: 110px; }.table-scroll table { min-width: 510px; }.upload-card { display: block; }.upload-card h3 { margin-top: 9px; }.file-drop { margin-top: 12px; }.judge-note { margin-top: 9px; } + .demo-section { grid-template-columns: 1fr; gap: 21px; margin-top: 50px; margin-bottom: 40px; padding: 20px 15px; }.demo-copy h2 { font-size: 38px; }.console-content { padding: 11px; }.timeline-step { grid-template-columns: 23px 1fr auto; gap: 7px; padding-right: 5px; padding-left: 5px; }.timeline-step > div > strong small { margin-left: 3px; font-size: 5px; }.demo-result { gap: 6px; }.result-tag { font-size: 5px; } + .site-footer { min-height: 92px; flex-wrap: wrap; justify-content: space-between; padding: 14px 5% 58px; }.site-footer > span { order: 3; width: 100%; }.site-footer > a:last-child { font-size: 7px; } +} +@media (max-width: 370px) { + .brand { gap: 6px; font-size: 12px; }.teach-switch { font-size: 8px; }.github-link { font-size: 8px; }.hero-diagram { height: 258px; }.agent-node { width: 79px; padding: 6px; font-size: 9px; }.agent-node span { font-size: 7px; }.orchestrator-node { left: 40%; width: 75px; height: 75px; }.filter-group { width: calc(50% - 4px); }.section-heading { flex-wrap: wrap; }.section-heading > .sample-badge { display: none; } +} +@media (prefers-reduced-motion: reduce) { + *,*::before,*::after { scroll-behavior: auto !important; animation-duration: .01ms !important; animation-iteration-count: 1 !important; transition-duration: .01ms !important; } +} + +.usage-section { padding: 45px 0 70px; border-top: 1px solid #202932; } +.usage-grid { display: grid; grid-template-columns: repeat(3,minmax(0,1fr)); gap: 12px; } +.usage-card,.lab-panel { min-width: 0; padding: 18px; border: 1px solid #26323b; border-radius: 9px; background: linear-gradient(145deg,#111820,#0e1319); } +.usage-card > span { color: var(--teal); font: 700 9px ui-monospace,SFMono-Regular,Consolas,monospace; letter-spacing: .1em; } +.usage-card h3 { margin: 12px 0 7px; color: #dfe5e9; font-size: 12px; } +.usage-card p,.usage-note { margin: 0; color: #8b98a3; font-size: 9px; line-height: 1.75; } +.usage-callout { display: flex; align-items: center; justify-content: space-between; gap: 20px; margin-top: 13px; padding: 15px 17px; border: 1px solid #2b4640; border-radius: 8px; background: #111c1b; } +.usage-callout p { margin: 0; color: #9baaa8; font-size: 10px; line-height: 1.7; } +.usage-callout strong { color: #d9e7e3; } +.usage-note { margin-top: 14px; } +.usage-note code,.section-heading code { color: #aeead9; font: 9px ui-monospace,SFMono-Regular,Consolas,monospace; } +.engineering-hero { padding-top: 68px; padding-bottom: 43px; } +.engineering-hero h1 { max-width: 760px; margin: 19px 0 12px; font-size: clamp(40px,6vw,60px); line-height: 1.02; letter-spacing: -.065em; } +.engineering-hero > p { max-width: 720px; margin: 0; color: #9aa6b0; font-size: 13px; line-height: 1.8; } +.engineering-hero-actions,.lab-actions { display: flex; align-items: center; flex-wrap: wrap; gap: 12px; margin-top: 20px; } +.engineering-back-link { color: #9aa7b0; font-size: 10px; } +.engineering-back-link:hover { color: var(--teal); } +.engineering-safety,.lab-warning { margin-top: 20px; padding: 12px 14px; border: 1px solid #584633; border-radius: 7px; color: #c4ad86; background: #1c1914; font-size: 9px; line-height: 1.7; } +.engineering-safety strong,.lab-warning strong { color: #ecd2a0; } +.lab-connection { margin-top: 12px; color: #8ed6bf; font-size: 9px; } +.lab-connection.is-warning { color: #e4be79; } +.lab-section { padding-top: 40px; padding-bottom: 35px; border-top: 1px solid #202932; } +.lab-section .section-heading { margin-bottom: 19px; } +.lab-tool-layout { display: grid; grid-template-columns: minmax(280px,.75fr) minmax(0,1.25fr); gap: 13px; } +.lab-tool-controls { display: grid; align-content: start; gap: 13px; } +.lab-field { display: grid; gap: 7px; color: #aab5bd; font-size: 9px; } +.lab-field select,.lab-field input,.lab-field textarea { width: 100%; min-height: 37px; padding: 9px 10px; border: 1px solid #303c46; border-radius: 5px; color: #d8e0e5; background: #0b1015; font: 10px ui-monospace,SFMono-Regular,Consolas,monospace; } +.lab-field textarea { min-height: 130px; resize: vertical; line-height: 1.6; } +.lab-field select { font-family: inherit; } +.lab-field input::placeholder { color: #62717c; } +.lab-hint,.lab-help { color: #74828e; font-size: 8px; line-height: 1.65; } +.lab-tool-controls .button:disabled,.lab-actions .button:disabled { opacity: .45; cursor: not-allowed; transform: none; } +.lab-result-panel { display: flex; min-height: 360px; flex-direction: column; } +.lab-panel-heading,.run-card-heading { display: flex; align-items: flex-start; justify-content: space-between; gap: 15px; } +.lab-panel-heading h3,.run-card-heading h3 { margin: 8px 0 0; color: #e2e8eb; font-size: 12px; } +.lab-panel-heading .status-pill { flex-shrink: 0; } +.lab-output { min-height: 230px; max-height: 580px; overflow: auto; margin: 16px 0 0; padding: 13px; border: 1px solid #222e37; border-radius: 6px; color: #b9c8cb; background: #090d11; font: 9px/1.65 ui-monospace,SFMono-Regular,Consolas,monospace; white-space: pre-wrap; overflow-wrap: anywhere; } +.lab-output.is-error,.is-error { color: #ffc0b3; } +.evidence-replay { margin-top: 16px; padding: 14px; border: 1px solid #2b403d; border-radius: 7px; background: linear-gradient(145deg,rgba(22,41,38,.52),rgba(12,18,23,.94) 55%); } +.replay-heading { display: flex; align-items: center; justify-content: space-between; gap: 12px; } +.replay-heading h3 { margin: 5px 0 0; color: #e2e8eb; font-size: 13px; } +.replay-count { padding: 5px 7px; border: 1px solid #2c4841; border-radius: 10px; color: #9de2cf; background: #13231f; font-size: 8px; white-space: nowrap; } +.replay-disclaimer { max-width: 760px; margin: 10px 0 12px; color: #8d9aa3; font-size: 9px; line-height: 1.6; } +.replay-controls { display: flex; align-items: center; gap: 6px; padding: 8px; border: 1px solid #242f37; border-radius: 6px; background: #0c1217; } +.replay-control { min-width: 29px; min-height: 27px; padding: 4px 8px; border: 1px solid #344149; border-radius: 4px; color: #c7d2d5; background: #141d22; font-size: 8px; cursor: pointer; } +.replay-control:hover { border-color: #4e8f7f; color: var(--teal); } +.replay-progress { min-width: 0; margin-left: 5px; overflow: hidden; color: #82919a; font-size: 8px; text-overflow: ellipsis; white-space: nowrap; } +.replay-timeline { display: grid; gap: 0; margin: 13px 0 0; padding: 0; list-style: none; } +.replay-step { position: relative; display: grid; grid-template-columns: 25px minmax(0,1fr); gap: 10px; padding: 0 0 13px; } +.replay-step:not(:last-child)::after { position: absolute; top: 25px; bottom: -1px; left: 11px; width: 1px; background: #29363c; content: ""; } +.replay-step-marker { z-index: 1; display: grid; width: 24px; height: 24px; place-items: center; border: 1px solid #344148; border-radius: 50%; color: #83939a; background: #11181d; font: 8px ui-monospace,SFMono-Regular,Consolas,monospace; } +.replay-step.is-current .replay-step-marker { border-color: #55a98f; color: #b6f4e1; background: #173128; box-shadow: 0 0 12px rgba(85,215,181,.13); } +.replay-step-content { min-width: 0; padding: 2px 8px 8px; border: 1px solid transparent; border-radius: 5px; } +.replay-step.is-current .replay-step-content { border-color: #2c413d; background: rgba(20,34,32,.48); } +.replay-select { display: grid; gap: 4px; width: 100%; padding: 0; border: 0; color: inherit; background: transparent; text-align: left; cursor: pointer; } +.replay-step-label { color: #7f9991; font-size: 7px; font-weight: 720; letter-spacing: .1em; } +.replay-step-title { color: #d1dadd; font-size: 9px; font-weight: 600; line-height: 1.55; overflow-wrap: anywhere; white-space: pre-wrap; } +.replay-step-description { margin: 5px 0 0; color: #9aa8ad; font-size: 8px; line-height: 1.55; overflow-wrap: anywhere; white-space: pre-wrap; } +.replay-evidence { margin-top: 7px; } +.replay-evidence summary { color: #91cdbb; font-size: 8px; cursor: pointer; } +.replay-evidence .detail-label { margin-top: 9px; } +.replay-evidence pre { max-height: 260px; overflow: auto; margin: 5px 0 0; padding: 9px; border: 1px solid #242f36; border-radius: 4px; color: #b8c8c5; background: #090e12; font: 8px/1.6 ui-monospace,SFMono-Regular,Consolas,monospace; white-space: pre-wrap; overflow-wrap: anywhere; } +.replay-source { margin-top: 4px; color: #64727d; font: 7px/1.5 ui-monospace,SFMono-Regular,Consolas,monospace; overflow-wrap: anywhere; } +.lab-form-grid { display: grid; grid-template-columns: minmax(0,1fr) minmax(220px,.8fr); gap: 14px; } +.runner-options { display: flex; flex-wrap: wrap; gap: 10px 22px; margin: 18px 0 0; padding: 12px; border: 1px solid #27343e; border-radius: 6px; } +.runner-options legend { padding: 0 7px; color: #9daab4; font-size: 9px; } +.runner-options label { display: flex; align-items: center; gap: 7px; color: #bac4cc; font-size: 9px; cursor: pointer; } +.runner-options input { accent-color: var(--teal); } +.lab-actions { margin-top: 15px; } +.compare-results,.history-list { display: grid; gap: 10px; margin-top: 17px; } +.run-card { min-width: 0; padding: 14px; border: 1px solid #28343d; border-radius: 7px; background: #0d1318; } +.run-card-heading { align-items: center; } +.run-card-heading h3 { margin: 0; } +.run-card-heading > span { color: #85939e; font-size: 8px; text-align: right; overflow-wrap: anywhere; } +.run-answer { margin: 11px 0 0; color: #b8c4ca; font-size: 9px; line-height: 1.7; white-space: pre-wrap; } +.run-evaluation { margin: 9px 0 0; color: #a1d5c2; font-size: 8px; line-height: 1.6; } +.run-card details { margin-top: 11px; border-top: 1px solid #232e36; padding-top: 9px; } +.run-card summary { color: #91b9ae; font-size: 8px; cursor: pointer; } +.history-output { min-height: 100px; max-height: 380px; } +.compact-run-card { padding: 12px; } +.benchmark-launcher { padding: 20px; } +.benchmark-detail { margin: 15px 0; padding: 12px; border-left: 2px solid var(--blue); color: #aebbc4; background: #111922; font-size: 9px; line-height: 1.7; } +.benchmark-detail.is-error { color: #ffc0b3; border-color: var(--rose); } +.benchmark-launcher > .lab-field { max-width: 520px; } +.history-actions { margin-top: -5px; } +.evaluation-summary { display: grid; grid-template-columns: repeat(3,minmax(0,1fr)); gap: 9px; margin-top: 15px; } +.evaluation-metric { display: grid; gap: 8px; padding: 12px; border: 1px solid #28363e; border-radius: 6px; background: #0e1519; } +.evaluation-metric span { color: #83919b; font-size: 8px; } +.evaluation-metric strong { color: #d9e8e2; font-size: 19px; font-weight: 600; letter-spacing: -.04em; } +.runner-score-summary { display: flex; grid-column: 1 / -1; flex-wrap: wrap; gap: 7px; } +.runner-score-summary span { padding: 7px 9px; border: 1px solid #293a3c; border-radius: 5px; color: #a8c4bb; background: #10191a; font-size: 8px; } +.how-to-section .usage-grid { grid-template-columns: repeat(4,minmax(0,1fr)); } +.engineering-page .site-footer { margin-top: 16px; } +body.teach-mode .engineering-hero > [data-simple] { color: #c4d8d1; } +@media (max-width: 820px) { + .how-to-section .usage-grid { grid-template-columns: repeat(2,minmax(0,1fr)); } + .lab-tool-layout { grid-template-columns: 1fr; } +} +@media (max-width: 620px) { + .engineering-hero { padding-top: 49px; } + .engineering-hero h1 { font-size: 42px; } + .engineering-hero-actions { align-items: flex-start; } + .engineering-hero-actions .button { min-height: 37px; padding: 0 10px; font-size: 9px; } + .usage-grid,.how-to-section .usage-grid { grid-template-columns: 1fr; } + .usage-callout { align-items: flex-start; flex-direction: column; } + .lab-form-grid { grid-template-columns: 1fr; } + .run-card-heading { align-items: flex-start; flex-direction: column; gap: 6px; } + .run-card-heading > span { text-align: left; } + .runner-options { display: grid; grid-template-columns: 1fr 1fr; } + .evaluation-summary { grid-template-columns: 1fr 1fr; } + .evaluation-metric:last-of-type { grid-column: 1 / -1; } + .evidence-replay { padding: 11px; } + .replay-heading { align-items: flex-start; flex-direction: column; } + .replay-step { grid-template-columns: 22px minmax(0,1fr); gap: 7px; } + .replay-step-marker { width: 21px; height: 21px; } + .replay-step:not(:last-child)::after { left: 10px; } + .engineering-page .main-nav { overflow-x: auto; justify-content: flex-start; padding: 0 8px; } + .engineering-page .main-nav a { flex: 0 0 auto; padding: 13px 10px; } +} diff --git a/dashboard/tests/test_server.py b/dashboard/tests/test_server.py new file mode 100644 index 000000000..79cefe3fc --- /dev/null +++ b/dashboard/tests/test_server.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from dashboard import server + + +class DashboardRepositoryTests(unittest.TestCase): + def test_scenarios_are_loaded_from_repository_files(self): + scenarios = server.load_scenarios() + by_id = {scenario["id"]: scenario for scenario in scenarios} + + self.assertGreaterEqual(len(scenarios), 25) + self.assertIn("304", by_id) + self.assertIn("scenario_3/question.txt", by_id["3"]["source"]) + self.assertIn("characteristic_form", by_id["304"]) + + def test_overview_uses_checked_in_data(self): + overview = server.load_data_overview() + + self.assertGreater(overview["telemetry"]["records"], 0) + self.assertGreater(overview["telemetry"]["observations"], 0) + self.assertEqual(overview["fmea"]["names"], ["seal leakage", "impeller wear"]) + self.assertEqual(overview["workorders"]["records"], 3) + self.assertEqual(overview["workorders"]["statuses"], {"WAPPR": 1, "APPR": 1, "COMP": 1}) + self.assertEqual(overview["kpis"]["completion_rate"], 33.3) + self.assertFalse(overview["alerts"]["available"]) + + def test_unknown_scenario_fails_before_agent_execution(self): + with self.assertRaises(LookupError): + server.run_scenario("not-a-repository-scenario", "plan-execute") + + def test_missing_provider_credentials_are_reported(self): + with patch.dict("os.environ", {}, clear=True): + with self.assertRaisesRegex(RuntimeError, "LITELLM_BASE_URL, LITELLM_API_KEY"): + server.run_scenario("304", "plan-execute", "litellm_proxy/test-model") + + def test_llm_judge_reports_absent_reference_answer(self): + scenario = { + "id": "1", + "text": "A repository task", + "category": "Repository scenario", + "type": "", + "characteristic_form": "", + } + + result = server.evaluate_result(scenario, "answer", {}, "litellm_proxy/test-model") + + self.assertFalse(result["passed"]) + self.assertIn("no characteristic_form", result["error"]) + + def test_dashboard_tool_policy_blocks_work_order_writes(self): + with self.assertRaisesRegex(ValueError, "not enabled"): + server.call_repository_tool("wo", "generate_work_order", {}) + + def test_dashboard_tool_policy_blocks_persistent_fmsr_edits(self): + with self.assertRaisesRegex(ValueError, "not enabled"): + server.call_repository_tool("fmsr", "add_failure_modes", {}) + + def test_local_profile_runs_only_scenarios_found_in_repository(self): + profile = { + "id": "mini", + "source": "benchmarks/scenario_suite/mini.yaml", + "scenario_count": 2, + "scenario_ids": ["304", "external-scenario"], + "locally_available": ["304"], + "requires_data_reset": True, + } + with ( + patch.object(server, "benchmark_profiles", return_value=[profile]), + patch.object(server, "run_scenario", return_value={"scenario_id": "304"}) as run, + ): + result = server.run_local_profile("mini", "plan-execute") + + run.assert_called_once_with("304", "plan-execute", None) + self.assertEqual(result["skipped_scenario_ids"], ["external-scenario"]) + self.assertFalse(result["data_reset_performed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/servers/fmsr/main.py b/src/servers/fmsr/main.py index ac97efe14..92a8ddecd 100644 --- a/src/servers/fmsr/main.py +++ b/src/servers/fmsr/main.py @@ -370,7 +370,6 @@ def generate_failure_modes( return ErrorResult(error=db_failure_text(exc)) -@mcp.tool(title="Add Failure Modes") def add_failure_modes( asset_class: str, failure_modes: List[str], @@ -454,6 +453,11 @@ def add_failure_modes( return ErrorResult(error=db_failure_text(exc)) +# Dashboard-launched runs inherit AOB_READONLY=1 and must not expose write tools. +if os.environ.get("AOB_READONLY") != "1": + mcp.tool(title="Add Failure Modes")(add_failure_modes) + + # generate_failure_mode_sensor_mapping is intentionally not registered. The # mapping workflow is disabled because large failure-mode/sensor matrices make # benchmark tool calls too expensive and timeout-prone.