diff --git a/apps/agent-forge/e2e/a11y.spec.ts b/apps/agent-forge/e2e/a11y.spec.ts index a3a396e0..38a0b9b5 100644 --- a/apps/agent-forge/e2e/a11y.spec.ts +++ b/apps/agent-forge/e2e/a11y.spec.ts @@ -140,6 +140,8 @@ const ITEM_H1: Record = { '/prompts/e2e-prompt?diff=1,2': 'e2e-prompt', '/prompts/:name': 'e2e-prompt', '/hub/:namespace': 'e2e-namespace', + '/evals/queues/:id': 'support-review', + '/evals/evaluators/:id': 'answered', }; for (const { label, path } of SCANS) { test(`${label} keeps the landmarks and passes axe`, async ({ page }) => { @@ -246,7 +248,7 @@ test('the axe scans see the stubbed daemon data, not the down state', async ({ p ['/hub', 'e2e-namespace'], ['/hub/e2e-namespace', 'order-policy'], ['/evals/queues', 'support-review'], - ['/evals/queues/e2e-queue', 'lookup_order'], + ['/evals/queues/e2e-queue', 'Where is my order?'], ]) { await page.goto(`/?token=e2e-studio-token#${path}`); await expect(main.getByText(text).first(), path).toBeVisible(); diff --git a/apps/agent-forge/e2e/docShots.spec.ts b/apps/agent-forge/e2e/docShots.spec.ts index e6a94e74..dfd5bfa2 100644 --- a/apps/agent-forge/e2e/docShots.spec.ts +++ b/apps/agent-forge/e2e/docShots.spec.ts @@ -393,7 +393,7 @@ test('datasets: one dataset with its examples', async ({ page }) => { test('evaluators: the rule list', async ({ page }) => { await open(page, '/evals/evaluators'); - await expect(rows(page)).toHaveCount(2); + await expect(page.getByRole('main').locator('.ev-card')).toHaveCount(2); await expect(page.getByRole('switch', { name: 'answered enabled' })).toBeChecked(); await shoot(page, 'evaluators'); }); @@ -1214,3 +1214,140 @@ test('x-threads: detail at 375 wide', async ({ page }) => { await shoot(page, 'x-thread-narrow'); await page.screenshot({ path: path.join(OUT, 'studio-x-thread-narrow-full.png'), fullPage: true }); }); + +// --- Evaluators and annotation queues (x- shots, P7) ------------------------- + +const RULES = [ + { rule_id: 'e2e-evaluator', name: 'answered', enabled: true, filter: { agent_id: 'e2e-agent', status: 'ok' }, sample_rate: 1, kind: 'code', config: { source: '(trace) => (trace.output ? 1 : 0)' } }, + { rule_id: 'e2e-judge', name: 'polite', enabled: false, filter: {}, sample_rate: 0.5, kind: 'judge', config: { model: 'openai/gpt-4o-mini', rubric: 'Is the reply polite? Score 1 when it is courteous and calm, 0 otherwise.' } }, + { rule_id: 'x-refund', name: 'refund-policy-cited', enabled: true, filter: { name_contains: 'refund' }, sample_rate: 0.25, kind: 'judge', config: { model: 'anthropic/claude-haiku', rubric: 'Does the reply cite the refund policy?' } }, + { rule_id: 'x-tools', name: 'tool-calls-under-budget-for-every-order-lookup-in-the-support-fleet', enabled: true, filter: { status: 'error' }, sample_rate: 0.1, kind: 'code', config: { source: '(trace) => trace.observations.length < 12 ? 1 : 0' } }, +].map((r) => ({ created_at: NOW - 30 * HOUR, updated_at: NOW - 30 * HOUR, ...r })); + +const QUEUES = [ + { queue_id: 'e2e-queue', name: 'support-review', rubric: [{ name: 'helpfulness', min: 0, max: 5 }, { name: 'tone', min: 1, max: 3 }], counts: { pending: 1, done: 2, skipped: 0 } }, + { queue_id: 'x-refunds', name: 'refund-escalations', rubric: [{ name: 'correct', min: 0, max: 1 }, { name: 'policy cited', min: 0, max: 1 }, { name: 'empathy', min: 1, max: 5 }], counts: { pending: 14, done: 41, skipped: 3 } }, + { queue_id: 'x-new', name: 'multilingual-greetings-weekly-spot-check', rubric: [{ name: 'fluency' }], counts: { pending: 25, done: 0, skipped: 0 } }, + { queue_id: 'x-done', name: 'october-regressions', rubric: [{ name: 'fixed', min: 0, max: 1 }], counts: { pending: 0, done: 57, skipped: 4 } }, +].map((q, i) => ({ created_at: NOW - (i + 1) * 26 * HOUR, ...q })); + +async function evaluatorList(page: Page, list: unknown[]) { + await page.route('**/observe/api/v1/evaluators?**', (route) => + route.request().method() === 'GET' ? json(route, { evaluators: list, total: list.length, limit: 50, offset: 0 }) : route.fallback(), + ); +} + +async function queueList(page: Page, list: unknown[]) { + await page.route('**/observe/api/v1/queues?**', (route) => + route.request().method() === 'GET' ? json(route, { queues: list, total: list.length, limit: 50, offset: 0 }) : route.fallback(), + ); +} + +const main = (page: Page) => page.getByRole('main'); + +test('evals x: evaluators grid', async ({ page }) => { + await evaluatorList(page, RULES); + await open(page, '/evals/evaluators'); + await expect(main(page).getByRole('switch', { name: 'polite enabled' })).toBeVisible(); + await shoot(page, 'x-evals-evaluators-grid'); +}); + +test('evals x: evaluators empty', async ({ page }) => { + await evaluatorList(page, []); + await open(page, '/evals/evaluators'); + await expect(main(page).locator('.empty-state')).toBeVisible(); + await shoot(page, 'x-evals-evaluators-empty'); +}); + +test('evals x: new evaluator form (code rule)', async ({ page }) => { + await open(page, '/evals/evaluators'); + await page.getByRole('button', { name: 'New evaluator' }).first().click(); + await expect(page.getByRole('form', { name: 'New evaluator' })).toBeVisible(); + await shoot(page, 'x-evals-evaluator-new-code'); +}); + +test('evals x: new evaluator form (LLM judge)', async ({ page }) => { + // A stored provider key enables the judge option (no real key is sent anywhere: the route answers in the browser). + await page.route('**/settings/providers', (route) => + route.request().method() === 'GET' + ? json(route, [ + { provider: 'openai', hasKey: true, masked: '', envKey: false }, + { provider: 'anthropic', hasKey: false, masked: null, envKey: false }, + ]) + : route.fallback(), + ); + await open(page, '/evals/evaluators'); + await page.getByRole('button', { name: 'New evaluator' }).first().click(); + const form = page.getByRole('form', { name: 'New evaluator' }); + await form.getByRole('radio', { name: 'LLM judge' }).check(); + await form.locator('[name="evaluator-name"]').fill('helpful'); + await form.locator('[name="evaluator-model"]').fill('gpt-4o-mini'); + await form.locator('[name="evaluator-rubric"]').fill('Score 1 when the reply answers the customer question with the order status, 0 otherwise.'); + await shoot(page, 'x-evals-evaluator-new-judge'); +}); + +test('evals x: evaluator edit view (code rule)', async ({ page }) => { + await open(page, '/evals/evaluators/e2e-evaluator'); + await expect(page.getByRole('form', { name: 'Edit evaluator' })).toBeVisible(); + await shoot(page, 'x-evals-evaluator-edit-code'); +}); + +test('evals x: evaluator edit view (LLM judge)', async ({ page }) => { + await page.route('**/observe/api/v1/evaluators/e2e-judge', (route) => + route.request().method() === 'GET' ? json(route, { evaluator: RULES[1] }) : route.fallback(), + ); + await open(page, '/evals/evaluators/e2e-judge'); + await expect(page.locator('[name="evaluator-rubric"]')).toBeVisible(); + await shoot(page, 'x-evals-evaluator-edit-judge'); +}); + +test('evals x: queues list', async ({ page }) => { + await queueList(page, QUEUES); + await open(page, '/evals/queues'); + await expect(main(page).getByText('refund-escalations').first()).toBeVisible(); + await shoot(page, 'x-evals-queues-list'); +}); + +test('evals x: queues empty', async ({ page }) => { + await queueList(page, []); + await open(page, '/evals/queues'); + await expect(main(page).locator('.empty-state')).toBeVisible(); + await shoot(page, 'x-evals-queues-empty'); +}); + +test('evals x: new queue form with rubric rows', async ({ page }) => { + await queueList(page, QUEUES); + await open(page, '/evals/queues'); + const toggle = page.getByRole('button', { name: 'New queue' }); + if (await toggle.count()) await toggle.first().click(); + const form = page.getByRole('form', { name: 'New annotation queue' }); + await form.locator('[name="queue-name"]').fill('billing-review'); + await form.locator('[name="rubric-name-0"]').fill('correct'); + await form.locator('[name="rubric-min-0"]').fill('0'); + await form.locator('[name="rubric-max-0"]').fill('1'); + await form.getByRole('button', { name: /Add rubric (row|entry)/ }).click(); + await form.locator('[name="rubric-name-1"]').fill('empathy'); + await form.locator('[name="rubric-description-1"]').fill('Did the reply acknowledge the problem?'); + await form.locator('[name="rubric-min-1"]').fill('1'); + await form.locator('[name="rubric-max-1"]').fill('5'); + await shoot(page, 'x-evals-queues-new'); +}); + +test('evals x: queue review at 375 wide', async ({ page }) => { + await page.setViewportSize({ width: 375, height: 800 }); + await open(page, '/evals/queues/e2e-queue'); + await expect(page.locator('[data-review-trace="e2e-trace"]')).toBeVisible(); + await shoot(page, 'x-evals-queue-review-375'); + await page.getByRole('form', { name: 'Scores' }).getByRole('button', { name: 'Skip' }).scrollIntoViewIfNeeded(); + await shoot(page, 'x-evals-queue-review-375-rubric'); +}); + +test('evals x: queue review, queue is empty', async ({ page }) => { + await page.route('**/observe/api/v1/queues/e2e-queue/next', (route) => route.fulfill({ status: 204 })); + await page.route('**/observe/api/v1/queues/e2e-queue', (route) => + json(route, { queue: { ...QUEUES[0], counts: { pending: 0, done: 2, skipped: 1 } } }), + ); + await open(page, '/evals/queues/e2e-queue'); + await expect(main(page).getByText('Queue is empty')).toBeVisible(); + await shoot(page, 'x-evals-queue-empty'); +}); diff --git a/apps/agent-forge/e2e/revamp.spec.ts b/apps/agent-forge/e2e/revamp.spec.ts index 89c9bf1d..e4390fc9 100644 --- a/apps/agent-forge/e2e/revamp.spec.ts +++ b/apps/agent-forge/e2e/revamp.spec.ts @@ -208,9 +208,11 @@ const CONTROLS: Record Promise> = { await expect(main.getByRole('combobox', { name: 'State' })).toHaveValue('true'); await expect(main.getByRole('switch', { name: 'answered enabled' })).toBeChecked(); }, - '/evals/evaluators/:id': async (main) => { + '/evals/evaluators/:id': async (main, page) => { await expect(main.getByRole('textbox', { name: 'Trace id' })).toHaveValue('e2e-trace'); - await expect(main.getByRole('link', { name: 'All evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true'); + // The way back is the header breadcrumb (the screen header sits outside
). + const crumbs = page.getByRole('navigation', { name: 'Breadcrumb' }); + await expect(crumbs.getByRole('link', { name: 'Evaluators' })).toHaveAttribute('href', '#/evals/evaluators?kind=code&enabled=true'); }, '/prompts/:name': async (main) => { await expect(main.getByRole('combobox', { name: 'Compare version A' })).toHaveValue('1'); @@ -323,6 +325,8 @@ test('queue review at 1280px: a trace tree row name never wraps one letter per l await page.setViewportSize({ width: 1280, height: 800 }); await page.goto(`${APP}#/evals/queues/e2e-queue`); await expect(page.locator('[data-review-trace="e2e-trace"]')).toBeVisible(); + // The review opens on the root step's messages; the tree is the Trace view. + await page.getByRole('group', { name: 'Item view' }).getByRole('button', { name: 'Trace' }).click(); const names = page.locator('[data-review-trace="e2e-trace"] .run-name'); await expect(names.first()).toBeVisible(); for (const name of await names.all()) { diff --git a/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx b/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx index 0510356c..cb8d8213 100644 --- a/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx +++ b/apps/agent-forge/src/screens/datasets/DatasetsScreen.tsx @@ -19,9 +19,9 @@ import { fmtInt } from '../../observe/format'; import { Card } from '../../ui/Card'; import { TablePager } from '../../ui/DataTable'; import { EmptyState } from '../../ui/EmptyState'; -import { Skeleton } from '../../ui/Skeleton'; import { useToast } from '../../ui/Toast'; import { EvalsHeader } from '../evals/EvalsTabs'; +import { GridSkeleton } from '../evals/GridSkeleton'; import { DatasetDetail } from './DatasetDetail'; import { DATASETS_PAGE_SIZE, datasetPath, datasetsPath, parseOffset } from './datasetsUrl'; @@ -149,27 +149,6 @@ function DatasetCard({ dataset: d }: { dataset: ObserveDataset }) { ); } -/** The grid's first-load placeholder: six skeleton cards and one status line. */ -function GridSkeleton() { - return ( -
- - Loading datasets - - -
- ); -} - function NewDatasetButton({ open, onClick }: { open: boolean; onClick: () => void }) { return ( @@ -65,7 +80,7 @@ function TestOnTrace({ rule, prefill }: { rule: ObserveEvaluator; prefill: strin

)} {result && ( -
+
Value
{result.value === null ? 'null' : String(result.value)}
Comment
@@ -106,7 +121,7 @@ function EditRule({ rule, back, onSaved }: { rule: ObserveEvaluator; back: strin }; return ( - <> +
void remove()}> + } /> - +
); } +/** The edit view while the rule loads: a card of skeleton lines. */ +const LoadingRule = () => ( + + + +); + /** `#/evals/evaluators/:id`; an unknown rule shows an empty state with the way back. */ export function EvaluatorDetail({ ruleId }: { ruleId: string }) { const { query } = useRoute(); const [refreshKey, setRefreshKey] = useState(0); const rule = useEvaluator(ruleId, refreshKey); const back = evaluatorsPath(parseListState(query)); - const backLink = All evaluators; - if (useDaemonDown(false, [rule])) return ; - let body; - if (rule.loading && !rule.data) body =

Loading evaluator...

; + const down = useDaemonDown(false, [rule]); + const name = rule.data?.name ?? ruleId; + let body: ReactNode; + if (down) body = ; + else if (rule.loading && !rule.data) body = ; else if (rule.error) body = ; - else if (!rule.data) body = ; + else if (!rule.data) + body = ( + + + All evaluators + + } + /> + + ); else body = setRefreshKey((k) => k + 1)} />; return ( <> - -

Evaluator {rule.data ? {rule.data.name} : null}

+ +

Evaluator {rule.data?.name}

{body} ); diff --git a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx index 4081377e..4b6ea627 100644 --- a/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx +++ b/apps/agent-forge/src/screens/evaluators/EvaluatorForm.tsx @@ -1,18 +1,21 @@ /** - * The evaluator form the list's "New evaluator" and the edit view share: - * name, kind (`code` with a function textarea, or `judge` with a model and a - * rubric), the trace filter (agent, status, name contains), the sample rate - * and the enabled flag. A judge needs a provider key, stored in Studio's - * settings or set as the provider's env var where Studio runs (judges run - * with either): without one the judge option is disabled, with a hint that - * links Settings. - * A failed save (a taken name is the daemon's 409) shows inline. + * The evaluator form the list's "New evaluator" and the edit view share, one + * card per section: General (name, enabled, kind), Logic (`code`: the function + * in a mono editor; `judge`: a model and the rubric) and Sampling (the trace + * filter - agent, status, name contains - and the sample rate), over a footer + * that sticks to the bottom of the screen with the save actions. A judge needs + * a provider key, stored in Studio's settings or set as the provider's env var + * where Studio runs (judges run with either): without one the judge option is + * disabled, with a hint that links Settings. + * A failed save (a taken name is the daemon's 409) shows inline in the footer. */ -import { useState, type FormEvent, type ReactNode } from 'react'; +import { useId, useState, type FormEvent, type ReactNode } from 'react'; import type { ProviderKeyStatus } from '../../../shared/wireTypes'; import { ObserveApiError } from '../../observe/observeClient'; import type { NewEvaluator, ObserveAgentRow } from '../../observe/types'; +import { Card } from '../../ui/Card'; import { MOCK_PROVIDER, defaultProvider, providerOptions } from '../playground/request'; +import { KIND_ICON } from './EvaluatorCard'; import { draftProblem, toNewEvaluator, type EvaluatorDraft } from './evaluatorDraft'; type SetField = (field: K, value: EvaluatorDraft[K]) => void; @@ -26,6 +29,54 @@ const saveError = (error: unknown): string => /** The providers a judge can run on: a key stored in Settings or the provider's env var (`envKey`) both count. */ const judgeKeys = (keys: readonly ProviderKeyStatus[]): ProviderKeyStatus[] => keys.map((k) => ({ ...k, hasKey: k.hasKey || k.envKey })); +const KINDS: { kind: EvaluatorDraft['kind']; title: string; description: string }[] = [ + { kind: 'code', title: 'Code', description: 'A JavaScript function of the trace returns the score.' }, + { kind: 'judge', title: 'LLM judge', description: 'A model grades the trace against your rubric.' }, +]; + +/** One kind as a radio card: the native radio, the kind icon, its title (the radio's name) and a line on what it does. */ +function KindOption({ + option, + checked, + disabled, + describedBy, + onChoose, +}: { + option: (typeof KINDS)[number]; + checked: boolean; + disabled: boolean; + describedBy?: string; + onChoose: () => void; +}) { + const id = useId(); + const Icon = KIND_ICON[option.kind]; + return ( + + ); +} + function KindField({ draft, choose, @@ -36,26 +87,25 @@ function KindField({ judgeAllowed: boolean; }) { return ( -
- Kind - - +
+ Kind +
+ {KINDS.map((option) => { + const blocked = option.kind === 'judge' && !judgeAllowed; + return ( + choose(option.kind)} + /> + ); + })} +
{!judgeAllowed && ( -

+

Add a provider key in Settings to use judge evaluators.

)} @@ -63,36 +113,83 @@ function KindField({ ); } -function FilterFields({ draft, set, agents }: { draft: EvaluatorDraft; set: SetField; agents: ObserveAgentRow[] }) { - const known = agents.some((a) => a.agent_id === draft.agentId); +function GeneralCard({ + draft, + set, + choose, + judgeAllowed, +}: { + draft: EvaluatorDraft; + set: SetField; + choose: (kind: EvaluatorDraft['kind']) => void; + judgeAllowed: boolean; +}) { return ( -
- Runs on traces where - - - -
+ +
+
+ + +
+ +
+
+ ); +} + +/** A labelled mono editor surface: the label is its header bar, `tag` a decorative language note. */ +function Editor({ + label, + tag, + name, + value, + placeholder, + onChange, +}: { + label: ReactNode; + tag: string; + name: string; + value: string; + placeholder?: string; + onChange: (value: string) => void; +}) { + return ( +