From 12dc742e6ac5b6dcc8d6e59e23192cf35779cb62 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Wed, 28 Oct 2026 15:41:29 +0600 Subject: [PATCH 01/16] feat: update 10xgraph and 10xgraph-api versions to 1.0.0rc1 in uv.lock --- 10xgraph.json | 24 +- 10xgraph.json.bak | 16 + CHANGELOG.md | 13 + config-editor-ui/src/actions.js | 182 ++++---- config-editor-ui/src/app.css | 12 +- config-editor-ui/src/fields.jsx | 207 +++++++--- config-editor-ui/src/icons.js | 16 +- config-editor-ui/src/inspector.jsx | 144 ++++--- config-editor-ui/src/main.jsx | 43 +- config-editor-ui/src/page.jsx | 211 +--------- config-editor-ui/src/pages.jsx | 326 +++++++++++++++ config-editor-ui/src/sidebar.jsx | 74 ++++ config-editor-ui/src/store.js | 109 ++++- config-editor-ui/src/ui.jsx | 20 +- tenxgraph_api/cli/config_editor/schema.py | 390 +++++++++++++++++- .../cli/config_editor/static/app.css | 2 +- tenxgraph_api/cli/config_editor/static/app.js | 6 +- tenxgraph_api/cli/config_editor/validation.py | 7 + tests/unit_tests/test_config_editor.py | 114 +++++ 19 files changed, 1457 insertions(+), 459 deletions(-) create mode 100644 10xgraph.json.bak create mode 100644 config-editor-ui/src/pages.jsx create mode 100644 config-editor-ui/src/sidebar.jsx diff --git a/10xgraph.json b/10xgraph.json index bad6e381..7c61ba86 100644 --- a/10xgraph.json +++ b/10xgraph.json @@ -6,11 +6,29 @@ "auth": null, "rate_limit": { "enabled": true, - "backend": "memory", + "backend": "redis", "requests": 100, "window": 60, "by": "ip", "trusted_proxy_headers": false, - "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"] + "exclude_paths": [ + "/health", + "/docs", + "/redoc", + "/openapi.json" + ] + }, + "test": { + "path": "tests", + "coverage": true, + "coverage_threshold": 60 + }, + "runs": { + "thread_ids": "server", + "max_concurrent_per_user": 10 + }, + "ag_ui": { + "enabled": false, + "allow_client_tools": false } -} \ No newline at end of file +} diff --git a/10xgraph.json.bak b/10xgraph.json.bak new file mode 100644 index 00000000..bad6e381 --- /dev/null +++ b/10xgraph.json.bak @@ -0,0 +1,16 @@ +{ + "agent": "graph.react:app", + "thread_name_generator": "graph.thread_name_generator:MyNameGenerator", + "store": "graph.dummy_store:store", + "env": ".env", + "auth": null, + "rate_limit": { + "enabled": true, + "backend": "memory", + "requests": 100, + "window": 60, + "by": "ip", + "trusted_proxy_headers": false, + "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"] + } +} \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 03acb6d9..532e8310 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,19 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed + +- **`10xgraph config` editor redesign.** A sidebar layout with a "Get running" page first, a + collapsible sidebar (Ctrl/Cmd+B), search across every setting, defaults shown as values with a + per-field reset, and a read-only reference of the related `.env` variables per section plus a + server environment page. Schema fixes: `graph_introspection`, `websocket.enabled` and + `observability.logfire.console` no longer show a misleading default and offer an explicit + "server default" choice; the websocket section no longer seeds `enabled: true`; + `runs.timeout` must be greater than 0 and can be set to no limit (`null`); the limit fields + offer "Unlimited"; `rate_limit.exclude_paths` is described as extra paths (`/ping` and + `/health/ready` are always skipped) and is no longer seeded; `observability.logfire.console: true` + is now an error, because Logfire accepts only `false` or a `ConsoleOptions` object. + ## [1.0.0rc1] - 2026-10-28 This is the 1.0 release candidate. It requires `10xgraph` 1.0 (`10xgraph>=1.0.0rc1`), and its diff --git a/config-editor-ui/src/actions.js b/config-editor-ui/src/actions.js index 09a41961..19f64c8b 100644 --- a/config-editor-ui/src/actions.js +++ b/config-editor-ui/src/actions.js @@ -1,5 +1,6 @@ import { - TIERS, TOKEN, clone, commit, counts, isDirty, isOn, plural, section, snapshot, store, + NAV, PAGES, TOKEN, clone, commit, counts, currentRouteId, fieldIdFor, isDirty, navItem, plural, + routeHash, section, snapshot, store, } from "./store.js"; // ------------------------------------------------------------------ network @@ -15,15 +16,6 @@ async function api(method, url, body) { const fatal = (title, text) => commit((s) => { s.fatal = { title, text }; }); -function rank(order, id) { - const i = order.indexOf(id); - return i === -1 ? order.length : i; -} - -function openIssues() { - for (const issue of store.issues) if (section(issue.section)) store.open.add(issue.section); -} - export async function load() { if (!TOKEN) return fatal("Open the link printed by 10xgraph config", "This page needs the session token included in that link."); let res; @@ -37,10 +29,14 @@ export async function load() { const data = res.data; commit((s) => { s.schema = data.schema; - const order = TIERS.flatMap((t) => t.sections); - s.sections = [...data.schema.sections].sort((a, b) => rank(order, a.id) - rank(order, b.id)); - s.tiers = TIERS.map((t) => ({ ...t, list: t.sections.map(section).filter(Boolean) })); - s.tiers[s.tiers.length - 1].list.push(...s.sections.filter((sec) => !order.includes(sec.id))); + s.sections = data.schema.sections; + const listed = new Set(NAV.flatMap((g) => g.items)); + const extra = s.sections.map((sec) => sec.id).filter((id) => id !== "core" && !listed.has(id)); + s.nav = NAV.map((g, i) => ({ + label: g.label, + // Sections the nav does not name join the last group, ahead of the environment page. + items: i < NAV.length - 1 ? g.items.filter((id) => PAGES[id] || section(id)) : [...extra, ...g.items], + })); for (const sec of s.sections) for (const f of sec.fields) s.fields.set(f.path.join("."), f); s.config = data.config; s.version = data.version; @@ -51,32 +47,28 @@ export async function load() { const clean = data.exists && !data.parse_error; s.saved = clean ? snapshot() : ""; s.validated = clean ? snapshot() : null; - // Open everything that is required, switched on, or has something to fix. - for (const sec of s.sections) if (!sec.optional || isOn(sec)) s.open.add(sec.id); - openIssues(); }); - const requested = new URLSearchParams(location.hash.slice(1)).get("section"); - if (requested && section(requested)) go(requested); + applyRoute({ focus: false }); + // Swap #token=... for a clean page route (the token stays in memory and sessionStorage). + history.replaceState(null, "", routeHash(store.page)); } +// Runs automatically a moment after edits (see main.jsx). Quiet: results show inline. +let lastTried = null; export async function validate() { - if (store.busy || !guardJson()) return; - commit((s) => { s.busy = "validate"; }); + const snap = snapshot(); + if (!store.schema || store.validated === snap || lastTried === snap) return; + if (Object.values(store.toolParams).some((t) => t.error)) return; + lastTried = snap; try { - const res = await api("POST", "/api/validate", { config: store.config }); - if (!res.ok) return toast("error", res.data.error || "Validation failed."); - store.issues = res.data.issues; - store.validated = snapshot(); - openIssues(); - const { errors, warnings } = counts(); - const show = { label: "Show", run: () => openInspector("problems") }; - if (errors) toast("error", `${plural(errors, "error")} to fix before saving.`, show); - else if (warnings) toast("warning", `Valid, with ${plural(warnings, "warning")}.`, show); - else toast("success", "Configuration is valid."); + const res = await api("POST", "/api/validate", { config: JSON.parse(snap) }); + if (!res.ok) return; + commit((s) => { + s.issues = res.data.issues; + s.validated = snap; + }); } catch { - toast("error", "Cannot reach the config server."); - } finally { - commit((s) => { s.busy = null; }); + // Offline or stopped: the next edit tries again. } } @@ -90,7 +82,6 @@ export async function save() { if (res.data.issues) { store.issues = res.data.issues; store.validated = snapshot(); - openIssues(); } if (res.ok) { store.version = res.data.version; @@ -106,7 +97,7 @@ export async function save() { toast("error", res.data.error, { label: "Reload", run: () => location.reload() }); } else if (res.status === 422) { const { errors } = counts(); - toast("error", `Not saved: ${plural(errors, "error")} to fix first.`, { label: "Show", run: () => openInspector("problems") }); + toast("error", `Not saved: ${plural(errors, "error")} to fix first.`, { label: "Show", run: showFirstError }); } else { toast("error", res.data.error || "Save failed."); } @@ -130,62 +121,115 @@ function guardJson() { export function setSection(sec, on) { commit((s) => { - if (on) { - s.config[sec.key] = clone(sec.default); - s.open.add(sec.id); - } else { - delete s.config[sec.key]; - } + if (on) s.config[sec.key] = clone(sec.default); + else delete s.config[sec.key]; if (sec.key === "remote_tools") s.toolParams = {}; }); } -export function toggleOpen(id) { - commit((s) => { - if (s.open.has(id)) s.open.delete(id); - else s.open.add(id); - }); +function showFirstError() { + const issue = store.issues.find((i) => i.level === "error") || store.issues[0]; + if (issue) go(issue.section, fieldIdFor(issue.path)); } -export function setAllOpen(open) { - commit((s) => { s.open = open ? new Set(s.sections.map((sec) => sec.id)) : new Set(); }); -} +// ------------------------------------------------------------------ routing const smooth = () => (window.matchMedia("(prefers-reduced-motion: reduce)").matches ? "auto" : "smooth"); -// Expand a section and bring it (or one of its fields) into view. -export function go(id, fieldId) { +function resolvePage(id) { + if (id === "core") return "get-running"; + return PAGES[id] || section(id) ? id : "get-running"; +} + +// Read the hash into store.page. A route change leaves search and moves focus to the heading. +export function applyRoute({ focus = true } = {}) { commit((s) => { - s.open.add(id); - s.inspectorOpen = false; + s.page = resolvePage(currentRouteId()); + s.query = ""; + s.drawerOpen = false; }); - history.replaceState(null, "", `#token=${encodeURIComponent(TOKEN)}§ion=${id}`); + if (!focus) return; requestAnimationFrame(() => { - const field = fieldId ? document.getElementById(fieldId) : null; - if (field) { - field.scrollIntoView({ block: "center", behavior: smooth() }); - field.focus({ preventScroll: true }); - return; - } - document.getElementById(`sec-${id}`)?.scrollIntoView({ block: "start", behavior: smooth() }); - document.getElementById(`acc-${id}`)?.focus({ preventScroll: true }); + window.scrollTo(0, 0); + document.getElementById("page-title")?.focus({ preventScroll: true }); }); } -export function scrollToTier(id) { - document.getElementById(`tier-${id}`)?.scrollIntoView({ block: "start", behavior: smooth() }); +// Open a page, and optionally bring one of its fields into view. +export function go(id, fieldId) { + const hash = routeHash(id); + if (location.hash !== hash) history.pushState(null, "", hash); + applyRoute(); + if (!fieldId) return; + requestAnimationFrame(() => requestAnimationFrame(() => { + const field = document.getElementById(fieldId); + if (!field) return; + field.scrollIntoView({ block: "center", behavior: smooth() }); + field.focus({ preventScroll: true }); + })); } -// ------------------------------------------------------------------ inspector +export const hasPage = (id) => !!navItem(id); -export function openInspector(tab) { +export function setQuery(query) { + commit((s) => { s.query = query; }); +} + +// ------------------------------------------------------------------ sidebar + +const reducedMotion = () => window.matchMedia("(prefers-reduced-motion: reduce)").matches; +let sideTimer; + +export function setSidebar(open, { instant = false } = {}) { + try { localStorage.setItem("tg-sidebar", open ? "shown" : "hidden"); } catch { /* private mode */ } + clearTimeout(sideTimer); + const still = instant || reducedMotion(); + commit((s) => { + s.sideOpen = open; + if (open) { s.sideRender = true; s.sideShown = still; } else s.sideShown = false; + if (!open && still) s.sideRender = false; + }); + if (open && !still) requestAnimationFrame(() => requestAnimationFrame(() => commit((s) => { s.sideShown = true; }))); + if (!open && !still) sideTimer = setTimeout(() => commit((s) => { if (!s.sideOpen) s.sideRender = false; }), 200); +} + +export function focusSearch() { + if (!store.sideOpen) setSidebar(true, { instant: true }); + requestAnimationFrame(() => { + const input = document.getElementById("q"); + input?.focus(); + input?.select(); + }); +} + +// ------------------------------------------------------------------ drawer + +export function openDrawer(tab) { commit((s) => { if (tab) s.tab = tab; - s.inspectorOpen = !window.matchMedia("(min-width: 1280px)").matches; + s.drawerOpen = true; }); } -export const closeInspector = () => commit((s) => { s.inspectorOpen = false; }); +export const closeDrawer = () => commit((s) => { s.drawerOpen = false; }); + +export async function copyCommand() { + const text = "10xgraph dev"; + try { + await navigator.clipboard.writeText(text); + } catch { + try { + const ta = document.createElement("textarea"); + ta.value = text; + document.body.append(ta); + ta.select(); + document.execCommand("copy"); + ta.remove(); + } catch { /* nothing more to try */ } + } + commit((s) => { s.copied = true; }); + setTimeout(() => commit((s) => { s.copied = false; }), 2000); +} export async function copyJson() { try { diff --git a/config-editor-ui/src/app.css b/config-editor-ui/src/app.css index 707c1f2c..7c3e614c 100644 --- a/config-editor-ui/src/app.css +++ b/config-editor-ui/src/app.css @@ -139,19 +139,19 @@ @layer components { .input { - @apply block h-10 w-full rounded-lg border border-line-strong bg-sunken px-3 text-base text-ink + @apply block h-11 w-full rounded-lg border border-line-strong bg-sunken px-3 text-base text-ink transition-[border-color,box-shadow] duration-150 placeholder:text-faint hover:border-faint/60 focus:border-brand focus:ring-4 focus:ring-brand-ring focus:outline-none - sm:h-9 sm:text-sm; + disabled:cursor-not-allowed disabled:opacity-50 sm:text-sm; } .input-invalid { @apply border-danger focus:border-danger focus:ring-danger-soft; } .btn { - @apply inline-flex h-10 shrink-0 cursor-pointer items-center justify-center gap-2 rounded-lg px-3.5 + @apply inline-flex h-11 shrink-0 cursor-pointer items-center justify-center gap-2 rounded-lg px-3.5 text-sm font-medium whitespace-nowrap transition-colors duration-150 focus-visible:ring-4 focus-visible:ring-brand-ring focus-visible:outline-none - disabled:cursor-not-allowed disabled:opacity-45 sm:h-9; + disabled:cursor-not-allowed disabled:opacity-45; } .btn-primary { @apply bg-brand-solid text-white shadow-card hover:bg-brand-solid-hover; @@ -163,9 +163,9 @@ @apply text-muted hover:bg-raised hover:text-ink; } .icon-btn { - @apply inline-flex size-10 shrink-0 cursor-pointer items-center justify-center rounded-lg text-muted + @apply inline-flex size-11 shrink-0 cursor-pointer items-center justify-center rounded-lg text-muted transition-colors duration-150 hover:bg-raised hover:text-ink - focus-visible:ring-4 focus-visible:ring-brand-ring focus-visible:outline-none sm:size-9; + focus-visible:ring-4 focus-visible:ring-brand-ring focus-visible:outline-none; } .panel { @apply rounded-xl border border-line bg-surface shadow-card; diff --git a/config-editor-ui/src/fields.jsx b/config-editor-ui/src/fields.jsx index 264feebe..b1f21ff7 100644 --- a/config-editor-ui/src/fields.jsx +++ b/config-editor-ui/src/fields.jsx @@ -1,12 +1,14 @@ import { useEffect, useState } from "preact/hooks"; -import { commit, deletePath, getPath, issueAt, setPath, store, valueOrDefault } from "./store.js"; -import { Chips, FieldIssue, FieldMeta, Row, Select, Toggle } from "./ui.jsx"; +import { + commit, deletePath, fieldChanged, fieldUnmet, getPath, issueAt, setPath, store, unmetHint, +} from "./store.js"; +import { Chips, FieldIssue, Segmented, Select } from "./ui.jsx"; -// Fields grouped by their `group` label, skipping ones whose show_if does not match. -export function fieldGroups(sec) { +// Fields grouped by their `group` label. Fields with an unmet show_if stay in the list; +// the Field itself renders dimmed with a "Used when ..." hint. +export function fieldGroups(fields) { const groups = []; - for (const f of sec.fields) { - if (f.show_if && valueOrDefault(f.show_if.path) !== f.show_if.equals) continue; + for (const f of fields) { const name = f.group || ""; let group = groups.find((g) => g.name === name); if (!group) groups.push(group = { name, fields: [] }); @@ -15,22 +17,32 @@ export function fieldGroups(sec) { return groups; } +// Editing a field of an optional section that is still on defaults switches the section on first. const put = (path, value) => commit((s) => { - if (value === undefined) deletePath(s.config, path); - else setPath(s.config, path, value); + if (value === undefined) { + deletePath(s.config, path); + return; + } + const sec = path.length > 1 ? s.sections.find((x) => x.optional && x.key === path[0]) : null; + if (sec && (s.config[sec.key] === undefined || s.config[sec.key] === null)) { + s.config[sec.key] = JSON.parse(JSON.stringify(sec.default)); + } + setPath(s.config, path, value); }); +export const fieldDomId = (f) => `f-${f.path.join("-")}`; + // Keeps the raw text while typing so "0." or "1e" do not get rewritten mid-edit. -function NumberInput({ id, value, field, invalid }) { +function NumberInput({ id, value, field, invalid, disabled }) { const [text, setText] = useState(String(value ?? "")); useEffect(() => { if (text === "" ? value !== undefined : Number(text) !== value) setText(String(value ?? "")); }, [value]); return ( { const raw = e.currentTarget.value; setText(raw); @@ -41,69 +53,150 @@ function NumberInput({ id, value, field, invalid }) { ); } -export function Field({ field: f }) { - const key = f.path.join("."); - const id = `f-${f.path.join("-")}`; - const value = getPath(store.config, f.path); - const issue = issueAt(key); - const invalid = issue?.level === "error"; +// exclusive_min is only a hint; the server decides what is valid. +function RangeHint({ field, value }) { + if (field.exclusive_min === undefined || typeof value !== "number" || value > field.exclusive_min) return null; + return

Must be greater than {field.exclusive_min}.

; +} + +function NumberControl({ id, field: f, value, invalid }) { + const unlimited = f.unlimited_value !== undefined && value === f.unlimited_value; + const input = ; - if (f.type === "bool") { + if (f.null_label !== undefined) { + const mode = value === null ? "n" : value === undefined ? "d" : "s"; + const seconds = typeof f.default === "number" ? f.default : 300; return ( -
-
- - -
-
- put(f.path, v)} /> +
+ put(f.path, m === "d" ? undefined : m === "n" ? null : seconds)} + /> + {mode === "s" ?
{input}
: null} +
+ ); + } + if (f.unlimited_value !== undefined) { + return ( +
+
+ {input} +
+
); } + return
{input}
; +} - if (f.type === "list") { +function BoolControl({ id, field: f, value }) { + if (f.unset_label !== undefined) { + const options = [["d", "Default"]]; + if (f.allow_true !== false) options.push(["t", "On"]); + options.push(["f", "Off"]); return ( -
- -
- put(f.path, next.length ? next : undefined)} - /> - -
+
+ put(f.path, v === "d" ? undefined : v === "t")} + /> +

Default: {f.unset_label}

); } + return ( + + ); +} - let control; - if (f.type === "select") { - control = ( - put(f.path, e.currentTarget.value === "" ? undefined : e.currentTarget.value)} + } + if (f.type === "select") { + const options = f.options.map((o) => [o, o]); + if (f.unset_label !== undefined) options.unshift(["", f.unset_label]); + return ( + put(f.path, e.currentTarget.value === "" ? undefined : e.currentTarget.value)} + /> + ); +} + +export function Field({ field: f }) { + const key = f.path.join("."); + const id = fieldDomId(f); + const value = getPath(store.config, f.path); + const issue = issueAt(key); + const changed = fieldChanged(f); + const unmet = fieldUnmet(f); + const checkbox = f.type === "bool" && f.unset_label === undefined; + const hasDefault = f.default !== undefined && !changed; + const defaultText = Array.isArray(f.default) ? f.default.join(", ") : String(f.default ?? ""); + + return ( +
+
+ {checkbox ? null : ( + + )} + {checkbox ? null : } + {hasDefault && !checkbox ? ( + default + ) : null} + + {changed ? ( + + ) : null}
- + {f.help ?

{f.help}

: null} +
+ +
+ {unmet ?

{unmetHint(f)}

: null} + +
); } diff --git a/config-editor-ui/src/icons.js b/config-editor-ui/src/icons.js index 4b0232c3..de0e1007 100644 --- a/config-editor-ui/src/icons.js +++ b/config-editor-ui/src/icons.js @@ -26,11 +26,21 @@ export const ICONS = { power: '', spinner: '', book: '', + flag: '', + lock: '', + play: '', + bolt: '', + terminal: '', + search: '', + chevronRight: '', + code: '', + circle: '', external: '', }; export const SECTION_ICONS = { - core: "box", auth: "key", authorization: "shield", rate_limit: "gauge", websocket: "arrows", - ag_ui: "external", - observability: "activity", plugins: "layers", remote_tools: "wrench", test: "flask", evaluation: "target", + "get-running": "flag", environment: "terminal", + core: "flag", auth: "lock", authorization: "shield", token_revocation: "key", rate_limit: "gauge", + observability: "activity", runs: "play", websocket: "bolt", ag_ui: "external", remote_tools: "wrench", + media: "file", plugins: "layers", test: "flask", evaluation: "target", }; diff --git a/config-editor-ui/src/inspector.jsx b/config-editor-ui/src/inspector.jsx index 12526fc7..acc18351 100644 --- a/config-editor-ui/src/inspector.jsx +++ b/config-editor-ui/src/inspector.jsx @@ -1,16 +1,25 @@ -import { closeInspector, copyJson, go, validate } from "./actions.js"; +import { useEffect, useRef } from "preact/hooks"; +import { closeDrawer, copyJson, go } from "./actions.js"; import { SECTION_ICONS } from "./icons.js"; -import { commit, counts, fieldIdFor, isDirty, isStale, issuesFor, plural, store } from "./store.js"; -import { EmptyPanel, Icon } from "./ui.jsx"; +import { commit, counts, fieldIdFor, isDirty, issuesFor, plural, store } from "./store.js"; +import { Icon } from "./ui.jsx"; -// Docked on the right from xl up (sticky, its own scroll); a drawer below that. -export function Inspector() { +// Right-hand drawer with the JSON preview and, only while there are issues, a Problems tab. +export function Drawer() { + const ref = useRef(null); + useEffect(() => { + const dialog = ref.current; + if (!dialog) return; + if (store.drawerOpen && !dialog.open) dialog.showModal(); + else if (!store.drawerOpen && dialog.open) dialog.close(); + }); + const hasIssues = store.issues.length > 0; + const tab = hasIssues && store.tab === "problems" ? "problems" : "json"; const { errors, warnings } = counts(); - const count = errors + warnings; - const tab = (id, label, badge) => ( + const tabButton = (id, label, badge) => ( ); return ( - + ); } function Problems() { - if (!store.schema) return null; - if (store.validated === null) { - return ( - - - - ); - } const { errors, warnings } = counts(); - const stale = isStale(); const groups = store.sections.map((sec) => [sec, issuesFor(sec.id)]).filter(([, list]) => list.length); return (
-
+

{plural(errors, "error")}{" · "} {plural(warnings, "warning")}

- {stale ? : null}
- {stale ? ( -
- You changed the form since this check. -
- ) : null} - {!store.issues.length ? ( - - ) : ( -
- {groups.map(([sec, list]) => ( -
-

- {sec.title} -

-
    - {list.map((issue, i) => { - const error = issue.level === "error"; - return ( -
  • - -
    • - ); - })} -
-
- ))} -
- )} +
+ {groups.map(([sec, list]) => ( +
+

+ {sec.title} +

+
    + {list.map((issue, i) => { + const error = issue.level === "error"; + return ( +
  • + +
    • + ); + })} +
+
+ ))} +
); } diff --git a/config-editor-ui/src/main.jsx b/config-editor-ui/src/main.jsx index f7d700c9..5e574ad8 100644 --- a/config-editor-ui/src/main.jsx +++ b/config-editor-ui/src/main.jsx @@ -1,9 +1,12 @@ import { render } from "preact"; import { useEffect, useReducer } from "preact/hooks"; -import { closeInspector, dismissToast, load, save } from "./actions.js"; -import { Inspector } from "./inspector.jsx"; +import { + applyRoute, dismissToast, focusSearch, load, save, setQuery, setSidebar, validate, +} from "./actions.js"; +import { Drawer } from "./inspector.jsx"; import { Header, Page } from "./page.jsx"; -import { isDirty, store, subscribe } from "./store.js"; +import { Sidebar } from "./sidebar.jsx"; +import { isDirty, snapshot, store, subscribe } from "./store.js"; import { Icon } from "./ui.jsx"; const TOAST_LOOK = { @@ -39,40 +42,60 @@ function Toasts() { ); } +const typing = () => /^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement?.tagName) || document.activeElement?.isContentEditable; + function App() { const [, rerender] = useReducer((n) => n + 1, 0); useEffect(() => { subscribe(rerender); + try { if (localStorage.getItem("tg-sidebar") === "hidden") setSidebar(false, { instant: true }); } catch { /* storage blocked */ } const onKey = (e) => { - if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === "s") { e.preventDefault(); save(); } - else if (e.key === "Escape" && store.inspectorOpen) closeInspector(); + const mod = e.ctrlKey || e.metaKey; + const key = e.key.toLowerCase(); + if (mod && key === "s") { e.preventDefault(); save(); } + else if (mod && key === "b") { e.preventDefault(); setSidebar(!store.sideOpen); } + else if (mod && key === "k") { e.preventDefault(); focusSearch(); } + else if (e.key === "/" && !mod && !e.altKey && !typing() && !store.drawerOpen) { e.preventDefault(); focusSearch(); } + else if (e.key === "Escape" && store.query && !store.drawerOpen) setQuery(""); }; const onUnload = (e) => { if (store.schema && isDirty()) e.preventDefault(); }; + const onHash = () => applyRoute(); document.addEventListener("keydown", onKey); window.addEventListener("beforeunload", onUnload); + window.addEventListener("hashchange", onHash); load(); return () => { document.removeEventListener("keydown", onKey); window.removeEventListener("beforeunload", onUnload); + window.removeEventListener("hashchange", onHash); }; }, []); + // Validate ~400 ms after the last edit. + useEffect(() => { + if (!store.schema || store.validated === snapshot()) return undefined; + const timer = setTimeout(validate, 400); + return () => clearTimeout(timer); + }); + + const withSide = store.sideRender; return ( <> Skip to content
-
-
-
+
+ +
+
-
- {store.inspectorOpen ?
: null} + +
{store.copied ? "Copied" : ""}
); diff --git a/config-editor-ui/src/page.jsx b/config-editor-ui/src/page.jsx index 3abf390a..e8402bbd 100644 --- a/config-editor-ui/src/page.jsx +++ b/config-editor-ui/src/page.jsx @@ -1,9 +1,7 @@ -import { openInspector, save, scrollToTier, setAllOpen, setSection, toggleOpen, validate } from "./actions.js"; -import { Field, fieldGroups } from "./fields.jsx"; -import { SECTION_ICONS } from "./icons.js"; -import { WIDGET_FIELD_IDS, counts, isDirty, isOn, issuesFor, plural, store } from "./store.js"; -import { Callout, EmptyPanel, Group, Icon, IssueCallout, Pill, Toggle } from "./ui.jsx"; -import { AuthWidget, AuthorizationWidget, RemoteToolsWidget } from "./widgets.jsx"; +import { openDrawer, save, setSidebar } from "./actions.js"; +import { PaneContent } from "./pages.jsx"; +import { isDirty, store } from "./store.js"; +import { Callout, EmptyPanel, Icon, Pill } from "./ui.jsx"; // ------------------------------------------------------------------ header @@ -28,11 +26,16 @@ function FileInfo() { } export function Header() { - const { errors, warnings } = counts(); const ready = !!store.schema && !store.fatal; - const validating = store.busy === "validate"; + const label = store.sideOpen ? "Hide sidebar" : "Show sidebar"; return ( -
+
+
{/* The 10xGraph mark, same geometry as agentflow-docs/src/components/Logo.astro. */} -
- ); -} - -function TierLink({ tier }) { - const on = tier.list.filter(isOn).length; - const { errors, warnings } = counts(store.issues.filter((i) => tier.list.some((s) => s.id === i.section))); - return ( - { e.preventDefault(); scrollToTier(tier.id); }} - > - {tier.title} - {on}/{tier.list.length} - {errors || warnings ? : null} - - ); -} - -function Tier({ tier }) { - const on = tier.list.filter(isOn).length; - return ( -
-
-
-

{tier.title}

- {on} of {tier.list.length} on -
-

{tier.tagline}

-
-
- {tier.list.map((sec) => )} -
-
- ); -} - -function SectionCard({ sec }) { - const on = isOn(sec); - const open = store.open.has(sec.id); - const { errors, warnings } = counts(issuesFor(sec.id)); - const bodyId = `body-${sec.id}`; - - let status = null; - if (errors) { - status = ( - - {errors} - - ); - } else if (warnings) { - status = ( - - {warnings} - - ); - } - - return ( -
-
-

- -

- {sec.optional ? ( -
- - setSection(sec, v)} /> -
- ) : null} -
- {/* grid-rows 0fr -> 1fr animates the height without measuring it. */} -
-
-
- -
-
-
-
- ); -} - -function SectionBody({ sec, on }) { - if (!on) { - return ( -
-

- {sec.key} is not in 10xgraph.json. {sec.off_summary || ""} -

- -
- ); - } - const inline = new Set([...sec.fields.map((f) => f.path.join(".")), ...Object.keys(WIDGET_FIELD_IDS)]); - const general = issuesFor(sec.id).filter((i) => !inline.has(i.path)); - return ( - <> - {general.length ?
{general.map((issue, i) => )}
: null} - {sec.notes?.length ? : null} - {sec.widget === "auth" ? : null} - {sec.widget === "authorization" ? : null} - {sec.widget === "remote_tools" ? : null} - {fieldGroups(sec).map((group) => ( - -
- {group.fields.map((f) => )} -
- - ))} + ); } - diff --git a/config-editor-ui/src/pages.jsx b/config-editor-ui/src/pages.jsx new file mode 100644 index 00000000..4b8d9afc --- /dev/null +++ b/config-editor-ui/src/pages.jsx @@ -0,0 +1,326 @@ +import { copyCommand, go, setSection } from "./actions.js"; +import { Field, fieldGroups } from "./fields.jsx"; +import { SECTION_ICONS } from "./icons.js"; +import { + PAGES, WIDGET_FIELD_IDS, getPath, isAgentReady, isOn, issueAt, issuesFor, navItem, section, store, + valueOrDefault, +} from "./store.js"; +import { Callout, Icon, IssueCallout, Segmented } from "./ui.jsx"; +import { AuthWidget, AuthorizationWidget, RemoteToolsWidget } from "./widgets.jsx"; + +const Heading = ({ children, code }) => ( +

+ {children} + {code ? {code} : null} +

+); + +const Lead = ({ children }) =>

{children}

; + +const Eyebrow = ({ children }) =>

{children}

; + +// Link rows: icon, title, one line of text. +function PageLink({ id, text, warn, compact }) { + const item = navItem(id); + if (!item) return null; + return ( + { e.preventDefault(); go(id); }} + class={`focus-ring group flex items-center gap-3.5 border-t border-line py-2 last:border-b ${compact ? "min-h-12" : "min-h-14"}`} + > + + + {item.title} + {text ?? item.description} + + + + ); +} + +// ------------------------------------------------------------------ get running + +function offText(id) { + return `Not set. ${section(id)?.off_summary || ""}`.trim(); +} + +function authSummary() { + const a = store.config.auth; + if (a === undefined || a === null) return { text: offText("auth") }; + if (a === "jwt") return { text: "JWT bearer tokens." }; + if (a && typeof a === "object") return { text: `Custom backend${a.path ? `: ${a.path}` : "."}` }; + return { text: String(a) }; +} + +function rateSummary() { + const r = store.config.rate_limit; + if (r === undefined || r === null) return { text: offText("rate_limit") }; + const v = (k) => valueOrDefault(["rate_limit", k]); + if (v("enabled") === false) return { text: "Settings kept, not enforced." }; + const by = { ip: "per IP", user: "per user", global: "across all clients" }[v("by")] || v("by"); + const warn = issueAt("rate_limit.backend")?.level === "warning"; + const custom = ["requests", "window", "by"].some((k) => getPath(store.config, ["rate_limit", k]) !== undefined + && getPath(store.config, ["rate_limit", k]) !== store.fields.get(`rate_limit.${k}`)?.default); + return { + text: `${v("requests")} requests per ${v("window")} s, ${by}${custom ? "" : " (default)"}.${warn ? " The memory backend counts per process." : ""}`, + warn, + }; +} + +function traceSummary() { + const o = store.config.observability; + if (o === undefined || o === null) return { text: offText("observability") }; + const on = []; + if (valueOrDefault(["observability", "logfire", "enabled"])) on.push("Logfire"); + if (valueOrDefault(["observability", "langsmith", "enabled"])) on.push("LangSmith"); + const level = valueOrDefault(["observability", "level"]); + return { text: `Level ${level}${on.length ? `, sending to ${on.join(" and ")}` : ", no destination chosen yet"}.` }; +} + +function GetRunning() { + const core = section("core"); + const ready = isAgentReady(); + const general = core ? issuesFor("core").filter((i) => !core.fields.some((f) => f.path.join(".") === i.path)) : []; + const rows = [["auth", authSummary()], ["rate_limit", rateSummary()], ["observability", traceSummary()]]; + const all = store.nav.flatMap((g) => g.items).filter((id) => id !== "get-running"); + return ( + <> + {ready ? "You're ready to run" : "One step to run your agent"} + {PAGES["get-running"].description} +
+ + Required: {ready ? "1" : "0"} of 1 done +
+ {general.length ?
{general.map((i, n) => )}
: null} +
+ {core ? fieldGroups(core.fields).flatMap((g) => g.fields).map((f) => ) : null} +
+ {ready ? ( +
+
+ Ready for 10xgraph dev +
+
+ 10xgraph dev + {store.copied ? Copied : null} + +
+

Everything else already has working defaults.

+
+ ) : ( +

Add the agent graph above, then run 10xgraph dev.

+ )} + Before production (optional) +
{rows.map(([id, s]) => )}
+ What you can configure +
{all.map((id) => )}
+ + ); +} + +// ------------------------------------------------------------------ section pages + +export function EnvList({ vars }) { + return ( +
+ {vars.map((v) => [ +
{v.name}
, +
{v.purpose}
, + ])} +
+ ); +} + +function EnvDisclosure({ vars, open }) { + if (!vars?.length) return null; + return ( +
+ + Set in .env + +

Read only. The editor never reads .env, so only names are shown.

+ +
+ ); +} + +function SectionWidget({ sec }) { + if (sec.widget === "auth") return ; + if (sec.widget === "authorization") return ; + if (sec.widget === "remote_tools") return ; + return null; +} + +// Issues that no rendered field or widget shows inline. +export function generalIssues(sec) { + const inline = new Set([...sec.fields.map((f) => f.path.join(".")), ...Object.keys(WIDGET_FIELD_IDS)]); + return issuesFor(sec.id).filter((i) => !inline.has(i.path)); +} + +// Fields and widget of a section, no heading. Shared by the page and search results. +export function SectionFields({ sec }) { + const on = isOn(sec); + const widget = sec.widget && (on || !sec.optional); + return ( + <> + {widget ?
: null} +
+ {fieldGroups(sec.fields).map((g, i) => ( +
+ {g.name ? ( +

{g.name}

+ ) : null} + {g.fields.map((f) => )} +
+ ))} +
+ + ); +} + +function SectionPage({ sec }) { + const on = isOn(sec); + const general = generalIssues(sec); + return ( + <> + {sec.title} + {sec.description} + {general.length ?
{general.map((i, n) => )}
: null} + {sec.optional ? ( +
+ setSection(sec, v === "c")} + /> +

+ {on ? `Saved in 10xgraph.json under ${sec.key}.` : sec.off_summary} +

+
+ ) : null} + {sec.notes?.length ?
: null} + + + + ); +} + +// ------------------------------------------------------------------ environment + +function EnvironmentPage() { + const groups = store.schema.environment || []; + return ( + <> + {PAGES.environment.title} + + These variables are set in .env or your deployment environment. The editor never reads that file, so only + names and purposes are shown. + + {groups.map((g) => ( +
+

{g.title}

+ {g.note ?

{g.note}

: null} + +
+ ))} + + ); +} + +// ------------------------------------------------------------------ search + +const norm = (s) => String(s ?? "").toLowerCase(); + +function search(terms) { + const hit = (text) => terms.every((w) => norm(text).includes(w)); + const results = []; + for (const sec of store.sections) { + const secHit = hit(`${sec.title} ${sec.description} ${sec.key || ""}`); + const fields = sec.fields.filter((f) => secHit || hit(`${f.label} ${f.help || ""} ${f.path.join(".")}`)); + const env = (sec.env || []).filter((v) => hit(`${v.name} ${v.purpose}`)); + const widget = !!sec.widget && secHit; + if (fields.length || env.length || widget) results.push({ id: sec.id, sec, fields, env, widget }); + } + const envGroups = (store.schema.environment || []) + .map((g) => ({ title: g.title, vars: g.vars.filter((v) => hit(`${v.name} ${v.purpose} ${g.title}`)) })) + .filter((g) => g.vars.length); + return { results, envGroups }; +} + +function SearchResults() { + const terms = norm(store.query).split(/\s+/).filter(Boolean); + const { results, envGroups } = search(terms); + const count = results.reduce((n, r) => n + r.fields.length + r.env.length + (r.widget ? 1 : 0), 0) + + envGroups.reduce((n, g) => n + g.vars.length, 0); + if (!count) { + return ( + <> + No matches +

Nothing matches "{store.query}". Try a shorter word, or press Esc to go back.

+ + ); + } + return ( + <> + {count} {count === 1 ? "setting matches" : "settings match"} + {results.map(({ id, sec, fields, env, widget }) => ( +
+

+ {sec.title} + { e.preventDefault(); go(id); }} + class="focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline" + > + Open page + +

+ {widget ?
: null} + {fields.map((f) => )} + {env.length ? ( +
+ + Set in .env + + +
+ ) : null} +
+ ))} + {envGroups.length ? ( +
+

+ {PAGES.environment.title} + { e.preventDefault(); go("environment"); }} + class="focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline" + > + Open page + +

+ {envGroups.map((g) => ( +
+

{g.title}

+ {g.note ?

{g.note}

: null} + +
+ ))} +
+ ) : null} + + ); +} + +// ------------------------------------------------------------------ router + +export function PaneContent() { + if (store.query.trim()) return ; + if (store.page === "get-running") return ; + if (store.page === "environment") return ; + const sec = section(store.page); + return sec ? : ; +} diff --git a/config-editor-ui/src/sidebar.jsx b/config-editor-ui/src/sidebar.jsx new file mode 100644 index 00000000..384aeca5 --- /dev/null +++ b/config-editor-ui/src/sidebar.jsx @@ -0,0 +1,74 @@ +import { go, setQuery } from "./actions.js"; +import { SECTION_ICONS } from "./icons.js"; +import { navItem, section, sectionChanged, store, worstIssue } from "./store.js"; +import { Icon } from "./ui.jsx"; + +function Markers({ id }) { + const changed = !!section(id) && sectionChanged(section(id)); + const issue = worstIssue(id); + if (!changed && !issue) return null; + return ( + + {changed ? : null} + {issue ? ( + + ) : null} + + ); +} + +function NavLink({ id }) { + const item = navItem(id); + if (!item) return null; + const current = !store.query && store.page === id; + return ( + { e.preventDefault(); go(id); }} + ref={(el) => { if (el && current && window.innerWidth < 1024) el.scrollIntoView({ block: "nearest", inline: "nearest" }); }} + class={`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${current + ? "bg-brand-soft text-ink" : "text-muted hover:bg-raised hover:text-ink"}`} + > + + {item.title} + + + ); +} + +// Sticky column from lg up; a horizontal tab row above the page below that. +export function Sidebar() { + if (!store.sideRender) return null; + return ( + + ); +} diff --git a/config-editor-ui/src/store.js b/config-editor-ui/src/store.js index daa5c3ca..54232982 100644 --- a/config-editor-ui/src/store.js +++ b/config-editor-ui/src/store.js @@ -2,34 +2,65 @@ // commit(), which re-renders the app. The tree is small, so a full render is cheap // and Preact's diff keeps focus, selection, and scroll position intact. -// Sections not listed here land in the last tier, so new schema entries still render. -export const TIERS = [ - { - id: "essential", title: "Essentials", - tagline: "What the server needs to boot. Start here.", - sections: ["core", "plugins"], - }, - { - id: "recommended", title: "Recommended", - tagline: "Lock down access and see what your agents do before you ship.", - sections: ["auth", "authorization", "rate_limit", "observability"], +// Sidebar structure. Section ids not listed here land in the last group (before the +// environment page), so new schema entries still render. "get-running" and "environment" +// are pages of their own rather than schema sections. +export const NAV = [ + { label: null, items: ["get-running"] }, + { label: "Before production", items: ["auth", "authorization", "rate_limit", "observability", "token_revocation"] }, + { label: "Runs and streaming", items: ["runs", "websocket", "ag_ui", "remote_tools", "media"] }, + { label: "Project wiring", items: ["plugins"] }, + { label: "Developer tools", items: ["test", "evaluation"] }, + { label: "Reference", items: ["environment"] }, +]; + +export const PAGES = { + "get-running": { + title: "Get running", icon: "flag", + description: "Only the agent graph is required. Everything else is optional.", }, - { - id: "advanced", title: "Advanced", - tagline: "Tune connection limits, client-side tools, and your test loop.", - sections: ["websocket", "ag_ui", "remote_tools", "test", "evaluation"], + environment: { + title: "Server environment (.env)", icon: "terminal", + description: "Variables the server reads from .env. The editor never reads that file, so values are not shown.", }, -]; +}; // Issues a widget renders beside its own input rather than at the top of the section. export const WIDGET_FIELD_IDS = { "auth.path": "auth-path", authorization: "authz-path" }; -export const TOKEN = new URLSearchParams(location.hash.slice(1)).get("token") || ""; +// The page token arrives in the URL hash (#token=...). Read it once, keep it in memory, and +// mirror it to sessionStorage so a reload still works after the hash switches to page routing. +const TOKEN_KEY = "tg-config-token"; +let tokenSaved = false; +function readToken() { + const fromHash = new URLSearchParams(location.hash.slice(1)).get("token") || ""; + try { + if (fromHash) sessionStorage.setItem(TOKEN_KEY, fromHash); + const stored = sessionStorage.getItem(TOKEN_KEY) || ""; + tokenSaved = stored !== "" && stored === (fromHash || stored); + return fromHash || stored; + } catch { + return fromHash; + } +} +export const TOKEN = readToken(); + +// "#id" normally. If sessionStorage is unavailable the token has to stay in the hash. +export const routeHash = (id) => (tokenSaved || !TOKEN ? `#${id}` : `#token=${encodeURIComponent(TOKEN)}&page=${id}`); + +export function currentRouteId() { + const raw = location.hash.slice(1); + if (raw.includes("=")) { + const params = new URLSearchParams(raw); + return params.get("page") || params.get("section") || ""; + } + try { return decodeURIComponent(raw); } catch { return raw; } +} export const store = { schema: null, sections: [], // in page order - tiers: [], // TIERS with the resolved section objects in `list` + nav: [], // NAV with unlisted sections appended: [{label, items: [id]}] fields: new Map(), // "a.b.c" -> field definition config: {}, version: null, @@ -39,10 +70,15 @@ export const store = { saved: "", // snapshot of the last loaded/saved config validated: null, // snapshot the current issues belong to issues: [], - open: new Set(), // ids of expanded sections - tab: "problems", - inspectorOpen: false, // drawer state below the xl breakpoint - busy: null, // "validate" | "save" | null + page: "get-running", // id of the current page (a nav item) + query: "", // search text; non-empty replaces the page with results + sideOpen: true, // sidebar wanted + sideRender: true, // sidebar in the layout (false only after the slide-out finished) + sideShown: true, // slide state, flips a frame after sideRender + drawerOpen: false, // 10xgraph.json drawer + tab: "json", + copied: false, // "10xgraph dev" copy feedback + busy: null, // "save" | null toolParams: {}, // remote tool index -> {text, error} while JSON is being edited toasts: [], fatal: null, // {title, text} when the page cannot load @@ -94,6 +130,7 @@ export function valueOrDefault(path) { return v !== undefined ? v : store.fields.get(path.join("."))?.default; } +export const deepEqual = (a, b) => JSON.stringify(a) === JSON.stringify(b); export const section = (id) => store.sections.find((s) => s.id === id); export const isDirty = () => snapshot() !== store.saved; export const isOn = (sec) => !sec.optional || (sec.key in store.config && store.config[sec.key] !== null); @@ -105,3 +142,31 @@ export function counts(list = store.issues) { return { errors, warnings: list.length - errors }; } export const fieldIdFor = (path) => WIDGET_FIELD_IDS[path] || `f-${path.replaceAll(".", "-")}`; + +// A field whose value is set and differs from its default. +export const fieldChanged = (f) => { + const v = getPath(store.config, f.path); + return v !== undefined && !deepEqual(v, f.default); +}; +export const sectionChanged = (sec) => { + if (sec.id === "core") return false; + return sec.optional ? isOn(sec) : sec.fields.some(fieldChanged); +}; +// Highest-severity issue of a section, or null. +export function worstIssue(id) { + const list = issuesFor(id); + return list.find((i) => i.level === "error") || list[0] || null; +} +// show_if that does not match the current value: the field is dimmed with a hint. +export const fieldUnmet = (f) => !!f.show_if && valueOrDefault(f.show_if.path) !== f.show_if.equals; +export function unmetHint(f) { + const dep = store.fields.get(f.show_if.path.join(".")); + const name = dep?.label || f.show_if.path.join("."); + const eq = f.show_if.equals; + return `Used when ${name} is ${eq === true ? "on" : eq === false ? "off" : String(eq)}.`; +} +export const navItem = (id) => { + const sec = section(id); + return PAGES[id] ? { id, ...PAGES[id] } : sec ? { id, title: sec.title, description: sec.description, icon: null } : null; +}; +export const isAgentReady = () => /^[\w.]+:[\w.]+$/.test(String(store.config.agent || "")); diff --git a/config-editor-ui/src/ui.jsx b/config-editor-ui/src/ui.jsx index 04d23e55..a3ef5352 100644 --- a/config-editor-ui/src/ui.jsx +++ b/config-editor-ui/src/ui.jsx @@ -2,6 +2,7 @@ import { useRef } from "preact/hooks"; import { ICONS } from "./icons.js"; export function Icon({ name, class: cls = "size-4" }) { + if (!/(^|\s)size-/.test(cls)) cls = `size-4 ${cls}`; return (
+ ); +} + export function Select({ id, options, value, invalid, onChange }) { return (
@@ -60,7 +78,7 @@ export function Chips({ id, values, placeholder, suggestions, onChange }) { const listId = suggestions ? `${id}-options` : undefined; return (
{ if (e.target === e.currentTarget) input.current?.focus(); }} > {values.map((v, i) => ( diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index 30dd5bd1..ad276b21 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -16,6 +16,7 @@ "widget": None, # custom renderer: "auth", "authorization", "remote_tools" "fields": [...], # generic fields, paths are relative to the root "notes": ["..."], # extra hints (secrets that belong in .env, ...) + "env": [{"name": "REDIS_URL", "purpose": "..."}], # related variables, read-only } Field shape:: @@ -31,9 +32,19 @@ "step": 1, # number only "required": False, "default": ..., # value the server assumes when the key is absent + "unset_label": "...", # offer a choice that removes the key; says what the server does + "allow_true": False, # bool only: render just Default / Off, never On + "unlimited_value": 0, # number only: an "Unlimited" checkbox that writes this value + "null_label": "No limit", # number only: a choice that writes JSON null + "exclusive_min": 0, # number only: the value must be greater than this "group": "Backend", # fields sharing a group render together "show_if": {"path": [...], "equals": value}, } + +``build_schema()`` also returns ``environment``: server environment variable groups that are +not tied to one section, as ``[{"title": ..., "vars": [{"name": ..., "purpose": ...}]}]``. +A group may also carry an optional ``note`` (a short string shown under the group title). +The editor only lists names; it never reads a ``.env`` file. """ from __future__ import annotations @@ -133,6 +144,24 @@ "JWT needs JWT_SECRET_KEY and JWT_ALGORITHM in your .env file.", "Custom auth points at a BaseAuth subclass, as module:attribute.", ], + "env": [ + { + "name": "JWT_SECRET_KEY", + "purpose": "Signing secret for JWT tokens. Required when auth is jwt.", + }, + { + "name": "JWT_ALGORITHM", + "purpose": "Signing algorithm for JWT tokens. Defaults to HS256.", + }, + { + "name": "JWT_ISSUER", + "purpose": "When set, tokens must carry a matching iss claim.", + }, + { + "name": "JWT_AUDIENCE", + "purpose": "When set, tokens must carry a matching aud claim.", + }, + ], }, { "id": "authorization", @@ -164,10 +193,10 @@ "label": "Graph introspection", "type": "select", "options": ["open", "scoped"], + "unset_label": "Automatic: scoped in production, open in development", "help": ( "Who may read the graph structure, tool schemas and state schema. scoped " - "requires the graph:read scope. Defaults to scoped in production, open in " - "development." + "requires the graph:read scope." ), }, { @@ -181,6 +210,12 @@ ), }, ], + "env": [ + { + "name": "REDIS_URL", + "purpose": "Redis for the revocation list when no redis key is set.", + }, + ], }, { "id": "rate_limit", @@ -195,7 +230,6 @@ "requests": 100, "window": 60, "by": "ip", - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"], }, "fields": [ { @@ -304,9 +338,23 @@ { "path": ["rate_limit", "exclude_paths"], "group": "Excluded paths", - "label": "Excluded paths", + "label": "Extra paths to skip", "type": "list", - "placeholder": "/ping", + "placeholder": "/docs", + "help": ( + "Paths that are never rate limited. /ping and /health/ready are always " + "skipped. API docs paths (/docs, /redoc, /openapi.json) are limited unless " + "listed here. The server default is an empty list." + ), + }, + ], + "env": [ + { + "name": "REDIS_URL", + "purpose": ( + "Not read by rate limiting itself. Reference it as ${REDIS_URL} in " + "rate_limit.redis.url; required then when the backend is redis." + ), }, ], }, @@ -323,16 +371,20 @@ "/v1/graph/ws and /v1/graph/live stay mounted with the default caps " "(1000 per process, 10 per user)." ), - "default": {"enabled": True, "max_connections": 1000, "max_connections_per_user": 10}, + "default": {"max_connections": 1000, "max_connections_per_user": 10}, "fields": [ { "path": ["websocket", "enabled"], - "default": True, "label": "Enabled", "type": "bool", + "unset_label": ( + "Default: routes stay mounted; only a warning if the websockets package " + "is missing" + ), "help": ( - "Mounts /v1/graph/ws and /v1/graph/live. Needs the websockets package " - '(pip install "10xgraph-api[websocket]"). Off: neither route exists.' + "On mounts /v1/graph/ws and /v1/graph/live and refuses to start without " + 'the websockets package (pip install "10xgraph-api[websocket]"). Off: ' + "neither route exists." ), }, { @@ -342,7 +394,8 @@ "min": 0, "step": 1, "placeholder": "1000", - "help": "Per worker process. Defaults to 1000; 0 means unlimited.", + "unlimited_value": 0, + "help": "Per worker process. Defaults to 1000; 0 or null means unlimited.", }, { "path": ["websocket", "max_connections_per_user"], @@ -351,7 +404,11 @@ "min": 0, "step": 1, "placeholder": "10", - "help": "Stops one account holding every slot. Defaults to 10; 0 means unlimited.", + "unlimited_value": 0, + "help": ( + "Stops one account holding every slot. Defaults to 10; 0 or null means " + "unlimited." + ), }, { "path": ["websocket", "realtime_models"], @@ -427,18 +484,21 @@ "step": 1, "placeholder": "10", "default": 10, - "help": "HTTP runs in flight per user, per process. 0 means unlimited.", + "unlimited_value": 0, + "help": "HTTP runs in flight per user, per process. 0 or null means unlimited.", }, { "path": ["runs", "timeout"], "label": "Run timeout (seconds)", "type": "number", - "min": 0, + "exclusive_min": 0, + "null_label": "No limit", "step": 1, "help": ( - "A run that takes longer is cancelled. Empty uses the default: 300 in " - "production (MODE=production), no limit in development. Set null in the " - "file for no limit in production." + "A run that takes longer is cancelled. Must be greater than 0; decimals " + "are allowed. Empty uses the default: 300 in production (MODE=production), " + "no limit in development. No limit writes null, which also removes the " + "production limit." ), }, { @@ -454,6 +514,22 @@ ), }, ], + "env": [ + { + "name": "TENXGRAPH_MAX_PARALLEL_TOOL_CALLS", + "purpose": ( + "Tool calls one turn may run at once when max_parallel_tool_calls is " + "not set above." + ), + }, + { + "name": "MODE", + "purpose": ( + "production gives runs a 300 second timeout when none is set above; " + "development sets no limit." + ), + }, + ], }, { "id": "ag_ui", @@ -510,6 +586,81 @@ ), }, ], + "env": [ + { + "name": "MEDIA_STORAGE_TYPE", + "purpose": "Where uploads are stored: memory, local or cloud. Defaults to local.", + }, + { + "name": "MEDIA_STORAGE_PATH", + "purpose": "Upload folder when the storage type is local. Defaults to ./uploads.", + }, + { + "name": "MEDIA_MAX_SIZE_MB", + "purpose": "Largest accepted upload in megabytes. Defaults to 25.", + }, + { + "name": "DOCUMENT_HANDLING", + "purpose": ( + "How uploaded documents reach the model: extract_text, pass_raw or skip. " + "Defaults to extract_text." + ), + }, + { + "name": "MEDIA_ALLOWED_CONTENT_TYPES", + "purpose": ( + "Comma-separated upload allowlist such as image/*,application/pdf. " + "Empty accepts any type." + ), + }, + { + "name": "MEDIA_REQUIRE_OWNER", + "purpose": ( + "Deny files with no recorded owner. Unset means on in production, off in " + "development." + ), + }, + { + "name": "MEDIA_CLOUD_PROVIDER", + "purpose": "Cloud provider, aws or gcp. Used when the storage type is cloud.", + }, + { + "name": "MEDIA_CLOUD_BUCKET", + "purpose": "Bucket name. Required when the storage type is cloud.", + }, + { + "name": "MEDIA_CLOUD_REGION", + "purpose": "Bucket region. Defaults to us-east-1.", + }, + { + "name": "MEDIA_CLOUD_PREFIX", + "purpose": "Key prefix for stored objects. Defaults to 10xgraph-media.", + }, + { + "name": "MEDIA_CLOUD_ACCESS_KEY_ID", + "purpose": "Access key id for cloud storage.", + }, + { + "name": "MEDIA_CLOUD_SECRET_ACCESS_KEY", + "purpose": "Secret access key for cloud storage.", + }, + { + "name": "MEDIA_CLOUD_SESSION_TOKEN", + "purpose": "Temporary session token for cloud storage credentials.", + }, + { + "name": "MEDIA_CLOUD_PROJECT_ID", + "purpose": "Project id when the cloud provider is gcp.", + }, + { + "name": "MEDIA_CLOUD_CREDENTIALS_JSON", + "purpose": "Service account credentials as JSON for cloud storage.", + }, + { + "name": "MEDIA_SIGNED_URL_TTL_SECONDS", + "purpose": "How long a signed download URL stays valid. Defaults to 3600.", + }, + ], }, { "id": "observability", @@ -571,9 +722,14 @@ { "path": ["observability", "logfire", "console"], "group": "Logfire", - "default": False, "label": "Print spans to console", "type": "bool", + "unset_label": "Default: Logfire's own console behavior", + "allow_true": False, + "help": ( + "Only Default or Off work from this file. Logfire takes False or a " + "ConsoleOptions object, which JSON cannot express, so true is rejected." + ), "show_if": {"path": ["observability", "logfire", "enabled"], "equals": True}, }, { @@ -601,6 +757,36 @@ }, ], "notes": ["Tokens stay in .env: LOGFIRE_TOKEN and LANGSMITH_API_KEY."], + "env": [ + { + "name": "LOGFIRE_TOKEN", + "purpose": "Logfire write token. Needed to send spans to Logfire.", + }, + { + "name": "LANGSMITH_API_KEY", + "purpose": "LangSmith API key. Required when LangSmith is enabled.", + }, + { + "name": "TENXGRAPH_TRACE_USER_ID_KEY", + "purpose": "Secret that keys the hash of user ids in traces when the mode is hash.", + }, + { + "name": "OTEL_ENABLED", + "purpose": "Export OpenTelemetry traces from the API server. Defaults to false.", + }, + { + "name": "OTEL_LEVEL", + "purpose": "Detail of the OpenTelemetry traces: spans, standard or full.", + }, + { + "name": "OTEL_SERVICE_NAME", + "purpose": "Service name on OpenTelemetry traces. Defaults to 10xgraph-api.", + }, + { + "name": "OTEL_EXPORTER_OTLP_ENDPOINT", + "purpose": "OTLP endpoint that receives the traces.", + }, + ], }, { "id": "plugins", @@ -654,6 +840,12 @@ ), }, ], + "env": [ + { + "name": "REDIS_URL", + "purpose": "Redis for the ownership cache when the redis key above is empty.", + }, + ], }, { "id": "remote_tools", @@ -731,10 +923,172 @@ }, ] +_ENVIRONMENT: list[dict[str, Any]] = [ + { + "title": "Mode and logging", + "vars": [ + { + "name": "MODE", + "purpose": ( + "development or production. Production turns on stricter defaults, " + "including the API docs being off." + ), + }, + { + "name": "IS_DEBUG", + "purpose": "Debug output. Always forced off when MODE is production.", + }, + {"name": "LOG_LEVEL", "purpose": "Python log level such as INFO or DEBUG."}, + { + "name": "STRICT_PRODUCTION_CHECKS", + "purpose": ( + "Refuse to start in production with wildcard ORIGINS or ALLOWED_HOST, or " + "docs enabled. Unset means on in production; false only warns." + ), + }, + ], + }, + { + "title": "CORS and hosts", + "vars": [ + { + "name": "ORIGINS", + "purpose": "Allowed CORS origins, comma-separated. Defaults to *.", + }, + { + "name": "ALLOWED_HOST", + "purpose": "Comma-separated host names the server answers to. Defaults to *.", + }, + { + "name": "CORS_ALLOW_CREDENTIALS", + "purpose": ( + "Let cross-origin requests carry cookies and auth headers. Defaults to " + "true; production refuses it together with ORIGINS=*." + ), + }, + ], + }, + { + "title": "Paths and limits", + "vars": [ + { + "name": "ROOT_PATH", + "purpose": "URL prefix when the app is served behind a proxy path.", + }, + { + "name": "DOCS_PATH", + "purpose": "Swagger docs path. Empty turns them off; off by default in production.", + }, + { + "name": "REDOCS_PATH", + "purpose": "ReDoc docs path. Empty turns them off; off by default in production.", + }, + { + "name": "MAX_REQUEST_SIZE", + "purpose": "Largest request body in bytes. Defaults to 10 MB.", + }, + ], + }, + { + "title": "Security headers", + "vars": [ + { + "name": "SECURITY_HEADERS_ENABLED", + "purpose": "Add security headers to every response. Defaults to true.", + }, + { + "name": "HSTS_ENABLED", + "purpose": "Send Strict-Transport-Security. Defaults to true.", + }, + {"name": "HSTS_MAX_AGE", "purpose": "HSTS max-age in seconds. Defaults to one year."}, + { + "name": "HSTS_INCLUDE_SUBDOMAINS", + "purpose": "Add includeSubDomains to the HSTS header. Defaults to true.", + }, + { + "name": "HSTS_PRELOAD", + "purpose": "Add preload to the HSTS header. Defaults to false.", + }, + {"name": "FRAME_OPTIONS", "purpose": "X-Frame-Options value. Defaults to DENY."}, + { + "name": "CONTENT_TYPE_OPTIONS", + "purpose": "X-Content-Type-Options value. Defaults to nosniff.", + }, + { + "name": "REFERRER_POLICY", + "purpose": "Referrer-Policy value. Defaults to strict-origin-when-cross-origin.", + }, + { + "name": "PERMISSIONS_POLICY", + "purpose": "Permissions-Policy value. Unset uses the built-in policy.", + }, + { + "name": "CSP_POLICY", + "purpose": "Content-Security-Policy value. Unset uses the built-in policy.", + }, + ], + }, + { + "title": "Error tracking", + "vars": [ + { + "name": "SENTRY_DSN", + "purpose": "Sentry project DSN. Error tracking is off without it.", + }, + { + "name": "SENTRY_TRACES_SAMPLE_RATE", + "purpose": "Share of requests traced, 0 to 1. Defaults to 0.1.", + }, + { + "name": "SENTRY_PROFILES_SAMPLE_RATE", + "purpose": "Share of requests profiled, 0 to 1. Defaults to 0.", + }, + ], + }, + { + "title": "Snowflake IDs (optional)", + "note": ( + "Only needed if you use SnowFlakeIdGenerator with no arguments " + "(needs the snowflakekit extra). Otherwise leave these unset." + ), + "vars": [ + { + "name": "SNOWFLAKE_EPOCH", + "purpose": "Epoch in milliseconds for Snowflake ids.", + }, + { + "name": "SNOWFLAKE_NODE_ID", + "purpose": "Node id for Snowflake ids. Give each machine a different one.", + }, + { + "name": "SNOWFLAKE_WORKER_ID", + "purpose": "Worker id for Snowflake ids. Make it unique per process on a node.", + }, + { + "name": "SNOWFLAKE_TIME_BITS", + "purpose": "Bits for the timestamp part of a Snowflake id.", + }, + {"name": "SNOWFLAKE_NODE_BITS", "purpose": "Bits for the node part of a Snowflake id."}, + { + "name": "SNOWFLAKE_WORKER_BITS", + "purpose": "Bits for the worker part of a Snowflake id.", + }, + { + "name": "SNOWFLAKE_TOTAL_BITS", + "purpose": "Total bits in a Snowflake id. Defaults to 64.", + }, + ], + }, +] + def build_schema() -> dict[str, Any]: """Return a fresh, JSON-serialisable copy of the editor schema.""" - return {"docs_url": DOCS_URL, "sections": copy.deepcopy(_SECTIONS)} + return { + "docs_url": DOCS_URL, + "sections": copy.deepcopy(_SECTIONS), + "environment": copy.deepcopy(_ENVIRONMENT), + } def section_for_key(key: str) -> str: diff --git a/tenxgraph_api/cli/config_editor/static/app.css b/tenxgraph_api/cli/config_editor/static/app.css index f21a5b2c..ead94f3e 100644 --- a/tenxgraph_api/cli/config_editor/static/app.css +++ b/tenxgraph_api/cli/config_editor/static/app.css @@ -1,2 +1,2 @@ /*! tailwindcss v4.3.3 | MIT License | https://tailwindcss.com */ -@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-backdrop-blur:initial;--tw-backdrop-brightness:initial;--tw-backdrop-contrast:initial;--tw-backdrop-grayscale:initial;--tw-backdrop-hue-rotate:initial;--tw-backdrop-invert:initial;--tw-backdrop-opacity:initial;--tw-backdrop-saturate:initial;--tw-backdrop-sepia:initial;--tw-duration:initial;--tw-ease:initial;--tw-content:""}}}@layer theme{:root,:host{--font-sans:"Inter", ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif;--font-mono:"JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;--color-black:#000;--color-white:#fff;--spacing:.25rem;--container-xs:20rem;--container-sm:24rem;--container-4xl:56rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-xl:1.25rem;--text-xl--line-height:calc(1.75 / 1.25);--font-weight-medium:500;--font-weight-semibold:600;--font-weight-bold:700;--tracking-tight:-.025em;--tracking-wide:.025em;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--ease-out:cubic-bezier(0, 0, .2, 1);--animate-spin:spin 1s linear infinite;--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:var(--font-sans);--default-mono-font-family:var(--font-mono);--color-canvas:var(--af-canvas);--color-surface:var(--af-surface);--color-raised:var(--af-raised);--color-hover:var(--af-hover);--color-sunken:var(--af-sunken);--color-line:var(--af-line);--color-line-strong:var(--af-line-strong);--color-ink:var(--af-ink);--color-body:var(--af-body);--color-muted:var(--af-muted);--color-faint:var(--af-faint);--color-brand:var(--af-brand);--color-brand-solid:var(--af-brand-solid);--color-brand-solid-hover:var(--af-brand-solid-hover);--color-brand-soft:var(--af-brand-soft);--color-brand-ring:var(--af-brand-ring);--color-brand-amber:var(--af-brand-amber);--color-ok:var(--af-ok);--color-ok-soft:var(--af-ok-soft);--color-warn:var(--af-warn);--color-warn-soft:var(--af-warn-soft);--color-danger:var(--af-danger);--color-danger-soft:var(--af-danger-soft)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}html{scrollbar-gutter:stable}body{background-color:var(--color-canvas);font-family:var(--font-sans);color:var(--color-body);-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}*{scrollbar-width:thin;scrollbar-color:var(--af-scrollbar) transparent}::-webkit-scrollbar{width:10px;height:10px}::-webkit-scrollbar-track{background:0 0}::-webkit-scrollbar-thumb{background:var(--af-scrollbar) padding-box;border:3px solid #0000;border-radius:9999px}::-webkit-scrollbar-thumb:hover{background:var(--af-scrollbar-hover) padding-box}::-webkit-scrollbar-corner{background:0 0}}@layer components{.input{height:calc(var(--spacing) * 10);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line-strong);background-color:var(--color-sunken);width:100%;padding-inline:calc(var(--spacing) * 3);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));color:var(--color-ink);transition-property:border-color,box-shadow;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s;display:block}.input::placeholder{color:var(--color-faint)}@media (hover:hover){.input:hover{border-color:var(--color-faint)}@supports (color:color-mix(in lab, red, red)){.input:hover{border-color:color-mix(in oklab, var(--color-faint) 60%, transparent)}}}.input:focus{border-color:var(--color-brand);--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}@media (min-width:40rem){.input{height:calc(var(--spacing) * 9);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}}.input-invalid{border-color:var(--color-danger)}.input-invalid:focus{border-color:var(--color-danger);--tw-ring-color:var(--color-danger-soft)}.btn{height:calc(var(--spacing) * 10);cursor:pointer;justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);border-radius:var(--radius-lg);padding-inline:calc(var(--spacing) * 3.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);white-space:nowrap;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;flex-shrink:0;transition-duration:.15s;display:inline-flex}.btn:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}.btn:disabled{cursor:not-allowed;opacity:.45}@media (min-width:40rem){.btn{height:calc(var(--spacing) * 9)}}.btn-primary{background-color:var(--color-brand-solid);color:var(--color-white);--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}@media (hover:hover){.btn-primary:hover{background-color:var(--color-brand-solid-hover)}}.btn-outline{border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line-strong);background-color:var(--color-surface);color:var(--color-body)}@media (hover:hover){.btn-outline:hover{background-color:var(--color-raised)}}.btn-ghost{color:var(--color-muted)}@media (hover:hover){.btn-ghost:hover{background-color:var(--color-raised);color:var(--color-ink)}}.icon-btn{width:calc(var(--spacing) * 10);height:calc(var(--spacing) * 10);cursor:pointer;border-radius:var(--radius-lg);color:var(--color-muted);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;flex-shrink:0;justify-content:center;align-items:center;transition-duration:.15s;display:inline-flex}@media (hover:hover){.icon-btn:hover{background-color:var(--color-raised);color:var(--color-ink)}}.icon-btn:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}@media (min-width:40rem){.icon-btn{width:calc(var(--spacing) * 9);height:calc(var(--spacing) * 9)}}.panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line);background-color:var(--color-surface);--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.kbd{height:calc(var(--spacing) * 5);border-style:var(--tw-border-style);border-width:1px;border-color:#ffffff40;border-radius:.25rem;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.kbd{border-color:color-mix(in oklab, var(--color-white) 25%, transparent)}}.kbd{padding-inline:calc(var(--spacing) * 1.5);font-family:var(--font-mono);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:#fffc}@supports (color:color-mix(in lab, red, red)){.kbd{color:color-mix(in oklab, var(--color-white) 80%, transparent)}}.eyebrow{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide);color:var(--color-faint);text-transform:uppercase}.focus-ring:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}}@layer utilities{.pointer-events-auto{pointer-events:auto}.pointer-events-none{pointer-events:none}.collapse{visibility:collapse}.sr-only{clip-path:inset(50%);white-space:nowrap;border-width:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.sticky{position:sticky}.inset-0{inset:0}.inset-y-0{inset-block:0}.top-0{top:0}.top-1{top:var(--spacing)}.top-1\/2{top:50%}.top-14{top:calc(var(--spacing) * 14)}.right-0{right:0}.right-1{right:var(--spacing)}.right-3{right:calc(var(--spacing) * 3)}.right-4{right:calc(var(--spacing) * 4)}.bottom-4{bottom:calc(var(--spacing) * 4)}.left-4{left:calc(var(--spacing) * 4)}.z-20{z-index:20}.z-30{z-index:30}.z-40{z-index:40}.z-50{z-index:50}.-mx-4{margin-inline:calc(var(--spacing) * -4)}.mx-4{margin-inline:calc(var(--spacing) * 4)}.mx-auto{margin-inline:auto}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:var(--spacing)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-3{margin-top:calc(var(--spacing) * 3)}.mt-4{margin-top:calc(var(--spacing) * 4)}.mt-5{margin-top:calc(var(--spacing) * 5)}.mt-6{margin-top:calc(var(--spacing) * 6)}.mt-10{margin-top:calc(var(--spacing) * 10)}.-mr-1{margin-right:calc(var(--spacing) * -1)}.mr-1\.5{margin-right:calc(var(--spacing) * 1.5)}.mb-1\.5{margin-bottom:calc(var(--spacing) * 1.5)}.mb-2\.5{margin-bottom:calc(var(--spacing) * 2.5)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.-ml-2\.5{margin-left:calc(var(--spacing) * -2.5)}.block{display:block}.flex{display:flex}.grid{display:grid}.hidden{display:none}.inline{display:inline}.inline-block{display:inline-block}.inline-flex{display:inline-flex}.size-1\.5{width:calc(var(--spacing) * 1.5);height:calc(var(--spacing) * 1.5)}.size-3{width:calc(var(--spacing) * 3);height:calc(var(--spacing) * 3)}.size-3\.5{width:calc(var(--spacing) * 3.5);height:calc(var(--spacing) * 3.5)}.size-4{width:calc(var(--spacing) * 4);height:calc(var(--spacing) * 4)}.size-5{width:calc(var(--spacing) * 5);height:calc(var(--spacing) * 5)}.size-6{width:calc(var(--spacing) * 6);height:calc(var(--spacing) * 6)}.size-7{width:calc(var(--spacing) * 7);height:calc(var(--spacing) * 7)}.size-9{width:calc(var(--spacing) * 9);height:calc(var(--spacing) * 9)}.size-11{width:calc(var(--spacing) * 11);height:calc(var(--spacing) * 11)}.size-\[18px\]{width:18px;height:18px}.size-full{width:100%;height:100%}.h-4{height:calc(var(--spacing) * 4)}.h-5{height:calc(var(--spacing) * 5)}.h-6{height:calc(var(--spacing) * 6)}.h-7{height:calc(var(--spacing) * 7)}.h-8{height:calc(var(--spacing) * 8)}.h-9{height:calc(var(--spacing) * 9)}.h-12{height:calc(var(--spacing) * 12)}.h-14{height:calc(var(--spacing) * 14)}.h-16{height:calc(var(--spacing) * 16)}.h-auto{height:auto}.min-h-0{min-height:0}.min-h-10{min-height:calc(var(--spacing) * 10)}.w-6{width:calc(var(--spacing) * 6)}.w-9{width:calc(var(--spacing) * 9)}.w-10{width:calc(var(--spacing) * 10)}.w-48{width:calc(var(--spacing) * 48)}.w-80{width:calc(var(--spacing) * 80)}.w-\[min\(100vw\,400px\)\]{width:min(100vw,400px)}.w-full{width:100%}.w-px{width:1px}.max-w-4xl{max-width:var(--container-4xl)}.max-w-full{max-width:100%}.max-w-prose{max-width:65ch}.max-w-sm{max-width:var(--container-sm)}.max-w-xs{max-width:var(--container-xs)}.min-w-0{min-width:0}.min-w-5{min-width:calc(var(--spacing) * 5)}.min-w-\[7rem\]{min-width:7rem}.flex-1{flex:1}.shrink-0{flex-shrink:0}.translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-0\.5{--tw-translate-x:calc(var(--spacing) * .5);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-\[18px\]{--tw-translate-x:18px;translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-full{--tw-translate-x:100%;translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1\/2{--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.-rotate-90{rotate:-90deg}.animate-pulse{animation:var(--animate-pulse)}.animate-spin{animation:var(--animate-spin)}.cursor-pointer{cursor:pointer}.cursor-text{cursor:text}.scroll-mt-28{scroll-margin-top:calc(var(--spacing) * 28)}.scroll-mt-32{scroll-margin-top:calc(var(--spacing) * 32)}.\[scrollbar-width\:none\]{scrollbar-width:none}.appearance-none{appearance:none}.grid-cols-\[10rem_minmax\(0\,1fr\)_auto\]{grid-template-columns:10rem minmax(0,1fr) auto}.grid-rows-\[0fr\]{grid-template-rows:0fr}.grid-rows-\[1fr\]{grid-template-rows:1fr}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-end{align-items:flex-end}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.gap-0\.5{gap:calc(var(--spacing) * .5)}.gap-1{gap:var(--spacing)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-2\.5{gap:calc(var(--spacing) * 2.5)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-4{gap:calc(var(--spacing) * 4)}.gap-6{gap:calc(var(--spacing) * 6)}:where(.space-y-0\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-3>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 3) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 3) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-6>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.gap-x-2{column-gap:calc(var(--spacing) * 2)}.gap-y-0\.5{row-gap:calc(var(--spacing) * .5)}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-line>:not(:last-child)){border-color:var(--color-line)}.justify-self-end{justify-self:flex-end}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-hidden{overflow:hidden}.overflow-x-auto{overflow-x:auto}.overflow-y-auto{overflow-y:auto}.overscroll-contain{overscroll-behavior:contain}.rounded{border-radius:.25rem}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius-lg)}.rounded-md{border-radius:var(--radius-md)}.rounded-xl{border-radius:var(--radius-xl)}.rounded-b-xl{border-bottom-right-radius:var(--radius-xl);border-bottom-left-radius:var(--radius-xl)}.border{border-style:var(--tw-border-style);border-width:1px}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-l{border-left-style:var(--tw-border-style);border-left-width:1px}.border-dashed{--tw-border-style:dashed;border-style:dashed}.border-brand{border-color:var(--color-brand)}.border-brand-solid{border-color:var(--color-brand-solid)}.border-brand\/20{border-color:var(--color-brand)}@supports (color:color-mix(in lab, red, red)){.border-brand\/20{border-color:color-mix(in oklab, var(--color-brand) 20%, transparent)}}.border-danger\/25{border-color:var(--color-danger)}@supports (color:color-mix(in lab, red, red)){.border-danger\/25{border-color:color-mix(in oklab, var(--color-danger) 25%, transparent)}}.border-danger\/40{border-color:var(--color-danger)}@supports (color:color-mix(in lab, red, red)){.border-danger\/40{border-color:color-mix(in oklab, var(--color-danger) 40%, transparent)}}.border-line{border-color:var(--color-line)}.border-line-strong{border-color:var(--color-line-strong)}.border-warn\/25{border-color:var(--color-warn)}@supports (color:color-mix(in lab, red, red)){.border-warn\/25{border-color:color-mix(in oklab, var(--color-warn) 25%, transparent)}}.bg-black\/50{background-color:#00000080}@supports (color:color-mix(in lab, red, red)){.bg-black\/50{background-color:color-mix(in oklab, var(--color-black) 50%, transparent)}}.bg-brand-soft{background-color:var(--color-brand-soft)}.bg-brand-solid{background-color:var(--color-brand-solid)}.bg-canvas\/60{background-color:var(--color-canvas)}@supports (color:color-mix(in lab, red, red)){.bg-canvas\/60{background-color:color-mix(in oklab, var(--color-canvas) 60%, transparent)}}.bg-canvas\/90{background-color:var(--color-canvas)}@supports (color:color-mix(in lab, red, red)){.bg-canvas\/90{background-color:color-mix(in oklab, var(--color-canvas) 90%, transparent)}}.bg-danger{background-color:var(--color-danger)}.bg-danger-soft{background-color:var(--color-danger-soft)}.bg-line{background-color:var(--color-line)}.bg-line-strong{background-color:var(--color-line-strong)}.bg-ok-soft{background-color:var(--color-ok-soft)}.bg-raised{background-color:var(--color-raised)}.bg-sunken{background-color:var(--color-sunken)}.bg-surface,.bg-surface\/95{background-color:var(--color-surface)}@supports (color:color-mix(in lab, red, red)){.bg-surface\/95{background-color:color-mix(in oklab, var(--color-surface) 95%, transparent)}}.bg-transparent{background-color:#0000}.bg-warn{background-color:var(--color-warn)}.bg-warn-soft{background-color:var(--color-warn-soft)}.bg-white{background-color:var(--color-white)}.fill-brand-amber{fill:var(--color-brand-amber)}.p-4{padding:calc(var(--spacing) * 4)}.px-1{padding-inline:var(--spacing)}.px-1\.5{padding-inline:calc(var(--spacing) * 1.5)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-5{padding-inline:calc(var(--spacing) * 5)}.px-6{padding-inline:calc(var(--spacing) * 6)}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-3\.5{padding-block:calc(var(--spacing) * 3.5)}.py-4{padding-block:calc(var(--spacing) * 4)}.py-5{padding-block:calc(var(--spacing) * 5)}.py-6{padding-block:calc(var(--spacing) * 6)}.py-10{padding-block:calc(var(--spacing) * 10)}.py-14{padding-block:calc(var(--spacing) * 14)}.pt-0\.5{padding-top:calc(var(--spacing) * .5)}.pt-8{padding-top:calc(var(--spacing) * 8)}.pr-0\.5{padding-right:calc(var(--spacing) * .5)}.pr-3{padding-right:calc(var(--spacing) * 3)}.pr-4{padding-right:calc(var(--spacing) * 4)}.pr-9{padding-right:calc(var(--spacing) * 9)}.pb-24{padding-bottom:calc(var(--spacing) * 24)}.pl-2{padding-left:calc(var(--spacing) * 2)}.pl-3{padding-left:calc(var(--spacing) * 3)}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.font-mono{font-family:var(--font-mono)}.text-base{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xl{font-size:var(--text-xl);line-height:var(--tw-leading,var(--text-xl--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.text-\[10px\]{font-size:10px}.text-\[11px\]{font-size:11px}.text-\[12\.5px\]{font-size:12.5px}.text-\[13px\]{font-size:13px}.text-\[15px\]{font-size:15px}.leading-4{--tw-leading:calc(var(--spacing) * 4);line-height:calc(var(--spacing) * 4)}.leading-5{--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5)}.leading-6{--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6)}.font-bold{--tw-font-weight:var(--font-weight-bold);font-weight:var(--font-weight-bold)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.break-all{word-break:break-all}.whitespace-pre-wrap{white-space:pre-wrap}.text-body{color:var(--color-body)}.text-brand{color:var(--color-brand)}.text-danger{color:var(--color-danger)}.text-faint,.text-faint\/70{color:var(--color-faint)}@supports (color:color-mix(in lab, red, red)){.text-faint\/70{color:color-mix(in oklab, var(--color-faint) 70%, transparent)}}.text-ink{color:var(--color-ink)}.text-muted{color:var(--color-muted)}.text-ok{color:var(--color-ok)}.text-warn{color:var(--color-warn)}.text-white{color:var(--color-white)}.tabular-nums{--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.shadow-card{--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-pop{--tw-shadow:var(--af-shadow-pop);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.ring-1{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.ring-black\/5{--tw-ring-color:#0000000d}@supports (color:color-mix(in lab, red, red)){.ring-black\/5{--tw-ring-color:color-mix(in oklab, var(--color-black) 5%, transparent)}}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.backdrop-blur{--tw-backdrop-blur:blur(8px);-webkit-backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,);backdrop-filter:var(--tw-backdrop-blur,) var(--tw-backdrop-brightness,) var(--tw-backdrop-contrast,) var(--tw-backdrop-grayscale,) var(--tw-backdrop-hue-rotate,) var(--tw-backdrop-invert,) var(--tw-backdrop-opacity,) var(--tw-backdrop-saturate,) var(--tw-backdrop-sepia,)}.transition-\[border-color\,box-shadow\]{transition-property:border-color,box-shadow;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-\[grid-template-rows\]{transition-property:grid-template-rows;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.duration-150{--tw-duration:.15s;transition-duration:.15s}.duration-200{--tw-duration:.2s;transition-duration:.2s}.ease-out{--tw-ease:var(--ease-out);transition-timing-function:var(--ease-out)}.outline-none{--tw-outline-style:none;outline-style:none}.select-none{-webkit-user-select:none;user-select:none}.placeholder\:text-faint::placeholder{color:var(--color-faint)}.before\:absolute:before{content:var(--tw-content);position:absolute}.before\:-inset-2\.5:before{content:var(--tw-content);inset:calc(var(--spacing) * -2.5)}.before\:content-\[\'\'\]:before{--tw-content:"";content:var(--tw-content)}.focus-within\:border-brand:focus-within{border-color:var(--color-brand)}.focus-within\:ring-4:focus-within{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus-within\:ring-brand-ring:focus-within{--tw-ring-color:var(--color-brand-ring)}@media (hover:hover){.hover\:border-brand:hover{border-color:var(--color-brand)}.hover\:border-line-strong:hover{border-color:var(--color-line-strong)}.hover\:bg-hover:hover{background-color:var(--color-hover)}.hover\:bg-raised:hover{background-color:var(--color-raised)}.hover\:text-brand:hover{color:var(--color-brand)}.hover\:text-danger:hover{color:var(--color-danger)}.hover\:text-ink:hover{color:var(--color-ink)}.hover\:underline:hover{text-decoration-line:underline}}.focus\:not-sr-only:focus{clip-path:none;white-space:normal;width:auto;height:auto;margin:0;padding:0;position:static;overflow:visible}.focus\:fixed:focus{position:fixed}.focus\:top-3:focus{top:calc(var(--spacing) * 3)}.focus\:left-3:focus{left:calc(var(--spacing) * 3)}.focus\:outline-none:focus{--tw-outline-style:none;outline-style:none}@media (min-width:40rem){.sm\:left-auto{left:auto}.sm\:-mx-8{margin-inline:calc(var(--spacing) * -8)}.sm\:grid{display:grid}.sm\:inline{display:inline}.sm\:size-5{width:calc(var(--spacing) * 5);height:calc(var(--spacing) * 5)}.sm\:h-6{height:calc(var(--spacing) * 6)}.sm\:min-h-9{min-height:calc(var(--spacing) * 9)}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.sm\:grid-cols-\[10rem_minmax\(0\,1fr\)_auto\]{grid-template-columns:10rem minmax(0,1fr) auto}.sm\:grid-cols-\[minmax\(0\,1fr\)_minmax\(0\,1\.1fr\)\]{grid-template-columns:minmax(0,1fr) minmax(0,1.1fr)}.sm\:flex-row{flex-direction:row}.sm\:items-center{align-items:center}.sm\:items-start{align-items:flex-start}.sm\:justify-between{justify-content:space-between}.sm\:gap-3{gap:calc(var(--spacing) * 3)}.sm\:gap-6{gap:calc(var(--spacing) * 6)}.sm\:gap-8{gap:calc(var(--spacing) * 8)}.sm\:px-4{padding-inline:calc(var(--spacing) * 4)}.sm\:px-5{padding-inline:calc(var(--spacing) * 5)}.sm\:px-8{padding-inline:calc(var(--spacing) * 8)}.sm\:py-1{padding-block:var(--spacing)}.sm\:pt-0\.5{padding-top:calc(var(--spacing) * .5)}.sm\:pr-5{padding-right:calc(var(--spacing) * 5)}.sm\:pl-4{padding-left:calc(var(--spacing) * 4)}.sm\:text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}}@media (min-width:48rem){.md\:block{display:block}.md\:flex{display:flex}}@media (min-width:64rem){.lg\:block{display:block}.lg\:inline-flex{display:inline-flex}}@media (min-width:80rem){.xl\:sticky{position:sticky}.xl\:top-14{top:calc(var(--spacing) * 14)}.xl\:bottom-auto{bottom:auto}.xl\:z-auto{z-index:auto}.xl\:hidden{display:none}.xl\:h-\[calc\(100dvh-3\.5rem\)\]{height:calc(100dvh - 3.5rem)}.xl\:w-\[380px\]{width:380px}.xl\:shrink-0{flex-shrink:0}.xl\:translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.xl\:self-start{align-self:flex-start}}.\[\&\:\:-webkit-scrollbar\]\:hidden::-webkit-scrollbar{display:none}}:root{color-scheme:light;--af-canvas:#f5f7fb;--af-surface:#fff;--af-raised:#f1f5f9;--af-hover:#e8eef6;--af-sunken:#fff;--af-line:#0f172a17;--af-line-strong:#0f172a2b;--af-ink:#0b1220;--af-body:#1e293b;--af-muted:#475569;--af-faint:#64748b;--af-brand:#2563eb;--af-brand-solid:#2563eb;--af-brand-solid-hover:#1d4ed8;--af-brand-soft:#2563eb14;--af-brand-ring:#2563eb38;--af-brand-amber:#c8692f;--af-ok:#047857;--af-ok-soft:#10b9811a;--af-warn:#b45309;--af-warn-soft:#f59e0b1f;--af-danger:#dc2626;--af-danger-soft:#ef444417;--af-shadow:0 1px 2px #0f172a0a, 0 1px 3px #0f172a0a;--af-shadow-pop:0 12px 32px #0f172a24;--af-scrollbar:#0f172a33;--af-scrollbar-hover:#0f172a59;--af-json-key:#1d4ed8;--af-json-string:#047857;--af-json-number:#b45309;--af-json-literal:#7c3aed}@media (prefers-color-scheme:dark){:root{color-scheme:dark;--af-canvas:#08111f;--af-surface:#0c1727;--af-raised:#131e33;--af-hover:#1a2741;--af-sunken:#07101d;--af-line:#94a3b824;--af-line-strong:#94a3b842;--af-ink:#f8fafc;--af-body:#e2e8f0;--af-muted:#a3b1c6;--af-faint:#8391a7;--af-brand:#60a5fa;--af-brand-solid:#2563eb;--af-brand-solid-hover:#3b82f6;--af-brand-soft:#3b82f624;--af-brand-ring:#3b82f659;--af-brand-amber:#e9a57c;--af-ok:#34d399;--af-ok-soft:#10b98124;--af-warn:#fbbf24;--af-warn-soft:#f59e0b21;--af-danger:#f87171;--af-danger-soft:#ef444421;--af-shadow:0 1px 2px #02061766;--af-shadow-pop:0 16px 40px #0206178c;--af-scrollbar:#94a3b838;--af-scrollbar-hover:#94a3b866;--af-json-key:#93c5fd;--af-json-string:#6ee7b7;--af-json-number:#fcd34d;--af-json-literal:#c4b5fd}}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:0s!important;animation-duration:0s!important}}@property --tw-translate-x{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-y{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-z{syntax:"*";inherits:false;initial-value:0}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-ordinal{syntax:"*";inherits:false}@property --tw-slashed-zero{syntax:"*";inherits:false}@property --tw-numeric-figure{syntax:"*";inherits:false}@property --tw-numeric-spacing{syntax:"*";inherits:false}@property --tw-numeric-fraction{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-backdrop-blur{syntax:"*";inherits:false}@property --tw-backdrop-brightness{syntax:"*";inherits:false}@property --tw-backdrop-contrast{syntax:"*";inherits:false}@property --tw-backdrop-grayscale{syntax:"*";inherits:false}@property --tw-backdrop-hue-rotate{syntax:"*";inherits:false}@property --tw-backdrop-invert{syntax:"*";inherits:false}@property --tw-backdrop-opacity{syntax:"*";inherits:false}@property --tw-backdrop-saturate{syntax:"*";inherits:false}@property --tw-backdrop-sepia{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@property --tw-ease{syntax:"*";inherits:false}@property --tw-content{syntax:"*";inherits:false;initial-value:""}@keyframes spin{to{transform:rotate(360deg)}}@keyframes pulse{50%{opacity:.5}} \ No newline at end of file +@layer properties{@supports (((-webkit-hyphens:none)) and (not (margin-trim:inline))) or ((-moz-orient:inline) and (not (color:rgb(from red r g b)))){*,:before,:after,::backdrop{--tw-translate-x:0;--tw-translate-y:0;--tw-translate-z:0;--tw-space-y-reverse:0;--tw-divide-y-reverse:0;--tw-border-style:solid;--tw-leading:initial;--tw-font-weight:initial;--tw-tracking:initial;--tw-ordinal:initial;--tw-slashed-zero:initial;--tw-numeric-figure:initial;--tw-numeric-spacing:initial;--tw-numeric-fraction:initial;--tw-shadow:0 0 #0000;--tw-shadow-color:initial;--tw-shadow-alpha:100%;--tw-inset-shadow:0 0 #0000;--tw-inset-shadow-color:initial;--tw-inset-shadow-alpha:100%;--tw-ring-color:initial;--tw-ring-shadow:0 0 #0000;--tw-inset-ring-color:initial;--tw-inset-ring-shadow:0 0 #0000;--tw-ring-inset:initial;--tw-ring-offset-width:0px;--tw-ring-offset-color:#fff;--tw-ring-offset-shadow:0 0 #0000;--tw-blur:initial;--tw-brightness:initial;--tw-contrast:initial;--tw-grayscale:initial;--tw-hue-rotate:initial;--tw-invert:initial;--tw-opacity:initial;--tw-saturate:initial;--tw-sepia:initial;--tw-drop-shadow:initial;--tw-drop-shadow-color:initial;--tw-drop-shadow-alpha:100%;--tw-drop-shadow-size:initial;--tw-duration:initial;--tw-ease:initial;--tw-content:""}}}@layer theme{:root,:host{--font-sans:"Inter", ui-sans-serif, system-ui, -apple-system, "Segoe UI", sans-serif;--font-mono:"JetBrains Mono", ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;--color-black:#000;--color-white:#fff;--spacing:.25rem;--container-xs:20rem;--container-sm:24rem;--text-xs:.75rem;--text-xs--line-height:calc(1 / .75);--text-sm:.875rem;--text-sm--line-height:calc(1.25 / .875);--text-base:1rem;--text-base--line-height:calc(1.5 / 1);--text-2xl:1.5rem;--text-2xl--line-height:calc(2 / 1.5);--font-weight-normal:400;--font-weight-medium:500;--font-weight-semibold:600;--tracking-tight:-.025em;--tracking-normal:0em;--tracking-wide:.025em;--tracking-wider:.05em;--radius-md:.375rem;--radius-lg:.5rem;--radius-xl:.75rem;--ease-out:cubic-bezier(0, 0, .2, 1);--animate-spin:spin 1s linear infinite;--animate-pulse:pulse 2s cubic-bezier(.4, 0, .6, 1) infinite;--default-transition-duration:.15s;--default-transition-timing-function:cubic-bezier(.4, 0, .2, 1);--default-font-family:var(--font-sans);--default-mono-font-family:var(--font-mono);--color-canvas:var(--af-canvas);--color-surface:var(--af-surface);--color-raised:var(--af-raised);--color-hover:var(--af-hover);--color-sunken:var(--af-sunken);--color-line:var(--af-line);--color-line-strong:var(--af-line-strong);--color-ink:var(--af-ink);--color-body:var(--af-body);--color-muted:var(--af-muted);--color-faint:var(--af-faint);--color-brand:var(--af-brand);--color-brand-solid:var(--af-brand-solid);--color-brand-solid-hover:var(--af-brand-solid-hover);--color-brand-soft:var(--af-brand-soft);--color-brand-ring:var(--af-brand-ring);--color-brand-amber:var(--af-brand-amber);--color-ok:var(--af-ok);--color-ok-soft:var(--af-ok-soft);--color-warn:var(--af-warn);--color-warn-soft:var(--af-warn-soft);--color-danger:var(--af-danger);--color-danger-soft:var(--af-danger-soft)}}@layer base{*,:after,:before,::backdrop{box-sizing:border-box;border:0 solid;margin:0;padding:0}::file-selector-button{box-sizing:border-box;border:0 solid;margin:0;padding:0}html,:host{-webkit-text-size-adjust:100%;tab-size:4;line-height:1.5;font-family:var(--default-font-family,-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", "Noto Sans", Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji");font-feature-settings:var(--default-font-feature-settings,normal);font-variation-settings:var(--default-font-variation-settings,normal);-webkit-tap-highlight-color:transparent}hr{height:0;color:inherit;border-top-width:1px}abbr:where([title]){-webkit-text-decoration:underline dotted;text-decoration:underline dotted}h1,h2,h3,h4,h5,h6{font-size:inherit;font-weight:inherit}a{color:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;-webkit-text-decoration:inherit;text-decoration:inherit}b,strong{font-weight:bolder}code,kbd,samp,pre{font-family:var(--default-mono-font-family,ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace);font-feature-settings:var(--default-mono-font-feature-settings,normal);font-variation-settings:var(--default-mono-font-variation-settings,normal);font-size:1em}small{font-size:80%}sub,sup{vertical-align:baseline;font-size:75%;line-height:0;position:relative}sub{bottom:-.25em}sup{top:-.5em}table{text-indent:0;border-color:inherit;border-collapse:collapse}:-moz-focusring:where(:not(iframe)){outline:auto}progress{vertical-align:baseline}summary{display:list-item}ol,ul,menu{list-style:none}img,svg,video,canvas,audio,iframe,embed,object{vertical-align:middle;display:block}img,video{max-width:100%;height:auto}button,input,select,optgroup,textarea{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}::file-selector-button{font:inherit;font-feature-settings:inherit;font-variation-settings:inherit;letter-spacing:inherit;color:inherit;opacity:1;background-color:#0000;border-radius:0}:where(select:is([multiple],[size])) optgroup{font-weight:bolder}:where(select:is([multiple],[size])) optgroup option{padding-inline-start:20px}::file-selector-button{margin-inline-end:4px}::placeholder{opacity:1}@supports (not ((-webkit-appearance:-apple-pay-button))) or (contain-intrinsic-size:1px){::placeholder{color:currentColor}@supports (color:color-mix(in lab, red, red)){::placeholder{color:color-mix(in oklab, currentcolor 50%, transparent)}}}textarea{resize:vertical}::-webkit-search-decoration{-webkit-appearance:none}::-webkit-date-and-time-value{min-height:1lh;text-align:inherit}::-webkit-datetime-edit{display:inline-flex}::-webkit-datetime-edit-fields-wrapper{padding:0}::-webkit-datetime-edit{padding-block:0}::-webkit-datetime-edit-year-field{padding-block:0}::-webkit-datetime-edit-month-field{padding-block:0}::-webkit-datetime-edit-day-field{padding-block:0}::-webkit-datetime-edit-hour-field{padding-block:0}::-webkit-datetime-edit-minute-field{padding-block:0}::-webkit-datetime-edit-second-field{padding-block:0}::-webkit-datetime-edit-millisecond-field{padding-block:0}::-webkit-datetime-edit-meridiem-field{padding-block:0}::-webkit-calendar-picker-indicator{line-height:1}:-moz-ui-invalid{box-shadow:none}button,input:where([type=button],[type=reset],[type=submit]){appearance:button}::file-selector-button{appearance:button}::-webkit-inner-spin-button{height:auto}::-webkit-outer-spin-button{height:auto}[hidden]:where(:not([hidden=until-found])){display:none!important}html{scrollbar-gutter:stable}body{background-color:var(--color-canvas);font-family:var(--font-sans);color:var(--color-body);-webkit-font-smoothing:antialiased;-moz-osx-font-smoothing:grayscale}*{scrollbar-width:thin;scrollbar-color:var(--af-scrollbar) transparent}::-webkit-scrollbar{width:10px;height:10px}::-webkit-scrollbar-track{background:0 0}::-webkit-scrollbar-thumb{background:var(--af-scrollbar) padding-box;border:3px solid #0000;border-radius:9999px}::-webkit-scrollbar-thumb:hover{background:var(--af-scrollbar-hover) padding-box}::-webkit-scrollbar-corner{background:0 0}}@layer components{.input{height:calc(var(--spacing) * 11);border-radius:var(--radius-lg);border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line-strong);background-color:var(--color-sunken);width:100%;padding-inline:calc(var(--spacing) * 3);font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height));color:var(--color-ink);transition-property:border-color,box-shadow;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;transition-duration:.15s;display:block}.input::placeholder{color:var(--color-faint)}@media (hover:hover){.input:hover{border-color:var(--color-faint)}@supports (color:color-mix(in lab, red, red)){.input:hover{border-color:color-mix(in oklab, var(--color-faint) 60%, transparent)}}}.input:focus{border-color:var(--color-brand);--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}.input:disabled{cursor:not-allowed;opacity:.5}@media (min-width:40rem){.input{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}}.input-invalid{border-color:var(--color-danger)}.input-invalid:focus{border-color:var(--color-danger);--tw-ring-color:var(--color-danger-soft)}.btn{height:calc(var(--spacing) * 11);cursor:pointer;justify-content:center;align-items:center;gap:calc(var(--spacing) * 2);border-radius:var(--radius-lg);padding-inline:calc(var(--spacing) * 3.5);font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height));--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium);white-space:nowrap;transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;flex-shrink:0;transition-duration:.15s;display:inline-flex}.btn:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}.btn:disabled{cursor:not-allowed;opacity:.45}.btn-primary{background-color:var(--color-brand-solid);color:var(--color-white);--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}@media (hover:hover){.btn-primary:hover{background-color:var(--color-brand-solid-hover)}}.btn-outline{border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line-strong);background-color:var(--color-surface);color:var(--color-body)}@media (hover:hover){.btn-outline:hover{background-color:var(--color-raised)}}.btn-ghost{color:var(--color-muted)}@media (hover:hover){.btn-ghost:hover{background-color:var(--color-raised);color:var(--color-ink)}}.icon-btn{width:calc(var(--spacing) * 11);height:calc(var(--spacing) * 11);cursor:pointer;border-radius:var(--radius-lg);color:var(--color-muted);transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration));--tw-duration:.15s;flex-shrink:0;justify-content:center;align-items:center;transition-duration:.15s;display:inline-flex}@media (hover:hover){.icon-btn:hover{background-color:var(--color-raised);color:var(--color-ink)}}.icon-btn:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}.panel{border-radius:var(--radius-xl);border-style:var(--tw-border-style);border-width:1px;border-color:var(--color-line);background-color:var(--color-surface);--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.kbd{height:calc(var(--spacing) * 5);border-style:var(--tw-border-style);border-width:1px;border-color:#ffffff40;border-radius:.25rem;align-items:center;display:inline-flex}@supports (color:color-mix(in lab, red, red)){.kbd{border-color:color-mix(in oklab, var(--color-white) 25%, transparent)}}.kbd{padding-inline:calc(var(--spacing) * 1.5);font-family:var(--font-mono);--tw-font-weight:var(--font-weight-medium);font-size:11px;font-weight:var(--font-weight-medium);color:#fffc}@supports (color:color-mix(in lab, red, red)){.kbd{color:color-mix(in oklab, var(--color-white) 80%, transparent)}}.eyebrow{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height));--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold);--tw-tracking:var(--tracking-wide);letter-spacing:var(--tracking-wide);color:var(--color-faint);text-transform:uppercase}.focus-ring:focus-visible{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow);--tw-ring-color:var(--color-brand-ring);--tw-outline-style:none;outline-style:none}}@layer utilities{.pointer-events-auto{pointer-events:auto}.pointer-events-none{pointer-events:none}.collapse{visibility:collapse}.sr-only{clip-path:inset(50%);white-space:nowrap;border-width:0;width:1px;height:1px;margin:-1px;padding:0;position:absolute;overflow:hidden}.absolute{position:absolute}.fixed{position:fixed}.relative{position:relative}.sticky{position:sticky}.top-0{top:0}.top-1\/2{top:50%}.right-3{right:calc(var(--spacing) * 3)}.right-4{right:calc(var(--spacing) * 4)}.bottom-4{bottom:calc(var(--spacing) * 4)}.left-3{left:calc(var(--spacing) * 3)}.left-4{left:calc(var(--spacing) * 4)}.z-40{z-index:40}.z-50{z-index:50}.m-0{margin:0}.mx-1{margin-inline:var(--spacing)}.mx-auto{margin-inline:auto}.mt-0\.5{margin-top:calc(var(--spacing) * .5)}.mt-1{margin-top:var(--spacing)}.mt-1\.5{margin-top:calc(var(--spacing) * 1.5)}.mt-2{margin-top:calc(var(--spacing) * 2)}.mt-2\.5{margin-top:calc(var(--spacing) * 2.5)}.mt-3{margin-top:calc(var(--spacing) * 3)}.mt-4{margin-top:calc(var(--spacing) * 4)}.mt-5{margin-top:calc(var(--spacing) * 5)}.mt-6{margin-top:calc(var(--spacing) * 6)}.mt-7{margin-top:calc(var(--spacing) * 7)}.mt-8{margin-top:calc(var(--spacing) * 8)}.mt-9{margin-top:calc(var(--spacing) * 9)}.mt-10{margin-top:calc(var(--spacing) * 10)}.-mr-1{margin-right:calc(var(--spacing) * -1)}.mr-1\.5{margin-right:calc(var(--spacing) * 1.5)}.mb-1{margin-bottom:var(--spacing)}.mb-1\.5{margin-bottom:calc(var(--spacing) * 1.5)}.mb-2{margin-bottom:calc(var(--spacing) * 2)}.mb-2\.5{margin-bottom:calc(var(--spacing) * 2.5)}.mb-3{margin-bottom:calc(var(--spacing) * 3)}.mb-6{margin-bottom:calc(var(--spacing) * 6)}.ml-auto{margin-left:auto}.block{display:block}.contents{display:contents}.flex{display:flex}.grid{display:grid}.hidden{display:none}.inline{display:inline}.inline-block{display:inline-block}.inline-flex{display:inline-flex}.size-1\.5{width:calc(var(--spacing) * 1.5);height:calc(var(--spacing) * 1.5)}.size-2{width:calc(var(--spacing) * 2);height:calc(var(--spacing) * 2)}.size-3{width:calc(var(--spacing) * 3);height:calc(var(--spacing) * 3)}.size-3\.5{width:calc(var(--spacing) * 3.5);height:calc(var(--spacing) * 3.5)}.size-4{width:calc(var(--spacing) * 4);height:calc(var(--spacing) * 4)}.size-5{width:calc(var(--spacing) * 5);height:calc(var(--spacing) * 5)}.size-6{width:calc(var(--spacing) * 6);height:calc(var(--spacing) * 6)}.size-7{width:calc(var(--spacing) * 7);height:calc(var(--spacing) * 7)}.size-11{width:calc(var(--spacing) * 11);height:calc(var(--spacing) * 11)}.size-\[18px\]{width:18px;height:18px}.size-full{width:100%;height:100%}.h-4{height:calc(var(--spacing) * 4)}.h-5{height:calc(var(--spacing) * 5)}.h-6{height:calc(var(--spacing) * 6)}.h-7{height:calc(var(--spacing) * 7)}.h-8{height:calc(var(--spacing) * 8)}.h-10{height:calc(var(--spacing) * 10)}.h-12{height:calc(var(--spacing) * 12)}.h-14{height:calc(var(--spacing) * 14)}.h-16{height:calc(var(--spacing) * 16)}.h-auto{height:auto}.h-dvh{height:100dvh}.h-full{height:100%}.max-h-none{max-height:none}.min-h-0{min-height:0}.min-h-6{min-height:calc(var(--spacing) * 6)}.min-h-8{min-height:calc(var(--spacing) * 8)}.min-h-10{min-height:calc(var(--spacing) * 10)}.min-h-11{min-height:calc(var(--spacing) * 11)}.min-h-12{min-height:calc(var(--spacing) * 12)}.min-h-14{min-height:calc(var(--spacing) * 14)}.w-9{width:calc(var(--spacing) * 9)}.w-10{width:calc(var(--spacing) * 10)}.w-48{width:calc(var(--spacing) * 48)}.w-80{width:calc(var(--spacing) * 80)}.w-\[min\(540px\,100vw\)\]{width:min(540px,100vw)}.w-full{width:100%}.w-px{width:1px}.max-w-40{max-width:calc(var(--spacing) * 40)}.max-w-\[62ch\]{max-width:62ch}.max-w-\[420px\]{max-width:420px}.max-w-\[520px\]{max-width:520px}.max-w-\[760px\]{max-width:760px}.max-w-full{max-width:100%}.max-w-prose{max-width:65ch}.max-w-sm{max-width:var(--container-sm)}.max-w-xs{max-width:var(--container-xs)}.min-w-0{min-width:0}.min-w-5{min-width:calc(var(--spacing) * 5)}.min-w-40{min-width:calc(var(--spacing) * 40)}.min-w-56{min-width:calc(var(--spacing) * 56)}.min-w-\[7rem\]{min-width:7rem}.flex-1{flex:1}.shrink-0{flex-shrink:0}.translate-x-0\.5{--tw-translate-x:calc(var(--spacing) * .5);translate:var(--tw-translate-x) var(--tw-translate-y)}.translate-x-\[18px\]{--tw-translate-x:18px;translate:var(--tw-translate-x) var(--tw-translate-y)}.-translate-y-1\/2{--tw-translate-y:calc(calc(1 / 2 * 100%) * -1);translate:var(--tw-translate-x) var(--tw-translate-y)}.animate-pulse{animation:var(--animate-pulse)}.animate-spin{animation:var(--animate-spin)}.cursor-pointer{cursor:pointer}.cursor-text{cursor:text}.list-none{list-style-type:none}.appearance-none{appearance:none}.grid-cols-\[10rem_minmax\(0\,1fr\)_auto\]{grid-template-columns:10rem minmax(0,1fr) auto}.flex-col{flex-direction:column}.flex-wrap{flex-wrap:wrap}.items-baseline{align-items:baseline}.items-center{align-items:center}.items-end{align-items:flex-end}.items-start{align-items:flex-start}.justify-between{justify-content:space-between}.justify-center{justify-content:center}.gap-0\.5{gap:calc(var(--spacing) * .5)}.gap-1{gap:var(--spacing)}.gap-1\.5{gap:calc(var(--spacing) * 1.5)}.gap-2{gap:calc(var(--spacing) * 2)}.gap-2\.5{gap:calc(var(--spacing) * 2.5)}.gap-3{gap:calc(var(--spacing) * 3)}.gap-3\.5{gap:calc(var(--spacing) * 3.5)}.gap-4{gap:calc(var(--spacing) * 4)}:where(.space-y-0\.5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * .5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * .5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-1>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(var(--spacing) * var(--tw-space-y-reverse));margin-block-end:calc(var(--spacing) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-2>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 2) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 2) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-4>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 4) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 4) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-5>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 5) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 5) * calc(1 - var(--tw-space-y-reverse)))}:where(.space-y-6>:not(:last-child)){--tw-space-y-reverse:0;margin-block-start:calc(calc(var(--spacing) * 6) * var(--tw-space-y-reverse));margin-block-end:calc(calc(var(--spacing) * 6) * calc(1 - var(--tw-space-y-reverse)))}.gap-x-3{column-gap:calc(var(--spacing) * 3)}.gap-x-4{column-gap:calc(var(--spacing) * 4)}.gap-y-1{row-gap:var(--spacing)}:where(.divide-y>:not(:last-child)){--tw-divide-y-reverse:0;border-bottom-style:var(--tw-border-style);border-top-style:var(--tw-border-style);border-top-width:calc(1px * var(--tw-divide-y-reverse));border-bottom-width:calc(1px * calc(1 - var(--tw-divide-y-reverse)))}:where(.divide-line>:not(:last-child)){border-color:var(--color-line)}.justify-self-end{justify-self:flex-end}.truncate{text-overflow:ellipsis;white-space:nowrap;overflow:hidden}.overflow-y-auto{overflow-y:auto}.overscroll-contain{overscroll-behavior:contain}.rounded{border-radius:.25rem}.rounded-\[10px\]{border-radius:10px}.rounded-full{border-radius:3.40282e38px}.rounded-lg{border-radius:var(--radius-lg)}.rounded-md{border-radius:var(--radius-md)}.rounded-xl{border-radius:var(--radius-xl)}.border{border-style:var(--tw-border-style);border-width:1px}.border-0{border-style:var(--tw-border-style);border-width:0}.border-t{border-top-style:var(--tw-border-style);border-top-width:1px}.border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.border-l{border-left-style:var(--tw-border-style);border-left-width:1px}.border-dashed{--tw-border-style:dashed;border-style:dashed}.border-brand{border-color:var(--color-brand)}.border-brand-solid{border-color:var(--color-brand-solid)}.border-brand\/20{border-color:var(--color-brand)}@supports (color:color-mix(in lab, red, red)){.border-brand\/20{border-color:color-mix(in oklab, var(--color-brand) 20%, transparent)}}.border-danger\/25{border-color:var(--color-danger)}@supports (color:color-mix(in lab, red, red)){.border-danger\/25{border-color:color-mix(in oklab, var(--color-danger) 25%, transparent)}}.border-line{border-color:var(--color-line)}.border-line-strong{border-color:var(--color-line-strong)}.border-warn\/25{border-color:var(--color-warn)}@supports (color:color-mix(in lab, red, red)){.border-warn\/25{border-color:color-mix(in oklab, var(--color-warn) 25%, transparent)}}.bg-brand{background-color:var(--color-brand)}.bg-brand-soft{background-color:var(--color-brand-soft)}.bg-brand-solid{background-color:var(--color-brand-solid)}.bg-danger{background-color:var(--color-danger)}.bg-danger-soft{background-color:var(--color-danger-soft)}.bg-line{background-color:var(--color-line)}.bg-line-strong{background-color:var(--color-line-strong)}.bg-ok-soft{background-color:var(--color-ok-soft)}.bg-raised{background-color:var(--color-raised)}.bg-sunken{background-color:var(--color-sunken)}.bg-surface{background-color:var(--color-surface)}.bg-transparent{background-color:#0000}.bg-warn{background-color:var(--color-warn)}.bg-warn-soft{background-color:var(--color-warn-soft)}.bg-white{background-color:var(--color-white)}.fill-brand-amber{fill:var(--color-brand-amber)}.p-0{padding:0}.p-0\.5{padding:calc(var(--spacing) * .5)}.p-4{padding:calc(var(--spacing) * 4)}.px-1{padding-inline:var(--spacing)}.px-1\.5{padding-inline:calc(var(--spacing) * 1.5)}.px-2{padding-inline:calc(var(--spacing) * 2)}.px-2\.5{padding-inline:calc(var(--spacing) * 2.5)}.px-3{padding-inline:calc(var(--spacing) * 3)}.px-3\.5{padding-inline:calc(var(--spacing) * 3.5)}.px-4{padding-inline:calc(var(--spacing) * 4)}.px-5{padding-inline:calc(var(--spacing) * 5)}.px-6{padding-inline:calc(var(--spacing) * 6)}.py-0{padding-block:0}.py-0\.5{padding-block:calc(var(--spacing) * .5)}.py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.py-2{padding-block:calc(var(--spacing) * 2)}.py-2\.5{padding-block:calc(var(--spacing) * 2.5)}.py-3{padding-block:calc(var(--spacing) * 3)}.py-4{padding-block:calc(var(--spacing) * 4)}.py-6{padding-block:calc(var(--spacing) * 6)}.py-10{padding-block:calc(var(--spacing) * 10)}.py-14{padding-block:calc(var(--spacing) * 14)}.pt-1{padding-top:var(--spacing)}.pt-2{padding-top:calc(var(--spacing) * 2)}.pt-4{padding-top:calc(var(--spacing) * 4)}.pt-5{padding-top:calc(var(--spacing) * 5)}.pt-6{padding-top:calc(var(--spacing) * 6)}.pr-0\.5{padding-right:calc(var(--spacing) * .5)}.pr-1{padding-right:var(--spacing)}.pr-3{padding-right:calc(var(--spacing) * 3)}.pr-4{padding-right:calc(var(--spacing) * 4)}.pr-9{padding-right:calc(var(--spacing) * 9)}.pb-1{padding-bottom:var(--spacing)}.pb-2{padding-bottom:calc(var(--spacing) * 2)}.pb-24{padding-bottom:calc(var(--spacing) * 24)}.pl-2{padding-left:calc(var(--spacing) * 2)}.pl-3\.5{padding-left:calc(var(--spacing) * 3.5)}.pl-10{padding-left:calc(var(--spacing) * 10)}.text-center{text-align:center}.text-left{text-align:left}.text-right{text-align:right}.font-mono{font-family:var(--font-mono)}.text-2xl{font-size:var(--text-2xl);line-height:var(--tw-leading,var(--text-2xl--line-height))}.text-base{font-size:var(--text-base);line-height:var(--tw-leading,var(--text-base--line-height))}.text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.text-xs{font-size:var(--text-xs);line-height:var(--tw-leading,var(--text-xs--line-height))}.text-\[11px\]{font-size:11px}.text-\[12\.5px\]{font-size:12.5px}.text-\[13px\]{font-size:13px}.text-\[15px\]{font-size:15px}.leading-4{--tw-leading:calc(var(--spacing) * 4);line-height:calc(var(--spacing) * 4)}.leading-5{--tw-leading:calc(var(--spacing) * 5);line-height:calc(var(--spacing) * 5)}.leading-6{--tw-leading:calc(var(--spacing) * 6);line-height:calc(var(--spacing) * 6)}.font-medium{--tw-font-weight:var(--font-weight-medium);font-weight:var(--font-weight-medium)}.font-normal{--tw-font-weight:var(--font-weight-normal);font-weight:var(--font-weight-normal)}.font-semibold{--tw-font-weight:var(--font-weight-semibold);font-weight:var(--font-weight-semibold)}.tracking-normal{--tw-tracking:var(--tracking-normal);letter-spacing:var(--tracking-normal)}.tracking-tight{--tw-tracking:var(--tracking-tight);letter-spacing:var(--tracking-tight)}.tracking-wider{--tw-tracking:var(--tracking-wider);letter-spacing:var(--tracking-wider)}.break-all{word-break:break-all}.whitespace-pre-wrap{white-space:pre-wrap}.text-body{color:var(--color-body)}.text-brand{color:var(--color-brand)}.text-danger{color:var(--color-danger)}.text-faint,.text-faint\/70{color:var(--color-faint)}@supports (color:color-mix(in lab, red, red)){.text-faint\/70{color:color-mix(in oklab, var(--color-faint) 70%, transparent)}}.text-ink{color:var(--color-ink)}.text-muted{color:var(--color-muted)}.text-ok{color:var(--color-ok)}.text-warn{color:var(--color-warn)}.text-white{color:var(--color-white)}.normal-case{text-transform:none}.uppercase{text-transform:uppercase}.tabular-nums{--tw-numeric-spacing:tabular-nums;font-variant-numeric:var(--tw-ordinal,) var(--tw-slashed-zero,) var(--tw-numeric-figure,) var(--tw-numeric-spacing,) var(--tw-numeric-fraction,)}.accent-brand-solid{accent-color:var(--color-brand-solid)}.opacity-50{opacity:.5}.opacity-80{opacity:.8}.shadow-card{--tw-shadow:var(--af-shadow);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-pop{--tw-shadow:var(--af-shadow-pop);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.shadow-sm{--tw-shadow:0 1px 3px 0 var(--tw-shadow-color,#0000001a), 0 1px 2px -1px var(--tw-shadow-color,#0000001a);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.ring-1{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(1px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.ring-black\/5{--tw-ring-color:#0000000d}@supports (color:color-mix(in lab, red, red)){.ring-black\/5{--tw-ring-color:color-mix(in oklab, var(--color-black) 5%, transparent)}}.blur{--tw-blur:blur(8px);filter:var(--tw-blur,) var(--tw-brightness,) var(--tw-contrast,) var(--tw-grayscale,) var(--tw-hue-rotate,) var(--tw-invert,) var(--tw-saturate,) var(--tw-sepia,) var(--tw-drop-shadow,)}.transition-\[border-color\,box-shadow\]{transition-property:border-color,box-shadow;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-colors{transition-property:color,background-color,border-color,outline-color,text-decoration-color,fill,stroke,--tw-gradient-from,--tw-gradient-via,--tw-gradient-to;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.transition-transform{transition-property:transform,translate,scale,rotate;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.duration-150{--tw-duration:.15s;transition-duration:.15s}.duration-200{--tw-duration:.2s;transition-duration:.2s}.ease-out{--tw-ease:var(--ease-out);transition-timing-function:var(--ease-out)}.outline-none{--tw-outline-style:none;outline-style:none}.select-none{-webkit-user-select:none;user-select:none}.group-open\:rotate-90:is(:where(.group):is([open],:popover-open,:open) *){rotate:90deg}@media (hover:hover){.group-hover\:text-brand:is(:where(.group):hover *){color:var(--color-brand)}}.placeholder\:text-faint::placeholder{color:var(--color-faint)}.backdrop\:bg-black\/45::backdrop{background-color:#00000073}@supports (color:color-mix(in lab, red, red)){.backdrop\:bg-black\/45::backdrop{background-color:color-mix(in oklab, var(--color-black) 45%, transparent)}}.before\:absolute:before{content:var(--tw-content);position:absolute}.before\:-inset-2\.5:before{content:var(--tw-content);inset:calc(var(--spacing) * -2.5)}.before\:top-6:before{content:var(--tw-content);top:calc(var(--spacing) * 6)}.before\:bottom-2:before{content:var(--tw-content);bottom:calc(var(--spacing) * 2)}.before\:-left-2\.5:before{content:var(--tw-content);left:calc(var(--spacing) * -2.5)}.before\:w-\[3px\]:before{content:var(--tw-content);width:3px}.before\:rounded:before{content:var(--tw-content);border-radius:.25rem}.before\:bg-brand:before{content:var(--tw-content);background-color:var(--color-brand)}.before\:content-\[\'\'\]:before{--tw-content:"";content:var(--tw-content)}.last\:border-b:last-child{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.focus-within\:border-brand:focus-within{border-color:var(--color-brand)}.focus-within\:ring-4:focus-within{--tw-ring-shadow:var(--tw-ring-inset,) 0 0 0 calc(4px + var(--tw-ring-offset-width)) var(--tw-ring-color,currentcolor);box-shadow:var(--tw-inset-shadow), var(--tw-inset-ring-shadow), var(--tw-ring-offset-shadow), var(--tw-ring-shadow), var(--tw-shadow)}.focus-within\:ring-brand-ring:focus-within{--tw-ring-color:var(--color-brand-ring)}@media (hover:hover){.hover\:border-brand:hover{border-color:var(--color-brand)}.hover\:border-line-strong:hover{border-color:var(--color-line-strong)}.hover\:bg-hover:hover{background-color:var(--color-hover)}.hover\:bg-raised:hover{background-color:var(--color-raised)}.hover\:text-brand:hover{color:var(--color-brand)}.hover\:text-danger:hover{color:var(--color-danger)}.hover\:text-ink:hover{color:var(--color-ink)}.hover\:underline:hover{text-decoration-line:underline}}.focus\:not-sr-only:focus{clip-path:none;white-space:normal;width:auto;height:auto;margin:0;padding:0;position:static;overflow:visible}.focus\:fixed:focus{position:fixed}.focus\:top-3:focus{top:calc(var(--spacing) * 3)}.focus\:left-3:focus{left:calc(var(--spacing) * 3)}.focus\:outline-none:focus{--tw-outline-style:none;outline-style:none}@media not all and (min-width:64rem){.max-lg\:sticky{position:sticky}.max-lg\:top-14{top:calc(var(--spacing) * 14)}.max-lg\:z-20{z-index:20}.max-lg\:mx-4{margin-inline:calc(var(--spacing) * 4)}.max-lg\:mt-3{margin-top:calc(var(--spacing) * 3)}.max-lg\:contents{display:contents}.max-lg\:flex{display:flex}.max-lg\:hidden{display:none}.max-lg\:min-h-11{min-height:calc(var(--spacing) * 11)}.max-lg\:shrink-0{flex-shrink:0}.max-lg\:gap-0\.5{gap:calc(var(--spacing) * .5)}.max-lg\:overflow-x-auto{overflow-x:auto}.max-lg\:border-b{border-bottom-style:var(--tw-border-style);border-bottom-width:1px}.max-lg\:border-line{border-color:var(--color-line)}.max-lg\:bg-surface{background-color:var(--color-surface)}.max-lg\:px-3{padding-inline:calc(var(--spacing) * 3)}.max-lg\:py-1\.5{padding-block:calc(var(--spacing) * 1.5)}.max-lg\:whitespace-nowrap{white-space:nowrap}}@media (min-width:40rem){.sm\:left-auto{left:auto}.sm\:flex{display:flex}.sm\:grid{display:grid}.sm\:inline{display:inline}.sm\:size-5{width:calc(var(--spacing) * 5);height:calc(var(--spacing) * 5)}.sm\:h-6{height:calc(var(--spacing) * 6)}.sm\:grid-cols-2{grid-template-columns:repeat(2,minmax(0,1fr))}.sm\:grid-cols-\[10rem_minmax\(0\,1fr\)_auto\]{grid-template-columns:10rem minmax(0,1fr) auto}.sm\:grid-cols-\[minmax\(0\,1fr\)_minmax\(0\,1\.1fr\)\]{grid-template-columns:minmax(0,1fr) minmax(0,1.1fr)}.sm\:grid-cols-\[minmax\(0\,260px\)_minmax\(0\,1fr\)\]{grid-template-columns:minmax(0,260px) minmax(0,1fr)}.sm\:items-start{align-items:flex-start}.sm\:gap-3{gap:calc(var(--spacing) * 3)}.sm\:gap-8{gap:calc(var(--spacing) * 8)}.sm\:px-4{padding-inline:calc(var(--spacing) * 4)}.sm\:px-8{padding-inline:calc(var(--spacing) * 8)}.sm\:py-0\.5{padding-block:calc(var(--spacing) * .5)}.sm\:py-1{padding-block:var(--spacing)}.sm\:pt-0\.5{padding-top:calc(var(--spacing) * .5)}.sm\:pt-8{padding-top:calc(var(--spacing) * 8)}.sm\:text-sm{font-size:var(--text-sm);line-height:var(--tw-leading,var(--text-sm--line-height))}.sm\:before\:-left-3\.5:before{content:var(--tw-content);left:calc(var(--spacing) * -3.5)}}@media (min-width:48rem){.md\:block{display:block}.md\:inline{display:inline}}@media (min-width:64rem){.lg\:sticky{position:sticky}.lg\:top-14{top:calc(var(--spacing) * 14)}.lg\:block{display:block}.lg\:grid{display:grid}.lg\:inline-flex{display:inline-flex}.lg\:h-\[calc\(100dvh-3\.5rem\)\]{height:calc(100dvh - 3.5rem)}.lg\:-translate-x-6{--tw-translate-x:calc(var(--spacing) * -6);translate:var(--tw-translate-x) var(--tw-translate-y)}.lg\:translate-x-0{--tw-translate-x:0px;translate:var(--tw-translate-x) var(--tw-translate-y)}.lg\:grid-cols-\[260px_minmax\(0\,1fr\)\]{grid-template-columns:260px minmax(0,1fr)}.lg\:items-start{align-items:flex-start}.lg\:overflow-y-auto{overflow-y:auto}.lg\:border-r{border-right-style:var(--tw-border-style);border-right-width:1px}.lg\:border-line{border-color:var(--color-line)}.lg\:bg-surface{background-color:var(--color-surface)}.lg\:px-3{padding-inline:calc(var(--spacing) * 3)}.lg\:pt-4{padding-top:calc(var(--spacing) * 4)}.lg\:pb-16{padding-bottom:calc(var(--spacing) * 16)}.lg\:opacity-0{opacity:0}.lg\:opacity-100{opacity:1}.lg\:transition-\[transform\,opacity\]{transition-property:transform,opacity;transition-timing-function:var(--tw-ease,var(--default-transition-timing-function));transition-duration:var(--tw-duration,var(--default-transition-duration))}.lg\:duration-200{--tw-duration:.2s;transition-duration:.2s}.lg\:ease-out{--tw-ease:var(--ease-out);transition-timing-function:var(--ease-out)}}.\[\&\:\:-webkit-details-marker\]\:hidden::-webkit-details-marker{display:none}}:root{color-scheme:light;--af-canvas:#f5f7fb;--af-surface:#fff;--af-raised:#f1f5f9;--af-hover:#e8eef6;--af-sunken:#fff;--af-line:#0f172a17;--af-line-strong:#0f172a2b;--af-ink:#0b1220;--af-body:#1e293b;--af-muted:#475569;--af-faint:#64748b;--af-brand:#2563eb;--af-brand-solid:#2563eb;--af-brand-solid-hover:#1d4ed8;--af-brand-soft:#2563eb14;--af-brand-ring:#2563eb38;--af-brand-amber:#c8692f;--af-ok:#047857;--af-ok-soft:#10b9811a;--af-warn:#b45309;--af-warn-soft:#f59e0b1f;--af-danger:#dc2626;--af-danger-soft:#ef444417;--af-shadow:0 1px 2px #0f172a0a, 0 1px 3px #0f172a0a;--af-shadow-pop:0 12px 32px #0f172a24;--af-scrollbar:#0f172a33;--af-scrollbar-hover:#0f172a59;--af-json-key:#1d4ed8;--af-json-string:#047857;--af-json-number:#b45309;--af-json-literal:#7c3aed}@media (prefers-color-scheme:dark){:root{color-scheme:dark;--af-canvas:#08111f;--af-surface:#0c1727;--af-raised:#131e33;--af-hover:#1a2741;--af-sunken:#07101d;--af-line:#94a3b824;--af-line-strong:#94a3b842;--af-ink:#f8fafc;--af-body:#e2e8f0;--af-muted:#a3b1c6;--af-faint:#8391a7;--af-brand:#60a5fa;--af-brand-solid:#2563eb;--af-brand-solid-hover:#3b82f6;--af-brand-soft:#3b82f624;--af-brand-ring:#3b82f659;--af-brand-amber:#e9a57c;--af-ok:#34d399;--af-ok-soft:#10b98124;--af-warn:#fbbf24;--af-warn-soft:#f59e0b21;--af-danger:#f87171;--af-danger-soft:#ef444421;--af-shadow:0 1px 2px #02061766;--af-shadow-pop:0 16px 40px #0206178c;--af-scrollbar:#94a3b838;--af-scrollbar-hover:#94a3b866;--af-json-key:#93c5fd;--af-json-string:#6ee7b7;--af-json-number:#fcd34d;--af-json-literal:#c4b5fd}}@media (prefers-reduced-motion:reduce){*,:before,:after{scroll-behavior:auto!important;transition-duration:0s!important;animation-duration:0s!important}}@property --tw-translate-x{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-y{syntax:"*";inherits:false;initial-value:0}@property --tw-translate-z{syntax:"*";inherits:false;initial-value:0}@property --tw-space-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-divide-y-reverse{syntax:"*";inherits:false;initial-value:0}@property --tw-border-style{syntax:"*";inherits:false;initial-value:solid}@property --tw-leading{syntax:"*";inherits:false}@property --tw-font-weight{syntax:"*";inherits:false}@property --tw-tracking{syntax:"*";inherits:false}@property --tw-ordinal{syntax:"*";inherits:false}@property --tw-slashed-zero{syntax:"*";inherits:false}@property --tw-numeric-figure{syntax:"*";inherits:false}@property --tw-numeric-spacing{syntax:"*";inherits:false}@property --tw-numeric-fraction{syntax:"*";inherits:false}@property --tw-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-shadow-color{syntax:"*";inherits:false}@property --tw-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-inset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-shadow-color{syntax:"*";inherits:false}@property --tw-inset-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-ring-color{syntax:"*";inherits:false}@property --tw-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-inset-ring-color{syntax:"*";inherits:false}@property --tw-inset-ring-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-ring-inset{syntax:"*";inherits:false}@property --tw-ring-offset-width{syntax:"";inherits:false;initial-value:0}@property --tw-ring-offset-color{syntax:"*";inherits:false;initial-value:#fff}@property --tw-ring-offset-shadow{syntax:"*";inherits:false;initial-value:0 0 #0000}@property --tw-blur{syntax:"*";inherits:false}@property --tw-brightness{syntax:"*";inherits:false}@property --tw-contrast{syntax:"*";inherits:false}@property --tw-grayscale{syntax:"*";inherits:false}@property --tw-hue-rotate{syntax:"*";inherits:false}@property --tw-invert{syntax:"*";inherits:false}@property --tw-opacity{syntax:"*";inherits:false}@property --tw-saturate{syntax:"*";inherits:false}@property --tw-sepia{syntax:"*";inherits:false}@property --tw-drop-shadow{syntax:"*";inherits:false}@property --tw-drop-shadow-color{syntax:"*";inherits:false}@property --tw-drop-shadow-alpha{syntax:"";inherits:false;initial-value:100%}@property --tw-drop-shadow-size{syntax:"*";inherits:false}@property --tw-duration{syntax:"*";inherits:false}@property --tw-ease{syntax:"*";inherits:false}@property --tw-content{syntax:"*";inherits:false;initial-value:""}@keyframes spin{to{transform:rotate(360deg)}}@keyframes pulse{50%{opacity:.5}} \ No newline at end of file diff --git a/tenxgraph_api/cli/config_editor/static/app.js b/tenxgraph_api/cli/config_editor/static/app.js index 0be4d8a2..7b4e32c9 100644 --- a/tenxgraph_api/cli/config_editor/static/app.js +++ b/tenxgraph_api/cli/config_editor/static/app.js @@ -1,3 +1,3 @@ -var pe,b,et,Kt,D,Ze,tt,nt,Ae,ae,K,ot,Ee,ze,Me,Zt,ce={},ue=[],Yt=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,fe=Array.isArray;function E(e,t){for(var n in t)e[n]=t[n];return e}function Oe(e){e&&e.parentNode&&e.parentNode.removeChild(e)}function Xt(e,t,n){var r,s,a,i={};for(a in t)a=="key"?r=t[a]:a=="ref"?s=t[a]:i[a]=t[a];if(arguments.length>2&&(i.children=arguments.length>3?pe.call(arguments,2):n),typeof e=="function"&&e.defaultProps!=null)for(a in e.defaultProps)i[a]===void 0&&(i[a]=e.defaultProps[a]);return ie(e,i,r,s,null)}function ie(e,t,n,r,s){var a={type:e,props:t,key:n,ref:r,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:s??++et,__i:-1,__u:0};return s==null&&b.vnode!=null&&b.vnode(a),a}function S(e){return e.children}function le(e,t){this.props=e,this.context=t}function L(e,t){if(t==null)return e.__?L(e.__,e.__i+1):null;for(var n;tt&&D.sort(nt),e=D.shift(),t=D.length,Qt(e)}finally{D.length=de.__r=0}}function st(e,t,n,r,s,a,i,c,u,p,_){var m,l,f,g,$,C,w=r&&r.__k||ue,v=t.length;for(u=en(n,t,w,u,v),m=0;m0?i=e.__k[a]=ie(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):e.__k[a]=i,u=a+l,i.__=e,i.__b=e.__b+1,c=null,(p=i.__i=tn(i,n,u,m))!=-1&&(m--,(c=n[p])&&(c.__u|=2)),c==null||c.__v==null?(p==-1&&(s>_?l--:s<_&&l++),typeof i.type!="function"&&(i.__u|=4)):p!=u&&(p==u-1?l--:p==u+1?l++:(p>u?l--:l++,i.__u|=4))):e.__k[a]=null;if(m)for(a=0;a<_;a++)(c=n[a])!=null&&(2&c.__u)==0&&(c.__e==r&&(r=L(c)),ut(c,c));return r}function at(e,t,n){var r,s;if(typeof e.type=="function"){for(r=e.__k,s=0;r&&s(_?1:0)){for(s=n-1,a=n+1;s>=0||a=0?s--:a++])!=null&&(2&p.__u)==0&&c==p.key&&u==p.type)return i}return-1}function Xe(e,t,n){t[0]=="-"?e.setProperty(t,n??""):e[t]=n==null?"":typeof n!="number"||Yt.test(t)?n:n+"px"}function se(e,t,n,r,s){var a,i;e:if(t=="style")if(typeof n=="string")e.style.cssText=n;else{if(typeof r=="string"&&(e.style.cssText=r=""),r)for(t in r)n&&t in n||Xe(e.style,t,"");if(n)for(t in n)r&&n[t]==r[t]||Xe(e.style,t,n[t])}else if(t[0]=="o"&&t[1]=="n")a=t!=(t=t.replace(ot,"$1")),i=t.toLowerCase(),t=i in e||t=="onFocusOut"||t=="onFocusIn"?i.slice(2):t.slice(2),e.l||(e.l={}),e.l[t+a]=n,n?r?n[K]=r[K]:(n[K]=Ee,e.addEventListener(t,a?Me:ze,a)):e.removeEventListener(t,a?Me:ze,a);else{if(s=="http://www.w3.org/2000/svg")t=t.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if(t!="width"&&t!="height"&&t!="href"&&t!="list"&&t!="form"&&t!="tabIndex"&&t!="download"&&t!="rowSpan"&&t!="colSpan"&&t!="role"&&t!="popover"&&t in e)try{e[t]=n??"";break e}catch{}typeof n=="function"||(n==null||n===!1&&t[4]!="-"?e.removeAttribute(t):e.setAttribute(t,t=="popover"&&n==1?"":n))}}function Qe(e){return function(t){if(this.l){var n=this.l[t.type+e];if(t[ae]==null)t[ae]=Ee++;else if(t[ae]0?e:fe(e)?e.map(ct):e.constructor!==void 0?null:E({},e)}function nn(e,t,n,r,s,a,i,c,u){var p,_,m,l,f,g,$,C=n.props||ce,w=t.props,v=t.type;if(v=="svg"?s="http://www.w3.org/2000/svg":v=="math"?s="http://www.w3.org/1998/Math/MathML":s||(s="http://www.w3.org/1999/xhtml"),a!=null){for(p=0;p=n.__.length&&n.__.push({}),n.__[e]}function me(e){return he=1,He(wt,e)}function He(e,t,n){var r=De(Z++,2);if(r.t=e,!r.__c&&(r.__=[n?n(t):wt(void 0,t),function(c){var u=r.__N?r.__N[0]:r.__[0],p=r.t(u,c);u!==p&&(r.__N=[p,r.__[1]],r.__c.setState({}))}],r.__c=y,!y.__f)){var s=function(c,u,p){if(!r.__c.__H)return!0;var _=!1,m=r.__c.props!==c;if(r.__c.__H.__.some(function(f){if(f.__N){_=!0;var g=f.__[0];f.__=f.__N,f.__N=void 0,g!==f.__[0]&&(m=!0)}}),a){var l=a.call(this,c,u,p);return _?l||m:l}return!_||m};y.__f=!0;var a=y.shouldComponentUpdate,i=y.componentWillUpdate;y.componentWillUpdate=function(c,u,p){if(this.__e){var _=a;a=void 0,s(c,u,p),a=_}i&&i.call(this,c,u,p)},y.shouldComponentUpdate=s}return r.__N||r.__}function W(e,t){var n=De(Z++,3);!k.__s&&kt(n.__H,t)&&(n.__=e,n.u=t,y.__H.__h.push(n))}function yt(e){return he=5,rn(function(){return{current:e}},[])}function rn(e,t){var n=De(Z++,7);return kt(n.__H,t)&&(n.__=e(),n.__H=t,n.__h=e),n.__}function sn(){for(var e;e=gt.shift();){var t=e.__H;if(e.__P&&t)try{t.__h.some(_e),t.__h.some(Re),t.__h=[]}catch(n){t.__h=[],k.__e(n,e.__v)}}}k.__b=function(e){y=null,ft&&ft(e)},k.__=function(e,t){e&&t.__k&&t.__k.__m&&(e.__m=t.__k.__m),bt&&bt(e,t)},k.__r=function(e){_t&&_t(e),Z=0;var t=(y=e.__c).__H;t&&(Ne===y?(t.__h=[],y.__h=[],t.__.some(function(n){n.__N&&(n.__=n.__N),n.u=n.__N=void 0})):(t.__h.some(_e),t.__h.some(Re),t.__h=[],Z=0)),Ne=y},k.diffed=function(e){ht&&ht(e);var t=e.__c;t&&t.__H&&(t.__H.__h.length&&(gt.push(t)!==1&&pt===k.requestAnimationFrame||((pt=k.requestAnimationFrame)||an)(sn)),t.__H.__.some(function(n){n.u&&(n.__H=n.u,n.u=void 0)})),Ne=y=null},k.__c=function(e,t){t.some(function(n){try{n.__h.some(_e),n.__h=n.__h.filter(function(r){return!r.__||Re(r)})}catch(r){t.some(function(s){s.__h&&(s.__h=[])}),t=[],k.__e(r,n.__v)}}),mt&&mt(e,t)},k.unmount=function(e){xt&&xt(e);var t,n=e.__c;n&&n.__H&&(n.__H.__.some(function(r){try{_e(r)}catch(s){t=s}}),n.__H=void 0,t&&k.__e(t,n.__v))};var vt=typeof requestAnimationFrame=="function";function an(e){var t,n=function(){clearTimeout(r),vt&&cancelAnimationFrame(t),setTimeout(e)},r=setTimeout(n,35);vt&&(t=requestAnimationFrame(n))}function _e(e){var t=y,n=e.__c;typeof n=="function"&&(e.__c=void 0,n()),y=t}function Re(e){var t=y;e.__c=e.__(),y=t}function kt(e,t){return!e||e.length!==t.length||t.some(function(n,r){return n!==e[r]})}function wt(e,t){return typeof t=="function"?t(e):t}var Le=[{id:"essential",title:"Essentials",tagline:"What the server needs to boot. Start here.",sections:["core","plugins"]},{id:"recommended",title:"Recommended",tagline:"Lock down access and see what your agents do before you ship.",sections:["auth","authorization","rate_limit","observability"]},{id:"advanced",title:"Advanced",tagline:"Tune connection limits, client-side tools, and your test loop.",sections:["websocket","ag_ui","remote_tools","test","evaluation"]}],Fe={"auth.path":"auth-path",authorization:"authz-path"},xe=new URLSearchParams(location.hash.slice(1)).get("token")||"",d={schema:null,sections:[],tiers:[],fields:new Map,config:{},version:null,exists:!1,path:"",parseError:null,saved:"",validated:null,issues:[],open:new Set,tab:"problems",inspectorOpen:!1,busy:null,toolParams:{},toasts:[],fatal:null},Ct=()=>{},Tt=e=>{Ct=e};function x(e){e&&e(d),Ct()}var St=e=>JSON.parse(JSON.stringify(e)),P=()=>JSON.stringify(d.config),O=e=>e!==null&&typeof e=="object"&&!Array.isArray(e),Ue=(e,t)=>t.reduce((n,r)=>O(n)?n[r]:void 0,e),I=(e,t)=>`${e} ${t}${e===1?"":"s"}`;function $t(e,t,n){let r=e;for(let s of t.slice(0,-1))O(r[s])||(r[s]={}),r=r[s];r[t[t.length-1]]=n}function It(e,t){let n=[],r=e;for(let s of t.slice(0,-1)){if(!O(r))return;n.push([r,s]),r=r[s]}O(r)&&delete r[t[t.length-1]];for(let s=n.length-1;s>=1;s--){let[a,i]=n[s];if(O(a[i])&&Object.keys(a[i]).length===0)delete a[i];else break}}function At(e){let t=Ue(d.config,e);return t!==void 0?t:d.fields.get(e.join("."))?.default}var be=e=>d.sections.find(t=>t.id===e),j=()=>P()!==d.saved,V=e=>!e.optional||e.key in d.config&&d.config[e.key]!==null,Y=e=>d.issues.filter(t=>t.section===e),X=e=>d.issues.find(t=>t.path===e),zt=()=>d.validated!==null&&d.validated!==P();function M(e=d.issues){let t=e.filter(n=>n.level==="error").length;return{errors:t,warnings:e.length-t}}var Mt=e=>Fe[e]||`f-${e.replaceAll(".","-")}`;async function Be(e,t,n){let r={"X-10xGraph-Token":xe};n!==void 0&&(r["Content-Type"]="application/json");let s=await fetch(t,{method:e,headers:r,body:n===void 0?void 0:JSON.stringify(n)}),a={};try{a=await s.json()}catch{}return{ok:s.ok,status:s.status,data:a}}var We=(e,t)=>x(n=>{n.fatal={title:e,text:t}});function Et(e,t){let n=e.indexOf(t);return n===-1?e.length:n}function Ge(){for(let e of d.issues)be(e.section)&&d.open.add(e.section)}async function Ot(){if(!xe)return We("Open the link printed by 10xgraph config","This page needs the session token included in that link.");let e;try{e=await Be("GET","/api/state")}catch{return We("Cannot reach the config server","Check that 10xgraph config is still running in your terminal, then reload.")}if(!e.ok)return We("This link has expired",e.data.error||"Restart 10xgraph config and open the new link it prints.");let t=e.data;x(r=>{r.schema=t.schema;let s=Le.flatMap(i=>i.sections);r.sections=[...t.schema.sections].sort((i,c)=>Et(s,i.id)-Et(s,c.id)),r.tiers=Le.map(i=>({...i,list:i.sections.map(be).filter(Boolean)})),r.tiers[r.tiers.length-1].list.push(...r.sections.filter(i=>!s.includes(i.id)));for(let i of r.sections)for(let c of i.fields)r.fields.set(c.path.join("."),c);r.config=t.config,r.version=t.version,r.exists=t.exists,r.path=t.path,r.parseError=t.parse_error,r.issues=t.issues;let a=t.exists&&!t.parse_error;r.saved=a?P():"",r.validated=a?P():null;for(let i of r.sections)(!i.optional||V(i))&&r.open.add(i.id);Ge()});let n=new URLSearchParams(location.hash.slice(1)).get("section");n&&be(n)&&ge(n)}async function Q(){if(!(d.busy||!Pt())){x(e=>{e.busy="validate"});try{let e=await Be("POST","/api/validate",{config:d.config});if(!e.ok)return T("error",e.data.error||"Validation failed.");d.issues=e.data.issues,d.validated=P(),Ge();let{errors:t,warnings:n}=M(),r={label:"Show",run:()=>ye("problems")};t?T("error",`${I(t,"error")} to fix before saving.`,r):n?T("warning",`Valid, with ${I(n,"warning")}.`,r):T("success","Configuration is valid.")}catch{T("error","Cannot reach the config server.")}finally{x(e=>{e.busy=null})}}}async function ve(){if(!(d.busy||!d.schema)){if(d.exists&&!j()&&!d.parseError)return T("success","No changes to save.");if(Pt()){x(e=>{e.busy="save"});try{let e=await Be("POST","/api/save",{config:d.config,version:d.version});if(e.data.issues&&(d.issues=e.data.issues,d.validated=P(),Ge()),e.ok){d.version=e.data.version,d.config=e.data.config,d.toolParams={},d.validated=P(),d.saved=P(),d.exists=!0,d.parseError=null;let t=e.data.path.split(/[\\/]/).pop();T("success",e.data.backup?`Saved ${t}. The previous version is in ${t}.bak.`:`Created ${t}.`)}else if(e.status===409)T("error",e.data.error,{label:"Reload",run:()=>location.reload()});else if(e.status===422){let{errors:t}=M();T("error",`Not saved: ${I(t,"error")} to fix first.`,{label:"Show",run:()=>ye("problems")})}else T("error",e.data.error||"Save failed.")}catch{T("error","Cannot reach the config server.")}finally{x(e=>{e.busy=null})}}}}function Pt(){let e=Object.values(d.toolParams).some(t=>t.error);return e&&(ge("remote_tools"),T("error","A remote tool has invalid parameters JSON.")),!e}function Je(e,t){x(n=>{t?(n.config[e.key]=St(e.default),n.open.add(e.id)):delete n.config[e.key],e.key==="remote_tools"&&(n.toolParams={})})}function jt(e){x(t=>{t.open.has(e)?t.open.delete(e):t.open.add(e)})}function Nt(e){x(t=>{t.open=e?new Set(t.sections.map(n=>n.id)):new Set})}var Ve=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth";function ge(e,t){x(n=>{n.open.add(e),n.inspectorOpen=!1}),history.replaceState(null,"",`#token=${encodeURIComponent(xe)}§ion=${e}`),requestAnimationFrame(()=>{let n=t?document.getElementById(t):null;if(n){n.scrollIntoView({block:"center",behavior:Ve()}),n.focus({preventScroll:!0});return}document.getElementById(`sec-${e}`)?.scrollIntoView({block:"start",behavior:Ve()}),document.getElementById(`acc-${e}`)?.focus({preventScroll:!0})})}function Rt(e){document.getElementById(`tier-${e}`)?.scrollIntoView({block:"start",behavior:Ve()})}function ye(e){x(t=>{e&&(t.tab=e),t.inspectorOpen=!window.matchMedia("(min-width: 1280px)").matches})}var ee=()=>x(e=>{e.inspectorOpen=!1});async function Dt(){try{await navigator.clipboard.writeText(JSON.stringify(d.config,null,2)+` -`),T("success","Copied 10xgraph.json to the clipboard.")}catch{T("error","The clipboard is not available here.")}}var ln=0;function T(e,t,n){let r=++ln;x(s=>{s.toasts=[...s.toasts,{id:r,kind:e,message:t,action:n}]}),setTimeout(()=>ke(r),n?8e3:4e3)}function ke(e){x(t=>{t.toasts=t.toasts.filter(n=>n.id!==e)})}var Ht={box:'',key:'',shield:'',gauge:'',arrows:'',activity:'',layers:'',wrench:'',flask:'',target:'',checkCircle:'',xCircle:'',alert:'',info:'',file:'',copy:'',x:'',plus:'',trash:'',panel:'',chevronDown:'',expand:'',collapse:'',power:'',spinner:'',book:'',external:''},we={core:"box",auth:"key",authorization:"shield",rate_limit:"gauge",websocket:"arrows",ag_ui:"external",observability:"activity",plugins:"layers",remote_tools:"wrench",test:"flask",evaluation:"target"};var cn=0;function o(e,t,n,r,s,a){t||(t={});var i,c,u=t;if("ref"in u)for(c in u={},t)c=="ref"?i=t[c]:u[c]=t[c];var p={type:e,props:u,key:n,ref:i,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:--cn,__i:-1,__u:0,__source:s,__self:a};if(typeof e=="function"&&(i=e.defaultProps))for(c in i)u[c]===void 0&&(u[c]=i[c]);return b.vnode&&b.vnode(p),p}function h({name:e,class:t="size-4"}){return o("span",{class:`inline-flex shrink-0 ${t}`,"aria-hidden":"true",children:o("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor","stroke-width":"1.75","stroke-linecap":"round","stroke-linejoin":"round",class:"size-full",dangerouslySetInnerHTML:{__html:Ht[e]}})})}function Ce({class:e,children:t}){return o("span",{class:`inline-flex h-5 shrink-0 items-center rounded-full px-2 text-[11px] font-semibold ${e}`,children:t})}function Te({id:e,on:t,label:n,onChange:r}){return o("button",{id:e,type:"button",role:"switch","aria-checked":t?"true":"false","aria-label":n,class:`focus-ring relative inline-flex h-6 w-10 shrink-0 cursor-pointer items-center rounded-full transition-colors duration-200 before:absolute before:-inset-2.5 before:content-[''] ${t?"bg-brand-solid":"bg-line-strong"}`,onClick:()=>r(!t),children:o("span",{class:`pointer-events-none inline-block size-5 rounded-full bg-white shadow-sm ring-1 ring-black/5 transition-transform duration-200 ease-out ${t?"translate-x-[18px]":"translate-x-0.5"}`})})}function Se({id:e,options:t,value:n,invalid:r,onChange:s}){return o("div",{class:"relative",children:[o("select",{id:e,value:n,class:`input cursor-pointer appearance-none pr-9 font-mono ${r?"input-invalid":""}`,onChange:a=>s(a.currentTarget.value),children:t.map(([a,i])=>o("option",{value:a,children:i},a))}),o(h,{name:"chevronDown",class:"pointer-events-none absolute top-1/2 right-3 size-4 -translate-y-1/2 text-faint"})]})}function te({id:e,values:t,placeholder:n,suggestions:r,onChange:s}){let a=yt(null),i=u=>{let p=u.trim();a.current&&(a.current.value=""),p&&!t.includes(p)&&s([...t,p])},c=r?`${e}-options`:void 0;return o("div",{class:"flex min-h-10 cursor-text flex-wrap items-center gap-1.5 rounded-lg border border-line-strong bg-sunken px-1.5 py-1.5 transition-[border-color,box-shadow] duration-150 focus-within:border-brand focus-within:ring-4 focus-within:ring-brand-ring sm:min-h-9 sm:py-1",onClick:u=>{u.target===u.currentTarget&&a.current?.focus()},children:[t.map((u,p)=>o("span",{class:"inline-flex h-7 items-center gap-0.5 rounded-md border border-line bg-raised pr-0.5 pl-2 font-mono text-xs text-body sm:h-6",children:[u,o("button",{type:"button","aria-label":`Remove ${u}`,class:"inline-flex size-6 cursor-pointer items-center justify-center rounded text-faint transition-colors hover:bg-hover hover:text-ink sm:size-5",onClick:()=>s(t.filter((_,m)=>m!==p)),children:o(h,{name:"x",class:"size-3"})})]},u)),o("input",{ref:a,id:e,type:"text",list:c,placeholder:t.length?"Add...":n,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:"h-7 min-w-[7rem] flex-1 bg-transparent px-1 font-mono text-base text-ink outline-none placeholder:text-faint sm:text-sm",onKeyDown:u=>{let p=u.currentTarget;(u.key==="Enter"||u.key===",")&&p.value.trim()?(u.preventDefault(),i(p.value)):u.key==="Backspace"&&!p.value&&t.length&&s(t.slice(0,-1))},onChange:u=>{r?.includes(u.currentTarget.value.trim())&&i(u.currentTarget.value)},onBlur:u=>i(u.currentTarget.value)}),r?o("datalist",{id:c,children:r.filter(u=>!t.includes(u)).map(u=>o("option",{value:u},u))}):null]})}function N({label:e,path:t,help:n,id:r,required:s}){return o("div",{class:"min-w-0",children:[o("label",{for:r,class:"block text-sm font-medium text-ink",children:[e,s?o("span",{class:"text-danger",children:" *"}):null]}),o("div",{class:"mt-0.5 font-mono text-xs text-faint",children:t}),n?o("p",{class:"mt-1.5 text-[13px] leading-5 text-muted",children:n}):null]})}function F({issue:e}){if(!e)return null;let t=e.level==="error";return o("p",{class:`mt-2 flex gap-1.5 text-[13px] leading-5 ${t?"text-danger":"text-warn"}`,role:t?"alert":void 0,children:[o(h,{name:t?"xCircle":"alert",class:"mt-0.5 size-3.5"}),o("span",{children:e.message})]})}function ne({children:e}){return o("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[minmax(0,1fr)_minmax(0,1.1fr)] sm:gap-8",children:e})}function B({id:e,label:t,path:n,value:r,placeholder:s,onInput:a,issue:i,help:c}){return o(ne,{children:[o(N,{label:t,path:n,help:c,id:e}),o("div",{class:"min-w-0 sm:pt-0.5",children:[o("input",{id:e,type:"text",value:r??"",placeholder:s,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${i?.level==="error"?"input-invalid":""}`,onInput:u=>a(u.currentTarget.value)}),o(F,{issue:i})]})]})}function qe({name:e,label:t,options:n,value:r,onPick:s}){return o("div",{role:"radiogroup","aria-label":t,class:"grid gap-3 sm:grid-cols-2",children:n.map(([a,i,c])=>{let u=a===r;return o("button",{type:"button",role:"radio","aria-checked":u?"true":"false",id:`${e}-${a}`,class:`focus-ring flex cursor-pointer items-start gap-3 rounded-xl border p-4 text-left transition-colors duration-150 ${u?"border-brand bg-brand-soft":"border-line bg-surface shadow-card hover:border-line-strong"}`,onClick:()=>{u||s(a)},children:[o("span",{class:`mt-0.5 flex size-4 shrink-0 items-center justify-center rounded-full border ${u?"border-brand-solid bg-brand-solid":"border-line-strong"}`,children:u?o("span",{class:"size-1.5 rounded-full bg-white"}):null}),o("span",{class:"min-w-0",children:[o("span",{class:"block text-sm font-medium text-ink",children:i}),o("span",{class:"mt-0.5 block text-[13px] leading-5 text-muted",children:c})]})]},a)})})}function R({title:e,children:t}){return o("section",{children:[e?o("h4",{class:"eyebrow mb-2.5",children:e}):null,t]})}var un={info:["info","border-brand/20 bg-brand-soft","text-brand"],warn:["alert","border-warn/25 bg-warn-soft","text-warn"],error:["xCircle","border-danger/25 bg-danger-soft","text-danger"]};function $e({kind:e,title:t,lines:n}){let[r,s,a]=un[e];return o("div",{class:`flex gap-3 rounded-xl border px-4 py-3 ${s}`,role:e==="error"?"alert":void 0,children:[o(h,{name:r,class:`mt-0.5 size-4 ${a}`}),o("div",{class:"min-w-0 space-y-1 text-[13px] leading-5 text-body",children:[t?o("p",{class:"font-medium text-ink",children:t}):null,n.map((i,c)=>o("p",{children:i},c))]})]})}function Lt({issue:e}){return o($e,{kind:e.level==="error"?"error":"warn",lines:[o("span",{children:[e.path?o("code",{class:"mr-1.5 font-mono text-xs text-muted",children:e.path}):null,e.message]})]})}function oe({icon:e,tint:t,title:n,text:r,children:s}){return o("div",{class:"px-6 py-14 text-center",children:[o("div",{class:`mx-auto flex size-11 items-center justify-center rounded-full bg-raised ${t}`,children:o(h,{name:e,class:"size-5"})}),o("p",{class:"mt-4 text-sm font-semibold text-ink",children:n}),o("p",{class:"mx-auto mt-1 max-w-xs text-sm leading-6 text-muted",children:r}),s]})}function Ft(){let{errors:e,warnings:t}=M(),n=e+t,r=(s,a,i)=>o("button",{type:"button",role:"tab",id:`tab-${s}`,"aria-selected":d.tab===s?"true":"false",class:`focus-ring inline-flex h-9 cursor-pointer items-center gap-2 rounded-lg px-3 text-sm font-medium transition-colors duration-150 ${d.tab===s?"bg-raised text-ink":"text-muted hover:text-ink"}`,onClick:()=>x(c=>{c.tab=s}),children:[a,i]});return o("aside",{"aria-label":"Inspector",class:`fixed inset-y-0 right-0 z-40 flex w-[min(100vw,400px)] flex-col border-l border-line bg-surface transition-transform duration-200 ease-out xl:sticky xl:top-14 xl:bottom-auto xl:z-auto xl:self-start xl:h-[calc(100dvh-3.5rem)] xl:w-[380px] xl:shrink-0 xl:translate-x-0 ${d.inspectorOpen?"translate-x-0":"translate-x-full"}`,children:[o("div",{class:"flex h-14 shrink-0 items-center gap-1 border-b border-line px-3",children:[o("div",{role:"tablist",class:"flex flex-1 gap-1",children:[r("problems","Problems",d.validated!==null&&n?o("span",{class:`inline-flex h-5 min-w-5 items-center justify-center rounded-full px-1.5 text-[11px] font-semibold ${e?"bg-danger-soft text-danger":"bg-warn-soft text-warn"}`,children:n}):null),r("json","10xgraph.json",null)]}),o("button",{type:"button",class:"icon-btn xl:hidden","aria-label":"Close inspector",onClick:ee,children:o(h,{name:"x",class:"size-5"})})]}),o("div",{class:"min-h-0 flex-1 overflow-y-auto overscroll-contain",children:d.tab==="problems"?o(dn,{}):o(fn,{})})]})}function dn(){if(!d.schema)return null;if(d.validated===null)return o(oe,{icon:"checkCircle",tint:"text-faint",title:"Not checked yet",text:"Validate runs the same rules 10xgraph api applies when it starts.",children:o("button",{type:"button",class:"btn btn-outline mt-5",disabled:!!d.busy,onClick:Q,children:"Validate now"})});let{errors:e,warnings:t}=M(),n=zt(),r=d.sections.map(s=>[s,Y(s.id)]).filter(([,s])=>s.length);return o("div",{children:[o("div",{class:"flex items-center justify-between gap-3 border-b border-line px-4 py-3",children:[o("p",{class:"text-sm text-muted",children:[o("span",{class:e?"font-medium text-danger":"",children:I(e,"error")})," \xB7 ",o("span",{class:t?"font-medium text-warn":"",children:I(t,"warning")})]}),n?o("button",{type:"button",class:"text-sm font-medium text-brand hover:underline",onClick:Q,children:"Re-check"}):null]}),n?o("div",{class:"mx-4 mt-3 flex gap-2 rounded-lg bg-raised px-3 py-2 text-[13px] text-muted",children:[o(h,{name:"info",class:"mt-0.5 size-3.5 text-faint"}),"You changed the form since this check."]}):null,d.issues.length?o("div",{class:"space-y-5 px-3 py-4",children:r.map(([s,a])=>o("div",{children:[o("p",{class:"eyebrow mb-1.5 flex items-center gap-2 px-2",children:[o(h,{name:we[s.id]||"box",class:"size-3.5"}),s.title]}),o("ul",{class:"space-y-0.5",children:a.map((i,c)=>{let u=i.level==="error";return o("li",{children:o("button",{type:"button",onClick:()=>ge(s.id,Mt(i.path)),class:"focus-ring flex w-full cursor-pointer gap-2.5 rounded-lg px-2 py-2 text-left transition-colors duration-150 hover:bg-raised",children:[o(h,{name:u?"xCircle":"alert",class:`mt-0.5 size-4 ${u?"text-danger":"text-warn"}`}),o("span",{class:"min-w-0",children:[o("span",{class:"block text-[13px] leading-5 text-body",children:i.message}),i.path?o("span",{class:"mt-0.5 block truncate font-mono text-xs text-faint",children:i.path}):null]})]})},c)})})]},s.id))}):o(oe,{icon:"checkCircle",tint:"text-ok",title:"No problems found",text:"This configuration passes every check."})]})}function pn(e){return e.replace(/&/g,"&").replace(//g,">").replace(/("(\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g,n=>{let r="var(--af-json-number)";return n.startsWith('"')?r=n.endsWith(":")?"var(--af-json-key)":"var(--af-json-string)":(n==="true"||n==="false"||n==="null")&&(r="var(--af-json-literal)"),`${n}`})}function fn(){let e=JSON.stringify(d.config,null,2).split(` -`);return o("div",{children:[o("div",{class:"sticky top-0 flex items-center justify-between gap-3 border-b border-line bg-surface px-4 py-2.5",children:[o("p",{class:"text-xs text-faint",children:j()||!d.exists?"Preview of what Save will write":"Matches the file on disk"}),o("button",{type:"button",class:"btn btn-ghost h-8 px-2.5 text-xs",onClick:Dt,children:[o(h,{name:"copy",class:"size-3.5"}),"Copy"]})]}),o("div",{class:"py-3 pr-4 font-mono text-[12.5px] leading-5",children:e.map((t,n)=>o("div",{class:"flex",children:[o("span",{class:"w-10 shrink-0 pr-3 text-right text-faint/70 select-none",children:n+1}),o("span",{class:"min-w-0 flex-1 break-all whitespace-pre-wrap",dangerouslySetInnerHTML:{__html:pn(t)}})]},n))})]})}function Ut(e){let t=[];for(let n of e.fields){if(n.show_if&&At(n.show_if.path)!==n.show_if.equals)continue;let r=n.group||"",s=t.find(a=>a.name===r);s||t.push(s={name:r,fields:[]}),s.fields.push(n)}return t}var G=(e,t)=>x(n=>{t===void 0?It(n.config,e):$t(n.config,e,t)});function _n({id:e,value:t,field:n,invalid:r}){let[s,a]=me(String(t??""));return W(()=>{(s===""?t!==void 0:Number(s)!==t)&&a(String(t??""))},[t]),o("input",{id:e,type:"number",inputmode:"decimal",value:s,min:n.min,step:n.step,placeholder:n.placeholder??n.default??"",class:`input font-mono tabular-nums ${r?"input-invalid":""}`,onInput:i=>{let c=i.currentTarget.value;a(c),c===""?G(n.path,void 0):Number.isNaN(Number(c))||G(n.path,Number(c))}})}function Wt({field:e}){let t=e.path.join("."),n=`f-${e.path.join("-")}`,r=Ue(d.config,e.path),s=X(t),a=s?.level==="error";if(e.type==="bool")return o("div",{class:"flex items-start justify-between gap-6 px-5 py-4",children:[o("div",{class:"min-w-0",children:[o(N,{label:e.label,path:t,help:e.help,id:n}),o(F,{issue:s})]}),o("div",{class:"pt-0.5",children:o(Te,{id:n,on:(r??e.default??!1)===!0,label:e.label,onChange:c=>G(e.path,c)})})]});if(e.type==="list")return o("div",{class:"px-5 py-4",children:[o(N,{label:e.label,path:t,help:e.help,id:n,required:e.required}),o("div",{class:"mt-3",children:[o(te,{id:n,values:Array.isArray(r)?r:[],placeholder:e.placeholder,onChange:c=>G(e.path,c.length?c:void 0)}),o(F,{issue:s})]})]});let i;return e.type==="select"?i=o(Se,{id:n,options:e.options.map(c=>[c,c]),value:r??e.default??e.options[0],invalid:a,onChange:c=>G(e.path,c)}):e.type==="number"?i=o(_n,{id:n,value:r,field:e,invalid:a}):i=o("input",{id:n,type:"text",value:r??"",placeholder:e.placeholder,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${a?"input-invalid":""}`,onInput:c=>G(e.path,c.currentTarget.value===""?void 0:c.currentTarget.value)}),o(ne,{children:[o(N,{label:e.label,path:t,help:e.help,id:n,required:e.required}),o("div",{class:"min-w-0 sm:pt-0.5",children:[i,o(F,{issue:s})]})]})}function Vt(){let e=d.config.auth,t=O(e)?"custom":"jwt";return o("div",{class:"space-y-6",children:[o(R,{title:"Method",children:o(qe,{name:"auth",label:"Authentication method",value:t,options:[["jwt","JWT","Verify bearer tokens signed with JWT_SECRET_KEY."],["custom","Custom backend","Your own BaseAuth subclass decides who is calling."]],onPick:n=>x(r=>{r.config.auth=n==="jwt"?"jwt":{method:"custom",path:""}})})}),t==="custom"?o(R,{title:"Backend",children:o("div",{class:"panel",children:o(B,{id:"auth-path",label:"Auth class",path:"auth.path",value:e.path,placeholder:"auth.custom:MyAuth",help:"Import path of a BaseAuth subclass, as module:attribute.",issue:X("auth.path"),onInput:n=>x(r=>{r.config.auth={...r.config.auth,method:"custom",path:n}})})})}):null]})}function hn(e){if(O(e))return"rbac";if(typeof e=="string"){let t=e.trim().toLowerCase();if(t==="ownership")return"ownership";if(["allow_all","default","none"].includes(t))return"allow_all"}return e==null?null:"custom"}var mn={ownership:()=>"ownership",allow_all:()=>"allow_all",custom:()=>"",rbac:()=>({backend:"rbac",roles:{admin:["*"],member:["graph:invoke","graph:stream","graph:read","checkpointer:read"]},default_scopes:[],isolation:"owner"})};function Bt({sec:e}){let t=d.config.authorization,n=hn(t)??"ownership";return o("div",{class:"space-y-6",children:[o(R,{title:"Policy",children:o(qe,{name:"authz",label:"Authorization policy",value:n,options:[["ownership","Ownership","Users can only reach threads they created."],["allow_all","Allow all","Any authenticated user can do anything."],["rbac","Roles and scopes","Map roles to scopes, on top of owner isolation."],["custom","Custom backend","Your own AuthorizationBackend class."]],onPick:r=>x(s=>{s.config.authorization=mn[r]()})})}),n==="custom"?o(R,{title:"Backend",children:o("div",{class:"panel",children:o(B,{id:"authz-path",label:"Backend class",path:"authorization",placeholder:"auth.permissions:MyBackend",value:typeof t=="string"?t:"",issue:X("authorization"),help:"Import path of an AuthorizationBackend, as module:attribute.",onInput:r=>x(s=>{s.config.authorization=r})})})}):null,n==="rbac"?o(bn,{sec:e,value:t}):null]})}function xn({id:e,name:t,onRename:n}){let[r,s]=me(t);return W(()=>s(t),[t]),o("div",{children:[o("label",{for:e,class:"sr-only",children:"Role name"}),o("input",{id:e,type:"text",value:r,class:"input font-mono",autocomplete:"off",spellcheck:!1,onInput:a=>s(a.currentTarget.value),onChange:a=>{n(a.currentTarget.value.trim())||s(t)}})]})}function bn({sec:e,value:t}){let n="roles"in t||!("role_scopes"in t)?"roles":"role_scopes",r=O(t[n])?t[n]:{},s=["*",...e.scopes||[]],a=Object.entries(r),i=u=>x(()=>{t[n]=u}),c=()=>{let u=a.length+1;for(;`role${u}`in r;)u++;i({...r,[`role${u}`]:[]})};return o(S,{children:[o(R,{title:"Roles",children:o("div",{class:"panel",children:[o("div",{class:"hidden grid-cols-[10rem_minmax(0,1fr)_auto] gap-3 border-b border-line px-5 py-2.5 sm:grid",children:[o("span",{class:"text-xs font-medium text-faint",children:"Role"}),o("span",{class:"text-xs font-medium text-faint",children:"Scopes"}),o("span",{class:"w-9"})]}),a.length?o("div",{class:"divide-y divide-line",children:a.map(([u,p],_)=>o("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[10rem_minmax(0,1fr)_auto] sm:items-start",children:[o(xn,{id:`role-name-${_}`,name:u,onRename:m=>!m||m!==u&&m in r?!1:(i(Object.fromEntries(Object.entries(r).map(([l,f])=>[l===u?m:l,f]))),!0)}),o(te,{id:`role-scopes-${_}`,values:Array.isArray(p)?p:[],placeholder:"Add a scope",suggestions:s,onChange:m=>i({...r,[u]:m})}),o("button",{type:"button",class:"icon-btn justify-self-end hover:text-danger","aria-label":`Remove role ${u}`,onClick:()=>i(Object.fromEntries(Object.entries(r).filter(([m])=>m!==u))),children:o(h,{name:"trash"})})]},_))}):o("p",{class:"px-5 py-6 text-center text-sm text-muted",children:"No roles yet. Users only get the default scopes."}),o("div",{class:"flex items-center justify-between gap-4 border-t border-line px-5 py-3",children:[o("p",{class:"text-[13px] text-muted",children:["Read from the user's roles (or role) claim. ",o("code",{class:"font-mono",children:"*"})," grants every scope."]}),o("button",{type:"button",class:"btn btn-ghost",onClick:c,children:[o(h,{name:"plus"}),"Add role"]})]})]})}),o(R,{title:"Defaults",children:o("div",{class:"panel divide-y divide-line",children:[o("div",{class:"px-5 py-4",children:[o(N,{label:"Default scopes",path:"authorization.default_scopes",id:"rbac-default-scopes",help:"Granted to every authenticated user, even without a role."}),o("div",{class:"mt-3",children:o(te,{id:"rbac-default-scopes",values:Array.isArray(t.default_scopes)?t.default_scopes:[],placeholder:"graph:read",suggestions:s,onChange:u=>x(()=>{t.default_scopes=u})})})]}),o(ne,{children:[o(N,{label:"Thread isolation",path:"authorization.isolation",id:"rbac-isolation",help:"owner keeps each user's threads private; none shares them."}),o("div",{class:"min-w-0 sm:pt-0.5",children:o(Se,{id:"rbac-isolation",options:[["owner","owner"],["none","none"]],value:t.isolation??"owner",onChange:u=>x(()=>{t.isolation=u})})})]})]})})]})}function Gt(){let e=Array.isArray(d.config.remote_tools)?d.config.remote_tools:[],t=()=>{x(n=>{e.push({node:"",name:"",description:"",parameters:{type:"object",properties:{},required:[]}}),n.config.remote_tools=e}),requestAnimationFrame(()=>document.getElementById(`tool-${e.length-1}-name`)?.focus())};return e.length?o("div",{class:"space-y-4",children:[e.map((n,r)=>o(vn,{tools:e,tool:n,index:r},r)),o("button",{type:"button",onClick:t,class:"focus-ring flex h-12 w-full cursor-pointer items-center justify-center gap-2 rounded-xl border border-dashed border-line-strong text-sm font-medium text-muted transition-colors duration-150 hover:border-brand hover:text-brand",children:[o(h,{name:"plus"}),"Add remote tool"]})]}):o("div",{class:"rounded-xl border border-dashed border-line-strong bg-surface px-6 py-10 text-center",children:[o("div",{class:"mx-auto flex size-11 items-center justify-center rounded-full bg-raised text-faint",children:o(h,{name:"wrench",class:"size-5"})}),o("p",{class:"mt-4 text-sm font-semibold text-ink",children:"No remote tools yet"}),o("p",{class:"mx-auto mt-1 max-w-sm text-sm leading-6 text-muted",children:"Declare a tool here and the client that calls the API runs it, for example to read the user's location."}),o("button",{type:"button",class:"btn btn-outline mt-5",onClick:t,children:[o(h,{name:"plus"}),"Add remote tool"]})]})}function vn({tools:e,tool:t,index:n}){let r=d.toolParams[n]??{text:JSON.stringify(t.parameters??{},null,2),error:null},s=i=>x(()=>{e[n]={...e[n],...i}}),a=i=>`remote_tools[${n}].${i}`;return o("section",{class:"panel",children:[o("div",{class:"flex items-center gap-3 border-b border-line px-5 py-3",children:[o(h,{name:"wrench",class:"size-4 text-faint"}),o("span",{class:"min-w-0 flex-1 truncate font-mono text-sm font-medium text-ink",children:t.name||`tool_${n+1}`}),t.node?o("span",{class:"rounded-md border border-line bg-raised px-1.5 py-0.5 font-mono text-xs text-muted",children:t.node}):null,o("button",{type:"button",class:"icon-btn hover:text-danger","aria-label":`Remove ${t.name||"tool"}`,onClick:()=>x(i=>{e.splice(n,1),i.toolParams={}}),children:o(h,{name:"trash"})})]}),o("div",{class:"divide-y divide-line",children:[o(B,{id:`tool-${n}-name`,label:"Tool name",path:a("name"),value:t.name,placeholder:"get_location",onInput:i=>s({name:i})}),o(B,{id:`tool-${n}-node`,label:"Graph node",path:a("node"),value:t.node,placeholder:"MAIN",help:"The node whose model may call this tool.",onInput:i=>s({node:i})}),o(B,{id:`tool-${n}-description`,label:"Description",path:a("description"),value:t.description,placeholder:"Returns the user's current city",help:"What the model reads when deciding to call it.",onInput:i=>s({description:i})}),o("div",{class:"px-5 py-4",children:[o(N,{label:"Parameters",path:a("parameters"),help:"JSON schema of the arguments. type must be object.",id:`tool-${n}-params`}),o("textarea",{id:`tool-${n}-params`,rows:7,spellcheck:!1,value:r.text,class:`input mt-3 h-auto py-2.5 font-mono text-[13px] leading-5 ${r.error?"input-invalid":""}`,onInput:i=>{let c=i.currentTarget.value;x(u=>{try{e[n]={...e[n],parameters:JSON.parse(c)},u.toolParams[n]={text:c,error:null}}catch(p){u.toolParams[n]={text:c,error:p.message}}})}}),o(F,{issue:r.error?{level:"error",message:r.error}:null})]})]})]})}function gn(){let e=d.path.split(/[\\/]/).pop()||"10xgraph.json",t=d.path.slice(0,d.path.length-e.length),n=null;return d.schema?d.exists?j()?n=o(Ce,{class:"bg-warn-soft text-warn",children:"Unsaved"}):n=o(Ce,{class:"bg-ok-soft text-ok",children:"Saved"}):n=o(Ce,{class:"bg-brand-soft text-brand",children:"New"}):n=null,o("div",{class:"flex min-w-0 flex-1 items-center gap-2",title:d.path||void 0,children:[o(h,{name:"file",class:"size-4 text-faint"}),o("span",{class:"truncate font-mono text-[13px] font-medium text-ink",children:e}),n,t?o("span",{class:"hidden min-w-0 truncate font-mono text-xs text-faint lg:block",dir:"rtl",children:o("bdi",{children:t})}):null]})}function Jt(){let{errors:e,warnings:t}=M(),n=!!d.schema&&!d.fatal,r=d.busy==="validate";return o("header",{class:"sticky top-0 z-30 flex h-14 items-center gap-2 border-b border-line bg-surface/95 px-3 backdrop-blur sm:gap-3 sm:px-4",children:[o("div",{class:"flex shrink-0 items-center gap-2.5",children:[o("svg",{viewBox:"0 0 48 48",class:"size-7 shrink-0 text-ink","aria-hidden":"true",children:[o("rect",{x:"4",y:"4",width:"40",height:"40",rx:"11",fill:"none",stroke:"currentColor","stroke-width":"4.4"}),o("path",{d:"M13 33L21 23L34 15L34 31L21 23",fill:"none",stroke:"currentColor","stroke-width":"3","stroke-linecap":"round","stroke-linejoin":"round"}),o("g",{fill:"currentColor",children:[o("circle",{cx:"21",cy:"23",r:"4"}),o("circle",{cx:"34",cy:"15",r:"4"}),o("circle",{cx:"34",cy:"31",r:"4"})]}),o("circle",{cx:"13",cy:"33",r:"4.6",class:"fill-brand-amber"})]}),o("div",{class:"hidden items-baseline gap-1.5 md:flex",children:[o("span",{class:"text-[15px] font-semibold tracking-tight text-ink",children:"10xGraph"}),o("span",{class:"text-[15px] text-faint",children:"Config"})]})]}),o("span",{class:"hidden h-6 w-px shrink-0 bg-line md:block","aria-hidden":"true"}),o(gn,{}),o("button",{type:"button",class:"icon-btn relative xl:hidden","aria-label":"Open problems and JSON preview",onClick:()=>ye(),children:[o(h,{name:"panel",class:"size-[18px]"}),e||t?o("span",{class:`absolute top-1 right-1 flex size-4 items-center justify-center rounded-full text-[10px] font-bold text-white ${e?"bg-danger":"bg-warn"}`,children:e||t}):null]}),o("button",{type:"button",class:"btn btn-outline","aria-label":"Validate configuration",disabled:!n||!!d.busy,onClick:Q,children:[o(h,{name:r?"spinner":"checkCircle",class:`size-4 ${r?"animate-spin":""}`}),o("span",{class:"hidden sm:inline",children:"Validate"})]}),o("button",{type:"button",class:"btn btn-primary",onClick:ve,disabled:!n||!!d.busy||d.exists&&!j()&&!d.parseError,children:[d.busy==="save"?o(h,{name:"spinner",class:"size-4 animate-spin"}):null,d.exists?"Save":"Create file",o("span",{class:"kbd hidden lg:inline-flex",children:"Ctrl S"})]})]})}function qt(){return d.fatal?o("div",{class:"panel mt-10",children:o(oe,{icon:"xCircle",tint:"text-danger",title:d.fatal.title,text:d.fatal.text})}):d.schema?o(S,{children:[o("div",{class:"pt-8",children:[o("h1",{class:"text-xl font-semibold tracking-tight text-ink",children:"Server configuration"}),o("p",{class:"mt-1 max-w-prose text-sm leading-6 text-muted",children:"Every 10xgraph.json option on one page. Fill in the essentials to get running, then switch on what you need on the way to production."})]}),o(yn,{}),d.parseError?o("div",{class:"mt-6",children:o($e,{kind:"warn",title:"The existing file is not valid JSON",lines:[`${d.parseError}. The form starts from defaults; saving replaces the file and keeps the old one as 10xgraph.json.bak.`]})}):null,d.tiers.map(e=>o(wn,{tier:e},e.id))]}):o("div",{class:"animate-pulse space-y-4 pt-8","aria-label":"Loading",children:[o("div",{class:"h-7 w-48 rounded-md bg-raised"}),o("div",{class:"h-4 w-80 max-w-full rounded-md bg-raised"}),[0,1,2,3].map(e=>o("div",{class:"h-16 rounded-xl bg-raised"},e))]})}function yn(){let e=d.sections.every(t=>d.open.has(t.id));return o("div",{class:"sticky top-14 z-20 -mx-4 mt-6 flex items-center gap-2 border-b border-line bg-canvas/90 px-4 py-2 backdrop-blur sm:-mx-8 sm:px-8",children:[o("nav",{class:"-ml-2.5 flex min-w-0 flex-1 gap-1 overflow-x-auto [scrollbar-width:none] [&::-webkit-scrollbar]:hidden","aria-label":"Jump to a tier",children:d.tiers.map(t=>o(kn,{tier:t},t.id))}),o("button",{type:"button",class:"btn btn-ghost","aria-label":e?"Collapse all sections":"Expand all sections",onClick:()=>Nt(!e),children:[o(h,{name:e?"collapse":"expand"}),o("span",{class:"hidden sm:inline",children:e?"Collapse all":"Expand all"})]}),o("a",{href:d.schema.docs_url,target:"_blank",rel:"noopener",class:"btn btn-ghost","aria-label":"Configuration reference",children:[o(h,{name:"book"}),o("span",{class:"hidden sm:inline",children:"Reference"})]})]})}function kn({tier:e}){let t=e.list.filter(V).length,{errors:n,warnings:r}=M(d.issues.filter(s=>e.list.some(a=>a.id===s.section)));return o("a",{href:`#tier-${e.id}`,title:`${t} of ${I(e.list.length,"section")} on`,class:"focus-ring inline-flex h-9 shrink-0 items-center gap-2 rounded-lg px-2.5 text-sm text-muted transition-colors duration-150 hover:bg-raised hover:text-ink",onClick:s=>{s.preventDefault(),Rt(e.id)},children:[o("span",{class:"font-medium",children:e.title}),o("span",{class:"font-mono text-xs text-faint tabular-nums",children:[t,"/",e.list.length]}),n||r?o("span",{class:`size-1.5 rounded-full ${n?"bg-danger":"bg-warn"}`}):null]})}function wn({tier:e}){let t=e.list.filter(V).length;return o("section",{id:`tier-${e.id}`,class:"mt-10 scroll-mt-28","aria-labelledby":`tier-${e.id}-title`,children:[o("div",{class:"mb-3",children:[o("div",{class:"flex items-baseline gap-3",children:[o("h2",{id:`tier-${e.id}-title`,class:"text-base font-semibold tracking-tight text-ink",children:e.title}),o("span",{class:"text-xs text-faint",children:[t," of ",e.list.length," on"]})]}),o("p",{class:"mt-0.5 text-sm text-muted",children:e.tagline})]}),o("div",{class:"space-y-3",children:e.list.map(n=>o(Cn,{sec:n},n.id))})]})}function Cn({sec:e}){let t=V(e),n=d.open.has(e.id),{errors:r,warnings:s}=M(Y(e.id)),a=`body-${e.id}`,i=null;return r?i=o("span",{class:"inline-flex shrink-0 items-center gap-1 text-xs font-medium text-danger",title:I(r,"error"),children:[o(h,{name:"xCircle",class:"size-3.5"}),r]}):s&&(i=o("span",{class:"inline-flex shrink-0 items-center gap-1 text-xs font-medium text-warn",title:I(s,"warning"),children:[o(h,{name:"alert",class:"size-3.5"}),s]})),o("section",{id:`sec-${e.id}`,class:`panel scroll-mt-32 ${r?"border-danger/40":""}`,children:[o("div",{class:"flex items-center gap-3 pr-4 sm:pr-5",children:[o("h3",{class:"min-w-0 flex-1",children:o("button",{type:"button",id:`acc-${e.id}`,"aria-expanded":n?"true":"false","aria-controls":a,class:"focus-ring flex w-full cursor-pointer items-center gap-3 rounded-xl py-3.5 pl-3 text-left sm:pl-4",onClick:()=>jt(e.id),children:[o(h,{name:"chevronDown",class:`size-4 text-faint transition-transform duration-200 ${n?"":"-rotate-90"}`}),o("span",{class:`flex size-9 shrink-0 items-center justify-center rounded-lg border border-line ${t?"bg-brand-soft text-brand":"bg-raised text-faint"}`,children:o(h,{name:we[e.id]||"box",class:"size-[18px]"})}),o("span",{class:"min-w-0 flex-1",children:[o("span",{class:"flex flex-wrap items-center gap-x-2 gap-y-0.5",children:[o("span",{class:`text-sm font-semibold ${t?"text-ink":"text-muted"}`,children:e.title}),e.key?o("code",{class:"hidden rounded border border-line bg-raised px-1.5 font-mono text-[11px] leading-4 text-faint sm:inline",children:e.key}):null]}),o("span",{class:"mt-0.5 block text-[13px] leading-5 text-muted",children:e.description})]}),i]})}),e.optional?o("div",{class:"flex shrink-0 items-center gap-2.5",children:[o("span",{class:"hidden w-6 text-xs font-medium text-faint sm:inline",children:t?"On":"Off"}),o(Te,{id:`toggle-${e.id}`,on:t,label:`${t?"Turn off":"Turn on"} ${e.title}`,onChange:c=>Je(e,c)})]}):null]}),o("div",{id:a,role:"region","aria-labelledby":`acc-${e.id}`,inert:!n,class:`grid transition-[grid-template-rows] duration-200 ease-out ${n?"grid-rows-[1fr]":"grid-rows-[0fr]"}`,children:o("div",{class:"min-h-0 overflow-hidden",children:o("div",{class:"space-y-6 rounded-b-xl border-t border-line bg-canvas/60 px-4 py-5 sm:px-5",children:o(Tn,{sec:e,on:t})})})})]})}function Tn({sec:e,on:t}){if(!t)return o("div",{class:"flex flex-col items-start gap-3 sm:flex-row sm:items-center sm:justify-between sm:gap-6",children:[o("p",{class:"text-sm leading-6 text-muted",children:[o("code",{class:"font-mono text-[13px] text-body",children:e.key})," is not in 10xgraph.json. ",e.off_summary||""]}),o("button",{type:"button",class:"btn btn-outline",onClick:()=>Je(e,!0),children:[o(h,{name:"power"}),"Turn on ",e.title.toLowerCase()]})]});let n=new Set([...e.fields.map(s=>s.path.join(".")),...Object.keys(Fe)]),r=Y(e.id).filter(s=>!n.has(s.path));return o(S,{children:[r.length?o("div",{class:"space-y-2",children:r.map((s,a)=>o(Lt,{issue:s},a))}):null,e.notes?.length?o($e,{kind:"info",lines:e.notes}):null,e.widget==="auth"?o(Vt,{}):null,e.widget==="authorization"?o(Bt,{sec:e}):null,e.widget==="remote_tools"?o(Gt,{}):null,Ut(e).map(s=>o(R,{title:s.name,children:o("div",{class:"panel divide-y divide-line",children:s.fields.map(a=>o(Wt,{field:a},a.path.join(".")))})},s.name))]})}var Sn={success:["checkCircle","text-ok"],warning:["alert","text-warn"],error:["xCircle","text-danger"]};function $n(){return o("div",{class:"pointer-events-none fixed right-4 bottom-4 left-4 z-50 flex flex-col items-end gap-2 sm:left-auto","aria-live":"polite",children:d.toasts.map(e=>{let[t,n]=Sn[e.kind];return o("div",{role:e.kind==="error"?"alert":"status",class:"pointer-events-auto flex w-full max-w-sm items-start gap-3 rounded-xl border border-line bg-surface px-4 py-3 text-sm shadow-pop",children:[o(h,{name:t,class:`mt-0.5 size-4 ${n}`}),o("p",{class:"min-w-0 flex-1 leading-5 text-body",children:e.message}),e.action?o("button",{type:"button",class:"shrink-0 font-medium text-brand hover:underline",onClick:()=>{ke(e.id),e.action.run()},children:e.action.label}):null,o("button",{type:"button",class:"-mr-1 shrink-0 cursor-pointer rounded text-faint hover:text-ink","aria-label":"Dismiss",onClick:()=>ke(e.id),children:o(h,{name:"x"})})]},e.id)})})}function In(){let[,e]=He(t=>t+1,0);return W(()=>{Tt(e);let t=r=>{(r.ctrlKey||r.metaKey)&&r.key.toLowerCase()==="s"?(r.preventDefault(),ve()):r.key==="Escape"&&d.inspectorOpen&&ee()},n=r=>{d.schema&&j()&&r.preventDefault()};return document.addEventListener("keydown",t),window.addEventListener("beforeunload",n),Ot(),()=>{document.removeEventListener("keydown",t),window.removeEventListener("beforeunload",n)}},[]),o(S,{children:[o("a",{href:"#main",class:"sr-only z-50 rounded-md bg-brand-solid px-3 py-2 text-white focus:not-sr-only focus:fixed focus:top-3 focus:left-3",children:"Skip to content"}),o(Jt,{}),o("div",{class:"flex",children:[o("main",{id:"main",tabindex:-1,class:"min-w-0 flex-1 focus:outline-none",children:o("div",{class:"mx-auto w-full max-w-4xl px-4 pb-24 sm:px-8",children:o(qt,{})})}),o(Ft,{})]}),d.inspectorOpen?o("div",{class:"fixed inset-0 z-30 bg-black/50 xl:hidden",onClick:ee}):null,o($n,{})]})}dt(o(In,{}),document.getElementById("app")); +var be,x,mt,bn,H,ut,ht,_t,De,fe,X,gt,Fe,He,Le,vn,_e={},ge=[],yn=/acit|ex(?:s|g|n|p|$)|rph|grid|ows|mnc|ntw|ine[ch]|zoo|^ord|itera/i,ve=Array.isArray;function P(e,t){for(var r in t)e[r]=t[r];return e}function qe(e){e&&e.parentNode&&e.parentNode.removeChild(e)}function kn(e,t,r){var o,s,a,i={};for(a in t)a=="key"?o=t[a]:a=="ref"?s=t[a]:i[a]=t[a];if(arguments.length>2&&(i.children=arguments.length>3?be.call(arguments,2):r),typeof e=="function"&&e.defaultProps!=null)for(a in e.defaultProps)i[a]===void 0&&(i[a]=e.defaultProps[a]);return me(e,i,o,s,null)}function me(e,t,r,o,s){var a={type:e,props:t,key:r,ref:o,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:s??++mt,__i:-1,__u:0};return s==null&&x.vnode!=null&&x.vnode(a),a}function w(e){return e.children}function he(e,t){this.props=e,this.context=t}function F(e,t){if(t==null)return e.__?F(e.__,e.__i+1):null;for(var r;tt&&H.sort(_t),e=H.shift(),t=H.length,wn(e)}finally{H.length=xe.__r=0}}function bt(e,t,r,o,s,a,i,d,c,p,m){var g,u,f,v,$,S,C=o&&o.__k||ge,b=t.length;for(c=Cn(r,t,C,c,b),g=0;g0?i=e.__k[a]=me(i.type,i.props,i.key,i.ref?i.ref:null,i.__v):e.__k[a]=i,c=a+u,i.__=e,i.__b=e.__b+1,d=null,(p=i.__i=Sn(i,r,c,g))!=-1&&(g--,(d=r[p])&&(d.__u|=2)),d==null||d.__v==null?(p==-1&&(s>m?u--:sc?u--:u++,i.__u|=4))):e.__k[a]=null;if(g)for(a=0;a(m?1:0)){for(s=r-1,a=r+1;s>=0||a=0?s--:a++])!=null&&(2&p.__u)==0&&d==p.key&&c==p.type)return i}return-1}function pt(e,t,r){t[0]=="-"?e.setProperty(t,r??""):e[t]=r==null?"":typeof r!="number"||yn.test(t)?r:r+"px"}function pe(e,t,r,o,s){var a,i;e:if(t=="style")if(typeof r=="string")e.style.cssText=r;else{if(typeof o=="string"&&(e.style.cssText=o=""),o)for(t in o)r&&t in r||pt(e.style,t,"");if(r)for(t in r)o&&r[t]==o[t]||pt(e.style,t,r[t])}else if(t[0]=="o"&&t[1]=="n")a=t!=(t=t.replace(gt,"$1")),i=t.toLowerCase(),t=i in e||t=="onFocusOut"||t=="onFocusIn"?i.slice(2):t.slice(2),e.l||(e.l={}),e.l[t+a]=r,r?o?r[X]=o[X]:(r[X]=Fe,e.addEventListener(t,a?Le:He,a)):e.removeEventListener(t,a?Le:He,a);else{if(s=="http://www.w3.org/2000/svg")t=t.replace(/xlink(H|:h)/,"h").replace(/sName$/,"s");else if(t!="width"&&t!="height"&&t!="href"&&t!="list"&&t!="form"&&t!="tabIndex"&&t!="download"&&t!="rowSpan"&&t!="colSpan"&&t!="role"&&t!="popover"&&t in e)try{e[t]=r??"";break e}catch{}typeof r=="function"||(r==null||r===!1&&t[4]!="-"?e.removeAttribute(t):e.setAttribute(t,t=="popover"&&r==1?"":r))}}function ft(e){return function(t){if(this.l){var r=this.l[t.type+e];if(t[fe]==null)t[fe]=Fe++;else if(t[fe]0?e:ve(e)?e.map(wt):e.constructor!==void 0?null:P({},e)}function $n(e,t,r,o,s,a,i,d,c){var p,m,g,u,f,v,$,S=r.props||_e,C=t.props,b=t.type;if(b=="svg"?s="http://www.w3.org/2000/svg":b=="math"?s="http://www.w3.org/1998/Math/MathML":s||(s="http://www.w3.org/1999/xhtml"),a!=null){for(p=0;p=r.__.length&&r.__.push({}),r.__[e]}function we(e){return ke=1,Je(Rt,e)}function Je(e,t,r){var o=Ge(ee++,2);if(o.t=e,!o.__c&&(o.__=[r?r(t):Rt(void 0,t),function(d){var c=o.__N?o.__N[0]:o.__[0],p=o.t(c,d);c!==p&&(o.__N=[p,o.__[1]],o.__c.setState({}))}],o.__c=y,!y.__f)){var s=function(d,c,p){if(!o.__c.__H)return!0;var m=!1,g=o.__c.props!==d;if(o.__c.__H.__.some(function(f){if(f.__N){m=!0;var v=f.__[0];f.__=f.__N,f.__N=void 0,v!==f.__[0]&&(g=!0)}}),a){var u=a.call(this,d,c,p);return m?u||g:u}return!m||g};y.__f=!0;var a=y.shouldComponentUpdate,i=y.componentWillUpdate;y.componentWillUpdate=function(d,c,p){if(this.__e){var m=a;a=void 0,s(d,c,p),a=m}i&&i.call(this,d,c,p)},y.shouldComponentUpdate=s}return o.__N||o.__}function M(e,t){var r=Ge(ee++,3);!k.__s&&Nt(r.__H,t)&&(r.__=e,r.u=t,y.__H.__h.push(r))}function Ce(e){return ke=5,An(function(){return{current:e}},[])}function An(e,t){var r=Ge(ee++,7);return Nt(r.__H,t)&&(r.__=e(),r.__H=t,r.__h=e),r.__}function En(){for(var e;e=zt.shift();){var t=e.__H;if(e.__P&&t)try{t.__h.some(ye),t.__h.some(Ve),t.__h=[]}catch(r){t.__h=[],k.__e(r,e.__v)}}}k.__b=function(e){y=null,Tt&&Tt(e)},k.__=function(e,t){e&&t.__k&&t.__k.__m&&(e.__m=t.__k.__m),Pt&&Pt(e,t)},k.__r=function(e){At&&At(e),ee=0;var t=(y=e.__c).__H;t&&(Be===y?(t.__h=[],y.__h=[],t.__.some(function(r){r.__N&&(r.__=r.__N),r.u=r.__N=void 0})):(t.__h.some(ye),t.__h.some(Ve),t.__h=[],ee=0)),Be=y},k.diffed=function(e){Et&&Et(e);var t=e.__c;t&&t.__H&&(t.__H.__h.length&&(zt.push(t)!==1&&$t===k.requestAnimationFrame||(($t=k.requestAnimationFrame)||In)(En)),t.__H.__.some(function(r){r.u&&(r.__H=r.u,r.u=void 0)})),Be=y=null},k.__c=function(e,t){t.some(function(r){try{r.__h.some(ye),r.__h=r.__h.filter(function(o){return!o.__||Ve(o)})}catch(o){t.some(function(s){s.__h&&(s.__h=[])}),t=[],k.__e(o,r.__v)}}),It&&It(e,t)},k.unmount=function(e){Ot&&Ot(e);var t,r=e.__c;r&&r.__H&&(r.__H.__.some(function(o){try{ye(o)}catch(s){t=s}}),r.__H=void 0,t&&k.__e(t,r.__v))};var Mt=typeof requestAnimationFrame=="function";function In(e){var t,r=function(){clearTimeout(o),Mt&&cancelAnimationFrame(t),setTimeout(e)},o=setTimeout(r,35);Mt&&(t=requestAnimationFrame(r))}function ye(e){var t=y,r=e.__c;typeof r=="function"&&(e.__c=void 0,r()),y=t}function Ve(e){var t=y;e.__c=e.__(),y=t}function Nt(e,t){return!e||e.length!==t.length||t.some(function(r,o){return r!==e[o]})}function Rt(e,t){return typeof t=="function"?t(e):t}var Se=[{label:null,items:["get-running"]},{label:"Before production",items:["auth","authorization","rate_limit","observability","token_revocation"]},{label:"Runs and streaming",items:["runs","websocket","ag_ui","remote_tools","media"]},{label:"Project wiring",items:["plugins"]},{label:"Developer tools",items:["test","evaluation"]},{label:"Reference",items:["environment"]}],N={"get-running":{title:"Get running",icon:"flag",description:"Only the agent graph is required. Everything else is optional."},environment:{title:"Server environment (.env)",icon:"terminal",description:"Variables the server reads from .env. The editor never reads that file, so values are not shown."}},Ke={"auth.path":"auth-path",authorization:"authz-path"},jt="tg-config-token",Dt=!1;function On(){let e=new URLSearchParams(location.hash.slice(1)).get("token")||"";try{e&&sessionStorage.setItem(jt,e);let t=sessionStorage.getItem(jt)||"";return Dt=t!==""&&t===(e||t),e||t}catch{return e}}var te=On(),Ye=e=>Dt||!te?`#${e}`:`#token=${encodeURIComponent(te)}&page=${e}`;function Ht(){let e=location.hash.slice(1);if(e.includes("=")){let t=new URLSearchParams(e);return t.get("page")||t.get("section")||""}try{return decodeURIComponent(e)}catch{return e}}var l={schema:null,sections:[],nav:[],fields:new Map,config:{},version:null,exists:!1,path:"",parseError:null,saved:"",validated:null,issues:[],page:"get-running",query:"",sideOpen:!0,sideRender:!0,sideShown:!0,drawerOpen:!1,tab:"json",copied:!1,busy:null,toolParams:{},toasts:[],fatal:null},Lt=()=>{},Ft=e=>{Lt=e};function _(e){e&&e(l),Lt()}var qt=e=>JSON.parse(JSON.stringify(e)),I=()=>JSON.stringify(l.config),z=e=>e!==null&&typeof e=="object"&&!Array.isArray(e),q=(e,t)=>t.reduce((r,o)=>z(r)?r[o]:void 0,e),ne=(e,t)=>`${e} ${t}${e===1?"":"s"}`;function Ut(e,t,r){let o=e;for(let s of t.slice(0,-1))z(o[s])||(o[s]={}),o=o[s];o[t[t.length-1]]=r}function Ze(e,t){let r=[],o=e;for(let s of t.slice(0,-1)){if(!z(o))return;r.push([o,s]),o=o[s]}z(o)&&delete o[t[t.length-1]];for(let s=r.length-1;s>=1;s--){let[a,i]=r[s];if(z(a[i])&&Object.keys(a[i]).length===0)delete a[i];else break}}function G(e){let t=q(l.config,e);return t!==void 0?t:l.fields.get(e.join("."))?.default}var Pn=(e,t)=>JSON.stringify(e)===JSON.stringify(t),T=e=>l.sections.find(t=>t.id===e),R=()=>I()!==l.saved,$e=e=>!e.optional||e.key in l.config&&l.config[e.key]!==null,J=e=>l.issues.filter(t=>t.section===e),U=e=>l.issues.find(t=>t.path===e);function re(e=l.issues){let t=e.filter(r=>r.level==="error").length;return{errors:t,warnings:e.length-t}}var Te=e=>Ke[e]||`f-${e.replaceAll(".","-")}`,Qe=e=>{let t=q(l.config,e.path);return t!==void 0&&!Pn(t,e.default)},Wt=e=>e.id==="core"?!1:e.optional?$e(e):e.fields.some(Qe);function Bt(e){let t=J(e);return t.find(r=>r.level==="error")||t[0]||null}var Vt=e=>!!e.show_if&&G(e.show_if.path)!==e.show_if.equals;function Gt(e){let r=l.fields.get(e.show_if.path.join("."))?.label||e.show_if.path.join("."),o=e.show_if.equals;return`Used when ${r} is ${o===!0?"on":o===!1?"off":String(o)}.`}var oe=e=>{let t=T(e);return N[e]?{id:e,...N[e]}:t?{id:e,title:t.title,description:t.description,icon:null}:null},Jt=()=>/^[\w.]+:[\w.]+$/.test(String(l.config.agent||""));async function et(e,t,r){let o={"X-10xGraph-Token":te};r!==void 0&&(o["Content-Type"]="application/json");let s=await fetch(t,{method:e,headers:o,body:r===void 0?void 0:JSON.stringify(r)}),a={};try{a=await s.json()}catch{}return{ok:s.ok,status:s.status,data:a}}var Xe=(e,t)=>_(r=>{r.fatal={title:e,text:t}});async function Zt(){if(!te)return Xe("Open the link printed by 10xgraph config","This page needs the session token included in that link.");let e;try{e=await et("GET","/api/state")}catch{return Xe("Cannot reach the config server","Check that 10xgraph config is still running in your terminal, then reload.")}if(!e.ok)return Xe("This link has expired",e.data.error||"Restart 10xgraph config and open the new link it prints.");let t=e.data;_(r=>{r.schema=t.schema,r.sections=t.schema.sections;let o=new Set(Se.flatMap(i=>i.items)),s=r.sections.map(i=>i.id).filter(i=>i!=="core"&&!o.has(i));r.nav=Se.map((i,d)=>({label:i.label,items:dN[c]||T(c)):[...s,...i.items]}));for(let i of r.sections)for(let d of i.fields)r.fields.set(d.path.join("."),d);r.config=t.config,r.version=t.version,r.exists=t.exists,r.path=t.path,r.parseError=t.parse_error,r.issues=t.issues;let a=t.exists&&!t.parse_error;r.saved=a?I():"",r.validated=a?I():null}),Ee({focus:!1}),history.replaceState(null,"",Ye(l.page))}var Kt=null;async function Qt(){let e=I();if(!(!l.schema||l.validated===e||Kt===e)&&!Object.values(l.toolParams).some(t=>t.error)){Kt=e;try{let t=await et("POST","/api/validate",{config:JSON.parse(e)});if(!t.ok)return;_(r=>{r.issues=t.data.issues,r.validated=e})}catch{}}}async function Ae(){if(!(l.busy||!l.schema)){if(l.exists&&!R()&&!l.parseError)return j("success","No changes to save.");if(Mn()){_(e=>{e.busy="save"});try{let e=await et("POST","/api/save",{config:l.config,version:l.version});if(e.data.issues&&(l.issues=e.data.issues,l.validated=I()),e.ok){l.version=e.data.version,l.config=e.data.config,l.toolParams={},l.validated=I(),l.saved=I(),l.exists=!0,l.parseError=null;let t=e.data.path.split(/[\\/]/).pop();j("success",e.data.backup?`Saved ${t}. The previous version is in ${t}.bak.`:`Created ${t}.`)}else if(e.status===409)j("error",e.data.error,{label:"Reload",run:()=>location.reload()});else if(e.status===422){let{errors:t}=re();j("error",`Not saved: ${ne(t,"error")} to fix first.`,{label:"Show",run:zn})}else j("error",e.data.error||"Save failed.")}catch{j("error","Cannot reach the config server.")}finally{_(e=>{e.busy=null})}}}}function Mn(){let e=Object.values(l.toolParams).some(t=>t.error);return e&&(O("remote_tools"),j("error","A remote tool has invalid parameters JSON.")),!e}function Xt(e,t){_(r=>{t?r.config[e.key]=qt(e.default):delete r.config[e.key],e.key==="remote_tools"&&(r.toolParams={})})}function zn(){let e=l.issues.find(t=>t.level==="error")||l.issues[0];e&&O(e.section,Te(e.path))}var Nn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches?"auto":"smooth";function Rn(e){return e==="core"?"get-running":N[e]||T(e)?e:"get-running"}function Ee({focus:e=!0}={}){_(t=>{t.page=Rn(Ht()),t.query="",t.drawerOpen=!1}),e&&requestAnimationFrame(()=>{window.scrollTo(0,0),document.getElementById("page-title")?.focus({preventScroll:!0})})}function O(e,t){let r=Ye(e);location.hash!==r&&history.pushState(null,"",r),Ee(),t&&requestAnimationFrame(()=>requestAnimationFrame(()=>{let o=document.getElementById(t);o&&(o.scrollIntoView({block:"center",behavior:Nn()}),o.focus({preventScroll:!0}))}))}function Ie(e){_(t=>{t.query=e})}var jn=()=>window.matchMedia("(prefers-reduced-motion: reduce)").matches,Yt;function K(e,{instant:t=!1}={}){try{localStorage.setItem("tg-sidebar",e?"shown":"hidden")}catch{}clearTimeout(Yt);let r=t||jn();_(o=>{o.sideOpen=e,e?(o.sideRender=!0,o.sideShown=r):o.sideShown=!1,!e&&r&&(o.sideRender=!1)}),e&&!r&&requestAnimationFrame(()=>requestAnimationFrame(()=>_(o=>{o.sideShown=!0}))),!e&&!r&&(Yt=setTimeout(()=>_(o=>{o.sideOpen||(o.sideRender=!1)}),200))}function tt(){l.sideOpen||K(!0,{instant:!0}),requestAnimationFrame(()=>{let e=document.getElementById("q");e?.focus(),e?.select()})}function en(e){_(t=>{e&&(t.tab=e),t.drawerOpen=!0})}var Oe=()=>_(e=>{e.drawerOpen=!1});async function tn(){let e="10xgraph dev";try{await navigator.clipboard.writeText(e)}catch{try{let t=document.createElement("textarea");t.value=e,document.body.append(t),t.select(),document.execCommand("copy"),t.remove()}catch{}}_(t=>{t.copied=!0}),setTimeout(()=>_(t=>{t.copied=!1}),2e3)}async function nn(){try{await navigator.clipboard.writeText(JSON.stringify(l.config,null,2)+` +`),j("success","Copied 10xgraph.json to the clipboard.")}catch{j("error","The clipboard is not available here.")}}var Dn=0;function j(e,t,r){let o=++Dn;_(s=>{s.toasts=[...s.toasts,{id:o,kind:e,message:t,action:r}]}),setTimeout(()=>Pe(o),r?8e3:4e3)}function Pe(e){_(t=>{t.toasts=t.toasts.filter(r=>r.id!==e)})}var rn={box:'',key:'',shield:'',gauge:'',arrows:'',activity:'',layers:'',wrench:'',flask:'',target:'',checkCircle:'',xCircle:'',alert:'',info:'',file:'',copy:'',x:'',plus:'',trash:'',panel:'',chevronDown:'',expand:'',collapse:'',power:'',spinner:'',book:'',flag:'',lock:'',play:'',bolt:'',terminal:'',search:'',chevronRight:'',code:'',circle:'',external:''},W={"get-running":"flag",environment:"terminal",core:"flag",auth:"lock",authorization:"shield",token_revocation:"key",rate_limit:"gauge",observability:"activity",runs:"play",websocket:"bolt",ag_ui:"external",remote_tools:"wrench",media:"file",plugins:"layers",test:"flask",evaluation:"target"};var Hn=0;function n(e,t,r,o,s,a){t||(t={});var i,d,c=t;if("ref"in c)for(d in c={},t)d=="ref"?i=t[d]:c[d]=t[d];var p={type:e,props:c,key:r,ref:i,__k:null,__:null,__b:0,__e:null,__c:null,constructor:void 0,__v:--Hn,__i:-1,__u:0,__source:s,__self:a};if(typeof e=="function"&&(i=e.defaultProps))for(d in i)c[d]===void 0&&(c[d]=i[d]);return x.vnode&&x.vnode(p),p}function h({name:e,class:t="size-4"}){return/(^|\s)size-/.test(t)||(t=`size-4 ${t}`),n("span",{class:`inline-flex shrink-0 ${t}`,"aria-hidden":"true",children:n("svg",{xmlns:"http://www.w3.org/2000/svg",viewBox:"0 0 24 24",fill:"none",stroke:"currentColor","stroke-width":"1.75","stroke-linecap":"round","stroke-linejoin":"round",class:"size-full",dangerouslySetInnerHTML:{__html:rn[e]}})})}function Me({class:e,children:t}){return n("span",{class:`inline-flex h-5 shrink-0 items-center rounded-full px-2 text-[11px] font-semibold ${e}`,children:t})}function ae({id:e,label:t,options:r,value:o,onPick:s}){return n("div",{id:e,role:"radiogroup","aria-label":t,class:"inline-flex max-w-full flex-wrap gap-0.5 rounded-[10px] border border-line-strong bg-raised p-0.5",children:r.map(([a,i])=>n("button",{type:"button",role:"radio","aria-checked":a===o?"true":"false",class:`focus-ring min-h-10 cursor-pointer rounded-lg px-3.5 text-sm font-medium transition-colors duration-150 ${a===o?"bg-surface text-ink shadow-card":"text-muted hover:text-ink"}`,onClick:()=>{a!==o&&s(a)},children:i},a))})}function ze({id:e,options:t,value:r,invalid:o,onChange:s}){return n("div",{class:"relative",children:[n("select",{id:e,value:r,class:`input cursor-pointer appearance-none pr-9 font-mono ${o?"input-invalid":""}`,onChange:a=>s(a.currentTarget.value),children:t.map(([a,i])=>n("option",{value:a,children:i},a))}),n(h,{name:"chevronDown",class:"pointer-events-none absolute top-1/2 right-3 size-4 -translate-y-1/2 text-faint"})]})}function se({id:e,values:t,placeholder:r,suggestions:o,onChange:s}){let a=Ce(null),i=c=>{let p=c.trim();a.current&&(a.current.value=""),p&&!t.includes(p)&&s([...t,p])},d=o?`${e}-options`:void 0;return n("div",{class:"flex min-h-11 cursor-text flex-wrap items-center gap-1.5 rounded-lg border border-line-strong bg-sunken px-1.5 py-1.5 transition-[border-color,box-shadow] duration-150 focus-within:border-brand focus-within:ring-4 focus-within:ring-brand-ring sm:py-1",onClick:c=>{c.target===c.currentTarget&&a.current?.focus()},children:[t.map((c,p)=>n("span",{class:"inline-flex h-7 items-center gap-0.5 rounded-md border border-line bg-raised pr-0.5 pl-2 font-mono text-xs text-body sm:h-6",children:[c,n("button",{type:"button","aria-label":`Remove ${c}`,class:"inline-flex size-6 cursor-pointer items-center justify-center rounded text-faint transition-colors hover:bg-hover hover:text-ink sm:size-5",onClick:()=>s(t.filter((m,g)=>g!==p)),children:n(h,{name:"x",class:"size-3"})})]},c)),n("input",{ref:a,id:e,type:"text",list:d,placeholder:t.length?"Add...":r,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:"h-7 min-w-[7rem] flex-1 bg-transparent px-1 font-mono text-base text-ink outline-none placeholder:text-faint sm:text-sm",onKeyDown:c=>{let p=c.currentTarget;(c.key==="Enter"||c.key===",")&&p.value.trim()?(c.preventDefault(),i(p.value)):c.key==="Backspace"&&!p.value&&t.length&&s(t.slice(0,-1))},onChange:c=>{o?.includes(c.currentTarget.value.trim())&&i(c.currentTarget.value)},onBlur:c=>i(c.currentTarget.value)}),o?n("datalist",{id:d,children:o.filter(c=>!t.includes(c)).map(c=>n("option",{value:c},c))}):null]})}function ie({label:e,path:t,help:r,id:o,required:s}){return n("div",{class:"min-w-0",children:[n("label",{for:o,class:"block text-sm font-medium text-ink",children:[e,s?n("span",{class:"text-danger",children:" *"}):null]}),n("div",{class:"mt-0.5 font-mono text-xs text-faint",children:t}),r?n("p",{class:"mt-1.5 text-[13px] leading-5 text-muted",children:r}):null]})}function le({issue:e}){if(!e)return null;let t=e.level==="error";return n("p",{class:`mt-2 flex gap-1.5 text-[13px] leading-5 ${t?"text-danger":"text-warn"}`,role:t?"alert":void 0,children:[n(h,{name:t?"xCircle":"alert",class:"mt-0.5 size-3.5"}),n("span",{children:e.message})]})}function nt({children:e}){return n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[minmax(0,1fr)_minmax(0,1.1fr)] sm:gap-8",children:e})}function Y({id:e,label:t,path:r,value:o,placeholder:s,onInput:a,issue:i,help:d}){return n(nt,{children:[n(ie,{label:t,path:r,help:d,id:e}),n("div",{class:"min-w-0 sm:pt-0.5",children:[n("input",{id:e,type:"text",value:o??"",placeholder:s,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${i?.level==="error"?"input-invalid":""}`,onInput:c=>a(c.currentTarget.value)}),n(le,{issue:i})]})]})}function rt({name:e,label:t,options:r,value:o,onPick:s}){return n("div",{role:"radiogroup","aria-label":t,class:"grid gap-3 sm:grid-cols-2",children:r.map(([a,i,d])=>{let c=a===o;return n("button",{type:"button",role:"radio","aria-checked":c?"true":"false",id:`${e}-${a}`,class:`focus-ring flex cursor-pointer items-start gap-3 rounded-xl border p-4 text-left transition-colors duration-150 ${c?"border-brand bg-brand-soft":"border-line bg-surface shadow-card hover:border-line-strong"}`,onClick:()=>{c||s(a)},children:[n("span",{class:`mt-0.5 flex size-4 shrink-0 items-center justify-center rounded-full border ${c?"border-brand-solid bg-brand-solid":"border-line-strong"}`,children:c?n("span",{class:"size-1.5 rounded-full bg-white"}):null}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-sm font-medium text-ink",children:i}),n("span",{class:"mt-0.5 block text-[13px] leading-5 text-muted",children:d})]})]},a)})})}function B({title:e,children:t}){return n("section",{children:[e?n("h4",{class:"eyebrow mb-2.5",children:e}):null,t]})}var Ln={info:["info","border-brand/20 bg-brand-soft","text-brand"],warn:["alert","border-warn/25 bg-warn-soft","text-warn"],error:["xCircle","border-danger/25 bg-danger-soft","text-danger"]};function ce({kind:e,title:t,lines:r}){let[o,s,a]=Ln[e];return n("div",{class:`flex gap-3 rounded-xl border px-4 py-3 ${s}`,role:e==="error"?"alert":void 0,children:[n(h,{name:o,class:`mt-0.5 size-4 ${a}`}),n("div",{class:"min-w-0 space-y-1 text-[13px] leading-5 text-body",children:[t?n("p",{class:"font-medium text-ink",children:t}):null,r.map((i,d)=>n("p",{children:i},d))]})]})}function ot({issue:e}){return n(ce,{kind:e.level==="error"?"error":"warn",lines:[n("span",{children:[e.path?n("code",{class:"mr-1.5 font-mono text-xs text-muted",children:e.path}):null,e.message]})]})}function on({icon:e,tint:t,title:r,text:o,children:s}){return n("div",{class:"px-6 py-14 text-center",children:[n("div",{class:`mx-auto flex size-11 items-center justify-center rounded-full bg-raised ${t}`,children:n(h,{name:e,class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:r}),n("p",{class:"mx-auto mt-1 max-w-xs text-sm leading-6 text-muted",children:o}),s]})}function an(){let e=Ce(null);M(()=>{let i=e.current;i&&(l.drawerOpen&&!i.open?i.showModal():!l.drawerOpen&&i.open&&i.close())});let t=l.issues.length>0,r=t&&l.tab==="problems"?"problems":"json",{errors:o,warnings:s}=re(),a=(i,d,c)=>n("button",{type:"button",role:"tab",id:`tab-${i}`,"aria-selected":r===i?"true":"false",class:`focus-ring inline-flex h-10 cursor-pointer items-center gap-2 rounded-lg px-3 text-sm font-medium transition-colors duration-150 ${r===i?"bg-raised text-ink":"text-muted hover:text-ink"}`,onClick:()=>_(p=>{p.tab=i}),children:[d,c]});return n("dialog",{ref:e,"aria-label":"10xgraph.json preview",class:"m-0 ml-auto h-dvh max-h-none w-[min(540px,100vw)] border-0 border-l border-line-strong bg-surface p-0 text-body shadow-pop backdrop:bg-black/45",onClose:()=>{l.drawerOpen&&Oe()},onClick:i=>{i.target===i.currentTarget&&Oe()},children:n("div",{class:"flex h-full flex-col",children:[n("div",{class:"flex h-14 shrink-0 items-center gap-1 border-b border-line px-3",children:[n("div",{role:"tablist",class:"flex flex-1 gap-1",children:[a("json","10xgraph.json",null),t?a("problems","Problems",n("span",{class:`inline-flex h-5 min-w-5 items-center justify-center rounded-full px-1.5 text-[11px] font-semibold ${o?"bg-danger-soft text-danger":"bg-warn-soft text-warn"}`,children:o+s})):null]}),n("button",{type:"button",class:"icon-btn","aria-label":"Close preview",onClick:Oe,children:n(h,{name:"x",class:"size-5"})})]}),n("div",{class:"min-h-0 flex-1 overflow-y-auto overscroll-contain",children:r==="problems"?n(Fn,{}):n(Un,{})})]})})}function Fn(){let{errors:e,warnings:t}=re(),r=l.sections.map(o=>[o,J(o.id)]).filter(([,o])=>o.length);return n("div",{children:[n("div",{class:"border-b border-line px-4 py-3",children:n("p",{class:"text-sm text-muted",children:[n("span",{class:e?"font-medium text-danger":"",children:ne(e,"error")})," \xB7 ",n("span",{class:t?"font-medium text-warn":"",children:ne(t,"warning")})]})}),n("div",{class:"space-y-5 px-3 py-4",children:r.map(([o,s])=>n("div",{children:[n("p",{class:"eyebrow mb-1.5 flex items-center gap-2 px-2",children:[n(h,{name:W[o.id]||"box",class:"size-3.5"}),o.title]}),n("ul",{class:"space-y-0.5",children:s.map((a,i)=>{let d=a.level==="error";return n("li",{children:n("button",{type:"button",onClick:()=>O(o.id,Te(a.path)),class:"focus-ring flex min-h-11 w-full cursor-pointer gap-2.5 rounded-lg px-2 py-2 text-left transition-colors duration-150 hover:bg-raised",children:[n(h,{name:d?"xCircle":"alert",class:`mt-0.5 size-4 ${d?"text-danger":"text-warn"}`}),n("span",{class:"min-w-0",children:[n("span",{class:"block text-[13px] leading-5 text-body",children:a.message}),a.path?n("span",{class:"mt-0.5 block truncate font-mono text-xs text-faint",children:a.path}):null]})]})},i)})})]},o.id))})]})}function qn(e){return e.replace(/&/g,"&").replace(//g,">").replace(/("(\\u[a-fA-F0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+-]?\d+)?)/g,r=>{let o="var(--af-json-number)";return r.startsWith('"')?o=r.endsWith(":")?"var(--af-json-key)":"var(--af-json-string)":(r==="true"||r==="false"||r==="null")&&(o="var(--af-json-literal)"),`${r}`})}function Un(){let e=JSON.stringify(l.config,null,2).split(` +`);return n("div",{children:[n("div",{class:"sticky top-0 flex items-center justify-between gap-3 border-b border-line bg-surface px-4 py-2.5",children:[n("p",{class:"text-xs text-faint",children:R()||!l.exists?"Preview of what Save will write":"Matches the file on disk"}),n("button",{type:"button",class:"btn btn-ghost h-8 px-2.5 text-xs",onClick:nn,children:[n(h,{name:"copy",class:"size-3.5"}),"Copy"]})]}),n("div",{class:"py-3 pr-4 font-mono text-[12.5px] leading-5",children:e.map((t,r)=>n("div",{class:"flex",children:[n("span",{class:"w-10 shrink-0 pr-3 text-right text-faint/70 select-none",children:r+1}),n("span",{class:"min-w-0 flex-1 break-all whitespace-pre-wrap",dangerouslySetInnerHTML:{__html:qn(t)}})]},r))})]})}function st(e){let t=[];for(let r of e){let o=r.group||"",s=t.find(a=>a.name===o);s||t.push(s={name:o,fields:[]}),s.fields.push(r)}return t}var D=(e,t)=>_(r=>{if(t===void 0){Ze(r.config,e);return}let o=e.length>1?r.sections.find(s=>s.optional&&s.key===e[0]):null;o&&(r.config[o.key]===void 0||r.config[o.key]===null)&&(r.config[o.key]=JSON.parse(JSON.stringify(o.default))),Ut(r.config,e,t)}),Wn=e=>`f-${e.path.join("-")}`;function Bn({id:e,value:t,field:r,invalid:o,disabled:s}){let[a,i]=we(String(t??""));return M(()=>{(a===""?t!==void 0:Number(a)!==t)&&i(String(t??""))},[t]),n("input",{id:e,type:"number",inputmode:"decimal",value:a,disabled:s,min:r.min,step:r.step??"any",placeholder:r.placeholder??r.default??"",class:`input max-w-40 font-mono tabular-nums ${o?"input-invalid":""}`,onInput:d=>{let c=d.currentTarget.value;i(c),c===""?D(r.path,void 0):Number.isNaN(Number(c))||D(r.path,Number(c))}})}function at({field:e,value:t}){return e.exclusive_min===void 0||typeof t!="number"||t>e.exclusive_min?null:n("p",{class:"mt-1.5 text-[13px] text-warn",children:["Must be greater than ",e.exclusive_min,"."]})}function Vn({id:e,field:t,value:r,invalid:o}){let s=t.unlimited_value!==void 0&&r===t.unlimited_value,a=n(Bn,{id:e,value:s?void 0:r,field:t,invalid:o,disabled:s});if(t.null_label!==void 0){let i=r===null?"n":r===void 0?"d":"s",d=typeof t.default=="number"?t.default:300;return n("div",{children:[n(ae,{id:e,label:t.label,value:i,options:[["d","Default"],["s","Seconds"],["n",t.null_label]],onPick:c=>D(t.path,c==="d"?void 0:c==="n"?null:d)}),i==="s"?n("div",{class:"mt-2.5",children:[a,n(at,{field:t,value:r})]}):null]})}return t.unlimited_value!==void 0?n("div",{children:[n("div",{class:"flex flex-wrap items-center gap-x-4",children:[a,n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm text-body",children:[n("input",{type:"checkbox",class:"size-[18px] accent-brand-solid",checked:s,onChange:i=>D(t.path,i.currentTarget.checked?t.unlimited_value:void 0)}),"Unlimited"]})]}),n(at,{field:t,value:r})]}):n("div",{children:[a,n(at,{field:t,value:r})]})}function Gn({id:e,field:t,value:r}){if(t.unset_label!==void 0){let o=[["d","Default"]];return t.allow_true!==!1&&o.push(["t","On"]),o.push(["f","Off"]),n("div",{children:[n(ae,{id:e,label:t.label,value:r===void 0?"d":r?"t":"f",options:o,onPick:s=>D(t.path,s==="d"?void 0:s==="t")}),n("p",{class:"mt-1.5 text-xs text-faint",children:["Default: ",t.unset_label]})]})}return n("label",{class:"inline-flex min-h-11 cursor-pointer items-center gap-2.5 text-sm font-semibold text-ink",children:[n("input",{id:e,type:"checkbox",class:"size-[18px] accent-brand-solid",checked:(r??t.default??!1)===!0,onChange:o=>D(t.path,o.currentTarget.checked)}),t.label]})}function Jn({id:e,field:t,value:r,invalid:o}){if(t.type==="bool")return n(Gn,{id:e,field:t,value:r});if(t.type==="number")return n(Vn,{id:e,field:t,value:r,invalid:o});if(t.type==="list")return n(se,{id:e,values:Array.isArray(r)?r:[],placeholder:t.placeholder,onChange:s=>D(t.path,s.length?s:void 0)});if(t.type==="select"){let s=t.options.map(a=>[a,a]);return t.unset_label!==void 0&&s.unshift(["",t.unset_label]),n(ze,{id:e,options:s,invalid:o,value:r??(t.unset_label!==void 0?"":t.default??t.options[0]),onChange:a=>D(t.path,a===""?void 0:a)})}return n("input",{id:e,type:"text",value:r??"",placeholder:t.placeholder,autocomplete:"off",spellcheck:!1,autocapitalize:"off",class:`input font-mono ${o?"input-invalid":""}`,onInput:s=>D(t.path,s.currentTarget.value===""?void 0:s.currentTarget.value)})}function Ne({field:e}){let t=e.path.join("."),r=Wn(e),o=q(l.config,e.path),s=U(t),a=Qe(e),i=Vt(e),d=e.type==="bool"&&e.unset_label===void 0,c=e.default!==void 0&&!a,p=Array.isArray(e.default)?e.default.join(", "):String(e.default??"");return n("div",{id:`row-${r}`,class:`relative pt-5 pb-1 ${a?"before:absolute before:top-6 before:-left-2.5 before:bottom-2 before:w-[3px] before:rounded before:bg-brand before:content-[''] sm:before:-left-3.5":""}`,children:[n("div",{class:"flex min-h-6 items-center gap-2.5",children:[d?null:n("label",{for:r,class:`text-sm font-semibold text-ink ${i?"opacity-50":""}`,children:[e.label,e.required?n("span",{class:"text-danger",children:" *"}):null]}),d?null:n("code",{class:"hidden font-mono text-[11px] text-faint sm:inline",children:t}),c&&!d?n("span",{class:"rounded border border-line px-1.5 text-[11px] leading-4 font-medium text-faint",children:"default"}):null,n("span",{class:"flex-1"}),a?n("button",{type:"button",class:"focus-ring min-h-8 cursor-pointer rounded px-1 text-[13px] font-medium text-brand hover:underline",title:`Reset to default${e.default!==void 0?` (${p})`:""}`,"aria-label":`Reset ${e.label} to default`,onClick:()=>_(m=>Ze(m.config,e.path)),children:"Reset"}):null]}),e.help?n("p",{class:`mt-0.5 max-w-prose text-[13px] leading-5 text-muted ${i?"opacity-50":""}`,children:e.help}):null,n("fieldset",{disabled:i,class:`m-0 mt-2.5 min-w-0 max-w-[520px] border-0 p-0 ${i?"opacity-50":""}`,children:n(Jn,{id:r,field:e,value:o,invalid:s?.level==="error"})}),i?n("p",{class:"mt-1.5 text-xs text-faint",children:Gt(e)}):null,n(le,{issue:s})]})}function sn(){let e=l.config.auth,t=z(e)?"custom":"jwt";return n("div",{class:"space-y-6",children:[n(B,{title:"Method",children:n(rt,{name:"auth",label:"Authentication method",value:t,options:[["jwt","JWT","Verify bearer tokens signed with JWT_SECRET_KEY."],["custom","Custom backend","Your own BaseAuth subclass decides who is calling."]],onPick:r=>_(o=>{o.config.auth=r==="jwt"?"jwt":{method:"custom",path:""}})})}),t==="custom"?n(B,{title:"Backend",children:n("div",{class:"panel",children:n(Y,{id:"auth-path",label:"Auth class",path:"auth.path",value:e.path,placeholder:"auth.custom:MyAuth",help:"Import path of a BaseAuth subclass, as module:attribute.",issue:U("auth.path"),onInput:r=>_(o=>{o.config.auth={...o.config.auth,method:"custom",path:r}})})})}):null]})}function Kn(e){if(z(e))return"rbac";if(typeof e=="string"){let t=e.trim().toLowerCase();if(t==="ownership")return"ownership";if(["allow_all","default","none"].includes(t))return"allow_all"}return e==null?null:"custom"}var Yn={ownership:()=>"ownership",allow_all:()=>"allow_all",custom:()=>"",rbac:()=>({backend:"rbac",roles:{admin:["*"],member:["graph:invoke","graph:stream","graph:read","checkpointer:read"]},default_scopes:[],isolation:"owner"})};function ln({sec:e}){let t=l.config.authorization,r=Kn(t)??"ownership";return n("div",{class:"space-y-6",children:[n(B,{title:"Policy",children:n(rt,{name:"authz",label:"Authorization policy",value:r,options:[["ownership","Ownership","Users can only reach threads they created."],["allow_all","Allow all","Any authenticated user can do anything."],["rbac","Roles and scopes","Map roles to scopes, on top of owner isolation."],["custom","Custom backend","Your own AuthorizationBackend class."]],onPick:o=>_(s=>{s.config.authorization=Yn[o]()})})}),r==="custom"?n(B,{title:"Backend",children:n("div",{class:"panel",children:n(Y,{id:"authz-path",label:"Backend class",path:"authorization",placeholder:"auth.permissions:MyBackend",value:typeof t=="string"?t:"",issue:U("authorization"),help:"Import path of an AuthorizationBackend, as module:attribute.",onInput:o=>_(s=>{s.config.authorization=o})})})}):null,r==="rbac"?n(Qn,{sec:e,value:t}):null]})}function Zn({id:e,name:t,onRename:r}){let[o,s]=we(t);return M(()=>s(t),[t]),n("div",{children:[n("label",{for:e,class:"sr-only",children:"Role name"}),n("input",{id:e,type:"text",value:o,class:"input font-mono",autocomplete:"off",spellcheck:!1,onInput:a=>s(a.currentTarget.value),onChange:a=>{r(a.currentTarget.value.trim())||s(t)}})]})}function Qn({sec:e,value:t}){let r="roles"in t||!("role_scopes"in t)?"roles":"role_scopes",o=z(t[r])?t[r]:{},s=["*",...e.scopes||[]],a=Object.entries(o),i=c=>_(()=>{t[r]=c}),d=()=>{let c=a.length+1;for(;`role${c}`in o;)c++;i({...o,[`role${c}`]:[]})};return n(w,{children:[n(B,{title:"Roles",children:n("div",{class:"panel",children:[n("div",{class:"hidden grid-cols-[10rem_minmax(0,1fr)_auto] gap-3 border-b border-line px-5 py-2.5 sm:grid",children:[n("span",{class:"text-xs font-medium text-faint",children:"Role"}),n("span",{class:"text-xs font-medium text-faint",children:"Scopes"}),n("span",{class:"w-9"})]}),a.length?n("div",{class:"divide-y divide-line",children:a.map(([c,p],m)=>n("div",{class:"grid gap-3 px-5 py-4 sm:grid-cols-[10rem_minmax(0,1fr)_auto] sm:items-start",children:[n(Zn,{id:`role-name-${m}`,name:c,onRename:g=>!g||g!==c&&g in o?!1:(i(Object.fromEntries(Object.entries(o).map(([u,f])=>[u===c?g:u,f]))),!0)}),n(se,{id:`role-scopes-${m}`,values:Array.isArray(p)?p:[],placeholder:"Add a scope",suggestions:s,onChange:g=>i({...o,[c]:g})}),n("button",{type:"button",class:"icon-btn justify-self-end hover:text-danger","aria-label":`Remove role ${c}`,onClick:()=>i(Object.fromEntries(Object.entries(o).filter(([g])=>g!==c))),children:n(h,{name:"trash"})})]},m))}):n("p",{class:"px-5 py-6 text-center text-sm text-muted",children:"No roles yet. Users only get the default scopes."}),n("div",{class:"flex items-center justify-between gap-4 border-t border-line px-5 py-3",children:[n("p",{class:"text-[13px] text-muted",children:["Read from the user's roles (or role) claim. ",n("code",{class:"font-mono",children:"*"})," grants every scope."]}),n("button",{type:"button",class:"btn btn-ghost",onClick:d,children:[n(h,{name:"plus"}),"Add role"]})]})]})}),n(B,{title:"Defaults",children:n("div",{class:"panel divide-y divide-line",children:[n("div",{class:"px-5 py-4",children:[n(ie,{label:"Default scopes",path:"authorization.default_scopes",id:"rbac-default-scopes",help:"Granted to every authenticated user, even without a role."}),n("div",{class:"mt-3",children:n(se,{id:"rbac-default-scopes",values:Array.isArray(t.default_scopes)?t.default_scopes:[],placeholder:"graph:read",suggestions:s,onChange:c=>_(()=>{t.default_scopes=c})})})]}),n(nt,{children:[n(ie,{label:"Thread isolation",path:"authorization.isolation",id:"rbac-isolation",help:"owner keeps each user's threads private; none shares them."}),n("div",{class:"min-w-0 sm:pt-0.5",children:n(ze,{id:"rbac-isolation",options:[["owner","owner"],["none","none"]],value:t.isolation??"owner",onChange:c=>_(()=>{t.isolation=c})})})]})]})})]})}function cn(){let e=Array.isArray(l.config.remote_tools)?l.config.remote_tools:[],t=()=>{_(r=>{e.push({node:"",name:"",description:"",parameters:{type:"object",properties:{},required:[]}}),r.config.remote_tools=e}),requestAnimationFrame(()=>document.getElementById(`tool-${e.length-1}-name`)?.focus())};return e.length?n("div",{class:"space-y-4",children:[e.map((r,o)=>n(Xn,{tools:e,tool:r,index:o},o)),n("button",{type:"button",onClick:t,class:"focus-ring flex h-12 w-full cursor-pointer items-center justify-center gap-2 rounded-xl border border-dashed border-line-strong text-sm font-medium text-muted transition-colors duration-150 hover:border-brand hover:text-brand",children:[n(h,{name:"plus"}),"Add remote tool"]})]}):n("div",{class:"rounded-xl border border-dashed border-line-strong bg-surface px-6 py-10 text-center",children:[n("div",{class:"mx-auto flex size-11 items-center justify-center rounded-full bg-raised text-faint",children:n(h,{name:"wrench",class:"size-5"})}),n("p",{class:"mt-4 text-sm font-semibold text-ink",children:"No remote tools yet"}),n("p",{class:"mx-auto mt-1 max-w-sm text-sm leading-6 text-muted",children:"Declare a tool here and the client that calls the API runs it, for example to read the user's location."}),n("button",{type:"button",class:"btn btn-outline mt-5",onClick:t,children:[n(h,{name:"plus"}),"Add remote tool"]})]})}function Xn({tools:e,tool:t,index:r}){let o=l.toolParams[r]??{text:JSON.stringify(t.parameters??{},null,2),error:null},s=i=>_(()=>{e[r]={...e[r],...i}}),a=i=>`remote_tools[${r}].${i}`;return n("section",{class:"panel",children:[n("div",{class:"flex items-center gap-3 border-b border-line px-5 py-3",children:[n(h,{name:"wrench",class:"size-4 text-faint"}),n("span",{class:"min-w-0 flex-1 truncate font-mono text-sm font-medium text-ink",children:t.name||`tool_${r+1}`}),t.node?n("span",{class:"rounded-md border border-line bg-raised px-1.5 py-0.5 font-mono text-xs text-muted",children:t.node}):null,n("button",{type:"button",class:"icon-btn hover:text-danger","aria-label":`Remove ${t.name||"tool"}`,onClick:()=>_(i=>{e.splice(r,1),i.toolParams={}}),children:n(h,{name:"trash"})})]}),n("div",{class:"divide-y divide-line",children:[n(Y,{id:`tool-${r}-name`,label:"Tool name",path:a("name"),value:t.name,placeholder:"get_location",onInput:i=>s({name:i})}),n(Y,{id:`tool-${r}-node`,label:"Graph node",path:a("node"),value:t.node,placeholder:"MAIN",help:"The node whose model may call this tool.",onInput:i=>s({node:i})}),n(Y,{id:`tool-${r}-description`,label:"Description",path:a("description"),value:t.description,placeholder:"Returns the user's current city",help:"What the model reads when deciding to call it.",onInput:i=>s({description:i})}),n("div",{class:"px-5 py-4",children:[n(ie,{label:"Parameters",path:a("parameters"),help:"JSON schema of the arguments. type must be object.",id:`tool-${r}-params`}),n("textarea",{id:`tool-${r}-params`,rows:7,spellcheck:!1,value:o.text,class:`input mt-3 h-auto py-2.5 font-mono text-[13px] leading-5 ${o.error?"input-invalid":""}`,onInput:i=>{let d=i.currentTarget.value;_(c=>{try{e[r]={...e[r],parameters:JSON.parse(d)},c.toolParams[r]={text:d,error:null}}catch(p){c.toolParams[r]={text:d,error:p.message}}})}}),n(le,{issue:o.error?{level:"error",message:o.error}:null})]})]})]})}var ue=({children:e,code:t})=>n("h1",{id:"page-title",tabindex:-1,class:"flex flex-wrap items-baseline gap-x-3 gap-y-1 text-2xl font-semibold tracking-tight text-ink focus:outline-none",children:[e,t?n("code",{class:"rounded border border-line bg-raised px-1.5 font-mono text-xs font-normal text-faint",children:t}):null]}),it=({children:e})=>n("p",{class:"mt-1.5 max-w-[62ch] text-[15px] leading-6 text-muted",children:e}),un=({children:e})=>n("h2",{class:"mt-10 mb-2 eyebrow",children:e});function dn({id:e,text:t,warn:r,compact:o}){let s=oe(e);return s?n("a",{href:`#${e}`,onClick:a=>{a.preventDefault(),O(e)},class:`focus-ring group flex items-center gap-3.5 border-t border-line py-2 last:border-b ${o?"min-h-12":"min-h-14"}`,children:[n(h,{name:W[e]||"box",class:"size-[18px] text-faint"}),n("span",{class:`min-w-0 flex-1 ${o?"flex flex-wrap items-baseline gap-x-4":""}`,children:[n("span",{class:`block text-sm text-ink group-hover:text-brand ${o?"min-w-40 font-medium":"font-semibold"}`,children:s.title}),n("span",{class:`block text-[13px] leading-5 ${r?"text-warn":"text-muted"}`,children:t??s.description})]}),n(h,{name:"chevronRight",class:"size-4 text-faint"})]}):null}function lt(e){return`Not set. ${T(e)?.off_summary||""}`.trim()}function er(){let e=l.config.auth;return e==null?{text:lt("auth")}:e==="jwt"?{text:"JWT bearer tokens."}:e&&typeof e=="object"?{text:`Custom backend${e.path?`: ${e.path}`:"."}`}:{text:String(e)}}function tr(){let e=l.config.rate_limit;if(e==null)return{text:lt("rate_limit")};let t=a=>G(["rate_limit",a]);if(t("enabled")===!1)return{text:"Settings kept, not enforced."};let r={ip:"per IP",user:"per user",global:"across all clients"}[t("by")]||t("by"),o=U("rate_limit.backend")?.level==="warning",s=["requests","window","by"].some(a=>q(l.config,["rate_limit",a])!==void 0&&q(l.config,["rate_limit",a])!==l.fields.get(`rate_limit.${a}`)?.default);return{text:`${t("requests")} requests per ${t("window")} s, ${r}${s?"":" (default)"}.${o?" The memory backend counts per process.":""}`,warn:o}}function nr(){let e=l.config.observability;if(e==null)return{text:lt("observability")};let t=[];return G(["observability","logfire","enabled"])&&t.push("Logfire"),G(["observability","langsmith","enabled"])&&t.push("LangSmith"),{text:`Level ${G(["observability","level"])}${t.length?`, sending to ${t.join(" and ")}`:", no destination chosen yet"}.`}}function pn(){let e=T("core"),t=Jt(),r=e?J("core").filter(a=>!e.fields.some(i=>i.path.join(".")===a.path)):[],o=[["auth",er()],["rate_limit",tr()],["observability",nr()]],s=l.nav.flatMap(a=>a.items).filter(a=>a!=="get-running");return n(w,{children:[n(ue,{children:t?"You're ready to run":"One step to run your agent"}),n(it,{children:N["get-running"].description}),n("div",{class:`mt-4 flex items-center gap-2 font-medium ${t?"text-ok":"text-muted"}`,children:[n(h,{name:t?"checkCircle":"circle",class:"size-[18px]"}),"Required: ",t?"1":"0"," of 1 done"]}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((a,i)=>n(ot,{issue:a},i))}):null,n("div",{class:"mt-2",children:e?st(e.fields).flatMap(a=>a.fields).map(a=>n(Ne,{field:a},a.path.join("."))):null}),t?n("div",{class:"mt-7 border-t border-line pt-5",children:[n("div",{class:"flex items-center gap-2.5 font-medium text-ok",children:[n(h,{name:"checkCircle",class:"size-[18px]"}),"Ready for 10xgraph dev"]}),n("div",{class:"mt-3 flex max-w-[420px] items-center gap-2 rounded-lg border border-line-strong bg-sunken py-0 pr-1 pl-3.5",children:[n("code",{class:"flex-1 font-mono text-[13px] font-medium text-ink",children:"10xgraph dev"}),l.copied?n("span",{class:"text-xs text-ok",children:"Copied"}):null,n("button",{type:"button",class:"icon-btn",onClick:tn,"aria-label":l.copied?"Copied 10xgraph dev":"Copy command 10xgraph dev",children:n(h,{name:"copy",class:"size-[18px]"})})]}),n("p",{class:"mt-3 text-muted",children:"Everything else already has working defaults."})]}):n("p",{class:"mt-4 text-xs text-faint",children:"Add the agent graph above, then run 10xgraph dev."}),n(un,{children:"Before production (optional)"}),n("div",{children:o.map(([a,i])=>n(dn,{id:a,text:i.text,warn:i.warn},a))}),n(un,{children:"What you can configure"}),n("div",{children:s.map(a=>n(dn,{id:a,compact:!0},a))})]})}function Re({vars:e}){return n("dl",{class:"mb-3 grid gap-x-4 gap-y-1 sm:grid-cols-[minmax(0,260px)_minmax(0,1fr)]",children:e.map(t=>[n("dt",{class:"py-0.5 font-mono text-xs font-medium break-all text-ink",children:t.name},`${t.name}-n`),n("dd",{class:"m-0 pb-2 text-[13px] text-muted sm:py-0.5",children:t.purpose},`${t.name}-p`)])})}function rr({vars:e,open:t}){return e?.length?n("details",{class:"group mt-9 border-t border-line pt-1",open:t,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n("p",{class:"mb-2 text-xs text-faint",children:"Read only. The editor never reads .env, so only names are shown."}),n(Re,{vars:e})]}):null}function fn({sec:e}){return e.widget==="auth"?n(sn,{}):e.widget==="authorization"?n(ln,{sec:e}):e.widget==="remote_tools"?n(cn,{}):null}function or(e){let t=new Set([...e.fields.map(r=>r.path.join(".")),...Object.keys(Ke)]);return J(e.id).filter(r=>!t.has(r.path))}function ar({sec:e}){let t=$e(e),r=e.widget&&(t||!e.optional);return n(w,{children:[r?n("div",{class:"mt-6",children:n(fn,{sec:e})}):null,n("div",{class:e.optional&&!t?"opacity-80":"",children:st(e.fields).map((o,s)=>n("div",{children:[o.name?n("h3",{class:`eyebrow ${s===0?"mt-6":"mt-8 border-t border-line pt-4"}`,children:o.name}):null,o.fields.map(a=>n(Ne,{field:a},a.path.join(".")))]},o.name))})]})}function sr({sec:e}){let t=$e(e),r=or(e);return n(w,{children:[n(ue,{code:e.key,children:e.title}),n(it,{children:e.description}),r.length?n("div",{class:"mt-4 space-y-2",children:r.map((o,s)=>n(ot,{issue:o},s))}):null,e.optional?n("div",{class:"mt-5 flex flex-wrap items-center gap-4",children:[n(ae,{label:`Settings for ${e.title}`,value:t?"c":"d",options:[["d","Use defaults"],["c","Customize"]],onPick:o=>Xt(e,o==="c")}),n("p",{class:"m-0 min-w-56 flex-1 text-[13px] text-muted",children:t?`Saved in 10xgraph.json under ${e.key}.`:e.off_summary})]}):null,e.notes?.length?n("div",{class:"mt-5",children:n(ce,{kind:"info",lines:e.notes})}):null,n(ar,{sec:e}),n(rr,{vars:e.env})]})}function ir(){let e=l.schema.environment||[];return n(w,{children:[n(ue,{children:N.environment.title}),n(it,{children:"These variables are set in .env or your deployment environment. The editor never reads that file, so only names and purposes are shown."}),e.map(t=>n("section",{children:[n("h2",{class:"mt-8 mb-2 text-sm font-semibold text-ink",children:t.title}),t.note?n("p",{class:"mb-2 text-xs text-faint",children:t.note}):null,n(Re,{vars:t.vars})]},t.title))]})}var mn=e=>String(e??"").toLowerCase();function lr(e){let t=s=>e.every(a=>mn(s).includes(a)),r=[];for(let s of l.sections){let a=t(`${s.title} ${s.description} ${s.key||""}`),i=s.fields.filter(p=>a||t(`${p.label} ${p.help||""} ${p.path.join(".")}`)),d=(s.env||[]).filter(p=>t(`${p.name} ${p.purpose}`)),c=!!s.widget&&a;(i.length||d.length||c)&&r.push({id:s.id,sec:s,fields:i,env:d,widget:c})}let o=(l.schema.environment||[]).map(s=>({title:s.title,vars:s.vars.filter(a=>t(`${a.name} ${a.purpose} ${s.title}`))})).filter(s=>s.vars.length);return{results:r,envGroups:o}}function cr(){let e=mn(l.query).split(/\s+/).filter(Boolean),{results:t,envGroups:r}=lr(e),o=t.reduce((s,a)=>s+a.fields.length+a.env.length+(a.widget?1:0),0)+r.reduce((s,a)=>s+a.vars.length,0);return o?n(w,{children:[n(ue,{code:l.query,children:[o," ",o===1?"setting matches":"settings match"]}),t.map(({id:s,sec:a,fields:i,env:d,widget:c})=>n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:W[s]||"box",class:"size-4"}),a.title,n("a",{href:`#${s}`,onClick:p=>{p.preventDefault(),O(s)},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),c?n("div",{class:"mt-4",children:n(fn,{sec:a})}):null,i.map(p=>n(Ne,{field:p},p.path.join("."))),d.length?n("details",{class:"group mt-5",open:!0,children:[n("summary",{class:"focus-ring flex min-h-11 cursor-pointer list-none items-center gap-2 font-medium text-muted [&::-webkit-details-marker]:hidden",children:[n(h,{name:"chevronRight",class:"size-4 transition-transform group-open:rotate-90"}),"Set in .env"]}),n(Re,{vars:d})]}):null]},s)),r.length?n("section",{class:"mt-8",children:[n("h2",{class:"eyebrow m-0 flex items-center gap-2",children:[n(h,{name:"terminal",class:"size-4"}),N.environment.title,n("a",{href:"#environment",onClick:s=>{s.preventDefault(),O("environment")},class:"focus-ring ml-auto text-xs font-medium tracking-normal text-brand normal-case hover:underline",children:"Open page"})]}),r.map(s=>n("div",{children:[n("h3",{class:"mt-4 mb-1.5 text-sm font-semibold text-ink",children:s.title}),s.note?n("p",{class:"mb-1.5 text-xs text-faint",children:s.note}):null,n(Re,{vars:s.vars})]},s.title))]}):null]}):n(w,{children:[n(ue,{children:"No matches"}),n("p",{class:"mt-8 text-muted",children:['Nothing matches "',l.query,'". Try a shorter word, or press Esc to go back.']})]})}function hn(){if(l.query.trim())return n(cr,{});if(l.page==="get-running")return n(pn,{});if(l.page==="environment")return n(ir,{});let e=T(l.page);return e?n(sr,{sec:e}):n(pn,{})}function ur(){let e=l.path.split(/[\\/]/).pop()||"10xgraph.json",t=l.path.slice(0,l.path.length-e.length),r=null;return l.schema?l.exists?R()?r=n(Me,{class:"bg-warn-soft text-warn",children:"Unsaved"}):r=n(Me,{class:"bg-ok-soft text-ok",children:"Saved"}):r=n(Me,{class:"bg-brand-soft text-brand",children:"New"}):r=null,n("div",{class:"flex min-w-0 flex-1 items-center gap-2",title:l.path||void 0,children:[n(h,{name:"file",class:"size-4 text-faint"}),n("span",{class:"truncate font-mono text-[13px] font-medium text-ink",children:e}),r,t?n("span",{class:"hidden min-w-0 truncate font-mono text-xs text-faint lg:block",dir:"rtl",children:n("bdi",{children:t})}):null]})}function _n(){let e=!!l.schema&&!l.fatal,t=l.sideOpen?"Hide sidebar":"Show sidebar";return n("header",{class:"sticky top-0 z-40 flex h-14 items-center gap-2 border-b border-line bg-surface px-3 sm:gap-3 sm:px-4",children:[n("button",{type:"button",class:"icon-btn","aria-controls":"side","aria-expanded":l.sideOpen?"true":"false","aria-label":t,title:`${t} (Ctrl/Cmd+B)`,onClick:()=>K(!l.sideOpen),children:n(h,{name:"panel",class:"size-[18px]"})}),n("div",{class:"flex shrink-0 items-center gap-2.5",children:[n("svg",{viewBox:"0 0 48 48",class:"size-7 shrink-0 text-ink","aria-hidden":"true",children:[n("rect",{x:"4",y:"4",width:"40",height:"40",rx:"11",fill:"none",stroke:"currentColor","stroke-width":"4.4"}),n("path",{d:"M13 33L21 23L34 15L34 31L21 23",fill:"none",stroke:"currentColor","stroke-width":"3","stroke-linecap":"round","stroke-linejoin":"round"}),n("g",{fill:"currentColor",children:[n("circle",{cx:"21",cy:"23",r:"4"}),n("circle",{cx:"34",cy:"15",r:"4"}),n("circle",{cx:"34",cy:"31",r:"4"})]}),n("circle",{cx:"13",cy:"33",r:"4.6",class:"fill-brand-amber"})]}),n("div",{class:"hidden items-baseline gap-1.5 sm:flex",children:[n("span",{class:"text-[15px] font-semibold tracking-tight text-ink",children:"10xGraph"}),n("span",{class:"text-[15px] text-faint",children:"Config"})]})]}),n("span",{class:"hidden h-6 w-px shrink-0 bg-line md:block","aria-hidden":"true"}),n(ur,{}),n("button",{type:"button",class:"btn btn-ghost","aria-label":"View 10xgraph.json","aria-haspopup":"dialog",disabled:!e,onClick:()=>en(),children:[n(h,{name:"code",class:"size-[18px]"}),n("span",{class:"hidden md:inline",children:"View 10xgraph.json"})]}),n("button",{type:"button",class:"btn btn-primary",onClick:Ae,disabled:!e||!!l.busy||l.exists&&!R()&&!l.parseError,children:[l.busy==="save"?n(h,{name:"spinner",class:"size-4 animate-spin"}):null,l.exists?"Save":"Create file",n("span",{class:"kbd hidden lg:inline-flex",children:"Ctrl S"})]})]})}function gn(){return l.fatal?n("div",{class:"panel mt-10",children:n(on,{icon:"xCircle",tint:"text-danger",title:l.fatal.title,text:l.fatal.text})}):l.schema?n(w,{children:[l.parseError?n("div",{class:"mb-6",children:n(ce,{kind:"warn",title:"The existing file is not valid JSON",lines:[`${l.parseError}. The form starts from defaults; saving replaces the file and keeps the old one as 10xgraph.json.bak.`]})}):null,n(hn,{})]}):n("div",{class:"animate-pulse space-y-4 pt-2","aria-label":"Loading",children:[n("div",{class:"h-7 w-48 rounded-md bg-raised"}),n("div",{class:"h-4 w-80 max-w-full rounded-md bg-raised"}),[0,1,2].map(e=>n("div",{class:"h-16 rounded-xl bg-raised"},e))]})}function dr({id:e}){let t=!!T(e)&&Wt(T(e)),r=Bt(e);return!t&&!r?null:n("span",{class:"flex shrink-0 gap-1",children:[t?n("span",{class:"size-2 rounded-full bg-brand",title:"Changed from default"}):null,r?n("span",{class:`size-2 rounded-full ${r.level==="error"?"bg-danger":"bg-warn"}`,title:r.level==="error"?"Needs attention":"Has a warning"}):null]})}function pr({id:e}){let t=oe(e);if(!t)return null;let r=!l.query&&l.page===e;return n("a",{href:`#${e}`,"aria-current":r?"page":void 0,onClick:o=>{o.preventDefault(),O(e)},ref:o=>{o&&r&&window.innerWidth<1024&&o.scrollIntoView({block:"nearest",inline:"nearest"})},class:`focus-ring flex min-h-10 items-center gap-2.5 rounded-lg px-3 font-medium transition-colors duration-150 max-lg:min-h-11 max-lg:shrink-0 max-lg:whitespace-nowrap ${r?"bg-brand-soft text-ink":"text-muted hover:bg-raised hover:text-ink"}`,children:[n(h,{name:W[e]||"box",class:`size-[18px] ${r?"text-brand":""}`}),n("span",{class:"min-w-0 flex-1 truncate",children:t.title}),n(dr,{id:e})]})}function xn(){return l.sideRender?n("aside",{id:"side","aria-label":"Settings navigation",class:`contents lg:sticky lg:top-14 lg:block lg:h-[calc(100dvh-3.5rem)] lg:overflow-y-auto lg:border-r lg:border-line lg:bg-surface lg:px-3 lg:pt-4 lg:pb-16 lg:transition-[transform,opacity] lg:duration-200 lg:ease-out ${l.sideShown?"lg:translate-x-0 lg:opacity-100":"lg:-translate-x-6 lg:opacity-0"}`,inert:!l.sideOpen,children:[n("div",{role:"search",class:"relative mx-1 mb-3 max-lg:mx-4 max-lg:mt-3",children:[n(h,{name:"search",class:"pointer-events-none absolute top-1/2 left-3 size-4 -translate-y-1/2 text-faint"}),n("label",{for:"q",class:"sr-only",children:"Search settings"}),n("input",{id:"q",type:"search",value:l.query,autocomplete:"off",spellcheck:!1,placeholder:"Search settings ( / )",class:"input pl-10",onInput:e=>Ie(e.currentTarget.value)})]}),n("nav",{"aria-label":"Pages",class:"max-lg:sticky max-lg:top-14 max-lg:z-20 max-lg:mt-3 max-lg:flex max-lg:gap-0.5 max-lg:overflow-x-auto max-lg:border-b max-lg:border-line max-lg:bg-surface max-lg:px-3 max-lg:py-1.5",children:l.nav.map(e=>n("div",{class:"max-lg:contents",children:[e.label?n("div",{class:"mt-4 mb-1 px-3 text-[11px] font-semibold tracking-wider text-faint uppercase max-lg:hidden",children:e.label}):null,e.items.map(t=>n(pr,{id:t},t))]},e.label||"top"))})]}):null}var fr={success:["checkCircle","text-ok"],warning:["alert","text-warn"],error:["xCircle","text-danger"]};function mr(){return n("div",{class:"pointer-events-none fixed right-4 bottom-4 left-4 z-50 flex flex-col items-end gap-2 sm:left-auto","aria-live":"polite",children:l.toasts.map(e=>{let[t,r]=fr[e.kind];return n("div",{role:e.kind==="error"?"alert":"status",class:"pointer-events-auto flex w-full max-w-sm items-start gap-3 rounded-xl border border-line bg-surface px-4 py-3 text-sm shadow-pop",children:[n(h,{name:t,class:`mt-0.5 size-4 ${r}`}),n("p",{class:"min-w-0 flex-1 leading-5 text-body",children:e.message}),e.action?n("button",{type:"button",class:"shrink-0 font-medium text-brand hover:underline",onClick:()=>{Pe(e.id),e.action.run()},children:e.action.label}):null,n("button",{type:"button",class:"-mr-1 shrink-0 cursor-pointer rounded text-faint hover:text-ink","aria-label":"Dismiss",onClick:()=>Pe(e.id),children:n(h,{name:"x"})})]},e.id)})})}var hr=()=>/^(INPUT|TEXTAREA|SELECT)$/.test(document.activeElement?.tagName)||document.activeElement?.isContentEditable;function _r(){let[,e]=Je(r=>r+1,0);M(()=>{Ft(e);try{localStorage.getItem("tg-sidebar")==="hidden"&&K(!1,{instant:!0})}catch{}let r=a=>{let i=a.ctrlKey||a.metaKey,d=a.key.toLowerCase();i&&d==="s"?(a.preventDefault(),Ae()):i&&d==="b"?(a.preventDefault(),K(!l.sideOpen)):i&&d==="k"?(a.preventDefault(),tt()):a.key==="/"&&!i&&!a.altKey&&!hr()&&!l.drawerOpen?(a.preventDefault(),tt()):a.key==="Escape"&&l.query&&!l.drawerOpen&&Ie("")},o=a=>{l.schema&&R()&&a.preventDefault()},s=()=>Ee();return document.addEventListener("keydown",r),window.addEventListener("beforeunload",o),window.addEventListener("hashchange",s),Zt(),()=>{document.removeEventListener("keydown",r),window.removeEventListener("beforeunload",o),window.removeEventListener("hashchange",s)}},[]),M(()=>{if(!l.schema||l.validated===I())return;let r=setTimeout(Qt,400);return()=>clearTimeout(r)});let t=l.sideRender;return n(w,{children:[n("a",{href:"#main",class:"sr-only z-50 rounded-md bg-brand-solid px-3 py-2 text-white focus:not-sr-only focus:fixed focus:top-3 focus:left-3",children:"Skip to content"}),n(_n,{}),n("div",{class:t?"lg:grid lg:grid-cols-[260px_minmax(0,1fr)] lg:items-start":"",children:[n(xn,{}),n("main",{id:"main",tabindex:-1,class:"min-w-0 px-4 pt-6 pb-24 focus:outline-none sm:px-8 sm:pt-8",children:n("div",{class:"mx-auto w-full max-w-[760px]",children:n(gn,{})})})]}),n(an,{}),n("div",{class:"sr-only",role:"status","aria-live":"polite",children:l.copied?"Copied":""}),n(mr,{})]})}St(n(_r,{}),document.getElementById("app")); diff --git a/tenxgraph_api/cli/config_editor/validation.py b/tenxgraph_api/cli/config_editor/validation.py index 008258f4..5caa4854 100644 --- a/tenxgraph_api/cli/config_editor/validation.py +++ b/tenxgraph_api/cli/config_editor/validation.py @@ -302,6 +302,13 @@ def _check_observability(config: dict, out: _Collector) -> None: if not isinstance(enabled, bool): out.error(f"observability.{backend}.enabled", "enabled must be true or false.") any_enabled = any_enabled or enabled is True + console = block.get("console") + if backend == "logfire" and console is not None and console is not False: + out.error( + "observability.logfire.console", + "console must be false or left out. Logfire takes False or a ConsoleOptions " + "object, which JSON cannot express.", + ) if not any_enabled: out.warning( "observability", "Neither Logfire nor LangSmith is enabled, so nothing is traced." diff --git a/tests/unit_tests/test_config_editor.py b/tests/unit_tests/test_config_editor.py index d92cac29..ede03b37 100644 --- a/tests/unit_tests/test_config_editor.py +++ b/tests/unit_tests/test_config_editor.py @@ -4,6 +4,7 @@ import http.client import json +import re from pathlib import Path import pytest @@ -47,6 +48,119 @@ def test_schema_scopes_match_api_scopes() -> None: assert set(AUTHORIZATION_SCOPES) == set(all_scopes()) +def _fields() -> dict[str, dict]: + return { + ".".join(field["path"]): field + for section in build_schema()["sections"] + for field in section["fields"] + } + + +def _env_entries() -> list[dict]: + schema = build_schema() + entries = [e for section in schema["sections"] for e in section.get("env", [])] + entries += [v for group in schema["environment"] for v in group["vars"]] + return entries + + +def test_env_entries_have_the_documented_shape() -> None: + schema = build_schema() + assert schema["environment"] + for group in schema["environment"]: + assert set(group) <= {"title", "vars", "note"} + assert {"title", "vars"} <= set(group) + assert group["title"] and group["vars"] + if "note" in group: + assert isinstance(group["note"], str) and group["note"].strip() + assert "\u2014" not in group["note"] + snowflake = next(g for g in schema["environment"] if "Snowflake" in g["title"]) + assert snowflake["note"] + entries = _env_entries() + assert entries + for entry in entries: + assert set(entry) == {"name", "purpose"} + assert re.fullmatch(r"[A-Z][A-Z0-9_]*", entry["name"]) + assert entry["purpose"].strip() + assert "\u2014" not in entry["purpose"] + + +def test_every_env_name_exists_in_the_source_tree() -> None: + root = Path(main_mod.__file__).resolve().parents[1] + sources = [ + p for p in root.rglob("*.py") if p.name != "schema.py" or "config_editor" not in str(p) + ] + core = Path(__file__).resolve().parents[3] / "agentflow" / "tenxgraph" + if core.is_dir(): + sources += list(core.rglob("*.py")) + text = "\n".join(p.read_text(encoding="utf-8", errors="ignore") for p in sources) + missing = sorted({e["name"] for e in _env_entries() if e["name"] not in text}) + assert not missing + + +def test_tri_state_fields_have_no_default() -> None: + fields = _fields() + for path in ( + "graph_introspection", + "websocket.enabled", + "observability.logfire.console", + ): + assert fields[path]["unset_label"] + assert "default" not in fields[path] + + +def test_logfire_console_does_not_offer_true() -> None: + assert _fields()["observability.logfire.console"]["allow_true"] is False + + +def test_websocket_seed_does_not_force_enabled() -> None: + section = next(s for s in build_schema()["sections"] if s["id"] == "websocket") + assert "enabled" not in section["default"] + assert not validate_config({"agent": "a:b", "websocket": section["default"]}) + + +def test_number_field_extensions() -> None: + fields = _fields() + timeout = fields["runs.timeout"] + assert timeout["exclusive_min"] == 0 + assert timeout["null_label"] == "No limit" + assert "min" not in timeout + for path in ( + "runs.max_concurrent_per_user", + "websocket.max_connections", + "websocket.max_connections_per_user", + ): + assert fields[path]["unlimited_value"] == 0 + + +def test_rate_limit_seed_does_not_claim_a_server_default_for_excluded_paths() -> None: + section = next(s for s in build_schema()["sections"] if s["id"] == "rate_limit") + assert "exclude_paths" not in section["default"] + assert _fields()["rate_limit.exclude_paths"]["label"] == "Extra paths to skip" + + +def test_unlimited_values_are_accepted_by_the_server() -> None: + for value in (0, None): + config = { + "agent": "a:b", + "runs": {"max_concurrent_per_user": value}, + "websocket": {"max_connections": value, "max_connections_per_user": value}, + } + assert not [i for i in validate_config(config) if i["level"] == "error"] + + +@pytest.mark.parametrize(("timeout", "ok"), [(0, False), (-1, False), (1.5, True), (None, True)]) +def test_runs_timeout_validation(timeout: object, ok: bool) -> None: + issues = validate_config({"agent": "a:b", "runs": {"timeout": timeout}}) + assert (not _levels(issues, "runs")) is ok + + +@pytest.mark.parametrize(("console", "ok"), [(False, True), (None, True), (True, False)]) +def test_logfire_console_only_accepts_false(console: object, ok: bool) -> None: + config = {"agent": "a:b", "observability": {"logfire": {"enabled": True, "console": console}}} + issues = validate_config(config) + assert (not _levels(issues, "observability.logfire.console")) is ok + + def test_build_schema_returns_independent_copies() -> None: first = build_schema() first["sections"][0]["title"] = "changed" From 7b4cdf90b417d35013f4076374890985d13b8dee Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Wed, 28 Oct 2026 15:43:51 +0600 Subject: [PATCH 02/16] feat: change rate limit backend to memory and clean up configuration --- 10xgraph.json | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/10xgraph.json b/10xgraph.json index 7c61ba86..bad6e381 100644 --- a/10xgraph.json +++ b/10xgraph.json @@ -6,29 +6,11 @@ "auth": null, "rate_limit": { "enabled": true, - "backend": "redis", + "backend": "memory", "requests": 100, "window": 60, "by": "ip", "trusted_proxy_headers": false, - "exclude_paths": [ - "/health", - "/docs", - "/redoc", - "/openapi.json" - ] - }, - "test": { - "path": "tests", - "coverage": true, - "coverage_threshold": 60 - }, - "runs": { - "thread_ids": "server", - "max_concurrent_per_user": 10 - }, - "ag_ui": { - "enabled": false, - "allow_client_tools": false + "exclude_paths": ["/health", "/docs", "/redoc", "/openapi.json"] } -} +} \ No newline at end of file From 3160bdb402fb2b3be88434dcbd315826557da07a Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Wed, 28 Oct 2026 21:30:38 +0600 Subject: [PATCH 03/16] feat: run trace, thread paging, file list, live options; remove eval endpoints - Run trace: move trace reconstruction to utils/run_trace.py and build a span tree (run > node visits > llm and tool calls) with inputs, outputs, status, tokens and timings; observability returns enabled/reason and notes. Still dev only: the store is not created with MODE=production. - Streams: rebuild a missing pause chunk from the checkpoint for every pause kind; continue interrupt_before/after pauses with an empty resume. - Threads and messages: has_more with limit/offset. - Media: GET /v1/files lists the caller's files. - Graph details: realtime_options for /v1/graph/live clients. - Remove /v1/evals/runs endpoints; `10xgraph eval` writes a JSON report only (no HTML, --open removed) for the playground to open. --- CHANGELOG.md | 5 + CLAUDE.md | 2 +- README.md | 2 +- tenxgraph_api/cli/commands/eval.py | 16 +- tenxgraph_api/cli/main.py | 8 +- .../prod/evals/user_simulator_eval.py | 2 +- .../10xgraph/references/cli-commands.md | 2 +- .../skills/10xgraph/references/evaluation.md | 5 +- .../src/app/core/auth/route_guard.py | 3 - tenxgraph_api/src/app/main.py | 2 - .../schemas/checkpointer_schemas.py | 6 + .../services/checkpointer_service.py | 31 +- .../src/app/routers/evals/__init__.py | 6 - tenxgraph_api/src/app/routers/evals/router.py | 38 -- .../app/routers/evals/services/__init__.py | 1 - .../evals/services/eval_report_service.py | 354 ------------ .../routers/graph/schemas/graph_schemas.py | 56 +- .../routers/graph/services/graph_service.py | 361 ++++++------- .../src/app/routers/media/__init__.py | 44 +- tenxgraph_api/src/app/routers/media/router.py | 40 +- .../src/app/routers/media/schemas.py | 18 + tenxgraph_api/src/app/routers/setup_router.py | 9 +- tenxgraph_api/src/app/utils/run_trace.py | 511 ++++++++++++++++++ tests/cli/test_cli_main.py | 2 - tests/cli/test_eval_command.py | 7 +- tests/unit_tests/test_eval_report_service.py | 290 ---------- tests/unit_tests/test_mode_validation.py | 2 +- tests/unit_tests/test_production_surface.py | 9 +- .../test_resume_binding_and_limits.py | 1 + tests/unit_tests/test_run_trace.py | 157 ++++++ tests/unit_tests/test_static_pause_resume.py | 66 +++ tests/unit_tests/test_stream_pause_chunk.py | 161 ++++++ .../test_threads_files_live_contract.py | 180 ++++++ tests/unit_tests/test_websocket_toggle.py | 10 +- 34 files changed, 1451 insertions(+), 956 deletions(-) delete mode 100644 tenxgraph_api/src/app/routers/evals/__init__.py delete mode 100644 tenxgraph_api/src/app/routers/evals/router.py delete mode 100644 tenxgraph_api/src/app/routers/evals/services/__init__.py delete mode 100644 tenxgraph_api/src/app/routers/evals/services/eval_report_service.py create mode 100644 tenxgraph_api/src/app/utils/run_trace.py delete mode 100644 tests/unit_tests/test_eval_report_service.py create mode 100644 tests/unit_tests/test_run_trace.py create mode 100644 tests/unit_tests/test_static_pause_resume.py create mode 100644 tests/unit_tests/test_stream_pause_chunk.py create mode 100644 tests/unit_tests/test_threads_files_live_contract.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 532e8310..973dcd12 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,11 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Removed + +- Removed the `/v1/evals/runs` endpoints; eval reports are JSON files opened in the playground. + `10xgraph eval` now writes only the JSON report and no longer has `--open`. + ### Changed - **`10xgraph config` editor redesign.** A sidebar layout with a "Get running" page first, a diff --git a/CLAUDE.md b/CLAUDE.md index 7f695f06..7006a320 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ Public exports from the package root (`from tenxgraph_api import ...`): `BaseAut | `10xgraph dev` | The only server command: local Uvicorn dev server (reload on) via `commands/api.py::APICommand`, opens the hosted playground when ready. `api` and `play` were removed (use `dev --no-open` / `dev`); production runs Gunicorn in the `10xgraph build` image | `--config/-c` (default `10xgraph.json`), `--host/-H`, `--port/-p` (8000), `--reload/--no-reload`, `--open/--no-open`, `-v/-q` | | `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | | `10xgraph build` | Generate a `Dockerfile` (and optionally `docker-compose.yml` / `k8s.yaml`). The builder stage strips CLI assets, runs `tenxgraph_api/_slim_image.py` (uninstalls typer/rich/questionary and what only they need, unless another package needs them) and removes pip, so the image carries no CLI code paths | `--output/-o`, `--force/-f`, `--python-version` (3.13), `--port`, `--docker-compose/--no-docker-compose`, `--k8s/--no-k8s`, `--service-name` | -| `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes HTML+JSON to `eval_reports/` | `--output/-o`, `--no-report`, `--threshold/-t`, `--open`, `--parallel/-p`, `--max-concurrency/-c` | +| `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes a JSON report to `eval_reports/` | `--output/-o`, `--no-report`, `--threshold/-t`, `--parallel/-p`, `--max-concurrency/-c` | | `10xgraph test` | Run project tests via pytest (args after `--` forwarded verbatim) | `--coverage/-C`, `--html`, `-k`, path arg | | `10xgraph skills` | Install the bundled 10xGraph skill (Agent Skills spec) for Codex/Claude/GitHub, or validate skills | `--agent/-a`, `--path/-p`, `--force/-f`, `--all`, `--list/-l`, `--validate PATH` | | `10xgraph version` | Show CLI + core framework version | both resolve from installed distribution metadata | diff --git a/README.md b/README.md index cc72f660..66b3cfdd 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Generate production Docker files. ### `10xgraph eval` / `10xgraph test` -Run agent evaluations (discovers `*_eval.py` / `eval_*.py`, writes HTML + JSON to `eval_reports/`) and project tests (pytest). +Run agent evaluations (discovers `*_eval.py` / `eval_*.py`, writes a JSON report to `eval_reports/`) and project tests (pytest). ```bash 10xgraph eval --parallel --threshold 0.8 diff --git a/tenxgraph_api/cli/commands/eval.py b/tenxgraph_api/cli/commands/eval.py index b1f9ff3b..d86502e6 100644 --- a/tenxgraph_api/cli/commands/eval.py +++ b/tenxgraph_api/cli/commands/eval.py @@ -1,4 +1,4 @@ -"""Eval command — discover and run 10xGraph evaluations, always generating reports.""" +"""Eval command — discover and run 10xGraph evaluations, always writing a JSON report.""" from __future__ import annotations @@ -8,7 +8,6 @@ import inspect import sys import typing -import webbrowser from collections import defaultdict from dataclasses import dataclass from pathlib import Path @@ -83,7 +82,7 @@ def _reset_inject_proxy(service_type: type) -> None: class EvalCommand(BaseCommand): - """Discover and run agent evaluations; always write HTML + JSON reports.""" + """Discover and run agent evaluations; always write a JSON report.""" # ------------------------------------------------------------------ # Discovery @@ -724,7 +723,6 @@ def execute( # noqa: PLR0912, PLR0915 output_dir: str = "eval_reports", no_report: bool = False, threshold: float | None = None, - open_report: bool = False, parallel: bool = False, max_concurrency: int = 4, verbose: bool = False, @@ -872,7 +870,7 @@ def execute( # noqa: PLR0912, PLR0915 manager = ReporterManager( ReporterConfig( output_dir=output_dir, - html=True, + html=False, json_report=True, console=False, timestamp_files=True, @@ -880,17 +878,15 @@ def execute( # noqa: PLR0912, PLR0915 ) report_result = manager.run_all(merged) - if report_result.html_path: - self.output.success(f"HTML report: {report_result.html_path}") if report_result.json_path: self.output.info(f"JSON report: {report_result.json_path}", emoji=False) + self.output.info( + "Open it in the playground: Evals > Open report file.", emoji=False + ) if report_result.has_errors: for name, err in report_result.errors: self.output.warning(f"Reporter error [{name}]: {err}") - if open_report and report_result.html_path: - webbrowser.open(Path(report_result.html_path).resolve().as_uri()) - summary = merged.summary self.output.info( f"Results: {summary.passed_cases}/{summary.total_cases} passed " diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index acc343ab..f366a004 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -754,11 +754,6 @@ def eval_cmd( "-t", help="Fail if overall pass rate is below this value (0.0-1.0)", ), - open_report: bool = typer.Option( - False, - "--open", - help="Open the HTML report in the default browser after the run", - ), parallel: bool = typer.Option( False, "--parallel", @@ -778,7 +773,7 @@ def eval_cmd( Discovers *_eval.py / eval_*.py files in the target directory (default: evals/). Collects all cases from all files into a flat pool, then runs them under a single - event loop throttled by --max-concurrency. Always generates HTML + JSON reports + event loop throttled by --max-concurrency. Always writes a JSON report in eval_reports/ unless --no-report is set. Each eval file must expose one of: @@ -795,7 +790,6 @@ def eval_cmd( output_dir=output_dir, no_report=no_report, threshold=threshold, - open_report=open_report, parallel=parallel, max_concurrency=max_concurrency, verbose=verbose, diff --git a/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py b/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py index 9487f608..3502cd90 100644 --- a/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +++ b/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py @@ -4,7 +4,7 @@ The CLI detects get_scenarios() and handles everything: - runs each scenario through UserSimulator - scores goal achievement with an LLM judge - - produces the same HTML + JSON report as regular eval cases + - produces the same JSON report as regular eval cases You only need to define your scenarios. No asyncio, no EvalSetBuilder, no boilerplate. diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md index 57bed3b7..dc5f1f66 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md @@ -52,7 +52,7 @@ agentflow = tenxgraph_api.cli.main:legacy_main # deprecated alias, removed in `10xgraph eval [TARGET]` - Runs agent evaluations: discovers `*_eval.py` and `eval_*.py` files in `TARGET` (default `evals/`) and runs their cases. -- Options include `--output/-o` (default `eval_reports`), `--no-report`, `--threshold/-t` (fail below this pass rate), `--open`, `--parallel/-p`, `--max-concurrency/-c` (default 4), `--verbose/-v`, and `--quiet/-q`. +- Options include `--output/-o` (default `eval_reports`), `--no-report`, `--threshold/-t` (fail below this pass rate), `--parallel/-p`, `--max-concurrency/-c` (default 4), `--verbose/-v`, and `--quiet/-q`. `10xgraph test [PATH]` diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md index d9a25aac..2581eb52 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md @@ -320,13 +320,12 @@ JUnit XML — for CI test summary tabs (GitHub Actions, Jenkins). ## 10xgraph eval CLI -Auto-discovers `*_eval.py` / `eval_*.py` in `evals/` (or configured directory). Each file must expose `get_eval_set()`, optionally `get_eval_config()` or `EVAL_CONFIG`, or a full `run()` function. +Auto-discovers `*_eval.py` / `eval_*.py` in `evals/` (or configured directory). Each file must expose `get_eval_set()`, optionally `get_eval_config()` or `EVAL_CONFIG`, or a full `run()` function. The CLI writes a JSON report only (no HTML, no browser); open it in the playground with Evals > Open report file. ```bash -10xgraph eval # scan evals/, write to eval_reports/ +10xgraph eval # scan evals/, write a JSON report to eval_reports/ 10xgraph eval evals/my_eval.py # single file 10xgraph eval evals/regression/ # subdirectory -10xgraph eval --open # open HTML report in browser after run 10xgraph eval --threshold 0.8 # exit non-zero if pass rate < 0.8 10xgraph eval --output ci/reports # custom output directory 10xgraph eval --no-report # console only, no files diff --git a/tenxgraph_api/src/app/core/auth/route_guard.py b/tenxgraph_api/src/app/core/auth/route_guard.py index 167d11e9..9c96afbb 100644 --- a/tenxgraph_api/src/app/core/auth/route_guard.py +++ b/tenxgraph_api/src/app/core/auth/route_guard.py @@ -23,13 +23,10 @@ logger = logging.getLogger("tenxgraph_api.route_guard") # Paths that are intentionally public (no authorization). Keep this list tiny and explicit. -# Evals is a dev-only report viewer over local files (no user data, not a production surface). DEFAULT_PUBLIC_PATHS = frozenset( { "/ping", "/health/ready", - "/v1/evals/runs", - "/v1/evals/runs/{run_id}", } ) diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index c8dc6824..0b4ccb41 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -189,11 +189,9 @@ async def _drain_background_tasks(timeout: float = SHUTDOWN_DRAIN_TIMEOUT_SECOND graph_config.media # noqa: B018 # init routes -# The evals viewer reads local report files and has no auth, so it is a development tool only. # The AG-UI endpoint is opt-in through 10xgraph.json; the WebSocket routes can be turned off there. init_routes( app, - evals=settings.MODE != "production", ag_ui=graph_config.ag_ui.enabled, websocket=graph_config.websocket.enabled, ) diff --git a/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py b/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py index a61956c4..943c7cb9 100644 --- a/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +++ b/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py @@ -163,6 +163,9 @@ class MessagesListResponseSchema(BaseModel): """Response schema for message list operations.""" messages: list[Message] | None = Field(None, description="List of messages") + has_more: bool | None = Field(None, description="Whether another page follows") + offset: int | None = Field(None, description="Offset applied to this page") + limit: int | None = Field(None, description="Page size applied to this page") class ThreadResponseSchema(BaseModel): @@ -175,3 +178,6 @@ class ThreadsListResponseSchema(BaseModel): """Response schema for thread list operations.""" threads: list[dict[str, Any]] | None = Field(None, description="List of threads") + has_more: bool | None = Field(None, description="Whether another page follows") + offset: int | None = Field(None, description="Offset applied to this page") + limit: int | None = Field(None, description="Page size applied to this page") diff --git a/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py b/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py index 8263b3f8..49ad9ce6 100644 --- a/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +++ b/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py @@ -1,3 +1,4 @@ +from collections.abc import Awaitable, Callable from typing import Any from fastapi import HTTPException @@ -48,6 +49,19 @@ def __init__(self) -> None: _CLEANUP_PAGE = 500 +async def _has_more( + count: int, offset: int | None, limit: int | None, probe: Callable[[int], Awaitable[list]] +) -> bool: + """Whether a row exists past this page. + + No checkpointer exposes a total, so a full page triggers a one-row read at the next + offset. A short page (or an unlimited read) is the end. + """ + if not limit or count < limit: + return False + return bool(await probe((offset or 0) + limit)) + + @singleton class CheckpointerService: @inject @@ -363,7 +377,12 @@ async def get_messages( ) -> MessagesListResponseSchema: cfg = self._config(config, user) res = await self.checkpointer.alist_messages(cfg, search, offset, limit) - return MessagesListResponseSchema(messages=res) + has_more = await _has_more( + len(res), offset, limit, lambda o: self.checkpointer.alist_messages(cfg, search, o, 1) + ) + return MessagesListResponseSchema( + messages=res, has_more=has_more, offset=offset or 0, limit=limit + ) async def delete_message( self, @@ -417,7 +436,15 @@ async def list_threads( ) -> ThreadsListResponseSchema: cfg = self._config({}, user) res = await self.checkpointer.alist_threads(cfg, search, offset, limit) - return ThreadsListResponseSchema(threads=[t.model_dump() for t in res]) + has_more = await _has_more( + len(res), offset, limit, lambda o: self.checkpointer.alist_threads(cfg, search, o, 1) + ) + return ThreadsListResponseSchema( + threads=[t.model_dump() for t in res], + has_more=has_more, + offset=offset or 0, + limit=limit, + ) async def delete_thread( self, diff --git a/tenxgraph_api/src/app/routers/evals/__init__.py b/tenxgraph_api/src/app/routers/evals/__init__.py deleted file mode 100644 index 71eefc19..00000000 --- a/tenxgraph_api/src/app/routers/evals/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""Evals router package.""" - -from .router import router - - -__all__ = ["router"] diff --git a/tenxgraph_api/src/app/routers/evals/router.py b/tenxgraph_api/src/app/routers/evals/router.py deleted file mode 100644 index 37b12266..00000000 --- a/tenxgraph_api/src/app/routers/evals/router.py +++ /dev/null @@ -1,38 +0,0 @@ -""" -Evals router — serves eval-report data over HTTP so the playground's Evals -inspector can exercise a real API. Reports are read from `10xgraph eval`'s -JSON output (`eval_reports/*.json`); see EvalReportService for how the -`EvalReport` schema (no run-number, no dollar cost, no built-in regression) -is adapted into the run/detail shape the UI expects. - -Endpoints: - GET /v1/evals/runs -> list of runs (summary rows) - GET /v1/evals/runs/{run_id} -> full drilldown for one run -""" - -from __future__ import annotations - -from fastapi import APIRouter, Request - -from tenxgraph_api.src.app.routers.evals.services.eval_report_service import EvalReportService -from tenxgraph_api.src.app.utils.response_helper import success_response - - -# Evals is a dev-only report viewer over local `eval_reports/*.json` files -- it exposes no -# user data and is not a production surface, so it carries no authorization (the paths are -# on the route-guard's public allowlist). See core/auth/route_guard.py. -router = APIRouter(tags=["evals"]) - - -@router.get("/v1/evals/runs", summary="List eval runs") -async def list_eval_runs(request: Request): - """List all eval runs (summary rows) found under eval_reports/.""" - service = EvalReportService() - return success_response({"runs": service.list_runs()}, request) - - -@router.get("/v1/evals/runs/{run_id}", summary="Get eval run detail") -async def get_eval_run(run_id: str, request: Request): - """Return the full drilldown for one eval run.""" - service = EvalReportService() - return success_response(service.get_run_detail(run_id), request) diff --git a/tenxgraph_api/src/app/routers/evals/services/__init__.py b/tenxgraph_api/src/app/routers/evals/services/__init__.py deleted file mode 100644 index ca540bc0..00000000 --- a/tenxgraph_api/src/app/routers/evals/services/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Evals services package.""" diff --git a/tenxgraph_api/src/app/routers/evals/services/eval_report_service.py b/tenxgraph_api/src/app/routers/evals/services/eval_report_service.py deleted file mode 100644 index 24a19395..00000000 --- a/tenxgraph_api/src/app/routers/evals/services/eval_report_service.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Reads real `10xgraph eval` reports for the evals API. - -Reports are written by `10xgraph eval` (tenxgraph_api/cli/commands/eval.py) via the core -`ReporterManager` / `JSONReporter` (tenxgraph/qa/evaluation/reporters/json.py) as -`eval_reports/*.json`, one file per run. That schema (`EvalReport` / `EvalCaseResult` / -`EvalSummary` in tenxgraph/qa/evaluation/eval_result.py) has no run-number, no dollar cost -(only token counts), no per-case "expected" value, and no built-in regression comparison -between runs — those are derived here so the response keeps the shape the playground's -Evals inspector already renders (see agentflow-playground/src/pages/evals/data.js). -""" - -from __future__ import annotations - -import json -import time -from pathlib import Path -from typing import Any - -from fastapi import HTTPException - - -DEFAULT_REPORTS_DIR = "eval_reports" -SECONDS_PER_MINUTE = 60 -MINUTES_PER_HOUR = 60 -HOURS_PER_DAY = 24 -TOKENS_PER_KILO = 1000 -REGRESSION_FLAT_EPSILON = 0.005 - - -def _format_ago(timestamp: float, now: float) -> str: - delta = max(0.0, now - timestamp) - if delta < SECONDS_PER_MINUTE: - return "just now" - minutes = delta / SECONDS_PER_MINUTE - if minutes < MINUTES_PER_HOUR: - return f"{int(minutes)}m ago" - hours = minutes / MINUTES_PER_HOUR - if hours < HOURS_PER_DAY: - return f"{int(hours)}h ago" - days = hours / HOURS_PER_DAY - return f"{int(days)}d ago" - - -def _format_duration(seconds: float) -> str: - return f"{seconds:.1f}s" - - -def _format_tokens(token_usage: dict[str, Any] | None) -> str: - total = (token_usage or {}).get("total_tokens", 0) - if total >= TOKENS_PER_KILO: - return f"{total / TOKENS_PER_KILO:.1f}k tok" - return f"{total} tok" - - -def _case_type(case: dict[str, Any]) -> str: - return "sim" if "turns" in (case.get("metadata") or {}) else "eval" - - -def _case_score(case: dict[str, Any]) -> float: - criteria = case.get("criterion_results") or [] - if not criteria: - return 1.0 if case.get("passed") else 0.0 - return sum(cr.get("score", 0.0) for cr in criteria) / len(criteria) - - -def _case_input(case: dict[str, Any]) -> str: - for message in case.get("messages") or []: - if message.get("role") != "user": - continue - content = message.get("content") - if isinstance(content, str) and content: - return content - if isinstance(content, list): - texts = [ - block.get("text", "") - for block in content - if isinstance(block, dict) and block.get("type") == "text" - ] - if texts: - return " ".join(texts) - return "—" - - -def _case_conversation(case: dict[str, Any]) -> list[dict[str, str]] | None: - if _case_type(case) != "sim": - return None - turns: list[dict[str, str]] = [] - for raw_line in (case.get("actual_response") or "").splitlines(): - stripped = raw_line.strip() - if not stripped: - continue - if stripped.startswith("USER:"): - turns.append({"role": "sim", "text": stripped[len("USER:") :].strip()}) - elif stripped.startswith("AGENT:"): - turns.append({"role": "agent", "text": stripped[len("AGENT:") :].strip()}) - else: - turns.append({"role": "agent", "text": stripped}) - return turns or None - - -def _case_rubric(case: dict[str, Any]) -> list[dict[str, Any]] | None: - criteria = case.get("criterion_results") or [] - if not criteria: - return None - rubric = [ - { - "key": cr.get("criterion", ""), - "value": round(cr.get("score", 0.0), 2), - "tone": "accent" if cr.get("passed") else "danger", - } - for cr in criteria - ] - rubric.append( - { - "key": "weighted score", - "value": round(_case_score(case), 2), - "tone": "accent" if case.get("passed") else "danger", - } - ) - return rubric - - -def _build_case_view(case: dict[str, Any]) -> dict[str, Any]: - return { - "id": case.get("eval_id", ""), - "name": case.get("name") or case.get("eval_id", ""), - "type": _case_type(case), - "score": round(_case_score(case), 2), - "status": "pass" if case.get("passed") else "fail", - "lat": _format_duration(case.get("duration_seconds", 0.0)), - "cost": _format_tokens(case.get("token_usage")), - "input": _case_input(case), - "expected": "—", - "actual": case.get("actual_response") or "—", - "rubric": _case_rubric(case), - "conversation": _case_conversation(case), - } - - -def _run_status(report: dict[str, Any]) -> str: - return "pass" if report.get("summary", {}).get("pass_rate", 0.0) >= 1.0 else "fail" - - -def _derive_threshold(report: dict[str, Any]) -> float: - thresholds = { - cr.get("threshold", 0.0) - for case in report.get("results", []) - for cr in case.get("criterion_results", []) - } - if not thresholds: - return 80.0 - return round(100 * sum(thresholds) / len(thresholds)) - - -def _build_stats(report: dict[str, Any]) -> list[dict[str, Any]]: - summary = report.get("summary", {}) - cases = report.get("results", []) - scores = [_case_score(c) for c in cases] - avg_score = sum(scores) / len(scores) if scores else 0.0 - return [ - {"label": "cases", "value": str(summary.get("total_cases", 0))}, - {"label": "passed", "value": str(summary.get("passed_cases", 0)), "tone": "ok"}, - { - "label": "failed", - "value": str(summary.get("failed_cases", 0) + summary.get("error_cases", 0)), - "tone": "bad", - }, - {"label": "avg score", "value": f"{avg_score:.2f}"}, - { - "label": "avg latency", - "value": _format_duration(summary.get("avg_duration_seconds", 0.0)), - }, - {"label": "total tokens", "value": _format_tokens(summary.get("total_token_usage"))}, - ] - - -def _build_run_row( - report: dict[str, Any], run_id: str, run_label: str, now: float -) -> dict[str, Any]: - summary = report.get("summary", {}) - return { - "id": run_id, - "name": report.get("eval_set_name") or report.get("eval_set_id", ""), - "run": run_label, - "rate": round(summary.get("pass_rate", 0.0) * 100, 1), - "status": _run_status(report), - "cases": summary.get("total_cases", 0), - "ago": _format_ago(report.get("timestamp", now), now), - } - - -def _build_regression( - current: dict[str, Any], - current_label: str, - previous: dict[str, Any], - previous_label: str, -) -> dict[str, Any]: - cur_cases = {c["eval_id"]: c for c in current.get("results", [])} - prev_cases = {c["eval_id"]: c for c in previous.get("results", [])} - - rows: list[dict[str, Any]] = [] - newly_failing = 0 - newly_passing = 0 - for eval_id, cur_case in cur_cases.items(): - prev_case = prev_cases.get(eval_id) - if prev_case is None: - continue - cur_score = _case_score(cur_case) - prev_score = _case_score(prev_case) - delta = cur_score - prev_score - - if prev_case["passed"] and not cur_case["passed"]: - newly_failing += 1 - flip = "pass → fail" - elif not prev_case["passed"] and cur_case["passed"]: - newly_passing += 1 - flip = "fail → pass" - else: - flip = f"{'pass' if cur_case['passed'] else 'fail'} (same)" - - if abs(delta) < REGRESSION_FLAT_EPSILON: - dir_, arrow = "flat", "—" - elif delta > 0: - dir_, arrow = "up", f"â–²{delta:.2f}" - else: - dir_, arrow = "down", f"â–¼{abs(delta):.2f}" - - rows.append( - { - "name": cur_case.get("name") or eval_id, - "delta": f"{prev_score:.2f} → {cur_score:.2f} {arrow}", - "dir": dir_, - "flip": flip, - "stay": cur_case["passed"] == prev_case["passed"], - } - ) - - cur_rate = current.get("summary", {}).get("pass_rate", 0.0) * 100 - prev_rate = previous.get("summary", {}).get("pass_rate", 0.0) * 100 - cur_scores = [_case_score(c) for c in current.get("results", [])] - prev_scores = [_case_score(c) for c in previous.get("results", [])] - avg_cur = sum(cur_scores) / len(cur_scores) if cur_scores else 0.0 - avg_prev = sum(prev_scores) / len(prev_scores) if prev_scores else 0.0 - score_drift = avg_cur - avg_prev - rate_drift = cur_rate - prev_rate - - return { - "note": { - "current": f"run {current_label} ({cur_rate:.1f}%)", - "prev": f"run {previous_label} ({prev_rate:.1f}%)", - "suite": "same suite", - }, - "summary": [ - { - "label": "pass-rate drift", - "value": f"{rate_drift:+.1f}pp", - "tone": "bad" if rate_drift < 0 else "ok", - }, - { - "label": "newly failing", - "value": str(newly_failing), - "tone": "bad" if newly_failing else "ok", - }, - {"label": "newly passing", "value": str(newly_passing), "tone": "ok"}, - { - "label": "avg score drift", - "value": f"{score_drift:+.2f}", - "tone": "warn" if score_drift < 0 else "ok", - }, - ], - "rows": rows, - } - - -class EvalReportService: - """Reads `eval_reports/*.json` and adapts them into the run/detail shape - the playground's Evals inspector expects.""" - - def __init__(self, reports_dir: str | Path | None = None) -> None: - base = Path(reports_dir) if reports_dir is not None else Path(DEFAULT_REPORTS_DIR) - self.reports_dir = base if base.is_absolute() else Path.cwd() / base - - def _load_reports(self) -> list[tuple[str, dict[str, Any]]]: - if not self.reports_dir.is_dir(): - return [] - reports = [] - for path in sorted(self.reports_dir.glob("*.json")): - try: - data = json.loads(path.read_text(encoding="utf-8")) - except (json.JSONDecodeError, OSError): - continue - reports.append((path.stem, data)) - reports.sort(key=lambda item: item[1].get("timestamp", 0.0)) - return reports - - def _run_labels(self) -> dict[str, tuple[int, dict[str, Any], str | None]]: - """run_id -> (run_number, report, previous_run_id), run_number and - previous_run_id computed per eval_set_id in chronological order.""" - counters: dict[str, int] = {} - previous_by_name: dict[str, str] = {} - labels: dict[str, tuple[int, dict[str, Any], str | None]] = {} - for run_id, report in self._load_reports(): - name = report.get("eval_set_id", "") - counters[name] = counters.get(name, 0) + 1 - labels[run_id] = (counters[name], report, previous_by_name.get(name)) - previous_by_name[name] = run_id - return labels - - def list_runs(self, now: float | None = None) -> list[dict[str, Any]]: - now = now if now is not None else time.time() - labels = self._run_labels() - rows = [ - (report.get("timestamp", 0.0), _build_run_row(report, run_id, f"#{run_number}", now)) - for run_id, (run_number, report, _prev) in labels.items() - ] - rows.sort(key=lambda item: item[0], reverse=True) - return [row for _, row in rows] - - def get_run_detail(self, run_id: str) -> dict[str, Any]: - labels = self._run_labels() - entry = labels.get(run_id) - if entry is None: - raise HTTPException(status_code=404, detail=f"Eval run '{run_id}' not found") - run_number, report, previous_run_id = entry - summary = report.get("summary", {}) - - regression = None - if previous_run_id is not None: - prev_entry = labels.get(previous_run_id) - if prev_entry is not None: - prev_number, prev_report, _ = prev_entry - regression = _build_regression( - report, f"#{run_number}", prev_report, f"#{prev_number}" - ) - - model = (report.get("config_used") or {}).get("model") - run_timestamp = time.gmtime(report.get("timestamp", 0.0)) - sub_parts = [report.get("eval_set_id", "")] - if model: - sub_parts.append(str(model)) - sub_parts.append(time.strftime("%Y-%m-%d %H:%M", run_timestamp)) - sub_parts.append(_format_duration(summary.get("total_duration_seconds", 0.0))) - - set_label = report.get("eval_set_name") or report.get("eval_set_id", "") - return { - "title": f"{set_label} · run #{run_number}", - "sub": " · ".join(sub_parts), - "rate": round(summary.get("pass_rate", 0.0) * 100, 1), - "status": _run_status(report), - "threshold": _derive_threshold(report), - "stats": _build_stats(report), - "cases": [_build_case_view(c) for c in report.get("results", [])], - "regression": regression, - } diff --git a/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py b/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py index e9ace05c..558a78ab 100644 --- a/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +++ b/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py @@ -152,6 +152,20 @@ class EdgeSchema(BaseModel): target: str = Field(..., description="Target node identifier") +class RealtimeOptionsSchema(BaseModel): + """Which /v1/graph/live init-frame options the server applies.""" + + client_overrides: list[str] = Field( + default_factory=list, + description="Init keys applied from the client (voice, modalities, vad, system_prompt)", + ) + models: list[str] = Field( + default_factory=list, description="Models a client may request; empty means none" + ) + model_selectable: bool = Field(False, description="Whether the init `model` is honored") + tools_tags: bool = Field(True, description="Whether the client may narrow the tool tags") + + class GraphInfoSchema(BaseModel): """Schema for graph metadata and configuration.""" @@ -161,6 +175,10 @@ class GraphInfoSchema(BaseModel): False, description="Whether the graph is a realtime/live agent (driven over /v1/graph/live)", ) + realtime_options: RealtimeOptionsSchema | None = Field( + None, + description="Live init options the server honors; null when the graph is not realtime", + ) checkpointer: bool = Field(..., description="Whether checkpointer is enabled") checkpointer_type: str | None = Field(None, description="Type of checkpointer if enabled") publisher: bool = Field(..., description="Whether publisher is enabled") @@ -220,7 +238,7 @@ class GraphToolsSchema(BaseModel): class ObsSpanSchema(BaseModel): - """A reconstructed span in the run trace.""" + """A reconstructed span in the run trace (root > node visit > llm / tool).""" id: str = Field(..., description="Span id (stable within the run)") name: str = Field(..., description="Display name, e.g. 'node: agent' or 'tool: get_weather'") @@ -228,9 +246,33 @@ class ObsSpanSchema(BaseModel): parent: str | None = Field(None, description="Parent span id") start_ms: float = Field(..., description="Start offset from run start, in ms") duration_ms: float = Field(..., description="Duration in ms") + status: Literal["ok", "error", "running"] = Field("ok", description="Span outcome") + error: str | None = Field(None, description="Error text when status is 'error'") + inputs: dict[str, Any] | None = Field( + None, + description=( + "What went in. root: user_message. node: incoming_message (approximate). " + "tool: args. llm: always null (the prompt is not recorded)." + ), + ) + outputs: dict[str, Any] | None = Field( + None, + description=( + "What came out. root: assistant_message. node: messages emitted in the visit. " + "llm: text and tool_calls. tool: result." + ), + ) + step: int | None = Field(None, description="Graph step of the visit (node spans)") + visit: int | None = Field( + None, description="1-based visit count of this node name (node spans)" + ) model: str | None = Field(None, description="LLM model (llm spans)") input_tokens: int | None = Field(None, description="Prompt tokens (llm spans)") output_tokens: int | None = Field(None, description="Completion tokens (llm spans)") + total_tokens: int | None = Field(None, description="Total tokens (llm spans)") + reasoning_tokens: int | None = Field(None, description="Reasoning tokens (llm spans)") + tool_name: str | None = Field(None, description="Tool name (tool spans)") + tool_call_id: str | None = Field(None, description="Tool call id (tool spans)") class ObsEventSchema(BaseModel): @@ -266,13 +308,23 @@ class ObsRunSchema(BaseModel): usage: ObsTokenUsageSchema = Field(default_factory=ObsTokenUsageSchema) llm_calls: int = 0 tool_calls: int = 0 - iterations: int = 0 + iterations: int = Field(0, description="Number of node visits") + error: str | None = Field(None, description="First error text seen in the run") + model: str | None = Field(None, description="First model seen in the run") + models: list[str] = Field(default_factory=list, description="Distinct models seen") + notes: list[str] = Field( + default_factory=list, description="Honest limits of this reconstruction" + ) class ObservabilitySchema(BaseModel): """Observability payload for a thread: available runs + the selected run.""" thread_id: str + enabled: bool = Field(True, description="False when run tracing is off on this server") + reason: Literal["disabled_in_production", "store_unavailable", "no_runs"] | None = Field( + None, description="Why there is no run: tracing off, store missing, or none captured yet" + ) run_count: int = 0 run_ids: list[str] = Field(default_factory=list) run: ObsRunSchema | None = Field(None, description="The requested/latest run, if any") diff --git a/tenxgraph_api/src/app/routers/graph/services/graph_service.py b/tenxgraph_api/src/app/routers/graph/services/graph_service.py index 83ebf7d0..4d6e5172 100644 --- a/tenxgraph_api/src/app/routers/graph/services/graph_service.py +++ b/tenxgraph_api/src/app/routers/graph/services/graph_service.py @@ -13,6 +13,7 @@ from tenxgraph.core.graph import CompiledGraph from tenxgraph.core.state import AgentState, Message, StreamChunk, StreamEvent from tenxgraph.storage.checkpointer import BaseCheckpointer +from tenxgraph.utils import ResponseGranularity from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.auth.request_config import ( @@ -39,10 +40,7 @@ GraphSchema, GraphToolsSchema, ObservabilitySchema, - ObsEventSchema, ObsRunSchema, - ObsSpanSchema, - ObsTokenUsageSchema, ToolNodeSchema, ToolSchema, ) @@ -55,6 +53,11 @@ ) from tenxgraph_api.src.app.routers.graph.thread_lock import acquire_thread_lock from tenxgraph_api.src.app.utils import DefaultThreadNameGenerator, ThreadNameGenerator +from tenxgraph_api.src.app.utils.run_trace import ( + event_summary, + reconstruct_run, + telemetry_record, +) from tenxgraph_api.src.app.utils.telemetry_store import TelemetryStore @@ -195,6 +198,22 @@ def generic_error_text(text: Any) -> Any: return _sanitize_error_message(text, code.group(1) if code else "GRAPH_STREAM_ERROR", True) +_PAUSE_KINDS = ("interrupt", "before", "after", "remote_tool") + + +def _is_pause_update(chunk: StreamChunk) -> bool: + data = chunk.data + return ( + chunk.event == StreamEvent.UPDATES + and isinstance(data, dict) + and ( + data.get("status") == "interrupted" + or "interrupt" in data + or data.get("pause") in _PAUSE_KINDS + ) + ) + + def sanitize_error_chunk(chunk: StreamChunk) -> StreamChunk: """In production, replace the exception text in an ``ERROR`` chunk with a generic message. @@ -482,59 +501,18 @@ def telemetry(self) -> TelemetryStore | None: return self._telemetry @staticmethod - def _telemetry_record(chunk: Any) -> dict[str, Any]: + def _telemetry_record(chunk: Any, ts: float | None = None) -> dict[str, Any]: """Flatten a StreamChunk into a compact record for the telemetry store.""" - md = getattr(chunk, "metadata", None) or {} - node = md.get("node") or md.get("current_node") or getattr(chunk, "node_name", "") or "" - - usages = None - message = getattr(chunk, "message", None) - if message is not None: - u = getattr(message, "usages", None) - if u is not None: - try: - usages = u.model_dump() if hasattr(u, "model_dump") else dict(u) - except Exception: - usages = None - - # Tool call / tool result hints for span reconstruction. - content_kinds: list[str] = [] - tool_names: list[str] = [] - if message is not None: - content = getattr(message, "content", None) - if isinstance(content, list): - for block in content: - btype = getattr(block, "type", None) or ( - block.get("type") if isinstance(block, dict) else None - ) - if btype: - content_kinds.append(btype) - if btype in ("tool_call", "tool_result"): - bname = getattr(block, "name", None) or ( - block.get("name") if isinstance(block, dict) else None - ) - if bname: - tool_names.append(bname) - - event = getattr(chunk, "event", None) - return { - "event": str(event.value) if hasattr(event, "value") else str(event or ""), - "node": node, - "timestamp": getattr(chunk, "timestamp", None), - "is_delta": bool(getattr(message, "delta", False)) if message else False, - "role": getattr(message, "role", None) if message else None, - "usages": usages, - "content_kinds": content_kinds, - "tool_names": tool_names, - "is_error": bool(getattr(chunk, "is_error", False)), - } + return telemetry_record(chunk, ts) - def _record_chunk(self, thread_id: str, run_id: str, chunk: Any) -> None: + def _record_chunk( + self, thread_id: str, run_id: str, chunk: Any, ts: float | None = None + ) -> None: store = self.telemetry if not store: return try: - store.record(thread_id, run_id, self._telemetry_record(chunk)) + store.record(thread_id, run_id, self._telemetry_record(chunk, ts)) except Exception as e: # - telemetry must never break a run logger.debug("Telemetry record failed: %s", e) @@ -745,6 +723,25 @@ async def _check_resume_target( f"interrupt '{pending.id}'" ) + async def _is_static_pause(self, config: dict[str, Any]) -> bool: + """Whether the thread is paused by ``interrupt_before`` / ``interrupt_after``. + + Such a pause has nothing to answer: core continues it from any input that carries no + ``resume`` key, so an empty resume must not reach core (it would fail with "Nothing to + resume"). A pause on an ``interrupt()`` or on a client-side tool call is not static. + """ + state = await self.checkpointer.aget_state(config) if self.checkpointer else None + if state is None or not state.is_interrupted(): + return False + try: + from tenxgraph.utils.interrupt import pending_interrupt + except ImportError: # core older than interrupt(): every pause is static + pending = None + else: + pending = pending_interrupt(state) + reason = getattr(state.execution_meta, "interrupt_reason", None) + return pending is None and reason != "remote_tool_call" + async def _prepare_input( self, graph_input: GraphInputSchema, @@ -794,7 +791,7 @@ async def _prepare_input( input_data: dict = { "messages": preprocessed, } - if graph_input.is_resume: + if graph_input.is_resume and not await self._is_static_pause(config): input_data["resume"] = graph_input.resume if graph_input.interrupt_id is not None: # Read by core 1.0+ (checks the id again and validates the answer); an older @@ -813,7 +810,7 @@ async def _prepare_input( }, ) - async def invoke_graph( # noqa: PLR0915 + async def invoke_graph( # noqa: PLR0912, PLR0915 self, graph_input: GraphInputSchema, user: dict[str, Any], @@ -871,7 +868,11 @@ async def invoke_graph( # noqa: PLR0915 # Record final messages into telemetry, then close the run. if self.telemetry: + inv_now = datetime.now().timestamp() for msg in messages: + msg_ts = getattr(msg, "timestamp", None) + if not isinstance(msg_ts, int | float) or not inv_started <= msg_ts <= inv_now: + msg_ts = inv_now self._record_chunk( inv_thread_id, inv_run_id, @@ -879,8 +880,9 @@ async def invoke_graph( # noqa: PLR0915 event=StreamEvent.MESSAGE, message=msg, metadata={"node": getattr(msg, "node", "") or ""}, - timestamp=datetime.now().timestamp(), + timestamp=msg_ts, ), + msg_ts, ) self.telemetry.finish_run( inv_thread_id, inv_run_id, datetime.now().timestamp(), "done" @@ -989,7 +991,7 @@ async def stream_graph( async for chunk in self.stream_chunks(graph_input, user): yield chunk.model_dump_json(serialize_as_any=True) + "\n" - async def stream_chunks( # noqa: PLR0915 + async def stream_chunks( # noqa: PLR0912, PLR0915 self, graph_input: GraphInputSchema, user: dict[str, Any], @@ -1045,6 +1047,7 @@ async def stream_chunks( # noqa: PLR0915 run_status = "done" # Stream the graph execution + saw_pause = False async for raw_chunk in self._astream_bounded( input_data, config, graph_input.response_granularity ): @@ -1052,7 +1055,9 @@ async def stream_chunks( # noqa: PLR0915 mt = chunk.metadata or {} mt.update(meta) chunk.metadata = mt - self._record_chunk(thread_id, run_id, chunk) + self._record_chunk(thread_id, run_id, chunk, datetime.now().timestamp()) + if _is_pause_update(chunk): + saw_pause = True if getattr(chunk, "is_error", False) or getattr(chunk, "event", None) in ( "error", StreamEvent.ERROR, @@ -1068,6 +1073,12 @@ async def stream_chunks( # noqa: PLR0915 ): messages_str.append(chunk.message.text()) + if not saw_pause and graph_input.response_granularity != ResponseGranularity.FULL: + pause = await self._missing_pause_chunk(config, meta) + if pause is not None: + self._record_chunk(thread_id, run_id, pause, datetime.now().timestamp()) + yield pause + logger.info("Graph streaming completed successfully") if self.telemetry: @@ -1109,6 +1120,74 @@ async def stream_chunks( # noqa: PLR0915 finally: thread_lock.release() + async def _missing_pause_chunk( + self, config: dict[str, Any], meta: dict[str, Any] + ) -> StreamChunk | None: + """The pause update, rebuilt from the checkpoint, when core did not send it. + + Core older than the fix drops it at ``low`` and ``partial`` granularity. A thread that + is paused after the run (``interrupt()``, interrupt_before/after, remote-tool wait) is + the signal; nothing is sent otherwise. + """ + if not self.checkpointer: + return None + try: + state = await self.checkpointer.aget_state(config) + except Exception as e: # the store is unavailable + logger.debug("Pause lookup failed: %s", e) + return None + if state is None: + return None + try: + from tenxgraph.utils.interrupt import pending_interrupt + + pending = pending_interrupt(state) + except Exception: # core older than interrupt() + pending = None + step = getattr(getattr(state, "execution_meta", None), "step", None) + if pending is not None: + data: dict[str, Any] = { + "status": "interrupted", + "node": pending.node, + "step": step, + "reason": "Graph execution paused by interrupt()", + "pause": "interrupt", + "interrupt": pending.model_dump(mode="json"), + } + else: + data = self._static_pause_data(state, step) + if data is None: + return None + return StreamChunk( + event=StreamEvent.UPDATES, + data=data, + thread_id=config.get("thread_id"), + run_id=config.get("run_id"), + metadata=dict(meta), + ) + + @staticmethod + def _static_pause_data(state: Any, step: int | None) -> dict[str, Any] | None: + """Pause data for interrupt_before / interrupt_after / remote-tool waits, from the state.""" + meta = getattr(state, "execution_meta", None) + status = str(getattr(getattr(meta, "status", None), "value", getattr(meta, "status", ""))) + node = getattr(meta, "interrupted_node", None) + if status not in ("interrupted_before", "interrupted_after") or not node: + return None + if getattr(meta, "interrupt_reason", None) == "remote_tool_call": + pause, name, reason = ( + "remote_tool", + "node_invoked", + "Remote tool call - graph interrupted", + ) + elif status == "interrupted_before": + pause, name = "before", "invoking_node" + reason = "Graph execution interrupted before node execution" + else: + pause, name = "after", "node_invoked" + reason = "Graph execution interrupted after node execution" + return {"status": name, "node": node, "step": step, "reason": reason, "pause": pause} + async def realtime_graph( self, input_queue: Any, @@ -1207,6 +1286,16 @@ def _restrict_realtime_overrides(self, overrides: dict[str, Any]) -> dict[str, A restricted["tools_tags"] = tags return restricted + def _realtime_options(self) -> dict[str, Any]: + """The /v1/graph/live init options this server applies (see ``_realtime_overrides``).""" + ws = self.config.websocket + return { + "client_overrides": list(ws.realtime_client_overrides), + "models": list(ws.realtime_models), + "model_selectable": bool(ws.realtime_models), + "tools_tags": True, + } + async def graph_details(self) -> GraphSchema: try: logger.info("Getting graph details") @@ -1216,6 +1305,7 @@ async def graph_details(self) -> GraphSchema: info = res.get("info") if isinstance(info, dict): info["is_realtime"] = self.is_live_agent + info["realtime_options"] = self._realtime_options() if self.is_live_agent else None return GraphSchema(**res) except ValueError as e: logger.warning(f"Graph details validation failed: {e}") @@ -1311,7 +1401,16 @@ async def get_observability( """ store = self.telemetry if not store: - return ObservabilitySchema(thread_id=str(thread_id), run_count=0) + from tenxgraph_api.src.app.core.config.settings import get_settings + + reason = ( + "disabled_in_production" + if get_settings().MODE == "production" + else "store_unavailable" + ) + return ObservabilitySchema( + thread_id=str(thread_id), enabled=False, reason=reason, run_count=0 + ) runs = store.get_runs(thread_id) run_ids = [r.run_id for r in runs] @@ -1328,6 +1427,7 @@ async def get_observability( run_count=len(runs), run_ids=run_ids, run=None, + reason="no_runs", ) run_schema = self._reconstruct_run(trace) @@ -1340,156 +1440,11 @@ async def get_observability( def _reconstruct_run(self, trace: Any) -> "ObsRunSchema": """Turn a captured RunTrace into spans + events + usage.""" - records = list(trace.records) - started = trace.started_at - # Fall back to the first record's timestamp if start wasn't stamped. - first_ts = next((r.get("timestamp") for r in records if r.get("timestamp")), started) - base = started or first_ts or 0.0 - last_ts = max([r.get("timestamp") or base for r in records] + [trace.finished_at or base]) - total_ms = max(0.0, (last_ts - base) * 1000.0) - - def off_ms(ts: float | None) -> float: - return max(0.0, ((ts or base) - base) * 1000.0) - - usage = ObsTokenUsageSchema() - events: list[ObsEventSchema] = [] - spans: list[ObsSpanSchema] = [] - - # Root span spans the whole run. - spans.append( - ObsSpanSchema( - id="root", - name="graph", - kind="root", - parent=None, - start_ms=0.0, - duration_ms=total_ms, - ) - ) - - # Walk records to build node spans (by node transitions) + llm/tool spans. - node_spans: dict[str, ObsSpanSchema] = {} - node_order: list[str] = [] - llm_calls = 0 - tool_calls = 0 - span_seq = 0 - - def new_span_id() -> str: - nonlocal span_seq - span_seq += 1 - return f"s{span_seq}" - - for idx, rec in enumerate(records): - node = rec.get("node") or "—" - ts = rec.get("timestamp") - ev_type = rec.get("event") or "message" - - # Node span: open on first sight of a node, extend its end as records arrive. - if node and node != "—": - span = node_spans.get(node) - if span is None: - span = ObsSpanSchema( - id=new_span_id(), - name=f"node: {node}", - kind="node", - parent="root", - start_ms=off_ms(ts), - duration_ms=0.0, - ) - node_spans[node] = span - node_order.append(node) - spans.append(span) - else: - span.duration_ms = max(span.duration_ms, off_ms(ts) - span.start_ms) - - # LLM span: a non-delta assistant message with usages under this node. - u = rec.get("usages") - if u: - usage.prompt_tokens += int(u.get("prompt_tokens", 0) or 0) - usage.completion_tokens += int(u.get("completion_tokens", 0) or 0) - usage.reasoning_tokens += int(u.get("reasoning_tokens", 0) or 0) - usage.total_tokens += int( - u.get("total_tokens", 0) - or (u.get("prompt_tokens", 0) or 0) + (u.get("completion_tokens", 0) or 0) - ) - llm_calls += 1 - parent = node_spans.get(node) - spans.append( - ObsSpanSchema( - id=new_span_id(), - name="llm.generate", - kind="llm", - parent=parent.id if parent else "root", - start_ms=off_ms(ts), - duration_ms=0.0, - model=(rec.get("model") or None), - input_tokens=int(u.get("prompt_tokens", 0) or 0), - output_tokens=int(u.get("completion_tokens", 0) or 0), - ) - ) - - # Tool spans: from tool_call / tool_result content blocks. - for tname in rec.get("tool_names", []) or []: - if "tool_call" in (rec.get("content_kinds") or []): - tool_calls += 1 - parent = node_spans.get(node) - spans.append( - ObsSpanSchema( - id=new_span_id(), - name=f"tool: {tname}", - kind="tool", - parent=parent.id if parent else "root", - start_ms=off_ms(ts), - duration_ms=0.0, - ) - ) - - # Event row. - events.append( - ObsEventSchema( - id=f"e{idx}", - type=ev_type, - node=node if node != "—" else "", - offset_ms=off_ms(ts), - summary=self._event_summary(rec), - ) - ) - - # Newest-first events for the pane. - events.reverse() - - return ObsRunSchema( - run_id=trace.run_id, - thread_id=trace.thread_id, - status=trace.status, - started_at=trace.started_at, - finished_at=trace.finished_at, - duration_ms=total_ms, - spans=spans, - events=events, - usage=usage, - llm_calls=llm_calls, - tool_calls=tool_calls, - iterations=len(node_order), - ) + return reconstruct_run(trace) @staticmethod def _event_summary(rec: dict[str, Any]) -> str: - kinds = rec.get("content_kinds") or [] - tools = rec.get("tool_names") or [] - if rec.get("is_error"): - return "error" - if tools and ("tool_call" in kinds or "tool_result" in kinds): - label = "tool_call" if "tool_call" in kinds else "tool_result" - return f"{label} {', '.join(tools)}" - if rec.get("usages"): - u = rec["usages"] - return f"llm.generate · {u.get('total_tokens', 0)} tokens" - if rec.get("is_delta"): - return "delta" - if kinds: - return f"{rec.get('role') or 'message'}: {', '.join(kinds)}" - return rec.get("event") or "chunk" + return event_summary(rec) async def get_state_schema(self) -> dict: try: diff --git a/tenxgraph_api/src/app/routers/media/__init__.py b/tenxgraph_api/src/app/routers/media/__init__.py index 92b4bb31..a6c9e2bb 100644 --- a/tenxgraph_api/src/app/routers/media/__init__.py +++ b/tenxgraph_api/src/app/routers/media/__init__.py @@ -300,7 +300,7 @@ async def upload_file( # Store binary in the configured MediaStore. The owner is written into the # file's own metadata, so ownership survives as long as the file does -- # unlike a cache entry, which would expire and silently un-own it. - file_metadata: dict[str, Any] = {"filename": filename} + file_metadata: dict[str, Any] = {"filename": filename, "created_at": int(time.time())} if user_id: file_metadata["owner_id"] = str(user_id) if self.supports_links: @@ -426,6 +426,48 @@ async def delete_file(self, file_id: str) -> bool: self._extraction_cache.pop(file_id, None) return deleted + async def list_files( + self, user_id: str | None, offset: int = 0, limit: int = 50 + ) -> tuple[list[dict[str, Any]], bool]: + """One page of the files ``user_id`` owns, newest first, and whether more follow. + + Files with no recorded owner are not listed for an identified user. With no ``user_id`` + (authentication off) every file is listed, matching ``ensure_can_access``. Reads each + stored file's metadata, so the cost grows with the number of files in the store. + + Raises: + NotImplementedError: The media store cannot enumerate its files. + """ + list_keys = getattr(self.store, "list_keys", None) # absent in core older than 1.0 + if not callable(list_keys): + raise NotImplementedError(f"{type(self.store).__name__} cannot list files") + keys = await list_keys() + owned: list[dict[str, Any]] = [] + for key in keys: + try: + meta = await self.store.get_metadata(key) + except Exception as exc: + logger.debug("Skipping unreadable file %s while listing: %s", log_safe(key), exc) + continue + if not meta: + continue + if user_id and str(meta.get("owner_id")) != str(user_id): + continue + owned.append( + { + "file_id": key, + "filename": meta.get("filename"), + "mime_type": meta.get("mime_type"), + "size_bytes": meta.get("size_bytes"), + "url": f"/v1/files/{key}", + "created_at": meta.get("created_at"), + } + ) + owned.sort(key=lambda f: f["file_id"]) + owned.sort(key=lambda f: f["created_at"] or 0, reverse=True) + page = owned[offset : offset + limit] + return page, offset + limit < len(owned) + async def get_file(self, file_id: str, user_id: str | None = None) -> tuple[bytes, str]: """Retrieve file bytes and MIME type.""" if user_id: diff --git a/tenxgraph_api/src/app/routers/media/router.py b/tenxgraph_api/src/app/routers/media/router.py index 17fcc01b..a392c107 100644 --- a/tenxgraph_api/src/app/routers/media/router.py +++ b/tenxgraph_api/src/app/routers/media/router.py @@ -5,7 +5,7 @@ import mimetypes from typing import Any -from fastapi import APIRouter, Depends, HTTPException, Request, UploadFile +from fastapi import APIRouter, Depends, HTTPException, Query, Request, UploadFile from fastapi.logger import logger from fastapi.responses import FileResponse, Response from injectq.integrations import InjectAPI @@ -19,6 +19,8 @@ FileAccessUrlResponse, FileDeleteResponse, FileInfoResponse, + FileListItem, + FileListResponse, FileUploadResponse, MultimodalConfigResponse, ) @@ -46,6 +48,7 @@ "text/plain", } ) +MAX_FILE_LIST_LIMIT = 200 _INLINE_PREFIXES = ("audio/", "video/") @@ -182,6 +185,41 @@ async def upload_file( return success_response(FileUploadResponse(**result), request) +# ------------------------------------------------------------------ +# GET /v1/files — the caller's own files, paged (declared before /{file_id}) +# ------------------------------------------------------------------ +@router.get( + "/v1/files", + summary="List the caller's files", + response_model=FileListResponse, + description=( + "Lists the files the authenticated user uploaded (files without a recorded owner are " + "not listed), newest first. With authentication off there is no user, so, like the " + "download route, every file is listed. Returns 501 when the configured media store cannot " + "enumerate its files." + ), +) +async def list_files( + request: Request, + limit: int = Query(50, ge=1, le=MAX_FILE_LIST_LIMIT), + offset: int = Query(0, ge=0), + service: MediaService = InjectAPI(MediaService), + user: dict[str, Any] = Depends(RequirePermission("files", "read")), +): + user_id = user.get("user_id") + try: + items, has_more = await service.list_files(user_id, offset, limit) + except NotImplementedError: + raise HTTPException(status_code=501, detail="The configured media store cannot list files") + payload = FileListResponse( + files=[FileListItem(**item) for item in items], + has_more=has_more, + offset=offset, + limit=limit, + ) + return success_response(payload, request) + + # ------------------------------------------------------------------ # 4.2 GET /v1/files/{file_id} — raw binary download # ------------------------------------------------------------------ diff --git a/tenxgraph_api/src/app/routers/media/schemas.py b/tenxgraph_api/src/app/routers/media/schemas.py index 08a858cc..c4a0933a 100644 --- a/tenxgraph_api/src/app/routers/media/schemas.py +++ b/tenxgraph_api/src/app/routers/media/schemas.py @@ -42,3 +42,21 @@ class MultimodalConfigResponse(BaseModel): media_storage_type: str media_max_size_mb: float document_handling: str + + +class FileListItem(BaseModel): + file_id: str + filename: str | None = None + mime_type: str | None = None + size_bytes: int | None = None + url: str = Field(..., description="Relative retrieval URL") + created_at: int | None = Field( + None, description="Upload time, unix seconds; null for files uploaded before it was kept" + ) + + +class FileListResponse(BaseModel): + files: list[FileListItem] + has_more: bool = Field(..., description="Whether another page follows") + offset: int + limit: int diff --git a/tenxgraph_api/src/app/routers/setup_router.py b/tenxgraph_api/src/app/routers/setup_router.py index ffa71e6f..e9ae475b 100644 --- a/tenxgraph_api/src/app/routers/setup_router.py +++ b/tenxgraph_api/src/app/routers/setup_router.py @@ -5,7 +5,6 @@ from fastapi.routing import APIWebSocketRoute from .checkpointer.router import router as checkpointer_router -from .evals import router as evals_router from .graph import router as graph_router from .media.router import router as media_router from .ping.router import router as ping_router @@ -72,9 +71,7 @@ def _graph_router(websocket: bool | None) -> APIRouter: return graph_router -def init_routes( - app: FastAPI, *, evals: bool = True, ag_ui: bool = False, websocket: bool | None = None -): +def init_routes(app: FastAPI, *, ag_ui: bool = False, websocket: bool | None = None): """ Initialize the routes for the FastAPI application. @@ -84,8 +81,6 @@ def init_routes( Args: app (FastAPI): The FastAPI application instance to which the routes will be added. - evals (bool): Mount the evals report viewer. It has no auth, so production - leaves it out. ag_ui (bool): Mount the AG-UI endpoint (``POST /v1/ag-ui``). Off unless ``ag_ui.enabled`` is set in 10xgraph.json. websocket (bool | None): ``websocket.enabled`` from 10xgraph.json. ``False`` leaves @@ -94,8 +89,6 @@ def init_routes( app.include_router(_graph_router(websocket)) app.include_router(checkpointer_router) app.include_router(store_router) - if evals: - app.include_router(evals_router) if ag_ui: app.include_router(_ag_ui_router()) app.include_router(ping_router) diff --git a/tenxgraph_api/src/app/utils/run_trace.py b/tenxgraph_api/src/app/utils/run_trace.py new file mode 100644 index 00000000..caeb56c4 --- /dev/null +++ b/tenxgraph_api/src/app/utils/run_trace.py @@ -0,0 +1,511 @@ +"""Run-trace capture and reconstruction for the observability endpoint. + +``telemetry_record`` flattens one ``StreamChunk`` into a compact, JSON-able record that the +in-memory ``TelemetryStore`` keeps. ``reconstruct_run`` turns the records of one run into a +span tree (root > node visit > llm / tool call) with real offsets, durations, status, and +inputs/outputs where the recorded chunks contain them. + +What the stream does not carry is not invented: the LLM prompt is never recorded, node inputs +are only the latest message seen before the visit, and ``invoke`` (non-stream) runs have no +per-node events. +""" + +from __future__ import annotations + +import json +import time +from typing import TYPE_CHECKING, Any + + +if TYPE_CHECKING: + from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( + ObsRunSchema, + ObsSpanSchema, + ) + + +CLIP_CHARS = 4000 +MAX_RECORDS = 2000 +MS_PER_SECOND = 1000.0 +_NO_MODEL = {"", "unknown", "none"} + +_STATUS_MAP = {"done": "ok", "error": "error", "running": "running", "stopped": "ok"} + + +def _get(obj: Any, key: str) -> Any: + if isinstance(obj, dict): + return obj.get(key) + return getattr(obj, key, None) + + +def clip(value: Any) -> Any: + """Return a JSON-able copy of ``value``, clipped to ``CLIP_CHARS`` characters.""" + if value is None or isinstance(value, bool | int | float): + return value + try: + text = value if isinstance(value, str) else json.dumps(value, default=str) + except (TypeError, ValueError): + text = str(value) + if len(text) <= CLIP_CHARS: + if isinstance(value, str): + return value + try: + return json.loads(text) + except ValueError: + return text + return f"{text[:CLIP_CHARS]}...[truncated {len(text) - CLIP_CHARS} chars]" + + +def _usage_dict(message: Any) -> dict[str, Any] | None: + usages = getattr(message, "usages", None) + if usages is None: + return None + try: + return usages.model_dump() if hasattr(usages, "model_dump") else dict(usages) + except Exception: + return None + + +def _model_name(message: Any) -> str | None: + meta = getattr(message, "metadata", None) + if not isinstance(meta, dict): + return None + name = meta.get("model") + if isinstance(name, str) and name.strip().lower() not in _NO_MODEL: + return name + return None + + +def telemetry_record(chunk: Any, ts: float | None = None) -> dict[str, Any]: + """Flatten a StreamChunk into a compact record for the telemetry store. + + ``ts`` is the arrival time. ``StreamChunk.timestamp`` is stamped once at import by core, + so it cannot order chunks; the caller passes the real clock instead. + """ + md = getattr(chunk, "metadata", None) or {} + data = getattr(chunk, "data", None) + data = data if isinstance(data, dict) else {} + node = ( + md.get("node") + or md.get("current_node") + or data.get("node") + or getattr(chunk, "node_name", "") + or "" + ) + step = data.get("step", md.get("step")) + + message = getattr(chunk, "message", None) + content_kinds: list[str] = [] + tool_names: list[str] = [] + tool_calls: list[dict[str, Any]] = [] + tool_results: list[dict[str, Any]] = [] + texts: list[str] = [] + block_errors: list[str] = [] + if message is not None: + content = getattr(message, "content", None) + if isinstance(content, list): + for block in content: + btype = _get(block, "type") + if btype: + content_kinds.append(btype) + if btype == "text" and _get(block, "text"): + texts.append(str(_get(block, "text"))) + elif btype == "tool_call": + name = _get(block, "name") + if name: + tool_names.append(name) + tool_calls.append( + { + "id": _get(block, "id"), + "name": name, + "args": clip(_get(block, "args") or {}), + } + ) + elif btype == "tool_result": + name = _get(block, "name") + if name: + tool_names.append(name) + tool_results.append( + { + "call_id": _get(block, "call_id"), + "output": clip(_get(block, "output")), + "is_error": bool(_get(block, "is_error")), + } + ) + elif btype == "error" and _get(block, "message"): + block_errors.append(str(_get(block, "message"))) + + event = getattr(chunk, "event", None) + event_name = str(event.value) if hasattr(event, "value") else str(event or "") + is_error = event_name == "error" or bool(getattr(chunk, "is_error", False)) + error = None + if is_error or block_errors: + error = clip( + data.get("error") + or data.get("reason") + or data.get("message") + or "; ".join(block_errors) + ) + error = str(error) if error else None + + text = "\n".join(texts) + return { + "event": event_name, + "node": node, + "step": step if isinstance(step, int) else None, + "timestamp": ts if ts is not None else time.time(), + "is_delta": bool(getattr(message, "delta", False)) if message else False, + "role": getattr(message, "role", None) if message else None, + "message_id": str(getattr(message, "message_id", "") or "") if message else "", + "usages": _usage_dict(message) if message is not None else None, + "content_kinds": content_kinds, + "tool_names": tool_names, + "is_error": is_error, + "status": data.get("status"), + "model": _model_name(message) if message is not None else None, + "text": clip(text) if text else None, + "tool_calls": tool_calls, + "tool_results": tool_results, + "tool_call_id": data.get("tool_call_id"), + "tool_name": data.get("tool_name"), + "args": clip(data.get("args")) if data.get("args") is not None else None, + "error": error, + } + + +def event_summary(rec: dict[str, Any]) -> str: + kinds = rec.get("content_kinds") or [] + tools = rec.get("tool_names") or [] + if rec.get("is_error"): + return "error" + if tools and ("tool_call" in kinds or "tool_result" in kinds): + label = "tool_call" if "tool_call" in kinds else "tool_result" + return f"{label} {', '.join(tools)}" + if rec.get("usages"): + u = rec["usages"] + return f"llm.generate · {u.get('total_tokens', 0)} tokens" + if rec.get("is_delta"): + return "delta" + if kinds: + return f"{rec.get('role') or 'message'}: {', '.join(kinds)}" + return rec.get("event") or "chunk" + + +def _summarize_message(rec: dict[str, Any]) -> dict[str, Any]: + summary: dict[str, Any] = {"role": rec.get("role")} + if rec.get("text"): + summary["text"] = rec["text"] + if rec.get("tool_calls"): + summary["tool_calls"] = [ + {"id": c.get("id"), "name": c.get("name")} for c in rec["tool_calls"] + ] + if rec.get("tool_results"): + summary["tool_results"] = [ + {"call_id": r.get("call_id"), "is_error": r.get("is_error", False)} + for r in rec["tool_results"] + ] + return summary + + +class _Visit: + def __init__(self, span: ObsSpanSchema, node: str, step: int | None, ts: float) -> None: + self.span = span + self.node = node + self.step = step + self.last_ts = ts + self.boundary = ts + self.messages: list[dict[str, Any]] = [] + self.seen_ids: set[str] = set() + self.error: str | None = None + self.delta_text: list[str] = [] + self.calls: list[dict[str, Any]] = [] + self.model: str | None = None + + +def reconstruct_run(trace: Any) -> ObsRunSchema: # noqa: PLR0912, PLR0915 + """Turn a captured RunTrace into spans + events + usage.""" + # Imported here: the routers package imports graph_service, which imports this module. + from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( + ObsEventSchema, + ObsRunSchema, + ObsSpanSchema, + ObsTokenUsageSchema, + ) + + records = list(trace.records) + started = trace.started_at + first_ts = next((r.get("timestamp") for r in records if r.get("timestamp")), started) + base = started or first_ts or 0.0 + last_ts = max([r.get("timestamp") or base for r in records] + [trace.finished_at or base]) + total_ms = max(0.0, (last_ts - base) * MS_PER_SECOND) + + def off_ms(ts: float | None) -> float: + return max(0.0, ((ts or base) - base) * MS_PER_SECOND) + + usage = ObsTokenUsageSchema() + events: list[ObsEventSchema] = [] + root = ObsSpanSchema( + id="root", + name="graph", + kind="root", + parent=None, + start_ms=0.0, + duration_ms=total_ms, + status=_STATUS_MAP.get(trace.status, "ok"), + ) + spans: list[ObsSpanSchema] = [root] + visits: list[_Visit] = [] + visit_counts: dict[str, int] = {} + cur: _Visit | None = None + tool_spans: dict[str, ObsSpanSchema] = {} + pending_calls: dict[str, dict[str, Any]] = {} + seen_llm_ids: set[str] = set() + models: list[str] = [] + run_error: str | None = None + first_user: str | None = None + last_assistant: str | None = None + last_message: dict[str, Any] | None = None + span_seq = 0 + + def new_span_id() -> str: + nonlocal span_seq + span_seq += 1 + return f"s{span_seq}" + + def parent_id() -> str: + return cur.span.id if cur else "root" + + for idx, rec in enumerate(records): + node = rec.get("node") or "" + ts = rec.get("timestamp") or base + step = rec.get("step") + + # A new node visit starts when the node changes, or the step counter moves on. + if node and ( + cur is None + or node != cur.node + or (step is not None and cur.step is not None and step != cur.step) + ): + visit_counts[node] = visit_counts.get(node, 0) + 1 + span = ObsSpanSchema( + id=new_span_id(), + name=f"node: {node}", + kind="node", + parent="root", + start_ms=off_ms(ts), + duration_ms=0.0, + step=step, + visit=visit_counts[node], + inputs={"incoming_message": last_message} if last_message else None, + ) + cur = _Visit(span, node, step, ts) + visits.append(cur) + spans.append(span) + if cur is not None and node: + if cur.step is None and step is not None: + cur.span.step = cur.step = step + cur.last_ts = max(cur.last_ts, ts) + cur.span.duration_ms = max(0.0, off_ms(cur.last_ts) - cur.span.start_ms) + + status = rec.get("status") + call_id = rec.get("tool_call_id") + + if rec.get("error") or rec.get("is_error"): + err = rec.get("error") or "error" + run_error = run_error or err + target = tool_spans.get(call_id) if call_id else None + if target is not None: + target.status = "error" + target.error = err + target.duration_ms = max(0.0, off_ms(ts) - target.start_ms) + elif cur is not None: + cur.error = cur.error or err + + if rec.get("model"): + if rec["model"] not in models: + models.append(rec["model"]) + if cur is not None: + cur.model = rec["model"] + + # Tool execution starts (tool node announces it) -> tool span. + if status == "invoking_tool" and call_id and call_id not in tool_spans: + name = rec.get("tool_name") or "tool" + call = pending_calls.get(call_id) or {} + tspan = ObsSpanSchema( + id=new_span_id(), + name=f"tool: {name}", + kind="tool", + parent=parent_id(), + start_ms=off_ms(ts), + duration_ms=0.0, + status="running" if trace.status == "running" else "ok", + inputs={ + "args": rec.get("args") if rec.get("args") is not None else call.get("args") + }, + tool_name=name, + tool_call_id=call_id, + ) + tool_spans[call_id] = tspan + spans.append(tspan) + + if rec.get("tool_calls"): + for call in rec["tool_calls"]: + cid = call.get("id") + if cid and cid not in pending_calls: + pending_calls[cid] = { + "name": call.get("name"), + "args": call.get("args"), + "ts": ts, + "parent": parent_id(), + } + if cur is not None and call not in cur.calls: + cur.calls.append(call) + + # Tool results close the tool span (create one for runs without tool updates). + for res in rec.get("tool_results") or []: + cid = res.get("call_id") + tspan = tool_spans.get(cid) if cid else None + if tspan is None and cid: + call = pending_calls.get(cid) or {} + name = call.get("name") or rec.get("tool_name") or "tool" + start = call.get("ts", ts) + tspan = ObsSpanSchema( + id=new_span_id(), + name=f"tool: {name}", + kind="tool", + parent=call.get("parent") or parent_id(), + start_ms=off_ms(start), + duration_ms=0.0, + inputs={"args": call.get("args")} if call else None, + tool_name=name, + tool_call_id=cid, + ) + tool_spans[cid] = tspan + spans.append(tspan) + if tspan is not None and tspan.outputs is None: + tspan.duration_ms = max(0.0, off_ms(ts) - tspan.start_ms) + tspan.outputs = {"result": res.get("output")} + if res.get("is_error"): + tspan.status = "error" + tspan.error = tspan.error or clip(res.get("output")) or "tool error" + else: + tspan.status = "ok" + if cur is not None: + cur.boundary = max(cur.boundary, ts) + + # Message bookkeeping (complete messages only). + if rec.get("role") and not rec.get("is_delta"): + mid = rec.get("message_id") + summary = _summarize_message(rec) + if rec.get("role") == "user" and first_user is None and rec.get("text"): + first_user = rec["text"] + if rec.get("role") == "assistant" and rec.get("text"): + last_assistant = rec["text"] + if cur is not None and (not mid or mid not in cur.seen_ids): + cur.messages.append(summary) + if mid: + cur.seen_ids.add(mid) + last_message = summary + elif rec.get("is_delta") and cur is not None and rec.get("text"): + cur.delta_text.append(rec["text"]) + + # LLM span: a message that carries token usage. + u = rec.get("usages") + mid = rec.get("message_id") + if u and not (mid and mid in seen_llm_ids): + if mid: + seen_llm_ids.add(mid) + prompt = int(u.get("prompt_tokens", 0) or 0) + completion = int(u.get("completion_tokens", 0) or 0) + total = int(u.get("total_tokens", 0) or prompt + completion) + usage.prompt_tokens += prompt + usage.completion_tokens += completion + usage.reasoning_tokens += int(u.get("reasoning_tokens", 0) or 0) + usage.total_tokens += total + start_ts = min(cur.boundary, ts) if cur else base + out: dict[str, Any] = {} + text = rec.get("text") or ("".join(cur.delta_text) if cur and cur.delta_text else "") + if text: + out["text"] = text + calls = rec.get("tool_calls") or (cur.calls if cur else []) + if calls: + out["tool_calls"] = list(calls) + spans.append( + ObsSpanSchema( + id=new_span_id(), + name="llm.generate", + kind="llm", + parent=parent_id(), + start_ms=off_ms(start_ts), + duration_ms=max(0.0, off_ms(ts) - off_ms(start_ts)), + outputs=out or None, + model=rec.get("model") or (cur.model if cur else None), + input_tokens=prompt, + output_tokens=completion, + total_tokens=total, + reasoning_tokens=int(u.get("reasoning_tokens", 0) or 0), + ) + ) + if cur is not None: + cur.boundary = max(cur.boundary, ts) + cur.delta_text = [] + cur.calls = [] + + events.append( + ObsEventSchema( + id=f"e{idx}", + type=rec.get("event") or "message", + node=node, + offset_ms=off_ms(ts), + summary=event_summary(rec), + ) + ) + + # Finish node spans: outputs, status. + for i, visit in enumerate(visits): + if visit.messages: + visit.span.outputs = {"messages": visit.messages} + if visit.error: + visit.span.status = "error" + visit.span.error = visit.error + elif trace.status == "running" and i == len(visits) - 1: + visit.span.status = "running" + + root.inputs = {"user_message": first_user} if first_user else None + root.outputs = {"assistant_message": last_assistant} if last_assistant else None + if trace.status == "error" or run_error: + root.status = "error" + root.error = run_error + + spans.sort(key=lambda s: s.start_ms) # stable: root stays first + events.reverse() + + notes = ["The LLM prompt is not recorded, so llm spans have no inputs."] + if records and not any(r.get("node") for r in records): + notes.append( + "No per-node events were recorded (non-streaming run): spans hang off the root " + "and timings come from message creation times." + ) + else: + notes.append("Node inputs show only the latest message seen before the visit.") + if len(records) >= MAX_RECORDS: + notes.append("Trace truncated: only the most recent 2000 chunks are kept.") + + return ObsRunSchema( + run_id=trace.run_id, + thread_id=trace.thread_id, + status=trace.status, + started_at=trace.started_at, + finished_at=trace.finished_at, + duration_ms=total_ms, + spans=spans, + events=events, + usage=usage, + llm_calls=sum(1 for s in spans if s.kind == "llm"), + tool_calls=sum(1 for s in spans if s.kind == "tool"), + iterations=len(visits), + error=run_error, + model=models[0] if models else None, + models=models, + notes=notes, + ) diff --git a/tests/cli/test_cli_main.py b/tests/cli/test_cli_main.py index d321886c..6a745f68 100644 --- a/tests/cli/test_cli_main.py +++ b/tests/cli/test_cli_main.py @@ -163,7 +163,6 @@ def test_eval_command(monkeypatch): "--no-report", "-t", "0.8", - "--open", "--parallel", "-c", "8", @@ -174,7 +173,6 @@ def test_eval_command(monkeypatch): assert called["output_dir"] == "out_dir" assert called["no_report"] is True assert called["threshold"] == 0.8 - assert called["open_report"] is True assert called["parallel"] is True assert called["max_concurrency"] == 8 diff --git a/tests/cli/test_eval_command.py b/tests/cli/test_eval_command.py index cc360c9a..b7f71d94 100644 --- a/tests/cli/test_eval_command.py +++ b/tests/cli/test_eval_command.py @@ -297,7 +297,6 @@ def test_execute_success(cmd, tmp_path): mock_report.summary.total_token_usage = TokenUsage(input_tokens=100, output_tokens=50) mock_rep_mgr_res = MagicMock() - mock_rep_mgr_res.html_path = "/absolute/path/to/report.html" mock_rep_mgr_res.json_path = "path/to/report.json" mock_rep_mgr_res.has_errors = False @@ -323,12 +322,10 @@ def test_execute_success(cmd, tmp_path): patch.object(cmd, "_run_flat_pool", return_value=quads), patch.object(cmd, "_merge_reports", return_value=mock_report), patch("tenxgraph_api.cli.commands.eval.ReporterManager", return_value=mock_rep_mgr), - patch("webbrowser.open") as mock_web_open, ): - code = cmd.execute(target=str(tmp_path), open_report=True) + code = cmd.execute(target=str(tmp_path)) assert code == 0 - mock_web_open.assert_called_once() - assert len(cmd.output.successes) > 0 + assert any("Open it in the playground" in m for m in cmd.output.infos) def test_execute_below_threshold(cmd, tmp_path): diff --git a/tests/unit_tests/test_eval_report_service.py b/tests/unit_tests/test_eval_report_service.py deleted file mode 100644 index c4c59f33..00000000 --- a/tests/unit_tests/test_eval_report_service.py +++ /dev/null @@ -1,290 +0,0 @@ -"""Unit tests for EvalReportService, which reads real `10xgraph eval` JSON -reports (eval_reports/*.json, written by tenxgraph.qa.evaluation.reporters.json.JSONReporter) -instead of the dummy in-module data the evals router used to serve.""" - -import json - -import pytest -from fastapi import HTTPException - -from tenxgraph_api.src.app.routers.evals.services.eval_report_service import ( - EvalReportService, - _case_conversation, - _case_input, - _case_rubric, - _case_score, - _case_type, - _format_ago, - _format_duration, -) - - -def _criterion(name, score, threshold=0.8): - return { - "criterion": name, - "score": score, - "passed": score >= threshold, - "threshold": threshold, - "details": {}, - "error": None, - "token_usage": { - "input_tokens": 10, - "output_tokens": 5, - "cache_read_tokens": 0, - "cache_creation_tokens": 0, - "total_tokens": 15, - }, - } - - -def _eval_case(eval_id, name, passed, score, duration=1.2): - return { - "eval_id": eval_id, - "name": name, - "passed": passed, - "criterion_results": [_criterion("correctness", score)], - "actual_trajectory": [], - "actual_tool_calls": [], - "actual_response": "The refund has been processed.", - "messages": [{"role": "user", "content": "I want a refund for order #4471."}], - "node_responses": [], - "node_visits": [], - "duration_seconds": duration, - "error": None, - "metadata": {}, - "turn_results": [], - "token_usage": { - "input_tokens": 100, - "output_tokens": 50, - "cache_read_tokens": 0, - "cache_creation_tokens": 0, - "total_tokens": 150, - }, - "agent_token_usage": { - "input_tokens": 0, - "output_tokens": 0, - "cache_read_tokens": 0, - "cache_creation_tokens": 0, - "total_tokens": 0, - }, - "node_details": [], - } - - -def _sim_case(eval_id, name, passed, score): - case = _eval_case(eval_id, name, passed, score, duration=4.3) - case["metadata"] = {"turns": 3, "goals_achieved": passed, "completed": True} - case["actual_response"] = ( - "USER: I've been charged three times, fix it.\n" - "AGENT: I understand, let me look into that.\n" - "USER: Just get me a person!" - ) - return case - - -def _report(eval_set_id, timestamp, cases, eval_set_name=""): - passed = sum(1 for c in cases if c["passed"]) - total = len(cases) - return { - "eval_set_id": eval_set_id, - "eval_set_name": eval_set_name, - "results": cases, - "summary": { - "total_cases": total, - "passed_cases": passed, - "failed_cases": total - passed, - "error_cases": 0, - "pass_rate": passed / total if total else 0.0, - "avg_duration_seconds": 1.5, - "total_duration_seconds": 1.5 * total, - "criterion_stats": {}, - "total_token_usage": { - "input_tokens": 100 * total, - "output_tokens": 50 * total, - "cache_read_tokens": 0, - "cache_creation_tokens": 0, - "total_tokens": 150 * total, - }, - "per_case_token_usage": {}, - "avg_tokens_per_case": 150.0, - }, - "config_used": {"model": "gemini-2.0-flash"}, - "timestamp": timestamp, - "metadata": {}, - } - - -class TestPureHelpers: - def test_format_duration(self): - assert _format_duration(1.234) == "1.2s" - - def test_format_ago_minutes_hours_days(self): - now = 1_000_000.0 - assert _format_ago(now - 30, now) == "just now" - assert _format_ago(now - 5 * 60, now) == "5m ago" - assert _format_ago(now - 3 * 3600, now) == "3h ago" - assert _format_ago(now - 2 * 86400, now) == "2d ago" - - def test_case_type_eval_vs_sim(self): - assert _case_type(_eval_case("c1", "n", True, 0.9)) == "eval" - assert _case_type(_sim_case("c2", "n", True, 0.9)) == "sim" - - def test_case_score_averages_criteria(self): - case = _eval_case("c1", "n", True, 0.8) - case["criterion_results"].append(_criterion("tone", 0.6)) - assert _case_score(case) == pytest.approx(0.7) - - def test_case_score_falls_back_to_passed_when_no_criteria(self): - case = _eval_case("c1", "n", True, 0.8) - case["criterion_results"] = [] - assert _case_score(case) == 1.0 - case["passed"] = False - assert _case_score(case) == 0.0 - - def test_case_input_reads_first_user_message(self): - case = _eval_case("c1", "n", True, 0.8) - assert _case_input(case) == "I want a refund for order #4471." - - def test_case_input_missing_falls_back(self): - case = _eval_case("c1", "n", True, 0.8) - case["messages"] = [] - assert _case_input(case) == "—" - - def test_case_rubric_includes_weighted_score_row(self): - case = _eval_case("c1", "n", True, 0.8) - rubric = _case_rubric(case) - assert rubric[0] == {"key": "correctness", "value": 0.8, "tone": "accent"} - assert rubric[-1]["key"] == "weighted score" - - def test_case_rubric_none_when_no_criteria(self): - case = _eval_case("c1", "n", True, 0.8) - case["criterion_results"] = [] - assert _case_rubric(case) is None - - def test_case_conversation_none_for_eval_case(self): - assert _case_conversation(_eval_case("c1", "n", True, 0.8)) is None - - def test_case_conversation_parses_sim_transcript(self): - turns = _case_conversation(_sim_case("c2", "n", False, 0.5)) - assert turns == [ - {"role": "sim", "text": "I've been charged three times, fix it."}, - {"role": "agent", "text": "I understand, let me look into that."}, - {"role": "sim", "text": "Just get me a person!"}, - ] - - -class TestEvalReportServiceListRuns: - def test_empty_directory_returns_no_runs(self, tmp_path): - service = EvalReportService(reports_dir=tmp_path) - assert service.list_runs() == [] - - def test_missing_directory_returns_no_runs(self, tmp_path): - service = EvalReportService(reports_dir=tmp_path / "does_not_exist") - assert service.list_runs() == [] - - def test_lists_runs_sorted_newest_first_with_run_numbers(self, tmp_path): - older = _report("customer_support", 1_000_000.0, [_eval_case("c1", "n", True, 0.9)]) - newer = _report("customer_support", 1_000_500.0, [_eval_case("c1", "n", False, 0.5)]) - (tmp_path / "customer_support_20260101_000000.json").write_text(json.dumps(older)) - (tmp_path / "customer_support_20260101_001000.json").write_text(json.dumps(newer)) - - service = EvalReportService(reports_dir=tmp_path) - runs = service.list_runs(now=1_000_500.0 + 60) - - assert [r["run"] for r in runs] == ["#2", "#1"] - assert runs[0]["status"] == "fail" - assert runs[1]["status"] == "pass" - assert runs[0]["cases"] == 1 - assert runs[0]["rate"] == 0.0 - assert runs[1]["rate"] == 100.0 - - def test_run_numbers_are_per_eval_set(self, tmp_path): - cs = _report("customer_support", 1_000_000.0, [_eval_case("c1", "n", True, 0.9)]) - rs = _report("refund_simulator", 1_000_100.0, [_sim_case("c1", "n", True, 0.9)]) - (tmp_path / "customer_support_1.json").write_text(json.dumps(cs)) - (tmp_path / "refund_simulator_1.json").write_text(json.dumps(rs)) - - service = EvalReportService(reports_dir=tmp_path) - runs = {r["name"]: r for r in service.list_runs(now=1_000_100.0)} - - assert runs["customer_support"]["run"] == "#1" - assert runs["refund_simulator"]["run"] == "#1" - - def test_skips_unparseable_json_files(self, tmp_path): - (tmp_path / "broken.json").write_text("{not valid json") - good = _report("customer_support", 1_000_000.0, [_eval_case("c1", "n", True, 0.9)]) - (tmp_path / "good.json").write_text(json.dumps(good)) - - service = EvalReportService(reports_dir=tmp_path) - runs = service.list_runs(now=1_000_000.0) - - assert len(runs) == 1 - assert runs[0]["name"] == "customer_support" - - -class TestEvalReportServiceRunDetail: - def test_unknown_run_id_raises_404(self, tmp_path): - service = EvalReportService(reports_dir=tmp_path) - with pytest.raises(HTTPException) as exc: - service.get_run_detail("nope") - assert exc.value.status_code == 404 - - def test_detail_shape_for_single_run(self, tmp_path): - report = _report( - "customer_support", - 1_000_000.0, - [ - _eval_case("c1", "greets user", True, 0.9), - _eval_case("c2", "refunds order", False, 0.4), - ], - eval_set_name="customer-support", - ) - (tmp_path / "customer_support_run1.json").write_text(json.dumps(report)) - - service = EvalReportService(reports_dir=tmp_path) - run_id = "customer_support_run1" - detail = service.get_run_detail(run_id) - - assert detail["title"] == "customer-support · run #1" - assert detail["status"] == "fail" - assert detail["rate"] == 50.0 - assert len(detail["cases"]) == 2 - assert detail["cases"][0]["id"] == "c1" - assert detail["cases"][0]["status"] == "pass" - assert detail["cases"][1]["status"] == "fail" - assert detail["regression"] is None - stat_labels = [s["label"] for s in detail["stats"]] - assert "cases" in stat_labels - assert "avg latency" in stat_labels - - def test_detail_includes_regression_against_previous_run_of_same_set(self, tmp_path): - older = _report( - "customer_support", - 1_000_000.0, - [_eval_case("c1", "n", True, 0.9), _eval_case("c2", "n", True, 0.85)], - ) - newer = _report( - "customer_support", - 1_000_500.0, - [_eval_case("c1", "n", True, 0.88), _eval_case("c2", "n", False, 0.4)], - ) - (tmp_path / "customer_support_a.json").write_text(json.dumps(older)) - (tmp_path / "customer_support_b.json").write_text(json.dumps(newer)) - - service = EvalReportService(reports_dir=tmp_path) - detail = service.get_run_detail("customer_support_b") - - assert detail["regression"] is not None - assert detail["regression"]["note"]["current"].startswith("run #2") - assert detail["regression"]["note"]["prev"].startswith("run #1") - newly_failing = next( - s for s in detail["regression"]["summary"] if s["label"] == "newly failing" - ) - assert newly_failing["value"] == "1" - row_c2 = next( - r for r in detail["regression"]["rows"] if r["name"] == "n" and r["stay"] is False - ) - assert row_c2["flip"] == "pass → fail" - - older_detail = service.get_run_detail("customer_support_a") - assert older_detail["regression"] is None diff --git a/tests/unit_tests/test_mode_validation.py b/tests/unit_tests/test_mode_validation.py index d0f764af..4a399ae0 100644 --- a/tests/unit_tests/test_mode_validation.py +++ b/tests/unit_tests/test_mode_validation.py @@ -65,6 +65,6 @@ def test_production_alias_turns_on_production_behaviour(): assert settings.REDOCS_PATH == "" # The comparison main.py uses to leave the evals router out. app = FastAPI() - init_routes(app, evals=settings.MODE != "production") + init_routes(app) paths = {path for path, _route, _guarded in _iter_routes(app.routes)} assert not any(path.startswith("/v1/evals") for path in paths) diff --git a/tests/unit_tests/test_production_surface.py b/tests/unit_tests/test_production_surface.py index 0cf34c99..1f3a5b68 100644 --- a/tests/unit_tests/test_production_surface.py +++ b/tests/unit_tests/test_production_surface.py @@ -1,6 +1,5 @@ """Production does not expose dev-only surfaces (M3, M4). -- The evals report viewer has no auth, so it is not mounted in production. - API docs and the OpenAPI schema are off in production unless a path is set explicitly. - Generated Docker builds do not copy local uploads or eval reports into the image. """ @@ -23,15 +22,9 @@ def _paths(app: FastAPI) -> set[str]: return {path for path, _route, _guarded in _iter_routes(app.routes)} -def test_evals_viewer_is_mounted_by_default(): +def test_no_evals_endpoints_are_mounted(): app = FastAPI() init_routes(app) - assert "/v1/evals/runs" in _paths(app) - - -def test_evals_viewer_can_be_left_out(): - app = FastAPI() - init_routes(app, evals=False) assert not any(path.startswith("/v1/evals") for path in _paths(app)) diff --git a/tests/unit_tests/test_resume_binding_and_limits.py b/tests/unit_tests/test_resume_binding_and_limits.py index f1567be8..b1338bb8 100644 --- a/tests/unit_tests/test_resume_binding_and_limits.py +++ b/tests/unit_tests/test_resume_binding_and_limits.py @@ -193,6 +193,7 @@ async def test_prepare_input_passes_the_id_to_core(): srv._graph = MagicMock() srv.checkpointer = MagicMock() srv.checkpointer.aget_thread_owner = AsyncMock(return_value=None) + srv.checkpointer.aget_state = AsyncMock(return_value=None) srv.config = MagicMock() srv._media_service = None srv.check_media_urls = MagicMock() diff --git a/tests/unit_tests/test_run_trace.py b/tests/unit_tests/test_run_trace.py new file mode 100644 index 00000000..ce4657a7 --- /dev/null +++ b/tests/unit_tests/test_run_trace.py @@ -0,0 +1,157 @@ +"""Run-trace capture and reconstruction (observability).""" + +from tenxgraph.core.state import Message +from tenxgraph.core.state.message import TokenUsages +from tenxgraph.core.state.message_block import TextBlock, ToolCallBlock, ToolResultBlock +from tenxgraph.core.state.stream_chunks import StreamChunk, StreamEvent + +from tenxgraph_api.src.app.utils.run_trace import ( + CLIP_CHARS, + clip, + reconstruct_run, + telemetry_record, +) +from tenxgraph_api.src.app.utils.telemetry_store import TelemetryStore + + +def _updates(status, node, step=None, **extra): + return StreamChunk( + event=StreamEvent.UPDATES, data={"status": status, "node": node, "step": step, **extra} + ) + + +def _msg(role, content, node, usage=None, model=None): + m = Message( + role=role, + content=content, + usages=usage, + metadata={"model": model} if model else {}, + ) + return StreamChunk(event=StreamEvent.MESSAGE, message=m, metadata={"node": node}) + + +def _build(): + store = TelemetryStore() + trace = store.start_run("t", "r", 100.0) + usage = TokenUsages(prompt_tokens=10, completion_tokens=5, total_tokens=15) + usage2 = TokenUsages(prompt_tokens=20, completion_tokens=6, total_tokens=26) + seq = [ + (100.0, _msg("user", [TextBlock(text="weather?")], "")), + (100.1, _updates("node_invoked", "agent", 0)), + ( + 100.6, + _msg( + "assistant", + [ToolCallBlock(id="c1", name="get_weather", args={"city": "Paris"})], + "agent", + usage, + "gpt-x", + ), + ), + (100.7, _updates("node_invoked", "tools", 1)), + ( + 100.7, + _updates( + "invoking_tool", + "tools", + None, + tool_name="get_weather", + tool_call_id="c1", + args={"city": "Paris"}, + ), + ), + (101.2, _msg("tool", [ToolResultBlock(call_id="c1", output="sunny")], "tools")), + (101.3, _updates("node_invoked", "agent", 2)), + (101.9, _msg("assistant", [TextBlock(text="Sunny.")], "agent", usage2, "gpt-x")), + ] + for ts, chunk in seq: + trace.add(telemetry_record(chunk, ts)) + store.finish_run("t", "r", 102.0, "done") + return trace + + +def test_visits_llm_tool_spans_and_timing(): + run = reconstruct_run(_build()) + nodes = [s for s in run.spans if s.kind == "node"] + assert [(s.name, s.visit) for s in nodes] == [ + ("node: agent", 1), + ("node: tools", 1), + ("node: agent", 2), + ] + assert run.iterations == 3 + llms = [s for s in run.spans if s.kind == "llm"] + assert len(llms) == 2 and llms[0].duration_ms > 0 + assert llms[0].model == "gpt-x" and run.model == "gpt-x" + assert llms[0].outputs["tool_calls"][0]["name"] == "get_weather" + assert llms[1].outputs["text"] == "Sunny." + tool = next(s for s in run.spans if s.kind == "tool") + assert abs(tool.duration_ms - 500.0) < 1 + assert tool.inputs == {"args": {"city": "Paris"}} + assert tool.outputs == {"result": "sunny"} + assert tool.status == "ok" + tools_node = nodes[1] + assert tool.parent == tools_node.id + root = run.spans[0] + assert root.inputs == {"user_message": "weather?"} + assert root.outputs == {"assistant_message": "Sunny."} + assert run.usage.total_tokens == 41 + assert run.tool_calls == 1 and run.llm_calls == 2 + + +def test_error_chunk_marks_span_and_run(): + store = TelemetryStore() + trace = store.start_run("t", "r", 10.0) + for ts, chunk in [ + (10.0, _updates("node_invoked", "tools", 0)), + (10.1, _updates("invoking_tool", "tools", None, tool_name="x", tool_call_id="c1", args={})), + ( + 10.4, + StreamChunk( + event=StreamEvent.ERROR, + data={ + "status": "tool_failed", + "node": "tools", + "tool_call_id": "c1", + "error": "boom", + }, + ), + ), + ]: + trace.add(telemetry_record(chunk, ts)) + store.finish_run("t", "r", 10.5, "error") + run = reconstruct_run(trace) + tool = next(s for s in run.spans if s.kind == "tool") + assert tool.status == "error" and tool.error == "boom" + assert run.error == "boom" and run.spans[0].status == "error" + + +def test_invoke_style_records_without_nodes_get_note(): + store = TelemetryStore() + trace = store.start_run("t", "r", 1.0) + trace.add(telemetry_record(_msg("assistant", [TextBlock(text="hi")], ""), 1.5)) + run = reconstruct_run(trace) + assert run.iterations == 0 + assert any("non-streaming" in n for n in run.notes) + + +def test_clip_truncates_and_keeps_json(): + assert clip({"a": 1}) == {"a": 1} + long = clip("x" * (CLIP_CHARS + 10)) + assert long.endswith("...[truncated 10 chars]") + + +def test_observability_flags(monkeypatch): + import asyncio + + from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + + svc = GraphService.__new__(GraphService) + svc._telemetry = None + monkeypatch.setattr(GraphService, "telemetry", property(lambda self: None)) + out = asyncio.run(svc.get_observability("t")) + assert out.enabled is False and out.reason in {"disabled_in_production", "store_unavailable"} + + store = TelemetryStore() + monkeypatch.setattr(GraphService, "telemetry", property(lambda self: store)) + out = asyncio.run(svc.get_observability("t")) + assert out.enabled is True and out.reason == "no_runs" and out.run is None diff --git a/tests/unit_tests/test_static_pause_resume.py b/tests/unit_tests/test_static_pause_resume.py new file mode 100644 index 00000000..feaa63b2 --- /dev/null +++ b/tests/unit_tests/test_static_pause_resume.py @@ -0,0 +1,66 @@ +"""An empty ``resume`` continues an interrupt_before / interrupt_after pause with no message.""" + +# ruff: noqa: S101 + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from tenxgraph.core.state import AgentState +from tenxgraph.core.state.execution_state import ExecutionStatus +from tenxgraph.utils.interrupt import Interrupt, pause_at + +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema + +from .test_run_limits import ALICE, _service + + +def _static_pause(status: ExecutionStatus) -> AgentState: + state = AgentState() + state.set_interrupt("TOOL", "HITL", status) + return state + + +async def _prepare(state, **fields): + service = _service(MagicMock(), timeout=None) + service.checkpointer.aget_state.return_value = state + graph_input = GraphInputSchema(config={"thread_id": "t1"}, **fields) + input_data, _, _ = await service._prepare_input(graph_input, ALICE["user_id"]) + return input_data + + +@pytest.mark.parametrize( + "status", [ExecutionStatus.INTERRUPTED_BEFORE, ExecutionStatus.INTERRUPTED_AFTER] +) +@pytest.mark.asyncio +async def test_empty_resume_continues_a_static_pause_without_a_message(status): + input_data = await _prepare(_static_pause(status), resume=None) + assert input_data["messages"] == [] + assert "resume" not in input_data + + +@pytest.mark.asyncio +async def test_empty_resume_on_an_unpaused_thread_still_reaches_core(): + # Core answers "Nothing to resume"; the API must not hide it. + input_data = await _prepare(AgentState(), resume=None) + assert "resume" in input_data + input_data = await _prepare(None, resume=None) + assert "resume" in input_data + + +@pytest.mark.asyncio +async def test_interrupt_resume_is_unchanged(): + state = AgentState() + pause_at(state, Interrupt(id="i1", key="k", node="ASK", value={"amount": 50})) + input_data = await _prepare(state, resume={"approved": True}, interrupt_id="i1") + assert input_data["resume"] == {"approved": True} + assert input_data["resume_interrupt_id"] == "i1" + + +@pytest.mark.asyncio +async def test_remote_tool_pause_keeps_the_resume_key(): + state = _static_pause(ExecutionStatus.INTERRUPTED_AFTER) + state.execution_meta.interrupt_reason = "remote_tool_call" + input_data = await _prepare(state, resume=None) + assert "resume" in input_data diff --git a/tests/unit_tests/test_stream_pause_chunk.py b/tests/unit_tests/test_stream_pause_chunk.py new file mode 100644 index 00000000..86506053 --- /dev/null +++ b/tests/unit_tests/test_stream_pause_chunk.py @@ -0,0 +1,161 @@ +"""The ``interrupt()`` pause update reaches clients at every granularity (old core included).""" + +# ruff: noqa: S101 + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from tenxgraph.core.state import AgentState, StreamChunk, StreamEvent +from tenxgraph.utils import ResponseGranularity +from tenxgraph.utils.interrupt import Interrupt, pause_at + +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema +from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + +from .test_run_limits import ALICE, _service + + +def _paused_state() -> AgentState: + state = AgentState() + pause_at(state, Interrupt(id="i1", key="k", node="ASK", value={"amount": 50})) + return state + + +def _stream_without_pause(): + """What core older than the fix yields at low granularity: messages only.""" + + async def astream(*args, **kwargs): + yield StreamChunk(event=StreamEvent.MESSAGE) + + return astream + + +def _input(granularity): + return GraphInputSchema( + messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + config={"thread_id": "t1"}, + response_granularity=granularity, + ) + + +async def _run(service, granularity): + return [c async for c in service.stream_chunks(_input(granularity), ALICE)] + + +@pytest.mark.parametrize("granularity", [ResponseGranularity.LOW, ResponseGranularity.PARTIAL]) +@pytest.mark.asyncio +async def test_pause_rebuilt_from_checkpoint_when_core_dropped_it(granularity): + graph = MagicMock() + graph.astream = _stream_without_pause() + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = _paused_state() + + chunks = await _run(service, granularity) + pause = chunks[-1] + assert pause.event == StreamEvent.UPDATES + assert pause.data["status"] == "interrupted" + assert pause.data["node"] == "ASK" + assert pause.data["interrupt"]["id"] == "i1" + assert pause.data["interrupt"]["value"] == {"amount": 50} + + +@pytest.mark.asyncio +async def test_no_duplicate_when_core_already_sent_it(): + sent = StreamChunk( + event=StreamEvent.UPDATES, data={"status": "interrupted", "node": "ASK", "interrupt": {}} + ) + + async def astream(*args, **kwargs): + yield sent + + graph = MagicMock() + graph.astream = astream + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = _paused_state() + + chunks = await _run(service, ResponseGranularity.LOW) + assert len([c for c in chunks if c.data and c.data.get("status") == "interrupted"]) == 1 + + +@pytest.mark.parametrize("granularity", list(ResponseGranularity)) +@pytest.mark.asyncio +async def test_finished_run_gets_no_pause_chunk(granularity): + graph = MagicMock() + graph.astream = _stream_without_pause() + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = AgentState() + + chunks = await _run(service, granularity) + assert all((c.data or {}).get("status") != "interrupted" for c in chunks) + + +@pytest.mark.asyncio +async def test_full_granularity_relies_on_core_only(): + graph = MagicMock() + graph.astream = _stream_without_pause() + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = _paused_state() + + chunks = await _run(service, ResponseGranularity.FULL) + assert len(chunks) == 1 + assert isinstance(service, GraphService) + + +def _static_state(status, reason): + from tenxgraph.core.state.execution_state import ExecutionStatus + + state = AgentState() + state.set_interrupt("N1", reason, ExecutionStatus(status)) + return state + + +@pytest.mark.parametrize("granularity", [ResponseGranularity.LOW, ResponseGranularity.PARTIAL]) +@pytest.mark.parametrize( + ("status", "reason", "pause", "update_status"), + [ + ("interrupted_before", "interrupt_before: N1", "before", "invoking_node"), + ("interrupted_after", "interrupt_after: N1", "after", "node_invoked"), + ("interrupted_after", "remote_tool_call", "remote_tool", "node_invoked"), + ], +) +@pytest.mark.asyncio +async def test_static_pauses_rebuilt_from_checkpoint( + granularity, status, reason, pause, update_status +): + graph = MagicMock() + graph.astream = _stream_without_pause() + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = _static_state(status, reason) + + chunks = await _run(service, granularity) + last = chunks[-1] + assert last.event == StreamEvent.UPDATES + assert last.data["pause"] == pause + assert last.data["status"] == update_status + assert last.data["node"] == "N1" + + +@pytest.mark.parametrize("granularity", [ResponseGranularity.LOW, ResponseGranularity.PARTIAL]) +@pytest.mark.parametrize("pause", ["before", "after", "remote_tool"]) +@pytest.mark.asyncio +async def test_no_duplicate_static_pause_when_core_already_sent_it(granularity, pause): + sent = StreamChunk( + event=StreamEvent.UPDATES, data={"status": "node_invoked", "node": "N1", "pause": pause} + ) + + async def astream(*args, **kwargs): + yield sent + + graph = MagicMock() + graph.astream = astream + service = _service(graph, timeout=None) + service.checkpointer.aget_state.return_value = _static_state( + "interrupted_before" if pause == "before" else "interrupted_after", + "remote_tool_call" if pause == "remote_tool" else f"interrupt_{pause}: N1", + ) + + chunks = await _run(service, granularity) + assert len([c for c in chunks if (c.data or {}).get("pause") == pause]) == 1 + assert len(chunks) == 1 diff --git a/tests/unit_tests/test_threads_files_live_contract.py b/tests/unit_tests/test_threads_files_live_contract.py new file mode 100644 index 00000000..2e254b6a --- /dev/null +++ b/tests/unit_tests/test_threads_files_live_contract.py @@ -0,0 +1,180 @@ +"""X8 (has_more on thread lists), X3 (GET /v1/files listing), X10 (realtime_options).""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch + +import pytest +from tenxgraph.core.state import Message +from tenxgraph.storage.checkpointer import BaseCheckpointer +from tenxgraph.storage.media.storage.local_store import LocalFileMediaStore +from tenxgraph.storage.media.storage.memory_store import InMemoryMediaStore + +from tenxgraph_api.src.app.core.config.graph_config import WebSocketConfig +from tenxgraph_api.src.app.routers.checkpointer.services.checkpointer_service import ( + CheckpointerService, +) +from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService +from tenxgraph_api.src.app.routers.media import MediaService + + +def _checkpointer_service(listed): + cp = MagicMock(spec=BaseCheckpointer) + + async def alist(cfg, search, offset, limit): + return listed[(offset or 0) : (offset or 0) + limit] + + cp.alist_messages = AsyncMock(side_effect=alist) + cp.alist_threads = AsyncMock(side_effect=alist) + svc = CheckpointerService.__new__(CheckpointerService) + svc.checkpointer = cp + svc.settings = MagicMock() + return svc + + +class _T: + def __init__(self, i): + self.i = i + + def model_dump(self): + return {"thread_id": str(self.i)} + + +class TestHasMore: + @pytest.mark.asyncio + async def test_threads_full_page_with_more(self): + svc = _checkpointer_service([_T(i) for i in range(5)]) + res = await svc.list_threads({"user_id": "u"}, None, 0, 3) + assert len(res.threads) == 3 + assert (res.has_more, res.offset, res.limit) == (True, 0, 3) + + @pytest.mark.asyncio + async def test_threads_exact_last_page(self): + svc = _checkpointer_service([_T(i) for i in range(3)]) + res = await svc.list_threads({"user_id": "u"}, None, 0, 3) + assert res.has_more is False + + @pytest.mark.asyncio + async def test_threads_short_page(self): + svc = _checkpointer_service([_T(i) for i in range(2)]) + res = await svc.list_threads({"user_id": "u"}, None, 0, 3) + assert res.has_more is False + + @pytest.mark.asyncio + async def test_messages_second_page(self): + svc = _checkpointer_service([Message.text_message(str(i)) for i in range(7)]) + res = await svc.get_messages({"thread_id": "t"}, {"user_id": "u"}, None, 3, 3) + assert [m.text() for m in res.messages] == ["3", "4", "5"] + assert (res.has_more, res.offset) == (True, 3) + res = await svc.get_messages({"thread_id": "t"}, {"user_id": "u"}, None, 6, 3) + assert res.has_more is False + + +class _ListableMemoryStore(InMemoryMediaStore): + """Memory store with ``list_keys``, so the tests also pass against an older core.""" + + async def list_keys(self): + return list(self._data) + + +needs_core_list_keys = pytest.mark.skipif( + not hasattr(InMemoryMediaStore, "list_keys"), reason="core without BaseMediaStore.list_keys" +) + + +def _media(store): + svc = MediaService.__new__(MediaService) + svc._settings = SimpleNamespace(MEDIA_REQUIRE_OWNER=False) + svc._store = store + return svc + + +class TestListFiles: + @pytest.mark.asyncio + async def test_only_own_files_paged_newest_first(self): + store = _ListableMemoryStore() + for i, owner in enumerate(["alice", "bob", "alice", "alice", None]): + meta = {"filename": f"f{i}", "created_at": 100 + i} + if owner: + meta["owner_id"] = owner + await store.store(b"x", "text/plain", meta) + svc = _media(store) + + page, more = await svc.list_files("alice", 0, 2) + assert [f["filename"] for f in page] == ["f3", "f2"] + assert more is True + assert page[0]["url"] == f"/v1/files/{page[0]['file_id']}" + page, more = await svc.list_files("alice", 2, 2) + assert [f["filename"] for f in page] == ["f0"] + assert more is False + page, _ = await svc.list_files("carol", 0, 10) + assert page == [] + + @pytest.mark.asyncio + async def test_no_user_lists_everything(self): + store = _ListableMemoryStore() + await store.store(b"x", "text/plain", {"owner_id": "alice"}) + await store.store(b"x", "text/plain", {}) + page, _ = await _media(store).list_files(None, 0, 10) + assert len(page) == 2 + + @pytest.mark.asyncio + async def test_store_without_listing_raises(self): + store = SimpleNamespace(get_metadata=AsyncMock()) # no list_keys + with pytest.raises(NotImplementedError): + await _media(store).list_files("alice", 0, 10) + + @needs_core_list_keys + @pytest.mark.asyncio + async def test_local_store_lists_keys(self, tmp_path): + store = LocalFileMediaStore(str(tmp_path)) + k = await store.store(b"hi", "text/plain", {"owner_id": "alice", "filename": "a.txt"}) + assert await store.list_keys() == [k] + page, _ = await _media(store).list_files("alice", 0, 10) + assert page[0]["file_id"] == k and page[0]["size_bytes"] == 2 + + +class TestRealtimeOptions: + def _service(self, ws, live): + srv = GraphService.__new__(GraphService) + srv._graph = MagicMock() + srv._graph.generate_graph.return_value = { + "info": { + "node_count": 1, + "edge_count": 0, + "checkpointer": False, + "publisher": False, + "store": False, + }, + "nodes": [], + "edges": [], + } + srv.config = SimpleNamespace(websocket=ws) + return srv + + @pytest.mark.asyncio + async def test_live_graph_reports_options(self): + ws = WebSocketConfig.from_dict( + {"realtime_models": ["m1"], "realtime_client_overrides": ["voice", "system_prompt"]} + ) + srv = self._service(ws, True) + with patch.object(GraphService, "is_live_agent", new_callable=PropertyMock) as live: + live.return_value = True + res = await srv.graph_details() + assert res.info.realtime_options.model_dump() == { + "client_overrides": ["voice", "system_prompt"], + "models": ["m1"], + "model_selectable": True, + "tools_tags": True, + } + + @pytest.mark.asyncio + async def test_default_and_non_live(self): + ws = WebSocketConfig.from_dict({}) + srv = self._service(ws, True) + with patch.object(GraphService, "is_live_agent", new_callable=PropertyMock) as live: + live.return_value = True + opts = (await srv.graph_details()).info.realtime_options + assert opts.client_overrides == ["voice", "modalities", "vad"] + assert opts.model_selectable is False + live.return_value = False + assert (await srv.graph_details()).info.realtime_options is None diff --git a/tests/unit_tests/test_websocket_toggle.py b/tests/unit_tests/test_websocket_toggle.py index 0ddf6f73..3b972822 100644 --- a/tests/unit_tests/test_websocket_toggle.py +++ b/tests/unit_tests/test_websocket_toggle.py @@ -48,8 +48,8 @@ def test_config_rejects_non_bool_enabled() -> None: def test_disabled_leaves_out_both_websocket_routes_only(monkeypatch) -> None: monkeypatch.setattr(setup_router, "websocket_library_available", lambda: False) on, off = FastAPI(), FastAPI() - setup_router.init_routes(on, evals=False, websocket=None) - setup_router.init_routes(off, evals=False, websocket=False) + setup_router.init_routes(on, websocket=None) + setup_router.init_routes(off, websocket=False) assert _ws_paths(on) == WS_PATHS assert _ws_paths(off) == set() @@ -62,14 +62,14 @@ def test_disabled_leaves_out_both_websocket_routes_only(monkeypatch) -> None: def test_enabled_without_a_websocket_library_refuses_to_boot(monkeypatch) -> None: monkeypatch.setattr(setup_router, "websocket_library_available", lambda: False) with pytest.raises(RuntimeError, match=r"10xgraph-api\[websocket\]"): - setup_router.init_routes(FastAPI(), evals=False, websocket=True) + setup_router.init_routes(FastAPI(), websocket=True) def test_unset_without_a_websocket_library_warns_and_keeps_routes(monkeypatch, caplog) -> None: monkeypatch.setattr(setup_router, "websocket_library_available", lambda: False) app = FastAPI() with caplog.at_level(logging.WARNING, logger="tenxgraph_api"): - setup_router.init_routes(app, evals=False, websocket=None) + setup_router.init_routes(app, websocket=None) assert _ws_paths(app) == WS_PATHS assert "10xgraph-api[websocket]" in caplog.text @@ -79,7 +79,7 @@ def test_enabled_with_a_websocket_library_mounts_quietly(monkeypatch, caplog) -> monkeypatch.setattr(setup_router, "websocket_library_available", lambda: True) app = FastAPI() with caplog.at_level(logging.WARNING, logger="tenxgraph_api"): - setup_router.init_routes(app, evals=False, websocket=True) + setup_router.init_routes(app, websocket=True) assert _ws_paths(app) == WS_PATHS assert "websocket" not in caplog.text.lower() From 07dba2148e1902bda4014375a8e5f7ca4b650bda Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 10:29:23 +0600 Subject: [PATCH 04/16] feat: enhance production setup with recommended configurations - Updated `10xgraph init --template production` to scaffold a production-ready setup with JWT auth and Redis rate limiting. - Introduced production-readiness warnings in `10xgraph config` and `10xgraph build` to alert users about potential misconfigurations. - Redesigned `10xgraph config` editor for improved user experience and added validation for production readiness. - Modified `10xgraph build` command to include warnings for non-production-ready configurations. - Updated templates and example configurations to reflect new production requirements, including PostgreSQL and Redis integration. - Added comprehensive tests for production readiness checks and configuration validation. --- CHANGELOG.md | 14 + tenxgraph_api/cli/commands/build.py | 25 +- tenxgraph_api/cli/commands/config.py | 14 + tenxgraph_api/cli/commands/init.py | 83 ++--- tenxgraph_api/cli/config_editor/validation.py | 36 +++ tenxgraph_api/cli/templates/defaults.py | 285 ------------------ tenxgraph_api/cli/templates/prod/.env.example | 11 +- .../cli/templates/prod/graph/agent.py | 16 +- .../cli/templates/prod/tests/conftest.py | 3 + tests/cli/test_init_non_interactive.py | 13 +- tests/cli/test_init_prod.py | 97 ++++++ tests/cli/test_production_readiness.py | 170 +++++++++++ tests/unit_tests/test_config_editor.py | 3 +- tests/unit_tests/test_runs_config.py | 9 +- tests/unit_tests/test_token_revocation.py | 3 +- ...370c5d6283c04a12a1c7a0a97afc59d6.meta.json | 1 + .../0c/370c5d6283c04a12a1c7a0a97afc59d6.png | 2 + 17 files changed, 435 insertions(+), 350 deletions(-) create mode 100644 tests/cli/test_production_readiness.py create mode 100644 uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.meta.json create mode 100644 uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.png diff --git a/CHANGELOG.md b/CHANGELOG.md index 973dcd12..a2d63377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,20 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Changed +- **`10xgraph init --template production` scaffolds the recommended production setup.** Auth + defaults to JWT (with ownership authorization) and the rate limiter to Redis, in both the + prompts and `--non-interactive` (the rate-limit question is now asked whatever the auth + answer). `graph/agent.py` builds a `PgCheckpointer` from `DATABASE_URL` and `REDIS_URL`, with + the in-memory checkpointer left as commented-out code for local testing. `.env.example` always + has the Postgres and Redis blocks, and `pyproject.toml` depends on `10xgraph[,pg_checkpoint]` + and `10xgraph-api[...redis...]`. Quick-start is unchanged. +- **Production-readiness warnings.** `10xgraph config` (the editor and the terminal on start-up) + and `10xgraph build` now warn, from the JSON alone, when `auth` is off, when `authorization` + is `allow_all`, `default` or `none` with auth on, and when `rate_limit` is missing or + disabled. They are warnings only: auth off stays allowed and `build` still exits 0. The + checks live in `production_readiness_warnings` (`cli/config_editor/validation.py`). The + `10xgraph init` next steps now point to `10xgraph config` (both templates) and + `10xgraph build` (production); the unused `InitCommand._print_next_steps` was removed. - **`10xgraph config` editor redesign.** A sidebar layout with a "Get running" page first, a collapsible sidebar (Ctrl/Cmd+B), search across every setting, defaults shown as values with a per-field reset, and a read-only reference of the related `.env` variables per section plus a diff --git a/tenxgraph_api/cli/commands/build.py b/tenxgraph_api/cli/commands/build.py index 67283f8f..369cd06b 100644 --- a/tenxgraph_api/cli/commands/build.py +++ b/tenxgraph_api/cli/commands/build.py @@ -1,12 +1,20 @@ """Build command implementation.""" +import json import re import tomllib from pathlib import Path from typing import Any from tenxgraph_api.cli.commands import BaseCommand -from tenxgraph_api.cli.constants import DEFAULT_PORT, DEFAULT_PYTHON_VERSION, DEFAULT_SERVICE_NAME +from tenxgraph_api.cli.config_editor.validation import production_readiness_warnings +from tenxgraph_api.cli.constants import ( + DEFAULT_CONFIG_FILE, + DEFAULT_PORT, + DEFAULT_PYTHON_VERSION, + DEFAULT_SERVICE_NAME, +) +from tenxgraph_api.cli.core.config import resolve_default_config from tenxgraph_api.cli.core.validation import Validator from tenxgraph_api.cli.exceptions import DockerError, FileOperationError, ValidationError from tenxgraph_api.cli.templates.defaults import ( @@ -23,6 +31,19 @@ class BuildCommand(BaseCommand): """Command to generate Dockerfile and docker-compose.yml for the application.""" + def _warn_not_production_ready(self) -> None: + """Warn about settings in the project config that are risky in production.""" + config_path = resolve_default_config(DEFAULT_CONFIG_FILE) + try: + config = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, ValueError) as exc: + self.logger.debug("Skipping the production-readiness check: %s", exc) + return + if not isinstance(config, dict): + return + for issue in production_readiness_warnings(config): + self.output.warning(f"{issue['path']}: {issue['message']}") + def execute( # noqa: PLR0912, PLR0915 self, output_file: str = "Dockerfile", @@ -108,6 +129,8 @@ def execute( # noqa: PLR0912, PLR0915 "cannot start the server." ) + self._warn_not_production_ready() + with timeline.step("dockerfile") as step: dockerfile_content = generate_dockerfile_content( python_version=validated_python_version, diff --git a/tenxgraph_api/cli/commands/config.py b/tenxgraph_api/cli/commands/config.py index 254d5951..17595d97 100644 --- a/tenxgraph_api/cli/commands/config.py +++ b/tenxgraph_api/cli/commands/config.py @@ -1,11 +1,13 @@ """Config command implementation: a browser editor for ``10xgraph.json``.""" +import json import webbrowser from pathlib import Path from typing import Any from tenxgraph_api.cli.commands import BaseCommand from tenxgraph_api.cli.config_editor import ConfigEditorServer +from tenxgraph_api.cli.config_editor.validation import production_readiness_warnings from tenxgraph_api.cli.constants import DEFAULT_CONFIG_FILE, LEGACY_CONFIG_FILE from tenxgraph_api.cli.exceptions import ServerError @@ -51,6 +53,7 @@ def execute( }, next_steps=["Press Ctrl+C to stop the editor when you are done."], ) + self._print_readiness_warnings(config_path) if open_browser: webbrowser.open_new_tab(server.url) @@ -62,6 +65,17 @@ def execute( server.shutdown() return 0 + def _print_readiness_warnings(self, config_path: Path) -> None: + # Skip silently when the file is missing or not valid JSON; the editor reports that. + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return + if not isinstance(data, dict): + return + for issue in production_readiness_warnings(data): + self.output.warning(f"{issue['path']}: {issue['message']}") + def _file_note(config_path: Path, seeded: bool, seed_path: Path | None) -> str: if config_path.exists(): diff --git a/tenxgraph_api/cli/commands/init.py b/tenxgraph_api/cli/commands/init.py index fab78df9..17743820 100644 --- a/tenxgraph_api/cli/commands/init.py +++ b/tenxgraph_api/cli/commands/init.py @@ -80,15 +80,21 @@ def _template_vars(context: dict[str, Any]) -> dict[str, Any]: """Derive the template variables that depend on the user's answers. The production pyproject.toml lists only what the answers need: the provider SDK - through the core extra, PyJWT for JWT auth, redis for the Redis rate limiter, and - websockets for WebSocket streaming / realtime. The generated Dockerfile installs - exactly this list. + and the Postgres + Redis checkpointer (``pg_checkpoint``) through the core extras, + PyJWT for JWT auth, redis (always, the checkpointer and the default rate limiter use + it), and websockets for WebSocket streaming / realtime. The generated Dockerfile + installs exactly this list. Quick-start has no Postgres or Redis, so its core + requirement is the provider extra alone. """ provider = _PROVIDERS[context.get("provider", _DEFAULT_PROVIDER)] + is_prod = context.get("setup_type") == "production" + core_extras = [provider["extra"]] + if is_prod: + core_extras.append("pg_checkpoint") api_extras = [] if context.get("auth") == "jwt": api_extras.append("jwt") - if context.get("rate_limit") == "redis": + if is_prod or context.get("rate_limit") == "redis": api_extras.append("redis") if context.get("websocket"): api_extras.append("websocket") @@ -97,7 +103,7 @@ def _template_vars(context: dict[str, Any]) -> dict[str, Any]: "llm_model": provider["model"], "llm_sim_model": provider["sim_model"], "llm_api_key_env": provider["api_key_env"], - "core_requirement": f"10xgraph[{provider['extra']}]", + "core_requirement": f"10xgraph[{','.join(core_extras)}]", "api_requirement": ( f"10xgraph-api[{','.join(api_extras)}]" if api_extras else "10xgraph-api" ), @@ -216,6 +222,17 @@ def execute( # noqa: PLR0913 else [f'pip install "{_template_vars(context)["core_requirement"]}"'] + (['pip install "10xgraph-api[websocket]"'] if context.get("websocket") else []) ) + prod_steps = ( + [ + "Set DATABASE_URL (Postgres) and REDIS_URL (Redis) in .env.", + "graph/agent.py keeps a commented InMemoryCheckpointer for local testing " + "without Postgres and Redis.", + "10xgraph config to review auth, rate limiting and the other settings.", + "10xgraph build to generate the Dockerfile for deployment.", + ] + if is_prod + else ["10xgraph config to review the settings in your browser."] + ) self.output.completion_screen( "Project ready", f'"{agent_name}" was created successfully', @@ -230,6 +247,7 @@ def execute( # noqa: PLR0913 f"cd {base_path}", *install_step, f"Copy .env.example to .env and set {provider_info['api_key_env']}.", + *prod_steps, "10xgraph dev", ], ) @@ -269,14 +287,17 @@ def _non_interactive_context( field="name", ) - resolved_auth = (auth or "none").strip().lower() + is_prod_template = resolved_template == "production" + resolved_auth = (auth or ("jwt" if is_prod_template else "none")).strip().lower() if resolved_auth not in {"none", "jwt", "custom"}: raise ValidationError( f"Invalid auth mode '{resolved_auth}'. Choose none, jwt, or custom.", field="auth", ) - resolved_rate_limit = (rate_limit or "none").strip().lower() + resolved_rate_limit = ( + (rate_limit or ("redis" if is_prod_template else "none")).strip().lower() + ) if resolved_rate_limit not in {"none", "memory", "redis"}: raise ValidationError( f"Invalid rate-limit mode '{resolved_rate_limit}'. " @@ -317,7 +338,7 @@ def _non_interactive_context( # Prompts # ------------------------------------------------------------------ - def _prompt_user(self) -> dict | None: # noqa: PLR0911, PLR0912 + def _prompt_user(self) -> dict | None: # noqa: PLR0911 prompts = self.output.prompts() prompts.require_interactive( field="template", @@ -342,7 +363,7 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911, PLR0912 Choice( "production", "Production", - "Auth, rate limiting, evals, and tests scaffolded in", + "JWT auth, Postgres + Redis, rate limiting, evals, and tests scaffolded in", ), ], default="quick_start", @@ -393,15 +414,12 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911, PLR0912 Choice("jwt", "JWT", "Requires JWT_SECRET_KEY and JWT_ALGORITHM"), Choice("custom", "Custom", "Scaffolds a BaseAuth subclass under auth/"), ], - default="none", + default="jwt", ) if auth is None: return None context["auth"] = auth - if auth == "none": - return context - rate_limit = prompts.select( "Rate limiting backend?", [ @@ -409,7 +427,7 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911, PLR0912 Choice("memory", "Memory", "Per-process counters — single instance only"), Choice("redis", "Redis", "Shared counters — requires REDIS_URL"), ], - default="none", + default="redis", ) if rate_limit is None: return None @@ -497,41 +515,6 @@ def _print_summary(self, context: dict) -> None: title="Project summary", ) - def _print_next_steps(self, context: dict, is_prod: bool) -> None: - steps: list[tuple[str, str]] = [] - - steps.append(("10xgraph skills", "Install coding agent skills")) - - if is_prod: - steps.append(("pre-commit install", "Set up Git hooks (ruff, bandit, etc.)")) - - provider = _PROVIDERS[context.get("provider", _DEFAULT_PROVIDER)] - if not is_prod: - # Production lists the provider SDK in pyproject.toml already. - steps.append( - (f'pip install "10xgraph[{provider["extra"]}]"', "Add the AI provider library") - ) - steps.append( - ("cp .env.example .env", f"Copy env template, then set {provider['api_key_env']}") - ) - - if context["auth"] == "jwt": - steps.append(("# Set JWT_SECRET_KEY in .env", "Required for JWT auth to work")) - - if context["rate_limit"] == "redis": - steps.append(("# Set REDIS_URL in .env", "Required for Redis rate limiting")) - - steps.append(("10xgraph dev", "Launch your agent")) - - self.output.print_table( - ["#", "Command", "Purpose"], - [ - [str(index), command, description] - for index, (command, description) in enumerate(steps, 1) - ], - title="Next steps", - ) - # ------------------------------------------------------------------ # Config generation # ------------------------------------------------------------------ @@ -608,7 +591,7 @@ def _render(self, src: Path, context: dict, is_prod: bool) -> str: if src.name == ".env.example" and is_prod: content = _strip_env_blocks( content, - keep_redis=context["rate_limit"] == "redis", + keep_redis=True, keep_jwt=context["auth"] == "jwt", ) diff --git a/tenxgraph_api/cli/config_editor/validation.py b/tenxgraph_api/cli/config_editor/validation.py index 5caa4854..343863b2 100644 --- a/tenxgraph_api/cli/config_editor/validation.py +++ b/tenxgraph_api/cli/config_editor/validation.py @@ -27,6 +27,8 @@ _OBSERVABILITY_LEVELS = ("spans", "standard", "full") _USER_ID_MODES = ("raw", "hash", "mask") _MAX_PERCENT = 100 +# Built-in authorization names that skip the per-thread ownership check. +_OPEN_AUTHORIZATION = tuple(name for name in BUILTIN_AUTHORIZATION if name != "ownership") class _Collector: @@ -80,6 +82,7 @@ def validate_config(config: Any, base_dir: Path | None = None) -> list[Issue]: _check_remote_tools(config, out) _check_test(config, out) _check_evaluation(config, out) + out.issues.extend(production_readiness_warnings(config)) for key in config: if key not in KNOWN_TOP_LEVEL_KEYS: @@ -87,6 +90,39 @@ def validate_config(config: Any, base_dir: Path | None = None) -> list[Issue]: return out.issues +def production_readiness_warnings(config: dict) -> list[Issue]: + """Return warnings for settings that are fine locally but risky in production. + + Reads only the JSON, never ``.env`` or the environment. Auth stays the developer's + choice, so these are warnings, never errors. + """ + out = _Collector() + auth = config.get("auth") + if not auth: + out.warning( + "auth", + "Auth is off, so any caller can use every endpoint. Fine for local development; " + 'set "jwt" or a custom BaseAuth before production.', + ) + else: + authorization = config.get("authorization") + if isinstance(authorization, str) and authorization.strip().lower() in _OPEN_AUTHORIZATION: + out.warning( + "authorization", + f"'{authorization}' lets any authenticated user read and delete any thread. " + 'Use "ownership" or a custom backend in production.', + ) + + rate_limit = config.get("rate_limit") + if rate_limit is None or (isinstance(rate_limit, dict) and rate_limit.get("enabled") is False): + out.warning( + "rate_limit", + "Rate limiting is off, so one caller can exhaust the service. Enable it, with the " + "redis backend for more than one worker, before production.", + ) + return out.issues + + def has_errors(issues: list[Issue]) -> bool: return any(issue["level"] == "error" for issue in issues) diff --git a/tenxgraph_api/cli/templates/defaults.py b/tenxgraph_api/cli/templates/defaults.py index b8a99a29..5b66b9f7 100644 --- a/tenxgraph_api/cli/templates/defaults.py +++ b/tenxgraph_api/cli/templates/defaults.py @@ -46,291 +46,6 @@ indent=2, ) -# Template for the default react agent graph -DEFAULT_REACT_PY: Final[str] = ''' -""" -Graph-based React Agent Implementation - -This module implements a reactive agent system using 10xGraph's StateGraph. -The agent can interact with tools (like weather checking) and maintain conversation -state through a checkpointer. The graph orchestrates the flow between the main -agent logic and tool execution. - -Key Components: -- Weather tool: Demonstrates tool calling with dependency injection -- Main agent: AI-powered assistant that can use tools -- Graph flow: Conditional routing based on tool usage -- Checkpointer: Maintains conversation state across interactions - -Architecture: -The system uses a state graph with two main nodes: -1. MAIN: Processes user input and generates AI responses -2. TOOL: Executes tool calls when requested by the AI - -The graph conditionally routes between these nodes based on whether -the AI response contains tool calls. Conversation history is maintained -through the checkpointer, allowing for multi-turn conversations. - -Tools are defined as functions with JSON schema docstrings that describe -their interface for the AI model. The ToolNode automatically extracts -these schemas for tool selection. - -Dependencies: -- 10xGraph: For graph and state management -- LiteLLM: For AI model interactions -- InjectQ: For dependency injection -- Python logging: For debug and info messages -""" - -import logging -from typing import Any - -from tenxgraph.adapters.llm.model_response_converter import ModelResponseConverter -from tenxgraph.checkpointer import InMemoryCheckpointer -from tenxgraph.graph import StateGraph, ToolNode -from tenxgraph.state.agent_state import AgentState -from tenxgraph.utils.callbacks import CallbackManager -from tenxgraph.utils.constants import END -from tenxgraph.utils.converter import convert_messages -from dotenv import load_dotenv -from injectq import Inject -from litellm import acompletion - - -# Configure logging for the module -logging.basicConfig( - level=logging.INFO, - format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", - handlers=[logging.StreamHandler()], -) -logger = logging.getLogger(__name__) - -# Load environment variables from .env file -load_dotenv() - -# Initialize in-memory checkpointer for maintaining conversation state -checkpointer = InMemoryCheckpointer() - - -""" -Note: The docstring below will be used as the tool description and it will be -passed to the AI model for tool selection, so keep it relevant and concise. -This function will be converted to a tool with the following schema: -[ - { - 'type': 'function', - 'function': { - 'name': 'get_weather', - 'description': 'Retrieve current weather information for a specified location.', - 'parameters': { - 'type': 'object', - 'properties': { - 'location': {'type': 'string'} - }, - 'required': ['location'] - } - } - } - ] - -Parameters like tool_call_id, state, and checkpointer are injected automatically -by InjectQ when the tool is called by the agent. -Available injected parameters: -The following parameters are automatically injected by InjectQ when the tool is called, -but need to keep them as same name and type for proper injection: -- tool_call_id: Unique ID for the tool call -- state: Current AgentState containing conversation context -- config: Configuration dictionary passed during graph invocation - -Below fields need to be used with Inject[] to get the instances: -- context_manager: ContextManager instance for managing context, like trimming -- publisher: Publisher instance for publishing events and logs -- checkpointer: InMemoryCheckpointer instance for state management -- store: InMemoryStore instance for temporary data storage -- callback: CallbackManager instance for handling callbacks - -""" - - -def get_weather( - location: str, - tool_call_id: str, - state: AgentState, - checkpointer: InMemoryCheckpointer = Inject[InMemoryCheckpointer], -) -> str: - """Retrieve current weather information for a specified location.""" - # Demonstrate access to injected parameters - logger.debug("***** Checkpointer instance: %s", checkpointer) - if tool_call_id: - logger.debug("Tool call ID: %s", tool_call_id) - if state and hasattr(state, "context"): - logger.debug("Number of messages in context: %d", len(state.context)) - - # Mock weather response - in production, this would call a real weather API - return f"The weather in {location} is sunny" - - -# Create a tool node containing all available tools -tool_node = ToolNode([get_weather]) - - -async def main_agent( - state: AgentState, - config: dict, - checkpointer: InMemoryCheckpointer = Inject[InMemoryCheckpointer], - callback: CallbackManager = Inject[CallbackManager], -) -> Any: - """ - Main agent logic that processes user messages and generates responses. - - This function implements the core AI agent behavior, handling both regular - conversation and tool-augmented responses. It uses LiteLLM for AI completion - and can access conversation history through the checkpointer. - - Args: - state: Current agent state containing conversation context - config: Configuration dictionary containing thread_id and other settings - checkpointer: Checkpointer for retrieving conversation history (injected) - callback: Callback manager for handling events (injected) - - Returns: - dict: AI completion response containing the agent's reply - - The agent follows this logic: - 1. If the last message was a tool result, generate a final response without tools - 2. Otherwise, generate a response with available tools for potential tool usage - """ - # System prompt defining the agent's role and capabilities - system_prompt = """ - You are a helpful assistant. - Your task is to assist the user in finding information and answering questions. - You have access to various tools that can help you provide accurate information. - """ - - # Convert state messages to the format expected by the AI model - messages = convert_messages( - system_prompts=[{"role": "system", "content": system_prompt}], - state=state, - ) - - # Retrieve conversation history from checkpointer - try: - thread_messages = await checkpointer.aget_thread({"thread_id": config["thread_id"]}) - logger.debug("Messages from checkpointer: %s", thread_messages) - except Exception as e: - logger.warning("Could not retrieve thread messages: %s", e) - thread_messages = [] - - # Log injected dependencies for debugging - logger.debug("Checkpointer in main_agent: %s", checkpointer) - logger.debug("CallbackManager in main_agent: %s", callback) - - # Placeholder for MCP (Model Context Protocol) tools - # These would be additional tools from external sources - mcp_tools = [] - is_stream = config.get("is_stream", False) - - # Determine response strategy based on conversation context - if state.context and len(state.context) > 0 and state.context[-1].role == "tool": - # Last message was a tool result - generate final response without tools - logger.info("Generating final response after tool execution") - response = await acompletion( - model="gemini/gemini-2.0-flash-exp", # Updated model name - messages=messages, - stream=is_stream, - ) - else: - # Regular response with tools available for potential usage - logger.info("Generating response with tools available") - tools = await tool_node.all_tools() - response = await acompletion( - model="gemini/gemini-2.0-flash-exp", # Updated model name - messages=messages, - tools=tools + mcp_tools, - stream=is_stream, - ) - - return ModelResponseConverter( - response, - converter="litellm", - ) - - -def should_use_tools(state: AgentState) -> str: - """ - Determine the next step in the graph execution based on the current state. - - This routing function decides whether to continue with tool execution, - end the conversation, or proceed with the main agent logic. - - Args: - state: Current agent state containing the conversation context - - Returns: - str: Next node to execute ("TOOL" or END constant) - - Routing Logic: - - If last message is from assistant and contains tool calls -> "TOOL" - - If last message is a tool result -> END (conversation complete) - - Otherwise -> END (default fallback) - """ - if not state.context or len(state.context) == 0: - return END - - last_message = state.context[-1] - if not last_message: - return END - - # Check if assistant wants to use tools - if ( - hasattr(last_message, "tools_calls") - and last_message.tools_calls - and len(last_message.tools_calls) > 0 - and last_message.role == "assistant" - ): - logger.debug("Routing to TOOL node for tool execution") - return "TOOL" - - # Check if we just received tool results - if last_message.role == "tool": - logger.info("Tool execution complete, ending conversation") - return END - - # Default case: end conversation - logger.debug("Default routing: ending conversation") - return END - - -# Initialize the state graph for orchestrating agent flow -graph = StateGraph() - -# Add nodes to the graph -graph.add_node("MAIN", main_agent) # Main agent processing node -graph.add_node("TOOL", tool_node) # Tool execution node - -# Define conditional edges from MAIN node -# Routes to TOOL if tools should be used, otherwise ends -graph.add_conditional_edges( - "MAIN", - should_use_tools, - {"TOOL": "TOOL", END: END}, -) - -# Define edge from TOOL back to MAIN for continued conversation -graph.add_edge("TOOL", "MAIN") - -# Set the entry point for graph execution -graph.set_entry_point("MAIN") - -# Compile the graph with checkpointer for state management -app = graph.compile( - checkpointer=checkpointer, -) - - - -''' - # Production templates (mirroring root repo tooling for convenience) DEFAULT_PRE_COMMIT: Final[str] = """repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/tenxgraph_api/cli/templates/prod/.env.example b/tenxgraph_api/cli/templates/prod/.env.example index 186aab6d..301c0184 100644 --- a/tenxgraph_api/cli/templates/prod/.env.example +++ b/tenxgraph_api/cli/templates/prod/.env.example @@ -58,12 +58,17 @@ SNOWFLAKE_WORKER_ID=2 ################################# SENTRY_DSN="" -##[IF_REDIS]## +################################# +###### Postgres Config ########## +################################# +# Durable thread state (PgCheckpointer). Local example value; replace it in production. +DATABASE_URL="postgresql://postgres:postgres@localhost:5432/agent" + ################################# ###### Redis Config ############# ################################# -REDIS_URL="" -##[/IF_REDIS]## +# Hot thread cache, rate limiting and run locks. Local example value; replace it in production. +REDIS_URL="redis://localhost:6379/0" ##[IF_JWT]## ################################# diff --git a/tenxgraph_api/cli/templates/prod/graph/agent.py b/tenxgraph_api/cli/templates/prod/graph/agent.py index 505108a2..5b20be2e 100644 --- a/tenxgraph_api/cli/templates/prod/graph/agent.py +++ b/tenxgraph_api/cli/templates/prod/graph/agent.py @@ -1,8 +1,9 @@ +import os from datetime import datetime from tenxgraph.core.state import MessageContextManager from tenxgraph.prebuilt.agent import ReactAgent -from tenxgraph.storage.checkpointer import InMemoryCheckpointer +from tenxgraph.storage.checkpointer import PgCheckpointer from dotenv import load_dotenv from injectq import InjectQ @@ -13,7 +14,18 @@ load_dotenv() -checkpointer = InMemoryCheckpointer() +# Durable thread state: Postgres keeps every thread, Redis caches the hot ones. +# Set DATABASE_URL and REDIS_URL in .env (see .env.example). +checkpointer = PgCheckpointer( + postgres_dsn=os.getenv("DATABASE_URL"), + redis_url=os.getenv("REDIS_URL"), +) + +# Local testing without Postgres and Redis: comment out the block above and use this. +# Threads live only in this process (lost on restart, not shared between workers), +# so switch back to PgCheckpointer once you are ready to deploy. +# from tenxgraph.storage.checkpointer import InMemoryCheckpointer +# checkpointer = InMemoryCheckpointer() # Dependency injection container; 10xgraph.json points at it ("injectq": "graph.agent:container"). container = InjectQ.get_instance() diff --git a/tenxgraph_api/cli/templates/prod/tests/conftest.py b/tenxgraph_api/cli/templates/prod/tests/conftest.py index 72ba7062..1f1d0b94 100644 --- a/tenxgraph_api/cli/templates/prod/tests/conftest.py +++ b/tenxgraph_api/cli/templates/prod/tests/conftest.py @@ -4,3 +4,6 @@ os.environ.setdefault("$llm_api_key_env", "test-key") +# PgCheckpointer connects lazily, so these only need to be well-formed to import the graph. +os.environ.setdefault("DATABASE_URL", "postgresql://postgres:postgres@localhost:5432/agent") +os.environ.setdefault("REDIS_URL", "redis://localhost:6379/0") diff --git a/tests/cli/test_init_non_interactive.py b/tests/cli/test_init_non_interactive.py index 2fbb1311..20115993 100644 --- a/tests/cli/test_init_non_interactive.py +++ b/tests/cli/test_init_non_interactive.py @@ -77,7 +77,7 @@ def test_production_dependencies_follow_the_answers(tmp_path: Path) -> None: non_interactive=True, ) assert result == 0 - assert _dependencies(target) == ["10xgraph[openai]", "10xgraph-api[jwt,redis]"] + assert _dependencies(target) == ["10xgraph[openai,pg_checkpoint]", "10xgraph-api[jwt,redis]"] import json @@ -93,7 +93,7 @@ def test_production_dependencies_follow_the_answers(tmp_path: Path) -> None: assert "$api_requirement" not in (target / "pyproject.toml").read_text() -def test_production_without_jwt_or_redis_installs_neither(tmp_path: Path) -> None: +def test_production_without_jwt_installs_redis_but_not_jwt(tmp_path: Path) -> None: target = tmp_path / "open-agent" command = InitCommand(OutputFormatter(stream=io.StringIO())) result = command.execute( @@ -106,7 +106,7 @@ def test_production_without_jwt_or_redis_installs_neither(tmp_path: Path) -> Non non_interactive=True, ) assert result == 0 - assert _dependencies(target) == ["10xgraph[anthropic]", "10xgraph-api"] + assert _dependencies(target) == ["10xgraph[anthropic,pg_checkpoint]", "10xgraph-api[redis]"] assert 'model="claude-sonnet-5-5"' in (target / "graph" / "agent.py").read_text() @@ -135,7 +135,7 @@ def test_websocket_off_by_default_and_not_installed(tmp_path: Path) -> None: path=str(target), template="production", provider="openai", non_interactive=True ) assert result == 0 - assert _dependencies(target) == ["10xgraph[openai]", "10xgraph-api"] + assert _dependencies(target) == ["10xgraph[openai,pg_checkpoint]", "10xgraph-api[jwt,redis]"] config = json.loads((target / "10xgraph.json").read_text()) assert config["websocket"] == {"enabled": False} @@ -154,7 +154,10 @@ def test_websocket_on_adds_the_extra_and_enables_the_routes(tmp_path: Path) -> N non_interactive=True, ) assert result == 0 - assert _dependencies(target) == ["10xgraph[anthropic]", "10xgraph-api[jwt,websocket]"] + assert _dependencies(target) == [ + "10xgraph[anthropic,pg_checkpoint]", + "10xgraph-api[jwt,redis,websocket]", + ] config = json.loads((target / "10xgraph.json").read_text()) assert config["websocket"] == {"enabled": True} diff --git a/tests/cli/test_init_prod.py b/tests/cli/test_init_prod.py index 60d774f0..374a335e 100644 --- a/tests/cli/test_init_prod.py +++ b/tests/cli/test_init_prod.py @@ -75,3 +75,100 @@ def test_init_prod_skips_binary_template_artifacts() -> None: {}, True, ) + + +def test_production_template_defaults_to_recommended_setup(tmp_path: Path) -> None: + """With no answers, production scaffolds JWT, ownership, Postgres + Redis and rate limits.""" + import json + import tomllib + + target = tmp_path / "default-prod" + command = InitCommand(OutputFormatter(stream=io.StringIO())) + result = command.execute( + path=str(target), agent_name="Default Prod", template="production", non_interactive=True + ) + assert result == 0 + + config = json.loads((target / "10xgraph.json").read_text()) + assert config["auth"] == "jwt" + assert config["authorization"] == "ownership" + assert config["rate_limit"]["enabled"] is True + assert config["rate_limit"]["backend"] == "redis" + + agent_py = (target / "graph" / "agent.py").read_text() + assert "PgCheckpointer(" in agent_py + memory_lines = [line for line in agent_py.splitlines() if "InMemoryCheckpointer" in line] + assert memory_lines + assert all(line.lstrip().startswith("#") for line in memory_lines) + + env_example = (target / ".env.example").read_text() + assert "DATABASE_URL=" in env_example + assert "REDIS_URL=" in env_example + + dependencies = tomllib.loads((target / "pyproject.toml").read_text())["project"]["dependencies"] + assert any("pg_checkpoint" in dep for dep in dependencies) + assert any(dep.startswith("10xgraph-api[") and "redis" in dep for dep in dependencies) + + +def test_interactive_production_prompts_default_to_jwt_and_redis( + monkeypatch, tmp_path: Path +) -> None: + """Accepting every default in the production flow gives JWT and the Redis limiter.""" + + class _AcceptDefaults: + def require_interactive(self, **_kwargs) -> None: + return None + + def text(self, _message, default="", **_kwargs): + return default + + def confirm(self, _message, default=False, **_kwargs): + return default + + def select(self, message, _choices, default=None, **_kwargs): + return "production" if message == "Which project template?" else default + + output = SilentOutput() + monkeypatch.setattr(output, "prompts", lambda: _AcceptDefaults()) + + context = InitCommand(output=output)._prompt_user() + + assert context is not None + assert context["setup_type"] == "production" + assert context["auth"] == "jwt" + assert context["rate_limit"] == "redis" + + +def test_production_graph_builds_a_pg_checkpointer_without_services(tmp_path: Path) -> None: + """The scaffolded agent imports with Postgres and Redis down (connections are lazy).""" + import subprocess + import sys + + import pytest + + pytest.importorskip("asyncpg") + pytest.importorskip("redis") + pytest.importorskip("google.genai") + target = tmp_path / "pg-agent" + command = InitCommand(OutputFormatter(stream=io.StringIO())) + assert ( + command.execute( + path=str(target), agent_name="Pg Agent", template="production", non_interactive=True + ) + == 0 + ) + script = ( + "import tests.conftest\n" + "from graph.agent import checkpointer\n" + "print(type(checkpointer).__name__)\n" + ) + done = subprocess.run( + [sys.executable, "-c", script], + cwd=target, + capture_output=True, + text=True, + timeout=120, + check=False, + ) + assert done.returncode == 0, done.stderr + assert done.stdout.strip().endswith("PgCheckpointer") diff --git a/tests/cli/test_production_readiness.py b/tests/cli/test_production_readiness.py new file mode 100644 index 00000000..3521d197 --- /dev/null +++ b/tests/cli/test_production_readiness.py @@ -0,0 +1,170 @@ +"""Production-readiness warnings in `10xgraph config`, `10xgraph build` and `10xgraph init`.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +import pytest + +from tenxgraph_api.cli.commands import config as config_module +from tenxgraph_api.cli.commands.build import BuildCommand +from tenxgraph_api.cli.commands.config import ConfigCommand +from tenxgraph_api.cli.commands.init import InitCommand +from tenxgraph_api.cli.config_editor.validation import ( + production_readiness_warnings, + validate_config, +) +from tenxgraph_api.cli.core.output import OutputFormatter + + +REDIS_RATE_LIMIT = {"enabled": True, "backend": "redis", "redis": {"url": "redis://x"}} +READY = {"agent": "graph.react:app", "auth": "jwt", "rate_limit": REDIS_RATE_LIMIT} +AUTH_OFF_TEXT = "Auth is off" + + +def _paths(issues: list[dict], level: str = "warning") -> list[str]: + return [i["path"] for i in issues if i["level"] == level] + + +def test_validate_config_warns_when_auth_and_rate_limit_are_missing() -> None: + paths = _paths(validate_config({"agent": "graph.react:app"})) + assert "auth" in paths + assert "rate_limit" in paths + + +def test_ready_config_has_no_readiness_warnings() -> None: + assert production_readiness_warnings(READY) == [] + assert not [i for i in validate_config(READY) if i["level"] == "warning"] + + +@pytest.mark.parametrize("value", ["allow_all", "default", "none"]) +def test_open_authorization_with_auth_warns(value: str) -> None: + issues = validate_config({**READY, "authorization": value}) + assert _paths(issues) == ["authorization"] + + +@pytest.mark.parametrize("value", [None, "ownership", "auth.perm:Backend"]) +def test_safe_authorization_does_not_warn(value: str | None) -> None: + config = {**READY, "authorization": value} + assert production_readiness_warnings(config) == [] + + +def test_disabled_rate_limit_warns() -> None: + config = {**READY, "rate_limit": {"enabled": False}} + assert _paths(production_readiness_warnings(config)) == ["rate_limit"] + + +# --------------------------------------------------------------------------- +# build +# --------------------------------------------------------------------------- + + +def _build(tmp_path: Path, monkeypatch, config: dict | str | None) -> tuple[int, str]: + monkeypatch.chdir(tmp_path) + if isinstance(config, dict): + (tmp_path / "10xgraph.json").write_text(json.dumps(config), encoding="utf-8") + elif isinstance(config, str): + (tmp_path / "10xgraph.json").write_text(config, encoding="utf-8") + stream = io.StringIO() + code = BuildCommand(OutputFormatter(stream=stream)).execute(force=True) + assert (tmp_path / "Dockerfile").exists() + return code, stream.getvalue() + + +def test_build_warns_when_auth_is_off(tmp_path: Path, monkeypatch) -> None: + code, text = _build(tmp_path, monkeypatch, {"agent": "graph.react:app", "auth": None}) + assert code == 0 + assert AUTH_OFF_TEXT in text + + +def test_build_is_quiet_for_a_ready_config(tmp_path: Path, monkeypatch) -> None: + code, text = _build(tmp_path, monkeypatch, READY) + assert code == 0 + assert AUTH_OFF_TEXT not in text + + +@pytest.mark.parametrize("config", [None, "{not json"]) +def test_build_skips_the_check_without_a_usable_config( + tmp_path: Path, monkeypatch, config: str | None +) -> None: + code, text = _build(tmp_path, monkeypatch, config) + assert code == 0 + assert AUTH_OFF_TEXT not in text + + +# --------------------------------------------------------------------------- +# init +# --------------------------------------------------------------------------- + + +def test_production_init_next_steps_point_to_config_and_build(tmp_path: Path) -> None: + stream = io.StringIO() + result = InitCommand(OutputFormatter(stream=stream)).execute( + path=str(tmp_path / "svc"), + agent_name="Svc", + template="production", + non_interactive=True, + ) + assert result == 0 + text = stream.getvalue() + assert "10xgraph config" in text + assert "10xgraph build" in text + + +def test_quick_start_init_mentions_config_but_not_build(tmp_path: Path) -> None: + stream = io.StringIO() + result = InitCommand(OutputFormatter(stream=stream)).execute( + path=str(tmp_path / "svc"), + agent_name="Svc", + template="quick-start", + non_interactive=True, + ) + assert result == 0 + text = stream.getvalue() + assert "10xgraph config" in text + assert "10xgraph build" not in text + + +# --------------------------------------------------------------------------- +# config command +# --------------------------------------------------------------------------- + + +class _StubServer: + url = "http://127.0.0.1:1" + + def __init__(self, path: Path, port: int = 0, seed_path: Path | None = None) -> None: + self.store = type("Store", (), {"seeded": False})() + + def serve_forever(self) -> None: + return None + + def shutdown(self) -> None: + return None + + +def _run_config(tmp_path: Path, monkeypatch, content: str | None) -> str: + monkeypatch.setattr(config_module, "ConfigEditorServer", _StubServer) + path = tmp_path / "10xgraph.json" + if content is not None: + path.write_text(content, encoding="utf-8") + stream = io.StringIO() + code = ConfigCommand(OutputFormatter(stream=stream)).execute( + config=str(path), open_browser=False + ) + assert code == 0 + return stream.getvalue() + + +def test_config_command_prints_readiness_warnings(tmp_path: Path, monkeypatch) -> None: + text = _run_config(tmp_path, monkeypatch, json.dumps({"agent": "graph.react:app"})) + assert AUTH_OFF_TEXT in text + + +@pytest.mark.parametrize("content", [None, "{not json"]) +def test_config_command_skips_warnings_without_a_usable_file( + tmp_path: Path, monkeypatch, content: str | None +) -> None: + assert AUTH_OFF_TEXT not in _run_config(tmp_path, monkeypatch, content) diff --git a/tests/unit_tests/test_config_editor.py b/tests/unit_tests/test_config_editor.py index ede03b37..ddf2a94d 100644 --- a/tests/unit_tests/test_config_editor.py +++ b/tests/unit_tests/test_config_editor.py @@ -115,7 +115,8 @@ def test_logfire_console_does_not_offer_true() -> None: def test_websocket_seed_does_not_force_enabled() -> None: section = next(s for s in build_schema()["sections"] if s["id"] == "websocket") assert "enabled" not in section["default"] - assert not validate_config({"agent": "a:b", "websocket": section["default"]}) + issues = validate_config({"agent": "a:b", "websocket": section["default"]}) + assert not [i for i in issues if i["path"].startswith("websocket")] def test_number_field_extensions() -> None: diff --git a/tests/unit_tests/test_runs_config.py b/tests/unit_tests/test_runs_config.py index c07a9dc6..144d5c30 100644 --- a/tests/unit_tests/test_runs_config.py +++ b/tests/unit_tests/test_runs_config.py @@ -125,7 +125,11 @@ def test_graph_config_property(tmp_path): def test_config_editor_knows_and_validates_runs(): base = {"agent": "graph:app"} - assert validate({**base, "runs": {"thread_ids": "server"}}) == [] + assert not [ + i + for i in validate({**base, "runs": {"thread_ids": "server"}}) + if i["path"].startswith("runs") + ] issues = validate({**base, "runs": {"thread_ids": "nobody"}}) assert [i["path"] for i in issues if i["level"] == "error"] == ["runs"] @@ -220,6 +224,7 @@ def test_loader_logs_when_the_mode_cannot_apply(caplog): def test_config_editor_validates_max_parallel_tool_calls(): base = {"agent": "graph:app"} - assert validate({**base, "runs": {"max_parallel_tool_calls": 5}}) == [] + issues = validate({**base, "runs": {"max_parallel_tool_calls": 5}}) + assert not [i for i in issues if i["path"].startswith("runs")] issues = validate({**base, "runs": {"max_parallel_tool_calls": 0}}) assert [i["path"] for i in issues if i["level"] == "error"] == ["runs"] diff --git a/tests/unit_tests/test_token_revocation.py b/tests/unit_tests/test_token_revocation.py index 1f3bed9e..a3e9052b 100644 --- a/tests/unit_tests/test_token_revocation.py +++ b/tests/unit_tests/test_token_revocation.py @@ -242,7 +242,8 @@ def test_config_editor_validates_token_revocation(): from tenxgraph_api.cli.config_editor import validate_config base = {"agent": "graph:app", "auth": "jwt"} - assert validate_config({**base, "token_revocation": "memory"}) == [] + issues = validate_config({**base, "token_revocation": "memory"}) + assert not [i for i in issues if i["path"].startswith("token_revocation")] assert any( i["level"] == "error" for i in validate_config({**base, "token_revocation": "bad path"}) ) diff --git a/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.meta.json b/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.meta.json new file mode 100644 index 00000000..9901667d --- /dev/null +++ b/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.meta.json @@ -0,0 +1 @@ +{"mime_type": "image/png", "size_bytes": 8, "filename": "p.png", "created_at": 1793187528} \ No newline at end of file diff --git a/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.png b/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.png new file mode 100644 index 00000000..0dd1608e --- /dev/null +++ b/uploads/37/0c/370c5d6283c04a12a1c7a0a97afc59d6.png @@ -0,0 +1,2 @@ +‰PNG + From 5621f03a8b003af1a56ce5d6e524e28c96fedfc4 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 16:06:47 +0600 Subject: [PATCH 05/16] Add unit tests for resume handling, run entry guards, and shutdown readiness - Implement tests for resume functionality requiring user_id in `test_resume_needs_user_id.py`. - Add tests for run entry guards and WebSocket lifecycle in `test_run_entry_guards.py`. - Create tests to ensure proper closure of run traces in `test_run_trace_close.py`. - Introduce tests for shutdown grace periods and readiness checks in `test_shutdown_readiness.py`. - Add tests for stream protocol behavior including keep-alive and terminal chunk handling in `test_stream_protocol.py`. - Implement tests for thread naming on the stream path in `test_stream_thread_naming.py`. --- CLAUDE.md | 2 +- tenxgraph_api/cli/commands/api.py | 28 ++ tenxgraph_api/cli/commands/audit.py | 29 +- tenxgraph_api/cli/commands/build.py | 7 + tenxgraph_api/cli/commands/eval.py | 17 +- tenxgraph_api/cli/commands/init.py | 69 ++- tenxgraph_api/cli/commands/revoke.py | 74 ++++ tenxgraph_api/cli/commands/skills.py | 51 ++- tenxgraph_api/cli/config_editor/schema.py | 3 +- tenxgraph_api/cli/config_editor/validation.py | 20 + tenxgraph_api/cli/core/validation.py | 7 +- tenxgraph_api/cli/main.py | 38 +- tenxgraph_api/cli/templates/defaults.py | 46 +- .../cli/templates/prod/10xgraph.json | 3 +- .../10xgraph/references/api-configuration.md | 7 +- .../references/dependency-injection.md | 3 +- .../10xgraph/references/rate-limiting.md | 4 +- .../src/app/core/auth/authorization.py | 5 +- tenxgraph_api/src/app/core/auth/revocation.py | 146 ++++++- tenxgraph_api/src/app/core/concurrency.py | 82 +++- .../src/app/core/config/graph_config.py | 2 +- tenxgraph_api/src/app/core/config/settings.py | 23 + .../src/app/core/config/setup_logs.py | 20 +- .../src/app/core/config/setup_middleware.py | 9 +- .../src/app/core/exceptions/handle_errors.py | 37 +- .../src/app/core/exceptions/server_errors.py | 159 +++++++ .../app/core/middleware/rate_limit/redis.py | 4 +- .../app/core/middleware/security_headers.py | 3 + tenxgraph_api/src/app/core/redis_client.py | 46 ++ tenxgraph_api/src/app/core/shutdown.py | 114 +++++ .../src/app/core/utils/log_sanitizer.py | 83 +++- tenxgraph_api/src/app/loader.py | 13 +- tenxgraph_api/src/app/main.py | 17 +- .../src/app/routers/ag_ui/converter.py | 5 +- tenxgraph_api/src/app/routers/ag_ui/router.py | 13 +- tenxgraph_api/src/app/routers/graph/router.py | 381 ++++++++++++---- .../src/app/routers/graph/run_guard.py | 51 +++ .../src/app/routers/graph/run_limits.py | 3 + .../routers/graph/services/graph_service.py | 118 ++++- .../src/app/routers/graph/thread_lock.py | 33 +- .../src/app/routers/ping/readiness.py | 38 +- tenxgraph_api/src/app/routers/ping/router.py | 14 +- .../routers/store/services/store_service.py | 27 +- .../src/app/utils/snowflake_id_generator.py | 91 ++-- .../src/app/utils/telemetry_store.py | 14 +- tenxgraph_api/src/app/uvicorn_worker.py | 80 ++++ tests/cli/test_cli_commands_ops.py | 8 + tests/cli/test_eval_command.py | 11 + tests/cli/test_group6_cli_templates.py | 93 ++++ tests/cli/test_group6_small_cli_bugs.py | 92 ++++ tests/cli/test_production_readiness.py | 44 +- tests/unit_tests/store/test_store_service.py | 31 ++ tests/unit_tests/test_cli_validation.py | 26 +- .../test_client_config_isolation.py | 3 +- tests/unit_tests/test_concurrency_limiter.py | 12 +- .../unit_tests/test_error_context_details.py | 2 +- tests/unit_tests/test_graph_service.py | 12 +- tests/unit_tests/test_group6_logging.py | 157 +++++++ tests/unit_tests/test_group6_small_fixes.py | 108 +++++ tests/unit_tests/test_handle_errors.py | 6 +- tests/unit_tests/test_lock_lease_integrity.py | 96 ++++ tests/unit_tests/test_middleware_order.py | 75 ++++ tests/unit_tests/test_production_surface.py | 2 +- tests/unit_tests/test_readiness.py | 6 +- tests/unit_tests/test_redis_hardening.py | 249 +++++++++++ tests/unit_tests/test_resume_needs_user_id.py | 94 ++++ tests/unit_tests/test_run_entry_guards.py | 410 ++++++++++++++++++ tests/unit_tests/test_run_limits.py | 1 + tests/unit_tests/test_run_trace_close.py | 93 ++++ tests/unit_tests/test_shutdown_readiness.py | 230 ++++++++++ .../unit_tests/test_snowflake_id_generator.py | 40 ++ tests/unit_tests/test_stream_protocol.py | 122 ++++++ tests/unit_tests/test_stream_thread_naming.py | 109 +++++ tests/unit_tests/test_thread_lock.py | 61 +++ tests/unit_tests/test_token_revocation.py | 14 +- tests/unit_tests/test_websocket_graph.py | 91 +++- tests/unit_tests/test_websocket_realtime.py | 5 + 77 files changed, 4023 insertions(+), 319 deletions(-) create mode 100644 tenxgraph_api/cli/commands/revoke.py create mode 100644 tenxgraph_api/src/app/core/exceptions/server_errors.py create mode 100644 tenxgraph_api/src/app/core/redis_client.py create mode 100644 tenxgraph_api/src/app/core/shutdown.py create mode 100644 tenxgraph_api/src/app/routers/graph/run_guard.py create mode 100644 tenxgraph_api/src/app/uvicorn_worker.py create mode 100644 tests/cli/test_group6_cli_templates.py create mode 100644 tests/cli/test_group6_small_cli_bugs.py create mode 100644 tests/unit_tests/test_group6_logging.py create mode 100644 tests/unit_tests/test_group6_small_fixes.py create mode 100644 tests/unit_tests/test_lock_lease_integrity.py create mode 100644 tests/unit_tests/test_redis_hardening.py create mode 100644 tests/unit_tests/test_resume_needs_user_id.py create mode 100644 tests/unit_tests/test_run_entry_guards.py create mode 100644 tests/unit_tests/test_run_trace_close.py create mode 100644 tests/unit_tests/test_shutdown_readiness.py create mode 100644 tests/unit_tests/test_stream_protocol.py create mode 100644 tests/unit_tests/test_stream_thread_naming.py diff --git a/CLAUDE.md b/CLAUDE.md index 7006a320..828b11aa 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -194,7 +194,7 @@ and for redis backend a `redis` sub-object `{ "url", "prefix" }` (or shorthand U `THREAD_BUSY_WAIT_SECONDS` (30), then 409 "thread is busy" in the standard error envelope (WS: error chunk `data.reason = "thread is busy"`, socket stays open). Request option `on_busy` (`"wait"` | `"cancel"`, top level next to `recursion_limit`; not on AG-UI): `cancel` calls - `GraphService.stop_graph` while waiting. `/v1/graph/live` is not locked. + `GraphService.stop_graph` while waiting. `/v1/graph/live` takes the lock for the session (and a run slot) too, refused with a fatal `thread_busy` / `too_many_runs` error event and close 1013. Every entry point takes lock + run slot through `routers/graph/run_guard.py::acquire_run_guard`. A Redis error falls back to a per-process count; a refresh that finds its lease gone sets `Lease.lost` and takes it again. - **Production error chunks:** `graph_service.sanitize_error_chunk` (called in `stream_chunks`, so stream, WS and AG-UI share it) replaces `data.reason` / `data.error` / `data.message` and `state.execution_meta.internal_data["error"]` of `ERROR` chunks with the diff --git a/tenxgraph_api/cli/commands/api.py b/tenxgraph_api/cli/commands/api.py index 170c80a2..41efcc7a 100644 --- a/tenxgraph_api/cli/commands/api.py +++ b/tenxgraph_api/cli/commands/api.py @@ -116,6 +116,11 @@ def execute( except ValueError as e: raise ConfigurationError(str(e)) from e + for warning in self.production_launch_warnings( + os.environ.get("MODE"), validated_options["host"], reload + ): + self.output.warning(warning) + os.environ["GRAPH_PATH"] = str(actual_config_path) # Add project root to sys.path for importing graph modules @@ -192,6 +197,29 @@ def execute( ) return self.handle_error(server_error) + @staticmethod + def production_launch_warnings(mode: str | None, host: str, reload: bool) -> list[str]: + """Warnings for starting the dev server with settings meant for a real deployment.""" + try: + production = normalize_mode_value(mode) == "production" + except ValueError: + return [] + if not production: + return [] + warnings = [] + if reload: + warnings.append( + "MODE=production with --reload: the file watcher restarts the server on " + "every change. `10xgraph dev` is for development; deploy the `10xgraph build` " + "image instead." + ) + if host not in ("127.0.0.1", "localhost", "::1", "[::1]"): + warnings.append( + f"MODE=production and --host {host} exposes the dev server to the network " + "without TLS. Use `10xgraph build` and put a TLS proxy in front." + ) + return warnings + @staticmethod def _port_in_use(host: str, port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: diff --git a/tenxgraph_api/cli/commands/audit.py b/tenxgraph_api/cli/commands/audit.py index 1bf76d26..157309c3 100644 --- a/tenxgraph_api/cli/commands/audit.py +++ b/tenxgraph_api/cli/commands/audit.py @@ -56,7 +56,7 @@ class Diagnostic: class AuditCommand(BaseCommand): """Audit the local CLI, core package, project config, and default port. - Runs six checks in a fixed order and reports them twice: live, through the + Runs seven checks in a fixed order and reports them twice: live, through the step timeline, and again as a summary table once every check has finished. ============ ========================================================== @@ -71,6 +71,9 @@ class AuditCommand(BaseCommand): config ``10xgraph.json`` exists here, parses, declares an ``agent`` key in ``module:attribute`` form, and contains valid ``remote_tools`` definitions if present. + security The settings in ``10xgraph.json`` that matter before production + (auth, authorization, rate limiting, token revocation), as + warnings only: they are the developer's choices, never failures. port The default API port is free to bind. ============ ========================================================== """ @@ -99,6 +102,7 @@ def execute(self, *, config: str = DEFAULT_CONFIG_FILE, **kwargs: Any) -> int: ("core", "Core framework", lambda: self._package_check("10xgraph")), ("evaluation", "Evaluation API", self._evaluation_api_check), ("config", "Project configuration", lambda: self._config_check(config)), + ("security", "Security posture", lambda: self._security_check(config)), ("port", f"Port {DEFAULT_PORT}", lambda: self._port_check(DEFAULT_PORT)), ) @@ -194,6 +198,29 @@ def _config_check(config: str = DEFAULT_CONFIG_FILE) -> Diagnostic: return Diagnostic("Project config", "fail", str(exc)) return Diagnostic("Project config", "pass", f"{config_path} ({len(tools)} remote tools)") + @staticmethod + def _security_check(config: str = DEFAULT_CONFIG_FILE) -> Diagnostic: + """Warn about production-readiness gaps in ``10xgraph.json``. + + Reads only the JSON file (never ``.env``). A missing or unreadable config is left to + the config check; this one then passes with a note. + """ + from tenxgraph_api.cli.config_editor.validation import production_readiness_warnings + + config_path = resolve_default_config(config).resolve() + try: + data = json.loads(config_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return Diagnostic("Security", "pass", "no readable config to check") + if not isinstance(data, dict): + return Diagnostic("Security", "pass", "no readable config to check") + issues = production_readiness_warnings(data) + if not issues: + return Diagnostic("Security", "pass", "no production-readiness warnings") + return Diagnostic( + "Security", "warn", "; ".join(f"{i['path']}: {i['message']}" for i in issues) + ) + @staticmethod def _port_check(port: int) -> Diagnostic: """Warn when ``port`` on loopback already has a listener. diff --git a/tenxgraph_api/cli/commands/build.py b/tenxgraph_api/cli/commands/build.py index 369cd06b..51c9f00a 100644 --- a/tenxgraph_api/cli/commands/build.py +++ b/tenxgraph_api/cli/commands/build.py @@ -71,6 +71,13 @@ def execute( # noqa: PLR0912, PLR0915 Exit code """ try: + if docker_compose and k8s: + raise ValidationError( + "--docker-compose and --k8s cannot be used together: the compose image has " + "no start command, so the pod would crash-loop. Run build once with " + "--k8s (Dockerfile and k8s.yaml) or once with --docker-compose.", + field="k8s", + ) self.output.command_header( "build", "Generate Dockerfile (and optional docker-compose.yml) for production image", diff --git a/tenxgraph_api/cli/commands/eval.py b/tenxgraph_api/cli/commands/eval.py index d86502e6..2702ae02 100644 --- a/tenxgraph_api/cli/commands/eval.py +++ b/tenxgraph_api/cli/commands/eval.py @@ -198,9 +198,12 @@ def _confeval_search_dirs(self, target_path: Path, evals_dir: Path) -> list[Path return [evals_dir] dirs: list[Path] = [] + # Walk up only inside the project: a target outside the working directory must not + # pick up a confeval.py (it is imported and run) from an unrelated parent folder. + inside_project = cur == cwd or cwd in cur.parents while True: dirs.append(cur) - if cur in (cwd, cur.parent): + if not inside_project or cur in (cwd, cur.parent): break cur = cur.parent @@ -713,6 +716,12 @@ def _resolve_eval_dir(self) -> Path: ) return Path.cwd() / directory + @staticmethod + def _exit_code(pass_rate: float, threshold: float | None) -> int: + """0 when the pass rate meets ``threshold``; without one, only a clean run passes.""" + required = 1.0 if threshold is None else threshold + return 0 if pass_rate >= required else 1 + # ------------------------------------------------------------------ # Main entry point # ------------------------------------------------------------------ @@ -857,13 +866,11 @@ def execute( # noqa: PLR0912, PLR0915 merged.metadata["criteria_by_file"] = self._criteria_by_file(pending) # 9. Determine exit code - if threshold is not None and merged.summary.pass_rate < threshold: + return_code = self._exit_code(merged.summary.pass_rate, threshold) + if threshold is not None and return_code: self.output.error( f"Pass rate {merged.summary.pass_rate:.1%} is below threshold {threshold:.1%}" ) - return_code = 1 - else: - return_code = 0 if merged.summary.pass_rate == 1.0 else 1 # 10. Generate reports if not no_report: diff --git a/tenxgraph_api/cli/commands/init.py b/tenxgraph_api/cli/commands/init.py index 17743820..2b6b3a08 100644 --- a/tenxgraph_api/cli/commands/init.py +++ b/tenxgraph_api/cli/commands/init.py @@ -1,6 +1,5 @@ """Init command implementation.""" -import contextlib import json import re from collections.abc import Callable @@ -52,6 +51,33 @@ _DEFAULT_PROVIDER = "google" +def _escape_agent_name(name: str, filename: str) -> str: + """Make ``name`` safe inside the quoted value of the file it is substituted into.""" + if filename.endswith(".toml"): + return name.replace("\\", "\\\\").replace('"', '\\"') + if filename.startswith(".env"): + escaped = name.replace("\\", "\\\\").replace('"', '\\"') + return escaped.replace("$", "\\$").replace("`", "\\`") + return name + + +def _agent_name_problem(value: str) -> bool | str: + """Prompt validator: True when the name is usable, else the message to show.""" + if not value.strip() or not _slugify(value): + return "Use at least one letter or number" + try: + _check_agent_name(value) + except ValidationError as exc: + return exc.message + return True + + +def _check_agent_name(name: str) -> None: + """Refuse a name with control characters (a newline would add lines to pyproject.toml).""" + if any(ord(ch) < 32 or ord(ch) == 127 for ch in name): # noqa: PLR2004 + raise ValidationError("Agent name cannot contain control characters.", field="name") + + def _slugify(name: str) -> str: """Convert 'WeatherBot' → 'weather-bot'.""" s = re.sub(r"([A-Z])", r"-\1", name).lstrip("-").lower() @@ -280,6 +306,7 @@ def _non_interactive_context( inferred_name = Path(path).resolve().name if path not in {"", "."} else "MyAgent" resolved_name = (agent_name or inferred_name).strip() + _check_agent_name(resolved_name) slug = _slugify(resolved_name) if not resolved_name or not slug: raise ValidationError( @@ -330,7 +357,8 @@ def _non_interactive_context( "rate_limit": resolved_rate_limit, "rl_requests": 100, "rl_window": 60, - "rl_by": "ip", + # With auth on, one budget per user; "ip" would share one budget behind a proxy. + "rl_by": "user" if resolved_auth in {"jwt", "custom"} else "ip", "rl_trusted_proxy": False, } @@ -348,7 +376,11 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911 ), ) - agent_name = prompts.text("What is your agent name?", default="MyAgent") + agent_name = prompts.text( + "What is your agent name?", + default="MyAgent", + validate=_agent_name_problem, + ) if agent_name is None: return None @@ -454,13 +486,19 @@ def _prompt_user(self) -> dict | None: # noqa: PLR0911 return None context["rl_window"] = int(rl_window) + auth_on = context.get("auth") in ("jwt", "custom") rl_by = prompts.select( "Count requests per?", [ - Choice("ip", "Per IP", "Recommended — each client gets its own budget"), + Choice( + "user", + "Per user", + "Recommended with auth — each signed-in user gets their own budget", + ), + Choice("ip", "Per IP", "Each client address gets its own budget"), Choice("global", "Global", "One shared budget across all clients"), ], - default="ip", + default="user" if auth_on else "ip", ) if rl_by is None: return None @@ -499,7 +537,7 @@ def _print_summary(self, context: dict) -> None: if rl == "none": rows.append(("Rate limit", "None")) else: - rl_by = "Global" if context.get("rl_by") == "global" else "Per IP" + rl_by = {"global": "Global", "user": "Per user"}.get(context.get("rl_by"), "Per IP") proxy = " · proxy headers on" if context.get("rl_trusted_proxy") else "" rows.append( ( @@ -559,14 +597,18 @@ def _build_config(self, context: dict, is_prod: bool) -> dict: "backend": rate_limit, "requests": context.get("rl_requests", 100), "window": context.get("rl_window", 60), - "by": context.get("rl_by", "ip"), + "by": context.get("rl_by", "user" if auth in ("jwt", "custom") else "ip"), "trusted_proxy_headers": context.get("rl_trusted_proxy", False), - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"], + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"], } if rate_limit == "redis": # Resolved from REDIS_URL in .env; the redis backend refuses to start # without a URL or a Redis client bound in InjectQ. config["rate_limit"]["redis"] = {"url": "${REDIS_URL}"} + if auth in ("jwt", "custom"): + # Redis is configured, so a leaked or logged-out token can be revoked + # for every worker. See `10xgraph revoke` and the token_revocation docs. + config["token_revocation"] = "redis" # noqa: S105 return config @@ -595,8 +637,15 @@ def _render(self, src: Path, context: dict, is_prod: bool) -> str: keep_jwt=context["auth"] == "jwt", ) - with contextlib.suppress(Exception): - content = Template(content).safe_substitute({**context, **_template_vars(context)}) + try: + values = {**context, **_template_vars(context)} + if "agent_name" in values: + values["agent_name"] = _escape_agent_name(str(values["agent_name"]), src.name) + content = Template(content).safe_substitute(values) + except Exception as exc: + raise FileOperationError( + f"Could not fill in the template {src.name}: {exc}", file_path=str(src) + ) from exc return content diff --git a/tenxgraph_api/cli/commands/revoke.py b/tenxgraph_api/cli/commands/revoke.py new file mode 100644 index 00000000..75de1906 --- /dev/null +++ b/tenxgraph_api/cli/commands/revoke.py @@ -0,0 +1,74 @@ +"""Revoke a token or every token of a user: ``10xgraph revoke``. + +Writes to the same Redis the server reads (``token_revocation: "redis"`` in 10xgraph.json), +so the revocation applies to every worker at once. The server needs no restart. +""" + +from __future__ import annotations + +import asyncio +import os +from typing import Any + +from tenxgraph_api.cli.commands import BaseCommand +from tenxgraph_api.cli.constants import DEFAULT_CONFIG_FILE +from tenxgraph_api.cli.core.config import resolve_default_config +from tenxgraph_api.cli.exceptions import ValidationError + + +class RevokeCommand(BaseCommand): + """Revoke one token (``--jti``) or all of a user's older tokens (``--user``).""" + + def _redis_url(self, redis_url: str | None, config: str) -> str: + if redis_url: + return redis_url + env_url = os.environ.get("REDIS_URL") + if env_url: + return env_url + path = resolve_default_config(config) + if path.exists(): + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + + configured = GraphConfig(str(path)).redis_url + if isinstance(configured, str) and configured and "${" not in configured: + return configured + raise ValidationError( + "No Redis URL. Pass --redis-url, set REDIS_URL, or set `redis` in 10xgraph.json.", + field="redis_url", + ) + + def execute( + self, + user: str | None = None, + jti: str | None = None, + before: float | None = None, + expires_at: float | None = None, + redis_url: str | None = None, + config: str = DEFAULT_CONFIG_FILE, + **kwargs: Any, + ) -> int: + try: + if bool(user) == bool(jti): + raise ValidationError("Pass exactly one of --user or --jti.", field="user") + url = self._redis_url(redis_url, config) + + from tenxgraph_api.src.app.core.auth.revocation import RedisTokenRevocation + + async def run() -> None: + backend = RedisTokenRevocation(url) + try: + if user: + await backend.revoke_user(user, before=before) + else: + await backend.revoke_token(str(jti), expires_at=expires_at) + finally: + await backend.aclose() + + asyncio.run(run()) + if user: + self.output.success(f"Revoked the tokens of user {user} issued so far.") + else: + self.output.success(f"Revoked token {jti}.") + return 0 + except Exception as e: + return self.handle_error(e) diff --git a/tenxgraph_api/cli/commands/skills.py b/tenxgraph_api/cli/commands/skills.py index dcaf2a16..fc5e458a 100644 --- a/tenxgraph_api/cli/commands/skills.py +++ b/tenxgraph_api/cli/commands/skills.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import os import shutil from dataclasses import dataclass from datetime import UTC, datetime @@ -434,31 +435,45 @@ def _install_one( file_path=str(existing[0]), ) - for _artifact, _source, dest in installs: - if dest.exists(): - if dest.is_dir(): - shutil.rmtree(dest) + # Build every artifact next to its destination first, then swap it in. Removing the + # old install before the copy would leave no skill folder if the copy failed. + staged: list[tuple[Path, Path]] = [] + try: + for artifact, source, dest in installs: + dest.parent.mkdir(parents=True, exist_ok=True) + temp = dest.parent / f".{dest.name}.installing-{os.getpid()}" + self._remove(temp) + staged.append((temp, dest)) + if artifact.kind == "folder": + shutil.copytree( + source, + temp, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".DS_Store"), + ) + if artifact.manifest: + self._write_manifest(temp, target.name) else: - dest.unlink() + shutil.copyfile(source, temp) + except BaseException: + for temp, _dest in staged: + self._remove(temp) + raise installed_paths: list[str] = [] - for artifact, source, dest in installs: - dest.parent.mkdir(parents=True, exist_ok=True) - - if artifact.kind == "folder": - shutil.copytree( - source, - dest, - ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".DS_Store"), - ) - if artifact.manifest: - self._write_manifest(dest, target.name) - else: - shutil.copyfile(source, dest) + for temp, dest in staged: + self._remove(dest) + temp.rename(dest) installed_paths.append(str(dest)) return installed_paths + @staticmethod + def _remove(path: Path) -> None: + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + elif path.exists() or path.is_symlink(): + path.unlink() + @staticmethod def _ensure_inside_project(project_root: Path, dest: Path) -> None: """Refuse a destination that is, or sits under, a symlink, or that leaves the project. diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index ad276b21..792d58da 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -69,7 +69,6 @@ "graph:fix", "graph:invoke", "graph:read", - "graph:setup", "graph:stop", "graph:stream", "store:delete", @@ -343,7 +342,7 @@ "placeholder": "/docs", "help": ( "Paths that are never rate limited. /ping and /health/ready are always " - "skipped. API docs paths (/docs, /redoc, /openapi.json) are limited unless " + "skipped. API docs paths (/docs, /redocs, /openapi.json) are limited unless " "listed here. The server default is an empty list." ), }, diff --git a/tenxgraph_api/cli/config_editor/validation.py b/tenxgraph_api/cli/config_editor/validation.py index 343863b2..495720c2 100644 --- a/tenxgraph_api/cli/config_editor/validation.py +++ b/tenxgraph_api/cli/config_editor/validation.py @@ -120,6 +120,26 @@ def production_readiness_warnings(config: dict) -> list[Issue]: "Rate limiting is off, so one caller can exhaust the service. Enable it, with the " "redis backend for more than one worker, before production.", ) + elif isinstance(rate_limit, dict) and rate_limit.get("enabled") is not False: + if auth and rate_limit.get("by", "ip") == "ip": + out.warning( + "rate_limit.by", + "Counting requests per IP with auth on shares one budget between users behind a " + 'proxy. Use "by": "user", or set trusted_proxy_headers behind your proxy.', + ) + exclude = rate_limit.get("exclude_paths") + if isinstance(exclude, list) and "/redoc" in exclude and "/redocs" not in exclude: + out.warning( + "rate_limit.exclude_paths", + 'The ReDoc page is served at /redocs, not /redoc; "/redoc" excludes nothing.', + ) + + if auth and not config.get("token_revocation"): + out.warning( + "token_revocation", + "Token revocation is off, so a leaked or logged-out token works until it expires. " + 'Set "token_revocation": "redis" and revoke with `10xgraph revoke`.', + ) return out.issues diff --git a/tenxgraph_api/cli/core/validation.py b/tenxgraph_api/cli/core/validation.py index ac5a9c64..841ae91a 100644 --- a/tenxgraph_api/cli/core/validation.py +++ b/tenxgraph_api/cli/core/validation.py @@ -108,9 +108,10 @@ def validate_python_version(version: str) -> str: parts = version.split(".") major, minor = int(parts[0]), int(parts[1]) - # Validate Python version range (3.8+) - if major < 3 or (major == 3 and minor < 8): # noqa: PLR2004 - raise ValidationError("Python version must be 3.8 or higher", field="python_version") + # The framework requires Python 3.12 (pyproject ``requires-python``); an older image + # would build and then fail at pip install. + if (major, minor) < (3, 12): + raise ValidationError("Python version must be 3.12 or higher", field="python_version") return version diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index f366a004..8cc0d1d3 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -67,6 +67,7 @@ def execute(self, *args: Any, **kwargs: Any) -> int: DemoCommand = _lazy_command("tenxgraph_api.cli.commands.demo", "DemoCommand") EvalCommand = _lazy_command("tenxgraph_api.cli.commands.eval", "EvalCommand") InitCommand = _lazy_command("tenxgraph_api.cli.commands.init", "InitCommand") +RevokeCommand = _lazy_command("tenxgraph_api.cli.commands.revoke", "RevokeCommand") SkillsCommand = _lazy_command("tenxgraph_api.cli.commands.skills", "SkillsCommand") TestCommand = _lazy_command("tenxgraph_api.cli.commands.test", "TestCommand") VersionCommand = _lazy_command("tenxgraph_api.cli.commands.version", "VersionCommand") @@ -358,7 +359,7 @@ def audit( ) -> None: """Audit package compatibility and the current project environment. - Runs six read-only checks and prints them as a table: the Python + Runs seven read-only checks and prints them as a table: the Python interpreter, the installed CLI and core packages, whether the core exposes the evaluation API this CLI expects, whether `10xgraph.json` declares a valid `agent` key and remote-tool schemas, and whether the default port is free. @@ -417,6 +418,41 @@ def version( sys.exit(handle_exception(e)) +@app.command(rich_help_panel="Manage") +def revoke( + user: str | None = typer.Option( + None, "--user", help="Revoke all tokens of this user_id issued so far (log out everywhere)." + ), + jti: str | None = typer.Option(None, "--jti", help="Revoke the one token with this jti claim."), + before: float | None = typer.Option( + None, "--before", help="With --user: revoke tokens issued at or before this epoch time." + ), + expires_at: float | None = typer.Option( + None, "--expires-at", help="With --jti: the token's exp (epoch seconds), to drop the entry." + ), + redis_url: str | None = typer.Option( + None, "--redis-url", help="Redis URL (default: REDIS_URL, then `redis` in 10xgraph.json)." + ), + config_file: str = typer.Option( + DEFAULT_CONFIG_FILE, "--config", "-c", help="Project configuration file." + ), +) -> None: + """Revoke a token or all of a user's tokens (needs "token_revocation": "redis").""" + try: + sys.exit( + RevokeCommand(output).execute( + user=user, + jti=jti, + before=before, + expires_at=expires_at, + redis_url=redis_url, + config=config_file, + ) + ) + except Exception as e: + sys.exit(handle_exception(e)) + + @app.command(rich_help_panel="Manage") def config( config_file: str = typer.Option( diff --git a/tenxgraph_api/cli/templates/defaults.py b/tenxgraph_api/cli/templates/defaults.py index 5b66b9f7..9caac93e 100644 --- a/tenxgraph_api/cli/templates/defaults.py +++ b/tenxgraph_api/cli/templates/defaults.py @@ -19,10 +19,30 @@ # Gunicorn kills a worker it considers hung. Long LLM calls must not look hung. WORKER_TIMEOUT_SECONDS: Final[int] = 660 +# Uvicorn stops waiting for open responses after this many seconds and cancels them, so the +# lifespan shutdown (30 s drain, graph.aclose) runs before gunicorn's --graceful-timeout +# SIGKILLs the worker. Open NDJSON streams get a final "interrupted" chunk just before it. +# Keep in step with ``uvicorn_worker.DEFAULT_GRACEFUL_SHUTDOWN_SECONDS``. +UVICORN_GRACEFUL_SHUTDOWN_SECONDS: Final[int] = GRACEFUL_TIMEOUT_SECONDS - 60 + +# The container runtime's own kill timer must be longer than gunicorn's, or SIGKILL lands +# while workers are still draining and the lifespan shutdown never runs. +SHUTDOWN_KILL_MARGIN_SECONDS: Final[int] = 60 + +# `docker compose down` waits this long before SIGKILL. +COMPOSE_STOP_GRACE_SECONDS: Final[int] = GRACEFUL_TIMEOUT_SECONDS + SHUTDOWN_KILL_MARGIN_SECONDS + # terminationGracePeriodSeconds must exceed the app's graceful timeout, plus the -# preStop sleep that lets the load balancer stop sending new traffic first. -K8S_TERMINATION_GRACE_SECONDS: Final[int] = GRACEFUL_TIMEOUT_SECONDS + 60 +# preStop sleep that lets the load balancer stop sending new traffic first (it runs +# inside the same period). K8S_PRESTOP_SLEEP_SECONDS: Final[int] = 15 +K8S_TERMINATION_GRACE_SECONDS: Final[int] = ( + GRACEFUL_TIMEOUT_SECONDS + K8S_PRESTOP_SLEEP_SECONDS + SHUTDOWN_KILL_MARGIN_SECONDS +) + +# Gunicorn worker class: uvicorn worker with timeout_graceful_shutdown, a draining flag for +# /health/ready and a final chunk for open streams. +GUNICORN_WORKER_CLASS: Final[str] = "tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker" # Fixed, numeric uid/gid for the image user. Kubernetes can only verify # ``runAsNonRoot`` against a numeric USER, and a fixed id keeps volume ownership @@ -401,7 +421,7 @@ def generate_dockerfile_content( "# hard-kill runs that were still mid-flight, truncating them. Give", "# in-flight runs time to finish draining instead.", ( - 'CMD ["gunicorn", "-k", "uvicorn_worker.UvicornWorker", ' + f'CMD ["gunicorn", "-k", "{GUNICORN_WORKER_CLASS}", ' f'"-b", "0.0.0.0:{port}", ' f'"--graceful-timeout", "{GRACEFUL_TIMEOUT_SECONDS}", ' f'"--timeout", "{WORKER_TIMEOUT_SECONDS}", ' @@ -446,8 +466,11 @@ def generate_k8s_manifest_content(service_name: str, port: int) -> str: "# one --from-literal per value (--from-env-file keeps quotes as part", "# of the value), or sync it from your secret manager:", f"# kubectl create secret generic {secret_name} \\", - "# --from-literal=JWT_SECRET_KEY=... --from-literal=GOOGLE_API_KEY=...", - "# 3. Replace the ORIGINS and ALLOWED_HOST placeholders.", + "# --from-literal=JWT_SECRET_KEY=... --from-literal=GOOGLE_API_KEY=... \\", + "# --from-literal=ORIGINS=https://your-frontend.example.com \\", + "# --from-literal=ALLOWED_HOST=your-app.example.com", + "# 3. ORIGINS (CORS origins, never *) and ALLOWED_HOST (your public hostname) are", + "# required in production and are read from the same Secret.", "# 4. Expose it through an Ingress or Gateway that terminates TLS.", "# The root filesystem is read-only: only /tmp and /app/uploads are writable.", "# If your graph writes anywhere else (a SQLite checkpointer under", @@ -505,12 +528,8 @@ def generate_k8s_manifest_content(service_name: str, port: int) -> str: ' value: "production"', " - name: IS_DEBUG", ' value: "false"', - " # CORS: production refuses wildcard origins with credentials.", - " - name: ORIGINS", - ' value: "https://your-frontend.example.com"', - " # Production refuses a wildcard host; set your public hostname.", - " - name: ALLOWED_HOST", - ' value: "your-app.example.com"', + # ORIGINS and ALLOWED_HOST come from the Secret (an env entry here would + # override it). Production refuses to start while either is missing or a wildcard. " lifecycle:", " preStop:", " exec:", @@ -599,7 +618,7 @@ def generate_docker_compose_content(service_name: str, port: int) -> str: " ports:", f" - '127.0.0.1:{port}:{port}'", ( - f" command: [ 'gunicorn', '-k', 'uvicorn_worker.UvicornWorker', " + f" command: [ 'gunicorn', '-k', '{GUNICORN_WORKER_CLASS}', " f"'-b', '0.0.0.0:{port}', " f"'--graceful-timeout', '{GRACEFUL_TIMEOUT_SECONDS}', " f"'--timeout', '{WORKER_TIMEOUT_SECONDS}', " @@ -607,7 +626,8 @@ def generate_docker_compose_content(service_name: str, port: int) -> str: ), # Give in-flight agent runs time to drain on `docker compose down` # instead of being killed after the default 10s. - f" stop_grace_period: {GRACEFUL_TIMEOUT_SECONDS}s", + # Longer than gunicorn's --graceful-timeout, or SIGKILL lands mid-drain. + f" stop_grace_period: {COMPOSE_STOP_GRACE_SECONDS}s", " restart: unless-stopped", " # Locked down: read-only filesystem, no Linux capabilities, no setuid", " # escalation. Only /tmp and the uploads folder are writable.", diff --git a/tenxgraph_api/cli/templates/prod/10xgraph.json b/tenxgraph_api/cli/templates/prod/10xgraph.json index 478cc551..3b683722 100644 --- a/tenxgraph_api/cli/templates/prod/10xgraph.json +++ b/tenxgraph_api/cli/templates/prod/10xgraph.json @@ -5,6 +5,7 @@ "method": "custom", "path": "auth.agent_auth:AgentAuth" }, + "token_revocation": "redis", "remote_tools": [], "ag_ui": { "enabled": false @@ -24,7 +25,7 @@ "trusted_proxy_headers": true, "trusted_proxy_hops": 1, "fail_open": false, - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"] + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"] }, "test": { "path": "tests", diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md index 352ef8be..664191b1 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-configuration.md @@ -17,7 +17,6 @@ Common full shape: ```json { "agent": "graph.react:app", - "checkpointer": "graph.dependencies:my_checkpointer", "store": "graph.dependencies:my_store", "injectq": "graph.dependencies:container", "thread_name_generator": "graph.thread_name_generator:MyNameGenerator", @@ -31,7 +30,7 @@ Common full shape: "requests": 100, "window": 60, "by": "ip", - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"] + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"] } } ``` @@ -39,7 +38,7 @@ Common full shape: ## Fields - `agent`: required import path to a compiled graph variable, in `module.path:attribute` format. -- `checkpointer`: optional import path to a `BaseCheckpointer` instance. +- `checkpointer`: not applied by the server (a warning is logged when it is set). Pass the checkpointer to `graph.compile(checkpointer=...)` in your agent module. - `store`: optional import path to a `BaseStore` instance; required for store endpoints. - `injectq`: optional import path to an `InjectQ` container. - `thread_name_generator`: optional import path to a thread-name generator class/instance. @@ -54,7 +53,7 @@ Common full shape: 1. Read `10xgraph.json`. 2. Load `.env` when configured. 3. Import the compiled graph from `agent`. -4. Attach `remote_tools`, then import and bind `checkpointer`, `store`, `injectq`, `thread_name_generator`, and `authorization` when configured. +4. Attach `remote_tools`, then import and bind `store`, `injectq`, `thread_name_generator`, and `authorization` when configured. 5. Configure auth. 6. Start FastAPI routes and services. diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md index b5e6c534..1b19b84c 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/dependency-injection.md @@ -36,13 +36,12 @@ In `10xgraph.json`, use import paths to wire dependencies into the server runtim ```json { "agent": "graph.react:app", - "checkpointer": "graph.dependencies:checkpointer", "store": "graph.dependencies:store", "injectq": "graph.dependencies:container" } ``` -The API loader imports these and binds them for graph execution. +The API loader imports these and binds them for graph execution. The checkpointer is not set here: pass it to `graph.compile(checkpointer=...)` in your agent module (a `checkpointer` key in `10xgraph.json` is ignored and logs a warning). ## Rules diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md index 42d90450..a94fa1a8 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/rate-limiting.md @@ -20,7 +20,7 @@ In-memory backend for local development or single-process services: "requests": 100, "window": 60, "by": "ip", - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"] + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"] } } ``` @@ -46,7 +46,7 @@ pip install "10xgraph-api[redis]" "window": 60, "by": "ip", "trusted_proxy_headers": true, - "exclude_paths": ["/ping", "/metrics", "/docs", "/redoc", "/openapi.json"], + "exclude_paths": ["/ping", "/metrics", "/docs", "/redocs", "/openapi.json"], "redis": { "url": "${RATE_LIMIT_REDIS_URL}", "prefix": "10xgraph:rate-limit" diff --git a/tenxgraph_api/src/app/core/auth/authorization.py b/tenxgraph_api/src/app/core/auth/authorization.py index fc0bad34..739483d2 100644 --- a/tenxgraph_api/src/app/core/auth/authorization.py +++ b/tenxgraph_api/src/app/core/auth/authorization.py @@ -21,6 +21,9 @@ # DELETE /v1/files/{file_id}. Core lists only files:read and files:upload, so it is added here. FILES_DELETE_SCOPES = frozenset({"files:delete"}) LEGACY_MEMORY_SCOPES = frozenset({"memory:read", "memory:write", "memory:delete"}) +# Core still lists it, but the setup endpoint was removed: no route checks it, so a role rule +# for it would guard nothing. +REMOVED_SCOPES = frozenset({"graph:setup"}) def all_scopes() -> frozenset[str]: @@ -28,7 +31,7 @@ def all_scopes() -> frozenset[str]: from tenxgraph.core.authz import ALL_SCOPES return ( - (frozenset(ALL_SCOPES) - LEGACY_MEMORY_SCOPES) + (frozenset(ALL_SCOPES) - LEGACY_MEMORY_SCOPES - REMOVED_SCOPES) | STORE_SCOPES | CHECKPOINTER_ADMIN_SCOPES | FILES_DELETE_SCOPES diff --git a/tenxgraph_api/src/app/core/auth/revocation.py b/tenxgraph_api/src/app/core/auth/revocation.py index 9b764664..df6d1ace 100644 --- a/tenxgraph_api/src/app/core/auth/revocation.py +++ b/tenxgraph_api/src/app/core/auth/revocation.py @@ -11,7 +11,12 @@ expired anyway; - **every token of a user issued before a moment** (``revoke_user``), compared against the token's ``iat`` claim. Use it for "log out everywhere" or after a password change. A token - without ``iat`` cannot be dated, so it counts as revoked once its user has been revoked. + without ``iat`` cannot be dated, so it counts as revoked while the user entry exists; give the + backend your ``token_lifetime`` and such a token is dated from its ``exp`` instead, so one + minted after the revoke is accepted. Tokens should carry ``iat``. + +To revoke from your own code use :func:`revoke_token` and :func:`revoke_user`; from a shell +use ``10xgraph revoke``. A newer ``revoke_user`` cutoff is never replaced by an older one. Enable it in ``10xgraph.json`` with ``"token_revocation": "redis"`` (shared by every worker; uses the ``redis`` key or ``REDIS_URL``), ``"token_revocation": "memory"`` (one process; tests @@ -24,6 +29,7 @@ from __future__ import annotations +import asyncio import importlib import inspect import logging @@ -69,17 +75,59 @@ def _user_key_of(identity: dict[str, Any]) -> str | None: return None -def _revoked_by_user_cutoff(identity: dict[str, Any], cutoff: float | None) -> bool: +# A revoke_user entry is kept this long unless told otherwise. It has to outlive the longest +# token you issue, or an old token comes back to life when the entry goes. +DEFAULT_USER_REVOCATION_TTL = 30 * 24 * 3600 + + +def _revoked_by_user_cutoff( + identity: dict[str, Any], cutoff: float | None, token_lifetime: float | None = None +) -> bool: if cutoff is None: return False issued_at = _number(identity.get("iat")) + if issued_at is None and token_lifetime is not None: + expires_at = _number(identity.get("exp")) + if expires_at is not None: + issued_at = expires_at - token_lifetime return issued_at is None or issued_at <= cutoff +async def _redis_now(client: Any) -> float: + """The Redis server's clock, so every pod agrees; this process's clock without ``TIME``.""" + server_time = getattr(client, "time", None) + if not callable(server_time): + return time.time() + seconds, microseconds = await server_time() + return float(seconds) + float(microseconds) / 1_000_000 + + +# Keeps the newest cutoff: an older ``revoke_user(before=...)`` must not un-revoke tokens a +# newer call already revoked. An empty cutoff means "now" on the Redis clock. +_REVOKE_USER_LUA = """ +local cutoff = tonumber(ARGV[1]) +if cutoff == nil then + local t = redis.call('TIME') + cutoff = tonumber(t[1]) + tonumber(t[2]) / 1000000 +end +local current = tonumber(redis.call('GET', KEYS[1]) or '') +if current ~= nil and current >= cutoff then + return 0 +end +redis.call('SET', KEYS[1], string.format('%.6f', cutoff)) +local ttl = tonumber(ARGV[2]) +if ttl ~= nil and ttl > 0 then + redis.call('EXPIRE', KEYS[1], ttl) +end +return 1 +""" + + class InMemoryTokenRevocation(TokenRevocationBackend): """Revocation list held in this process. Fine for tests and single-worker servers.""" - def __init__(self) -> None: + def __init__(self, token_lifetime: float | None = None) -> None: + self._token_lifetime = token_lifetime self._tokens: dict[str, float] = {} # jti -> expiry (epoch seconds) self._users: dict[str, float] = {} # user_id -> revoke tokens issued at or before @@ -89,7 +137,9 @@ def revoke_token(self, jti: str, expires_at: float | None = None) -> None: def revoke_user(self, user_id: str, before: float | None = None) -> None: """Revoke every token of ``user_id`` issued at or before ``before`` (default: now).""" - self._users[str(user_id)] = time.time() if before is None else float(before) + cutoff = time.time() if before is None else float(before) + key = str(user_id) + self._users[key] = max(cutoff, self._users.get(key, cutoff)) async def is_revoked(self, identity: dict[str, Any]) -> bool: now = time.time() @@ -99,14 +149,20 @@ async def is_revoked(self, identity: dict[str, Any]) -> bool: return True user_id = _user_key_of(identity) cutoff = self._users.get(user_id) if user_id is not None else None - return _revoked_by_user_cutoff(identity, cutoff) + return _revoked_by_user_cutoff(identity, cutoff, self._token_lifetime) class RedisTokenRevocation(TokenRevocationBackend): """Revocation list in Redis, shared by every worker and instance. Keys: ``:jti:`` (expires with the token) and ``:user:`` - (holds the cutoff timestamp). One ``MGET`` per check. + (holds the cutoff timestamp). A check reads both keys with two parallel ``GET`` calls (a + single ``MGET`` over two keys fails with CROSSSLOT on Redis Cluster). Times come from the + Redis clock, so pods with skewed clocks agree. + + ``token_lifetime`` is the longest lifetime of the tokens you issue, in seconds. It dates + a token that has no ``iat`` from its ``exp``. ``user_ttl`` is how long a ``revoke_user`` + entry is kept when the call gives no ``ttl`` (``None`` keeps it forever). """ def __init__( @@ -115,18 +171,23 @@ def __init__( *, client: Any | None = None, prefix: str = "10xgraph:revoked", + token_lifetime: float | None = None, + user_ttl: int | None = DEFAULT_USER_REVOCATION_TTL, ) -> None: + self._token_lifetime = token_lifetime + self._user_ttl = user_ttl if client is None: if not url: raise ValueError("RedisTokenRevocation needs a Redis url or client") + from tenxgraph_api.src.app.core.redis_client import create_redis + try: - from redis.asyncio import Redis # type: ignore[import] + client = create_redis(url) except ImportError as exc: # pragma: no cover - depends on installed extras raise ImportError( 'token_revocation "redis" needs the redis package: ' 'pip install "10xgraph-api[redis]"' ) from exc - client = Redis.from_url(url, decode_responses=False) self._client = client self._prefix = prefix @@ -137,7 +198,9 @@ def _user_key(self, user_id: Any) -> str: return f"{self._prefix}:user:{user_id}" async def revoke_token(self, jti: str, expires_at: float | None = None) -> None: - ttl = None if expires_at is None else max(1, int(expires_at - time.time())) + ttl = None + if expires_at is not None: + ttl = max(1, int(expires_at - await _redis_now(self._client))) await self._client.set(self._jti_key(jti), b"1", ex=ttl) async def revoke_user( @@ -145,30 +208,30 @@ async def revoke_user( ) -> None: """Revoke tokens of ``user_id`` issued at or before ``before`` (default: now). - ``ttl`` (seconds) can expire the entry once every older token has expired; pass your - longest token lifetime. + ``ttl`` (seconds) expires the entry once every older token has expired; it defaults to + ``user_ttl``. An older ``before`` than the one already stored changes nothing. """ - cutoff = time.time() if before is None else float(before) - await self._client.set(self._user_key(user_id), str(cutoff).encode(), ex=ttl) + cutoff = "" if before is None else repr(float(before)) + keep_for = self._user_ttl if ttl is None else ttl + await self._client.eval( + _REVOKE_USER_LUA, 1, self._user_key(user_id), cutoff, str(keep_for or 0) + ) async def is_revoked(self, identity: dict[str, Any]) -> bool: jti, user_id = identity.get("jti"), _user_key_of(identity) - keys = [] - if jti is not None: - keys.append(self._jti_key(jti)) - if user_id is not None: - keys.append(self._user_key(user_id)) - if not keys: + if jti is None and user_id is None: return False - values = await self._client.mget(keys) - values = dict(zip(keys, values, strict=True)) - if jti is not None and values.get(self._jti_key(jti)) is not None: + # Two plain GETs, not one MGET: the keys hash to different slots on Redis Cluster. + jti_value, raw = await asyncio.gather( + self._client.get(self._jti_key(jti)) if jti is not None else _none(), + self._client.get(self._user_key(user_id)) if user_id is not None else _none(), + ) + if jti_value is not None: return True - raw = values.get(self._user_key(user_id)) if user_id is not None else None if raw is None: return False cutoff = float(raw.decode() if isinstance(raw, bytes | bytearray) else raw) - return _revoked_by_user_cutoff(identity, cutoff) + return _revoked_by_user_cutoff(identity, cutoff, self._token_lifetime) async def aclose(self) -> None: close = getattr(self._client, "aclose", None) @@ -176,6 +239,10 @@ async def aclose(self) -> None: await close() +async def _none() -> None: + return None + + def build_token_revocation(spec: Any, redis_url: str | None = None) -> TokenRevocationBackend: """Build the backend named by the ``token_revocation`` key of 10xgraph.json.""" if spec == "memory": @@ -231,3 +298,34 @@ async def token_revoked( except Exception as exc: logger.error("Token revocation check failed: %s", exc) raise TokenRevocationUnavailableError("Token revocation check unavailable") from exc + + +async def _call(method: Any, *args: Any, **kwargs: Any) -> None: + """Call a backend method that may be sync (in-memory) or async (Redis).""" + result = method(*args, **kwargs) + if inspect.isawaitable(result): + await result + + +def _backend_or_raise(backend: TokenRevocationBackend | None) -> Any: + backend = backend or _bound_backend() + if backend is None: + raise RuntimeError( + 'No token revocation backend is configured. Set "token_revocation": "redis" in ' + "10xgraph.json (with Redis configured), or bind a TokenRevocationBackend." + ) + return backend + + +async def revoke_token( + jti: str, expires_at: float | None = None, *, backend: TokenRevocationBackend | None = None +) -> None: + """Revoke one token by its ``jti`` claim. ``expires_at`` is its ``exp`` (epoch seconds).""" + await _call(_backend_or_raise(backend).revoke_token, jti, expires_at) + + +async def revoke_user( + user_id: str, before: float | None = None, *, backend: TokenRevocationBackend | None = None +) -> None: + """Revoke every token of ``user_id`` issued at or before ``before`` (default: now).""" + await _call(_backend_or_raise(backend).revoke_user, user_id, before) diff --git a/tenxgraph_api/src/app/core/concurrency.py b/tenxgraph_api/src/app/core/concurrency.py index 66f3d934..d8d9daac 100644 --- a/tenxgraph_api/src/app/core/concurrency.py +++ b/tenxgraph_api/src/app/core/concurrency.py @@ -13,8 +13,12 @@ Acquiring runs one transaction (drop expired leases, add ours, count, set a key TTL) and backs out if the count is over the limit. Two callers racing for the last slot may both back out, -but the cap is never exceeded. A Redis error fails open: the request is allowed and the error -logged, because refusing every request while Redis is down would be an outage of its own. +but the cap is never exceeded. Expiry times use the Redis clock (``TIME``), not each pod's +own, so clock skew between pods cannot trim a live lease. A Redis error fails open to this +process's own count: the error is logged and the slot is taken from an in-process counter, +because refusing every request while Redis is down would be an outage of its own. The cap then +holds per process, not across workers, until Redis is back (so the per-thread run lock still +stops a double run in one worker). """ from __future__ import annotations @@ -36,6 +40,9 @@ # Release coroutines scheduled from sync code, referenced until they finish. _pending_releases: set[asyncio.Task[Any]] = set() +# Longest ``aclose`` waits for those releases before it closes the client anyway. +SHUTDOWN_RELEASE_WAIT_SECONDS = 5.0 + class Lease: """A held slot. ``release`` is idempotent and safe to call from sync code.""" @@ -43,6 +50,8 @@ class Lease: def __init__(self, release: Any = None) -> None: self._release = release self._released = release is None + # Set when a shared lease was found missing on refresh (expired or evicted while held). + self.lost = False @property def released(self) -> bool: @@ -118,20 +127,31 @@ def __init__( self._prefix = prefix self._lease_ttl = lease_ttl self._refreshers: set[asyncio.Task[None]] = set() + # Counts used while Redis is unreachable, so one process still honours the cap. + self._local = MemoryConcurrencyLimiter() @classmethod def from_url(cls, url: str, **kwargs: Any) -> RedisConcurrencyLimiter: - from redis.asyncio import Redis # type: ignore[import] + from tenxgraph_api.src.app.core.redis_client import create_redis - return cls(Redis.from_url(url, decode_responses=False), **kwargs) + return cls(create_redis(url), **kwargs) def _key(self, key: str) -> str: return f"{self._prefix}:{key}" + async def _now(self) -> float: + """The Redis server's clock. Lease expiry is compared across pods, so a pod's own + clock (skewed by more than the lease ttl) would trim another pod's live lease.""" + server_time = getattr(self._client, "time", None) + if not callable(server_time): + return time.time() + seconds, microseconds = await server_time() + return float(seconds) + float(microseconds) / 1_000_000 + async def acquire(self, key: str, limit: int) -> Lease | None: redis_key, member = self._key(key), uuid.uuid4().hex - now = time.time() try: + now = await self._now() pipe = self._client.pipeline(transaction=True) pipe.zremrangebyscore(redis_key, "-inf", now) pipe.zadd(redis_key, {member: now + self._lease_ttl}) @@ -141,13 +161,21 @@ async def acquire(self, key: str, limit: int) -> Lease | None: if int(results[2]) > limit: await self._client.zrem(redis_key, member) return None + except asyncio.CancelledError: + # The caller gave up (a client left while waiting). The transaction may already + # have added our member; take it out now rather than leave it for the lease ttl. + self._discard_later(redis_key, member) + raise except Exception as exc: logger.warning( - "Shared concurrency limit unavailable (%s); allowing %s", exc, log_safe(key) + "Shared concurrency limit unavailable (%s); counting %s in this process", + exc, + log_safe(key), ) - return Lease(None) + return await self._local.acquire(key, limit) - refresher = asyncio.create_task(self._refresh(redis_key, member)) + lease = Lease(lambda: release()) # release is defined below, it needs the refresher + refresher = asyncio.create_task(self._refresh(redis_key, member, lease)) self._refreshers.add(refresher) refresher.add_done_callback(self._refreshers.discard) @@ -158,22 +186,52 @@ async def release() -> None: except Exception as exc: # the lease expires on its own logger.debug("Lease release failed for %s: %s", log_safe(key), exc) - return Lease(release) + return lease + + def _discard_later(self, redis_key: str, member: str) -> None: + """Remove ``member`` on a task of its own, for callers that are being cancelled.""" + + async def discard() -> None: + try: + await self._client.zrem(redis_key, member) + except Exception as exc: # the lease expires on its own + logger.debug("Discarding a lease failed for %s: %s", log_safe(redis_key), exc) + + task = asyncio.get_running_loop().create_task(discard()) + _pending_releases.add(task) + task.add_done_callback(_pending_releases.discard) - async def _refresh(self, redis_key: str, member: str) -> None: + async def _refresh(self, redis_key: str, member: str, lease: Lease) -> None: """Keep a held lease alive; it expires ``lease_ttl`` after the last refresh.""" interval = max(self._lease_ttl / 3, 0.01) while True: await asyncio.sleep(interval) try: - await self._client.zadd(redis_key, {member: time.time() + self._lease_ttl}, xx=True) + expiry = await self._now() + self._lease_ttl + # ``xx`` never adds, and without ``ch`` Redis reports 0 for a refreshed member + # as well as for a missing one, so the reply could not tell them apart. + changed = await self._client.zadd(redis_key, {member: expiry}, xx=True, ch=True) + if not changed: + lease.lost = True + logger.warning( + "Lease %s expired or was evicted while held; taking it again", redis_key + ) + await self._client.zadd(redis_key, {member: expiry}) await self._client.expire(redis_key, int(self._lease_ttl) + 1) except Exception as exc: - logger.debug("Lease refresh failed for %s: %s", log_safe(redis_key), exc) + logger.warning("Lease refresh failed for %s: %s", log_safe(redis_key), exc) async def aclose(self) -> None: for task in list(self._refreshers): task.cancel() + # Releases scheduled from sync code run on tasks of their own; the client has to stay + # open until they have, or the leases stay until their ttl (a 409 on the next pod). + while _pending_releases: + pending = set(_pending_releases) + _, still = await asyncio.wait(pending, timeout=SHUTDOWN_RELEASE_WAIT_SECONDS) + if still: + logger.warning("%d lease releases did not finish before shutdown", len(still)) + break close = getattr(self._client, "aclose", None) if callable(close): with contextlib.suppress(Exception): diff --git a/tenxgraph_api/src/app/core/config/graph_config.py b/tenxgraph_api/src/app/core/config/graph_config.py index 627f3f67..d1df4a97 100644 --- a/tenxgraph_api/src/app/core/config/graph_config.py +++ b/tenxgraph_api/src/app/core/config/graph_config.py @@ -148,7 +148,7 @@ class RateLimitConfig: "window": 60, "by": "ip", "trusted_proxy_headers": false, - "exclude_paths": ["/ping", "/docs", "/redoc", "/openapi.json"] + "exclude_paths": ["/ping", "/docs", "/redocs", "/openapi.json"] } Example (Redis backend):: diff --git a/tenxgraph_api/src/app/core/config/settings.py b/tenxgraph_api/src/app/core/config/settings.py index 5ed48a9f..994521eb 100644 --- a/tenxgraph_api/src/app/core/config/settings.py +++ b/tenxgraph_api/src/app/core/config/settings.py @@ -135,6 +135,16 @@ class Settings(BaseSettings): ###### REDIS Config ########## ################################# REDIS_URL: str | None = None + # Limits for every Redis client the API opens (run locks, caps, rate limit, token + # revocation, ownership cache, readiness probe). Without them a Redis that stops + # answering (not refusing) would hang each request that touches it. Seconds; 0 turns the + # timeout off. REDIS_MAX_CONNECTIONS bounds each client's pool; a request waits up to + # REDIS_POOL_TIMEOUT for a free connection. 0 means no bound. + REDIS_SOCKET_TIMEOUT: float = Field(default=5.0, ge=0.0) + REDIS_CONNECT_TIMEOUT: float = Field(default=5.0, ge=0.0) + REDIS_MAX_CONNECTIONS: int = Field(default=50, ge=0) + REDIS_POOL_TIMEOUT: float = Field(default=5.0, ge=0.0) + REDIS_HEALTH_CHECK_INTERVAL: float = Field(default=30.0, ge=0.0) ################################# ###### sentry Config ############ @@ -245,6 +255,19 @@ def check_production_security(self): if warnings: logger.warning("\n".join(["\n🔒 PRODUCTION SECURITY WARNINGS:", *warnings])) + # Not security holes, but both leak request content into the logs and traces. + if (self.LOG_LEVEL or "").upper() in ("DEBUG", "NOTSET"): + logger.warning( + "LOG_LEVEL=%s in production writes request and run details to the logs. " + "Use INFO or WARNING.", + self.LOG_LEVEL, + ) + if (self.OTEL_LEVEL or "").lower() == "full" and self.OTEL_ENABLED: + logger.warning( + "OTEL_LEVEL=full in production exports message content to your trace " + "backend. Use 'standard' unless you need it." + ) + return self model_config = ConfigDict(extra="allow") diff --git a/tenxgraph_api/src/app/core/config/setup_logs.py b/tenxgraph_api/src/app/core/config/setup_logs.py index 74fcba6b..6b128f81 100644 --- a/tenxgraph_api/src/app/core/config/setup_logs.py +++ b/tenxgraph_api/src/app/core/config/setup_logs.py @@ -4,12 +4,25 @@ from fastapi.logger import logger as fastapi_logger from tenxgraph_api.src.app.core.config.settings import LOGGER_NAME -from tenxgraph_api.src.app.core.utils.log_sanitizer import SanitizingFormatter +from tenxgraph_api.src.app.core.utils.log_sanitizer import SanitizingFormatter, redact_text # Top-level names of the loggers the API server writes to; children ("tenxgraph_api.media", # "tenxgraph_api.rate_limit", ...) reach these handlers through propagation. -APP_LOGGER_NAMES = (LOGGER_NAME, "10xgraph-api", "tenxgraph_api", "tenxgraph_api") +# "tenxgraph" is the core framework: its records (checkpoint failures, provider retries, +# publisher errors) used to be dropped because the library only installs a NullHandler. +APP_LOGGER_NAMES = (LOGGER_NAME, "10xgraph-api", "tenxgraph_api", "tenxgraph") + + +class AccessLogTokenFilter(logging.Filter): + """Redact credentials in the request line of uvicorn's access log.""" + + def filter(self, record: logging.LogRecord) -> bool: + if isinstance(record.args, tuple): + record.args = tuple( + redact_text(arg) if isinstance(arg, str) else arg for arg in record.args + ) + return True def init_logger(level: int | str = logging.INFO) -> None: @@ -34,6 +47,9 @@ def init_logger(level: int | str = logging.INFO) -> None: # gunicorn_logger = logging.getLogger("gunicorn") uvicorn_access_logger = logging.getLogger("uvicorn.access") uvicorn_access_logger.handlers = gunicorn_error_logger.handlers + # The request line is logged as written, query string included (``?token=...``). + if not any(isinstance(f, AccessLogTokenFilter) for f in uvicorn_access_logger.filters): + uvicorn_access_logger.addFilter(AccessLogTokenFilter()) fastapi_logger.handlers = gunicorn_error_logger.handlers fastapi_logger.setLevel(level) diff --git a/tenxgraph_api/src/app/core/config/setup_middleware.py b/tenxgraph_api/src/app/core/config/setup_middleware.py index 8bcfc3d5..fe863b4c 100644 --- a/tenxgraph_api/src/app/core/config/setup_middleware.py +++ b/tenxgraph_api/src/app/core/config/setup_middleware.py @@ -12,6 +12,7 @@ from starlette.requests import Request from starlette.types import ASGIApp, Message, Receive, Scope, Send +from tenxgraph_api.src.app.core.exceptions.server_errors import UnhandledErrorMiddleware from tenxgraph_api.src.app.core.middleware.rate_limit import RateLimitMiddleware, build_backend from tenxgraph_api.src.app.core.middleware.request_limits import RequestSizeLimitMiddleware from tenxgraph_api.src.app.core.middleware.security_headers import SecurityHeadersMiddleware @@ -438,13 +439,17 @@ def setup_middleware( # resulting order, outermost first: # # CORS -> request id -> security headers -> trusted host -> rate limit -> size limit - # -> gzip -> app + # -> gzip -> unhandled-error catch -> app # # Anything an inner layer answers on its own (429, 413, a bad Host) still passes back out # through CORS, request id and security headers. With CORS innermost, as before, a browser # could not read those responses at all. origins, allow_credentials = _resolve_cors_policy(settings) + # Innermost: an exception nobody handled becomes the standard JSON error here, so it still + # passes out through the layers below (request id, security headers, CORS). + app.add_middleware(UnhandledErrorMiddleware) + # Exclude streaming endpoints from compression: they must send data as it is produced. app.add_middleware(SelectiveGZipMiddleware, minimum_size=1000) @@ -481,7 +486,7 @@ def setup_middleware( app.add_middleware( HealthCheckAwareTrustedHostMiddleware, - allowed_hosts=settings.ALLOWED_HOST.split(","), + allowed_hosts=[h.strip() for h in settings.ALLOWED_HOST.split(",") if h.strip()], ) if settings.SECURITY_HEADERS_ENABLED: diff --git a/tenxgraph_api/src/app/core/exceptions/handle_errors.py b/tenxgraph_api/src/app/core/exceptions/handle_errors.py index 27a46a1e..03ed860c 100644 --- a/tenxgraph_api/src/app/core/exceptions/handle_errors.py +++ b/tenxgraph_api/src/app/core/exceptions/handle_errors.py @@ -18,6 +18,11 @@ from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.auth.request_config import ClientInputError from tenxgraph_api.src.app.core.config.settings import get_settings +from tenxgraph_api.src.app.core.exceptions.server_errors import ( + RETRY_AFTER_SECONDS, + classify, + unexpected_error_response, +) from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe from tenxgraph_api.src.app.utils import error_response from tenxgraph_api.src.app.utils.schemas import ErrorSchemas @@ -134,12 +139,17 @@ async def http_exception_handler(request: Request, exc: HTTPException): if is_production and is_server_error: logger.error(f"Request {request_id} - HTTPException details: {exc.detail}") - return error_response( + response = error_response( request, error_code="HTTPException", message=message, status_code=exc.status_code, ) + # Headers the route set on the exception (Retry-After, WWW-Authenticate) must reach + # the client. + for name, value in (exc.headers or {}).items(): + response.headers[name] = value + return response @app.exception_handler(RequestValidationError) async def validation_exception_handler(request: Request, exc: RequestValidationError): @@ -237,17 +247,17 @@ async def resource_not_found_exception_handler(request: Request, exc: APIResourc ## Need to handle 10xGraph specific exceptions here @app.exception_handler(ValidationError) async def graph_validation_exception_handler(request: Request, exc: ValidationError): - _log_handled(request, exc, 422, "AGENTFLOW_VALIDATION_ERROR", "10xGraph ValidationError") + _log_handled(request, exc, 422, "VALIDATION_ERROR", "10xGraph ValidationError") request_id = getattr(request.state, "request_id", "unknown") - message = _sanitize_error_message(str(exc), "AGENTFLOW_VALIDATION_ERROR", is_production) + message = _sanitize_error_message(str(exc), "VALIDATION_ERROR", is_production) if is_production: logger.warning("Request %s - 10xGraph ValidationError: %s", request_id, log_safe(exc)) return error_response( request, - error_code="AGENTFLOW_VALIDATION_ERROR", + error_code="VALIDATION_ERROR", message=message, status_code=422, ) @@ -294,7 +304,9 @@ async def node_error_exception_handler(request: Request, exc: NodeError): @app.exception_handler(GraphRecursionError) async def graph_recursion_error_exception_handler(request: Request, exc: GraphRecursionError): - logger.error(f"GraphRecursionError: url: {request.base_url}", exc_info=exc) + # The run hit its recursion limit: not a server fault, and the client can raise the + # limit, so it is a 422 and not a 500. + _log_handled(request, exc, 422, "RECURSION", "GraphRecursionError") request_id = getattr(request.state, "request_id", "unknown") error_code = getattr(exc, "error_code", "GRAPH_RECURSION_000") @@ -302,20 +314,27 @@ async def graph_recursion_error_exception_handler(request: Request, exc: GraphRe message = _sanitize_error_message(original_message, error_code, is_production) if is_production: - logger.error(f"Request {request_id} - GraphRecursionError: {original_message}") + logger.warning( + "Request %s - GraphRecursionError: %s", request_id, log_safe(original_message) + ) return error_response( request, error_code=error_code, message=message, details=_context_details(exc) if not is_production else None, - status_code=500, + status_code=422, ) @app.exception_handler(StorageError) async def storage_error_exception_handler(request: Request, exc: StorageError): logger.error(f"StorageError: url: {request.base_url}", exc_info=exc) + kind = classify(exc) + if kind is not None and kind.status_code == 503: # noqa: PLR2004 + # The database or cache could not be reached: retryable, not a bug. + return unexpected_error_response(request, exc) + request_id = getattr(request.state, "request_id", "unknown") error_code = getattr(exc, "error_code", "STORAGE_000") original_message = getattr(exc, "message", str(exc)) @@ -346,13 +365,15 @@ async def transient_storage_error_exception_handler( if is_production: logger.error(f"Request {request_id} - TransientStorageError: {original_message}") - return error_response( + response = error_response( request, error_code=error_code, message=message, details=_context_details(exc) if not is_production else None, status_code=503, ) + response.headers["Retry-After"] = str(RETRY_AFTER_SECONDS) + return response @app.exception_handler(MetricsError) async def metrics_error_exception_handler(request: Request, exc: MetricsError): diff --git a/tenxgraph_api/src/app/core/exceptions/server_errors.py b/tenxgraph_api/src/app/core/exceptions/server_errors.py new file mode 100644 index 00000000..9b0d87f0 --- /dev/null +++ b/tenxgraph_api/src/app/core/exceptions/server_errors.py @@ -0,0 +1,159 @@ +"""Answers for failures nobody planned for, so clients can tell a retry from a bug. + +``UnhandledErrorMiddleware`` sits innermost in the middleware stack. An exception that no +handler took becomes the same JSON envelope as every other error (with the request id, and +the security headers and CORS that the outer layers add), instead of Starlette's bare +``Internal Server Error`` text that skips them. The exception is also logged with its +traceback and sent to Sentry when it is set up. + +``classify`` tells three kinds of failure from a plain bug: a database or Redis driver that +cannot be reached (503 with ``Retry-After``), a model provider's rate limit (429 with +``Retry-After``) and a graph that hit its recursion limit (422). +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Any + +from starlette.requests import Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from tenxgraph_api.src.app.core.config.settings import get_settings +from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe +from tenxgraph_api.src.app.utils import error_response + + +logger = logging.getLogger("tenxgraph_api.errors") + +RETRY_AFTER_SECONDS = 5 + +_DRIVER_MODULES = ("redis", "asyncpg", "psycopg", "psycopg2", "psycopg_pool") +_MAX_CHAIN = 8 + + +@dataclass(frozen=True) +class Classified: + status_code: int + error_code: str + message: str + retry_after: int | None = None + + +def _chain(exc: BaseException): + """``exc`` and the exceptions it was raised from, without looping.""" + seen: set[int] = set() + current: BaseException | None = exc + while current is not None and id(current) not in seen and len(seen) < _MAX_CHAIN: + seen.add(id(current)) + yield current + current = current.__cause__ or current.__context__ + + +def _retry_after_of(exc: BaseException) -> int: + response = getattr(exc, "response", None) + headers = getattr(response, "headers", None) + try: + value = headers.get("retry-after") if headers is not None else None + return max(1, int(float(value))) if value is not None else RETRY_AFTER_SECONDS + except (TypeError, ValueError): + return RETRY_AFTER_SECONDS + + +def classify(exc: BaseException) -> Classified | None: + """A retryable or client-fixable reading of ``exc``, or ``None`` for a plain server bug.""" + for item in _chain(exc): + cls = type(item) + module = (cls.__module__ or "").split(".")[0] + name = cls.__name__ + if name == "GraphRecursionError": + return Classified( + 422, + "GRAPH_RECURSION_LIMIT", + "The graph reached its recursion limit before finishing. " + "Raise recursion_limit or check the graph for a loop.", + ) + if "RateLimit" in name or name in {"ResourceExhausted", "TooManyRequests"}: + return Classified( + 429, + "PROVIDER_RATE_LIMITED", + "The model provider is rate limiting this server. Retry shortly.", + _retry_after_of(item), + ) + if module in _DRIVER_MODULES and name.endswith( + ("Error", "Exception", "Timeout", "Interrupted") + ): + return Classified( + 503, + "SERVICE_UNAVAILABLE", + "A backing service (database or cache) is unavailable. Retry shortly.", + RETRY_AFTER_SECONDS, + ) + return None + + +def unexpected_error_response(request: Request, exc: BaseException): + """The JSON answer for ``exc``: classified when it is retryable, else a generic 500.""" + kind = classify(exc) + if kind is None: + kind = Classified(500, "INTERNAL_SERVER_ERROR", "An unexpected error occurred.") + message = kind.message + else: + message = kind.message + if get_settings().MODE != "production" and kind.status_code == 500: # noqa: PLR2004 + message = f"{type(exc).__name__}: {log_safe(exc)}" + response = error_response( + request, error_code=kind.error_code, message=message, status_code=kind.status_code + ) + if kind.retry_after is not None: + response.headers["Retry-After"] = str(kind.retry_after) + response.headers["Cache-Control"] = "no-store" + return response + + +def report(request: Request, exc: BaseException) -> None: + """Log with the traceback, and tell Sentry when it is configured.""" + logger.error( + "Unhandled %s on %s %s", + type(exc).__name__, + request.method, + log_safe(request.url.path), + exc_info=exc, + ) + try: + import sentry_sdk + + sentry_sdk.capture_exception(exc) + except Exception: # noqa: S110 - Sentry is optional and must never break the error path + pass + + +class UnhandledErrorMiddleware: + """Turn an exception nobody handled into the standard JSON error response.""" + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + started = False + + async def tracked_send(message: Message) -> None: + nonlocal started + if message["type"] == "http.response.start": + started = True + await send(message) + + try: + await self.app(scope, receive, tracked_send) + except Exception as exc: + if started: # a streaming body already left; nothing better can be sent + raise + request = Request(scope) + report(request, exc) + response: Any = unexpected_error_response(request, exc) + await response(scope, receive, send) diff --git a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py b/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py index ef29e858..c8e060c5 100644 --- a/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py +++ b/tenxgraph_api/src/app/core/middleware/rate_limit/redis.py @@ -85,7 +85,9 @@ def from_url( "Redis backend requires the 'redis' package. " f"Install the optional Redis extra with: {_REDIS_EXTRA_INSTALL}" ) - redis = AsyncRedis.from_url(redis_url, decode_responses=False) + from tenxgraph_api.src.app.core.redis_client import create_redis + + redis = create_redis(redis_url) return cls(redis=redis, prefix=prefix, fail_open=fail_open, close_redis=True) async def check(self, key: str, *, limit: int, window: int) -> RateLimitDecision: diff --git a/tenxgraph_api/src/app/core/middleware/security_headers.py b/tenxgraph_api/src/app/core/middleware/security_headers.py index cb0df4c2..097c876f 100644 --- a/tenxgraph_api/src/app/core/middleware/security_headers.py +++ b/tenxgraph_api/src/app/core/middleware/security_headers.py @@ -132,6 +132,9 @@ def _headers_for(self, is_https: bool) -> list[tuple[str, str]]: ("X-Frame-Options", self.frame_options), ("X-XSS-Protection", self.xss_protection), # legacy browsers ("Referrer-Policy", self.referrer_policy), + # API answers carry per-user data; a shared cache or the browser must not keep them. + # A route that sets its own Cache-Control (a file download) keeps it. + ("Cache-Control", "no-store"), ] if self.permissions_policy: headers.append(("Permissions-Policy", self.permissions_policy)) diff --git a/tenxgraph_api/src/app/core/redis_client.py b/tenxgraph_api/src/app/core/redis_client.py new file mode 100644 index 00000000..7a562e15 --- /dev/null +++ b/tenxgraph_api/src/app/core/redis_client.py @@ -0,0 +1,46 @@ +"""One place that builds the API's Redis clients, with timeouts and a bounded pool. + +Every Redis client the API opens goes through :func:`create_redis`, so a Redis that stops +answering fails a call after ``REDIS_SOCKET_TIMEOUT`` seconds instead of hanging the request. +The limits come from the settings (``REDIS_SOCKET_TIMEOUT``, ``REDIS_CONNECT_TIMEOUT``, +``REDIS_MAX_CONNECTIONS``, ``REDIS_POOL_TIMEOUT``, ``REDIS_HEALTH_CHECK_INTERVAL``). +""" + +from __future__ import annotations + +from typing import Any + + +def redis_options() -> dict[str, Any]: + """Connection keyword arguments built from the settings (0 turns a limit off).""" + from tenxgraph_api.src.app.core.config.settings import get_settings + + settings = get_settings() + options: dict[str, Any] = {} + if settings.REDIS_SOCKET_TIMEOUT > 0: + options["socket_timeout"] = settings.REDIS_SOCKET_TIMEOUT + if settings.REDIS_CONNECT_TIMEOUT > 0: + options["socket_connect_timeout"] = settings.REDIS_CONNECT_TIMEOUT + if settings.REDIS_HEALTH_CHECK_INTERVAL > 0: + options["health_check_interval"] = settings.REDIS_HEALTH_CHECK_INTERVAL + return options + + +def create_redis(url: str, *, decode_responses: bool = False) -> Any: + """An async Redis client for ``url`` with the configured timeouts and pool bound.""" + from redis.asyncio import BlockingConnectionPool, Redis # type: ignore[import] + + from tenxgraph_api.src.app.core.config.settings import get_settings + + settings = get_settings() + options = redis_options() + if settings.REDIS_MAX_CONNECTIONS > 0: + pool = BlockingConnectionPool.from_url( + url, + decode_responses=decode_responses, + max_connections=settings.REDIS_MAX_CONNECTIONS, + timeout=settings.REDIS_POOL_TIMEOUT or None, + **options, + ) + return Redis.from_pool(pool) + return Redis.from_url(url, decode_responses=decode_responses, **options) diff --git a/tenxgraph_api/src/app/core/shutdown.py b/tenxgraph_api/src/app/core/shutdown.py new file mode 100644 index 00000000..47c9445c --- /dev/null +++ b/tenxgraph_api/src/app/core/shutdown.py @@ -0,0 +1,114 @@ +"""Server shutdown signal shared by the worker, the lifespan and open response streams. + +Uvicorn stops accepting connections on SIGTERM, waits for open responses up to +``timeout_graceful_shutdown`` and then cancels them, which a client sees as a cut connection. +The worker (``tenxgraph_api.src.app.uvicorn_worker``) calls :func:`interrupt_streams` shortly +before that deadline; :func:`end_on_shutdown` then stops each open stream and sends a final +chunk saying why, so the client can resume the thread instead of guessing. +""" + +from __future__ import annotations + +import asyncio +import contextlib +from collections.abc import AsyncIterator, Callable + +from tenxgraph.core.state import StreamChunk, StreamEvent + + +INTERRUPTED_REASON = ( + "The server is shutting down and this run was interrupted. " + "The thread is saved: send a new request on the same thread_id to resume." +) + +# Sent on an idle stream so a proxy's idle timeout does not cut a run that is still working. +# A blank line: NDJSON readers skip it (the TypeScript client does), and it is still bytes on +# the wire. +KEEPALIVE_LINE = "\n" +KEEPALIVE_SECONDS = 15.0 + +_interrupted = False +_waiters: set[asyncio.Future[None]] = set() + + +def interrupt_streams() -> None: + """Tell every open stream wrapped by :func:`end_on_shutdown` to finish now.""" + global _interrupted # noqa: PLW0603 + _interrupted = True + for waiter in list(_waiters): + if not waiter.done(): + waiter.set_result(None) + + +def reset() -> None: + """Clear the signal (tests).""" + global _interrupted # noqa: PLW0603 + _interrupted = False + _waiters.clear() + + +def interrupted_line() -> str: + """The final NDJSON line of a stream cut by shutdown.""" + chunk = StreamChunk( + event=StreamEvent.ERROR, + data={"reason": INTERRUPTED_REASON, "interrupted": True}, + ) + return chunk.model_dump_json(serialize_as_any=True) + "\n" + + +async def _next(stream: AsyncIterator[str]) -> str: + return await anext(stream) + + +async def end_on_shutdown( + stream: AsyncIterator[str], + final: Callable[[], str] = interrupted_line, + keepalive_seconds: float | None = None, +) -> AsyncIterator[str]: + """Yield ``stream``; when shutdown is signalled, stop it and yield ``final()`` last. + + With ``keepalive_seconds``, a blank line is yielded each time the stream has been silent + that long, so idle proxies keep the connection open. + """ + loop = asyncio.get_running_loop() + waiter: asyncio.Future[None] = loop.create_future() + if _interrupted: + waiter.set_result(None) + _waiters.add(waiter) + pending: asyncio.Future[str] | None = None + try: + while True: + # Eager: a chunk that is ready without waiting costs no event-loop turn. + pending = asyncio.Task(_next(stream), loop=loop, eager_start=True) + while not pending.done() and not waiter.done(): + await asyncio.wait( + {pending, waiter}, + timeout=keepalive_seconds, + return_when=asyncio.FIRST_COMPLETED, + ) + if not pending.done() and not waiter.done(): + yield KEEPALIVE_LINE + if not pending.done(): + pending.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await pending + pending = None + yield final() + return + try: + item = pending.result() + except StopAsyncIteration: + pending = None + return + pending = None + yield item + finally: + _waiters.discard(waiter) + if pending is not None and not pending.done(): + pending.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await pending + aclose = getattr(stream, "aclose", None) + if callable(aclose): + with contextlib.suppress(Exception): + await aclose() diff --git a/tenxgraph_api/src/app/core/utils/log_sanitizer.py b/tenxgraph_api/src/app/core/utils/log_sanitizer.py index cf6104dd..be8158c4 100644 --- a/tenxgraph_api/src/app/core/utils/log_sanitizer.py +++ b/tenxgraph_api/src/app/core/utils/log_sanitizer.py @@ -10,33 +10,50 @@ from typing import Any -# Patterns for detecting sensitive field names +# Words that mark a field name as sensitive. A key is split into words (on ``_``, ``-``, ``.``, +# spaces and camelCase) and redacted when a word matches, so ``api_key`` and ``apiKey`` are +# redacted but ``monkey``, ``shipping`` and ``author`` are not. SENSITIVE_PATTERNS = { "token", + "tokens", "password", + "passwd", "secret", + "secrets", "key", + "keys", "credential", + "credentials", "authorization", - "api_key", - "access_token", - "refresh_token", "auth", "bearer", "jwt", "session", "cookie", - "private_key", "passphrase", "pin", "ssn", + # Compound names, kept for code that reads this set; matched by word or compound. + "api_key", + "access_token", + "refresh_token", + "private_key", "credit_card", } +# Names whose words are harmless alone but sensitive together (matched with separators removed). +_SENSITIVE_COMPOUNDS = {"apikey", "accesstoken", "refreshtoken", "privatekey", "creditcard"} + +_KEY_WORD_SPLIT = re.compile(r"[^A-Za-z0-9]+|(?<=[a-z0-9])(?=[A-Z])") + # Regex pattern to detect JWT tokens (three base64url parts separated by dots) # Each part excludes ".", so there is one way to split a string: no polynomial backtracking. JWT_PATTERN = re.compile(r"^[A-Za-z0-9_=-]+\.[A-Za-z0-9_=-]+(?:\.[A-Za-z0-9_.+/=-]*)?$") +_UUID_PATTERN = re.compile( + r"^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$" +) + # Regex pattern to detect bearer tokens BEARER_PATTERN = re.compile(r"^Bearer\s+[A-Za-z0-9\-_.~+/]+=*$", re.IGNORECASE) @@ -44,6 +61,20 @@ _INLINE_SECRET_PATTERNS = ( (re.compile(r"Bearer\s+[A-Za-z0-9\-_.~+/]+=*", re.IGNORECASE), "Bearer ***REDACTED***"), (re.compile(r"\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]*"), "***JWT_TOKEN***"), + # Provider and cloud keys: OpenAI/Anthropic (sk-...), Google (AIza...), GitHub, Slack, AWS. + (re.compile(r"\bsk-[A-Za-z0-9_-]{16,}"), "***API_KEY***"), + (re.compile(r"\bAIza[0-9A-Za-z_-]{20,}"), "***API_KEY***"), + (re.compile(r"\bgh[pousr]_[A-Za-z0-9]{20,}"), "***API_KEY***"), + (re.compile(r"\bxox[abpr]-[A-Za-z0-9-]{10,}"), "***API_KEY***"), + (re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"), "***API_KEY***"), + # A credential in a query string (the WebSocket handshake passes ``?token=``). + ( + re.compile( + r"([?&](?:token|access_token|id_token|api_key|apikey|key|secret)=)[^&\s\"']+", + re.IGNORECASE, + ), + r"\1***REDACTED***", + ), ) @@ -112,6 +143,14 @@ def sanitize_for_logging(data: Any, max_depth: int = 10, _current_depth: int = 0 return data +def _is_sensitive_key(key: Any) -> bool: + text = str(key) + words = {word.lower() for word in _KEY_WORD_SPLIT.split(text) if word} + if words & SENSITIVE_PATTERNS: + return True + return "".join(_KEY_WORD_SPLIT.split(text)).lower() in _SENSITIVE_COMPOUNDS + + def _sanitize_value(key: str, value: Any, max_depth: int, current_depth: int) -> Any: """ Sanitize a value based on its key name and content. @@ -126,8 +165,7 @@ def _sanitize_value(key: str, value: Any, max_depth: int, current_depth: int) -> Sanitized value """ # Check if key name contains sensitive patterns - key_lower = key.lower() - if any(pattern in key_lower for pattern in SENSITIVE_PATTERNS): + if _is_sensitive_key(key): return "***REDACTED***" # Recursively sanitize the value @@ -145,6 +183,9 @@ def _sanitize_string(value: str) -> str: Original string or redaction marker """ max_value = 32 + # An id is not a secret, and operators grep for it: leave UUIDs as they are. + if _UUID_PATTERN.match(value): + return value # Check for JWT token pattern if len(value) > max_value and JWT_PATTERN.match(value): return "***JWT_TOKEN***" @@ -161,6 +202,29 @@ def _sanitize_string(value: str) -> str: return value +def _clean_arg(arg: Any) -> Any: + cleaned = sanitize_for_logging(arg) + return log_safe(cleaned) if isinstance(cleaned, str) else cleaned + + +def summarize_validation_error(exc: Any) -> str: + """A pydantic ``ValidationError`` as ``loc: message`` pairs, without the submitted values. + + ``str(exc)`` quotes the offending input (``input_value=...``), which can be a message, a + file or a secret; this is safe for logs and for text sent back to a client. + """ + try: + errors = exc.errors(include_input=False, include_url=False, include_context=False) + except TypeError: # an older pydantic + errors = exc.errors() + parts = [ + f"{'.'.join(str(item) for item in error.get('loc', ())) or 'input'}: " + f"{error.get('msg', 'invalid')}" + for error in errors + ] + return "; ".join(parts) or "invalid input" + + def sanitize_log_message(message: str, *args: Any, **kwargs: Any) -> tuple[str, tuple, dict]: """ Sanitize log message arguments. @@ -223,9 +287,10 @@ def format(self, record: logging.LogRecord) -> str: Returns: Formatted and sanitized log string """ - # Sanitize the message arguments + # Sanitize the message arguments. Text arguments also lose CR/LF and other control + # characters, so a value that carries a newline cannot forge a log line. if isinstance(record.args, tuple) and record.args: - record.args = tuple(sanitize_for_logging(arg) for arg in record.args) + record.args = tuple(_clean_arg(arg) for arg in record.args) elif isinstance(record.args, dict): record.args = sanitize_for_logging(record.args) diff --git a/tenxgraph_api/src/app/loader.py b/tenxgraph_api/src/app/loader.py index 45b01962..93c75971 100644 --- a/tenxgraph_api/src/app/loader.py +++ b/tenxgraph_api/src/app/loader.py @@ -325,7 +325,7 @@ def _build_ownership_redis(redis_url: str | None) -> object | None: if not redis_url: return None try: - from redis.asyncio import Redis as AsyncRedis # type: ignore[import] + import redis.asyncio # type: ignore[import] # noqa: F401 except ImportError: logger.warning( "REDIS configured but the 'redis' package is not installed; ownership cache " @@ -333,7 +333,9 @@ def _build_ownership_redis(redis_url: str | None) -> object | None: ) return None try: - client = AsyncRedis.from_url(redis_url, decode_responses=False) + from tenxgraph_api.src.app.core.redis_client import create_redis + + client = create_redis(redis_url) logger.info("Ownership authorization L2 cache enabled (shared Redis).") return client except Exception as exc: # pragma: no cover - defensive @@ -525,6 +527,13 @@ async def attach_all_modules( logger.info("All modules attached successfully") + if config.checkpointer_path: + logger.warning( + 'The "checkpointer" key in 10xgraph.json is not applied by the server and is ' + "ignored. Pass the checkpointer to graph.compile(checkpointer=...) in your agent " + "module instead." + ) + # This binding we have done already in the library # # Bind checkpointer instance if configured # checkpointer = load_checkpointer(config.checkpointer_path) diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index 0b4ccb41..33cc0230 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -1,3 +1,4 @@ +import asyncio import logging import os @@ -31,6 +32,10 @@ # draining for a fixed window and then moving on. SHUTDOWN_DRAIN_TIMEOUT_SECONDS = 30.0 +# Boot-time temp media cleanup is best effort. It must not hold the app (and so the first +# readiness probe) behind a slow or wedged checkpointer or media store. +BOOT_CLEANUP_TIMEOUT_SECONDS = 15.0 + # GraphConfig loads the ``env`` file named in 10xgraph.json, so it must run before Settings # is first built. `10xgraph dev` loads that file itself, but plain `gunicorn ...main:app` # (the generated Dockerfile) does not, and Settings is cached after the first read. @@ -78,7 +83,13 @@ async def lifespan(app: FastAPI): ) # Clean up expired temporary media cache on startup - await _cleanup_temp_media_cache() + try: + await asyncio.wait_for(_cleanup_temp_media_cache(), BOOT_CLEANUP_TIMEOUT_SECONDS) + except TimeoutError: + logger.warning( + "Temporary media cache cleanup timed out after %ss; continuing startup", + BOOT_CLEANUP_TIMEOUT_SECONDS, + ) # load Store # store = load_store(graph_config.store_path) @@ -86,6 +97,8 @@ async def lifespan(app: FastAPI): yield + app.state.draining = True # readiness answers 503 from here on + # Shutdown. Order matters: drain in-flight work BEFORE tearing down the # resources that work depends on. # @@ -165,7 +178,7 @@ async def _drain_background_tasks(timeout: float = SHUTDOWN_DRAIN_TIMEOUT_SECOND app = FastAPI( title=settings.APP_NAME, version=settings.APP_VERSION, - debug=settings.MODE == "DEVELOPMENT", + debug=settings.IS_DEBUG, summary=settings.SUMMARY, docs_url=settings.DOCS_PATH if settings.DOCS_PATH else None, redoc_url=settings.REDOCS_PATH if settings.REDOCS_PATH else None, diff --git a/tenxgraph_api/src/app/routers/ag_ui/converter.py b/tenxgraph_api/src/app/routers/ag_ui/converter.py index 09852a7f..4e68a151 100644 --- a/tenxgraph_api/src/app/routers/ag_ui/converter.py +++ b/tenxgraph_api/src/app/routers/ag_ui/converter.py @@ -31,6 +31,7 @@ ClientInputError, request_size_error, ) +from tenxgraph_api.src.app.core.utils.log_sanitizer import summarize_validation_error _MEDIA_BLOCKS = { @@ -69,7 +70,9 @@ def parse_run_input(body: Any) -> RunAgentInput: try: return RunAgentInput.model_validate(body) except ValidationError as exc: - raise ClientInputError(f"Invalid AG-UI RunAgentInput: {exc}") from exc + raise ClientInputError( + f"Invalid AG-UI RunAgentInput: {summarize_validation_error(exc)}" + ) from exc def _trailing_turn(messages: list[Any]) -> list[Any]: diff --git a/tenxgraph_api/src/app/routers/ag_ui/router.py b/tenxgraph_api/src/app/routers/ag_ui/router.py index dfed634a..2122edee 100644 --- a/tenxgraph_api/src/app/routers/ag_ui/router.py +++ b/tenxgraph_api/src/app/routers/ag_ui/router.py @@ -15,12 +15,12 @@ from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.config.settings import get_settings from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe -from tenxgraph_api.src.app.routers.graph.run_limits import acquire_run_slot, release_when_done +from tenxgraph_api.src.app.routers.graph.run_guard import acquire_run_guard +from tenxgraph_api.src.app.routers.graph.run_limits import release_when_done from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService from tenxgraph_api.src.app.routers.graph.services.multimodal_preprocessor import ( MediaURLBlockedError, ) -from tenxgraph_api.src.app.routers.graph.thread_lock import acquire_thread_lock, combine_leases from .converter import check_client_tools, parse_run_input, to_tenxgraph_messages from .service import stream_ag_ui @@ -98,12 +98,7 @@ async def run_ag_ui( encoder = EventEncoder(accept=request.headers.get("accept") or "") # One run at a time per thread (AG-UI always waits); taken before the response starts so a # busy thread is a real 409. - thread_lock = await acquire_thread_lock(thread_id) - try: - slot = await acquire_run_slot(user, service.runs_policy()) - except BaseException: - thread_lock.release() - raise + guard = await acquire_run_guard(user=user, runs=service.runs_policy(), thread_id=thread_id) return StreamingResponse( release_when_done( stream_ag_ui( @@ -113,7 +108,7 @@ async def run_ag_ui( encoder, allow_client_tools=config.ag_ui.allow_client_tools, ), - combine_leases(slot, thread_lock), + guard, ), media_type=encoder.get_content_type(), headers={ diff --git a/tenxgraph_api/src/app/routers/graph/router.py b/tenxgraph_api/src/app/routers/graph/router.py index a645e22d..ecd40f29 100644 --- a/tenxgraph_api/src/app/routers/graph/router.py +++ b/tenxgraph_api/src/app/routers/graph/router.py @@ -1,6 +1,8 @@ import asyncio import contextlib import json +import time +from collections import deque from typing import Any from fastapi import APIRouter, Depends, HTTPException, Request, WebSocket, WebSocketDisconnect @@ -24,17 +26,14 @@ token_revoked, ) from tenxgraph_api.src.app.core.concurrency import Lease -from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe +from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe, summarize_validation_error from tenxgraph_api.src.app.routers.graph.realtime_guard import ( realtime_connection_guard, ws_identity_still_valid, ws_run_allowed, ) -from tenxgraph_api.src.app.routers.graph.run_limits import ( - acquire_run_slot, - release_when_done, - run_slot, -) +from tenxgraph_api.src.app.routers.graph.run_guard import acquire_run_guard +from tenxgraph_api.src.app.routers.graph.run_limits import release_when_done from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( FixGraphRequestSchema, GraphInputSchema, @@ -46,12 +45,7 @@ WsGraphInputSchema, ) from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService -from tenxgraph_api.src.app.routers.graph.thread_lock import ( - THREAD_BUSY_DETAIL, - acquire_thread_lock, - combine_leases, - lock_requested_thread, -) +from tenxgraph_api.src.app.routers.graph.thread_lock import THREAD_BUSY_DETAIL, requested_thread_id from tenxgraph_api.src.app.utils import success_response from tenxgraph_api.src.app.utils.swagger_helper import generate_swagger_responses @@ -92,11 +86,94 @@ # session while keeping the exposure window short next to a session that can run for hours. REALTIME_RECHECK_INTERVAL = 30.0 +# RFC 6455 close code 1013 "Try Again Later", for a refused or shed run. +WS_TRY_AGAIN_LATER = 1013 + +# Longest one write to a client socket may take. A reader that stops reading would otherwise +# park the run (and the thread lock and run slot it holds) on a full send buffer for good. This +# bounds our own socket write only, never the run: a quiet run, however long, is not affected. +WS_SEND_TIMEOUT_SECONDS = 30.0 + +# Frames of a ``/v1/graph/ws`` socket read ahead while a run is in progress. Reading is what +# notices a client that left, so its run can be stopped; the cap keeps a chatty client from +# buffering without bound (past it the socket is left to TCP back-pressure). +WS_READ_AHEAD_FRAMES = 2 + +# Frames in a row that fail to parse or validate before the socket is closed (1008). Valid +# frames reset the count. Each one also counts against the rate limit, like a run does. +WS_MAX_INVALID_FRAMES = 20 + class WsFrameTooLarge(Exception): """A JSON text frame passed the size cap; the socket must be closed with 1009.""" +class WsSendTimeout(Exception): + """A client socket did not take a write in time; the socket must be closed.""" + + +async def _ws_send(websocket: WebSocket, text: str) -> None: + """``send_text`` that gives up on a client that is not reading (``WsSendTimeout``).""" + try: + await asyncio.wait_for(websocket.send_text(text), WS_SEND_TIMEOUT_SECONDS) + except TimeoutError: + raise WsSendTimeout from None + + +async def _ws_send_bytes(websocket: WebSocket, data: bytes) -> None: + try: + await asyncio.wait_for(websocket.send_bytes(data), WS_SEND_TIMEOUT_SECONDS) + except TimeoutError: + raise WsSendTimeout from None + + +async def _cancel_tasks(*tasks: asyncio.Task[Any]) -> None: + """Cancel ``tasks`` and wait until they are gone. + + Their own cancellation is absorbed; a cancel aimed at the caller still propagates (it is + not swallowed the way ``suppress(CancelledError)`` around ``await task`` would). + """ + pending = [task for task in tasks if not task.done()] + for task in pending: + task.cancel() + if pending: + await asyncio.gather(*pending, return_exceptions=True) + + +class _WsFrames: + """The text frames of a ``/v1/graph/ws`` socket. + + While a run is in progress :meth:`read_ahead` keeps reading, so a client that leaves is + noticed (``gone``) and its run can be stopped; frames it sent meanwhile are kept, in order, + for the next :meth:`next`. + """ + + def __init__(self, websocket: WebSocket) -> None: + self._websocket = websocket + self._ahead: deque[str] = deque() + self.gone = False + + async def next(self) -> str: + if self._ahead: + return self._ahead.popleft() + if self.gone: + raise WebSocketDisconnect + return await self._websocket.receive_text() + + async def read_ahead(self) -> None: + try: + while len(self._ahead) < WS_READ_AHEAD_FRAMES: + self._ahead.append(await self._websocket.receive_text()) + except Exception: # the client left (or the socket is already closed) + self.gone = True + + +def _parse_json_frame(text: str, max_chars: int) -> Any: + if len(text) > max_chars: + raise WsFrameTooLarge(f"Frame of {len(text)} characters exceeds the {max_chars} limit") + return json.loads(text) + + async def _receive_json_frame(websocket: WebSocket, max_chars: int) -> Any: """Receive one JSON text frame, refusing it before parsing when it is over ``max_chars``. @@ -105,10 +182,7 @@ async def _receive_json_frame(websocket: WebSocket, max_chars: int) -> Any: frames far larger than the 10 MB HTTP body cap. ``websocket.receive_text`` raises ``WebSocketDisconnect`` on disconnect, like ``receive_json``. """ - text = await websocket.receive_text() - if len(text) > max_chars: - raise WsFrameTooLarge(f"Frame of {len(text)} characters exceeds the {max_chars} limit") - return json.loads(text) + return _parse_json_frame(await websocket.receive_text(), max_chars) def _ws_max_frame_chars() -> int: @@ -135,15 +209,66 @@ async def _send_ws_error(websocket: WebSocket, reason: str) -> None: ) -async def _next_ws_input(websocket: WebSocket, max_frame: int) -> Any: +def _seconds_until_expiry(user: dict[str, Any]) -> float | None: + """Seconds until the identity's ``exp`` claim, or ``None`` when it has none.""" + exp = user.get("exp") if isinstance(user, dict) else None + if isinstance(exp, bool) or not isinstance(exp, int | float): + return None + return max(float(exp) - time.time(), 0.0) + 0.5 + + +class _SessionExpired(Exception): + """The socket sat idle past its token's expiry.""" + + +async def _next_frame(frames: _WsFrames, websocket: WebSocket, user: dict[str, Any]) -> str: + """The next text frame; an idle socket is cut once its credential stops verifying. + + Without this, a socket that sends nothing keeps its slot long after the token that opened + it expired (the per-run check only runs when a frame arrives). + """ + while True: + wait = _seconds_until_expiry(user) + if wait is None: + return await frames.next() + try: + return await asyncio.wait_for(frames.next(), wait) + except TimeoutError: + if not ws_identity_still_valid(websocket, user): + raise _SessionExpired from None + + +async def _close_expired(websocket: WebSocket) -> None: + logger.info("WebSocket credential expired while idle; closing") + with contextlib.suppress(Exception): + await websocket.send_text( + StreamChunk( + event=StreamEvent.ERROR, + data={"reason": "Session expired. Reconnect with a valid token."}, + ).model_dump_json() + ) + with contextlib.suppress(Exception): + await websocket.close(code=1008) + + +async def _next_ws_input( + frames: _WsFrames, + websocket: WebSocket, + max_frame: int, + user: dict[str, Any] | None = None, +) -> Any: """Read the next run request from ``/v1/graph/ws``. Returns the parsed ``WsGraphInputSchema``, ``_WS_SKIP`` for a frame that was rejected with an error message (the socket stays open), or ``None`` when the socket is done: the client - left, or sent a frame over the size cap (closed with 1009). + left, sent a frame over the size cap (closed with 1009), or its token expired while idle. """ try: - data = await _receive_json_frame(websocket, max_frame) + text = await _next_frame(frames, websocket, user or {}) + data = _parse_json_frame(text, max_frame) + except _SessionExpired: + await _close_expired(websocket) + return None except WebSocketDisconnect: logger.info("WebSocket client disconnected") return None @@ -157,8 +282,9 @@ async def _next_ws_input(websocket: WebSocket, max_frame: int) -> Any: try: return WsGraphInputSchema(**data) except Exception as e: - logger.warning("WebSocket received invalid input: %s", e) - await _send_ws_error(websocket, f"Invalid request: {e}") + logger.warning("WebSocket received invalid input: %s", _error_summary(e)) + # The validation text can quote the submitted values; production sends the generic one. + await _send_ws_error(websocket, f"Invalid request: {_client_error(e)}") return _WS_SKIP @@ -197,6 +323,24 @@ async def _bind_ws_container() -> None: logger.exception("Failed to bind InjectQ container for WebSocket scope") +async def _http_run_guard( + service: GraphService, + graph_input: GraphInputSchema, + user: dict[str, Any], + request: Request | None = None, +) -> Lease: + """The thread lock and run slot for an HTTP run (``on_busy`` as the request set it).""" + thread_id = requested_thread_id(graph_input.config) + return await acquire_run_guard( + user=user, + runs=service.runs_policy(), + thread_id=thread_id, + on_busy=graph_input.on_busy, + stop=lambda: service.stop_graph(thread_id, user), + is_disconnected=request.is_disconnected if request is not None else None, + ) + + @router.post( "/v1/graph/invoke", summary="Invoke graph execution", @@ -216,20 +360,15 @@ async def invoke_graph( logger.info(f"Graph invoke request received with {len(graph_input.messages)} messages") service.check_media_urls(graph_input.messages) - # One run at a time per thread; 409 when the thread stays busy. - thread_lock = await lock_requested_thread( - graph_input.config, - graph_input.on_busy, - lambda thread_id: service.stop_graph(thread_id, user), - ) + # One run at a time per thread (409 when it stays busy) and the per-user run cap (429). + guard = await _http_run_guard(service, graph_input, user, request) try: - async with run_slot(user, service.runs_policy()): - result: GraphInvokeOutputSchema = await service.invoke_graph( - graph_input, - user, - ) + result: GraphInvokeOutputSchema = await service.invoke_graph( + graph_input, + user, + ) finally: - thread_lock.release() + guard.release() logger.info("Graph invoke completed successfully") @@ -250,6 +389,7 @@ async def stream_graph( graph_input: GraphInputSchema, service: GraphService = InjectAPI(GraphService), user: dict[str, Any] = Depends(RequirePermission("graph", "stream")), + request: Request = None, # type: ignore[assignment] # injected by FastAPI ): """ Stream the graph execution with real-time output. @@ -258,25 +398,20 @@ async def stream_graph( # Before the response starts, so a blocked URL is a real 400 and not an error chunk. service.check_media_urls(graph_input.messages) - # Both taken before the response starts, so a busy thread still gets a real 409 and an + # Taken before the response starts, so a busy thread still gets a real 409 and an # over-cap request a 429. - thread_lock = await lock_requested_thread( - graph_input.config, - graph_input.on_busy, - lambda thread_id: service.stop_graph(thread_id, user), - ) + guard = await _http_run_guard(service, graph_input, user, request) try: - slot = await acquire_run_slot(user, service.runs_policy()) + # Same for a resume aimed at the wrong interrupt: a 409, as invoke answers it. + await service.check_resume_target(graph_input, user) except BaseException: - thread_lock.release() + guard.release() raise - result = release_when_done( - service.stream_graph(graph_input, user), combine_leases(slot, thread_lock) - ) + result = release_when_done(service.stream_graph(graph_input, user), guard) return StreamingResponse( result, - media_type="text/event-stream", + media_type="application/x-ndjson", headers={ "Cache-Control": "no-cache, no-transform", "Connection": "keep-alive", @@ -618,15 +753,46 @@ async def _ws_media_urls_allowed( return True -async def _ws_thread_lock( - service: GraphService, user: dict[str, Any], thread_id: str | None, on_busy: Any -) -> Lease: - """The run lock for a WebSocket run (a no-op lease for a thread that does not exist yet).""" - if thread_id is None: - return Lease(None) - return await acquire_thread_lock( - thread_id, on_busy=on_busy, stop=lambda: service.stop_graph(thread_id, user) - ) +def _guard_refusal(exc: HTTPException) -> str: + """The reason a run was refused (thread busy, run cap) as a client-facing sentence.""" + return THREAD_BUSY_DETAIL if exc.status_code == 409 else str(exc.detail) # noqa: PLR2004 + + +async def _ws_run( + websocket: WebSocket, + service: GraphService, + graph_input: GraphInputSchema, + user: dict[str, Any], + frames: _WsFrames, +) -> bool: + """Stream one run to the socket; ``False`` when the client left and the run was stopped. + + The run is its own task, so a client that leaves while the run is quiet (a long tool call) + is noticed and the run cancelled, instead of holding the lock and slot until its next + write. The run generator is closed before this returns, so the caller can release the + thread lock knowing the run is over. + """ + stream = service.stream_graph(graph_input, user=user) + + async def pump() -> None: + async with contextlib.aclosing(stream): + async for chunk_json in stream: + await _ws_send(websocket, chunk_json) + + run_task = asyncio.create_task(pump()) + reader = asyncio.create_task(frames.read_ahead()) + try: + await asyncio.wait({run_task, reader}, return_when=asyncio.FIRST_COMPLETED) + if not run_task.done() and frames.gone: + logger.info("WebSocket client left during a run; stopping the run") + run_task.cancel() + await asyncio.wait({run_task}) + finally: + await _cancel_tasks(run_task, reader) + if run_task.cancelled(): + return False + run_task.result() + return True @router.websocket("/v1/graph/ws") @@ -713,13 +879,26 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 return max_frame = _ws_max_frame_chars() + frames = _WsFrames(websocket) + invalid_frames = 0 try: while True: - ws_input = await _next_ws_input(websocket, max_frame) + ws_input = await _next_ws_input(frames, websocket, max_frame, user) if ws_input is None: break if ws_input is _WS_SKIP: + # A frame that was refused still cost a parse: it counts against the rate + # limit, and a socket that only sends junk is closed. + invalid_frames += 1 + retry_after = await ws_run_allowed(websocket) + if retry_after is not None or invalid_frames > WS_MAX_INVALID_FRAMES: + code = 1008 if retry_after is None else WS_TRY_AGAIN_LATER + logger.warning("WebSocket closed after too many invalid frames") + with contextlib.suppress(Exception): + await websocket.close(code=code) + break continue + invalid_frames = 0 # The handshake checks happen once; these apply to every run on the socket. gate = await _ws_run_gate(websocket, user) @@ -754,29 +933,41 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 if not await _ws_media_urls_allowed(websocket, service, graph_input): continue try: - thread_lock = await _ws_thread_lock(service, user, thread_id, ws_input.on_busy) - except HTTPException: + guard = await acquire_run_guard( + user=user, + runs=service.runs_policy(), + thread_id=thread_id, + on_busy=ws_input.on_busy, + stop=lambda tid=thread_id: service.stop_graph(tid, user), + ) + except HTTPException as exc: await websocket.send_text( StreamChunk( - event=StreamEvent.ERROR, data={"reason": THREAD_BUSY_DETAIL} + event=StreamEvent.ERROR, data={"reason": _guard_refusal(exc)} ).model_dump_json() ) continue try: - async for chunk_json in service.stream_graph(graph_input, user=user): - await websocket.send_text(chunk_json) + finished = await _ws_run(websocket, service, graph_input, user, frames) finally: - thread_lock.release() + guard.release() + if not finished: + break - await websocket.send_text( + await _ws_send( + websocket, StreamChunk( event=StreamEvent.UPDATES, data={"status": "done"}, - ).model_dump_json() + ).model_dump_json(), ) except WebSocketDisconnect: logger.info("WebSocket client disconnected") + except WsSendTimeout: + logger.warning("WebSocket client is not reading; closing the socket") + with contextlib.suppress(Exception): + await websocket.close(code=WS_TRY_AGAIN_LATER) except Exception as e: logger.error("WebSocket graph connection error: %s", e) with contextlib.suppress(Exception): @@ -793,7 +984,14 @@ def _client_error(exc: Exception) -> str: from tenxgraph_api.src.app.core.exceptions.handle_errors import _sanitize_error_message is_production = get_settings().MODE == "production" - return _sanitize_error_message(str(exc), "VALIDATION_ERROR", is_production) + return _sanitize_error_message(_error_summary(exc), "VALIDATION_ERROR", is_production) + + +def _error_summary(exc: Exception) -> str: + """``exc`` as text; a pydantic validation error without the values it quotes.""" + if isinstance(exc, ValidationError): + return summarize_validation_error(exc) + return str(exc) class _ByteBoundedLiveInputQueue(LiveInputQueue): @@ -972,6 +1170,7 @@ async def realtime_graph_ws( # noqa: PLR0911, PLR0912, PLR0915 await websocket.close(code=1003) return + thread_id: str | None = None if isinstance(init, dict): # Same rules as /v1/graph/ws: "new" starts a fresh thread, never a shared one. thread_id = _resolve_ws_thread(init) @@ -1024,6 +1223,28 @@ async def close_session_expired() -> None: await close_session_expired() return + # The same guard as every other run: one session at a time on a thread, and the session + # counts against the user's run cap for as long as it lasts. + try: + guard = await acquire_run_guard( + user=user, runs=service.runs_policy(), thread_id=thread_id, on_busy="wait" + ) + except HTTPException as exc: + busy = exc.status_code == 409 # noqa: PLR2004 + with contextlib.suppress(Exception): + await websocket.send_text( + _realtime_event_json( + ErrorEvent( + code="thread_busy" if busy else "too_many_runs", + message=_guard_refusal(exc), + fatal=True, + ) + ) + ) + with contextlib.suppress(Exception): + await websocket.close(code=WS_TRY_AGAIN_LATER) + return + queue = _ByteBoundedLiveInputQueue(maxsize=REALTIME_INPUT_QUEUE_MAXSIZE) async def upstream() -> None: @@ -1057,9 +1278,9 @@ async def downstream() -> None: try: async for event in service.realtime_graph(queue, init, user): if getattr(event, "type", None) == "audio_delta": - await websocket.send_bytes(event.data) + await _ws_send_bytes(websocket, event.data) else: - await websocket.send_text(_realtime_event_json(event)) + await _ws_send(websocket, _realtime_event_json(event)) except (ValidationError, ValueError) as e: # Bad session config from the init frame (e.g. an invalid ``modalities`` value) # is a client error, not a server fault. Send a normalized, fatal error event @@ -1080,10 +1301,14 @@ async def credential_watchdog() -> None: await close_session_expired() return - up_task = asyncio.create_task(upstream()) - down_task = asyncio.create_task(downstream()) - watch_task = asyncio.create_task(credential_watchdog()) + tasks: list[asyncio.Task[None]] = [] try: + up_task = asyncio.create_task(upstream()) + tasks.append(up_task) + down_task = asyncio.create_task(downstream()) + tasks.append(down_task) + watch_task = asyncio.create_task(credential_watchdog()) + tasks.append(watch_task) done, pending = await asyncio.wait( {up_task, down_task, watch_task}, return_when=asyncio.FIRST_COMPLETED ) @@ -1091,10 +1316,7 @@ async def credential_watchdog() -> None: # Credential gone: no grace period, stop at once. if watch_task in done: queue.close() - for task in (up_task, down_task): - task.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await task + await _cancel_tasks(up_task, down_task) return # If the client side finished first (sent ``close`` or disconnected) while the @@ -1108,11 +1330,7 @@ async def credential_watchdog() -> None: # Cancel whatever is still running: downstream that overran the grace window, or # upstream once the session ended. - for task in (up_task, down_task, watch_task): - if not task.done(): - task.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await task + await _cancel_tasks(up_task, down_task, watch_task) # asyncio.wait never re-raises; surface any failure from the finished task(s) # (e.g. arealtime rejecting a non-live graph, or a provider/checkpointer error). @@ -1123,11 +1341,18 @@ async def credential_watchdog() -> None: # The client going away mid-session is a normal termination, not a server fault; # don't log it as an error or attempt a 1011 close on an already-closed socket. logger.info("Realtime WebSocket client disconnected") + except WsSendTimeout: + logger.warning("Realtime client is not reading; closing the socket") + with contextlib.suppress(Exception): + await websocket.close(code=WS_TRY_AGAIN_LATER) except Exception as e: logger.error("Realtime WebSocket error: %s", e) with contextlib.suppress(Exception): await websocket.close(code=1011) finally: queue.close() + # Also when this handler itself is cancelled (shutdown): no child task outlives it. + await _cancel_tasks(*tasks) + guard.release() with contextlib.suppress(Exception): await websocket.close() diff --git a/tenxgraph_api/src/app/routers/graph/run_guard.py b/tenxgraph_api/src/app/routers/graph/run_guard.py new file mode 100644 index 00000000..7245d467 --- /dev/null +++ b/tenxgraph_api/src/app/routers/graph/run_guard.py @@ -0,0 +1,51 @@ +"""The guard every run entry point takes before a graph run starts. + +A run needs two things held for as long as it lasts: the thread's one-run lock +(``thread_lock``) and the user's run slot (``run_limits``). Invoke, stream, AG-UI, +``/v1/graph/ws`` and ``/v1/graph/live`` all take both through :func:`acquire_run_guard`, so a +new entry point cannot pick up one and forget the other. The result is one lease; release it +when the run is over (``release_when_done`` for a response stream, ``finally`` otherwise). +""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from tenxgraph_api.src.app.core.concurrency import Lease +from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.routers.graph.run_limits import acquire_run_slot +from tenxgraph_api.src.app.routers.graph.thread_lock import ( + OnBusy, + acquire_thread_lock, + combine_leases, +) + + +async def acquire_run_guard( + *, + user: dict[str, Any], + runs: RunsConfig, + thread_id: str | None, + on_busy: OnBusy = "wait", + stop: Callable[[], Awaitable[Any]] | None = None, + is_disconnected: Callable[[], Awaitable[bool]] | None = None, +) -> Lease: + """Lock ``thread_id`` and take a run slot for ``user``; release both with the one lease. + + ``thread_id`` is ``None`` for a run that starts a new thread (the service locks that one + once its id exists). Raises ``HTTPException``: 409 when the thread stays busy, 429 over the + run cap, 499 when ``is_disconnected`` says the client left while waiting. Nothing stays held + when it raises. + """ + thread_lock = Lease(None) + if thread_id is not None: + thread_lock = await acquire_thread_lock( + thread_id, on_busy=on_busy, stop=stop, is_disconnected=is_disconnected + ) + try: + slot = await acquire_run_slot(user, runs) + except BaseException: + thread_lock.release() + raise + return combine_leases(slot, thread_lock) diff --git a/tenxgraph_api/src/app/routers/graph/run_limits.py b/tenxgraph_api/src/app/routers/graph/run_limits.py index 4e9d4950..a9b32777 100644 --- a/tenxgraph_api/src/app/routers/graph/run_limits.py +++ b/tenxgraph_api/src/app/routers/graph/run_limits.py @@ -28,6 +28,8 @@ HTTP_TOO_MANY_REQUESTS = 429 +# Seconds a client is told to wait before it starts another run. +RUN_CAP_RETRY_AFTER_SECONDS = 5 # A slot that was never counted (no identity, or no cap). RunSlot = Lease @@ -58,6 +60,7 @@ async def acquire_run_slot(user: dict[str, Any], runs: RunsConfig) -> Lease: f"Too many runs in progress for this user (limit {limit}). " "Wait for one to finish and retry." ), + headers={"Retry-After": str(RUN_CAP_RETRY_AFTER_SECONDS)}, ) return lease diff --git a/tenxgraph_api/src/app/routers/graph/services/graph_service.py b/tenxgraph_api/src/app/routers/graph/services/graph_service.py index 4d6e5172..78b38b5b 100644 --- a/tenxgraph_api/src/app/routers/graph/services/graph_service.py +++ b/tenxgraph_api/src/app/routers/graph/services/graph_service.py @@ -1,6 +1,7 @@ import asyncio import contextlib import re +import sys from collections.abc import AsyncIterable, AsyncIterator from datetime import datetime from typing import Any @@ -33,6 +34,7 @@ ) from tenxgraph_api.src.app.core.concurrency import Lease from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RunsConfig +from tenxgraph_api.src.app.core.shutdown import KEEPALIVE_SECONDS, end_on_shutdown from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe, sanitize_for_logging from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( GraphInputSchema, @@ -312,6 +314,10 @@ async def produce() -> None: # Longest a background thread-naming call may run (it is a model call). THREAD_NAME_TIMEOUT_SECONDS = 60.0 +# Naming tasks kept by the service itself (no core task manager bound) are capped; over it the +# thread simply stays unnamed, as the core manager does with its own limit. +MAX_PENDING_THREAD_NAMES = 100 + # A taken generated id is retried with a fresh one; each retry is another prediction the # claimer would have had to get right. MAX_THREAD_ID_ATTEMPTS = 5 @@ -451,11 +457,28 @@ async def name_thread() -> None: name_thread(), name="thread_name", timeout=THREAD_NAME_TIMEOUT_SECONDS ) return - task = asyncio.create_task(name_thread()) tasks: set[asyncio.Task[None]] = self.__dict__.setdefault("_naming_tasks", set()) + if len(tasks) >= MAX_PENDING_THREAD_NAMES: + logger.warning( + "Too many thread names pending (%d); leaving thread %s unnamed", + len(tasks), + log_safe(thread_id), + ) + return + task = asyncio.create_task(name_thread()) tasks.add(task) task.add_done_callback(tasks.discard) + async def drain_naming_tasks(self) -> None: + """Wait for the naming tasks this service holds itself (call at shutdown). + + Tasks on the core ``BackgroundTaskManager`` are drained with that manager. There is + no timeout: a naming model call is the developer's own. + """ + tasks = list(self.__dict__.get("_naming_tasks", ())) + if tasks: + await asyncio.gather(*tasks, return_exceptions=True) + async def _save_thread(self, config: dict[str, Any], thread_id: str): """ Save the generated thread name to the database. @@ -500,6 +523,20 @@ def telemetry(self) -> TelemetryStore | None: self._telemetry = None return self._telemetry + def _close_run_trace(self, thread_id: str | None, run_id: str | None) -> None: + """Close a run's trace that nothing else closed: cancelled, disconnected or failed.""" + store = self.telemetry + if not store or not thread_id or not run_id: + return + exc = sys.exc_info()[1] + status = "stopped" if isinstance(exc, asyncio.CancelledError | GeneratorExit) else "error" + if exc is None: + status = "stopped" + try: + store.close_run(thread_id, run_id, datetime.now().timestamp(), status) + except Exception as e: # - telemetry must never break a run + logger.debug("Telemetry close_run failed: %s", e) + @staticmethod def _telemetry_record(chunk: Any, ts: float | None = None) -> dict[str, Any]: """Flatten a StreamChunk into a compact record for the telemetry store.""" @@ -723,6 +760,33 @@ async def _check_resume_target( f"interrupt '{pending.id}'" ) + async def check_resume_target( + self, graph_input: GraphInputSchema, user: dict[str, Any] + ) -> None: + """409 for a resume aimed at the wrong interrupt, before a stream response starts. + + A stream answers 200 as soon as it starts, so the router asks this first, as invoke + gets the same check inside its run. Nothing happens unless the request is a resume + that names an ``interrupt_id``. + + Raises: + HTTPException: 409 when the thread is not paused at that interrupt. + """ + if not graph_input.is_resume or graph_input.interrupt_id is None: + return + thread_id = normalize_thread_id((graph_input.config or {}).get("thread_id")) + user_id = user.get("user_id") + config = {"thread_id": thread_id, "user_id": user_id if user_id else "anonymous"} + config["user"] = user + try: + if thread_id is None: # a new thread has nothing to resume + raise InterruptConflictError( + "Nothing to resume: this thread is not paused at an interrupt" + ) + await self._check_resume_target(graph_input, config) + except _resume_error_types() as exc: + raise _resume_http_error(exc) from exc + async def _is_static_pause(self, config: dict[str, Any]) -> bool: """Whether the thread is paused by ``interrupt_before`` / ``interrupt_after``. @@ -766,6 +830,10 @@ async def _prepare_input( # update thread id config["thread_id"] = str(thread_id) + # The authenticated user, before any checkpointer read below: PgCheckpointer refuses a + # config without ``user_id``. Never taken from the request (``client_config`` strips it). + config["user_id"] = user_id if user_id else "anonymous" + # check recursion limit set or not config["recursion_limit"] = graph_input.recursion_limit or 25 @@ -828,6 +896,8 @@ async def invoke_graph( # noqa: PLR0912, PLR0915 HTTPException: If graph execution fails. """ thread_lock = Lease(None) + inv_thread_id: str | None = None + inv_run_id: str | None = None try: logger.debug("Invoking graph with input: %s", log_safe(graph_input.messages)) @@ -929,6 +999,7 @@ async def invoke_graph( # noqa: PLR0912, PLR0915 logger.error(f"Graph execution failed: {e}") raise HTTPException(status_code=500, detail=f"Graph execution failed: {e!s}") finally: + self._close_run_trace(inv_thread_id, inv_run_id) thread_lock.release() @staticmethod @@ -988,8 +1059,33 @@ async def stream_graph( Yields: str: Individual JSON chunks from graph execution with newline delimiters. """ - async for chunk in self.stream_chunks(graph_input, user): - yield chunk.model_dump_json(serialize_as_any=True) + "\n" + + async def lines() -> AsyncIterable[str]: + last: StreamChunk | None = None + errored = False + async for chunk in self.stream_chunks(graph_input, user): + errored = errored or chunk.event == StreamEvent.ERROR + yield chunk.model_dump_json(serialize_as_any=True) + "\n" + # The graph reuses one chunk object, so keep only the ids, not the object. + last = StreamChunk( + event=StreamEvent.UPDATES, + thread_id=chunk.thread_id, + run_id=chunk.run_id, + ) + if not errored: + # The documented terminal chunk: a client that waits for it can tell a + # finished stream from a cut one. WebSocket runs send the same chunk. + done = StreamChunk( + event=StreamEvent.UPDATES, + data={"status": "done"}, + thread_id=last.thread_id if last else None, + run_id=last.run_id if last else None, + ) + yield done.model_dump_json(serialize_as_any=True) + "\n" + + # On server shutdown the stream ends with a clear final chunk, not a cut connection. + async for line in end_on_shutdown(lines(), keepalive_seconds=KEEPALIVE_SECONDS): + yield line async def stream_chunks( # noqa: PLR0912, PLR0915 self, @@ -1013,6 +1109,7 @@ async def stream_chunks( # noqa: PLR0912, PLR0915 meta: dict[str, Any] = {} thread_id: str | None = None run_id: str | None = None + naming: tuple[dict[str, Any], Any, list[str]] | None = None thread_lock = Lease(None) try: logger.debug("Streaming graph with input: %s", log_safe(graph_input.messages)) @@ -1085,16 +1182,7 @@ async def stream_chunks( # noqa: PLR0912, PLR0915 self.telemetry.finish_run(thread_id, run_id, datetime.now().timestamp(), run_status) if meta["is_new_thread"] and self.config.thread_name_generator_path: - thread_name = await self._save_thread_name( - config, config["thread_id"], messages_str - ) - meta["thread_name"] = thread_name - - yield StreamChunk( - event=StreamEvent.UPDATES, - data={"status": "completed"}, - metadata=meta, - ) + naming = (dict(config), config["thread_id"], messages_str) except Exception as e: # Never raise HTTPException from inside an async generator that is @@ -1118,7 +1206,11 @@ async def stream_chunks( # noqa: PLR0912, PLR0915 metadata=meta, ) finally: + self._close_run_trace(thread_id, run_id) thread_lock.release() + if naming is not None: + # An extra model call: started once the thread is free, never awaited here. + self._name_thread_in_background(*naming) async def _missing_pause_chunk( self, config: dict[str, Any], meta: dict[str, Any] diff --git a/tenxgraph_api/src/app/routers/graph/thread_lock.py b/tenxgraph_api/src/app/routers/graph/thread_lock.py index 50b3f9bf..312fe64d 100644 --- a/tenxgraph_api/src/app/routers/graph/thread_lock.py +++ b/tenxgraph_api/src/app/routers/graph/thread_lock.py @@ -9,7 +9,9 @@ (key ``runs:thread:``): shared through Redis when it is configured, otherwise per process. A Redis lease is renewed in the background while the run is alive and expires on its own if the worker dies, so a crashed worker cannot hold a thread forever. A Redis error -lets the run proceed (the limiter fails open), as for the per-user run cap. +falls back to a per-process lock (the limiter fails open to its own count), so one worker still +runs one thing at a time on a thread. Run entry points take this lock together with the run +slot through ``run_guard.acquire_run_guard``. A run that finds the thread busy waits up to ``THREAD_BUSY_WAIT_SECONDS`` and is then rejected with 409. With ``on_busy="cancel"`` it first asks the active run to stop (and @@ -32,6 +34,10 @@ HTTP_CONFLICT = 409 THREAD_BUSY_DETAIL = "thread is busy" +# nginx's code for a client that closed the request; the response is never read. +CLIENT_CLOSED_REQUEST = 499 +# Seconds a client is told to wait before it tries a busy thread again. +BUSY_RETRY_AFTER_SECONDS = 5 # Longest a run waits for a busy thread before it is rejected. THREAD_BUSY_WAIT_SECONDS = 30.0 @@ -54,7 +60,20 @@ class ThreadBusyError(HTTPException): """The thread still has an active run after the wait limit.""" def __init__(self) -> None: - super().__init__(status_code=HTTP_CONFLICT, detail=THREAD_BUSY_DETAIL) + super().__init__( + status_code=HTTP_CONFLICT, + detail=THREAD_BUSY_DETAIL, + headers={"Retry-After": str(BUSY_RETRY_AFTER_SECONDS)}, + ) + + +class ClientGoneError(HTTPException): + """The client closed the request while it waited for the thread.""" + + def __init__(self) -> None: + super().__init__( + status_code=CLIENT_CLOSED_REQUEST, detail="client closed the request while waiting" + ) async def acquire_thread_lock( @@ -62,11 +81,14 @@ async def acquire_thread_lock( *, on_busy: OnBusy = "wait", stop: Callable[[], Awaitable[Any]] | None = None, + is_disconnected: Callable[[], Awaitable[bool]] | None = None, ) -> Lease: """Take the run lock for ``thread_id``, waiting for the active run if there is one. ``stop`` asks the active run to stop; it is called only for ``on_busy="cancel"`` and only while the thread is busy. Raises :class:`ThreadBusyError` (409) after the wait limit. + ``is_disconnected`` tells whether the client is gone: a waiter whose client left raises + :class:`ClientGoneError` instead of taking the lock and running for nobody. """ limiter = concurrency_limiter() key = thread_lock_key(thread_id) @@ -74,10 +96,17 @@ async def acquire_thread_lock( deadline = loop.time() + THREAD_BUSY_WAIT_SECONDS delay = THREAD_BUSY_POLL_MIN_SECONDS next_stop = 0.0 + waited = False while True: lease = await limiter.acquire(key, 1) if lease is not None: + if waited and is_disconnected is not None and await is_disconnected(): + lease.release() + raise ClientGoneError return lease + waited = True + if is_disconnected is not None and await is_disconnected(): + raise ClientGoneError now = loop.time() if now >= deadline: logger.warning("Thread %s is still busy after the wait limit", log_safe(thread_id)) diff --git a/tenxgraph_api/src/app/routers/ping/readiness.py b/tenxgraph_api/src/app/routers/ping/readiness.py index bf3715b5..f3400270 100644 --- a/tenxgraph_api/src/app/routers/ping/readiness.py +++ b/tenxgraph_api/src/app/routers/ping/readiness.py @@ -9,6 +9,13 @@ public. Results are cached for ``CACHE_SECONDS`` with one check in flight at a time, so a flood of probe requests cannot turn into a flood of database and Redis round trips. +Redis is the one dependency the server survives losing: the run lock, run cap, WebSocket caps +and (by default) the rate limiter fail open, and ownership falls back to the database. A Redis +outage therefore must not take every pod out of rotation. Its result is still reported, but only +counts against readiness when a configured feature fails closed without it (see +:func:`redis_required`). While the app is shutting down readiness answers 503 (``draining``) so +the load balancer stops sending new requests. + A component that knows best how to check itself can expose ``async def readiness_check() -> None`` (raise when not ready), which is used before the built-in probes. @@ -83,11 +90,11 @@ async def check_store(store: Any) -> None: async def check_redis(url: str | None) -> None: if not url: raise _Skip - from redis.asyncio import Redis # type: ignore[import] + from tenxgraph_api.src.app.core.redis_client import create_redis # A fresh connection per probe: probes are seconds apart, and a cached client would be # tied to the event loop that created it. - client = Redis.from_url(url) + client = create_redis(url) try: await client.ping() finally: @@ -108,6 +115,33 @@ async def run_check(name: str, check: Callable[[], Awaitable[None]]) -> str: return OK +def redis_required(graph_config: Any) -> bool: + """True when a configured feature fails closed without Redis. + + Token revocation on Redis answers 503 when Redis is down, and a Redis rate limiter with + ``fail_open`` off denies requests. Every other Redis use fails open. + """ + try: + if getattr(graph_config, "token_revocation", None) == "redis": + return True + limit = getattr(graph_config, "rate_limit", None) + return bool( + limit is not None + and getattr(limit, "backend", None) == "redis" + and getattr(limit, "fail_open", True) is False + ) + except Exception: + return False + + +def is_ready(checks: dict[str, str], *, redis_is_required: bool) -> bool: + """Whether ``checks`` allow traffic; a failed optional Redis does not count.""" + return all( + result != FAIL or (name == "redis" and not redis_is_required) + for name, result in checks.items() + ) + + async def readiness(checkpointer: Any, store: Any, redis_url: str | None) -> dict[str, str]: """Run every check concurrently; ``{component: ok | fail | skipped}``.""" names = ("checkpointer", "redis", "store") diff --git a/tenxgraph_api/src/app/routers/ping/router.py b/tenxgraph_api/src/app/routers/ping/router.py index 044c8e42..95757881 100644 --- a/tenxgraph_api/src/app/routers/ping/router.py +++ b/tenxgraph_api/src/app/routers/ping/router.py @@ -34,12 +34,14 @@ async def ping_server( summary="Readiness probe", description=( "Checks the dependencies a request needs (checkpointer, Redis, store). Answers 200 " - "when every configured one is reachable and 503 otherwise. /ping stays the liveness " + "when every configured one is reachable and 503 otherwise, and 503 while the server is " + "shutting down. A Redis outage alone stays 200 unless a configured feature (token " + "revocation, a fail-closed rate limiter) needs it. /ping stays the liveness " "probe. Public, like /ping: it reports only ok or fail per configured component, and the " "result is cached for a few seconds." ), ) -async def readiness_probe(): +async def readiness_probe(request: Request): from injectq import InjectQ from tenxgraph.storage.checkpointer import BaseCheckpointer from tenxgraph.storage.store import BaseStore @@ -48,7 +50,11 @@ async def readiness_probe(): from tenxgraph_api.src.app.core.config.settings import get_settings from tenxgraph_api.src.app.core.utils.container import current_container - from .readiness import cached_readiness + from .readiness import cached_readiness, is_ready, redis_required + + if getattr(request.app.state, "draining", False): + # Shutting down: leave the load balancer's rotation before the sockets close. + return JSONResponse(status_code=503, content={"status": "draining", "checks": {}}) container: InjectQ = current_container() graph_config = container.try_get(GraphConfig) if GraphConfig in container else None @@ -58,7 +64,7 @@ async def readiness_probe(): container.try_get(BaseStore) if BaseStore in container else None, redis_url if isinstance(redis_url, str) else None, ) - ready = all(result != "fail" for result in checks.values()) + ready = is_ready(checks, redis_is_required=redis_required(graph_config)) return JSONResponse( status_code=200 if ready else 503, content={"status": "ready" if ready else "not_ready", "checks": checks}, diff --git a/tenxgraph_api/src/app/routers/store/services/store_service.py b/tenxgraph_api/src/app/routers/store/services/store_service.py index c31f2d59..72ea5d4b 100644 --- a/tenxgraph_api/src/app/routers/store/services/store_service.py +++ b/tenxgraph_api/src/app/routers/store/services/store_service.py @@ -50,6 +50,29 @@ def _client_metadata(metadata: dict[str, Any] | None) -> dict[str, Any] | None: return {k: v for k, v in metadata.items() if k not in SYSTEM_METADATA_KEYS} +# Keys that choose whose memories a store call touches. The server sets them from the +# authenticated user, so a client ``filters`` dict may never carry them (J-4). +RESERVED_FILTER_KEYS = frozenset({"user_id", "agent_id", "app_id", "run_id"}) +_LOGICAL_KEYS = frozenset({"AND", "OR", "NOT"}) + + +def _client_filters(filters: Any) -> Any: + """Copy of ``filters`` without reserved owner keys, also inside AND / OR / NOT groups.""" + if not isinstance(filters, dict): + return filters + clean: dict[str, Any] = {} + for key, value in filters.items(): + if key in RESERVED_FILTER_KEYS: + continue + if key in _LOGICAL_KEYS and isinstance(value, list): + groups = [g for g in (_client_filters(v) for v in value) if g not in ({}, None)] + if groups: + clean[key] = groups + continue + clean[key] = value + return clean + + def _client_options(options: dict[str, Any] | None) -> dict[str, Any]: return {k: v for k, v in (options or {}).items() if k in CLIENT_OPTION_KEYS} @@ -116,7 +139,7 @@ async def search_memories( category=payload.category, limit=payload.limit, score_threshold=payload.score_threshold, - filters=payload.filters, + filters=_client_filters(payload.filters), retrieval_strategy=payload.retrieval_strategy, distance_metric=payload.distance_metric, max_tokens=payload.max_tokens, @@ -190,7 +213,7 @@ async def forget_memory( forget_kwargs: dict[str, Any] = { "memory_type": payload.memory_type, "category": payload.category, - "filters": payload.filters, + "filters": _client_filters(payload.filters), } # Remove None values before forwarding to the store forget_kwargs = {k: v for k, v in forget_kwargs.items() if v is not None} diff --git a/tenxgraph_api/src/app/utils/snowflake_id_generator.py b/tenxgraph_api/src/app/utils/snowflake_id_generator.py index b2e9c128..a4599555 100644 --- a/tenxgraph_api/src/app/utils/snowflake_id_generator.py +++ b/tenxgraph_api/src/app/utils/snowflake_id_generator.py @@ -1,4 +1,7 @@ +import logging import os +import socket +import zlib from importlib.util import find_spec from tenxgraph.utils.id_generator import BaseIDGenerator, IDType @@ -7,6 +10,16 @@ # Check if snowflakekit is available HAS_SNOWFLAKE = find_spec("snowflakekit") is not None +logger = logging.getLogger("tenxgraph_api.snowflake") + + +def _setting(value: int | None, env_name: str, default: int | None) -> int | None: + """``value`` when given, else the environment variable, else ``default``.""" + if value is not None: + return value + raw = os.environ.get(env_name) + return int(raw) if raw not in (None, "") else default + class SnowFlakeIdGenerator(BaseIDGenerator): def __init__( @@ -19,8 +32,6 @@ def __init__( snowflake_worker_id: int | None = None, snowflake_worker_bits: int | None = None, ): - # IF all these are None then try to read from env - config = None if not HAS_SNOWFLAKE: raise ImportError( "snowflakekit is not installed. Please install it to use SnowFlakeIdGenerator." @@ -28,53 +39,43 @@ def __init__( from snowflakekit import SnowflakeConfig, SnowflakeGenerator - if ( - snowflake_epoch is None - and total_bits is None - and snowflake_time_bits is None - and snowflake_node_bits is None - and snowflake_node_id is None - and snowflake_worker_id is None - and snowflake_worker_bits is None - ): - snowflake_epoch = int(os.environ.get("SNOWFLAKE_EPOCH", "1723323246031")) - total_bits = int(os.environ.get("SNOWFLAKE_TOTAL_BITS", "64")) - snowflake_time_bits = int(os.environ.get("SNOWFLAKE_TIME_BITS", "39")) - snowflake_node_bits = int(os.environ.get("SNOWFLAKE_NODE_BITS", "7")) - snowflake_node_id = int(os.environ.get("SNOWFLAKE_NODE_ID", "0")) - snowflake_worker_id = int(os.environ.get("SNOWFLAKE_WORKER_ID", "0")) - snowflake_worker_bits = int(os.environ.get("SNOWFLAKE_WORKER_BITS", "5")) + # Each value is, in order: the argument, its SNOWFLAKE_* environment variable, a default. + # A partly given configuration therefore still builds a complete config. + epoch = _setting(snowflake_epoch, "SNOWFLAKE_EPOCH", 1723323246031) + total = _setting(total_bits, "SNOWFLAKE_TOTAL_BITS", 64) + time_bits = _setting(snowflake_time_bits, "SNOWFLAKE_TIME_BITS", 39) + node_bits = _setting(snowflake_node_bits, "SNOWFLAKE_NODE_BITS", 7) + worker_bits = _setting(snowflake_worker_bits, "SNOWFLAKE_WORKER_BITS", 5) - config = SnowflakeConfig( - epoch=snowflake_epoch, - total_bits=total_bits, - time_bits=snowflake_time_bits, - node_bits=snowflake_node_bits, - node_id=snowflake_node_id, - worker_id=snowflake_worker_id, - worker_bits=snowflake_worker_bits, + # Two processes with the same node and worker id mint the same ids. When the ids are + # not set, derive them from this process (worker id from the pid, node id from the + # host name) instead of giving every gunicorn worker the same 0/0. + node_id = _setting(snowflake_node_id, "SNOWFLAKE_NODE_ID", None) + if node_id is None: + node_id = zlib.crc32(socket.gethostname().encode()) % (1 << node_bits) + logger.warning( + "SNOWFLAKE_NODE_ID is not set; using %d derived from the host name. " + "Set it explicitly to keep ids unique across hosts.", + node_id, ) - - elif ( - snowflake_epoch is not None - and total_bits is not None - and snowflake_time_bits is not None - and snowflake_node_bits is not None - and snowflake_node_id is not None - and snowflake_worker_id is not None - and snowflake_worker_bits is not None - ): - config = SnowflakeConfig( - epoch=snowflake_epoch, - total_bits=total_bits, - time_bits=snowflake_time_bits, - node_bits=snowflake_node_bits, - node_id=snowflake_node_id, - worker_id=snowflake_worker_id, - worker_bits=snowflake_worker_bits, + worker_id = _setting(snowflake_worker_id, "SNOWFLAKE_WORKER_ID", None) + if worker_id is None: + worker_id = os.getpid() % (1 << worker_bits) + logger.warning( + "SNOWFLAKE_WORKER_ID is not set; using %d derived from the process id. " + "Set it per worker to rule out collisions.", + worker_id, ) - # Now create generator + config = SnowflakeConfig( + epoch=epoch, + total_bits=total, + time_bits=time_bits, + node_bits=node_bits, + node_id=node_id, + worker_id=worker_id, + worker_bits=worker_bits, + ) self.generator = SnowflakeGenerator(config=config) @property diff --git a/tenxgraph_api/src/app/utils/telemetry_store.py b/tenxgraph_api/src/app/utils/telemetry_store.py index 79886a11..1898f304 100644 --- a/tenxgraph_api/src/app/utils/telemetry_store.py +++ b/tenxgraph_api/src/app/utils/telemetry_store.py @@ -44,7 +44,9 @@ def __init__(self) -> None: self._lock = threading.Lock() # thread_id -> ordered dict-like: run_id -> RunTrace (kept insertion order) self._threads: dict[str, dict[str, RunTrace]] = {} - self._thread_order: deque[str] = deque(maxlen=MAX_THREADS) + # No maxlen: a deque that drops its oldest id on append would leave that thread in the + # dict for good. The eviction loop in start_run is the only bound. + self._thread_order: deque[str] = deque() def start_run(self, thread_id: str, run_id: str, started_at: float) -> RunTrace: thread_id = str(thread_id) @@ -99,6 +101,16 @@ def finish_run( trace.finished_at = finished_at trace.status = status + def close_run(self, thread_id: str, run_id: str, finished_at: float, status: str) -> None: + """Finish a run only if it is still open (a cancelled run, an unhandled error).""" + thread_id = str(thread_id) + run_id = str(run_id) + with self._lock: + trace = (self._threads.get(thread_id) or {}).get(run_id) + if trace and trace.finished_at is None: + trace.finished_at = finished_at + trace.status = status + def get_runs(self, thread_id: str) -> list[RunTrace]: """Runs for a thread, newest last (insertion order).""" with self._lock: diff --git a/tenxgraph_api/src/app/uvicorn_worker.py b/tenxgraph_api/src/app/uvicorn_worker.py new file mode 100644 index 00000000..9ef60f57 --- /dev/null +++ b/tenxgraph_api/src/app/uvicorn_worker.py @@ -0,0 +1,80 @@ +"""Gunicorn worker class used by the generated image (``10xgraph build``). + +Differences from ``uvicorn_worker.UvicornWorker``: + +- ``timeout_graceful_shutdown`` is set, so uvicorn gives up on open responses before gunicorn's + ``--graceful-timeout`` SIGKILLs the worker. Without it a long stream holds shutdown until the + kill and the lifespan drain never runs. +- Shutdown marks the app as draining first (``/health/ready`` answers 503), and shortly before + the deadline tells open NDJSON streams to end with a final "interrupted by shutdown" chunk + (``core/shutdown.py``), so clients see why the stream ended and can resume the thread. + +Open WebSockets are still closed by uvicorn with 1012 (service restart) at that point; the run +is saved INTERRUPTED and the client reconnects. +""" + +from __future__ import annotations + +import asyncio +import os +import sys +from typing import Any + +from gunicorn.arbiter import Arbiter +from uvicorn.server import Server +from uvicorn_worker import UvicornWorker + +from tenxgraph_api.src.app.core.shutdown import interrupt_streams + + +# Seconds uvicorn waits for open responses after SIGTERM. Must stay below the gunicorn +# ``--graceful-timeout`` of the generated image (600) with room for the lifespan drain +# (30 s) and ``graph.aclose``. +DEFAULT_GRACEFUL_SHUTDOWN_SECONDS = 540 +# Open streams are told to finish this long before uvicorn would cancel them. +STREAM_INTERRUPT_MARGIN_SECONDS = 5.0 + + +def graceful_shutdown_seconds() -> int: + raw = os.environ.get("TENXGRAPH_GRACEFUL_SHUTDOWN_SECONDS", "") + try: + value = int(raw) + except ValueError: + return DEFAULT_GRACEFUL_SHUTDOWN_SECONDS + return value if value > 0 else DEFAULT_GRACEFUL_SHUTDOWN_SECONDS + + +class DrainingServer(Server): + """Uvicorn server that flags the app as draining before it stops serving.""" + + app_state: Any = None + + async def shutdown(self, sockets: Any = None) -> None: + if self.app_state is not None: + self.app_state.draining = True + timeout = self.config.timeout_graceful_shutdown + timer = None + if timeout: + delay = max(timeout - STREAM_INTERRUPT_MARGIN_SECONDS, 0.0) + timer = asyncio.get_running_loop().call_later(delay, interrupt_streams) + try: + await super().shutdown(sockets=sockets) + finally: + if timer is not None: + timer.cancel() + + +class TenxGraphUvicornWorker(UvicornWorker): + CONFIG_KWARGS = { + **UvicornWorker.CONFIG_KWARGS, + "timeout_graceful_shutdown": graceful_shutdown_seconds(), + } + + async def _serve(self) -> None: + self.config.app = self.wsgi + server = DrainingServer(config=self.config) + server.app_state = getattr(self.wsgi, "state", None) + self._install_sigquit_handler() + await server.serve(sockets=self.sockets) + if not server.started: + sys.exit(Arbiter.WORKER_BOOT_ERROR) diff --git a/tests/cli/test_cli_commands_ops.py b/tests/cli/test_cli_commands_ops.py index 438c5730..0d1b9e51 100644 --- a/tests/cli/test_cli_commands_ops.py +++ b/tests/cli/test_cli_commands_ops.py @@ -332,3 +332,11 @@ def test_ws_max_size_follows_max_request_size(monkeypatch, raw, expected): else: monkeypatch.setenv("MAX_REQUEST_SIZE", raw) assert _max_request_size() == expected + + +def test_build_command_refuses_compose_together_with_k8s(tmp_path, monkeypatch, silent_output): + monkeypatch.chdir(tmp_path) + cmd = BuildCommand(output=silent_output) + code = cmd.execute(output_file="Dockerfile", force=True, docker_compose=True, k8s=True) + assert code != 0 + assert list(tmp_path.iterdir()) == [] diff --git a/tests/cli/test_eval_command.py b/tests/cli/test_eval_command.py index b7f71d94..d396a3cc 100644 --- a/tests/cli/test_eval_command.py +++ b/tests/cli/test_eval_command.py @@ -620,3 +620,14 @@ def test_execute_load_config_error(cmd, tmp_path): ): code = cmd.execute(target=str(tmp_path)) assert code == 1 + + +def test_threshold_decides_the_exit_code(): + """A pass rate at or above --threshold exits 0 even when it is below 100 percent.""" + from tenxgraph_api.cli.commands.eval import EvalCommand + + assert EvalCommand._exit_code(0.8, 0.75) == 0 + assert EvalCommand._exit_code(0.75, 0.75) == 0 + assert EvalCommand._exit_code(0.7, 0.75) == 1 + assert EvalCommand._exit_code(0.9, None) == 1 + assert EvalCommand._exit_code(1.0, None) == 0 diff --git a/tests/cli/test_group6_cli_templates.py b/tests/cli/test_group6_cli_templates.py new file mode 100644 index 00000000..20e066f8 --- /dev/null +++ b/tests/cli/test_group6_cli_templates.py @@ -0,0 +1,93 @@ +"""Production template defaults: rate limit key (G-168), token revocation (G-167), docs path +(G-120) and the Kubernetes manifest env (G-95).""" + +from __future__ import annotations + +import io +import json +from pathlib import Path + +from tenxgraph_api.cli.commands.init import InitCommand +from tenxgraph_api.cli.core.output import OutputFormatter +from tenxgraph_api.cli.templates.defaults import generate_k8s_manifest_content + + +def _cmd() -> InitCommand: + return InitCommand(output=OutputFormatter(stream=io.StringIO(), quiet=True)) + + +def _context(auth: str, rate_limit: str = "redis") -> dict: + return _cmd()._non_interactive_context( + path=".", + agent_name="demo", + template="production", + auth=auth, + rate_limit=rate_limit, + ) + + +def test_production_rate_limit_is_per_user_when_auth_is_on() -> None: + for auth in ("jwt", "custom"): + config = _cmd()._build_config(_context(auth), True) + assert config["rate_limit"]["by"] == "user" + config = _cmd()._build_config(_context("none"), True) + assert config["rate_limit"]["by"] == "ip" + + +def test_production_config_enables_token_revocation_with_redis() -> None: + config = _cmd()._build_config(_context("jwt"), True) + assert config["token_revocation"] == "redis" + assert "token_revocation" not in _cmd()._build_config(_context("jwt", "memory"), True) + assert "token_revocation" not in _cmd()._build_config(_context("none"), True) + + +def test_static_production_template_matches() -> None: + path = ( + Path(__file__).parents[2] / "tenxgraph_api" / "cli" / "templates" / "prod" / "10xgraph.json" + ) + config = json.loads(path.read_text(encoding="utf-8")) + assert config["token_revocation"] == "redis" + assert "/redocs" in config["rate_limit"]["exclude_paths"] + assert "/redoc" not in config["rate_limit"]["exclude_paths"] + + +def test_generated_exclude_paths_use_the_real_redoc_path() -> None: + config = _cmd()._build_config(_context("jwt"), True) + assert "/redocs" in config["rate_limit"]["exclude_paths"] + + +def test_k8s_env_does_not_shadow_the_secret() -> None: + yaml = __import__("pytest").importorskip("yaml") + deployment, _ = yaml.safe_load_all(generate_k8s_manifest_content("svc", 8000)) + env = {e["name"] for e in deployment["spec"]["template"]["spec"]["containers"][0]["env"]} + assert "ORIGINS" not in env + assert "ALLOWED_HOST" not in env + + +def test_revoke_command_writes_to_redis(monkeypatch) -> None: + from tenxgraph_api.cli.commands.revoke import RevokeCommand + from tenxgraph_api.src.app.core.auth import revocation + + calls: list[tuple] = [] + + class _Backend: + def __init__(self, url): + calls.append(("url", url)) + + async def revoke_user(self, user, before=None): + calls.append(("user", user, before)) + + async def revoke_token(self, jti, expires_at=None): + calls.append(("jti", jti, expires_at)) + + async def aclose(self): + calls.append(("closed",)) + + monkeypatch.setattr(revocation, "RedisTokenRevocation", _Backend) + cmd = RevokeCommand(output=OutputFormatter(stream=io.StringIO(), quiet=True)) + assert cmd.execute(user="u1", redis_url="redis://x") == 0 + assert cmd.execute(jti="j1", expires_at=5.0, redis_url="redis://x") == 0 + assert ("user", "u1", None) in calls and ("jti", "j1", 5.0) in calls + assert calls.count(("closed",)) == 2 + assert cmd.execute(redis_url="redis://x") != 0 # neither --user nor --jti + assert cmd.execute(user="u", jti="j", redis_url="redis://x") != 0 diff --git a/tests/cli/test_group6_small_cli_bugs.py b/tests/cli/test_group6_small_cli_bugs.py new file mode 100644 index 00000000..fac0acd0 --- /dev/null +++ b/tests/cli/test_group6_small_cli_bugs.py @@ -0,0 +1,92 @@ +"""G-121: small CLI bugs.""" + +from __future__ import annotations + +import io +import tomllib +from pathlib import Path + +import pytest + +from tenxgraph_api.cli.commands.eval import EvalCommand +from tenxgraph_api.cli.commands.init import InitCommand +from tenxgraph_api.cli.core.output import OutputFormatter +from tenxgraph_api.cli.exceptions import FileOperationError, ValidationError + + +def _out() -> OutputFormatter: + return OutputFormatter(stream=io.StringIO(), quiet=True) + + +def test_agent_name_with_quotes_does_not_break_pyproject(tmp_path: Path) -> None: + cmd = InitCommand(output=_out()) + context = cmd._non_interactive_context( + path=str(tmp_path), + agent_name='My "Agent" \\ x', + template="production", + auth="jwt", + rate_limit="memory", + ) + template = Path(__file__).parents[2] / "tenxgraph_api/cli/templates/prod/pyproject.toml" + rendered = cmd._render(template, context, True) + assert tomllib.loads(rendered)["project"]["description"].startswith('My "Agent" \\ x') + env = cmd._render(template.with_name(".env.example"), context, True) + assert 'APP_NAME="My \\"Agent\\" \\\\ x"' in env + + +@pytest.mark.parametrize("name", ["bad\nname", "tab\tname", "nul\x00"]) +def test_agent_name_with_control_characters_is_refused(name: str) -> None: + with pytest.raises(ValidationError): + InitCommand(output=_out())._non_interactive_context( + path=".", agent_name=name, template="production", auth="jwt", rate_limit="memory" + ) + + +def test_render_failure_is_reported_not_swallowed(tmp_path: Path, monkeypatch) -> None: + from tenxgraph_api.cli.commands import init as init_mod + + def boom(_context): + raise KeyError("provider") + + monkeypatch.setattr(init_mod, "_template_vars", boom) + src = tmp_path / "f.txt" + src.write_text("$agent_name", encoding="utf-8") + with pytest.raises(FileOperationError): + InitCommand(output=_out())._render(src, {"agent_name": "x", "auth": "none"}, False) + + +def test_confeval_is_not_searched_above_the_project(tmp_path: Path, monkeypatch) -> None: + project = tmp_path / "project" + outside = tmp_path / "outside" / "evals" + project.mkdir() + outside.mkdir(parents=True) + (tmp_path / "confeval.py").write_text("raise SystemExit('loaded')", encoding="utf-8") + monkeypatch.chdir(project) + dirs = EvalCommand(output=_out())._confeval_search_dirs(outside, project / "evals") + assert tmp_path.resolve() not in dirs + assert tmp_path.parent.resolve() not in dirs + assert dirs[0] == outside.resolve() + + +def test_skills_force_keeps_the_old_install_when_the_copy_fails( + tmp_path: Path, monkeypatch +) -> None: + import shutil + + from tenxgraph_api.cli.commands import skills as skills_mod + + cmd = skills_mod.SkillsCommand(output=_out()) + templates_root = Path(skills_mod.__file__).parents[1] / "templates" / "skills" + target = next(t for t in skills_mod._TARGETS if any(a.kind == "folder" for a in t.artifacts)) + cmd._install_one(templates_root, tmp_path, target, force=False) + folder = next(tmp_path / a.install_relpath for a in target.artifacts if a.kind == "folder") + marker = folder / "keep.txt" + marker.write_text("old", encoding="utf-8") + + def fail(*_a, **_k): + raise OSError("disk full") + + monkeypatch.setattr(shutil, "copytree", fail) + with pytest.raises(OSError): + cmd._install_one(templates_root, tmp_path, target, force=True) + assert marker.read_text(encoding="utf-8") == "old" diff --git a/tests/cli/test_production_readiness.py b/tests/cli/test_production_readiness.py index 3521d197..8ae1d628 100644 --- a/tests/cli/test_production_readiness.py +++ b/tests/cli/test_production_readiness.py @@ -19,8 +19,18 @@ from tenxgraph_api.cli.core.output import OutputFormatter -REDIS_RATE_LIMIT = {"enabled": True, "backend": "redis", "redis": {"url": "redis://x"}} -READY = {"agent": "graph.react:app", "auth": "jwt", "rate_limit": REDIS_RATE_LIMIT} +REDIS_RATE_LIMIT = { + "enabled": True, + "backend": "redis", + "redis": {"url": "redis://x"}, + "by": "user", +} +READY = { + "agent": "graph.react:app", + "auth": "jwt", + "rate_limit": REDIS_RATE_LIMIT, + "token_revocation": "redis", +} AUTH_OFF_TEXT = "Auth is off" @@ -56,6 +66,36 @@ def test_disabled_rate_limit_warns() -> None: assert _paths(production_readiness_warnings(config)) == ["rate_limit"] +def test_auth_without_token_revocation_warns() -> None: + config = {key: value for key, value in READY.items() if key != "token_revocation"} + assert _paths(production_readiness_warnings(config)) == ["token_revocation"] + + +def test_per_ip_rate_limit_with_auth_warns() -> None: + config = {**READY, "rate_limit": {**REDIS_RATE_LIMIT, "by": "ip"}} + assert _paths(production_readiness_warnings(config)) == ["rate_limit.by"] + + +def test_redoc_exclusion_with_the_wrong_path_warns() -> None: + limit = {**REDIS_RATE_LIMIT, "exclude_paths": ["/ping", "/redoc"]} + issues = production_readiness_warnings({**READY, "rate_limit": limit}) + assert _paths(issues) == ["rate_limit.exclude_paths"] + + +def test_audit_reports_the_security_posture(tmp_path: Path, monkeypatch) -> None: + from tenxgraph_api.cli.commands.audit import AuditCommand + + monkeypatch.chdir(tmp_path) + (tmp_path / "10xgraph.json").write_text( + json.dumps({"agent": "graph.react:app", "auth": None}), encoding="utf-8" + ) + weak = AuditCommand._security_check() + assert weak.status == "warn" + assert "auth" in weak.detail + (tmp_path / "10xgraph.json").write_text(json.dumps(READY), encoding="utf-8") + assert AuditCommand._security_check().status == "pass" + + # --------------------------------------------------------------------------- # build # --------------------------------------------------------------------------- diff --git a/tests/unit_tests/store/test_store_service.py b/tests/unit_tests/store/test_store_service.py index bf073cd5..25f1cece 100644 --- a/tests/unit_tests/store/test_store_service.py +++ b/tests/unit_tests/store/test_store_service.py @@ -525,3 +525,34 @@ async def test_forget_memory_excludes_none_values(self, store_service, mock_stor assert "memory_type" not in call_args[1] assert "category" not in call_args[1] assert "filters" not in call_args[1] + + +@pytest.mark.asyncio +class TestReservedFilterKeys: + """J-4: client filters never carry owner keys to the store.""" + + HOSTILE = { + "user_id": "victim", + "agent_id": "a", + "app_id": "b", + "run_id": "c", + "source": "chat", + "OR": [{"user_id": "victim"}, {"tag": "x"}], + } + + async def test_search_drops_reserved_keys(self, store_service, mock_store, mock_user): + mock_store.asearch.return_value = [] + await store_service.search_memories( + SearchMemorySchema(query="q", filters=dict(self.HOSTILE)), mock_user + ) + assert mock_store.asearch.call_args.kwargs["filters"] == { + "source": "chat", + "OR": [{"tag": "x"}], + } + + async def test_forget_drops_reserved_keys(self, store_service, mock_store, mock_user): + await store_service.forget_memory(ForgetMemorySchema(filters=dict(self.HOSTILE)), mock_user) + assert mock_store.aforget_memory.call_args.kwargs["filters"] == { + "source": "chat", + "OR": [{"tag": "x"}], + } diff --git a/tests/unit_tests/test_cli_validation.py b/tests/unit_tests/test_cli_validation.py index 769cc3ff..e0d51f66 100644 --- a/tests/unit_tests/test_cli_validation.py +++ b/tests/unit_tests/test_cli_validation.py @@ -141,33 +141,35 @@ class TestValidatorPythonVersion: def test_validate_python_version_valid_two_parts(self): """Test validating valid Python versions with two parts.""" - assert Validator.validate_python_version("3.8") == "3.8" - assert Validator.validate_python_version("3.9") == "3.9" assert Validator.validate_python_version("3.12") == "3.12" assert Validator.validate_python_version("3.13") == "3.13" def test_validate_python_version_valid_three_parts(self): """Test validating valid Python versions with three parts.""" - assert Validator.validate_python_version("3.8.10") == "3.8.10" - assert Validator.validate_python_version("3.9.5") == "3.9.5" assert Validator.validate_python_version("3.12.0") == "3.12.0" def test_validate_python_version_too_old(self): """Test validating Python version that's too old.""" with pytest.raises(ValidationError) as exc_info: - Validator.validate_python_version("3.7.0") - assert "3.8 or higher" in str(exc_info.value) + Validator.validate_python_version("3.11.9") + assert "3.12 or higher" in str(exc_info.value) + + @pytest.mark.parametrize("version", ["3.8", "3.9", "3.10", "3.11", "3.8.10"]) + def test_validate_python_version_below_the_framework_minimum(self, version): + """The framework needs Python 3.12; an older image would fail at install time.""" + with pytest.raises(ValidationError): + Validator.validate_python_version(version) def test_validate_python_version_very_old(self): """Test validating very old Python version.""" with pytest.raises(ValidationError) as exc_info: Validator.validate_python_version("2.7.18") - assert "3.8 or higher" in str(exc_info.value) + assert "3.12 or higher" in str(exc_info.value) def test_validate_python_version_invalid_format(self): """Test validating Python version with invalid format.""" with pytest.raises(ValidationError) as exc_info: - Validator.validate_python_version("3.8.10.1") + Validator.validate_python_version("3.12.0.1") assert "format" in str(exc_info.value) def test_validate_python_version_not_string(self): @@ -344,19 +346,19 @@ def test_validate_cli_options_with_config(self): def test_validate_cli_options_with_python_version(self): """Test validating CLI options with Python version.""" - result = validate_cli_options(host="localhost", port=8000, python_version="3.9") - assert result["python_version"] == "3.9" + result = validate_cli_options(host="localhost", port=8000, python_version="3.13") + assert result["python_version"] == "3.13" def test_validate_cli_options_all_options(self): """Test validating CLI options with all options.""" with tempfile.NamedTemporaryFile(suffix=".json") as f: result = validate_cli_options( - host="localhost", port=8000, config=f.name, python_version="3.11" + host="localhost", port=8000, config=f.name, python_version="3.12" ) assert result["host"] == "localhost" assert result["port"] == 8000 assert "config" in result - assert result["python_version"] == "3.11" + assert result["python_version"] == "3.12" def test_validate_cli_options_invalid_host(self): """Test validating CLI options with invalid host.""" diff --git a/tests/unit_tests/test_client_config_isolation.py b/tests/unit_tests/test_client_config_isolation.py index 4eb065eb..b08047ce 100644 --- a/tests/unit_tests/test_client_config_isolation.py +++ b/tests/unit_tests/test_client_config_isolation.py @@ -363,7 +363,8 @@ async def test_reserved_keys_are_stripped_not_counted(graph_service): _, config, _ = await graph_service._prepare_input( _input(config={"user_id": "forged", "authz": {}, "_node_name": "x", "thread_id": "t1"}) ) - assert "user_id" not in config and "authz" not in config + # The forged id is gone; the server sets its own (anonymous when no user was passed). + assert config["user_id"] == "anonymous" and "authz" not in config @pytest.mark.asyncio diff --git a/tests/unit_tests/test_concurrency_limiter.py b/tests/unit_tests/test_concurrency_limiter.py index 81cecf05..3b6a61e6 100644 --- a/tests/unit_tests/test_concurrency_limiter.py +++ b/tests/unit_tests/test_concurrency_limiter.py @@ -36,13 +36,18 @@ async def zrem(self, key, member): self._check() self.zsets.get(key, {}).pop(member, None) - async def zadd(self, key, mapping, xx=False): + async def zadd(self, key, mapping, xx=False, ch=False): + """Like redis-py: the count of new members (plus changed scores with ``ch``).""" self._check() zset = self.zsets.setdefault(key, {}) + count = 0 for member, score in mapping.items(): if xx and member not in zset: continue + if member not in zset or ch and zset[member] != score: + count += 1 zset[member] = score + return count async def expire(self, key, seconds): self._check() @@ -184,7 +189,10 @@ async def test_redis_outage_fails_open(): redis.fail = True limiter = RedisConcurrencyLimiter(redis) lease = await limiter.acquire("k", 1) - assert lease is not None and lease.released # allowed, nothing to release + assert lease is not None # allowed, counted in this process only + assert await limiter.acquire("k", 1) is None # still one holder at a time per process + lease.release() + assert await limiter.acquire("k", 1) is not None @pytest.mark.asyncio diff --git a/tests/unit_tests/test_error_context_details.py b/tests/unit_tests/test_error_context_details.py index 14988c1a..bd82f65a 100644 --- a/tests/unit_tests/test_error_context_details.py +++ b/tests/unit_tests/test_error_context_details.py @@ -19,7 +19,7 @@ CASES = [ (GraphError, "GRAPH_000", 500), (NodeError, "NODE_000", 500), - (GraphRecursionError, "RECURSION_000", 500), + (GraphRecursionError, "RECURSION_000", 422), (StorageError, "STORAGE_000", 500), (TransientStorageError, "STORAGE_TRANSIENT_000", 503), (MetricsError, "METRICS_000", 500), diff --git a/tests/unit_tests/test_graph_service.py b/tests/unit_tests/test_graph_service.py index d83c1ea6..3c2ca1a0 100644 --- a/tests/unit_tests/test_graph_service.py +++ b/tests/unit_tests/test_graph_service.py @@ -280,16 +280,14 @@ async def mock_stream(*args, **kwargs): async for c in service.stream_graph(gi, {}): chunks.append(c) - assert ( - len(chunks) == 2 - ) # message chunk + final completed status chunk (due to thread name generator path branch) + # Naming runs in the background after the run: no extra chunk on the stream, only the + # terminal "done" update. + assert len(chunks) == 2 data0 = json.loads(chunks[0]) assert data0["event"] == "message" - - data1 = json.loads(chunks[1]) - assert data1["event"] == "updates" - assert data1["data"]["status"] == "completed" + assert json.loads(chunks[1])["data"] == {"status": "done"} + await service.drain_naming_tasks() @pytest.mark.asyncio async def test_stream_graph_exception_handling(self, service, mock_graph, monkeypatch): diff --git a/tests/unit_tests/test_group6_logging.py b/tests/unit_tests/test_group6_logging.py new file mode 100644 index 00000000..56a67c99 --- /dev/null +++ b/tests/unit_tests/test_group6_logging.py @@ -0,0 +1,157 @@ +"""G-114, G-115, G-116: log sanitizer and logging setup.""" + +# ruff: noqa: S101 + +import io +import logging +import uuid + +import pytest +from pydantic import BaseModel, ValidationError + +from tenxgraph_api.src.app.core.config import setup_logs +from tenxgraph_api.src.app.core.utils.log_sanitizer import ( + SanitizingFormatter, + redact_text, + sanitize_for_logging, + summarize_validation_error, +) + + +# G-115 + + +def test_uuid_ids_are_not_rewritten(): + thread_id = str(uuid.uuid4()) + assert sanitize_for_logging({"thread_id": thread_id}) == {"thread_id": thread_id} + assert sanitize_for_logging(thread_id) == thread_id + + +def test_long_opaque_tokens_are_still_shortened(): + token = "a1b2c3d4e5" * 5 + assert sanitize_for_logging(token) != token + + +@pytest.mark.parametrize("key", ["monkey", "shipping", "author", "keyboard", "spinning", "turkey"]) +def test_innocent_keys_are_not_redacted(key): + assert sanitize_for_logging({key: "visible"}) == {key: "visible"} + + +@pytest.mark.parametrize( + "key", + [ + "token", + "api_key", + "apiKey", + "Authorization", + "x-api-key", + "access_token", + "password", + "client_secret", + "refreshToken", + "Cookie", + "private_key", + ], +) +def test_secret_keys_are_redacted(key): + assert sanitize_for_logging({key: "value"}) == {key: "***REDACTED***"} + + +def _format(message, *args): + formatter = SanitizingFormatter(logging.Formatter("%(message)s")) + record = logging.LogRecord("x", logging.INFO, __file__, 1, message, args, None) + return formatter.format(record) + + +def test_newline_in_a_logged_value_cannot_forge_a_line(): + out = _format("user %s logged in", "bob\n2026-01-01 - x - INFO - forged line") + assert "\n" not in out + assert "\\n" in out + + +# G-114 + + +@pytest.mark.parametrize( + "secret", + [ + "sk-proj-abcdefghijklmnopqrstuvwx", + "sk-ant-api03-abcdefghijklmnopqrstuvwx", + "AIzaSyA1234567890abcdefghijklmnopqrstu", + "ghp_abcdefghijklmnopqrstuvwxyz0123456789", + "AKIAABCDEFGHIJKLMNOP", + ], +) +def test_provider_keys_are_redacted_in_text(secret): + assert secret not in redact_text(f"failed calling provider with key {secret} now") + + +def test_query_tokens_are_redacted_in_text(): + out = redact_text("GET /v1/graph/ws?token=abc.def.ghi&x=1 HTTP/1.1") + assert "abc.def.ghi" not in out + assert "x=1" in out + + +def test_access_log_hides_the_query_token(): + stream = io.StringIO() + handler = logging.StreamHandler(stream) + logger = logging.getLogger("uvicorn.access") + saved = (list(logger.handlers), list(logger.filters)) + try: + setup_logs.init_logger(logging.INFO) + for f in logger.filters: + handler.addFilter(f) + record = logger.makeRecord( + "uvicorn.access", + logging.INFO, + __file__, + 1, + '%s - "%s %s HTTP/%s" %d', + ("127.0.0.1:1", "GET", "/v1/graph/ws?token=SECRETVALUE", "1.1", 101), + None, + ) + assert logger.filter(record) + assert "SECRETVALUE" not in record.getMessage() + finally: + logger.handlers, logger.filters = saved[0], saved[1] + + +class _Model(BaseModel): + name: str + + +def test_validation_summary_has_no_input_values(): + with pytest.raises(ValidationError) as exc: + _Model(name=12345678) # type: ignore[arg-type] + assert "12345678" in str(exc.value) + summary = summarize_validation_error(exc.value) + assert "12345678" not in summary + assert "name" in summary + + +def test_ag_ui_invalid_input_message_does_not_echo_values(): + from tenxgraph_api.src.app.routers.ag_ui.converter import parse_run_input + + with pytest.raises(ValueError) as exc: + parse_run_input({"threadId": "t", "runId": "r", "messages": "SECRET-BODY-TEXT"}) + assert "SECRET-BODY-TEXT" not in str(exc.value) + + +# G-116 + + +def test_core_logger_records_reach_the_console_handler(): + stream = io.StringIO() + core = logging.getLogger("tenxgraph") + saved = (list(core.handlers), core.propagate, core.level) + try: + core.addHandler(logging.NullHandler()) # what a library does + setup_logs.init_logger(logging.INFO) + handlers = [h for h in core.handlers if isinstance(h.formatter, SanitizingFormatter)] + assert handlers, "core records have no sanitizing handler" + handlers[0].setStream(stream) if hasattr(handlers[0], "setStream") else None + logging.getLogger("tenxgraph.storage.checkpointer").warning("checkpoint failed") + assert "checkpoint failed" in stream.getvalue() + finally: + core.handlers[:], core.propagate = saved[0], saved[1] + core.setLevel(saved[2]) diff --git a/tests/unit_tests/test_group6_small_fixes.py b/tests/unit_tests/test_group6_small_fixes.py new file mode 100644 index 00000000..ffa2e88f --- /dev/null +++ b/tests/unit_tests/test_group6_small_fixes.py @@ -0,0 +1,108 @@ +"""G-161 (checkpointer key warning), G-169 (stale graph:setup scope).""" + +# ruff: noqa: S101 + +import logging +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tenxgraph_api.cli.config_editor.schema import AUTHORIZATION_SCOPES +from tenxgraph_api.src.app import loader + + +@pytest.mark.asyncio +async def test_checkpointer_key_logs_a_warning(caplog): + config = MagicMock() + config.remote_tools = [] + config.graph_path = "graph:app" + config.checkpointer_path = "graph.deps:cp" + with ( + patch.object(loader, "load_graph", AsyncMock(return_value=None)), + patch.object(loader, "load_store", return_value=None), + patch.object(loader, "load_container", return_value=None), + caplog.at_level(logging.WARNING, logger="tenxgraph_api"), + ): + try: + await loader.attach_all_modules(config, MagicMock()) + except Exception: # - only the warning matters here + pass + assert any('"checkpointer" key' in r.getMessage() for r in caplog.records) + + +def test_skill_templates_do_not_tell_developers_to_set_the_checkpointer_key(): + root = Path(loader.__file__).parents[2] / "cli" / "templates" / "skills" + for name in ("api-configuration.md", "dependency-injection.md"): + text = (root / "10xgraph" / "references" / name).read_text(encoding="utf-8") + assert '"checkpointer":' not in text, name + + +def test_setup_scope_is_gone_from_the_editor(): + assert "graph:setup" not in AUTHORIZATION_SCOPES + + +# --------------------------------------------------------------------------- +# G-119 +# --------------------------------------------------------------------------- + + +def test_allowed_host_list_is_stripped_for_the_middleware(monkeypatch): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from tenxgraph_api.src.app.core.config import setup_middleware as module + from tenxgraph_api.src.app.core.config.settings import Settings + + settings = Settings(MODE="development", ALLOWED_HOST="testserver, example.com ", _env_file=None) + monkeypatch.setattr(module, "get_settings", lambda: settings, raising=False) + app = FastAPI() + with patch.object(module, "settings", settings, create=True): + module.setup_middleware(app) + + @app.get("/x") + def x(): + return {"ok": True} + + assert TestClient(app, base_url="http://example.com").get("/x").status_code == 200 + + +def test_is_debug_reaches_the_app(): + import inspect + + from tenxgraph_api.src.app import main + + assert "debug=settings.IS_DEBUG" in inspect.getsource(main) + + +def test_production_warns_about_debug_logging_and_full_otel(caplog): + from tenxgraph_api.src.app.core.config.settings import Settings + + with caplog.at_level(logging.WARNING): + Settings( + MODE="production", + LOG_LEVEL="DEBUG", + OTEL_ENABLED=True, + OTEL_LEVEL="full", + ORIGINS="https://a.example", + ALLOWED_HOST="a.example", + _env_file=None, + ) + text = " ".join(r.getMessage() for r in caplog.records) + assert "LOG_LEVEL=DEBUG" in text + assert "OTEL_LEVEL=full" in text + + +def test_dev_command_warns_about_reload_and_open_host_in_production(): + from tenxgraph_api.cli.commands.api import APICommand + + warn = APICommand.production_launch_warnings + assert len(warn("production", "0.0.0.0", True)) == 2 + assert warn("development", "0.0.0.0", True) == [] + assert warn("production", "127.0.0.1", False) == [] + + +def test_star_role_is_not_granted_the_removed_setup_scope(): + from tenxgraph_api.src.app.core.auth.authorization import all_scopes + + assert "graph:setup" not in all_scopes() diff --git a/tests/unit_tests/test_handle_errors.py b/tests/unit_tests/test_handle_errors.py index be29acc7..c00c0c19 100644 --- a/tests/unit_tests/test_handle_errors.py +++ b/tests/unit_tests/test_handle_errors.py @@ -250,7 +250,7 @@ def validation_error(): r = client.get("/validation-error") assert r.status_code == 422 body = r.json() - assert body["error"]["code"] == "AGENTFLOW_VALIDATION_ERROR" + assert body["error"]["code"] == "VALIDATION_ERROR" cleanup_env() @@ -267,7 +267,7 @@ def validation_error(): r = client.get("/validation-error") assert r.status_code == 422 body = r.json() - assert body["error"]["code"] == "AGENTFLOW_VALIDATION_ERROR" + assert body["error"]["code"] == "VALIDATION_ERROR" cleanup_env() @@ -334,7 +334,7 @@ def recursion_error(): client = TestClient(app) r = client.get("/recursion-error") - assert r.status_code == 500 + assert r.status_code == 422 body = r.json() assert body["error"]["code"] == "GRAPH_RECURSION_001" diff --git a/tests/unit_tests/test_lock_lease_integrity.py b/tests/unit_tests/test_lock_lease_integrity.py new file mode 100644 index 00000000..1feea77f --- /dev/null +++ b/tests/unit_tests/test_lock_lease_integrity.py @@ -0,0 +1,96 @@ +"""Lease integrity of the shared limiter behind the thread lock (G-91, G-94).""" + +# ruff: noqa: S101 + +import asyncio + +import pytest + +from tenxgraph_api.src.app.core import concurrency +from tenxgraph_api.src.app.core.concurrency import RedisConcurrencyLimiter + +from .test_concurrency_limiter import _FakeRedis + + +KEY = "10xgraph:concurrency:runs:thread:t1" + + +class _SlowPipeRedis(_FakeRedis): + """A Redis whose transaction takes a while, so a caller can be cancelled inside it.""" + + def pipeline(self, transaction=True): + pipe = super().pipeline(transaction) + execute = pipe.execute + + async def slow_execute(): + results = await execute() # the server applied it ... + await asyncio.sleep(10) # ... but the reply is slow + return results + + pipe.execute = slow_execute + return pipe + + +class _StrictCloseRedis(_FakeRedis): + """Refuses commands once closed, like a real client.""" + + async def zrem(self, key, member): + await asyncio.sleep(0.01) + if self.closed: + raise ConnectionError("client closed") + await super().zrem(key, member) + + +@pytest.mark.asyncio +async def test_redis_outage_keeps_one_run_per_thread_in_this_process(): + redis = _FakeRedis() + redis.fail = True + limiter = RedisConcurrencyLimiter(redis) + first = await limiter.acquire("runs:thread:t1", 1) + assert first is not None + assert await limiter.acquire("runs:thread:t1", 1) is None + first.release() + assert await limiter.acquire("runs:thread:t1", 1) is not None + await limiter.aclose() + + +@pytest.mark.asyncio +async def test_lost_lease_is_detected_and_restored(): + redis = _FakeRedis() + limiter = RedisConcurrencyLimiter(redis, lease_ttl=0.06) + lease = await limiter.acquire("runs:thread:t1", 1) + assert lease is not None and not lease.lost + redis.zsets[KEY].clear() # evicted, or expired during a stall + await asyncio.sleep(0.05) + assert lease.lost + assert len(redis.zsets[KEY]) == 1 # held again, so a second run is still refused + assert await limiter.acquire("runs:thread:t1", 1) is None + lease.release() + await limiter.aclose() + + +@pytest.mark.asyncio +async def test_a_cancelled_waiter_leaves_no_ghost_lease(): + redis = _SlowPipeRedis() + limiter = RedisConcurrencyLimiter(redis, lease_ttl=60) + waiter = asyncio.create_task(limiter.acquire("runs:thread:t1", 1)) + await asyncio.sleep(0.05) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + await asyncio.sleep(0.05) + assert redis.zsets.get(KEY, {}) == {} + await limiter.aclose() + + +@pytest.mark.asyncio +async def test_aclose_waits_for_pending_releases_before_closing_the_client(): + redis = _StrictCloseRedis() + limiter = RedisConcurrencyLimiter(redis, lease_ttl=60) + lease = await limiter.acquire("runs:thread:t1", 1) + assert lease is not None + lease.release() # scheduled, not yet run + await limiter.aclose() + assert redis.zsets[KEY] == {} + assert redis.closed + assert not concurrency._pending_releases diff --git a/tests/unit_tests/test_middleware_order.py b/tests/unit_tests/test_middleware_order.py index bccfc39b..e88d1e02 100644 --- a/tests/unit_tests/test_middleware_order.py +++ b/tests/unit_tests/test_middleware_order.py @@ -62,6 +62,7 @@ def test_cors_is_outermost(): "RateLimitMiddleware", "RequestSizeLimitMiddleware", "SelectiveGZipMiddleware", + "UnhandledErrorMiddleware", ] @@ -95,3 +96,77 @@ def test_preflight_is_answered_before_the_rate_limiter(path): for _ in range(3): # CORS answers preflights itself; they never reach the limiter assert client.options(path, headers=headers).status_code == 200 assert client.get(path, headers={"Origin": ORIGIN}).status_code == 200 + + +def test_unhandled_exception_is_a_json_500_with_request_id_and_headers(): + app = _app() + + @app.get("/boom") + async def boom(): + raise RuntimeError("secret internal detail") + + client = TestClient(app, raise_server_exceptions=False) + response = client.get("/boom", headers={"Origin": ORIGIN}) + assert response.status_code == 500 + assert response.headers["content-type"].startswith("application/json") + _assert_decorated(response) + assert response.headers["cache-control"] == "no-store" + body = response.json() + assert body["error"]["code"] == "INTERNAL_SERVER_ERROR" + assert body["metadata"]["request_id"] == response.headers["x-request-id"] + + +@pytest.mark.parametrize( + ("make", "status", "retry"), + [ + (lambda: __import__("redis").exceptions.ConnectionError("down"), 503, True), + ( + lambda: type("RateLimitError", (Exception,), {})("slow down"), + 429, + True, + ), + (lambda: RuntimeError("bug"), 500, False), + ], +) +def test_retryable_failures_are_not_plain_500s(make, status, retry): + app = _app() + + @app.get("/boom") + async def boom(): + raise make() + + response = TestClient(app, raise_server_exceptions=False).get("/boom") + assert response.status_code == status + assert ("retry-after" in response.headers) is retry + + +def test_a_wrapped_driver_error_is_still_a_503(): + app = _app() + + @app.get("/boom") + async def boom(): + try: + raise __import__("redis").exceptions.TimeoutError("slow") + except Exception as exc: + raise RuntimeError("checkpoint failed") from exc + + response = TestClient(app, raise_server_exceptions=False).get("/boom") + assert response.status_code == 503 + assert response.headers["retry-after"] + + +def test_http_exception_headers_reach_the_client(): + from fastapi import HTTPException + + from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler + + app = _app() + init_errors_handler(app) + + @app.get("/busy") + async def busy(): + raise HTTPException(409, "busy", headers={"Retry-After": "3"}) + + response = TestClient(app).get("/busy") + assert response.status_code == 409 + assert response.headers["retry-after"] == "3" diff --git a/tests/unit_tests/test_production_surface.py b/tests/unit_tests/test_production_surface.py index 1f3a5b68..622aa5b3 100644 --- a/tests/unit_tests/test_production_surface.py +++ b/tests/unit_tests/test_production_surface.py @@ -180,7 +180,7 @@ def test_dockerfile_installs_only_what_the_dependency_file_lists(): def test_dockerfile_uses_the_uvicorn_worker_package(): content = _dockerfile(has_requirements=False, has_pyproject=True) - assert '"uvicorn_worker.UvicornWorker"' in content + assert '"tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker"' in content assert "uvicorn.workers" not in content diff --git a/tests/unit_tests/test_readiness.py b/tests/unit_tests/test_readiness.py index 753cca85..93acf86b 100644 --- a/tests/unit_tests/test_readiness.py +++ b/tests/unit_tests/test_readiness.py @@ -131,9 +131,9 @@ async def ping(self): async def aclose(self): calls.append("closed") - import redis.asyncio - - monkeypatch.setattr(redis.asyncio.Redis, "from_url", classmethod(lambda cls, url: _Client())) + monkeypatch.setattr( + "tenxgraph_api.src.app.core.redis_client.create_redis", lambda url, **kw: _Client() + ) assert (await readiness.readiness(None, None, "redis://cache"))["redis"] == "ok" assert calls == ["ping", "closed"] diff --git a/tests/unit_tests/test_redis_hardening.py b/tests/unit_tests/test_redis_hardening.py new file mode 100644 index 00000000..c0df5638 --- /dev/null +++ b/tests/unit_tests/test_redis_hardening.py @@ -0,0 +1,249 @@ +"""Redis client limits (G-92), Redis clock (G-131) and revocation fixes (G-117, G-167).""" + +# ruff: noqa: S101 + +import time +from typing import Any + +import pytest + +from tenxgraph_api.src.app.core import redis_client +from tenxgraph_api.src.app.core.auth import revocation +from tenxgraph_api.src.app.core.auth.revocation import ( + DEFAULT_USER_REVOCATION_TTL, + InMemoryTokenRevocation, + RedisTokenRevocation, +) +from tenxgraph_api.src.app.core.concurrency import RedisConcurrencyLimiter +from tenxgraph_api.src.app.core.config.settings import get_settings + + +@pytest.fixture(autouse=True) +def _fresh_settings(monkeypatch): + for name in ( + "REDIS_SOCKET_TIMEOUT", + "REDIS_CONNECT_TIMEOUT", + "REDIS_MAX_CONNECTIONS", + "REDIS_POOL_TIMEOUT", + "REDIS_HEALTH_CHECK_INTERVAL", + ): + monkeypatch.delenv(name, raising=False) + get_settings.cache_clear() + yield + get_settings.cache_clear() + + +# --------------------------------------------------------------------------- +# G-92: timeouts and a bounded pool on every client +# --------------------------------------------------------------------------- + + +def test_default_client_has_timeouts_and_a_bounded_pool(): + client = redis_client.create_redis("redis://localhost:6379/0") + kwargs = client.connection_pool.connection_kwargs + assert kwargs["socket_timeout"] == 5.0 + assert kwargs["socket_connect_timeout"] == 5.0 + assert kwargs["health_check_interval"] == 30.0 + assert client.connection_pool.max_connections == 50 + + +def test_client_limits_are_configurable(monkeypatch): + monkeypatch.setenv("REDIS_SOCKET_TIMEOUT", "1.5") + monkeypatch.setenv("REDIS_CONNECT_TIMEOUT", "2") + monkeypatch.setenv("REDIS_MAX_CONNECTIONS", "7") + get_settings.cache_clear() + client = redis_client.create_redis("redis://localhost:6379/0") + assert client.connection_pool.connection_kwargs["socket_timeout"] == 1.5 + assert client.connection_pool.connection_kwargs["socket_connect_timeout"] == 2.0 + assert client.connection_pool.max_connections == 7 + + +def test_zero_turns_a_limit_off(monkeypatch): + monkeypatch.setenv("REDIS_SOCKET_TIMEOUT", "0") + monkeypatch.setenv("REDIS_MAX_CONNECTIONS", "0") + get_settings.cache_clear() + client = redis_client.create_redis("redis://localhost:6379/0") + assert "socket_timeout" not in client.connection_pool.connection_kwargs or ( + client.connection_pool.connection_kwargs["socket_timeout"] is None + ) + + +def test_every_api_redis_client_uses_the_shared_builder(monkeypatch): + built = [] + monkeypatch.setattr( + redis_client, "create_redis", lambda url, **kw: built.append(url) or object() + ) + RedisConcurrencyLimiter.from_url("redis://a") + RedisTokenRevocation("redis://b") + from tenxgraph_api.src.app.core.middleware.rate_limit.redis import RedisRateLimitBackend + + class _Client: + def register_script(self, _): + return None + + monkeypatch.setattr( + redis_client, "create_redis", lambda url, **kw: built.append(url) or _Client() + ) + RedisRateLimitBackend.from_url("redis://c", "p") + from tenxgraph_api.src.app.loader import _build_ownership_redis + + _build_ownership_redis("redis://d") + assert built == ["redis://a", "redis://b", "redis://c", "redis://d"] + + +# --------------------------------------------------------------------------- +# G-131: lease times come from the Redis clock +# --------------------------------------------------------------------------- + + +class _SkewedRedis: + """Redis whose clock is an hour ahead of this process.""" + + SKEW = 3600.0 + + def __init__(self): + self.scores: list[float] = [] + + async def time(self): + now = time.time() + self.SKEW + return int(now), int((now % 1) * 1_000_000) + + def pipeline(self, transaction=True): + scores = self.scores + + class _Pipe: + def zremrangebyscore(self, key, low, high): + self.high = high + + def zadd(self, key, mapping): + scores.extend(mapping.values()) + + def zcard(self, key): + pass + + def expire(self, key, seconds): + pass + + async def execute(self): + return [None, 1, 1, True] + + return _Pipe() + + async def zrem(self, key, member): + pass + + +@pytest.mark.asyncio +async def test_lease_expiry_uses_the_redis_clock(): + client = _SkewedRedis() + limiter = RedisConcurrencyLimiter(client, lease_ttl=60.0) + lease = await limiter.acquire("k", 1) + assert lease is not None + # With the pod's own clock the score would be about now + 60; the Redis clock is +3600. + assert client.scores[0] > time.time() + 3000 + lease.release() + await limiter.aclose() + + +# --------------------------------------------------------------------------- +# G-117 / G-167: revocation +# --------------------------------------------------------------------------- + + +class _FakeRedis: + def __init__(self): + self.data: dict[str, Any] = {} + self.ttl: dict[str, Any] = {} + self.mget_calls = 0 + + async def time(self): + now = time.time() + return int(now), int((now % 1) * 1_000_000) + + async def set(self, key, value, ex=None): + self.data[key] = value + self.ttl[key] = ex + + async def get(self, key): + return self.data.get(key) + + async def mget(self, keys): + self.mget_calls += 1 + raise AssertionError("MGET over two keys fails with CROSSSLOT on Redis Cluster") + + async def eval(self, script, numkeys, key, cutoff, ttl): + new = float(cutoff) if cutoff else time.time() + current = self.data.get(key) + if current is not None and float(current) >= new: + return 0 + self.data[key] = repr(new).encode() + self.ttl[key] = int(ttl) or None + return 1 + + +@pytest.mark.asyncio +async def test_check_does_not_use_mget(): + client = _FakeRedis() + backend = RedisTokenRevocation(client=client, prefix="p") + await backend.revoke_user("a", before=100.0) + assert await backend.is_revoked({"user_id": "a", "jti": "x", "iat": 50}) is True + assert client.mget_calls == 0 + + +@pytest.mark.asyncio +async def test_an_older_cutoff_does_not_replace_a_newer_one(): + client = _FakeRedis() + backend = RedisTokenRevocation(client=client, prefix="p") + await backend.revoke_user("a", before=2000.0) + await backend.revoke_user("a", before=1000.0) + assert await backend.is_revoked({"user_id": "a", "iat": 1500}) is True + + +def test_memory_older_cutoff_does_not_replace_a_newer_one(): + backend = InMemoryTokenRevocation() + backend.revoke_user("a", before=2000.0) + backend.revoke_user("a", before=1000.0) + assert backend._users["a"] == 2000.0 + + +@pytest.mark.asyncio +async def test_user_entries_expire_by_default(): + client = _FakeRedis() + backend = RedisTokenRevocation(client=client, prefix="p") + await backend.revoke_user("a") + assert client.ttl["p:user:a"] == DEFAULT_USER_REVOCATION_TTL + + +@pytest.mark.asyncio +async def test_token_without_iat_is_dated_from_exp_when_the_lifetime_is_known(): + now = time.time() + backend = InMemoryTokenRevocation(token_lifetime=3600) + backend.revoke_user("a", before=now) + old = {"user_id": "a", "exp": now + 100} # minted about 3500 s ago + new = {"user_id": "a", "exp": now + 3600 + 100} # minted after the revoke + assert await backend.is_revoked(old) is True + assert await backend.is_revoked(new) is False + # Without a lifetime an undated token cannot be told apart and stays revoked. + plain = InMemoryTokenRevocation() + plain.revoke_user("a", before=now) + assert await plain.is_revoked(new) is True + + +@pytest.mark.asyncio +async def test_exported_helpers_revoke_through_the_backend(): + backend = InMemoryTokenRevocation() + await revocation.revoke_token("j1", backend=backend) + await revocation.revoke_user("u1", before=time.time(), backend=backend) + assert await backend.is_revoked({"jti": "j1"}) is True + assert await backend.is_revoked({"user_id": "u1", "iat": 1}) is True + + redis = RedisTokenRevocation(client=_FakeRedis(), prefix="p") + await revocation.revoke_token("j2", backend=redis) + assert await redis.is_revoked({"jti": "j2"}) is True + + +@pytest.mark.asyncio +async def test_helpers_need_a_backend(monkeypatch): + monkeypatch.setattr(revocation, "_bound_backend", lambda: None) + with pytest.raises(RuntimeError, match="token_revocation"): + await revocation.revoke_user("u") diff --git a/tests/unit_tests/test_resume_needs_user_id.py b/tests/unit_tests/test_resume_needs_user_id.py new file mode 100644 index 00000000..c43526d4 --- /dev/null +++ b/tests/unit_tests/test_resume_needs_user_id.py @@ -0,0 +1,94 @@ +"""Resume at ``interrupt()`` must work on a checkpointer that requires ``config['user_id']``. + +PgCheckpointer raises ``user_id must be provided in config`` on every read without it, so the +service has to put the authenticated user into the run config before its first state read. +""" + +# ruff: noqa: S101 + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from tenxgraph.core.state import AgentState +from tenxgraph.utils.interrupt import Interrupt, pause_at + +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema + +from .test_run_limits import ALICE, _service + + +def _strict_service(graph): + service = _service(graph, timeout=None) + state = AgentState() + pause_at(state, Interrupt(id="i1", key="k", node="ASK", value={"amount": 50})) + + async def aget_state(config): + if not config.get("user_id"): + raise ValueError("user_id must be provided in config") + return state + + service.checkpointer.aget_state = AsyncMock(side_effect=aget_state) + return service + + +def _resume(**fields): + return GraphInputSchema(config={"thread_id": "t1"}, resume={"ok": True}, **fields) + + +@pytest.mark.parametrize("interrupt_id", [None, "i1"]) +@pytest.mark.asyncio +async def test_invoke_resume_reads_state_with_user_id(interrupt_id): + graph = MagicMock() + graph.ainvoke = AsyncMock(return_value={"messages": [], "state": AgentState()}) + service = _strict_service(graph) + await service.invoke_graph(_resume(interrupt_id=interrupt_id), ALICE) + input_data = graph.ainvoke.call_args.args[0] + assert input_data["resume"] == {"ok": True} + + +@pytest.mark.asyncio +async def test_stream_resume_reads_state_with_user_id(): + graph = MagicMock() + seen = {} + + async def astream(input_data, **kwargs): + seen.update(input_data) + return + yield + + graph.astream = astream + service = _strict_service(graph) + chunks = [c async for c in service.stream_chunks(_resume(interrupt_id="i1"), ALICE)] + assert not any(getattr(c, "is_error", False) for c in chunks) + assert seen["resume"] == {"ok": True} + + +@pytest.mark.asyncio +async def test_websocket_path_shares_stream_chunks(): + # /v1/graph/ws drives GraphService.stream_graph, which wraps stream_chunks. + graph = MagicMock() + seen = {} + + async def astream(input_data, **kwargs): + seen.update(input_data) + return + yield + + graph.astream = astream + service = _strict_service(graph) + out = [c async for c in service.stream_graph(_resume(), ALICE)] + assert not any('"error"' in c for c in out) + assert seen["resume"] == {"ok": True} + + +@pytest.mark.asyncio +async def test_client_cannot_supply_user_id(): + service = _strict_service(MagicMock()) + graph_input = GraphInputSchema(config={"thread_id": "t1", "user_id": "mallory"}, resume={}) + try: + _, config, _ = await service._prepare_input(graph_input, ALICE["user_id"]) + except Exception: # rejected outright is also fine + return + assert config["user_id"] == "alice" diff --git a/tests/unit_tests/test_run_entry_guards.py b/tests/unit_tests/test_run_entry_guards.py new file mode 100644 index 00000000..797ed803 --- /dev/null +++ b/tests/unit_tests/test_run_entry_guards.py @@ -0,0 +1,410 @@ +"""One guard step for every run entry point, and the WebSocket run lifecycle (G-98 to G-100, G-158). + +``/v1/graph/stream``, ``/v1/graph/ws``, ``/v1/graph/live`` and AG-UI all take the same thread +lock and per-user run slot, hold them until the run is really over, and cancel cleanly. +""" + +# ruff: noqa: S101 + +import asyncio +import json +import sys +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException, WebSocketDisconnect + +from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.routers.graph import run_limits, thread_lock +from tenxgraph_api.src.app.routers.graph.router import ( + realtime_graph_ws, + stream_graph, + websocket_graph, +) +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema + + +ALICE = {"user_id": "alice"} +MSG = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] +FRAME = {"invoke_type": "fresh", "messages": MSG, "config": {"thread_id": "t1"}} +THREAD_KEY = "runs:thread:t1" +SLOT_KEY = "runs:user:alice" + + +def _router(): + return sys.modules[websocket_graph.__module__] + + +@pytest.fixture(autouse=True) +def limiter(monkeypatch): + """One in-process limiter behind both the thread lock and the run slot.""" + shared = MemoryConcurrencyLimiter() + monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: shared) + monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: shared) + monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.2) + router = _router() + monkeypatch.setattr(router, "ws_identity_still_valid", lambda ws, user: True) + monkeypatch.setattr(router, "_ws_token_revoked", AsyncMock(return_value=False)) + return shared + + +def _production(monkeypatch): + from tenxgraph_api.src.app.core.config import settings + + monkeypatch.setattr( + settings, + "get_settings", + lambda: SimpleNamespace(MODE="production", MAX_REQUEST_SIZE=10_000_000), + ) + + +def _service(runs=None): + service = MagicMock() + service.is_live_agent = False + service.runs_policy.return_value = runs or RunsConfig(max_concurrent_per_user=1) + service.stop_graph = AsyncMock(return_value={"ok": True}) + service.check_media_urls = MagicMock() + service.check_resume_target = AsyncMock() + return service + + +def _ws(frames): + ws = MagicMock() + ws.accept = AsyncMock() + ws.send_text = AsyncMock() + ws.send_bytes = AsyncMock() + ws.close = AsyncMock() + ws.receive_text = AsyncMock( + side_effect=[json.dumps(f) if isinstance(f, dict) else f for f in frames] + ) + return ws + + +def _sent(ws): + return [json.loads(c.args[0]) for c in ws.send_text.call_args_list] + + +async def _until(predicate, timeout=2.0): + loop = asyncio.get_running_loop() + end = loop.time() + timeout + while not predicate(): + assert loop.time() < end, "condition not reached" + await asyncio.sleep(0.01) + + +# --------------------------------------------------------------------------- +# /v1/graph/ws: run slot, error text +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_ws_run_holds_a_run_slot_and_the_thread_lock(limiter): + seen = {} + + def stream(*args, **kwargs): + async def gen(): + seen.update(limiter._held) + yield "chunk\n" + + return gen() + + service = _service() + service.stream_graph = stream + ws = _ws([FRAME, WebSocketDisconnect()]) + await websocket_graph(websocket=ws, service=service, user=ALICE) + assert seen == {THREAD_KEY: 1, SLOT_KEY: 1} + assert limiter._held == {} + + +@pytest.mark.asyncio +async def test_ws_over_the_run_cap_sends_an_error_chunk_and_stays_open(limiter): + held = await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) + service = _service() + service.stream_graph = MagicMock() + ws = _ws([FRAME, WebSocketDisconnect()]) + await websocket_graph(websocket=ws, service=service, user=ALICE) + chunk = _sent(ws)[0] + assert chunk["event"] == "error" + assert "Too many runs" in chunk["data"]["reason"] + service.stream_graph.assert_not_called() + ws.close.assert_not_awaited() + assert limiter._held == {SLOT_KEY: 1} # the thread lock was given back + held.release() + + +@pytest.mark.asyncio +async def test_ws_invalid_input_does_not_echo_the_validation_text_in_production(monkeypatch): + _production(monkeypatch) + bad = {"invoke_type": "fresh", "messages": [], "config": {"api_key": "hunter2-secret"}} + service = _service() + ws = _ws([bad, WebSocketDisconnect()]) + await websocket_graph(websocket=ws, service=service, user=ALICE) + reason = _sent(ws)[0]["data"]["reason"] + assert "messages must not be empty" not in reason + assert "hunter2" not in reason + + +# --------------------------------------------------------------------------- +# /v1/graph/ws: run lifecycle +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_ws_lock_is_released_only_after_the_run_generator_closed(limiter): + held_when_closed = [] + + def stream(*args, **kwargs): + async def gen(): + try: + yield "one\n" + yield "two\n" + finally: + await asyncio.sleep(0) # the generator's own async cleanup + held_when_closed.append(dict(limiter._held)) + + return gen() + + service = _service() + service.stream_graph = stream + ws = _ws([FRAME, WebSocketDisconnect()]) + ws.send_text = AsyncMock(side_effect=[RuntimeError("socket gone")]) + await websocket_graph(websocket=ws, service=service, user=ALICE) + await asyncio.sleep(0.05) + assert held_when_closed == [{THREAD_KEY: 1, SLOT_KEY: 1}] + assert limiter._held == {} + + +@pytest.mark.asyncio +async def test_ws_client_leaving_mid_run_cancels_the_run(limiter): + started, closed = asyncio.Event(), asyncio.Event() + + def stream(*args, **kwargs): + async def gen(): + try: + started.set() + await asyncio.sleep(30) # a long tool call: nothing to send + yield "never\n" + finally: + closed.set() + + return gen() + + service = _service() + service.stream_graph = stream + ws = _ws([FRAME]) + + async def disconnect_after_start(): + await started.wait() + await asyncio.sleep(0.02) + raise WebSocketDisconnect + + ws.receive_text.side_effect = _frames_then(json.dumps(FRAME), disconnect_after_start) + await asyncio.wait_for(websocket_graph(websocket=ws, service=service, user=ALICE), 3) + assert closed.is_set() + assert limiter._held == {} + + +def _frames_then(first, later): + calls = [] + + async def receive(): + calls.append(1) + if len(calls) == 1: + return first + return await later() + + return receive + + +@pytest.mark.asyncio +async def test_ws_slow_consumer_is_cut_off_and_frees_the_lock(limiter, monkeypatch): + monkeypatch.setattr(_router(), "WS_SEND_TIMEOUT_SECONDS", 0.05) + + def stream(*args, **kwargs): + async def gen(): + yield "chunk\n" + + return gen() + + service = _service() + service.stream_graph = stream + ws = _ws([FRAME]) + ws.receive_text.side_effect = _frames_then(json.dumps(FRAME), _forever) + + async def stalled(text): + await asyncio.sleep(30) + + ws.send_text = AsyncMock(side_effect=stalled) + await asyncio.wait_for(websocket_graph(websocket=ws, service=service, user=ALICE), 3) + ws.close.assert_awaited() + assert limiter._held == {} + + +@pytest.mark.asyncio +async def test_ws_handler_cancel_propagates_and_frees_everything(limiter): + started = asyncio.Event() + + def stream(*args, **kwargs): + async def gen(): + started.set() + await asyncio.sleep(30) + yield "never\n" + + return gen() + + service = _service() + service.stream_graph = stream + ws = _ws([FRAME]) + ws.receive_text.side_effect = _frames_then(json.dumps(FRAME), _forever) + before = asyncio.all_tasks() + task = asyncio.create_task(websocket_graph(websocket=ws, service=service, user=ALICE)) + await asyncio.wait_for(started.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.sleep(0.05) + assert limiter._held == {} + assert asyncio.all_tasks() - before - {task} == set() + + +async def _forever(): + await asyncio.sleep(30) + + +# --------------------------------------------------------------------------- +# /v1/graph/live +# --------------------------------------------------------------------------- + + +def _live_ws(init): + ws = MagicMock() + ws.accept = AsyncMock() + ws.receive_text = AsyncMock(return_value=json.dumps(init)) + ws.receive = AsyncMock(side_effect=_forever) + ws.send_bytes = AsyncMock() + ws.send_text = AsyncMock() + ws.close = AsyncMock() + return ws + + +def _live_service(events=None, hold=None): + service = _service() + service.is_live_agent = True + captured = {} + + async def realtime(queue, init, user): + captured["started"] = True + if hold is not None: + captured.update(held=dict(hold._held)) + for event in events or []: + yield event + await asyncio.sleep(30) + + service.realtime_graph = realtime + service._captured = captured + return service + + +@pytest.mark.asyncio +async def test_live_session_takes_the_thread_lock_and_a_run_slot(limiter): + service = _live_service(hold=limiter) + ws = _live_ws({"model": "m", "thread_id": "t1"}) + task = asyncio.create_task(realtime_graph_ws(websocket=ws, service=service, user=ALICE)) + await _until(lambda: service._captured.get("started")) + assert service._captured["held"] == {THREAD_KEY: 1, SLOT_KEY: 1} + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert limiter._held == {} + + +@pytest.mark.asyncio +async def test_live_session_on_a_busy_thread_is_refused(limiter): + held = await thread_lock.acquire_thread_lock("t1") + service = _live_service() + ws = _live_ws({"model": "m", "thread_id": "t1"}) + await realtime_graph_ws(websocket=ws, service=service, user=ALICE) + assert "started" not in service._captured + sent = [json.loads(c.args[0]) for c in ws.send_text.call_args_list] + assert sent[0]["code"] == "thread_busy" + ws.close.assert_awaited() + assert limiter._held == {THREAD_KEY: 1} + held.release() + + +@pytest.mark.asyncio +async def test_live_session_over_the_run_cap_is_refused(limiter): + held = await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) + service = _live_service() + ws = _live_ws({"model": "m", "thread_id": "t1"}) + await realtime_graph_ws(websocket=ws, service=service, user=ALICE) + assert "started" not in service._captured + sent = [json.loads(c.args[0]) for c in ws.send_text.call_args_list] + assert sent[0]["code"] == "too_many_runs" + assert limiter._held == {SLOT_KEY: 1} + held.release() + + +@pytest.mark.asyncio +async def test_live_handler_cancel_propagates_and_leaves_no_child_tasks(limiter): + service = _live_service() + ws = _live_ws({"model": "m", "thread_id": "t1"}) + before = asyncio.all_tasks() + task = asyncio.create_task(realtime_graph_ws(websocket=ws, service=service, user=ALICE)) + await _until(lambda: service._captured.get("started")) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.sleep(0.05) + assert asyncio.all_tasks() - before - {task} == set() + assert limiter._held == {} + + +# --------------------------------------------------------------------------- +# POST /v1/graph/stream: interrupt_id mismatch is a 409 before the stream starts +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_stream_interrupt_mismatch_is_a_real_409_and_frees_the_guard(limiter): + service = _service() + service.check_resume_target = AsyncMock(side_effect=HTTPException(409, "wrong interrupt")) + service.stream_graph = MagicMock() + graph_input = GraphInputSchema( + config={"thread_id": "t1"}, resume={"ok": True}, interrupt_id="i-1" + ) + with pytest.raises(HTTPException) as exc: + await stream_graph(graph_input=graph_input, service=service, user=ALICE) + assert exc.value.status_code == 409 + service.stream_graph.assert_not_called() + assert limiter._held == {} + + +@pytest.mark.asyncio +async def test_check_resume_target_answers_409_for_the_wrong_interrupt(monkeypatch): + from .test_resume_binding_and_limits import USER, _paused_at, _resume + from .test_resume_binding_and_limits import _service as real + + _paused_at(monkeypatch, "i-2") + srv = real(state=object()) + with pytest.raises(HTTPException) as exc: + await srv.check_resume_target(_resume(interrupt_id="i-1", config={"thread_id": "t1"}), USER) + assert exc.value.status_code == 409 + assert "does not match the open interrupt 'i-2'" in exc.value.detail + + await srv.check_resume_target(_resume(interrupt_id="i-2", config={"thread_id": "t1"}), USER) + with pytest.raises(HTTPException) as new_thread: # no thread named: nothing to resume + await srv.check_resume_target(_resume(interrupt_id="i-2"), USER) + assert new_thread.value.status_code == 409 + + +@pytest.mark.asyncio +async def test_check_resume_target_reads_nothing_without_an_interrupt_id(): + from .test_resume_binding_and_limits import USER, _resume + from .test_resume_binding_and_limits import _service as real + + srv = real(state=object()) + await srv.check_resume_target(_resume(config={"thread_id": "t1"}), USER) + srv.checkpointer.aget_state.assert_not_awaited() diff --git a/tests/unit_tests/test_run_limits.py b/tests/unit_tests/test_run_limits.py index 3ea43c0b..deb619ce 100644 --- a/tests/unit_tests/test_run_limits.py +++ b/tests/unit_tests/test_run_limits.py @@ -173,6 +173,7 @@ async def test_stream_route_refuses_before_streaming(fresh_slots): service = MagicMock() service.runs_policy.return_value = RunsConfig(max_concurrent_per_user=1) + service.check_resume_target = AsyncMock() service.stream_graph = MagicMock(return_value=_items("a")) response = await stream_graph(graph_input=_input(), service=service, user=ALICE) diff --git a/tests/unit_tests/test_run_trace_close.py b/tests/unit_tests/test_run_trace_close.py new file mode 100644 index 00000000..d4decae4 --- /dev/null +++ b/tests/unit_tests/test_run_trace_close.py @@ -0,0 +1,93 @@ +"""The run-trace store closes every run it opens, and evicts threads consistently (G-134).""" + +# ruff: noqa: S101 + +import asyncio +from unittest.mock import MagicMock + +import pytest +from tenxgraph.core.state import StreamChunk, StreamEvent + +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema +from tenxgraph_api.src.app.utils import telemetry_store +from tenxgraph_api.src.app.utils.telemetry_store import TelemetryStore + +from .test_run_limits import ALICE, _input, _service + + +def test_eviction_keeps_the_dict_and_the_order_in_step(monkeypatch): + monkeypatch.setattr(telemetry_store, "MAX_THREADS", 5) + store = TelemetryStore() + for i in range(12): + store.start_run(f"t{i}", "r", 1.0) + assert len(store._threads) == 5 + assert set(store._threads) == set(store._thread_order) + assert set(store._threads) == {f"t{i}" for i in range(7, 12)} + + +def _hanging_graph(started: asyncio.Event): + graph = MagicMock() + + async def astream(*args, **kwargs): + yield StreamChunk(event=StreamEvent.UPDATES, data={"status": "node_invoked"}) + started.set() + await asyncio.sleep(30) + + async def ainvoke(*args, **kwargs): + started.set() + await asyncio.sleep(30) + + graph.astream = astream + graph.ainvoke = ainvoke + return graph + + +def _traced_service(started): + service = _service(_hanging_graph(started), timeout=None) + service._telemetry = TelemetryStore() + return service + + +def _graph_input(): + return GraphInputSchema(messages=_input().messages, config={"thread_id": "t1"}) + + +@pytest.mark.asyncio +async def test_a_cancelled_stream_closes_its_run(): + started = asyncio.Event() + service = _traced_service(started) + + async def consume(): + async for _ in service.stream_chunks(_graph_input(), ALICE): + pass + + task = asyncio.create_task(consume()) + await asyncio.wait_for(started.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + run = service._telemetry.get_latest_run("t1") + assert run.status == "stopped" + assert run.finished_at is not None + + +@pytest.mark.asyncio +async def test_a_stream_the_client_closes_closes_its_run(): + started = asyncio.Event() + service = _traced_service(started) + stream = service.stream_chunks(_graph_input(), ALICE) + await anext(stream) + await stream.aclose() + assert service._telemetry.get_latest_run("t1").status == "stopped" + + +@pytest.mark.asyncio +async def test_a_cancelled_invoke_closes_its_run(): + started = asyncio.Event() + service = _traced_service(started) + task = asyncio.create_task(service.invoke_graph(_graph_input(), ALICE)) + await asyncio.wait_for(started.wait(), 2) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert service._telemetry.get_latest_run("t1").status == "stopped" diff --git a/tests/unit_tests/test_shutdown_readiness.py b/tests/unit_tests/test_shutdown_readiness.py new file mode 100644 index 00000000..cb586ae8 --- /dev/null +++ b/tests/unit_tests/test_shutdown_readiness.py @@ -0,0 +1,230 @@ +"""G-93 / G-108: shutdown grace periods, draining readiness, boot cleanup bound, final chunk.""" + +# ruff: noqa: S101 + +import asyncio +import json +import re +from types import SimpleNamespace + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from injectq import InjectQ +from injectq.integrations.fastapi import setup_fastapi +from tenxgraph.storage.checkpointer import BaseCheckpointer, InMemoryCheckpointer + +from tenxgraph_api.cli.templates import defaults +from tenxgraph_api.src.app import uvicorn_worker +from tenxgraph_api.src.app.core import shutdown +from tenxgraph_api.src.app.routers.ping import readiness +from tenxgraph_api.src.app.routers.ping.router import router as ping_router + + +# ---------------------------------------------------------------- build templates + + +def test_container_grace_exceeds_gunicorn_graceful_timeout(): + graceful = defaults.GRACEFUL_TIMEOUT_SECONDS + compose = defaults.generate_docker_compose_content("svc", 8000) + match = re.search(r"stop_grace_period: (\d+)s", compose) + assert match + assert int(match.group(1)) > graceful + assert ( + graceful + defaults.K8S_PRESTOP_SLEEP_SECONDS + 30 + ) <= defaults.K8S_TERMINATION_GRACE_SECONDS + + +def test_uvicorn_gives_up_before_gunicorn_kills_the_worker(): + uv = uvicorn_worker.TenxGraphUvicornWorker.CONFIG_KWARGS["timeout_graceful_shutdown"] + assert uv == defaults.UVICORN_GRACEFUL_SHUTDOWN_SECONDS + assert uv + 30 < defaults.GRACEFUL_TIMEOUT_SECONDS # room for the lifespan drain + + +def test_dockerfile_and_compose_use_the_draining_worker(): + worker = "tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker" + assert worker in defaults.generate_docker_compose_content("svc", 8000) + assert worker in defaults.generate_dockerfile_content( + "3.13", 8000, "requirements.txt", has_requirements=False, has_pyproject=True + ) + + +# ---------------------------------------------------------------- readiness + + +def _client(monkeypatch, draining=False): + from tenxgraph_api.src.app.core.config import settings as settings_mod + + monkeypatch.setattr(settings_mod, "get_settings", lambda: SimpleNamespace(REDIS_URL=None)) + container = InjectQ() + container.bind_instance(BaseCheckpointer, InMemoryCheckpointer()) + app = FastAPI() + setup_fastapi(container, app) + app.include_router(ping_router) + app.state.draining = draining + return TestClient(app) + + +@pytest.fixture(autouse=True) +def _fresh(): + readiness.reset_cache() + shutdown.reset() + yield + readiness.reset_cache() + shutdown.reset() + + +def test_draining_app_is_not_ready(monkeypatch): + assert _client(monkeypatch).get("/health/ready").status_code == 200 + resp = _client(monkeypatch, draining=True).get("/health/ready") + assert resp.status_code == 503 + assert resp.json()["status"] == "draining" + + +def test_redis_outage_alone_stays_ready_unless_a_feature_fails_closed(): + checks = {"checkpointer": "ok", "redis": "fail"} + assert readiness.is_ready(checks, redis_is_required=False) + assert not readiness.is_ready(checks, redis_is_required=True) + assert not readiness.is_ready({"checkpointer": "fail", "redis": "ok"}, redis_is_required=False) + assert readiness.redis_required(SimpleNamespace(token_revocation="redis", rate_limit=None)) + assert readiness.redis_required( + SimpleNamespace( + token_revocation=None, rate_limit=SimpleNamespace(backend="redis", fail_open=False) + ) + ) + assert not readiness.redis_required( + SimpleNamespace( + token_revocation="memory", rate_limit=SimpleNamespace(backend="redis", fail_open=True) + ) + ) + + +# ---------------------------------------------------------------- boot cleanup + + +@pytest.mark.asyncio +async def test_boot_cleanup_is_bounded(monkeypatch): + from tenxgraph_api.src.app import main + + monkeypatch.setattr(main, "BOOT_CLEANUP_TIMEOUT_SECONDS", 0.05) + + async def hang(): + await asyncio.sleep(30) + + monkeypatch.setattr(main, "_cleanup_temp_media_cache", hang) + monkeypatch.setattr(main, "attach_all_modules", lambda *a, **k: _none()) + app = FastAPI() + async with asyncio.timeout(5): + async with main.lifespan(app): + assert not getattr(app.state, "draining", False) + assert app.state.draining is True + + +async def _none(): + return None + + +# ---------------------------------------------------------------- final chunk + + +async def _slow_stream(): + yield '{"event":"message"}\n' + await asyncio.sleep(30) + yield "never\n" + + +@pytest.mark.asyncio +async def test_stream_ends_with_an_interrupted_chunk_on_shutdown(): + lines = [] + + async def consume(): + async for line in shutdown.end_on_shutdown(_slow_stream()): + lines.append(line) + + task = asyncio.create_task(consume()) + await asyncio.sleep(0.05) + shutdown.interrupt_streams() + await asyncio.wait_for(task, 2) + assert lines[0] == '{"event":"message"}\n' + final = json.loads(lines[-1]) + assert final["event"] == "error" + assert final["data"]["interrupted"] is True + assert "resume" in final["data"]["reason"] + assert len(lines) == 2 + + +@pytest.mark.asyncio +async def test_stream_closes_the_run_when_interrupted(): + closed = asyncio.Event() + + async def stream(): + try: + yield "a\n" + await asyncio.sleep(30) + finally: + closed.set() + + task = asyncio.create_task(_drain(shutdown.end_on_shutdown(stream()))) + await asyncio.sleep(0.05) + shutdown.interrupt_streams() + await asyncio.wait_for(task, 2) + assert closed.is_set() + + +async def _drain(gen): + return [line async for line in gen] + + +@pytest.mark.asyncio +async def test_normal_stream_is_unchanged(): + async def stream(): + yield "a\n" + yield "b\n" + + assert await _drain(shutdown.end_on_shutdown(stream())) == ["a\n", "b\n"] + + +@pytest.mark.asyncio +async def test_http_stream_graph_ends_with_the_interrupted_chunk(): + from tenxgraph.core.state import StreamChunk, StreamEvent + + from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + + async def chunks(self, graph_input, user, server_config=None): + yield StreamChunk(event=StreamEvent.UPDATES, data={"status": "started"}) + await asyncio.sleep(30) + + service = object.__new__(GraphService) + service.stream_chunks = chunks.__get__(service) + lines = [] + + async def consume(): + async for line in service.stream_graph(None, {}): + lines.append(line) + + task = asyncio.create_task(consume()) + await asyncio.sleep(0.05) + shutdown.interrupt_streams() + await asyncio.wait_for(task, 2) + assert json.loads(lines[-1])["data"]["interrupted"] is True + + +@pytest.mark.asyncio +async def test_worker_server_flags_draining_and_interrupts_streams_before_deadline(monkeypatch): + from uvicorn.config import Config + from uvicorn.server import Server + + monkeypatch.setattr(uvicorn_worker, "STREAM_INTERRUPT_MARGIN_SECONDS", 0.0) + seen = {} + + async def fake_shutdown(self, sockets=None): + seen["draining"] = self.app_state.draining + await asyncio.sleep(0.1) + seen["interrupted"] = shutdown._interrupted + + monkeypatch.setattr(Server, "shutdown", fake_shutdown) + state = SimpleNamespace(draining=False) + server = uvicorn_worker.DrainingServer(Config(app=None, timeout_graceful_shutdown=0.05)) + server.app_state = state + await server.shutdown() + assert seen == {"draining": True, "interrupted": True} diff --git a/tests/unit_tests/test_snowflake_id_generator.py b/tests/unit_tests/test_snowflake_id_generator.py index f5fdbd83..03cd575c 100644 --- a/tests/unit_tests/test_snowflake_id_generator.py +++ b/tests/unit_tests/test_snowflake_id_generator.py @@ -205,3 +205,43 @@ def test_init_with_custom_env_values(self, mock_snowflake_kit): assert call_args[1]["node_id"] == 5 assert call_args[1]["worker_id"] == 10 assert call_args[1]["worker_bits"] == 8 + + +class TestSnowFlakeIdGeneratorConfig: + """G-103: complete config from partial input, and distinct ids per process.""" + + @staticmethod + def _env(**overrides): + keep = {k: v for k, v in os.environ.items() if not k.startswith("SNOWFLAKE_")} + return patch.dict(os.environ, {**keep, **overrides}, clear=True) + + def test_partial_params_build_a_complete_config(self, mock_snowflake_kit): + from tenxgraph_api.src.app.utils.snowflake_id_generator import SnowFlakeIdGenerator + + with self._env(): + SnowFlakeIdGenerator(snowflake_node_id=3, snowflake_worker_id=4) + + kwargs = mock_snowflake_kit["config_class"].call_args.kwargs + assert kwargs["node_id"] == 3 + assert kwargs["worker_id"] == 4 + assert kwargs["epoch"] == 1723323246031 + assert mock_snowflake_kit["generator_class"].call_args.kwargs["config"] is not None + + def test_worker_id_differs_between_processes_when_unset(self, mock_snowflake_kit): + from tenxgraph_api.src.app.utils import snowflake_id_generator as module + + ids = [] + with self._env(): + for pid in (1001, 1002): + with patch.object(module.os, "getpid", return_value=pid): + module.SnowFlakeIdGenerator() + ids.append(mock_snowflake_kit["config_class"].call_args.kwargs["worker_id"]) + assert ids[0] != ids[1] + + def test_explicit_env_ids_are_used_as_given(self, mock_snowflake_kit): + from tenxgraph_api.src.app.utils.snowflake_id_generator import SnowFlakeIdGenerator + + with self._env(SNOWFLAKE_NODE_ID="5", SNOWFLAKE_WORKER_ID="6"): + SnowFlakeIdGenerator() + kwargs = mock_snowflake_kit["config_class"].call_args.kwargs + assert (kwargs["node_id"], kwargs["worker_id"]) == (5, 6) diff --git a/tests/unit_tests/test_stream_protocol.py b/tests/unit_tests/test_stream_protocol.py new file mode 100644 index 00000000..6acbe017 --- /dev/null +++ b/tests/unit_tests/test_stream_protocol.py @@ -0,0 +1,122 @@ +"""HTTP run stream: content type (G-104), keep-alive (G-104), terminal chunk (G-165).""" + +# ruff: noqa: S101 + +import asyncio +import json +from unittest.mock import MagicMock, patch + +import pytest +from tenxgraph.core.state import StreamChunk, StreamEvent + +from tenxgraph_api.src.app.core import shutdown +from tenxgraph_api.src.app.routers.graph.router import stream_graph +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema +from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + + +@pytest.fixture(autouse=True) +def _reset_shutdown(): + shutdown.reset() + yield + shutdown.reset() + + +def _service(chunks) -> GraphService: + async def stream_chunks(self, graph_input, user, server_config=None): + for chunk in chunks: + yield chunk + + service = object.__new__(GraphService) + service.stream_chunks = stream_chunks.__get__(service) + return service + + +async def _lines(service) -> list[dict]: + return [json.loads(line) async for line in service.stream_graph(None, {})] + + +@pytest.mark.asyncio +async def test_stream_ends_with_the_done_chunk(): + last = StreamChunk(event=StreamEvent.MESSAGE, data={}, thread_id="t1", run_id="r1") + out = await _lines(_service([StreamChunk(event=StreamEvent.UPDATES, data={"a": 1}), last])) + assert out[-1]["event"] == "updates" + assert out[-1]["data"] == {"status": "done"} + assert out[-1]["thread_id"] == "t1" + assert len(out) == 3 + + +@pytest.mark.asyncio +async def test_a_stream_that_errored_has_no_done_chunk(): + out = await _lines(_service([StreamChunk(event=StreamEvent.ERROR, data={"reason": "x"})])) + assert [c["event"] for c in out] == ["error"] + + +@pytest.mark.asyncio +async def test_done_chunk_follows_a_pause_update(): + pause = StreamChunk(event=StreamEvent.UPDATES, data={"status": "interrupted"}) + out = await _lines(_service([pause])) + assert out[-1]["data"] == {"status": "done"} + + +@pytest.mark.asyncio +async def test_router_labels_the_stream_ndjson(): + graph_input = GraphInputSchema( + messages=[{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + ) + service = MagicMock() + guard = MagicMock() + with ( + patch( + "tenxgraph_api.src.app.routers.graph.router._http_run_guard", + new=MagicMock(side_effect=lambda *a, **k: _done(guard)), + ), + patch( + "tenxgraph_api.src.app.routers.graph.router.release_when_done", + side_effect=lambda gen, g: gen, + ), + ): + service.check_resume_target = MagicMock(side_effect=lambda *a, **k: _done(None)) + response = await stream_graph(graph_input=graph_input, service=service, user={}) + assert response.media_type == "application/x-ndjson" + + +async def _done(value): + return value + + +@pytest.mark.asyncio +async def test_idle_stream_gets_keep_alive_lines(): + async def stream(): + yield "a\n" + await asyncio.sleep(0.35) + yield "b\n" + + lines = [line async for line in shutdown.end_on_shutdown(stream(), keepalive_seconds=0.1)] + assert lines[0] == "a\n" + assert lines[-1] == "b\n" + assert shutdown.KEEPALIVE_LINE in lines[1:-1] + assert len([line for line in lines if line == shutdown.KEEPALIVE_LINE]) >= 2 + + +@pytest.mark.asyncio +async def test_keep_alive_does_not_lose_chunks_or_delay_shutdown(): + async def stream(): + yield "a\n" + await asyncio.sleep(30) + + lines: list[str] = [] + + async def consume(): + async for line in shutdown.end_on_shutdown(stream(), keepalive_seconds=0.05): + lines.append(line) + + task = asyncio.create_task(consume()) + await asyncio.sleep(0.2) + shutdown.interrupt_streams() + await asyncio.wait_for(task, 2) + assert json.loads(lines[-1])["data"]["interrupted"] is True + + +def test_keep_alive_line_is_a_blank_line(): + assert shutdown.KEEPALIVE_LINE == "\n" diff --git a/tests/unit_tests/test_stream_thread_naming.py b/tests/unit_tests/test_stream_thread_naming.py new file mode 100644 index 00000000..44749e0f --- /dev/null +++ b/tests/unit_tests/test_stream_thread_naming.py @@ -0,0 +1,109 @@ +"""Thread naming on the stream path stays off the critical path (G-102).""" + +# ruff: noqa: S101 + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest +from injectq import InjectQ +from tenxgraph.core.state import Message, StreamChunk, StreamEvent +from tenxgraph.core.state.message_block import TextBlock + +from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.routers.graph import thread_lock +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema +from tenxgraph_api.src.app.routers.graph.services import graph_service + +from .test_run_limits import ALICE, _input, _service + + +@pytest.fixture(autouse=True) +def no_task_manager(monkeypatch): + """No core task manager bound: naming runs on the service's own tasks.""" + container = InjectQ() + monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: container)) + + +@pytest.fixture +def locks(monkeypatch): + limiter = MemoryConcurrencyLimiter() + monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + return limiter + + +def _naming_service(generate_name): + graph = MagicMock() + + async def astream(*args, **kwargs): + yield StreamChunk( + event=StreamEvent.MESSAGE, + message=Message(role="assistant", content=[TextBlock(text="hello")]), + ) + + graph.astream = astream + service = _service(graph, timeout=None) + service.config.thread_name_generator_path = "pkg:namer" + service.checkpointer.aget_thread_owner = AsyncMock(return_value=None) + service.thread_name_generator = MagicMock() + service.thread_name_generator.generate_name = generate_name + service.check_media_urls = MagicMock() + return service + + +async def _drain(service): + tasks = list(service.__dict__.get("_naming_tasks", ())) + if tasks: + await asyncio.gather(*tasks) + + +@pytest.mark.asyncio +async def test_a_slow_namer_does_not_hold_the_stream_or_the_lock(locks): + release_name = asyncio.Event() + + async def slow_name(messages): + await release_name.wait() + return "named" + + service = _naming_service(slow_name) + graph_input = GraphInputSchema(messages=_input().messages) # no thread id: a new thread + chunks = await asyncio.wait_for(_collect(service.stream_chunks(graph_input, ALICE)), timeout=2) + assert [c.event for c in chunks] == [StreamEvent.MESSAGE] + assert not locks._held # the thread lock is already free + release_name.set() + await _drain(service) + saved = [c.args[1] for c in service.checkpointer.aput_thread.await_args_list] + assert any(info.thread_name == "named" for info in saved) + + +@pytest.mark.asyncio +async def test_a_naming_failure_does_not_turn_a_good_run_into_an_error(locks): + service = _naming_service(AsyncMock(side_effect=RuntimeError("llm down"))) + graph_input = GraphInputSchema(messages=_input().messages) + chunks = await _collect(service.stream_chunks(graph_input, ALICE)) + await _drain(service) + assert StreamEvent.ERROR not in [c.event for c in chunks] + + +@pytest.mark.asyncio +async def test_naming_tasks_are_capped(locks, monkeypatch): + monkeypatch.setattr(graph_service, "MAX_PENDING_THREAD_NAMES", 2) + gate = asyncio.Event() + + async def blocked(messages): + await gate.wait() + return "n" + + service = _naming_service(blocked) + for i in range(5): + service._name_thread_in_background({"thread_id": f"t{i}"}, f"t{i}", ["hi"]) + await asyncio.sleep(0) + assert service.thread_name_generator.generate_name is blocked + assert len(service.__dict__["_naming_tasks"]) == 2 + gate.set() + await service.drain_naming_tasks() + assert not service.__dict__["_naming_tasks"] + + +async def _collect(stream): + return [chunk async for chunk in stream] diff --git a/tests/unit_tests/test_thread_lock.py b/tests/unit_tests/test_thread_lock.py index c705a1d2..9e398b79 100644 --- a/tests/unit_tests/test_thread_lock.py +++ b/tests/unit_tests/test_thread_lock.py @@ -46,6 +46,7 @@ def _input(thread_id="t1", **extra): def _request(): request = MagicMock() request.state = SimpleNamespace(request_id="r", timestamp="t") + request.is_disconnected = AsyncMock(return_value=False) return request @@ -53,6 +54,7 @@ def _service(invoke): service = MagicMock() service.runs_policy.return_value = RunsConfig() service.invoke_graph = invoke + service.check_resume_target = AsyncMock() service.stop_graph = AsyncMock(return_value={"ok": True}) return service @@ -514,3 +516,62 @@ async def test_development_still_shows_error_text(monkeypatch): assert SECRET in stream assert SECRET in await _ws_output(service) assert SECRET in await _ag_ui_output(service) + + +# --------------------------------------------------------------------------- +# G-105: Retry-After and a waiter whose client left +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_busy_409_and_run_cap_429_carry_retry_after(fresh_locks, monkeypatch): + held = await thread_lock.acquire_thread_lock("t-retry") + with pytest.raises(HTTPException) as busy: + await thread_lock.acquire_thread_lock("t-retry") + assert busy.value.status_code == 409 + assert busy.value.headers["Retry-After"] + held.release() + + from tenxgraph_api.src.app.routers.graph import run_limits + + limiter = MemoryConcurrencyLimiter() + monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: limiter) + slot = await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) + with pytest.raises(HTTPException) as capped: + await run_limits.acquire_run_slot(ALICE, RunsConfig(max_concurrent_per_user=1)) + assert capped.value.status_code == 429 + assert capped.value.headers["Retry-After"] + slot.release() + + +@pytest.mark.asyncio +async def test_a_waiter_whose_client_left_gives_up_and_takes_nothing(fresh_locks): + held = await thread_lock.acquire_thread_lock("t-gone") + gone = AsyncMock(return_value=True) + with pytest.raises(HTTPException) as exc: + await thread_lock.acquire_thread_lock("t-gone", is_disconnected=gone) + assert exc.value.status_code == thread_lock.CLIENT_CLOSED_REQUEST + held.release() + # The lock is free again: the abandoned waiter did not queue up behind it. + again = await thread_lock.acquire_thread_lock("t-gone") + again.release() + + +@pytest.mark.asyncio +async def test_a_client_that_left_just_before_the_lock_freed_does_not_start_a_run(fresh_locks): + held = await thread_lock.acquire_thread_lock("t-late") + calls = {"n": 0} + + async def disconnected(): + calls["n"] += 1 + return calls["n"] > 1 # present while waiting, gone once the lock is free + + waiter = asyncio.create_task( + thread_lock.acquire_thread_lock("t-late", is_disconnected=disconnected) + ) + await asyncio.sleep(0.1) + held.release() + with pytest.raises(HTTPException): + await waiter + free = await thread_lock.acquire_thread_lock("t-late") # not left held by the waiter + free.release() diff --git a/tests/unit_tests/test_token_revocation.py b/tests/unit_tests/test_token_revocation.py index a3e9052b..edead7dc 100644 --- a/tests/unit_tests/test_token_revocation.py +++ b/tests/unit_tests/test_token_revocation.py @@ -71,8 +71,18 @@ async def set(self, key, value, ex=None): self.data[key] = value self.ttl[key] = ex - async def mget(self, keys): - return [self.data.get(k) for k in keys] + async def get(self, key): + return self.data.get(key) + + async def eval(self, script, numkeys, key, cutoff, ttl): + """Python stand-in for the revoke_user script (keeps the newest cutoff).""" + new = float(cutoff) if cutoff else time.time() + current = self.data.get(key) + if current is not None and float(current) >= new: + return 0 + self.data[key] = repr(new).encode() + self.ttl[key] = int(ttl) or None + return 1 async def aclose(self): self.closed = True diff --git a/tests/unit_tests/test_websocket_graph.py b/tests/unit_tests/test_websocket_graph.py index a643a8b8..2a1d7a63 100644 --- a/tests/unit_tests/test_websocket_graph.py +++ b/tests/unit_tests/test_websocket_graph.py @@ -78,7 +78,23 @@ def _side_effect(graph_input, user): "config": {"thread_id": "thread-abc"}, } -_DONE_JSON = StreamChunk(event=StreamEvent.UPDATES, data={"status": "done"}).model_dump_json() + +class _Done(str): + """The ``done`` chunk, equal to any serialisation of it whatever its timestamp.""" + + def __eq__(self, other): + try: + chunk = json.loads(other) + except (TypeError, ValueError): + return False + return chunk.get("event") == "updates" and chunk.get("data") == {"status": "done"} + + __hash__ = str.__hash__ + + +_DONE_JSON = _Done( + StreamChunk(event=StreamEvent.UPDATES, data={"status": "done"}).model_dump_json() +) # ───────────────────────────────────────────────────────────────────────────── @@ -358,3 +374,76 @@ async def test_default_cap_is_max_request_size(self): mod = importlib.import_module("tenxgraph_api.src.app.routers.graph.router") assert mod._ws_max_frame_chars() == get_settings().MAX_REQUEST_SIZE + + +@pytest.mark.asyncio +class TestIdleAndInvalidFrames: + """G-101: an idle socket does not outlive its token; junk frames are not free.""" + + async def test_idle_socket_is_closed_when_its_token_expires(self, monkeypatch): + import asyncio + import importlib + import time + + router_module = importlib.import_module("tenxgraph_api.src.app.routers.graph.router") + + monkeypatch.setattr(router_module, "ws_identity_still_valid", lambda ws, user: False) + ws = _make_websocket([]) + + async def never(*_a, **_k): + await asyncio.sleep(30) + + ws.receive_text = AsyncMock(side_effect=never) + service = _make_service([[]]) + user = {"user_id": "u1", "exp": time.time() + 0.2} + + await asyncio.wait_for(websocket_graph(websocket=ws, service=service, user=user), 5) + + ws.close.assert_awaited() + assert ws.close.call_args.kwargs["code"] == 1008 + assert "expired" in ws.send_text.call_args_list[-1].args[0] + + async def test_a_flood_of_invalid_frames_closes_the_socket(self): + from tenxgraph_api.src.app.routers.graph.router import WS_MAX_INVALID_FRAMES + + ws = _make_websocket([]) + ws.receive_text = AsyncMock(side_effect=["{bad"] * (WS_MAX_INVALID_FRAMES + 5)) + service = _make_service([[]]) + + await websocket_graph(websocket=ws, service=service, user={}) + + ws.close.assert_awaited() + assert ws.close.call_args.kwargs["code"] == 1008 + assert ws.receive_text.await_count == WS_MAX_INVALID_FRAMES + 1 + + async def test_invalid_frames_count_against_the_rate_limit(self, monkeypatch): + import importlib + + router_module = importlib.import_module("tenxgraph_api.src.app.routers.graph.router") + + monkeypatch.setattr(router_module, "ws_run_allowed", AsyncMock(return_value=7)) + ws = _make_websocket([]) + ws.receive_text = AsyncMock(side_effect=["{bad", "{bad", "{bad"]) + service = _make_service([[]]) + + await websocket_graph(websocket=ws, service=service, user={}) + + assert ws.close.call_args.kwargs["code"] == 1013 + assert ws.receive_text.await_count == 1 + + async def test_valid_frames_reset_the_invalid_count(self): + from tenxgraph_api.src.app.routers.graph.router import WS_MAX_INVALID_FRAMES + + frames = ( + ["{bad"] * (WS_MAX_INVALID_FRAMES - 1) + + [json.dumps(_FRESH_REQUEST)] + + ["{bad"] * (WS_MAX_INVALID_FRAMES - 1) + + [WebSocketDisconnect()] + ) + ws = _make_websocket([]) + ws.receive_text = AsyncMock(side_effect=frames) + service = _make_service([[]]) + + await websocket_graph(websocket=ws, service=service, user={}) + + assert len(service._stream_calls) == 1 diff --git a/tests/unit_tests/test_websocket_realtime.py b/tests/unit_tests/test_websocket_realtime.py index dc8ae498..75e3a3b0 100644 --- a/tests/unit_tests/test_websocket_realtime.py +++ b/tests/unit_tests/test_websocket_realtime.py @@ -13,6 +13,7 @@ import pytest from fastapi import WebSocketDisconnect +from tenxgraph_api.src.app.core.config.graph_config import RunsConfig from tenxgraph_api.src.app.routers.graph.router import realtime_graph_ws @@ -54,6 +55,7 @@ def _make_websocket(receive_side_effects: list, init: dict): def _make_service(events: list): service = MagicMock() service.is_live_agent = True # live graph: allowed on /v1/graph/live + service.runs_policy.return_value = RunsConfig() captured = {} async def _gen(input_queue, init, user): @@ -157,6 +159,7 @@ async def test_upstream_text_control_frames_dispatch(self): async def test_downstream_error_closes_with_1011(self): ws = _make_websocket([WebSocketDisconnect()], init={"model": "gemini-2.5-flash-live"}) service = MagicMock() + service.runs_policy.return_value = RunsConfig() async def _boom(input_queue, init, user): raise RuntimeError("graph has no LiveAgent") @@ -230,6 +233,7 @@ async def _slow_gen(input_queue, init, user): yield e service = MagicMock() + service.runs_policy.return_value = RunsConfig() service.realtime_graph = _slow_gen ws = _make_websocket( @@ -254,6 +258,7 @@ async def _boom(input_queue, init, user): yield # make it an async generator service = MagicMock() + service.runs_policy.return_value = RunsConfig() service.realtime_graph = _boom ws = _make_websocket([WebSocketDisconnect()], init={"model": "gemini-x"}) From 105b544bde4423f1c2cceb5b07ad615303c8ffb2 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 21:03:56 +0600 Subject: [PATCH 06/16] Add unit tests for media ownership, production template validators, resource ID rules, and shutdown wiring - Implement tests to ensure that user B cannot access user A's media files through various routes and methods. - Create a validator test to confirm that ordinary text passes through the production template validator without issues. - Add tests for client-named IDs to validate proper formatting and error handling for thread, message, and memory IDs. - Introduce tests to verify that the shutdown process correctly calls the graph's close method and handles errors gracefully. --- .gitignore | 3 + CHANGELOG.md | 278 ++++++++- README.md | 4 +- pyproject.toml | 9 +- tenxgraph_api/cli/commands/__init__.py | 4 +- tenxgraph_api/cli/commands/build.py | 1 + tenxgraph_api/cli/templates/defaults.py | 7 +- .../cli/templates/prod/graph/agent.py | 3 +- .../prod/graph/validators/validators.py | 7 +- tenxgraph_api/src/app/core/auth/jwt_auth.py | 151 ++++- .../src/app/core/auth/ownership_resolver.py | 13 +- .../src/app/core/auth/permissions.py | 26 +- .../src/app/core/auth/request_config.py | 102 +++- tenxgraph_api/src/app/core/auth/revocation.py | 10 +- tenxgraph_api/src/app/core/concurrency.py | 10 +- .../src/app/core/config/graph_config.py | 15 +- .../src/app/core/config/media_settings.py | 24 + tenxgraph_api/src/app/core/config/settings.py | 48 +- .../src/app/core/exceptions/handle_errors.py | 63 ++- tenxgraph_api/src/app/core/redis_client.py | 17 + tenxgraph_api/src/app/loader.py | 41 ++ tenxgraph_api/src/app/main.py | 141 +++-- tenxgraph_api/src/app/routers/ag_ui/router.py | 59 +- .../src/app/routers/checkpointer/router.py | 56 +- .../schemas/checkpointer_schemas.py | 9 +- .../services/checkpointer_service.py | 45 +- .../src/app/routers/graph/idempotency.py | 275 +++++++++ .../src/app/routers/graph/realtime_guard.py | 5 +- tenxgraph_api/src/app/routers/graph/router.py | 185 +++++- .../src/app/routers/graph/run_limits.py | 3 +- .../routers/graph/schemas/graph_schemas.py | 38 +- .../routers/graph/services/graph_service.py | 36 +- .../graph/services/multimodal_preprocessor.py | 39 +- .../src/app/routers/graph/thread_lock.py | 32 +- .../src/app/routers/media/__init__.py | 303 +++++++++- tenxgraph_api/src/app/routers/media/router.py | 39 +- .../src/app/routers/ping/readiness.py | 17 + tenxgraph_api/src/app/routers/store/router.py | 4 +- .../routers/store/schemas/store_schemas.py | 29 +- .../routers/store/services/store_service.py | 13 +- .../src/app/utils/media/extractor.py | 62 +- tenxgraph_api/src/app/uvicorn_worker.py | 4 +- tests/cli/test_group6_small_cli_bugs.py | 37 ++ tests/integration_tests/conftest.py | 3 + .../integration_tests/test_idempotency_key.py | 210 +++++++ .../test_server_thread_ids.py | 73 ++- .../test_upload_auth_order.py | 41 ++ .../unit_tests/auth/test_graph_config_auth.py | 8 +- tests/unit_tests/auth/test_jwt_hardening.py | 44 ++ tests/unit_tests/auth/test_jwt_rotation.py | 115 ++++ tests/unit_tests/store/test_store_service.py | 4 +- tests/unit_tests/test_closing_and_cancel.py | 213 +++++++ tests/unit_tests/test_extraction_limits.py | 7 +- tests/unit_tests/test_g127_leftovers.py | 78 +++ tests/unit_tests/test_group3a_storage.py | 308 ++++++++++ tests/unit_tests/test_group6_nesting_depth.py | 65 +++ tests/unit_tests/test_handle_errors.py | 62 ++ tests/unit_tests/test_loader.py | 56 ++ tests/unit_tests/test_media_g123_g125.py | 152 +++++ tests/unit_tests/test_media_group3b.py | 365 ++++++++++++ tests/unit_tests/test_media_lifecycle.py | 35 +- tests/unit_tests/test_nc23_media_ownership.py | 535 ++++++++++++++++++ .../test_ownership_authorization.py | 15 +- .../test_prod_template_validators.py | 33 ++ tests/unit_tests/test_readiness.py | 31 + tests/unit_tests/test_resource_id_rule.py | 114 ++++ tests/unit_tests/test_shutdown_wiring.py | 92 +++ tests/unit_tests/test_static_pause_resume.py | 13 + tests/unit_tests/test_token_revocation.py | 6 +- tests/unit_tests/test_websocket_realtime.py | 142 +++++ uv.lock | 120 ++-- 71 files changed, 4884 insertions(+), 323 deletions(-) create mode 100644 tenxgraph_api/src/app/routers/graph/idempotency.py create mode 100644 tests/integration_tests/test_idempotency_key.py create mode 100644 tests/integration_tests/test_upload_auth_order.py create mode 100644 tests/unit_tests/auth/test_jwt_rotation.py create mode 100644 tests/unit_tests/test_closing_and_cancel.py create mode 100644 tests/unit_tests/test_g127_leftovers.py create mode 100644 tests/unit_tests/test_group3a_storage.py create mode 100644 tests/unit_tests/test_group6_nesting_depth.py create mode 100644 tests/unit_tests/test_media_g123_g125.py create mode 100644 tests/unit_tests/test_media_group3b.py create mode 100644 tests/unit_tests/test_nc23_media_ownership.py create mode 100644 tests/unit_tests/test_prod_template_validators.py create mode 100644 tests/unit_tests/test_resource_id_rule.py create mode 100644 tests/unit_tests/test_shutdown_wiring.py diff --git a/.gitignore b/.gitignore index da959a24..6313b1c0 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,6 @@ cython_debug/ # uv lockfile: not tracked (this is a library; CI resolves fresh) uv.lock + +# Media uploads written by local runs and tests +uploads/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a2d63377..028a5d7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,12 +27,163 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- -## [Unreleased] +## [1.0.0] - Unreleased -### Removed +The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is aligned with +`10xgraph-client` 1.0. It is the `1.0.0rc1` code plus the changes below. If you upgrade from +0.9.x, the `1.0.0rc1` breaking list further down applies to you as well. + +### Breaking + +- **Ids in paths and bodies must match `^[A-Za-z0-9._:@-]{1,128}$`.** A `thread_id`, + `message_id` or memory id that is longer or has other characters gets 422 + (`VALIDATION_ERROR`). Snowflake integers, UUIDs and the ids the stores return all fit. Ids + that start with `__system__` stay reserved. +- **Nested `initial_state`, `config` and `resume` values deeper than 64 levels get 422** + (`VALIDATION_ERROR`). Before, about 5,000 levels passed validation and failed inside the run + with `RecursionError`. +- **Resume errors answer 409 or 422.** A `resume_interrupt_id` or `interrupt_id` that is not the + pending interrupt's, a `resume` sent to a thread that is not paused, and a plain new message + sent to a thread paused at `interrupt()` return 409. A resume value that fails the + interrupt's `response_schema` returns 422. The 409 now comes before the stream starts on + `POST /v1/graph/stream` (it was a 200 with an error chunk). +- **Writes during a run return 409.** `PUT` and `DELETE /v1/threads/{thread_id}/state`, + `POST /v1/threads/{thread_id}/messages`, `DELETE` of a thread message, `DELETE` of a thread + and `POST /v1/graph/fix` are refused at once with 409 while a run holds the thread. Retry + after the run ends or stop it first. They used to be overwritten by the run, or brought a + deleted thread back. +- **Thread delete can return `success=false`.** When file cleanup, trace cleanup or cache + eviction failed, `DELETE /v1/threads/{thread_id}` answers `success=false` with + `data.cleanup_failed` listing the steps (`files`, `traces`, `cache`) and logs a warning. The + thread rows are deleted. Treat `success=false` as "retry the cleanup", not as "thread still + there". +- **`JWT_ALGORITHM` must be on the allowlist.** `HS256`, `HS384`, `HS512`, `RS256`, `RS384`, + `RS512`, `ES256`, `ES384`, `ES512`, `PS256`, `PS384`, `PS512` and `EdDSA`. `none`, an empty + value or any other value stops startup. Startup also stops when `"auth": "jwt"` is set and + PyJWT is missing; before, every request failed with a 500. +- **The `auth` value must be exactly `"jwt"`.** Before, any string that contained `jwt` was + accepted. A value such as `"jwt-custom"` now fails with `Unsupported auth method`. +- **The `graph:setup` scope is gone.** The setup endpoint no longer exists, and a rule that + names the scope guards nothing. Remove it from your authorization config. +- **Graph validation errors use the code `VALIDATION_ERROR`.** The duplicate + `AGENTFLOW_VALIDATION_ERROR` is no longer sent. The client and docs already branch on + `VALIDATION_ERROR`. +- **A run that hits its recursion limit answers 422 (`GRAPH_RECURSION_LIMIT`), not 500.** +- **`POST /v1/graph/stream` is labelled `application/x-ndjson`** (it was `text/event-stream` + while sending NDJSON). Idle streams send a blank keep-alive line every 15 s; NDJSON readers + skip it. Proxy rules or clients that matched on `text/event-stream` need updating. +- **`10xgraph build` refuses `--docker-compose` together with `--k8s`.** The combined image had + no start command. Run it once for each target. +- **Uploads are checked against new default limits** (see Added): images over 50,000,000 + pixels and PDFs over 10 MB or 500 pages are refused with 413. +- **The server needs the new dependency floors** (see Upgrade notes) and `10xgraph` 1.0. +- **AG-UI and `/v1/graph/live` follow `runs.thread_ids` like REST.** Under the default + `"server"`, a thread id the server never issued gets 403 on `POST /v1/ag-ui`, and on live an + error event with code `not_authorized` and close code 1008. Before, both accepted any id. A + blank or `"new"` id, or no id, makes the server create the thread: AG-UI returns the new id in + `RUN_STARTED`, and live sends `{"type": "session_started", "thread_id": ""}` as its first + server event. Frontends that make their own thread ids (CopilotKit does by default) must set + `"runs": {"thread_ids": "client"}` in `10xgraph.json`. + +### Upgrade notes + +- **Postgres:** `10xgraph` 1.0 runs schema migration v4 on first use. Stop all old pods, run + the migration (start one new pod and wait for it to be ready), then start the rest. A + rollback needs a database restore. `/health/ready` now fails while the schema version is + below the one the code expects, so a pod that cannot create or upgrade the schema stays out + of rotation. See the `10xgraph` changelog for the full note. +- **Dependency minimums:** `python-multipart>=0.0.31`, `starlette>=1.3.1`, `PyJWT>=2.15.0` + (the `jwt` extra) and `opentelemetry-api` and `opentelemetry-sdk` `>=1.45.1` (the `otel` + extra). `10xgraph>=1.0.0rc1,<2.0` is the core floor in `pyproject.toml`. +- **Shutdown timing:** the generated Gunicorn command uses a new worker class, + `tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker`. Regenerate Docker and + Kubernetes files with `10xgraph build`, or point your own command at it. Keep Gunicorn + `--graceful-timeout` above the sum of the three `SHUTDOWN_*_SECONDS` settings (50 s by + default) and the pod grace period above that. +- **Alerting:** handled client errors log one WARNING line, not an ERROR with a traceback. + Alerts that matched ERROR for 4xx responses must match WARNING. + +### Added + +- **Shutdown settings.** `SHUTDOWN_GRACE_SECONDS` (default 10), `SHUTDOWN_STOP_SECONDS` + (default 10) and `SHUTDOWN_TIMEOUT_SECONDS` (default 30). The app lifespan calls + `graph.aclose(grace, stop_timeout, timeout)` once with these values, then closes the API's + own clients. Runs that are cut are saved INTERRUPTED and the thread resumes with the next + request. +- **`503` while closing.** A run refused because the graph is closing (`GraphClosedError`) + answers 503 `SERVICE_UNAVAILABLE` with `Retry-After`. `/health/ready` answers 503 + (`status: draining`) once shutdown starts. +- A run refused while the graph is closing sends a stream error chunk or WebSocket frame with + `data.code = "SERVICE_UNAVAILABLE"` and `retry_after_seconds`. On a WebSocket the socket then + closes with code 1012. +- **Final chunk on shutdown.** An open HTTP stream cut by a restart ends with an error chunk + that has `data.interrupted: true` and a reason, so the client knows to resume the thread. +- **Terminal chunk on HTTP streams.** A run stream ends with + `{"event": "updates", "data": {"status": "done"}}`, as the WebSocket path already did. A + client can tell a finished stream from a cut one. +- **`Retry-After` on 429 and 409.** The run-cap 429 and the busy-thread 409 now carry it, and + a queued request whose client has left no longer runs. +- **Typed server errors.** An unhandled exception returns the JSON error envelope with the + request id, the security headers and `Cache-Control: no-store`, not bare text. A database or + Redis driver error that cannot connect answers 503 with `Retry-After`, and a model + provider's rate limit answers 429 `PROVIDER_RATE_LIMITED` with `Retry-After`. In production a + plain `ValueError` still answers 422 with "Invalid input provided."; only errors written for + the client (`ClientInputError`) keep their message. +- **Redis client settings.** `REDIS_SOCKET_TIMEOUT` (5 s), `REDIS_CONNECT_TIMEOUT` (5 s), + `REDIS_POOL_TIMEOUT` (5 s), `REDIS_MAX_CONNECTIONS` (50) and `REDIS_HEALTH_CHECK_INTERVAL` + (30 s) apply to every Redis client the API opens (run locks and caps, rate limit, token + revocation, ownership cache, readiness). `0` turns a timeout off. A Redis that stops + answering no longer hangs requests. +- **`REDIS_KEY_PREFIX`.** A prefix for every key the API writes to Redis, so deployments that + share one Redis do not collide. Give the core checkpointer the same value as + `redis_key_prefix`. +- **Media limits.** `MEDIA_MAX_FILES_PER_USER` and `MEDIA_MAX_USER_STORAGE_MB` (both `0`, off), + `MEDIA_MAX_IMAGE_PIXELS` (50,000,000), `MEDIA_MAX_PDF_MB` (10) and `MEDIA_MAX_PDF_PAGES` + (500; checked before parsing, 413 before the file is stored), `MEDIA_INLINE_MAX_BYTES` + (50,000; larger inline base64 media is moved to the media store, `0` turns it off) and + `MEDIA_SWEEP_INTERVAL_SECONDS` (3600, `0` turns it off; removes orphan data files left by a + crash). The sweeper does not run on the cloud store, which cannot list blobs. The per-user + quotas do work on S3 and GCS (see the owner index under Changed). +- `GET /v1/files` lists the caller's files (`limit` up to 200, `offset`, `has_more`). It works + on cloud stores too, for files stored with `10xgraph` 1.0 or later. +- `has_more`, `offset` and `limit` on the thread and thread-message lists, and `offset`, + `has_more` and `next_offset` on the memory list. +- **Run trace.** The observability route returns a span tree (run, node visits, LLM and tool + calls) with inputs, outputs, status, tokens and timings, plus `enabled` and `reason` so a UI + can tell tracing off from no runs yet. It is still dev only; the store is not created with + `MODE=production`. +- `realtime_options` in the graph details: the Live init options `/v1/graph/live` honours. +- **`10xgraph revoke`** revokes one token (`--jti`) or all of a user's tokens (`--user`) in the + Redis the server reads (`"token_revocation": "redis"`), with no restart. `10xgraph init + --template production` now sets `token_revocation` to `redis`. +- `10xgraph build` and `10xgraph config` warn when auth is off or the production setup is + incomplete (see Changed). +- `TenxGraphUvicornWorker`, a Gunicorn worker class that ends open streams with the final + chunk before the shutdown deadline. +- The production template validators allow ordinary text (see Fixed) and the template enables + the rate limit per user when auth is on. +- **Idempotency keys (opt-in).** Send `Idempotency-Key` (1 to 255 printable ASCII characters) on + `POST /v1/graph/invoke`, `POST /v1/graph/stream` and the AG-UI route, or `idempotency_key` in + the WebSocket run frame, and a gateway retry no longer runs the tools twice. Keys are scoped + to the user, and a record lives `IDEMPOTENCY_TTL_SECONDS` (default 86400) in Redis when + configured, else in the process. + - The same key while the first run is going: 409 with `Retry-After`. + - After the run finished: invoke returns the stored result, stream replays the final state + line plus `done`, and AG-UI and WebSocket answer 409 (their event output is not stored). + - The same key with a different body: 422. A malformed key: 422. + - A failed run stores nothing, so the key can be retried. A result over 1 MiB is not stored, + and the key then answers 409. + - A request without the header behaves as before. +- **JWT key rotation (opt-in).** `JWT_SECRET_KEYS` is a JSON list of `{"kid": ..., "key": ...}` + objects or plain strings, checked beside `JWT_SECRET_KEY`, which works unchanged. A token whose + header has a `kid` is checked against the keys with that `kid`; a token without one is tried + against every key. +- **`JWT_MAX_TOKEN_AGE_SECONDS` (opt-in).** A token older than this by its `iat` claim gets 401 + `EXPIRED_TOKEN`, and a token with no `iat` is refused. Off by default. +- **`session_started` event on `/v1/graph/live`.** The first server event of every live session is + `{"type": "session_started", "thread_id": ""}`, with the server-made id when the init frame + had none. Reconnect with that id to continue the thread. -- Removed the `/v1/evals/runs` endpoints; eval reports are JSON files opened in the playground. - `10xgraph eval` now writes only the JSON report and no longer has `--open`. ### Changed @@ -60,6 +211,121 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). offer "Unlimited"; `rate_limit.exclude_paths` is described as extra paths (`/ping` and `/health/ready` are always skipped) and is no longer seeded; `observability.logfire.console: true` is now an error, because Logfire accepts only `false` or a `ConsoleOptions` object. +- **Concurrency leases and revocation use the Redis clock.** Lease and revocation timing used + each pod's own clock, so skew over 60 s could trim another pod's live lease. A cancelled + waiter no longer leaves a ghost lease, a Redis error no longer lets a second run start on the + thread, and shutdown releases leases before it closes the limiter's Redis client. +- **Thread-lock responses.** A busy thread answers 409 with `Retry-After`. `/v1/graph/live` and + `/v1/graph/ws` now take the thread lock and a run slot like the other run routes, release the + lock only after the run generator closes, and stop their child tasks when the handler is + cancelled. +- **WebSocket.** `/v1/graph/ws` checks the token of an idle socket, rate limits invalid frames, + and applies a send timeout to slow consumers. +- **Thread naming.** Thread naming no longer runs inline while a stream holds the + lock and run slot, and a failed naming call no longer turns a good run into an error chunk. +- **Readiness** checks the database schema version, and a Redis blip no longer takes a pod out + of rotation by itself. +- **Boot.** The lifespan calls `asetup()` on the checkpointer at startup and retries on first + use if it fails, so a fresh Postgres no longer answers the first request with a 500. +- **Realtime sessions.** `/v1/graph/live` and `/v1/graph/ws` give each connection its own + `LiveAgent`, so concurrent users no longer share one provider session. +- **Logging.** The API no longer drops core `tenxgraph.*` log records. The log sanitizer + redacts `sk-` and `AIza` secrets as well as Bearer and JWT values, keeps UUID ids readable, + redacts keys by whole name, and replaces CR and LF. Validation errors on WebSocket and AG-UI + no longer echo pydantic `input_value` text, and the access log does not print `?token=`. +- **`ALLOWED_HOST` is stripped** before the middleware compares it, `IS_DEBUG=true` is reported + as ignored in production, and `10xgraph dev --host 0.0.0.0` and `--reload` in production log + a warning. +- **Token revocation.** A `revoke_user` cutoff is never replaced by an older one, entries + expire (30 days by default), and a token without `iat` is dated from its `exp` when you give + the backend your `token_lifetime`. Revocation reads one key at a time, so Redis Cluster no + longer fails with CROSSSLOT. +- **Snowflake ids.** The generator reads its settings from the environment when you pass none, + so workers no longer all share one node and worker id, and a partial configuration no longer + raises on the first id. +- **Kubernetes manifest.** `ORIGINS` and `ALLOWED_HOST` entries no longer override the values in + the `envFrom` Secret. +- **Rate limit.** `exclude_paths` in the production template names `/redocs`, the real path. +- **`10xgraph audit`** has a seventh check, "Security posture", that warns about the same + production-readiness gaps as `10xgraph build` (auth, authorization, rate limit, token + revocation). It only warns. +- **Media delete.** Deleting a file or a thread removes the extracted text from the 24 h cache, + and thread delete releases media linked through `PUT .../state`. +- **Uploads.** A file's thread links are updated on every route that accepts media, and the + startup temp-cache cleanup now runs against the media store. +- **Request size caps.** Client dicts and lists (`config`, `initial_state`, `resume`, + `metadata`, `filters` and the store and checkpointer request bodies, and WebSocket frames) are + limited to 1000 keys per object, 10000 items per list and 200000 values in all, beside the + 64-level depth cap. A larger body gets 422 (`VALIDATION_ERROR`). The caps are fixed and have + no setting. +- **The id rule covers more inputs.** The WebSocket `config.thread_id`, the thread id in the + live init frame and the AG-UI `threadId` must match the id pattern from Breaking. An invalid + one gets 422 or an error frame with `VALIDATION_ERROR`. +- **Uploads are authenticated before the body is parsed,** so an unauthenticated caller can no + longer make the server spool a multipart body. An upload with no `file` part answers 422. +- **Cloud media (S3, GCS).** `GET /v1/files`, the per-user quotas and release on thread delete + now work through the per-owner index of `10xgraph` 1.0. Files stored before 1.0, or with no + owner, are not in the index. Listing with auth off still answers 501 on a cloud store. +- **A stored thread at a missing node answers 409** (`GRAPH_NODE_MISSING_001`), and stored state + from a newer `10xgraph` answers 500 with `STATE_FORMAT_001` and a message to upgrade. Before, + both were a generic 500. + +### Removed + +- Removed the `/v1/evals/runs` endpoints; eval reports are JSON files opened in the playground. + `10xgraph eval` now writes only the JSON report and no longer has `--open`. +- The `graph:setup` scope is removed from the config editor and the docs (see Breaking). + +### Fixed + +- A thread that was cancelled, disconnected or cut by a restart is usable on the next request + (the core fix), and a client disconnect during a stream no longer leaves the thread lock held. +- `POST /v1/threads/{thread_id}/messages` and the other write routes no longer race an active + run (see Breaking). +- Resuming a thread paused at `interrupt()` works on Postgres over HTTP and WebSocket. +- `DELETE` of a message removes it from the thread state and cache, not only from the message + list, and removes media only when state no longer references it. +- Mem0 memory search: client `filters` cannot replace the owner `user_id`. The keys `user_id`, + `agent_id`, `app_id` and `run_id` are dropped from client filters. +- Memory list paging: a user with more than 1,000 memories can page through them. +- `10xgraph eval --threshold` exits 1 only when the pass rate is below the threshold, not below + 100 percent. The production template's own gate (0.75) passes. +- `10xgraph init` escapes the agent name before it writes `pyproject.toml` and `.env.example`, + `--python-version` accepts only supported versions, template substitution errors are no + longer swallowed, `10xgraph skills --force` copies before it deletes, and `10xgraph eval` + loads `confeval.py` only from the project. +- The `checkpointer` key in `10xgraph.json` was ignored without a word. The server now warns + at load when it is set, and the skill templates no longer tell you to set it. +- The in-memory checkpointer fallback and per-process locks now log a startup warning in + production or with more than one worker. +- The run-trace store closes a cancelled run, so the playground no longer shows a disconnected + run as still running, and its memory use is bounded. +- The first REST invoke example in the docs now sends a content block and no client-invented + `thread_id`. + +### Security + +- `POST /v1/threads/{id}/messages` and `PUT /v1/threads/{id}/state` now check `file_id` media + blocks for ownership the same way invoke does; provider file ids are refused there as on invoke. +- Ids in requests are bounded (see Breaking), which keeps oversized ids out of log lines, Redis + keys and database keys. +- Redis keys for ownership, run and WebSocket slots, the thread lock and revocation are built + from percent-encoded parts, so an id with a colon cannot collide with another key. +- Redis timeouts and a bounded pool stop a blackholed Redis from hanging every request that + reaches revocation, the thread lock or the limiter. +- JWT startup checks: the algorithm allowlist, PyJWT required at boot, and an exact `auth` + parse (see Breaking). The `user_id` and scope rules from `1.0.0rc1` still apply. +- Rotating the JWT secret no longer has to invalidate every token (`JWT_SECRET_KEYS`), and a + maximum token age can be enforced (`JWT_MAX_TOKEN_AGE_SECONDS`). Both are opt-in. +- Upload bodies are parsed after authentication, and client dicts and lists are size-capped + (see Changed), so an unauthenticated or oversized request costs less memory. +- Secrets and request text no longer reach logs or clients through validation error echoes, + the access log or unsanitized log lines. +- Extracted document text is removed with the file, and uploads are bounded by image pixel, + PDF size and page limits. +- The production template defaults to JWT, ownership authorization, Redis rate limiting and + `by: user` rate limiting when auth is on, and `10xgraph build` and `10xgraph config` warn + about a missing auth or authorization setup. ## [1.0.0rc1] - 2026-10-28 @@ -74,8 +340,8 @@ This is the 1.0 release candidate. It requires `10xgraph` 1.0 (`10xgraph>=1.0.0r `/v1/graph/ws` and the checkpointer write routes (`PUT /v1/threads/{id}/state`, message writes). Migrate by omitting the id on the first request and using the `meta.thread_id` the server returns from then on (`"new"` over `/v1/graph/ws`), or set - `"runs": {"thread_ids": "client"}` in `10xgraph.json`. AG-UI and `/v1/graph/live` need no change. Only the built-in ownership and - rbac backends refuse; `allow_all` and custom backends do not unless they read + `"runs": {"thread_ids": "client"}` in `10xgraph.json`. AG-UI and `/v1/graph/live` follow the same setting (see the next item). Only the built-in + ownership and rbac backends refuse; `allow_all` and custom backends do not unless they read `server_thread_ids` themselves. - **Production deployments with the default or wildcard `ALLOWED_HOST` or `ORIGINS` refuse to start.** Set explicit values, or `STRICT_PRODUCTION_CHECKS=false`. A public, token-less API diff --git a/README.md b/README.md index 66b3cfdd..d0153526 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Python](https://img.shields.io/pypi/pyversions/10xgraph-api)](https://pypi.org/project/10xgraph-api/) [![License](https://img.shields.io/github/license/10xGraph/10xgraph-api)](https://github.com/10xGraph/10xgraph-api/blob/main/LICENSE) -`10xgraph-api` generates the production server around a [10xGraph](https://github.com/10xGraph/10xGraph) agent. Point `10xgraph.json` at your compiled graph and it serves REST, SSE streaming, WebSocket and realtime-audio endpoints with JWT or custom auth, scoped authorization on every endpoint, owner-only threads and rate limiting, then writes your Docker Compose and Kubernetes files. MIT licensed and self-hosted. +`10xgraph-api` generates the production server around a [10xGraph](https://github.com/10xGraph/10xGraph) agent. Point `10xgraph.json` at your compiled graph and it serves REST, NDJSON streaming, WebSocket and realtime-audio endpoints with JWT or custom auth, scoped authorization on every endpoint, owner-only threads and rate limiting, then writes your Docker Compose and Kubernetes files. MIT licensed and self-hosted. Other frameworks give you the graph. 10xGraph gives you the graph and the server around it, in the same open-source project. @@ -16,7 +16,7 @@ Other frameworks give you the graph. 10xGraph gives you the graph and the server ## What the server gives you -- **Endpoints generated from the graph.** Invoke, SSE stream, stop, WebSocket and realtime audio; threads, state and messages; the memory store; file uploads for multimodal input; and an optional [AG-UI](https://10xgraph.com/docs/server/ag-ui) endpoint for CopilotKit and other AG-UI clients. +- **Endpoints generated from the graph.** Invoke, NDJSON stream, stop, WebSocket and realtime audio; threads, state and messages; the memory store; file uploads for multimodal input; and an optional [AG-UI](https://10xgraph.com/docs/server/ag-ui) endpoint for CopilotKit and other AG-UI clients. - **Authentication.** `"auth": "jwt"`, or your own `BaseAuth` subclass. - **Scoped authorization on every endpoint.** Routes require `resource:action` scopes such as `graph:invoke`, `graph:stream` or `checkpointer:read`, checked by a pluggable `AuthorizationBackend`. The server refuses to start if any non-public route has no guard. - **Owner-only threads.** In production mode a thread is readable and writable only by the user who created it, on every step that touches it. Ownership checks are cached in-process and optionally in Redis, so they do not hit the database on each request. diff --git a/pyproject.toml b/pyproject.toml index 50a3aa99..e1e3934a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,8 @@ dependencies = [ # Pre-1.0 and major-version-risky deps are capped; floors are the versions # the suite is verified against. "fastapi>=0.116,<1.0", - "python-multipart>=0.0.19,<0.1", + "python-multipart>=0.0.31,<0.1", + "starlette>=1.3.1", "pydantic>=2.13,<3", "pydantic-settings>=2.3,<3", "uvicorn>=0.36,<1.0", @@ -96,7 +97,7 @@ redis = [ "redis>=5.0.7,<7", ] jwt = [ - "PyJWT>=2.8,<3", + "PyJWT>=2.15.0,<3", ] media = [ "textxtract[pdf,docx,html,xml,md]>=0.2.0", @@ -110,8 +111,8 @@ gcloud = [ "google-cloud-logging", ] otel = [ - "opentelemetry-api>=1.20.0", - "opentelemetry-sdk>=1.20.0", + "opentelemetry-api>=1.45.1", + "opentelemetry-sdk>=1.45.1", "opentelemetry-instrumentation-fastapi>=0.40b0", "opentelemetry-exporter-otlp>=1.20.0", ] diff --git a/tenxgraph_api/cli/commands/__init__.py b/tenxgraph_api/cli/commands/__init__.py index fdcaf03e..86ea0282 100644 --- a/tenxgraph_api/cli/commands/__init__.py +++ b/tenxgraph_api/cli/commands/__init__.py @@ -36,7 +36,9 @@ def handle_error(self, error: Exception) -> int: Returns: Appropriate exit code """ - self.logger.error("Command failed: %s", error) + # The user sees the message once, through the output formatter below. The log line + # carries the traceback (and the chained cause) for --verbose only. + self.logger.debug("Command failed: %s", error, exc_info=error) # Import here to avoid circular imports from tenxgraph_api.cli.exceptions import CLIError diff --git a/tenxgraph_api/cli/commands/build.py b/tenxgraph_api/cli/commands/build.py index 51c9f00a..31e47187 100644 --- a/tenxgraph_api/cli/commands/build.py +++ b/tenxgraph_api/cli/commands/build.py @@ -209,6 +209,7 @@ def execute( # noqa: PLR0912, PLR0915 return self.handle_error(e) except Exception as e: docker_error = DockerError(f"Failed to generate Docker files: {e}") + docker_error.__cause__ = e return self.handle_error(docker_error) def _discover_requirements(self, current_dir: Path) -> tuple[list[Path], str]: diff --git a/tenxgraph_api/cli/templates/defaults.py b/tenxgraph_api/cli/templates/defaults.py index 9caac93e..23a5eb1d 100644 --- a/tenxgraph_api/cli/templates/defaults.py +++ b/tenxgraph_api/cli/templates/defaults.py @@ -20,8 +20,9 @@ WORKER_TIMEOUT_SECONDS: Final[int] = 660 # Uvicorn stops waiting for open responses after this many seconds and cancels them, so the -# lifespan shutdown (30 s drain, graph.aclose) runs before gunicorn's --graceful-timeout -# SIGKILLs the worker. Open NDJSON streams get a final "interrupted" chunk just before it. +# lifespan shutdown (graph.aclose: the three SHUTDOWN_*_SECONDS settings, 50 s by default) runs +# before gunicorn's --graceful-timeout SIGKILLs the worker. Open NDJSON streams get a final +# "interrupted" chunk just before it. # Keep in step with ``uvicorn_worker.DEFAULT_GRACEFUL_SHUTDOWN_SECONDS``. UVICORN_GRACEFUL_SHUTDOWN_SECONDS: Final[int] = GRACEFUL_TIMEOUT_SECONDS - 60 @@ -238,7 +239,7 @@ # Docker templates -# The published name on PyPI. The bare name "10xgraph-api" belongs to someone else. +# The published name on PyPI (pyproject.toml `name`). CLI_PACKAGE = "10xgraph-api" diff --git a/tenxgraph_api/cli/templates/prod/graph/agent.py b/tenxgraph_api/cli/templates/prod/graph/agent.py index 5b20be2e..049ac671 100644 --- a/tenxgraph_api/cli/templates/prod/graph/agent.py +++ b/tenxgraph_api/cli/templates/prod/graph/agent.py @@ -15,7 +15,8 @@ load_dotenv() # Durable thread state: Postgres keeps every thread, Redis caches the hot ones. -# Set DATABASE_URL and REDIS_URL in .env (see .env.example). +# Set DATABASE_URL and REDIS_URL in .env (see .env.example). The tables are created (or +# upgraded) when the server starts, so a fresh database needs no setup step. checkpointer = PgCheckpointer( postgres_dsn=os.getenv("DATABASE_URL"), redis_url=os.getenv("REDIS_URL"), diff --git a/tenxgraph_api/cli/templates/prod/graph/validators/validators.py b/tenxgraph_api/cli/templates/prod/graph/validators/validators.py index 952cf0db..38eecbf4 100644 --- a/tenxgraph_api/cli/templates/prod/graph/validators/validators.py +++ b/tenxgraph_api/cli/templates/prod/graph/validators/validators.py @@ -3,19 +3,14 @@ prompt_validator = PromptInjectionValidator( strict_mode=True, - max_length=1000, + max_length=10000, blocked_patterns=[], suspicious_keywords=[ "ignore previous", "forget previous", "disregard previous", - "bypass", "circumvent", - "override", - "disable", "remove restrictions", - "token", "coupon", - "free", ], ) diff --git a/tenxgraph_api/src/app/core/auth/jwt_auth.py b/tenxgraph_api/src/app/core/auth/jwt_auth.py index 7a9d2661..6a2a6869 100644 --- a/tenxgraph_api/src/app/core/auth/jwt_auth.py +++ b/tenxgraph_api/src/app/core/auth/jwt_auth.py @@ -1,3 +1,4 @@ +import time from typing import Any from fastapi import Request, Response @@ -19,19 +20,58 @@ # RFC 7518 3.2: an HMAC key must be at least as long as the hash output (256 bits for HS256). MIN_HMAC_BYTES = 32 +# Signature algorithms JWT_ALGORITHM may name. ``none`` and anything not listed is refused. +ALLOWED_JWT_ALGORITHMS: frozenset[str] = frozenset( + { + "HS256", + "HS384", + "HS512", + "RS256", + "RS384", + "RS512", + "ES256", + "ES384", + "ES512", + "PS256", + "PS384", + "PS512", + "EdDSA", + } +) + + +def _require_pyjwt() -> None: + if jwt is None: + raise ImportError( + "PyJWT is required for JWT authentication. " + 'Install with `pip install "10xgraph-api[jwt]"`' + ) + def check_jwt_settings(settings: Settings) -> None: - """Refuse a guessable HMAC secret at startup (production), warn about it otherwise. + """Check the JWT settings at startup. + + Refuses a missing PyJWT and an algorithm outside ``ALLOWED_JWT_ALGORITHMS`` (``none`` + included), always. Refuses a guessable HMAC secret in production, warns about it otherwise. A short HS* secret can be brute-forced offline from any one token, after which anyone can mint a token for any ``user_id``. """ - algorithm = (settings.JWT_ALGORITHM or "").upper() - secret = settings.JWT_SECRET_KEY or "" - if not algorithm.startswith("HS") or len(secret.encode()) >= MIN_HMAC_BYTES: + _require_pyjwt() + configured = settings.JWT_ALGORITHM or "" + if configured not in ALLOWED_JWT_ALGORITHMS: + raise ValueError( + f"JWT_ALGORITHM {configured!r} is not allowed. Use one of: " + f"{', '.join(sorted(ALLOWED_JWT_ALGORITHMS))} (HS256 is the default)." + ) + algorithm = configured.upper() + if not algorithm.startswith("HS"): + return + secrets = [key for _, key in verification_keys(settings)] + if all(len(secret.encode()) >= MIN_HMAC_BYTES for secret in secrets): return message = ( - f"JWT_SECRET_KEY is shorter than {MIN_HMAC_BYTES} bytes, which is too weak for " + f"A JWT secret key is shorter than {MIN_HMAC_BYTES} bytes, which is too weak for " f"{algorithm}. Generate one with: " "python -c 'import secrets; print(secrets.token_urlsafe(48))'" ) @@ -40,6 +80,66 @@ def check_jwt_settings(settings: Settings) -> None: logger.warning(message) +def verification_keys(settings: Settings) -> list[tuple[str | None, str]]: + """``(kid, key)`` for every configured verification key: ``JWT_SECRET_KEY`` first (no kid), + then each entry of ``JWT_SECRET_KEYS``.""" + keys: list[tuple[str | None, str]] = [] + if settings.JWT_SECRET_KEY: + keys.append((None, settings.JWT_SECRET_KEY)) + keys.extend((entry.kid, entry.key) for entry in settings.JWT_SECRET_KEYS or []) + return keys + + +def _candidate_keys(token: str, keys: list[tuple[str | None, str]]) -> list[str]: + """The keys to try for ``token``: those matching its header ``kid``, else the kid-less ones. + + A header ``kid`` that names a configured key selects only the keys with that ``kid``. A + token without ``kid`` (or with one nobody configured) is tried against every key that has + no ``kid`` of its own, then the rest, so ``JWT_SECRET_KEY`` keeps verifying as before. + """ + try: + header = jwt.get_unverified_header(token) + except jwt.InvalidTokenError: + return [key for _, key in keys] + kid = header.get("kid") + if isinstance(kid, str) and kid: + matched = [key for key_id, key in keys if key_id == kid] + if matched: + return matched + return [key for key_id, key in keys if key_id is None] + return [key for _, key in keys] + + +def _decode_with_keys( + token: str, + keys: list[tuple[str | None, str]], + algorithm: str, + settings: Settings, + required: list[str], +) -> dict[str, Any]: + """Verify ``token`` against the candidate keys; the first key that verifies wins.""" + failure: Exception | None = None + for candidate in _candidate_keys(token, keys): + try: + return jwt.decode( + token, + candidate, + algorithms=[algorithm], + issuer=settings.JWT_ISSUER or None, + audience=settings.JWT_AUDIENCE or None, + options={"require": required}, + ) + except jwt.InvalidSignatureError as err: + failure = failure or err # another key may verify it + except jwt.ExpiredSignatureError: + raise _unauthorized("Token has expired, please login again", "EXPIRED_TOKEN") + except jwt.InvalidTokenError as err: + failure = err + break + logger.warning("JWT rejected: %s", type(failure).__name__) + raise _unauthorized("Invalid token, please login again", "INVALID_TOKEN") + + def _unauthorized(message: str, error_code: str) -> UserAccountError: return UserAccountError(message=message, error_code=error_code, status_code=401) @@ -91,18 +191,21 @@ def authenticate( raise _unauthorized("Invalid token, please login again", "REVOKED_TOKEN") settings = get_settings() - jwt_secret_key = settings.JWT_SECRET_KEY + keys = verification_keys(settings) jwt_algorithm = settings.JWT_ALGORITHM token = credential.credentials - if jwt is None: - raise ImportError( - "PyJWT is required for JWT authentication. " - 'Install with `pip install "10xgraph-api[jwt]"`' + _require_pyjwt() + + if not keys or jwt_algorithm is None: + raise UserAccountError( + message="JWT settings are not configured", + error_code="JWT_SETTINGS_NOT_CONFIGURED", + status_code=500, ) - if jwt_secret_key is None or jwt_algorithm is None: + if jwt_algorithm not in ALLOWED_JWT_ALGORITHMS: raise UserAccountError( message="JWT settings are not configured", error_code="JWT_SETTINGS_NOT_CONFIGURED", @@ -110,24 +213,22 @@ def authenticate( ) required = ["exp"] + if settings.JWT_MAX_TOKEN_AGE_SECONDS: + required.append("iat") if settings.JWT_ISSUER: required.append("iss") if settings.JWT_AUDIENCE: required.append("aud") - try: - decoded_token = jwt.decode( - token, - jwt_secret_key, - algorithms=[jwt_algorithm], - issuer=settings.JWT_ISSUER or None, - audience=settings.JWT_AUDIENCE or None, - options={"require": required}, - ) - except jwt.ExpiredSignatureError: - raise _unauthorized("Token has expired, please login again", "EXPIRED_TOKEN") - except jwt.InvalidTokenError as err: - logger.warning("JWT rejected: %s", type(err).__name__) - raise _unauthorized("Invalid token, please login again", "INVALID_TOKEN") + decoded_token = _decode_with_keys(token, keys, jwt_algorithm, settings, required) + max_age = settings.JWT_MAX_TOKEN_AGE_SECONDS + if max_age: + issued_at = decoded_token.get("iat") + if ( + isinstance(issued_at, bool) + or not isinstance(issued_at, int | float) + or time.time() - issued_at > max_age + ): + raise _unauthorized("Token is too old, please login again", "EXPIRED_TOKEN") response.headers["WWW-Authenticate"] = 'Bearer realm="auth_required"' diff --git a/tenxgraph_api/src/app/core/auth/ownership_resolver.py b/tenxgraph_api/src/app/core/auth/ownership_resolver.py index f1180ea6..9e94e5cd 100644 --- a/tenxgraph_api/src/app/core/auth/ownership_resolver.py +++ b/tenxgraph_api/src/app/core/auth/ownership_resolver.py @@ -33,6 +33,8 @@ from collections import OrderedDict from collections.abc import Awaitable, Callable +from tenxgraph_api.src.app.core.redis_client import key_part, prefixed + logger = logging.getLogger("tenxgraph_api.authorization") @@ -56,7 +58,8 @@ class ThreadOwnershipResolver: When absent the resolver is L1-only. Redis errors degrade to the DB lookup and never fail a request. max_size: L1 LRU capacity (entries). - redis_prefix: key namespace for L2 entries. + redis_prefix: key namespace for L2 entries (default ``af:authz:owner`` behind + ``REDIS_KEY_PREFIX``). l1_ttl: seconds an L1 entry is trusted; bounds staleness after a delete elsewhere. l2_ttl: seconds an L2 entry lives in Redis. """ @@ -67,14 +70,14 @@ def __init__( *, redis: object | None = None, max_size: int = 10_000, - redis_prefix: str = "af:authz:owner", + redis_prefix: str | None = None, l1_ttl: float = DEFAULT_L1_TTL_SECONDS, l2_ttl: int = DEFAULT_L2_TTL_SECONDS, ) -> None: self._lookup = lookup self._redis = redis self._max_size = max(1, max_size) - self._prefix = redis_prefix + self._prefix = redis_prefix if redis_prefix is not None else prefixed("af:authz:owner") self._l1_ttl = l1_ttl self._l2_ttl = l2_ttl # key -> (owner, monotonic expiry) @@ -178,7 +181,7 @@ async def _l1_put(self, key: str, owner: str) -> None: # Owner entries and delete markers live under different fixed segments, so no thread id # (not even one like "deleted:X") can make an owner key collide with a marker key. def _redis_key(self, key: str) -> str: - return f"{self._prefix}:id:{key}" + return f"{self._prefix}:id:{key_part(key)}" async def _redis_get(self, key: str) -> str | None: if self._redis is None: @@ -204,7 +207,7 @@ async def _redis_set(self, key: str, owner: str) -> None: logger.debug("Ownership L2 set failed for %s: %s", key, exc) def _tombstone_key(self, key: str) -> str: - return f"{self._prefix}:tomb:{key}" + return f"{self._prefix}:tomb:{key_part(key)}" async def _redis_tombstone(self, key: str, owner: str) -> None: if self._redis is None: diff --git a/tenxgraph_api/src/app/core/auth/permissions.py b/tenxgraph_api/src/app/core/auth/permissions.py index b7542536..0fca0ce5 100644 --- a/tenxgraph_api/src/app/core/auth/permissions.py +++ b/tenxgraph_api/src/app/core/auth/permissions.py @@ -22,7 +22,11 @@ scopes_from_claims, ) from tenxgraph_api.src.app.core.auth.identity_cache import recall_identity -from tenxgraph_api.src.app.core.auth.request_config import normalize_thread_id +from tenxgraph_api.src.app.core.auth.request_config import ( + ClientInputError, + normalize_thread_id, + resource_id_error, +) from tenxgraph_api.src.app.core.auth.reserved_identity import ( RESERVED_USER_ID_MESSAGE, identity_is_reserved, @@ -329,6 +333,7 @@ async def __call__( # noqa: PLR0912 resource_id = self._extract_resource_id_from_path(connection) if resource_id is None: resource_id = await self._extract_resource_id_from_body(connection) + self._check_resource_ids(connection, resource_id) # Step 4: Scope check -- does this identity carry the permission for this # endpoint at all? The required scope is ":". Scopes come from @@ -387,6 +392,25 @@ async def __call__( # noqa: PLR0912 return user + @staticmethod + def _check_resource_ids(connection: HTTPConnection, resource_id: str | None) -> None: + """Refuse an id that breaks the shared resource-id rule (422 ``VALIDATION_ERROR``). + + Covers the thread id from the path or body and the message and memory ids in the path. + """ + problem = None + params = connection.path_params + for name in ("thread_id", "message_id", "memory_id"): + if name in params: + problem = resource_id_error(params[name], name) or problem + if problem is None: + problem = resource_id_error(resource_id, "thread_id") + if problem is None: + return + if isinstance(connection, WebSocket): + _reject(connection, 422, problem) + raise ClientInputError(problem) + def _extract_resource_id_from_path(self, connection: HTTPConnection) -> str | None: """ Extract resource ID from request path parameters. diff --git a/tenxgraph_api/src/app/core/auth/request_config.py b/tenxgraph_api/src/app/core/auth/request_config.py index 9f5cfa66..2a385585 100644 --- a/tenxgraph_api/src/app/core/auth/request_config.py +++ b/tenxgraph_api/src/app/core/auth/request_config.py @@ -53,7 +53,10 @@ from __future__ import annotations -from typing import Any +import re +from typing import Annotated, Any + +from pydantic import BeforeValidator, ValidationInfo from tenxgraph_api.src.app.core import logger @@ -188,6 +191,79 @@ def _field(message: Any, name: str) -> Any: MAX_AG_UI_HISTORY_MESSAGES = 10_000 +# Deepest nesting of dicts and lists accepted in client ``initial_state``, ``config`` and +# ``resume``. Deeper input would fail with RecursionError inside the run (copy, serialise). +MAX_NESTING_DEPTH = 64 + + +def nesting_depth_error(value: Any, field: str, max_depth: int = MAX_NESTING_DEPTH) -> str | None: + """Describe a ``value`` nested deeper than ``max_depth`` dicts and lists, or ``None``. + + Walks with an explicit stack, so a hostile value cannot trigger RecursionError here. + """ + stack: list[tuple[Any, int]] = [(value, 1)] + while stack: + item, depth = stack.pop() + if isinstance(item, dict): + children: Any = item.values() + elif isinstance(item, list | tuple): + children = item + else: + continue + if depth > max_depth: + return f"{field}: nested more than {max_depth} levels deep" + stack.extend((child, depth + 1) for child in children) + return None + + +# Size caps for the open dicts and lists a client sends (``config``, ``initial_state``, +# ``metadata``, ``filters``, ``resume`` ...). Per container, plus a cap on all values together, +# so a wide or bushy body cannot cost more memory than a normal request does. Fixed: no setting. +MAX_CONTAINER_ITEMS = 10_000 +MAX_DICT_KEYS = 1_000 +MAX_TOTAL_NODES = 200_000 + + +def structure_error(value: Any, field: str) -> str | None: + """Describe why a client dict or list is too deep or too large, or ``None`` when it fits. + + One walk with an explicit stack: nesting beyond ``MAX_NESTING_DEPTH``, a dict with more than + ``MAX_DICT_KEYS`` keys, a list with more than ``MAX_CONTAINER_ITEMS`` items, or more than + ``MAX_TOTAL_NODES`` values in all. + """ + if (error := nesting_depth_error(value, field)) is not None: + return error + stack: list[Any] = [value] + total = 0 + while stack: + item = stack.pop() + if isinstance(item, dict): + if len(item) > MAX_DICT_KEYS: + return f"{field}: an object has more than {MAX_DICT_KEYS} keys" + children: Any = item.values() + elif isinstance(item, list | tuple): + if len(item) > MAX_CONTAINER_ITEMS: + return f"{field}: a list has more than {MAX_CONTAINER_ITEMS} items" + children = item + else: + continue + total += len(item) + if total > MAX_TOTAL_NODES: + return f"{field}: more than {MAX_TOTAL_NODES} values in all" + stack.extend(children) + return None + + +def _bounded(value: Any, info: ValidationInfo) -> Any: + if (error := structure_error(value, info.field_name or "value")) is not None: + raise ClientInputError(error) + return value + + +# A client-supplied open dict, checked for depth and size before it is parsed further. +BoundedDict = Annotated[dict[str, Any], BeforeValidator(_bounded)] + + class InterruptConflictError(Exception): """A resume names an interrupt the thread is not paused at (HTTP 409, or an error chunk).""" @@ -334,6 +410,30 @@ def unanswered_tool_calls(context: Any) -> dict[str, str]: return calls +# One rule for every id a client names in a path or body (thread, message, memory). The API's +# own ids (Snowflake integers, UUIDs, memory ids from the stores) all fit; a longer or odder id +# would reach log lines, Redis keys and database keys unbounded. +RESOURCE_ID_PATTERN = re.compile(r"^[A-Za-z0-9._:@-]{1,128}$") + + +def resource_id_error(value: Any, field: str) -> str | None: + """Why ``value`` is not an acceptable resource id, or ``None`` when it is. + + Checked in one place, ``RequirePermission``, for the ``thread_id``, ``message_id`` and + ``memory_id`` path parameters and the ``thread_id`` / ``config.thread_id`` body fields. + """ + if value is None or isinstance(value, bool): + return None + if not isinstance(value, str | int): + return f"{field} must be a string or integer" + if not RESOURCE_ID_PATTERN.fullmatch(str(value)): + return ( + f"{field} must be 1 to 128 characters of letters, digits and . _ : @ - " + "(no spaces or slashes)" + ) + return None + + def normalize_thread_id(value: Any) -> str | None: """The one canonical form of a thread id, shared by the permission check and the services. diff --git a/tenxgraph_api/src/app/core/auth/revocation.py b/tenxgraph_api/src/app/core/auth/revocation.py index df6d1ace..548323b5 100644 --- a/tenxgraph_api/src/app/core/auth/revocation.py +++ b/tenxgraph_api/src/app/core/auth/revocation.py @@ -37,6 +37,8 @@ from abc import ABC, abstractmethod from typing import Any +from tenxgraph_api.src.app.core.redis_client import key_part, prefixed + logger = logging.getLogger("tenxgraph_api.revocation") @@ -170,7 +172,7 @@ def __init__( url: str | None = None, *, client: Any | None = None, - prefix: str = "10xgraph:revoked", + prefix: str | None = None, token_lifetime: float | None = None, user_ttl: int | None = DEFAULT_USER_REVOCATION_TTL, ) -> None: @@ -189,13 +191,13 @@ def __init__( 'pip install "10xgraph-api[redis]"' ) from exc self._client = client - self._prefix = prefix + self._prefix = prefix if prefix is not None else prefixed("10xgraph:revoked") def _jti_key(self, jti: Any) -> str: - return f"{self._prefix}:jti:{jti}" + return f"{self._prefix}:jti:{key_part(jti)}" def _user_key(self, user_id: Any) -> str: - return f"{self._prefix}:user:{user_id}" + return f"{self._prefix}:user:{key_part(user_id)}" async def revoke_token(self, jti: str, expires_at: float | None = None) -> None: ttl = None diff --git a/tenxgraph_api/src/app/core/concurrency.py b/tenxgraph_api/src/app/core/concurrency.py index d8d9daac..bf8e9966 100644 --- a/tenxgraph_api/src/app/core/concurrency.py +++ b/tenxgraph_api/src/app/core/concurrency.py @@ -31,6 +31,7 @@ from abc import ABC, abstractmethod from typing import Any +from tenxgraph_api.src.app.core.redis_client import prefixed from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -120,16 +121,21 @@ def __init__( self, client: Any, *, - prefix: str = "10xgraph:concurrency", + prefix: str | None = None, lease_ttl: float = 60.0, ) -> None: self._client = client - self._prefix = prefix + self._prefix = prefix if prefix is not None else prefixed("10xgraph:concurrency") self._lease_ttl = lease_ttl self._refreshers: set[asyncio.Task[None]] = set() # Counts used while Redis is unreachable, so one process still honours the cap. self._local = MemoryConcurrencyLimiter() + @property + def client(self) -> Any: + """The Redis client, so other shared state (idempotency records) uses the same one.""" + return self._client + @classmethod def from_url(cls, url: str, **kwargs: Any) -> RedisConcurrencyLimiter: from tenxgraph_api.src.app.core.redis_client import create_redis diff --git a/tenxgraph_api/src/app/core/config/graph_config.py b/tenxgraph_api/src/app/core/config/graph_config.py index d1df4a97..bb00c8df 100644 --- a/tenxgraph_api/src/app/core/config/graph_config.py +++ b/tenxgraph_api/src/app/core/config/graph_config.py @@ -439,9 +439,10 @@ class RunsConfig: thread under any id it picks; under owner-only authorization the first writer of an id then owns it, so a user who claims a guessable id (``"default"``) first locks its intended owner out, and a request on an unknown id answers differently from one on someone else's. AG-UI - (``POST /v1/ag-ui``) and realtime (``/v1/graph/live``) always accept an id the frontend - made, even with ``"server"``, because CopilotKit and the realtime client create their own; - the caller owns a new one. + (``POST /v1/ag-ui``) and realtime (``/v1/graph/live``) follow the same rule as REST: with + ``"server"`` an id the server has never seen is refused (403). An AG-UI client that has no + thread yet sends ``threadId`` blank or ``"new"`` and reads the server's id from + ``RUN_STARTED``; a frontend that makes its own ids needs ``"client"``. ``config_keys`` and ``state_keys`` are allow-lists for client input. When set, a request whose ``config`` or ``initial_state`` carries any other key is rejected with 422. @@ -705,13 +706,15 @@ def _parse_auth_config(res: Any) -> dict | None: if not res: return None - if isinstance(res, str) and "jwt" in res: + if isinstance(res, str) and res == "jwt": # Now check jwt secret and algorithm available in env secret = os.environ.get("JWT_SECRET_KEY", None) algorithm = os.environ.get("JWT_ALGORITHM", None) - if not secret or not algorithm: + rotation_keys = os.environ.get("JWT_SECRET_KEYS", None) + if not (secret or rotation_keys) or not algorithm: raise ValueError( - "JWT_SECRET_KEY and JWT_ALGORITHM must be set in environment variables", + "JWT_SECRET_KEY (or JWT_SECRET_KEYS) and JWT_ALGORITHM must be set in " + "environment variables", ) return { "method": "jwt", diff --git a/tenxgraph_api/src/app/core/config/media_settings.py b/tenxgraph_api/src/app/core/config/media_settings.py index fdccd00c..9f92f20d 100644 --- a/tenxgraph_api/src/app/core/config/media_settings.py +++ b/tenxgraph_api/src/app/core/config/media_settings.py @@ -27,6 +27,30 @@ class MediaSettings(BaseSettings): MEDIA_MAX_SIZE_MB: float = 25.0 DOCUMENT_HANDLING: str = "extract_text" # extract_text | pass_raw | skip + # Per-user upload quotas, checked on upload. 0 means no limit. Counting reads the + # metadata of every stored file, so it needs a store that can list its files (memory and + # local); on one that cannot (cloud) the quota is skipped with a warning. + MEDIA_MAX_FILES_PER_USER: int = 0 + MEDIA_MAX_USER_STORAGE_MB: float = 0.0 + + # Refuse an image with more pixels than this before it is stored (a small file can + # decode to a huge bitmap). 0 means no limit. Needs Pillow; without it the check is skipped. + MEDIA_MAX_IMAGE_PIXELS: int = 50_000_000 + + # PDFs are refused (413) before text extraction when larger than this many MB or with more + # than this many pages. 0 means no limit. Only applies while DOCUMENT_HANDLING=extract_text. + MEDIA_MAX_PDF_MB: float = 10.0 + MEDIA_MAX_PDF_PAGES: int = 500 + + # Inline base64 media in a run request larger than this many bytes is moved into the media + # store and replaced by a reference, so it is not copied into state at every step. + # 0 turns the offload off. + MEDIA_INLINE_MAX_BYTES: int = 50_000 + + # Seconds between sweeps that delete files a crash left behind in the media store + # (local store only). 0 turns the sweeper off. + MEDIA_SWEEP_INTERVAL_SECONDS: float = 3600.0 + # Comma-separated content-type allowlist for uploads. Empty (the default) means # accept any type -- the developer's call. Set e.g. # ``MEDIA_ALLOWED_CONTENT_TYPES=image/*,application/pdf`` to restrict. Entries may diff --git a/tenxgraph_api/src/app/core/config/settings.py b/tenxgraph_api/src/app/core/config/settings.py index 994521eb..6f3f5670 100644 --- a/tenxgraph_api/src/app/core/config/settings.py +++ b/tenxgraph_api/src/app/core/config/settings.py @@ -2,7 +2,7 @@ import os from functools import lru_cache -from pydantic import ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator from pydantic_settings import BaseSettings @@ -47,6 +47,18 @@ def _csv_tokens(value: str) -> list[str]: return [token.strip() for token in value.split(",") if token.strip()] +class JwtVerificationKey(BaseModel): + """One JWT verification key, optionally named by the ``kid`` its tokens carry.""" + + kid: str | None = None + key: str = Field(min_length=1) + + @model_validator(mode="before") + @classmethod + def _from_plain_string(cls, value): + return {"key": value} if isinstance(value, str) else value + + class Settings(BaseSettings): """ This class defines the configuration settings for the application. @@ -145,6 +157,24 @@ class Settings(BaseSettings): REDIS_MAX_CONNECTIONS: int = Field(default=50, ge=0) REDIS_POOL_TIMEOUT: float = Field(default=5.0, ge=0.0) REDIS_HEALTH_CHECK_INTERVAL: float = Field(default=30.0, ge=0.0) + # Put in front of every key the API writes to Redis (run locks and caps, ownership cache, + # token revocation), so two deployments that share one Redis cannot read each other's + # entries. Empty means no prefix. Give the core checkpointer the same value through its + # ``redis_key_prefix`` option so its cache keys are separated too. + REDIS_KEY_PREFIX: str = "" + + ################################# + ###### Shutdown ################# + ################################# + # Passed to ``graph.aclose()`` when the app stops (after uvicorn has finished or cut open + # responses). GRACE: seconds runs still going get to finish on their own. STOP: seconds + # after the stop flag, while each run saves its thread INTERRUPTED. TIMEOUT: seconds for the + # hard cancel and the background-task drain. Their sum must stay under the margin between + # uvicorn's graceful shutdown and gunicorn's --graceful-timeout (60 s in the generated + # image), or the worker is killed mid-close. + SHUTDOWN_GRACE_SECONDS: float = Field(default=10.0, ge=0.0) + SHUTDOWN_STOP_SECONDS: float = Field(default=10.0, ge=0.0) + SHUTDOWN_TIMEOUT_SECONDS: float = Field(default=30.0, ge=0.0) ################################# ###### sentry Config ############ @@ -174,6 +204,22 @@ class Settings(BaseSettings): # another service that shares the signing key is not accepted here. JWT_ISSUER: str | None = None JWT_AUDIENCE: str | None = None + # Extra verification keys for key rotation (JSON in the environment), e.g. + # ``[{"kid": "2026-10", "key": "..."}, {"key": "old-secret-without-kid"}]``. They are tried + # beside ``JWT_SECRET_KEY``, which keeps working unchanged. A token whose header names a + # ``kid`` is checked against the keys with that ``kid`` (then the keys without one); a token + # without ``kid`` is tried against every key. Off by default. + JWT_SECRET_KEYS: list[JwtVerificationKey] | None = None + # When set, a token older than this many seconds (by its ``iat`` claim) is refused, and a + # token without ``iat`` is refused too. Off by default. + JWT_MAX_TOKEN_AGE_SECONDS: int | None = Field(default=None, gt=0) + + ################################# + ###### Idempotency ############## + ################################# + # How long a finished run is remembered under its ``Idempotency-Key``. The header is opt-in: + # a request without it behaves as before. + IDEMPOTENCY_TTL_SECONDS: int = Field(default=86400, gt=0) ################################# ###### OTEL Config ############## diff --git a/tenxgraph_api/src/app/core/exceptions/handle_errors.py b/tenxgraph_api/src/app/core/exceptions/handle_errors.py index 03ed860c..5b62b74f 100644 --- a/tenxgraph_api/src/app/core/exceptions/handle_errors.py +++ b/tenxgraph_api/src/app/core/exceptions/handle_errors.py @@ -4,13 +4,16 @@ from starlette.exceptions import HTTPException from starlette.requests import Request from tenxgraph.core.exceptions import ( + GraphClosedError, GraphError, GraphRecursionError, MetricsError, NodeError, SchemaVersionError, SerializationError, + StateFormatVersionError, StorageError, + StoredNodeMissingError, TransientStorageError, ) from tenxgraph.utils.validators import ValidationError @@ -53,7 +56,16 @@ def _sanitize_error_message(message: str, error_code: str, is_production: bool) generic_messages = { "VALIDATION_ERROR": "The request data is invalid. Please check your input.", "HTTPException": "An error occurred processing your request.", + # Before "GRAPH_": the first matching prefix wins. + "GRAPH_NODE_MISSING_": ( + "This thread was saved at a step that no longer exists in the graph. " + "Start a new thread, or ask the operator to map the old step." + ), "GRAPH_": "An error occurred executing the graph.", + "STATE_": ( + "This thread was saved by a newer server version. " + "The server must be upgraded to read it." + ), "NODE_": "An error occurred in a graph node.", "STORAGE_": "An error occurred accessing storage.", "METRICS_": "An error occurred collecting metrics.", @@ -197,7 +209,8 @@ async def value_exception_handler(request: Request, exc: ValueError): request_id = getattr(request.state, "request_id", "unknown") message = str(exc) - if is_production: + # A ClientInputError is written for the client, so its message survives production. + if is_production and not isinstance(exc, ClientInputError): logger.warning("Request %s - ValueError details: %s", request_id, log_safe(message)) message = "Invalid input provided." @@ -262,6 +275,54 @@ async def graph_validation_exception_handler(request: Request, exc: ValidationEr status_code=422, ) + @app.exception_handler(GraphClosedError) + async def graph_closed_exception_handler(request: Request, exc: GraphClosedError): + # The server is shutting down: this replica takes no new runs, another one will. + _log_handled(request, exc, 503, "SERVICE_UNAVAILABLE", "Graph closed") + response = error_response( + request, + error_code="SERVICE_UNAVAILABLE", + message="The server is shutting down. Retry the request.", + status_code=503, + ) + response.headers["Retry-After"] = str(RETRY_AFTER_SECONDS) + return response + + @app.exception_handler(StoredNodeMissingError) + async def stored_node_missing_exception_handler(request: Request, exc: StoredNodeMissingError): + # The thread's stored position no longer exists in this graph: the client can start a + # new thread, so it is a 409 and not a 500. + error_code = getattr(exc, "error_code", "GRAPH_NODE_MISSING_001") + _log_handled(request, exc, 409, error_code, "StoredNodeMissingError") + message = _sanitize_error_message( + getattr(exc, "message", str(exc)), error_code, is_production + ) + return error_response( + request, + error_code=error_code, + message=message, + details=_context_details(exc) if not is_production else None, + status_code=409, + ) + + @app.exception_handler(StateFormatVersionError) + async def state_format_version_exception_handler( + request: Request, exc: StateFormatVersionError + ): + # Stored state is newer than this server reads (a rollback): an operator problem. + error_code = getattr(exc, "error_code", "STATE_FORMAT_001") + logger.error("StateFormatVersionError: url: %s", request.base_url, exc_info=exc) + message = _sanitize_error_message( + getattr(exc, "message", str(exc)), error_code, is_production + ) + return error_response( + request, + error_code=error_code, + message=message, + details=_context_details(exc) if not is_production else None, + status_code=500, + ) + @app.exception_handler(GraphError) async def graph_error_exception_handler(request: Request, exc: GraphError): logger.error(f"GraphError: url: {request.base_url}", exc_info=exc) diff --git a/tenxgraph_api/src/app/core/redis_client.py b/tenxgraph_api/src/app/core/redis_client.py index 7a562e15..1a3e2c3e 100644 --- a/tenxgraph_api/src/app/core/redis_client.py +++ b/tenxgraph_api/src/app/core/redis_client.py @@ -11,6 +11,23 @@ from typing import Any +def key_part(value: Any) -> str: + """An id made safe to sit between ``:`` separators in a Redis key. + + ``:`` and ``%`` are percent-encoded, so ``a:b`` cannot read as two parts. Ids without + either (UUIDs, snowflake ids, most user ids) are unchanged, so existing keys stay valid. + """ + return str(value).replace("%", "%25").replace(":", "%3A") + + +def prefixed(namespace: str) -> str: + """``namespace`` with the deployment prefix (``REDIS_KEY_PREFIX``) in front, if one is set.""" + from tenxgraph_api.src.app.core.config.settings import get_settings + + prefix = get_settings().REDIS_KEY_PREFIX + return f"{prefix}:{namespace}" if prefix else namespace + + def redis_options() -> dict[str, Any]: """Connection keyword arguments built from the settings (0 turns a limit off).""" from tenxgraph_api.src.app.core.config.settings import get_settings diff --git a/tenxgraph_api/src/app/loader.py b/tenxgraph_api/src/app/loader.py index 93c75971..744a93e1 100644 --- a/tenxgraph_api/src/app/loader.py +++ b/tenxgraph_api/src/app/loader.py @@ -508,6 +508,46 @@ def _bind_shared_concurrency(container: InjectQ, redis_url: str | None) -> None: logger.info("Run and WebSocket connection caps are shared through Redis.") +def _web_concurrency() -> int: + """Worker count from ``WEB_CONCURRENCY`` (Gunicorn's own variable); 1 when unset or bad.""" + try: + return max(int(os.environ.get("WEB_CONCURRENCY", "1")), 1) + except ValueError: + return 1 + + +def warn_per_process_state(container: InjectQ, redis_url: str | None) -> None: + """Warn when threads or caps live in one process and the deployment may run several. + + The in-memory and SQLite checkpointers keep threads in one process (or one file), and + without Redis the run locks, run caps and rate-limit counters are per process too. That is + fine for one process; with more than one worker or replica every process has its own view. + Warns in production or when ``WEB_CONCURRENCY`` is above 1 (the API cannot see replicas, + so production always gets the line). The choice stays the developer's. + """ + workers = _web_concurrency() + if not (_is_production() or workers > 1): + return + checkpointer = container.try_get(BaseCheckpointer) + kind = type(checkpointer).__name__ + if kind in {"InMemoryCheckpointer", "SqliteCheckpointer"}: + logger.warning( + "The checkpointer is %s: threads live in this process only (workers=%d). With more " + "than one worker or replica, a thread saved by one is invisible to the others and " + "is lost on restart. Use PgCheckpointer for a deployment with more than one process.", + kind, + workers, + ) + if not redis_url: + logger.warning( + "No Redis configured: the per-thread run lock, run and WebSocket caps and " + "in-memory rate limits are counted per process (workers=%d). With more than one " + "worker or replica each limit multiplies and two processes can run the same thread " + "at once. Set `redis` in 10xgraph.json for a shared view.", + workers, + ) + + async def attach_all_modules( config: GraphConfig, container: InjectQ, @@ -609,6 +649,7 @@ async def attach_all_modules( _bind_token_revocation(container, config, bool(auth_config), ownership_redis_url) _bind_shared_concurrency(container, ownership_redis_url) + warn_per_process_state(container, ownership_redis_url) # --- Media service wiring --- from tenxgraph_api.src.app.core.config.media_settings import ( diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index 33cc0230..1780888c 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -7,7 +7,6 @@ from injectq import InjectQ from injectq.integrations.fastapi import setup_fastapi from tenxgraph.core.graph import CompiledGraph -from tenxgraph.utils.background_task_manager import BackgroundTaskManager # # Prometheus Instrumentator import # from prometheus_fastapi_instrumentator import Instrumentator @@ -26,12 +25,6 @@ logger = logging.getLogger("tenxgraph_api") -# How long shutdown waits for in-flight background work (event publishing, -# checkpoint writes) before giving up. Bounded on purpose: a wedged sink must not -# hold the pod open until the kubelet SIGKILLs it, which would lose more than -# draining for a fixed window and then moving on. -SHUTDOWN_DRAIN_TIMEOUT_SECONDS = 30.0 - # Boot-time temp media cleanup is best effort. It must not hold the app (and so the first # readiness probe) behind a slow or wedged checkpointer or media store. BOOT_CLEANUP_TIMEOUT_SECONDS = 15.0 @@ -50,13 +43,79 @@ container.bind_instance(GraphConfig, graph_config) +def _media_store(): + """The media store the API serves files from, or None when media is not wired. + + The app binds ``MediaService`` (never a bare ``media_store``), so that is where it lives. + """ + try: + from tenxgraph_api.src.app.routers.media import MediaService + + service = container.try_get(MediaService) + return service.store if service is not None else None + except Exception as exc: + logger.debug("No media store available: %s", exc) + return None + + +# Files younger than this are never swept: an upload may still be writing them. +MEDIA_SWEEP_MIN_AGE_SECONDS = 3600.0 + + +async def _sweep_media_loop(interval: float) -> None: + """Delete files a crash left in the media store, every ``interval`` seconds, forever. + + Runs once straight away (the crash that left them was probably the last shutdown). A store + without ``sweep_orphans`` (memory, cloud, older core) makes this return at once. + """ + while True: + store = _media_store() + sweep = getattr(store, "sweep_orphans", None) + if not callable(sweep): + return + try: + removed = await sweep(MEDIA_SWEEP_MIN_AGE_SECONDS) + if removed: + logger.info("Media sweeper removed %d orphan files", removed) + except asyncio.CancelledError: + raise + except Exception as exc: + logger.warning("Media sweep failed: %s", exc) + await asyncio.sleep(interval) + + +async def _prepare_checkpointer() -> None: + """Create or upgrade the checkpointer's schema at boot when it has a ``asetup`` step. + + A fresh Postgres then works without a manual setup call, and a bad connection string or + missing permission shows in the boot log instead of on the first request. Best effort: + the checkpointer also creates its schema on its first query, so a failure here only logs. + """ + checkpointer = container.try_get("checkpointer") or container.try_get("BaseCheckpointer") + asetup = getattr(checkpointer, "asetup", None) + if not callable(asetup): + return + try: + await asyncio.wait_for(asetup(), BOOT_CLEANUP_TIMEOUT_SECONDS) + except Exception as exc: + logger.error( + "Checkpointer setup failed at startup (%s: %s); it will be retried on first use", + type(exc).__name__, + exc, + ) + + async def _cleanup_temp_media_cache() -> None: """Run best-effort cleanup of expired temporary media cache entries.""" try: from tenxgraph.storage.media.temp_cache import TemporaryMediaCache checkpointer = container.try_get("checkpointer") or container.try_get("BaseCheckpointer") - media_store = container.try_get("media_store") or container.try_get("BaseMediaStore") + media_store = ( + container.try_get("media_store") + or container.try_get("BaseMediaStore") + or _media_store() + ) if checkpointer is None: logger.debug("No checkpointer available, skipping temp media cache cleanup") @@ -82,6 +141,8 @@ async def lifespan(app: FastAPI): container=container, ) + await _prepare_checkpointer() + # Clean up expired temporary media cache on startup try: await asyncio.wait_for(_cleanup_temp_media_cache(), BOOT_CLEANUP_TIMEOUT_SECONDS) @@ -95,22 +156,35 @@ async def lifespan(app: FastAPI): # store = load_store(graph_config.store_path) # injector.binder.bind(BaseStore, store) - yield + sweeper: asyncio.Task | None = None + from tenxgraph_api.src.app.core.config.media_settings import get_media_settings - app.state.draining = True # readiness answers 503 from here on + sweep_interval = get_media_settings().MEDIA_SWEEP_INTERVAL_SECONDS + if sweep_interval > 0: + sweeper = asyncio.create_task(_sweep_media_loop(sweep_interval)) - # Shutdown. Order matters: drain in-flight work BEFORE tearing down the - # resources that work depends on. - # - # Previously this went straight to `graph.aclose()`, which released the graph's - # resources while background work (publisher emits, checkpoint writes) could - # still be in flight -- so a rolling deploy could drop the tail end of a run's - # events and its final checkpoint. - await _drain_background_tasks() + yield + app.state.draining = True # readiness answers 503 from here on + if sweeper is not None: + sweeper.cancel() + await asyncio.gather(sweeper, return_exceptions=True) + + # Shutdown. Order matters: the sweeper is already stopped, so nothing touches the media + # store while the graph closes. One core call ends active runs in phases (finish, stop, + # cancel; each saved INTERRUPTED), drains its own background work (events, checkpoint + # writes) and then releases the checkpointer, publisher and store. The API's own clients + # close after it because that work may still use them. if graph: - # release all the resources - await graph.aclose() + try: + stats = await graph.aclose( + grace=settings.SHUTDOWN_GRACE_SECONDS, + stop_timeout=settings.SHUTDOWN_STOP_SECONDS, + timeout=settings.SHUTDOWN_TIMEOUT_SECONDS, + ) + logger.info("Graph closed: %s", stats) + except Exception as exc: + logger.warning("Graph close did not finish cleanly: %s", exc) # Close rate-limit backend (e.g. Redis connection pool) backend = getattr(app.state, "rate_limit_backend", None) @@ -148,33 +222,6 @@ async def lifespan(app: FastAPI): logger.debug("Authorization backend close failed: %s", exc) -async def _drain_background_tasks(timeout: float = SHUTDOWN_DRAIN_TIMEOUT_SECONDS) -> None: - """Wait for in-flight background tasks (event publishing, checkpoint writes). - - Bounded: if the sink is wedged we still shut down, rather than hanging the pod - until the kubelet SIGKILLs it -- which would lose strictly more than draining - for a bounded window and then giving up. - """ - try: - task_manager = InjectQ.get_instance().try_get(BackgroundTaskManager) - except Exception: # container not configured; nothing to drain - return - - if task_manager is None: - return - - pending = getattr(task_manager, "pending_count", 0) - if not pending: - return - - logger.info("Draining %d in-flight background tasks before shutdown", pending) - try: - await task_manager.wait_for_all(timeout=timeout) - logger.info("Background tasks drained") - except Exception as e: - logger.warning("Background task drain did not finish cleanly: %s", e) - - app = FastAPI( title=settings.APP_NAME, version=settings.APP_VERSION, diff --git a/tenxgraph_api/src/app/routers/ag_ui/router.py b/tenxgraph_api/src/app/routers/ag_ui/router.py index 2122edee..6f01d146 100644 --- a/tenxgraph_api/src/app/routers/ag_ui/router.py +++ b/tenxgraph_api/src/app/routers/ag_ui/router.py @@ -9,12 +9,15 @@ from tenxgraph_api.src.app.core.auth.authorization import AuthorizationBackend from tenxgraph_api.src.app.core.auth.permissions import RequirePermission from tenxgraph_api.src.app.core.auth.request_config import ( + ClientInputError, client_error_detail, normalize_thread_id, + resource_id_error, ) from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.config.settings import get_settings from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe +from tenxgraph_api.src.app.routers.graph import idempotency from tenxgraph_api.src.app.routers.graph.run_guard import acquire_run_guard from tenxgraph_api.src.app.routers.graph.run_limits import release_when_done from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService @@ -26,6 +29,9 @@ from .service import stream_ag_ui +# A threadId that asks the server to make the thread (same word as the WebSocket routes). +NEW_THREAD = "new" + router = APIRouter( tags=["AG-UI"], ) @@ -71,16 +77,39 @@ async def run_ag_ui( except ValueError as exc: raise HTTPException(status_code=422, detail=client_error_detail(exc)) from exc + # Opt-in Idempotency-Key: a running run is 409; a finished one is 409 too (SSE output is + # not stored); a different body under the same key is 422. + idem = None + idem_key = request.headers.get(idempotency.IDEMPOTENCY_HEADER) + if isinstance(idem_key, str): + idem, _ = await idempotency.check(user, idem_key, "ag-ui", body) + if idem is None: + raise idempotency.refuse_repeat() + try: + return await _run_ag_ui(request, run_input, service, user, authz, config, idem) + except BaseException: + if idem is not None: + idem.release() + raise + + +async def _run_ag_ui(request, run_input, service, user, authz, config, idem): thread_id = normalize_thread_id(run_input.thread_id) - if thread_id is None: - raise HTTPException(status_code=422, detail="threadId is required") - run_input.thread_id = thread_id - # AG-UI clients make their own thread ids, so a new one is allowed even under - # runs.thread_ids "server" (client_thread_ids); an existing thread is still owner-checked. - if config.auth_config() and not await authz.authorize( - user, "graph", "stream", resource_id=thread_id, client_thread_ids=True + if thread_id not in (None, NEW_THREAD) and ( + problem := resource_id_error(thread_id, "threadId") + ): + raise ClientInputError(problem) + if thread_id is None or thread_id == NEW_THREAD: + # The protocol makes threadId required, so a client that wants a server-made thread + # sends it blank or "new". The new id comes back in RUN_STARTED. + thread_id = await service.new_thread_id() + elif config.auth_config() and not await authz.authorize( + user, "graph", "stream", resource_id=thread_id ): + # Same as REST: with runs.thread_ids "server" (the default) an id the server has never + # seen is refused; "client" lets the caller name a new thread. raise HTTPException(status_code=403, detail="Not allowed to access this thread") + run_input.thread_id = thread_id # External media URLs in the request, checked before the response starts so a blocked # one is a 400. Anything the converter rejects is left to the run, which reports it. @@ -99,17 +128,13 @@ async def run_ag_ui( # One run at a time per thread (AG-UI always waits); taken before the response starts so a # busy thread is a real 409. guard = await acquire_run_guard(user=user, runs=service.runs_policy(), thread_id=thread_id) + events = stream_ag_ui( + service, run_input, user, encoder, allow_client_tools=config.ag_ui.allow_client_tools + ) + if idem is not None: + events = idempotency.mark_when_done(events, idem, failed_marker="RUN_ERROR") return StreamingResponse( - release_when_done( - stream_ag_ui( - service, - run_input, - user, - encoder, - allow_client_tools=config.ag_ui.allow_client_tools, - ), - guard, - ), + release_when_done(events, guard), media_type=encoder.get_content_type(), headers={ "Cache-Control": "no-cache, no-transform", diff --git a/tenxgraph_api/src/app/routers/checkpointer/router.py b/tenxgraph_api/src/app/routers/checkpointer/router.py index a9b326df..144f04eb 100644 --- a/tenxgraph_api/src/app/routers/checkpointer/router.py +++ b/tenxgraph_api/src/app/routers/checkpointer/router.py @@ -9,6 +9,7 @@ from tenxgraph_api.src.app.core.auth.permissions import RequirePermission from tenxgraph_api.src.app.core.auth.request_config import client_config from tenxgraph_api.src.app.core.config.settings import get_settings +from tenxgraph_api.src.app.routers.graph.thread_lock import thread_write_guard from tenxgraph_api.src.app.utils.parse_output import parse_message_output from tenxgraph_api.src.app.utils.response_helper import success_response from tenxgraph_api.src.app.utils.swagger_helper import generate_swagger_responses @@ -120,11 +121,12 @@ async def put_state( # body config cannot redirect the call to another thread. config = {**client_config(payload.config), "thread_id": thread_id} - res = await service.put_state( - config, - user, - payload.state, - ) + async with thread_write_guard(thread_id): + res = await service.put_state( + config, + user, + payload.state, + ) return success_response( res, @@ -159,10 +161,11 @@ async def clear_state( validate_thread_id(thread_id) config = {"thread_id": thread_id} - res = await service.clear_state( - config, - user, - ) + async with thread_write_guard(thread_id): + res = await service.clear_state( + config, + user, + ) return success_response( res, @@ -212,12 +215,13 @@ async def put_messages( # body config cannot redirect the call to another thread. config = {**client_config(payload.config), "thread_id": thread_id} - res = await service.put_messages( - config, - user, - payload.messages, - payload.metadata, - ) + async with thread_write_guard(thread_id): + res = await service.put_messages( + config, + user, + payload.messages, + payload.metadata, + ) return success_response( res, @@ -401,11 +405,12 @@ async def delete_message( # body config cannot redirect the call to another thread. config = {**client_config(payload.config), "thread_id": thread_id} - await service.delete_message( - config, - user, - message_id, - ) + async with thread_write_guard(thread_id): + await service.delete_message( + config, + user, + message_id, + ) return success_response( {"success": True, "message": "Message deleted successfully"}, @@ -526,11 +531,12 @@ async def delete_thread( # body config cannot redirect the call to another thread. config = {**client_config(payload.config), "thread_id": thread_id} - res = await service.delete_thread( - config, - user, - thread_id, - ) + async with thread_write_guard(thread_id): + res = await service.delete_thread( + config, + user, + thread_id, + ) return success_response( res, diff --git a/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py b/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py index 943c7cb9..5bd676c5 100644 --- a/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py +++ b/tenxgraph_api/src/app/routers/checkpointer/schemas/checkpointer_schemas.py @@ -7,6 +7,7 @@ from tenxgraph.core.state import Message from tenxgraph_api.src.app.core.auth.request_config import ( + BoundedDict, ClientInputError, client_message_error, request_size_error, @@ -16,7 +17,7 @@ class ConfigSchema(BaseModel): """Schema for state data.""" - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default_factory=dict, description="Configuration for the state" ) @@ -30,7 +31,7 @@ class StateResponseSchema(BaseModel): class StateSchema(ConfigSchema): """Schema for putting state.""" - state: dict[str, Any] = Field(..., description="State data") + state: BoundedDict = Field(..., description="State data") @field_validator("state") @classmethod @@ -45,7 +46,7 @@ class PutMessagesSchema(ConfigSchema): """Schema for putting messages.""" messages: list[Message] = Field(..., description="List of messages to store") - metadata: dict[str, Any] | None = Field(None, description="Optional metadata") + metadata: BoundedDict | None = Field(None, description="Optional metadata") @field_validator("messages", mode="before") @classmethod @@ -121,7 +122,7 @@ class DeleteMessageSchema(ConfigSchema): class PutThreadSchema(ConfigSchema): """Schema for putting thread info.""" - thread_info: dict[str, Any] = Field(..., description="Thread information to store") + thread_info: BoundedDict = Field(..., description="Thread information to store") class GetThreadSchema(ConfigSchema): diff --git a/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py b/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py index 49ad9ce6..3b4b7095 100644 --- a/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py +++ b/tenxgraph_api/src/app/routers/checkpointer/services/checkpointer_service.py @@ -138,7 +138,7 @@ def _link_tracking_media(self) -> Any: async def _thread_media_keys( self, cfg: dict[str, Any], skip_message_id: Any = None ) -> list[str]: - """Media keys referenced by the thread's messages (all pages), minus one message. + """Media keys referenced by the thread's messages and state context, minus one message. A read failure raises, so a partial list is never used: cleanup can only miss a file (it stays), never delete one that is still referenced. @@ -146,6 +146,14 @@ async def _thread_media_keys( keys: list[str] = [] offset = 0 try: + # Files linked through PUT state live in the state's context, not in message rows. + state = await self.checkpointer.aget_state(cfg) + for message in getattr(state, "context", None) or []: + if skip_message_id is not None and str(getattr(message, "message_id", None)) == str( + skip_message_id + ): + continue + keys.extend(internal_media_keys(message)) while True: page = await self.checkpointer.alist_messages(cfg, None, offset, _CLEANUP_PAGE) for message in page: @@ -458,6 +466,9 @@ async def delete_thread( log_safe(sanitize_for_logging(user)), log_safe(thread_id), ) + # Cleanup after the thread row is gone is best effort, but a delete that left data + # behind must not read as a clean success: each step that failed is named in the reply. + incomplete: list[str] = [] # Collect the files the thread's messages reference before they are gone. thread_keys: list[str] = [] media = self._link_tracking_media() @@ -465,10 +476,19 @@ async def delete_thread( try: thread_keys = list(dict.fromkeys(await self._thread_media_keys(cfg))) except Exception as exc: - logger.debug("File cleanup skipped for thread %s: %s", log_safe(thread_id), exc) + incomplete.append("files") + logger.warning("File cleanup skipped for thread %s: %s", log_safe(thread_id), exc) res = await self.checkpointer.aclean_thread(cfg) - if thread_keys and media is not None: - await media.unlink_threads(thread_keys, str(thread_id)) + if ( + thread_keys + and media is not None + and await media.unlink_threads(thread_keys, str(thread_id)) + ): + incomplete.append("files") + # Files still linked to the thread through messages that are gone (cleared or replaced + # state) are not in `thread_keys`; release them by their own metadata. + if media is not None and await media.release_thread(str(thread_id), cfg.get("user_id")): + incomplete.append("files") # Clean telemetry traces if TelemetryStore is bound in InjectQ try: @@ -480,7 +500,8 @@ async def delete_thread( if telemetry_store: telemetry_store.delete_thread(str(thread_id)) except Exception as exc: - logger.debug( + incomplete.append("traces") + logger.warning( "Telemetry store cleanup failed for thread %s: %s", log_safe(thread_id), exc ) @@ -497,10 +518,22 @@ async def delete_thread( if pending is not None: await pending except Exception as exc: - logger.debug( + incomplete.append("cache") + logger.warning( "Ownership cache eviction failed for thread %s: %s", log_safe(thread_id), exc ) + if incomplete: + steps = ", ".join(dict.fromkeys(incomplete)) + return ResponseSchema( + success=False, + message=f"Thread deleted, but cleanup did not finish: {steps}", + data={ + "deleted": True, + "cleanup_failed": list(dict.fromkeys(incomplete)), + "result": res, + }, + ) return ResponseSchema(success=True, message="Thread deleted successfully", data=res) # ------------------------------------------------- diff --git a/tenxgraph_api/src/app/routers/graph/idempotency.py b/tenxgraph_api/src/app/routers/graph/idempotency.py new file mode 100644 index 00000000..0b797190 --- /dev/null +++ b/tenxgraph_api/src/app/routers/graph/idempotency.py @@ -0,0 +1,275 @@ +"""Opt-in ``Idempotency-Key`` for graph runs (invoke, stream, AG-UI and WebSocket). + +A gateway that retries a request after a timeout would run the tools twice. A client that sends +an ``Idempotency-Key`` header (WebSocket: ``idempotency_key`` in the run frame) gets this +instead, per user and key: + +- the first request runs normally and holds a lease while it runs; +- the same key while that run is still going: 409 with ``Retry-After``; +- the same key after it finished: invoke returns the stored result; stream replays the stored + final state and the ``done`` chunk as NDJSON; AG-UI and WebSocket answer 409 (their output is + an event sequence that is not stored); +- the same key with a different request body: 422; +- a run that failed stores nothing, so the same key may be retried. + +No header, no change. Records live in Redis when it is configured (the client of the shared +concurrency limiter, keys under ``REDIS_KEY_PREFIX``), else in this process. The running state +is the same kind of lease the thread lock uses, so a crashed worker's key frees itself. A Redis +error is logged and the request runs without idempotency, as the concurrency limiter fails open. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import time +import weakref +from collections.abc import AsyncIterator +from dataclasses import dataclass +from typing import Any + +from fastapi import HTTPException + +from tenxgraph_api.src.app.core import logger +from tenxgraph_api.src.app.core.auth.request_config import ClientInputError +from tenxgraph_api.src.app.core.concurrency import ( + Lease, + RedisConcurrencyLimiter, + concurrency_limiter, +) +from tenxgraph_api.src.app.core.config.settings import get_settings +from tenxgraph_api.src.app.core.redis_client import key_part, prefixed + + +IDEMPOTENCY_HEADER = "Idempotency-Key" +KEY_PATTERN = re.compile(r"^[\x21-\x7e]{1,255}$") +RETRY_AFTER_SECONDS = 5 +# A result larger than this is not stored; the key then answers 409 instead of replaying. +MAX_STORED_BYTES = 1024 * 1024 +MAX_MEMORY_RECORDS = 10_000 + +HTTP_CONFLICT = 409 +HTTP_UNPROCESSABLE = 422 + + +def idempotency_key_error(value: Any) -> str | None: + """Why ``value`` is not an acceptable key (1 to 255 printable ASCII characters), or None.""" + if not isinstance(value, str) or not KEY_PATTERN.fullmatch(value): + return f"{IDEMPOTENCY_HEADER} must be 1 to 255 printable ASCII characters" + return None + + +def fingerprint(kind: str, body: Any) -> str: + """A stable hash of the request body (JSON is canonicalised), per route kind.""" + if isinstance(body, bytes | bytearray): + try: + body = json.loads(body) + except ValueError: + body = bytes(body).decode("utf-8", "replace") + canonical = json.dumps(body, sort_keys=True, separators=(",", ":"), default=str) + return hashlib.sha256(f"{kind}\n{canonical}".encode()).hexdigest() + + +class IdempotencyConflict(HTTPException): + def __init__(self, detail: str) -> None: + super().__init__( + status_code=HTTP_CONFLICT, + detail=detail, + headers={"Retry-After": str(RETRY_AFTER_SECONDS)}, + ) + + +class IdempotencyMismatch(HTTPException): + def __init__(self) -> None: + super().__init__( + status_code=HTTP_UNPROCESSABLE, + detail=f"{IDEMPOTENCY_HEADER} was already used with a different request", + ) + + +_memory: dict[str, tuple[float, str]] = {} + + +def _redis_client() -> Any | None: + try: + from tenxgraph_api.src.app.core.concurrency import shared_concurrency_limiter + + limiter = shared_concurrency_limiter() + except Exception: + return None + return limiter.client if isinstance(limiter, RedisConcurrencyLimiter) else None + + +async def _load(name: str) -> dict[str, Any] | None: + client = _redis_client() + raw: Any = None + if client is not None: + try: + raw = await client.get(name) + except Exception as exc: + logger.warning("Idempotency lookup failed; running without it: %s", exc) + return None + else: + entry = _memory.get(name) + if entry and entry[0] > time.monotonic(): + raw = entry[1] + elif entry: + _memory.pop(name, None) + if raw is None: + return None + try: + record = json.loads(raw) + except ValueError: + return None + return record if isinstance(record, dict) else None + + +async def _save(name: str, record: dict[str, Any]) -> None: + text = json.dumps(record, separators=(",", ":"), default=str) + ttl = get_settings().IDEMPOTENCY_TTL_SECONDS + client = _redis_client() + if client is not None: + try: + await client.set(name, text, ex=ttl) + except Exception as exc: + logger.warning("Idempotency record could not be stored: %s", exc) + return + now = time.monotonic() + if len(_memory) >= MAX_MEMORY_RECORDS: + for stale in [k for k, (exp, _) in _memory.items() if exp <= now]: + _memory.pop(stale, None) + while len(_memory) >= MAX_MEMORY_RECORDS: # still full: drop the oldest + _memory.pop(next(iter(_memory))) + _memory[name] = (now + ttl, text) + + +@dataclass +class IdempotentRun: + """A request that took its key: finish it with :meth:`complete`, always :meth:`release`.""" + + name: str + fp: str + lease: Lease + replay: dict[str, Any] | None = None + + async def complete(self, result: str | None) -> None: + """Store the finished run's ``result`` (JSON text) so a repeat can be answered. + + ``None``, or a result over ``MAX_STORED_BYTES``, stores a marker only: a repeat then + answers 409 because there is nothing safe to replay. + """ + record: dict[str, Any] = {"fp": self.fp, "stored": False} + if result is not None and len(result) <= MAX_STORED_BYTES: + record = {"fp": self.fp, "stored": True, "result": result} + await _save(self.name, record) + + def release(self) -> None: + self.lease.release() + + +async def check( + user: dict[str, Any], key: str, kind: str, body: Any +) -> tuple[IdempotentRun | None, dict[str, Any] | None]: + """Look the key up. Returns ``(run, None)`` to proceed, or ``(None, record)`` to replay. + + Raises :class:`IdempotencyConflict` (409) while the first run is going, or when it finished + but nothing is stored; :class:`IdempotencyMismatch` (422) for a different body; and + ``ClientInputError`` (422) for a malformed key. + """ + if problem := idempotency_key_error(key): + raise ClientInputError(problem) + user_id = user.get("user_id") if isinstance(user, dict) else None + ident = f"{key_part(user_id or 'anonymous')}:{key_part(key)}" + name = f"{prefixed('10xgraph:idempotency')}:{ident}" + fp = fingerprint(kind, body) + + def _answer(record: dict[str, Any]) -> tuple[None, dict[str, Any]]: + if record.get("fp") != fp: + raise IdempotencyMismatch + return None, record + + if (record := await _load(name)) is not None: + return _answer(record) + lease = await concurrency_limiter().acquire(f"runs:idem:{ident}", 1) + if lease is None: + raise IdempotencyConflict("a request with this Idempotency-Key is still running") + if (record := await _load(name)) is not None: # finished while we took the lease + lease.release() + return _answer(record) + return IdempotentRun(name=name, fp=fp, lease=lease), None + + +def refuse_repeat() -> IdempotencyConflict: + """The 409 for a finished key on a route that does not replay (AG-UI, WebSocket).""" + return IdempotencyConflict( + "a run with this Idempotency-Key already completed; its output is not replayable here" + ) + + +async def _record_stream(lines: AsyncIterator[str], run: IdempotentRun) -> AsyncIterator[str]: + """Pass NDJSON ``lines`` through; remember the final state and ``done`` chunk on success. + + A run that errored, was cut off or never sent ``done`` stores nothing and frees the key. + """ + last_state: str | None = None + done: str | None = None + errored = False + try: + async for line in lines: + if line.startswith('{"event":"state"'): + last_state = line + elif line.startswith('{"event":"error"'): + errored = True + elif '"status":"done"' in line[:200] and line.startswith('{"event":"updates"'): + done = line + yield line + if done is not None and not errored: + if last_state is not None and len(last_state) > MAX_STORED_BYTES: + await run.complete(None) + else: + await run.complete(json.dumps({"state": last_state, "done": done})) + finally: + run.release() + + +def replay_lines(record: dict[str, Any]) -> list[str]: + """The NDJSON lines for a stored stream result: final state (if any), then ``done``.""" + stored = json.loads(record["result"]) + return [line for line in (stored.get("state"), stored["done"]) if line] + + +async def _mark_stream( + stream: AsyncIterator[Any], run: IdempotentRun, *, failed_marker: str | None = None +) -> AsyncIterator[Any]: + """Pass ``stream`` through and mark the key completed (no replayable output stored). + + For AG-UI, whose output is an SSE event sequence. An item containing ``failed_marker`` + means the run failed: nothing is marked and the key is free to retry. + """ + failed = False + try: + async for item in stream: + if failed_marker and isinstance(item, str) and failed_marker in item: + failed = True + yield item + if not failed: + await run.complete(None) + finally: + run.release() + + +def stream_with_record(lines: AsyncIterator[str], run: IdempotentRun) -> AsyncIterator[str]: + """``lines`` that remember their final state on success; the key is freed however it ends.""" + wrapped = _record_stream(lines, run) + weakref.finalize(wrapped, run.release) # dropped before it was ever iterated + return wrapped + + +def mark_when_done( + stream: AsyncIterator[Any], run: IdempotentRun, *, failed_marker: str | None = None +) -> AsyncIterator[Any]: + """``stream`` that marks the key completed (nothing replayable) when it ends without error.""" + wrapped = _mark_stream(stream, run, failed_marker=failed_marker) + weakref.finalize(wrapped, run.release) + return wrapped diff --git a/tenxgraph_api/src/app/routers/graph/realtime_guard.py b/tenxgraph_api/src/app/routers/graph/realtime_guard.py index c9409380..179e07c4 100644 --- a/tenxgraph_api/src/app/routers/graph/realtime_guard.py +++ b/tenxgraph_api/src/app/routers/graph/realtime_guard.py @@ -36,6 +36,7 @@ from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.middleware.rate_limit import keying from tenxgraph_api.src.app.core.middleware.rate_limit.keying import client_key_for +from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -162,7 +163,9 @@ async def _shared_connection_slots( leases.append(lease) user_key = keying.verified_user_id(websocket) if user_key and ws_config.max_connections_per_user is not None: - lease = await limiter.acquire(f"ws:user:{user_key}", ws_config.max_connections_per_user) + lease = await limiter.acquire( + f"ws:user:{key_part(user_key)}", ws_config.max_connections_per_user + ) if lease is None: logger.warning( "WebSocket per-user connection limit reached for %s (max=%s)", diff --git a/tenxgraph_api/src/app/routers/graph/router.py b/tenxgraph_api/src/app/routers/graph/router.py index ecd40f29..7757a115 100644 --- a/tenxgraph_api/src/app/routers/graph/router.py +++ b/tenxgraph_api/src/app/routers/graph/router.py @@ -20,13 +20,18 @@ RequirePermission, ws_bearer_subprotocol, ) -from tenxgraph_api.src.app.core.auth.request_config import normalize_thread_id +from tenxgraph_api.src.app.core.auth.request_config import ( + ClientInputError, + normalize_thread_id, + resource_id_error, +) from tenxgraph_api.src.app.core.auth.revocation import ( TokenRevocationUnavailableError, token_revoked, ) from tenxgraph_api.src.app.core.concurrency import Lease from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe, summarize_validation_error +from tenxgraph_api.src.app.routers.graph import idempotency from tenxgraph_api.src.app.routers.graph.realtime_guard import ( realtime_connection_guard, ws_identity_still_valid, @@ -44,8 +49,15 @@ ObservabilitySchema, WsGraphInputSchema, ) -from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService -from tenxgraph_api.src.app.routers.graph.thread_lock import THREAD_BUSY_DETAIL, requested_thread_id +from tenxgraph_api.src.app.routers.graph.services.graph_service import ( + SERVICE_UNAVAILABLE_CODE, + GraphService, +) +from tenxgraph_api.src.app.routers.graph.thread_lock import ( + THREAD_BUSY_DETAIL, + requested_thread_id, + thread_write_guard, +) from tenxgraph_api.src.app.utils import success_response from tenxgraph_api.src.app.utils.swagger_helper import generate_swagger_responses @@ -89,6 +101,9 @@ # RFC 6455 close code 1013 "Try Again Later", for a refused or shed run. WS_TRY_AGAIN_LATER = 1013 +# RFC 6455 close code 1012 "Service Restart", for a run refused because the graph is closing. +WS_SERVICE_RESTART = 1012 + # Longest one write to a client socket may take. A reader that stops reading would otherwise # park the run (and the thread lock and run slot it holds) on a full send buffer for good. This # bounds our own socket write only, never the run: a quiet run, however long, is not affected. @@ -280,7 +295,10 @@ async def _next_ws_input( await _send_ws_error(websocket, "Invalid request: frame is not valid JSON") return _WS_SKIP try: - return WsGraphInputSchema(**data) + ws_input = WsGraphInputSchema(**data) + if problem := resource_id_error((ws_input.config or {}).get("thread_id"), "thread_id"): + raise ClientInputError(problem) + return ws_input except Exception as e: logger.warning("WebSocket received invalid input: %s", _error_summary(e)) # The validation text can quote the submitted values; production sends the generic one. @@ -341,6 +359,16 @@ async def _http_run_guard( ) +async def _idempotency( + request: Request, user: dict[str, Any], kind: str +) -> tuple[idempotency.IdempotentRun | None, dict[str, Any] | None]: + """Take the request's ``Idempotency-Key``, if it sent one: ``(run, replay_record)``.""" + key = request.headers.get(idempotency.IDEMPOTENCY_HEADER) if request is not None else None + if not isinstance(key, str): + return None, None + return await idempotency.check(user, key, kind, await request.body()) + + @router.post( "/v1/graph/invoke", summary="Invoke graph execution", @@ -360,15 +388,33 @@ async def invoke_graph( logger.info(f"Graph invoke request received with {len(graph_input.messages)} messages") service.check_media_urls(graph_input.messages) + # Opt-in Idempotency-Key: a finished run's stored result is returned, a running one is 409. + idem, replay = await _idempotency(request, user, "invoke") + if replay is not None: + if not replay.get("stored"): + raise idempotency.refuse_repeat() + return success_response( + GraphInvokeOutputSchema.model_validate_json(replay["result"]), request + ) + # One run at a time per thread (409 when it stays busy) and the per-user run cap (429). - guard = await _http_run_guard(service, graph_input, user, request) + try: + guard = await _http_run_guard(service, graph_input, user, request) + except BaseException: + if idem: + idem.release() + raise try: result: GraphInvokeOutputSchema = await service.invoke_graph( graph_input, user, ) + if idem: + await idem.complete(result.model_dump_json()) finally: guard.release() + if idem: + idem.release() logger.info("Graph invoke completed successfully") @@ -398,16 +444,38 @@ async def stream_graph( # Before the response starts, so a blocked URL is a real 400 and not an error chunk. service.check_media_urls(graph_input.messages) + # Opt-in Idempotency-Key: a finished run's final state is replayed, a running one is 409. + idem, replay = await _idempotency(request, user, "stream") + if replay is not None: + if not replay.get("stored"): + raise idempotency.refuse_repeat() + + async def replayed(): + for line in idempotency.replay_lines(replay): + yield line + + return StreamingResponse(replayed(), media_type="application/x-ndjson") + # Taken before the response starts, so a busy thread still gets a real 409 and an # over-cap request a 429. - guard = await _http_run_guard(service, graph_input, user, request) + try: + guard = await _http_run_guard(service, graph_input, user, request) + except BaseException: + if idem: + idem.release() + raise try: # Same for a resume aimed at the wrong interrupt: a 409, as invoke answers it. await service.check_resume_target(graph_input, user) except BaseException: guard.release() + if idem: + idem.release() raise - result = release_when_done(service.stream_graph(graph_input, user), guard) + lines = service.stream_graph(graph_input, user) + if idem: + lines = idempotency.stream_with_record(lines, idem) + result = release_when_done(lines, guard) return StreamingResponse( result, @@ -612,11 +680,12 @@ async def fix_graph( if not _is_production(): logger.info("Graph fix request received for thread: %s", log_safe(fix_request.thread_id)) - result = await service.fix_graph( - fix_request.thread_id, - user, - fix_request.config, - ) + async with thread_write_guard(fix_request.thread_id): + result = await service.fix_graph( + fix_request.thread_id, + user, + fix_request.config, + ) if not _is_production(): logger.info("Graph fix completed for thread %s", log_safe(fix_request.thread_id)) @@ -702,13 +771,11 @@ async def _ws_thread_authorized( user: dict[str, Any], thread_id: str | None, action: str, - *, - client_thread_ids: bool = False, ) -> bool: """Owner-check ``thread_id`` for a WebSocket run. - ``client_thread_ids`` is for ``/v1/graph/live``, whose client makes its own thread ids: it - lets a new unknown id through under ``runs.thread_ids: "server"`` (the caller claims it). + Follows ``runs.thread_ids`` like REST: under ``"server"`` (the default) an id the server + has never seen is refused; ``"client"`` lets the caller name a new thread. Returns ``True`` when the run may proceed. Ownership is only enforced when auth is configured (``GraphConfig.auth_config``); an unauthenticated dev graph and fresh @@ -732,10 +799,6 @@ async def _ws_thread_authorized( if not has_auth: return True - if client_thread_ids: - return await authz.authorize( - user, "graph", action, resource_id=str(thread_id), client_thread_ids=True - ) return await authz.authorize(user, "graph", action, resource_id=str(thread_id)) @@ -758,6 +821,21 @@ def _guard_refusal(exc: HTTPException) -> str: return THREAD_BUSY_DETAIL if exc.status_code == 409 else str(exc.detail) # noqa: PLR2004 +class _ServiceClosing(Exception): + """A run was refused because the graph is closing; the error frame is already sent.""" + + +def _is_service_unavailable_chunk(chunk_json: str) -> bool: + """Whether a serialized chunk is the error chunk of a run refused while closing.""" + if not chunk_json.startswith('{"event":"error"'): + return False + try: + data = json.loads(chunk_json).get("data") + except ValueError: + return False + return isinstance(data, dict) and data.get("code") == SERVICE_UNAVAILABLE_CODE + + async def _ws_run( websocket: WebSocket, service: GraphService, @@ -778,6 +856,8 @@ async def pump() -> None: async with contextlib.aclosing(stream): async for chunk_json in stream: await _ws_send(websocket, chunk_json) + if _is_service_unavailable_chunk(chunk_json): + raise _ServiceClosing run_task = asyncio.create_task(pump()) reader = asyncio.create_task(frames.read_ahead()) @@ -932,6 +1012,23 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 graph_input = ws_input.to_graph_input() if not await _ws_media_urls_allowed(websocket, service, graph_input): continue + idem = None + if ws_input.idempotency_key is not None: + try: + idem, _ = await idempotency.check( + user, + ws_input.idempotency_key, + "ws", + ws_input.model_dump(mode="json", exclude={"idempotency_key"}), + ) + if idem is None: # finished before: a socket run's output is not replayable + raise idempotency.refuse_repeat() + except HTTPException as exc: + await _send_ws_error(websocket, str(exc.detail)) + continue + except ClientInputError as exc: + await _send_ws_error(websocket, str(exc)) + continue try: guard = await acquire_run_guard( user=user, @@ -941,6 +1038,8 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 stop=lambda tid=thread_id: service.stop_graph(tid, user), ) except HTTPException as exc: + if idem: + idem.release() await websocket.send_text( StreamChunk( event=StreamEvent.ERROR, data={"reason": _guard_refusal(exc)} @@ -949,8 +1048,12 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 continue try: finished = await _ws_run(websocket, service, graph_input, user, frames) + if idem and finished: + await idem.complete(None) finally: guard.release() + if idem: + idem.release() if not finished: break @@ -964,6 +1067,11 @@ async def websocket_graph( # noqa: PLR0912, PLR0915 except WebSocketDisconnect: logger.info("WebSocket client disconnected") + except _ServiceClosing: + # The SERVICE_UNAVAILABLE error frame went out; now tell the client to reconnect. + logger.info("WebSocket run refused: the graph is closing") + with contextlib.suppress(Exception): + await websocket.close(code=WS_SERVICE_RESTART) except WsSendTimeout: logger.warning("WebSocket client is not reading; closing the socket") with contextlib.suppress(Exception): @@ -1109,6 +1217,9 @@ async def realtime_graph_ws( # noqa: PLR0911, PLR0912, PLR0915 Protocol (provider-neutral; the client never sees Gemini vs OpenAI) ------------------------------------------------------------------ First frame : JSON control ``{model, thread_id?, voice?, modalities?, vad?, ...}`` + (``thread_id`` absent, blank or ``"new"`` makes the server create the thread) + First reply : JSON ``{type:"session_started", thread_id}``; reconnect with that id to + continue the thread Upstream : binary frame = PCM16 input audio JSON control = ``{type:"activity_start"|"activity_end"|"text"|"close", ...}`` Downstream : binary frame = PCM16 model audio (``audio_delta``) @@ -1174,9 +1285,18 @@ async def realtime_graph_ws( # noqa: PLR0911, PLR0912, PLR0915 if isinstance(init, dict): # Same rules as /v1/graph/ws: "new" starts a fresh thread, never a shared one. thread_id = _resolve_ws_thread(init) - if not await _ws_thread_authorized( - authz, user, thread_id, "stream", client_thread_ids=True - ): + if problem := resource_id_error(thread_id, "thread_id"): + logger.warning("Realtime init frame refused: %s", problem) + with contextlib.suppress(Exception): + await websocket.send_text( + _realtime_event_json( + ErrorEvent(code="VALIDATION_ERROR", message=problem, fatal=True) + ) + ) + with contextlib.suppress(Exception): + await websocket.close(code=1003) + return + if not await _ws_thread_authorized(authz, user, thread_id, "stream"): logger.warning( "Realtime WebSocket authorization failed for user %s on thread %s", log_safe(user.get("user_id")), @@ -1202,6 +1322,12 @@ async def realtime_graph_ws( # noqa: PLR0911, PLR0912, PLR0915 await websocket.close(code=1003) return + if thread_id is None: + # No id (absent, blank or "new"): the server makes the thread, as AG-UI does, so there + # is nothing to owner-check. The id goes back to the client in ``session_started``. + thread_id = await service.new_thread_id() + init["thread_id"] = thread_id + async def close_session_expired() -> None: logger.info("Realtime credential no longer valid; closing") with contextlib.suppress(Exception): @@ -1247,6 +1373,19 @@ async def close_session_expired() -> None: queue = _ByteBoundedLiveInputQueue(maxsize=REALTIME_INPUT_QUEUE_MAXSIZE) + # First server event of the session: the thread it runs on (server-made when the client + # sent none). A reconnect sends this id back to continue the same thread. + try: + await _ws_send(websocket, json.dumps({"type": "session_started", "thread_id": thread_id})) + except Exception as e: + # The client went away (or stopped reading) before the session began. + logger.info("Realtime client gone before session start: %s", type(e).__name__) + queue.close() + guard.release() + with contextlib.suppress(Exception): + await websocket.close() + return + async def upstream() -> None: """Pump client frames into the input queue until close/disconnect.""" try: diff --git a/tenxgraph_api/src/app/routers/graph/run_limits.py b/tenxgraph_api/src/app/routers/graph/run_limits.py index a9b32777..61ed878f 100644 --- a/tenxgraph_api/src/app/routers/graph/run_limits.py +++ b/tenxgraph_api/src/app/routers/graph/run_limits.py @@ -24,6 +24,7 @@ from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.concurrency import Lease, concurrency_limiter from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -36,7 +37,7 @@ def run_slot_key(user_id: str) -> str: - return f"runs:user:{user_id}" + return f"runs:user:{key_part(user_id)}" async def acquire_run_slot(user: dict[str, Any], runs: RunsConfig) -> Lease: diff --git a/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py b/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py index 558a78ab..cb820879 100644 --- a/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py +++ b/tenxgraph_api/src/app/routers/graph/schemas/graph_schemas.py @@ -5,9 +5,11 @@ from tenxgraph.utils import ResponseGranularity from tenxgraph_api.src.app.core.auth.request_config import ( + BoundedDict, ClientInputError, client_message_error, request_size_error, + structure_error, with_fresh_message_ids, ) @@ -43,11 +45,11 @@ class GraphInputSchema(BaseModel): "is rejected with 409 before anything runs." ), ) - initial_state: dict[str, Any] | None = Field( + initial_state: BoundedDict | None = Field( default=None, description="Initial state for the graph execution", ) - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default=None, description="Optional configuration for graph execution", ) @@ -75,6 +77,14 @@ def messages_within_limits(cls, v: Any) -> Any: raise ClientInputError(error) return v + @field_validator("initial_state", "config", "resume", mode="before") + @classmethod + def nesting_within_limit(cls, v: Any, info: ValidationInfo) -> Any: + # Before parsing: a very deep value would fail with RecursionError inside the run. + if error := structure_error(v, info.field_name or "input"): + raise ClientInputError(error) + return v + @field_validator("messages") @classmethod def client_messages_only(cls, v: list[Message], info: ValidationInfo) -> list[Message]: @@ -342,7 +352,7 @@ def thread_id_not_blank(cls, v: str) -> str: raise ValueError("thread_id cannot be empty or whitespace") return v.strip() - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default=None, description="Optional configuration for the stop operation" ) @@ -359,7 +369,7 @@ def thread_id_not_blank(cls, v: str) -> str: raise ValueError("thread_id cannot be empty or whitespace") return v.strip() - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default=None, description="Optional configuration for the fix operation" ) @@ -395,6 +405,15 @@ class WsGraphInputSchema(BaseModel): can resume the correct checkpointed thread. ``messages`` is unused. """ + idempotency_key: str | None = Field( + default=None, + max_length=255, + description=( + "Optional Idempotency-Key for this run. The same key from the same user while the " + "first run is going, or after it finished, is refused with an error frame; a " + "different request under the same key is refused too." + ), + ) invoke_type: Literal["fresh", "resume"] = Field( default="fresh", description=( @@ -435,11 +454,11 @@ class WsGraphInputSchema(BaseModel): ) # ── Shared fields (mirror GraphInputSchema) ─────────────────────────── - initial_state: dict[str, Any] | None = Field( + initial_state: BoundedDict | None = Field( default=None, description="Initial state for the graph execution.", ) - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default=None, description="Optional configuration. Must include 'thread_id' for resume runs.", ) @@ -470,6 +489,13 @@ def messages_within_limits(cls, v: Any, info: ValidationInfo) -> Any: raise ClientInputError(error) return v + @field_validator("resume", mode="before") + @classmethod + def resume_within_limits(cls, v: Any) -> Any: + if error := structure_error(v, "resume"): + raise ClientInputError(error) + return v + @model_validator(mode="after") def validate_by_invoke_type(self) -> "WsGraphInputSchema": if self.invoke_type == "fresh": diff --git a/tenxgraph_api/src/app/routers/graph/services/graph_service.py b/tenxgraph_api/src/app/routers/graph/services/graph_service.py index 78b38b5b..44919bb4 100644 --- a/tenxgraph_api/src/app/routers/graph/services/graph_service.py +++ b/tenxgraph_api/src/app/routers/graph/services/graph_service.py @@ -10,6 +10,7 @@ from fastapi import HTTPException from injectq import InjectQ, inject, singleton from pydantic import BaseModel, TypeAdapter, ValidationError +from tenxgraph.core.exceptions import GraphClosedError from tenxgraph.core.exceptions.media_exceptions import UnsupportedMediaInputError from tenxgraph.core.graph import CompiledGraph from tenxgraph.core.state import AgentState, Message, StreamChunk, StreamEvent @@ -91,6 +92,8 @@ def _resume_error_types() -> tuple[type[Exception], ...]: return ( InterruptConflictError, *_core_error("InterruptMismatchError"), + *_core_error("NothingToResumeError"), + *_core_error("InterruptPendingError"), *_core_error("ResumeValidationError"), ) @@ -98,7 +101,12 @@ def _resume_error_types() -> tuple[type[Exception], ...]: def _resume_http_error(exc: Exception) -> HTTPException: """409 for a resume aimed at the wrong (or no) interrupt, 422 for an answer that fails the interrupt's schema. Both messages are written for the client, so production keeps them.""" - conflict = (InterruptConflictError, *_core_error("InterruptMismatchError")) + conflict = ( + InterruptConflictError, + *_core_error("InterruptMismatchError"), + *_core_error("NothingToResumeError"), + *_core_error("InterruptPendingError"), + ) return HTTPException( status_code=409 if isinstance(exc, conflict) else 422, detail=str(getattr(exc, "message", None) or exc), @@ -157,6 +165,9 @@ def __init__(self, timeout: float) -> None: _STREAM_DONE = object() +# ``data.code`` of the error chunk for a run refused while the graph is closing. +SERVICE_UNAVAILABLE_CODE = "SERVICE_UNAVAILABLE" + def _stream_error_reason(exc: Exception) -> str: """The client-facing reason for a failed stream. @@ -178,6 +189,23 @@ def _stream_error_reason(exc: Exception) -> str: return _sanitize_error_message(str(exc), "GRAPH_STREAM_ERROR", is_production) +def _stream_error_data(exc: Exception) -> dict[str, Any]: + """The ``data`` of a stream's terminal ERROR chunk. + + A run refused because the graph is closing carries the same code and retry hint as the + HTTP 503 (``SERVICE_UNAVAILABLE``), so a client can tell it from a failure and retry. + """ + if isinstance(exc, GraphClosedError): + from tenxgraph_api.src.app.core.exceptions.server_errors import RETRY_AFTER_SECONDS + + return { + "reason": "The server is shutting down. Retry the request.", + "code": SERVICE_UNAVAILABLE_CODE, + "retry_after_seconds": RETRY_AFTER_SECONDS, + } + return {"reason": _stream_error_reason(exc)} + + # Chunk ``data`` fields that carry exception text. ``message`` is what a tool's own # ``emitter.error()`` sends. _ERROR_TEXT_FIELDS = ("reason", "error", "message") @@ -709,6 +737,10 @@ def runs_policy(self) -> RunsConfig: runs = getattr(self.config, "runs", None) return runs if isinstance(runs, RunsConfig) else RunsConfig() + async def new_thread_id(self) -> str: + """A fresh server-made thread id (public name of :meth:`_new_thread_id`).""" + return await self._new_thread_id() + async def _new_thread_id(self) -> str: """Generate a thread id that nobody owns yet. @@ -1202,7 +1234,7 @@ async def stream_chunks( # noqa: PLR0912, PLR0915 logger.debug("Telemetry finish_run failed on stream error: %s", telemetry_exc) yield StreamChunk( event=StreamEvent.ERROR, - data={"reason": _stream_error_reason(e)}, + data=_stream_error_data(e), metadata=meta, ) finally: diff --git a/tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py b/tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py index edd2fcac..08ff2554 100644 --- a/tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py +++ b/tenxgraph_api/src/app/routers/graph/services/multimodal_preprocessor.py @@ -128,6 +128,29 @@ def walk(node: Any) -> None: return keys +def media_file_ids(message: Message | dict[str, Any]) -> list[str]: + """The ``file_id`` of every top-level media block (``media.kind == "file_id"``). + + These are the blocks ``preprocess_multimodal_messages`` rewrites to ``graph://media/`` + references, so they are held to the same ownership rule. + """ + content = ( + message.get("content") if isinstance(message, dict) else getattr(message, "content", None) + ) + ids: list[str] = [] + for block in content or []: + media = block.get("media") if isinstance(block, dict) else getattr(block, "media", None) + if media is None: + continue + kind = media.get("kind") if isinstance(media, dict) else media.kind + file_id = ( + media.get("file_id") if isinstance(media, dict) else getattr(media, "file_id", None) + ) + if kind == "file_id" and isinstance(file_id, str) and file_id: + ids.append(file_id) + return ids + + async def _ensure_access(media_service: MediaService, file_id: str, user_id: str) -> None: """Ownership check with one answer for missing and foreign files. @@ -152,7 +175,7 @@ async def check_media_references( written straight into a thread, since the next run resolves those references too. """ for message in messages: - keys = list(dict.fromkeys(internal_media_keys(message))) + keys = list(dict.fromkeys([*internal_media_keys(message), *media_file_ids(message)])) if not keys: continue if media_service is None: @@ -182,6 +205,18 @@ async def _get_cached_extraction(media_service: MediaService, file_id: str) -> s return None +async def _offload_inline( + media_service: MediaService, messages: list[Message], user_id: str | None +) -> None: + """Move large inline base64 blobs into the media store before anything copies them into state. + + Skipped for a stand-in service (it has no store to offload into). + """ + offload = getattr(media_service, "offload_inline", None) + if inspect.iscoroutinefunction(offload): + await offload(messages, user_id) + + async def preprocess_multimodal_messages( messages: list[Message], media_service: MediaService | None, @@ -206,6 +241,8 @@ async def preprocess_multimodal_messages( if media_service is None: return messages + await _offload_inline(media_service, messages, user_id) + processed: list[Message] = [] for msg in messages: new_content = [] diff --git a/tenxgraph_api/src/app/routers/graph/thread_lock.py b/tenxgraph_api/src/app/routers/graph/thread_lock.py index 312fe64d..fca78d8b 100644 --- a/tenxgraph_api/src/app/routers/graph/thread_lock.py +++ b/tenxgraph_api/src/app/routers/graph/thread_lock.py @@ -21,7 +21,8 @@ from __future__ import annotations import asyncio -from collections.abc import Awaitable, Callable +import contextlib +from collections.abc import AsyncIterator, Awaitable, Callable from typing import Any, Literal from fastapi import HTTPException @@ -29,6 +30,7 @@ from tenxgraph_api.src.app.core import logger from tenxgraph_api.src.app.core.auth.request_config import normalize_thread_id from tenxgraph_api.src.app.core.concurrency import Lease, concurrency_limiter +from tenxgraph_api.src.app.core.redis_client import key_part from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe @@ -53,7 +55,7 @@ def thread_lock_key(thread_id: str) -> str: - return f"runs:thread:{thread_id}" + return f"runs:thread:{key_part(thread_id)}" class ThreadBusyError(HTTPException): @@ -82,6 +84,7 @@ async def acquire_thread_lock( on_busy: OnBusy = "wait", stop: Callable[[], Awaitable[Any]] | None = None, is_disconnected: Callable[[], Awaitable[bool]] | None = None, + wait_seconds: float | None = None, ) -> Lease: """Take the run lock for ``thread_id``, waiting for the active run if there is one. @@ -89,11 +92,14 @@ async def acquire_thread_lock( while the thread is busy. Raises :class:`ThreadBusyError` (409) after the wait limit. ``is_disconnected`` tells whether the client is gone: a waiter whose client left raises :class:`ClientGoneError` instead of taking the lock and running for nobody. + ``wait_seconds`` defaults to ``THREAD_BUSY_WAIT_SECONDS``, read at call time. """ + if wait_seconds is None: + wait_seconds = THREAD_BUSY_WAIT_SECONDS limiter = concurrency_limiter() key = thread_lock_key(thread_id) loop = asyncio.get_running_loop() - deadline = loop.time() + THREAD_BUSY_WAIT_SECONDS + deadline = loop.time() + wait_seconds delay = THREAD_BUSY_POLL_MIN_SECONDS next_stop = 0.0 waited = False @@ -123,6 +129,26 @@ async def acquire_thread_lock( delay = min(delay * 2, THREAD_BUSY_POLL_MAX_SECONDS) +@contextlib.asynccontextmanager +async def thread_write_guard(thread_id: Any) -> AsyncIterator[None]: + """Hold the thread's run lock around a state-changing REST call. + + A state edit, message import, message or thread delete, or graph fix that races an active + run is overwritten by that run or brings a deleted thread back. This takes the lock the run + routes take, so the call is refused with 409 (``ThreadBusyError``) at once while a run holds + it, instead of waiting behind a run that can last minutes. + """ + key = normalize_thread_id(thread_id) + if key is None: + yield + return + lease = await acquire_thread_lock(key, wait_seconds=0) + try: + yield + finally: + lease.release() + + def requested_thread_id(config: dict[str, Any] | None) -> str | None: """The thread a request names, or ``None`` when it starts a new one.""" return normalize_thread_id((config or {}).get("thread_id")) diff --git a/tenxgraph_api/src/app/routers/media/__init__.py b/tenxgraph_api/src/app/routers/media/__init__.py index a6c9e2bb..571d3764 100644 --- a/tenxgraph_api/src/app/routers/media/__init__.py +++ b/tenxgraph_api/src/app/routers/media/__init__.py @@ -7,19 +7,22 @@ import logging import time from collections import OrderedDict +from collections.abc import Callable from pathlib import Path from typing import Any from injectq import InjectQ, inject, singleton from tenxgraph.storage.checkpointer import BaseCheckpointer from tenxgraph.storage.media.config import DocumentHandling +from tenxgraph.storage.media.offload import MediaOffloadPolicy, ensure_media_offloaded +from tenxgraph.storage.media.security import check_image_pixels from tenxgraph.storage.media.storage.base import BaseMediaStore from tenxgraph_api.src.app.core.auth.request_config import ClientInputError from tenxgraph_api.src.app.core.config.graph_config import MediaConfig from tenxgraph_api.src.app.core.config.media_settings import MediaSettings, MediaStorageType from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe -from tenxgraph_api.src.app.utils.media.extractor import DocumentExtractor +from tenxgraph_api.src.app.utils.media.extractor import DocumentExtractor, DocumentTooLargeError from tenxgraph_api.src.app.utils.media.pipeline import DocumentPipeline @@ -101,16 +104,19 @@ def _build_config_instance(config_cls: type[Any], values: dict[str, Any]) -> Any def _create_media_store(settings: MediaSettings) -> BaseMediaStore: """Factory that instantiates the right MediaStore from settings.""" stype = settings.MEDIA_STORAGE_TYPE + # The store enforces the upload cap itself, so offloaded and other non-upload writes + # cannot go past it either. + max_bytes = int(settings.MEDIA_MAX_SIZE_MB * 1024 * 1024) if stype == MediaStorageType.MEMORY: from tenxgraph.storage.media.storage.memory_store import InMemoryMediaStore - return InMemoryMediaStore() + return InMemoryMediaStore(max_size_bytes=max_bytes) if stype == MediaStorageType.LOCAL: from tenxgraph.storage.media.storage.local_store import LocalFileMediaStore - return LocalFileMediaStore(base_dir=settings.MEDIA_STORAGE_PATH) + return LocalFileMediaStore(base_dir=settings.MEDIA_STORAGE_PATH, max_size_bytes=max_bytes) if stype == MediaStorageType.CLOUD: from cloud_storage_manager import ( @@ -166,6 +172,7 @@ def _create_media_store(settings: MediaSettings) -> BaseMediaStore: return CloudMediaStore( storage=storage, prefix=settings.MEDIA_CLOUD_PREFIX, + max_size_bytes=max_bytes, ) raise ValueError(f"Unknown MEDIA_STORAGE_TYPE: {stype}") @@ -179,10 +186,37 @@ def _create_document_pipeline(settings: MediaSettings) -> DocumentPipeline: } handling = handling_map.get(settings.DOCUMENT_HANDLING, DocumentHandling.EXTRACT_TEXT) - extractor = DocumentExtractor() if handling == DocumentHandling.EXTRACT_TEXT else None + extractor = ( + DocumentExtractor( + max_pdf_bytes=int(settings.MEDIA_MAX_PDF_MB * 1024 * 1024), + max_pdf_pages=settings.MEDIA_MAX_PDF_PAGES, + ) + if handling == DocumentHandling.EXTRACT_TEXT + else None + ) return DocumentPipeline(document_extractor=extractor, handling=handling) +class _OwnedStore: + """Wraps a media store so offloaded blobs get the same metadata as an upload.""" + + def __init__(self, store: BaseMediaStore, user_id: str | None, track_threads: bool): + self._store = store + self._user_id = user_id + self._track_threads = track_threads + + async def store(self, data: bytes, mime_type: str, metadata: dict[str, Any] | None = None): + meta: dict[str, Any] = {"created_at": int(time.time()), **(metadata or {})} + if self._user_id: + meta["owner_id"] = str(self._user_id) + if self._track_threads: + meta[THREADS_KEY] = [] + return await self._store.store(data, mime_type, metadata=meta) + + def to_media_ref(self, *args: Any, **kwargs: Any): + return self._store.to_media_ref(*args, **kwargs) + + @singleton class MediaService: """Central media service providing storage + document extraction.""" @@ -297,6 +331,26 @@ async def upload_file( f"File size {len(data)} exceeds maximum {self._settings.MEDIA_MAX_SIZE_MB}MB" ) + if mime_type.lower().startswith("image/"): + try: + check_image_pixels(data, self._settings.MEDIA_MAX_IMAGE_PIXELS or None) + except ValueError as exc: + raise ClientInputError(str(exc)) from exc + + extracting = self.pipeline.handling == DocumentHandling.EXTRACT_TEXT and ( + self._is_extractable(mime_type) + ) + check_limits = ( + getattr(self.pipeline.extractor, "check_limits", None) if extracting else None + ) + if callable(check_limits): + try: + check_limits(data, mime_type) + except DocumentTooLargeError as exc: + raise ClientInputError(str(exc)) from exc + + await self._check_quota(user_id, len(data)) + # Store binary in the configured MediaStore. The owner is written into the # file's own metadata, so ownership survives as long as the file does -- # unlike a cache entry, which would expire and silently un-own it. @@ -324,9 +378,7 @@ async def upload_file( } # If it's a document, try extraction - if self.pipeline.handling == DocumentHandling.EXTRACT_TEXT and self._is_extractable( - mime_type - ): + if extracting: try: text = await self.pipeline.extractor.extract(data, filename, mime_type) if text: @@ -345,6 +397,68 @@ async def upload_file( return result + async def _check_quota(self, user_id: str | None, incoming_bytes: int) -> None: + """Refuse an upload that would take ``user_id`` past the per-user file or byte quota. + + Does nothing without a user, with both quotas off, or when the store cannot list its + files (a warning is logged once). Counts only files that record this user as owner. + """ + max_files = self._settings.MEDIA_MAX_FILES_PER_USER + max_bytes = int(self._settings.MEDIA_MAX_USER_STORAGE_MB * 1024 * 1024) + if not user_id or (not max_files and not max_bytes): + return + try: + keys = await self._keys_of(user_id) + except NotImplementedError: + if not getattr(self, "_quota_notice_logged", False): + self._quota_notice_logged = True + logger.warning( + "MEDIA_MAX_FILES_PER_USER / MEDIA_MAX_USER_STORAGE_MB are set but %s " + "cannot list files; the per-user quota is not enforced.", + type(self.store).__name__, + ) + return + count = 0 + used = 0 + for key in keys: + try: + meta = await self.store.get_metadata(key) + except Exception as exc: + logger.debug("Skipping unreadable file %s in quota count: %s", log_safe(key), exc) + continue + if meta and str(meta.get("owner_id")) == str(user_id): + count += 1 + used += int(meta.get("size_bytes") or 0) + if max_files and count >= max_files: + raise ClientInputError(f"Upload limit reached: at most {max_files} files per user") + if max_bytes and used + incoming_bytes > max_bytes: + raise ClientInputError( + f"Storage limit reached: at most {self._settings.MEDIA_MAX_USER_STORAGE_MB}MB " + "of files per user" + ) + + async def offload_inline(self, messages: list[Any], user_id: str | None = None) -> None: + """Move large inline base64 media in ``messages`` into the store, in place. + + Uses core ``ensure_media_offloaded`` with ``MEDIA_INLINE_MAX_BYTES`` as the threshold + (0 turns it off). Offloaded files are owned by ``user_id`` and carry the same metadata + as an upload, so the caller can link them to the thread and they are deleted with it. + + Raises: + ClientInputError: The blob is not valid base64 or exceeds ``MEDIA_MAX_SIZE_MB``. + """ + threshold = self._settings.MEDIA_INLINE_MAX_BYTES + if threshold <= 0: + return + store = _OwnedStore(self.store, user_id, self.supports_links) + try: + for message in messages: + await ensure_media_offloaded( + message, store, MediaOffloadPolicy.THRESHOLD, max_inline_bytes=threshold + ) + except ValueError as exc: # bad base64 (binascii.Error) or MediaTooLargeError + raise ClientInputError(f"Inline media rejected: {exc}") from exc + # ------------------------------------------------------------------ # Thread links and deletion # ------------------------------------------------------------------ @@ -370,6 +484,42 @@ def delete_with_thread(self) -> bool: return True return bool(media.delete_with_thread) if isinstance(media, MediaConfig) else True + async def _keys_of(self, user_id: str | None) -> list[str]: + """Keys worth inspecting for ``user_id``; every key when there is no user. + + Uses the store's per-owner listing when it has one (the cloud store's owner index), + else lists all keys. Callers still check each file's recorded owner. + + Raises: + NotImplementedError: The store cannot enumerate the files. + """ + if user_id: + for_owner = getattr(self.store, "list_keys_for_owner", None) # core before 1.0: none + if callable(for_owner): + return list(await for_owner(str(user_id))) + list_keys = getattr(self.store, "list_keys", None) # absent in core older than 1.0 + if not callable(list_keys): + raise NotImplementedError(f"{type(self.store).__name__} cannot list files") + return list(await list_keys()) + + async def _modify_metadata( + self, key: str, fn: Callable[[dict[str, Any]], dict[str, Any] | None] + ) -> bool: + """Change one file's metadata from its current value as a single step. + + Uses the store's ``modify_metadata`` (atomic in the bundled stores). A store from an + older core falls back to read then ``update_metadata``, which can lose a concurrent + update. ``fn`` may run more than once, so it must only compute. + """ + modify = getattr(self.store, "modify_metadata", None) + if callable(modify): + return bool(await modify(key, fn)) + metadata = await self.store.get_metadata(key) + if metadata is None: + return False + updates = fn(dict(metadata)) + return await self._update_metadata(key, updates) if updates else True + async def link_threads(self, keys: list[str], thread_id: str, user_id: str) -> None: """Record ``thread_id`` on each file the caller owns that an accepted message references. @@ -388,44 +538,140 @@ async def link_threads(self, keys: list[str], thread_id: str, user_id: str) -> N return for key in dict.fromkeys(keys): try: - metadata = await self.store.get_metadata(key) - if not metadata or str(metadata.get("owner_id")) != str(user_id): - continue - threads = metadata.get(THREADS_KEY) - if not isinstance(threads, list) or thread_id in threads: - continue - await self._update_metadata(key, {THREADS_KEY: [*threads, thread_id]}) + + def add_link(metadata: dict[str, Any]) -> dict[str, Any] | None: + threads = metadata.get(THREADS_KEY) + if ( + str(metadata.get("owner_id")) != str(user_id) + or not isinstance(threads, list) + or thread_id in threads + ): + return None + return {THREADS_KEY: [*threads, thread_id]} + + await self._modify_metadata(key, add_link) except Exception as exc: logger.warning("Could not link file %s to a thread: %s", log_safe(key), exc) - async def unlink_threads(self, keys: list[str], thread_id: str) -> None: + async def unlink_threads(self, keys: list[str], thread_id: str) -> list[str]: """Drop ``thread_id`` from each file's links; delete a file left with none. - The delete only happens with ``media.delete_with_thread`` on. Never raises. + The delete only happens with ``media.delete_with_thread`` on. Never raises; returns the + keys that could not be released (empty when all were), so a caller can report a partial + cleanup. """ + failed: list[str] = [] if not keys or not self.supports_links: - return + return failed delete = self.delete_with_thread() for key in dict.fromkeys(keys): try: - metadata = await self.store.get_metadata(key) - threads = metadata.get(THREADS_KEY) if metadata else None - if not isinstance(threads, list) or thread_id not in threads: - continue - remaining = [t for t in threads if t != thread_id] - if remaining or not delete: - await self._update_metadata(key, {THREADS_KEY: remaining}) - else: + last_link = False + + def drop_link(metadata: dict[str, Any]) -> dict[str, Any] | None: + nonlocal last_link + last_link = False + threads = metadata.get(THREADS_KEY) + if not isinstance(threads, list) or thread_id not in threads: + return None + remaining = [t for t in threads if t != thread_id] + if not remaining and delete: + last_link = True # deleted below, not rewritten + return None + return {THREADS_KEY: remaining} + + await self._modify_metadata(key, drop_link) + if last_link: await self.delete_file(key) except Exception as exc: + failed.append(key) logger.warning("Could not release file %s from a thread: %s", log_safe(key), exc) + return failed + + async def release_thread(self, thread_id: str, user_id: str | None) -> list[str]: + """Release every file ``user_id`` linked to ``thread_id``, whatever the messages say. + + ``unlink_threads`` works from the keys a thread's messages reference. A file linked + through a message that is gone since (cleared state, replaced messages) would stay + linked to a deleted thread forever; this finds those by scanning the store's metadata + (needs ``list_keys()`` or ``list_keys_for_owner()``, so stores that cannot + enumerate are skipped). Same rules as ``unlink_threads``: a file left with no thread is + deleted only with ``media.delete_with_thread`` on. Never raises; returns the keys that + could not be released. + """ + if not self.supports_links or user_id is None: + return [] + try: + all_keys = await self._keys_of(user_id) + except NotImplementedError: + return [] + except Exception as exc: + logger.warning("Could not list files for thread %s: %s", log_safe(thread_id), exc) + return ["*"] + linked: list[str] = [] + failed: list[str] = [] + for key in all_keys: + try: + metadata = await self.store.get_metadata(key) + except Exception as exc: + failed.append(key) + logger.warning("Could not read file %s to release it: %s", log_safe(key), exc) + continue + threads = metadata.get(THREADS_KEY) if metadata else None + if ( + isinstance(threads, list) + and thread_id in threads + and str(metadata.get("owner_id")) == str(user_id) + ): + linked.append(key) + failed.extend(await self.unlink_threads(linked, thread_id)) + return failed async def delete_file(self, file_id: str) -> bool: """Delete a file's bytes and metadata; False when it does not exist.""" + mime_type: str | None = None + try: + metadata = await self.store.get_metadata(file_id) + mime_type = metadata.get("mime_type") if metadata else None + except Exception: + metadata = None deleted = bool(await self.store.delete(file_id)) - self._extraction_cache.pop(file_id, None) + await self._forget_cached(file_id, mime_type) return deleted + async def _forget_cached(self, file_id: str, mime_type: str | None) -> None: + """Drop the file's extracted text and signed URLs from the local and shared caches. + + The shared (checkpointer) entries outlive the file for 24 h otherwise, so a deleted + file's text could still be read back. Never raises. + """ + self._extraction_cache.pop(file_id, None) + prefix = f"{file_id}:" + for key in [k for k in self._signed_url_cache if str(k).startswith(prefix)]: + self._signed_url_cache.pop(key, None) + checkpointer = self.checkpointer + if checkpointer is None: + return + keys = [(_EXTRACTION_NAMESPACE, file_id)] + signed_keys: list[str] = [] + try: # every ttl variant of the key; not all backends can list keys + signed_keys = list( + await checkpointer.alist_cache_keys(_SIGNED_URL_NAMESPACE, prefix=prefix) or [] + ) + except Exception as exc: + logger.debug("Could not list signed-URL cache keys: %s", exc) + if mime_type: + ttl = self._settings.MEDIA_SIGNED_URL_TTL_SECONDS + signed_keys.append(f"{file_id}:{mime_type}:{ttl}") + keys += [(_SIGNED_URL_NAMESPACE, key) for key in dict.fromkeys(signed_keys)] + for namespace, key in keys: + try: + await checkpointer.aclear_cache_value(namespace, key) + except Exception as exc: + logger.warning( + "Could not clear cached %s for %s: %s", namespace, log_safe(file_id), exc + ) + async def list_files( self, user_id: str | None, offset: int = 0, limit: int = 50 ) -> tuple[list[dict[str, Any]], bool]: @@ -438,10 +684,7 @@ async def list_files( Raises: NotImplementedError: The media store cannot enumerate its files. """ - list_keys = getattr(self.store, "list_keys", None) # absent in core older than 1.0 - if not callable(list_keys): - raise NotImplementedError(f"{type(self.store).__name__} cannot list files") - keys = await list_keys() + keys = await self._keys_of(user_id) owned: list[dict[str, Any]] = [] for key in keys: try: diff --git a/tenxgraph_api/src/app/routers/media/router.py b/tenxgraph_api/src/app/routers/media/router.py index a392c107..f2dd7f42 100644 --- a/tenxgraph_api/src/app/routers/media/router.py +++ b/tenxgraph_api/src/app/routers/media/router.py @@ -9,6 +9,7 @@ from fastapi.logger import logger from fastapi.responses import FileResponse, Response from injectq.integrations import InjectAPI +from starlette.datastructures import UploadFile as StarletteUploadFile from tenxgraph_api.src.app.core.auth.permissions import RequirePermission from tenxgraph_api.src.app.core.auth.request_config import client_error_detail @@ -127,6 +128,26 @@ def _file_download_response(file_id: str, path: Any, mime_type: str | None) -> R return FileResponse(path, media_type=mime, headers=headers) +# One instance, so FastAPI resolves the permission check once per request and the file +# dependency below runs only after it passed. +_UPLOAD_PERMISSION = RequirePermission("files", "upload") + + +async def _uploaded_file( + request: Request, _user: dict[str, Any] = Depends(_UPLOAD_PERMISSION) +) -> UploadFile: + """The multipart ``file`` part, parsed only once the caller is authorized. + + A ``file: UploadFile`` parameter would make FastAPI parse (and spool) the body before any + dependency runs, so an unauthenticated caller could force that work. + """ + form = await request.form() + file = form.get("file") + if not isinstance(file, StarletteUploadFile): + raise HTTPException(status_code=422, detail="multipart field 'file' is required") + return file # type: ignore[return-value] + + # ------------------------------------------------------------------ # 4.1 POST /v1/files/upload # ------------------------------------------------------------------ @@ -138,12 +159,26 @@ def _file_download_response(file_id: str, path: Any, mime_type: str | None) -> R "Accepts a multipart file upload. Stores binary in the configured " "MediaStore and optionally extracts text for documents." ), + openapi_extra={ + "requestBody": { + "required": True, + "content": { + "multipart/form-data": { + "schema": { + "type": "object", + "required": ["file"], + "properties": {"file": {"type": "string", "format": "binary"}}, + } + } + }, + } + }, ) async def upload_file( request: Request, - file: UploadFile, + file: UploadFile = Depends(_uploaded_file), service: MediaService = InjectAPI(MediaService), - user: dict[str, Any] = Depends(RequirePermission("files", "upload")), + user: dict[str, Any] = Depends(_UPLOAD_PERMISSION), ): if file.filename is None: raise HTTPException(status_code=400, detail="filename is required") diff --git a/tenxgraph_api/src/app/routers/ping/readiness.py b/tenxgraph_api/src/app/routers/ping/readiness.py index f3400270..3c7631e5 100644 --- a/tenxgraph_api/src/app/routers/ping/readiness.py +++ b/tenxgraph_api/src/app/routers/ping/readiness.py @@ -56,6 +56,22 @@ async def _own_check(component: Any) -> bool: return True +async def _check_schema_version(checkpointer: Any) -> None: + """Fail while the database schema is older than the one this code expects. + + Reads the recorded version without creating anything (``aget_schema_version``), so a + missing or half-upgraded schema (boot setup failed, or the database was restored from an + older backup) keeps the pod out of rotation instead of failing every request. + """ + read_version = getattr(checkpointer, "aget_schema_version", None) + expected_fn = getattr(checkpointer, "_get_current_schema_version", None) + if not callable(read_version) or not callable(expected_fn): + return + found, expected = await read_version(), expected_fn() + if found < expected: + raise RuntimeError(f"checkpointer schema version {found} is below the expected {expected}") + + async def check_checkpointer(checkpointer: Any) -> None: if checkpointer is None: raise _Skip @@ -64,6 +80,7 @@ async def check_checkpointer(checkpointer: Any) -> None: run_query = getattr(checkpointer, "_run_query", None) if callable(run_query): # PgCheckpointer: one round trip on its pool await run_query(lambda conn: conn.fetchval("SELECT 1")) + await _check_schema_version(checkpointer) redis = getattr(checkpointer, "redis", None) if redis is not None and callable(getattr(redis, "ping", None)): # its cache tier await _maybe_await(redis.ping()) diff --git a/tenxgraph_api/src/app/routers/store/router.py b/tenxgraph_api/src/app/routers/store/router.py index f4970909..77df685f 100644 --- a/tenxgraph_api/src/app/routers/store/router.py +++ b/tenxgraph_api/src/app/routers/store/router.py @@ -99,7 +99,9 @@ async def list_memories( payload = ListMemoriesSchema() cfg = payload.config or {} opts = payload.options - result = await service.list_memories(cfg, user, limit=payload.limit, options=opts) + result = await service.list_memories( + cfg, user, limit=payload.limit, options=opts, offset=payload.offset + ) return success_response(result, request) diff --git a/tenxgraph_api/src/app/routers/store/schemas/store_schemas.py b/tenxgraph_api/src/app/routers/store/schemas/store_schemas.py index c583e087..ebba2a35 100644 --- a/tenxgraph_api/src/app/routers/store/schemas/store_schemas.py +++ b/tenxgraph_api/src/app/routers/store/schemas/store_schemas.py @@ -14,11 +14,14 @@ RetrievalStrategy, ) +from tenxgraph_api.src.app.core.auth.request_config import BoundedDict + # Upper bounds so one request cannot make the store (and the embedding / vector backend) # return or rank an unbounded number of rows. MAX_SEARCH_LIMIT = 100 MAX_LIST_LIMIT = 1000 +MAX_LIST_OFFSET = 100_000 MAX_SEARCH_TOKENS = 16_000 # Character caps on free text. The text is embedded, so an unbounded body costs embedding # tokens and memory. A query is a short question; stored content may be a long note. @@ -31,11 +34,11 @@ class BaseConfigSchema(BaseModel): """Base schema containing configuration overrides and store options.""" - config: dict[str, Any] | None = Field( + config: BoundedDict | None = Field( default_factory=dict, description="Configuration values forwarded to the store backend.", ) - options: dict[str, Any] | None = Field( + options: BoundedDict | None = Field( default=None, description="Extra keyword arguments to forward to the store backend.", ) @@ -50,7 +53,7 @@ class StoreMemorySchema(BaseConfigSchema): description="Memory classification used by the backend store.", ) category: str = Field(default="general", description="Category label for the memory.") - metadata: dict[str, Any] | None = Field( + metadata: BoundedDict | None = Field( default=None, description="Arbitrary metadata associated with the memory.", ) @@ -82,7 +85,7 @@ class SearchMemorySchema(BaseConfigSchema): default=None, description="Minimum similarity score required for results.", ) - filters: dict[str, Any] | None = Field( + filters: BoundedDict | None = Field( default=None, description="Additional store-specific filters.", ) @@ -106,7 +109,7 @@ class UpdateMemorySchema(BaseConfigSchema): """Schema for updating a memory.""" content: MemoryText | Message = Field(..., description="Updated memory content or message.") - metadata: dict[str, Any] | None = Field( + metadata: BoundedDict | None = Field( default=None, description="Updated metadata for the memory.", ) @@ -129,6 +132,12 @@ class ListMemoriesSchema(BaseConfigSchema): le=MAX_LIST_LIMIT, description="Maximum number of memories to return.", ) + offset: int = Field( + default=0, + ge=0, + le=MAX_LIST_OFFSET, + description="Number of memories to skip, for paging. Use next_offset from the response.", + ) class ForgetMemorySchema(BaseConfigSchema): @@ -142,7 +151,7 @@ class ForgetMemorySchema(BaseConfigSchema): default=None, description="Optional category to target for deletion.", ) - filters: dict[str, Any] | None = Field( + filters: BoundedDict | None = Field( default=None, description="Additional filters to control which memories are forgotten.", ) @@ -170,6 +179,14 @@ class MemoryListResponseSchema(BaseModel): default_factory=list, description="Collection of memories returned from the store.", ) + has_more: bool = Field( + default=False, + description="True when more memories follow this page.", + ) + next_offset: int | None = Field( + default=None, + description="Offset to send for the next page, or null on the last page.", + ) class MemorySearchResponseSchema(BaseModel): diff --git a/tenxgraph_api/src/app/routers/store/services/store_service.py b/tenxgraph_api/src/app/routers/store/services/store_service.py index 72ea5d4b..33e0579f 100644 --- a/tenxgraph_api/src/app/routers/store/services/store_service.py +++ b/tenxgraph_api/src/app/routers/store/services/store_service.py @@ -165,11 +165,20 @@ async def list_memories( user: dict[str, Any], limit: int = 100, options: dict[str, Any] | None = None, + offset: int = 0, ) -> MemoryListResponseSchema: + """One page of memories. The store has no offset, so it is asked for ``offset + limit + + 1`` rows and the page is cut from those; the extra row only tells ``has_more``.""" store = self._get_store() cfg = self._config(config, user) - memories = await store.aget_all(cfg, limit=limit, **_client_options(options)) - return MemoryListResponseSchema(memories=memories) + rows = await store.aget_all(cfg, limit=offset + limit + 1, **_client_options(options)) + page = list(rows[offset : offset + limit]) + has_more = len(rows) > offset + limit + return MemoryListResponseSchema( + memories=page, + has_more=has_more, + next_offset=offset + limit if has_more else None, + ) async def update_memory( self, diff --git a/tenxgraph_api/src/app/utils/media/extractor.py b/tenxgraph_api/src/app/utils/media/extractor.py index 1738ad7f..88420fc7 100644 --- a/tenxgraph_api/src/app/utils/media/extractor.py +++ b/tenxgraph_api/src/app/utils/media/extractor.py @@ -10,6 +10,7 @@ import asyncio import io import logging +import re import zipfile from pathlib import PurePosixPath from typing import Any @@ -57,8 +58,7 @@ # # Markup and text parsers expand their input: a 12 MB ``text/html`` file used about 1.2 GB RSS # (roughly 100x) and a 20 MB one took 32.8 s of CPU. 5 MB keeps one parse near 500 MB and -# inside the timeout. PDF and DOCX are not capped here: they are bounded by the upload cap and, -# for DOCX, by the archive checks above. +# inside the timeout. DOCX is bounded by the archive checks above and PDF by the caps below. MAX_TEXT_DOCUMENT_BYTES = 5 * 1024 * 1024 _SIZE_CAPPED_EXTENSIONS = frozenset({".html", ".xml", ".csv", ".md", ".json", ".txt"}) # At most this many extractions run (or are still running after a timeout) at once. @@ -66,6 +66,44 @@ MAX_EXTRACTED_CHARS = 1_000_000 +# PDF caps. A 0.9 MB PDF can inflate to ~1 GB while parsed and a 100,000-page one outlives the +# timeout, so both are checked before any parser sees the bytes. The page count is read from the +# raw bytes (page objects and page-tree ``/Count`` values), with no parsing: a PDF that hides +# its page tree in compressed object streams is bounded by the size cap alone. +MAX_PDF_BYTES = 10 * 1024 * 1024 +MAX_PDF_PAGES = 500 +_PDF_PAGE_RE = re.compile(rb"/Type\s*/Page(?![A-Za-z])") +_PDF_COUNT_RE = re.compile(rb"/Count\s+(\d{1,9})") + + +class DocumentTooLargeError(ValueError): + """A document is over a configured size or page limit (the upload route answers 413).""" + + +def pdf_page_estimate(data: bytes) -> int: + """Best-effort page count read from the raw bytes, without parsing the PDF.""" + pages = len(_PDF_PAGE_RE.findall(data)) + counts = [int(m) for m in _PDF_COUNT_RE.findall(data)] + return max([pages, *counts]) + + +def check_pdf_limits(data: bytes, max_bytes: int | None, max_pages: int | None) -> None: + """Raise :class:`DocumentTooLargeError` when a PDF is over the size or page cap. + + ``None`` or ``0`` disables a cap. Runs before parsing. + """ + if max_bytes and len(data) > max_bytes: + raise DocumentTooLargeError( + f"PDF is {len(data)} bytes; the limit for text extraction is {max_bytes} bytes" + ) + if max_pages: + pages = pdf_page_estimate(data) + if pages > max_pages: + raise DocumentTooLargeError( + f"PDF has about {pages} pages; the limit for text extraction is {max_pages} pages" + ) + + def _handler_extension(filename: str | None, mime_type: str | None) -> str | None: """The extension (and so the handler) to extract with, or None to skip extraction.""" if mime_type: @@ -115,7 +153,14 @@ class DocumentExtractor: text = await extractor.extract(pdf_bytes, "report.pdf") """ - def __init__(self, extractor: Any | None = None): + def __init__( + self, + extractor: Any | None = None, + max_pdf_bytes: int | None = MAX_PDF_BYTES, + max_pdf_pages: int | None = MAX_PDF_PAGES, + ): + self.max_pdf_bytes = max_pdf_bytes + self.max_pdf_pages = max_pdf_pages if extractor is not None: self.extractor = extractor return @@ -128,6 +173,15 @@ def __init__(self, extractor: Any | None = None): self.extractor = AsyncTextExtractor() + def check_limits(self, data: bytes, mime_type: str | None) -> None: + """Raise :class:`DocumentTooLargeError` when ``data`` is a PDF over the caps. + + Lets the upload route refuse the file up front instead of storing it and then skipping + its text. + """ + if _handler_extension(None, mime_type) == ".pdf": + check_pdf_limits(data, self.max_pdf_bytes, self.max_pdf_pages) + def _slots(self) -> asyncio.Semaphore: slots = getattr(self, "_slot_semaphore", None) if slots is None: @@ -196,6 +250,8 @@ async def extract( "Not extracting %s (%s): not a supported document", filename, mime_type ) return None + if extension == ".pdf": + check_pdf_limits(data, self.max_pdf_bytes, self.max_pdf_pages) if extension == ".docx": _check_zip_container(data) if extension in _SIZE_CAPPED_EXTENSIONS and len(data) > MAX_TEXT_DOCUMENT_BYTES: diff --git a/tenxgraph_api/src/app/uvicorn_worker.py b/tenxgraph_api/src/app/uvicorn_worker.py index 9ef60f57..2a0ffd8e 100644 --- a/tenxgraph_api/src/app/uvicorn_worker.py +++ b/tenxgraph_api/src/app/uvicorn_worker.py @@ -28,8 +28,8 @@ # Seconds uvicorn waits for open responses after SIGTERM. Must stay below the gunicorn -# ``--graceful-timeout`` of the generated image (600) with room for the lifespan drain -# (30 s) and ``graph.aclose``. +# ``--graceful-timeout`` of the generated image (600) with room for the lifespan close +# (``graph.aclose``: the three SHUTDOWN_*_SECONDS settings, 50 s in total by default). DEFAULT_GRACEFUL_SHUTDOWN_SECONDS = 540 # Open streams are told to finish this long before uvicorn would cancel them. STREAM_INTERRUPT_MARGIN_SECONDS = 5.0 diff --git a/tests/cli/test_group6_small_cli_bugs.py b/tests/cli/test_group6_small_cli_bugs.py index fac0acd0..1685083d 100644 --- a/tests/cli/test_group6_small_cli_bugs.py +++ b/tests/cli/test_group6_small_cli_bugs.py @@ -90,3 +90,40 @@ def fail(*_a, **_k): with pytest.raises(OSError): cmd._install_one(templates_root, tmp_path, target, force=True) assert marker.read_text(encoding="utf-8") == "old" + + +def test_build_unexpected_error_keeps_its_cause(tmp_path: Path, monkeypatch) -> None: + from tenxgraph_api.cli.commands import build as build_module + from tenxgraph_api.cli.exceptions import DockerError + + monkeypatch.chdir(tmp_path) + original = RuntimeError("disk exploded") + + def boom(*_a, **_k): + raise original + + monkeypatch.setattr(build_module, "generate_dockerfile_content", boom) + cmd = build_module.BuildCommand(output=_out()) + seen: list[Exception] = [] + monkeypatch.setattr(cmd, "handle_error", lambda e: seen.append(e) or 1) + assert cmd.execute(output_file="Dockerfile", force=True) == 1 + assert isinstance(seen[0], DockerError) + assert seen[0].__cause__ is original + + +def test_handle_error_does_not_print_the_error_twice(caplog) -> None: + import logging + + from tenxgraph_api.cli.commands.build import BuildCommand + + cmd = BuildCommand(output=_out()) + with caplog.at_level(logging.INFO, logger="tenxgraph_api.cli"): + cmd.handle_error(RuntimeError("only once")) + assert [r for r in caplog.records if r.levelno >= logging.INFO] == [] + + +def test_generated_dockerfile_comment_is_not_stale() -> None: + from tenxgraph_api.cli.templates import defaults + + source = Path(defaults.__file__).read_text(encoding="utf-8") + assert "belongs to someone else" not in source diff --git a/tests/integration_tests/conftest.py b/tests/integration_tests/conftest.py index 7dbce43d..bb12a6d8 100644 --- a/tests/integration_tests/conftest.py +++ b/tests/integration_tests/conftest.py @@ -25,6 +25,7 @@ from tenxgraph_api.src.app.core.config.graph_config import GraphConfig from tenxgraph_api.src.app.core.config.media_settings import MediaSettings, MediaStorageType from tenxgraph_api.src.app.core.config.setup_middleware import setup_middleware +from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler class HeaderAuth(BaseAuth): @@ -109,6 +110,8 @@ def build_app( """Assemble a test app with real auth/authz wiring and the given routers.""" app = FastAPI() setup_middleware(app) + # Same handlers as main.py, so a ClientInputError answers 422 as in production. + init_errors_handler(app) container = InjectQ() container.bind_instance(GraphConfig, _AuthOnConfig()) diff --git a/tests/integration_tests/test_idempotency_key.py b/tests/integration_tests/test_idempotency_key.py new file mode 100644 index 00000000..a16e2b77 --- /dev/null +++ b/tests/integration_tests/test_idempotency_key.py @@ -0,0 +1,210 @@ +"""G-107: opt-in Idempotency-Key on invoke, stream and WebSocket runs.""" + +# ruff: noqa: S101, PLR2004 + +from __future__ import annotations + +import asyncio +import json +from unittest.mock import AsyncMock, MagicMock + +import pytest +from tenxgraph.core.state import StreamChunk, StreamEvent + +from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.routers.graph import idempotency +from tenxgraph_api.src.app.routers.graph.router import router as graph_router +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInvokeOutputSchema +from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + +from .conftest import build_app, make_client, user_headers + + +MESSAGES = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] +OTHER = [{"role": "user", "content": [{"type": "text", "text": "different"}]}] +KEY = {"Idempotency-Key": "k-1"} + + +@pytest.fixture(autouse=True) +def _clean(): + idempotency._memory.clear() + yield + idempotency._memory.clear() + + +def _client(): + runs = {"count": 0} + + async def _stream(*_a, **_k): + runs["count"] += 1 + yield json.dumps({"event": "message", "thread_id": "t1"}, separators=(",", ":")) + "\n" + state = StreamChunk(event=StreamEvent.STATE, thread_id="t1").model_dump_json() + yield state + "\n" + done = StreamChunk(event=StreamEvent.UPDATES, data={"status": "done"}, thread_id="t1") + yield done.model_dump_json() + "\n" + + async def _invoke(*_a, **_k): + runs["count"] += 1 + return GraphInvokeOutputSchema(messages=[], meta={"thread_id": "t1"}) + + service = MagicMock(spec=GraphService) + service.check_media_urls = MagicMock() + service.check_resume_target = AsyncMock() + service.runs_policy = MagicMock(return_value=RunsConfig()) + service.invoke_graph = AsyncMock(side_effect=_invoke) + service.stream_graph = MagicMock(side_effect=_stream) + app = build_app(routers=[graph_router], extra_bindings={GraphService: service}) + return make_client(app), runs, service + + +def _post(client, path, key=KEY, messages=MESSAGES, user="alice"): + headers = {**user_headers(user), **(key or {})} + return client.post(path, json={"messages": messages}, headers=headers) + + +def test_invoke_same_key_returns_stored_result_without_running_again(): + client, runs, _ = _client() + first = _post(client, "/v1/graph/invoke") + second = _post(client, "/v1/graph/invoke") + assert first.status_code == second.status_code == 200 + assert runs["count"] == 1 + assert second.json()["data"]["meta"] == first.json()["data"]["meta"] + + +def test_no_header_runs_every_time(): + client, runs, _ = _client() + _post(client, "/v1/graph/invoke", key=None) + _post(client, "/v1/graph/invoke", key=None) + assert runs["count"] == 2 + + +def test_different_body_under_the_same_key_is_422(): + client, runs, _ = _client() + _post(client, "/v1/graph/invoke") + resp = _post(client, "/v1/graph/invoke", messages=OTHER) + assert resp.status_code == 422 + assert runs["count"] == 1 + + +def test_key_is_per_user(): + client, runs, _ = _client() + _post(client, "/v1/graph/invoke", user="alice") + _post(client, "/v1/graph/invoke", user="bob") + assert runs["count"] == 2 + + +def test_malformed_key_is_422(): + client, _, _ = _client() + resp = _post(client, "/v1/graph/invoke", key={"Idempotency-Key": "a" * 300}) + assert resp.status_code == 422 + + +def test_failed_run_frees_the_key(): + client, runs, service = _client() + service.invoke_graph = AsyncMock(side_effect=RuntimeError("boom")) + assert _post(client, "/v1/graph/invoke").status_code == 500 + service.invoke_graph = AsyncMock(return_value=GraphInvokeOutputSchema(messages=[])) + assert _post(client, "/v1/graph/invoke").status_code == 200 + + +def test_stream_replays_final_state_and_done(): + client, runs, _ = _client() + first = _post(client, "/v1/graph/stream") + second = _post(client, "/v1/graph/stream") + assert runs["count"] == 1 + lines = [json.loads(x) for x in second.text.splitlines()] + assert [x["event"] for x in lines] == ["state", "updates"] + assert lines[-1]["data"] == {"status": "done"} + assert first.text.count("\n") == 3 + + +@pytest.mark.asyncio +async def test_running_key_is_409_with_retry_after(): + user = {"user_id": "alice"} + run, record = await idempotency.check(user, "k", "invoke", {"a": 1}) + assert run is not None and record is None + with pytest.raises(idempotency.IdempotencyConflict) as exc: + await idempotency.check(user, "k", "invoke", {"a": 1}) + assert exc.value.status_code == 409 + assert exc.value.headers["Retry-After"] + await run.complete("{}") + run.release() + again, stored = await idempotency.check(user, "k", "invoke", {"a": 1}) + assert again is None and stored["stored"] is True + + +@pytest.mark.asyncio +async def test_oversized_result_is_marked_not_stored(): + user = {"user_id": "alice"} + run, _ = await idempotency.check(user, "big", "invoke", {}) + await run.complete("x" * (idempotency.MAX_STORED_BYTES + 1)) + run.release() + _, record = await idempotency.check(user, "big", "invoke", {}) + assert record["stored"] is False + + +@pytest.mark.asyncio +async def test_redis_records_use_the_shared_client(monkeypatch): + store: dict[str, tuple[str, int]] = {} + client = MagicMock() + client.get = AsyncMock(side_effect=lambda name: store.get(name, (None,))[0]) + + async def _set(name, value, ex=None): + store[name] = (value, ex) + + client.set = _set + monkeypatch.setattr(idempotency, "_redis_client", lambda: client) + user = {"user_id": "alice"} + run, _ = await idempotency.check(user, "r", "invoke", {}) + await run.complete("{}") + run.release() + (name, (_, ttl)) = next(iter(store.items())) + assert "idempotency" in name and ":alice:r" in name + assert ttl == 86400 + assert asyncio.iscoroutinefunction(client.get) or client.get.await_count >= 1 + + +def test_ws_frame_carries_the_key(): + from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import WsGraphInputSchema + + parsed = WsGraphInputSchema(messages=MESSAGES, idempotency_key="abc") + assert parsed.idempotency_key == "abc" + assert WsGraphInputSchema(messages=MESSAGES).idempotency_key is None + + +@pytest.mark.asyncio +async def test_ag_ui_repeat_of_a_finished_key_is_409_and_other_body_422(): + pytest.importorskip("ag_ui") + from types import SimpleNamespace + + from fastapi import HTTPException + + from tenxgraph_api.src.app.routers.ag_ui.router import run_ag_ui + + body = { + "threadId": "t1", + "runId": "r1", + "state": {}, + "messages": [{"id": "m1", "role": "user", "content": "hi"}], + "tools": [], + "context": [], + "forwardedProps": {}, + } + user = {"user_id": "alice"} + run, _ = await idempotency.check(user, "ak", "ag-ui", body) + await run.complete(None) + run.release() + + def request(payload): + req = MagicMock() + req.json = AsyncMock(return_value=payload) + req.headers = {"Idempotency-Key": "ak"} + return req + + config = SimpleNamespace(auth_config=lambda: None, ag_ui=SimpleNamespace()) + with pytest.raises(HTTPException) as same: + await run_ag_ui(request(body), MagicMock(), user, MagicMock(), config) + assert same.value.status_code == 409 + with pytest.raises(HTTPException) as other: + await run_ag_ui(request({**body, "runId": "r2"}), MagicMock(), user, MagicMock(), config) + assert other.value.status_code == 422 diff --git a/tests/integration_tests/test_server_thread_ids.py b/tests/integration_tests/test_server_thread_ids.py index dc75c95b..31823905 100644 --- a/tests/integration_tests/test_server_thread_ids.py +++ b/tests/integration_tests/test_server_thread_ids.py @@ -11,7 +11,7 @@ import importlib from types import SimpleNamespace -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import anyio import pytest @@ -150,8 +150,8 @@ def test_put_state_on_an_unknown_id_is_refused_by_default(): @pytest.mark.asyncio -async def test_ws_run_refuses_an_unknown_id_but_live_accepts_it(monkeypatch): - """/v1/graph/ws follows the server mode; /v1/graph/live passes client_thread_ids.""" +async def test_ws_and_live_refuse_an_unknown_id_in_server_mode(monkeypatch): + """G-113: /v1/graph/ws and /v1/graph/live follow runs.thread_ids like REST.""" from injectq import InjectQ container = MagicMock() @@ -165,14 +165,13 @@ async def test_ws_run_refuses_an_unknown_id_but_live_accepts_it(monkeypatch): check = graph_router_module._ws_thread_authorized assert await check(authz, alice, "frontend-id", "stream") is False - assert await check(authz, alice, "frontend-id", "stream", client_thread_ids=True) is True - # Fresh threads and owned threads are unchanged; a foreign thread stays refused on live. + # Fresh threads and owned threads are unchanged; a foreign thread stays refused. assert await check(authz, alice, None, "stream") is True assert await check(authz, alice, "taken", "stream") is True - assert ( - await check(authz, {"user_id": "mallory"}, "taken", "stream", client_thread_ids=True) - is False - ) + assert await check(authz, {"user_id": "mallory"}, "taken", "stream") is False + # Opting in with runs.thread_ids "client" lets the caller name a new thread. + opted = _backend(checkpointer, thread_ids="client") + assert await check(opted, alice, "frontend-id", "stream") is True @pytest.mark.asyncio @@ -199,7 +198,7 @@ async def test_the_caller_owns_a_frontend_id_after_claiming_it(): @pytest.mark.asyncio -async def test_ag_ui_asks_the_backend_with_client_thread_ids(): +async def test_ag_ui_asks_the_backend_without_the_client_exemption(): pytest.importorskip("ag_ui") from tenxgraph_api.src.app.routers.ag_ui.router import run_ag_ui @@ -227,5 +226,57 @@ async def test_ag_ui_asks_the_backend_with_client_thread_ids(): await run_ag_ui(request, MagicMock(), {"user_id": "alice"}, authz, config) assert exc.value.status_code == HTTP_FORBIDDEN authz.authorize.assert_awaited_once_with( - {"user_id": "alice"}, "graph", "stream", resource_id="copilot-9", client_thread_ids=True + {"user_id": "alice"}, "graph", "stream", resource_id="copilot-9" + ) + + +def _ag_ui_request(thread_id): + request = MagicMock() + request.json = AsyncMock( + return_value={ + "threadId": thread_id, + "runId": "r1", + "state": {}, + "messages": [{"id": "m1", "role": "user", "content": "hi"}], + "tools": [], + "context": [], + "forwardedProps": {}, + } + ) + request.headers = {} + return request + + +@pytest.mark.asyncio +async def test_ag_ui_blank_thread_id_gets_a_server_id_without_authz(): + pytest.importorskip("ag_ui") + from tenxgraph_api.src.app.routers.ag_ui.router import run_ag_ui + + service = MagicMock() + service.new_thread_id = AsyncMock(return_value="srv-1") + service.check_media_urls = MagicMock() + authz = MagicMock() + authz.authorize = AsyncMock(return_value=False) + config = SimpleNamespace( + auth_config=lambda: "jwt", ag_ui=SimpleNamespace(allow_client_tools=True) ) + with patch( + "tenxgraph_api.src.app.routers.ag_ui.router.acquire_run_guard", new=AsyncMock() + ) as guard: + response = await run_ag_ui(_ag_ui_request(""), service, {"user_id": "a"}, authz, config) + assert response.status_code == HTTP_OK + authz.authorize.assert_not_awaited() + assert guard.await_args.kwargs["thread_id"] == "srv-1" + + +@pytest.mark.asyncio +async def test_ag_ui_bad_thread_id_is_a_validation_error(): + pytest.importorskip("ag_ui") + from tenxgraph_api.src.app.core.auth.request_config import ClientInputError + from tenxgraph_api.src.app.routers.ag_ui.router import run_ag_ui + + config = SimpleNamespace(auth_config=lambda: None, ag_ui=SimpleNamespace()) + with pytest.raises(ClientInputError): + await run_ag_ui( + _ag_ui_request("a/b"), MagicMock(), {"user_id": "a"}, MagicMock(), config + ) diff --git a/tests/integration_tests/test_upload_auth_order.py b/tests/integration_tests/test_upload_auth_order.py new file mode 100644 index 00000000..8ac71072 --- /dev/null +++ b/tests/integration_tests/test_upload_auth_order.py @@ -0,0 +1,41 @@ +"""G-127: a multipart upload is refused (401) before its body is parsed.""" + +# ruff: noqa: S101 + +from starlette.requests import Request + +from tenxgraph_api.src.app.routers.media import MediaService +from tenxgraph_api.src.app.routers.media.router import router as media_router + +from .conftest import build_app, make_client + + +def _app(settings): + return build_app( + routers=[media_router], extra_bindings={MediaService: MediaService(settings=settings)} + ) + + +def test_unauthenticated_upload_is_not_parsed(monkeypatch, memory_media_settings): + parsed = [] + original = Request.form + + async def spy(self, *args, **kwargs): + parsed.append(True) + return await original(self, *args, **kwargs) + + monkeypatch.setattr(Request, "form", spy) + client = make_client(_app(memory_media_settings)) + resp = client.post("/v1/files/upload", files={"file": ("a.txt", b"hello")}) + assert resp.status_code == 401 + assert parsed == [] + + +def test_missing_file_field_is_422(memory_media_settings): + client = make_client(_app(memory_media_settings)) + resp = client.post( + "/v1/files/upload", + files={"other": ("a.txt", b"hello")}, + headers={"X-Test-User": "alice"}, + ) + assert resp.status_code == 422 diff --git a/tests/unit_tests/auth/test_graph_config_auth.py b/tests/unit_tests/auth/test_graph_config_auth.py index 0d050b4e..4b00d8f7 100644 --- a/tests/unit_tests/auth/test_graph_config_auth.py +++ b/tests/unit_tests/auth/test_graph_config_auth.py @@ -99,8 +99,7 @@ def test_auth_config_jwt_string_with_jwt_substring( self, temp_config_file, ): - """Test that auth config works with strings containing 'jwt' substring.""" - # The code uses 'jwt' in res, so "my-jwt-auth" would also match + """A string that only contains 'jwt' is not the jwt method (G-117: exact match).""" config_path = temp_config_file({"agent": "path/to/agent.py", "auth": "my-jwt-auth"}) with patch.dict( @@ -111,9 +110,8 @@ def test_auth_config_jwt_string_with_jwt_substring( }, ): graph_config = GraphConfig(path=config_path) - result = graph_config.auth_config() - - assert result == {"method": "jwt"} + with pytest.raises(ValueError, match="Unsupported auth method"): + graph_config.auth_config() def test_auth_config_jwt_uppercase_raises_error( self, diff --git a/tests/unit_tests/auth/test_jwt_hardening.py b/tests/unit_tests/auth/test_jwt_hardening.py index c6f13ee1..e04d6bd2 100644 --- a/tests/unit_tests/auth/test_jwt_hardening.py +++ b/tests/unit_tests/auth/test_jwt_hardening.py @@ -86,3 +86,47 @@ def test_asymmetric_algorithms_are_not_length_checked(): def test_strong_secret_passes(): check_jwt_settings(_settings(MODE="production")) + + +# G-117: algorithm allowlist, PyJWT checked at boot, exact auth parse + + +@pytest.mark.parametrize("algorithm", ["none", "NONE", "None", "", "HS1", "hs256", "RS999"]) +def test_unknown_or_none_algorithm_is_refused_at_boot(algorithm): + with pytest.raises(ValueError, match="not allowed"): + check_jwt_settings(_settings(JWT_ALGORITHM=algorithm)) + + +@pytest.mark.parametrize("algorithm", ["HS256", "HS384", "HS512", "RS256", "ES256", "EdDSA"]) +def test_allowed_algorithms_pass_boot(algorithm): + check_jwt_settings(_settings(JWT_SECRET_KEY="x" * 64, JWT_ALGORITHM=algorithm)) + + +def test_algorithm_none_never_authenticates(): + credential = HTTPAuthorizationCredentials( + scheme="Bearer", credentials=jwt.encode({"user_id": "u1"}, None, algorithm="none") + ) + with patch( + "tenxgraph_api.src.app.core.auth.jwt_auth.get_settings", + return_value=_settings(JWT_ALGORITHM="none"), + ): + with pytest.raises(UserAccountError) as exc: + JwtAuth().authenticate(None, Response(), credential) + assert exc.value.status_code == 500 + + +def test_missing_pyjwt_fails_at_boot(): + with patch("tenxgraph_api.src.app.core.auth.jwt_auth.jwt", None): + with pytest.raises(ImportError, match="PyJWT is required"): + check_jwt_settings(_settings()) + + +def test_auth_string_is_matched_exactly(monkeypatch): + from tenxgraph_api.src.app.core.config.graph_config import GraphConfig + + monkeypatch.setenv("JWT_SECRET_KEY", SECRET) + monkeypatch.setenv("JWT_ALGORITHM", "HS256") + assert GraphConfig._parse_auth_config("jwt") == {"method": "jwt"} + for bad in ("JWT", " jwt", "no-jwt-here", "jwt_custom", "notjwt", "custom jwt"): + with pytest.raises(ValueError, match="Unsupported auth method"): + GraphConfig._parse_auth_config(bad) diff --git a/tests/unit_tests/auth/test_jwt_rotation.py b/tests/unit_tests/auth/test_jwt_rotation.py new file mode 100644 index 00000000..916ba7c7 --- /dev/null +++ b/tests/unit_tests/auth/test_jwt_rotation.py @@ -0,0 +1,115 @@ +"""JWT key rotation (JWT_SECRET_KEYS with kid) and the optional maximum token age.""" + +# ruff: noqa: S101, S105, S106, PLR2004 + +import time +from unittest.mock import patch + +import jwt +import pytest +from fastapi import Response +from fastapi.security import HTTPAuthorizationCredentials + +from tenxgraph_api.src.app.core.auth.jwt_auth import JwtAuth, check_jwt_settings +from tenxgraph_api.src.app.core.config.settings import Settings +from tenxgraph_api.src.app.core.exceptions import UserAccountError + + +OLD = "o" * 48 +NEW = "n" * 48 +OTHER = "x" * 48 + + +def _settings(**overrides) -> Settings: + values = { + "ALLOWED_HOST": "api.example.com", + "ORIGINS": "https://app.example.com", + "_env_file": None, + } + values.update(overrides) + return Settings(**values) + + +def _auth(secret: str, settings: Settings, headers: dict | None = None, **claims): + token = jwt.encode( + {"user_id": "u1", "exp": int(time.time()) + 60, **claims}, + secret, + algorithm="HS256", + headers=headers, + ) + credential = HTTPAuthorizationCredentials(scheme="Bearer", credentials=token) + with patch("tenxgraph_api.src.app.core.auth.jwt_auth.get_settings", return_value=settings): + return JwtAuth().authenticate(None, Response(), credential) + + +def _rotating() -> Settings: + return _settings( + JWT_SECRET_KEY=OLD, + JWT_SECRET_KEYS=[{"kid": "2026-10", "key": NEW}], + ) + + +def test_old_and_new_key_both_verify_during_rotation(): + settings = _rotating() + assert _auth(OLD, settings)["user_id"] == "u1" + assert _auth(NEW, settings, headers={"kid": "2026-10"})["user_id"] == "u1" + assert _auth(NEW, settings)["user_id"] == "u1" # no kid: every key is tried + + +def test_kid_selects_only_the_matching_key(): + settings = _settings(JWT_SECRET_KEYS=[{"kid": "a", "key": OLD}, {"kid": "b", "key": NEW}]) + with pytest.raises(UserAccountError) as exc: + _auth(OLD, settings, headers={"kid": "b"}) # signed with a's key but claims b + assert exc.value.status_code == 401 + assert _auth(OLD, settings, headers={"kid": "a"})["user_id"] == "u1" + + +def test_unknown_kid_falls_back_to_keys_without_kid_only(): + settings = _settings(JWT_SECRET_KEY=OLD, JWT_SECRET_KEYS=[{"kid": "a", "key": NEW}]) + assert _auth(OLD, settings, headers={"kid": "zzz"})["user_id"] == "u1" + with pytest.raises(UserAccountError): + _auth(NEW, settings, headers={"kid": "zzz"}) + + +def test_unlisted_key_is_refused(): + with pytest.raises(UserAccountError) as exc: + _auth(OTHER, _rotating()) + assert exc.value.status_code == 401 + + +def test_keys_list_alone_is_enough_and_plain_strings_work(): + settings = _settings(JWT_SECRET_KEYS=[NEW]) + assert _auth(NEW, settings)["user_id"] == "u1" + + +def test_expired_token_is_still_expired_with_several_keys(): + with pytest.raises(UserAccountError) as exc: + _auth(NEW, _rotating(), exp=int(time.time()) - 5) + assert exc.value.error_code == "EXPIRED_TOKEN" + + +def test_secret_key_alone_unchanged(): + settings = _settings(JWT_SECRET_KEY=OLD) + assert _auth(OLD, settings, headers={"kid": "anything"})["user_id"] == "u1" + + +def test_every_rotation_key_is_length_checked_in_production(): + settings = _settings( + MODE="production", JWT_SECRET_KEY=OLD, JWT_SECRET_KEYS=[{"kid": "a", "key": "short"}] + ) + with pytest.raises(ValueError, match="shorter"): + check_jwt_settings(settings) + + +def test_max_token_age_off_by_default(): + old = int(time.time()) - 10_000_000 + assert _auth(OLD, _settings(JWT_SECRET_KEY=OLD), iat=old)["user_id"] == "u1" + + +def test_max_token_age_refuses_old_and_missing_iat(): + settings = _settings(JWT_SECRET_KEY=OLD, JWT_MAX_TOKEN_AGE_SECONDS=300) + assert _auth(OLD, settings, iat=int(time.time()) - 10)["user_id"] == "u1" + for claims in ({"iat": int(time.time()) - 1000}, {}): + with pytest.raises(UserAccountError) as exc: + _auth(OLD, settings, **claims) + assert exc.value.status_code == 401 diff --git a/tests/unit_tests/store/test_store_service.py b/tests/unit_tests/store/test_store_service.py index 25f1cece..4ef89101 100644 --- a/tests/unit_tests/store/test_store_service.py +++ b/tests/unit_tests/store/test_store_service.py @@ -285,7 +285,7 @@ async def test_list_memories_default( assert len(result.memories) == 2 mock_store.aget_all.assert_called_once() call_args = mock_store.aget_all.call_args - assert call_args[1]["limit"] == 100 + assert call_args[1]["limit"] == 101 # one extra row tells has_more async def test_list_memories_custom_limit( self, store_service, mock_store, mock_user, sample_memory_results @@ -300,7 +300,7 @@ async def test_list_memories_custom_limit( # Assert assert len(result.memories) == 1 call_args = mock_store.aget_all.call_args - assert call_args[1]["limit"] == 1 + assert call_args[1]["limit"] == 2 # one extra row tells has_more async def test_list_memories_with_options( self, store_service, mock_store, mock_user, sample_memory_results diff --git a/tests/unit_tests/test_closing_and_cancel.py b/tests/unit_tests/test_closing_and_cancel.py new file mode 100644 index 00000000..a5b2b130 --- /dev/null +++ b/tests/unit_tests/test_closing_and_cancel.py @@ -0,0 +1,213 @@ +"""A run refused while the graph closes (BF3), client disconnect (BF6), cancelled run (BF7).""" + +# ruff: noqa: S101 + +import asyncio +import json +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import WebSocketDisconnect +from tenxgraph.core.graph import StateGraph +from tenxgraph.core.state import AgentState, Message +from tenxgraph.storage.checkpointer import InMemoryCheckpointer +from tenxgraph.utils.constants import END + +from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter +from tenxgraph_api.src.app.core.config.graph_config import RunsConfig +from tenxgraph_api.src.app.routers.graph import run_limits, thread_lock +from tenxgraph_api.src.app.routers.graph.router import ( + WS_SERVICE_RESTART, + invoke_graph, + stream_graph, + websocket_graph, +) +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema +from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + + +ALICE = {"user_id": "alice"} +MSG = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + +@pytest.fixture(autouse=True) +def limiters(monkeypatch): + locks, slots = MemoryConcurrencyLimiter(), MemoryConcurrencyLimiter() + monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: locks) + monkeypatch.setattr(run_limits, "concurrency_limiter", lambda: slots) + monkeypatch.setattr(thread_lock, "THREAD_BUSY_WAIT_SECONDS", 0.3) + return SimpleNamespace(locks=locks, slots=slots) + + +def _input(thread_id="t1"): + return GraphInputSchema(messages=MSG, config={"thread_id": thread_id}) + + +def _graph(node): + g = StateGraph() + g.add_node("A", node) + g.add_edge("A", END) + g.set_entry_point("A") + return g.compile(checkpointer=InMemoryCheckpointer()) + + +def _service(graph) -> GraphService: + service = GraphService.__new__(GraphService) + service._graph = graph + service.config = SimpleNamespace(runs=RunsConfig(max_concurrent_per_user=1)) + service.config.thread_name_generator_path = None + service.checkpointer = graph._checkpointer + service.thread_name_generator = None + service._media_service = None + service._telemetry = None + return service + + +def _quick(state: AgentState, config: dict) -> Message: + return Message.text_message("ok", role="assistant") + + +async def _closed_service() -> GraphService: + graph = _graph(_quick) + await graph.aclose() + return _service(graph) + + +def _ws(frames): + ws = MagicMock() + ws.accept = AsyncMock() + ws.send_text = AsyncMock() + ws.close = AsyncMock() + ws.receive_text = AsyncMock( + side_effect=[json.dumps(f) if isinstance(f, dict) else f for f in frames] + ) + return ws + + +# --------------------------------------------------------------------------- +# BF3: GraphClosedError after the stream headers went out, and on a WebSocket +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_closing_graph_ends_the_ndjson_stream_with_a_terminal_error_chunk(limiters): + service = await _closed_service() + # The route returns the response first (headers are sent); the refusal comes while streaming. + response = await stream_graph(graph_input=_input(), service=service, user=ALICE) + lines = [json.loads(line) async for line in response.body_iterator if line.strip()] + assert len(lines) == 1 + assert lines[0]["event"] == "error" + assert lines[0]["data"]["code"] == "SERVICE_UNAVAILABLE" + assert lines[0]["data"]["retry_after_seconds"] >= 1 + assert limiters.locks._held == {} + assert limiters.slots._held == {} + + +@pytest.mark.asyncio +async def test_closing_graph_sends_an_error_frame_then_closes_the_websocket(limiters): + service = await _closed_service() + frame = {"invoke_type": "fresh", "messages": MSG, "config": {"thread_id": "t1"}} + ws = _ws([frame, WebSocketDisconnect()]) + await websocket_graph(websocket=ws, service=service, user={}) + sent = [json.loads(c.args[0]) for c in ws.send_text.call_args_list] + assert [c["event"] for c in sent] == ["error"] # no "done" frame + assert sent[0]["data"]["code"] == "SERVICE_UNAVAILABLE" + ws.close.assert_awaited_once_with(code=WS_SERVICE_RESTART) + assert limiters.locks._held == {} + assert limiters.slots._held == {} + + +# --------------------------------------------------------------------------- +# BF6 and BF7: a client that leaves, or a cancelled request, stops the run +# --------------------------------------------------------------------------- + + +def _blocking_node(): + probe = SimpleNamespace(started=asyncio.Event(), cancelled=False, block=True) + + async def node(state: AgentState, config: dict) -> Message: + if probe.block: + probe.started.set() + try: + await asyncio.sleep(30) + except asyncio.CancelledError: + probe.cancelled = True + raise + return Message.text_message("ok", role="assistant") + + return node, probe + + +async def _next_run_works(service) -> None: + """A fresh run on the same thread starts and ends: the thread is not stuck.""" + response = await stream_graph(graph_input=_input(), service=service, user=ALICE) + lines = [json.loads(line) async for line in response.body_iterator if line.strip()] + assert lines[-1]["data"] == {"status": "done"} + + +@pytest.mark.asyncio +async def test_client_disconnect_during_a_stream_cancels_the_run_and_frees_the_thread(limiters): + node, probe = _blocking_node() + service = _service(_graph(node)) + response = await stream_graph(graph_input=_input(), service=service, user=ALICE) + body = response.body_iterator + + await body.__anext__() # the client got the first chunk, then leaves + assert limiters.locks.held(thread_lock.thread_lock_key("t1")) == 1 + await body.aclose() # what Starlette does when the client goes away + + assert not probe.started.is_set() or probe.cancelled # the run did not carry on + assert limiters.locks._held == {} + assert limiters.slots._held == {} + probe.block = False + await _next_run_works(service) + + +@pytest.mark.asyncio +async def test_cancelled_stream_request_releases_thread_lock_and_run_slot(limiters): + node, probe = _blocking_node() + service = _service(_graph(node)) + response = await stream_graph(graph_input=_input(), service=service, user=ALICE) + + async def read(): + async for _ in response.body_iterator: + pass + + reader = asyncio.create_task(read()) + await asyncio.wait_for(probe.started.wait(), 5) + assert limiters.slots.held(run_limits.run_slot_key("alice")) == 1 + reader.cancel() # the server cancels the request task + await asyncio.gather(reader, return_exceptions=True) + await asyncio.sleep(0) + + assert probe.cancelled + assert limiters.locks._held == {} + assert limiters.slots._held == {} + probe.block = False + await _next_run_works(service) + + +@pytest.mark.asyncio +async def test_cancelled_invoke_request_releases_thread_lock_and_run_slot(limiters): + node, probe = _blocking_node() + service = _service(_graph(node)) + request = MagicMock() + request.state = SimpleNamespace(request_id="r", timestamp="t") + request.is_disconnected = AsyncMock(return_value=False) + + task = asyncio.create_task( + invoke_graph(request=request, graph_input=_input(), service=service, user=ALICE) + ) + await asyncio.wait_for(probe.started.wait(), 5) + assert limiters.locks.held(thread_lock.thread_lock_key("t1")) == 1 + assert limiters.slots.held(run_limits.run_slot_key("alice")) == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + + assert probe.cancelled + assert limiters.locks._held == {} + assert limiters.slots._held == {} + probe.block = False + await _next_run_works(service) diff --git a/tests/unit_tests/test_extraction_limits.py b/tests/unit_tests/test_extraction_limits.py index 4cddbff4..c8f87102 100644 --- a/tests/unit_tests/test_extraction_limits.py +++ b/tests/unit_tests/test_extraction_limits.py @@ -150,7 +150,12 @@ async def test_upload_uses_its_declared_type(): fake = RecordingExtractor() service = MediaService.__new__(MediaService) - service._settings = MagicMock(MEDIA_MAX_SIZE_MB=1) + service._settings = MagicMock( + MEDIA_MAX_SIZE_MB=1, + MEDIA_MAX_FILES_PER_USER=0, + MEDIA_MAX_USER_STORAGE_MB=0, + MEDIA_MAX_IMAGE_PIXELS=0, + ) service._store = MagicMock(store=AsyncMock(return_value="k1")) service._pipeline = MagicMock( handling=DocumentHandling.EXTRACT_TEXT, extractor=DocumentExtractor(extractor=fake) diff --git a/tests/unit_tests/test_g127_leftovers.py b/tests/unit_tests/test_g127_leftovers.py new file mode 100644 index 00000000..715218d8 --- /dev/null +++ b/tests/unit_tests/test_g127_leftovers.py @@ -0,0 +1,78 @@ +"""G-127 leftovers: size caps on open dicts, upload parsed after auth, WS and AG-UI id rule.""" + +# ruff: noqa: S101, PLR2004 + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from tenxgraph_api.src.app.core.auth.request_config import ( + MAX_CONTAINER_ITEMS, + MAX_DICT_KEYS, + structure_error, +) +from tenxgraph_api.src.app.routers.checkpointer.schemas.checkpointer_schemas import ( + PutThreadSchema, +) +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( + GraphInputSchema, + WsGraphInputSchema, +) +from tenxgraph_api.src.app.routers.store.schemas.store_schemas import SearchMemorySchema + + +MSG = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + +def test_structure_error_caps_keys_items_and_total(): + assert structure_error({str(i): 1 for i in range(MAX_DICT_KEYS)}, "x") is None + assert "keys" in structure_error({str(i): 1 for i in range(MAX_DICT_KEYS + 1)}, "x") + assert structure_error([0] * MAX_CONTAINER_ITEMS, "x") is None + assert "items" in structure_error([0] * (MAX_CONTAINER_ITEMS + 1), "x") + bushy = [[0] * 5000 for _ in range(50)] # each list is fine, the total is not + assert "in all" in structure_error(bushy, "x") + + +def test_graph_input_refuses_wide_dicts_and_lists(): + wide = {str(i): 1 for i in range(MAX_DICT_KEYS + 1)} + for field in ("initial_state", "config"): + with pytest.raises(ValidationError, match="keys"): + GraphInputSchema(messages=MSG, **{field: wide}) + with pytest.raises(ValidationError, match="items"): + GraphInputSchema(resume=[0] * (MAX_CONTAINER_ITEMS + 1)) + with pytest.raises(ValidationError, match="keys"): + WsGraphInputSchema(messages=MSG, config=wide) + GraphInputSchema(messages=MSG, config={"a": 1}) + + +def test_store_and_checkpointer_schemas_are_bounded(): + wide = {str(i): 1 for i in range(MAX_DICT_KEYS + 1)} + with pytest.raises(ValidationError, match="keys"): + SearchMemorySchema(query="q", filters=wide) + with pytest.raises(ValidationError, match="keys"): + PutThreadSchema(thread_info=wide) + + +@pytest.mark.asyncio +async def test_ws_frame_with_a_bad_thread_id_gets_an_error_frame_and_skip(): + import importlib + import json + from unittest.mock import AsyncMock, MagicMock + + graph_router = importlib.import_module("tenxgraph_api.src.app.routers.graph.router") + + frames = MagicMock() + frames.next = AsyncMock( + side_effect=[ + json.dumps({"messages": MSG, "config": {"thread_id": "../etc/passwd"}}), + json.dumps({"messages": MSG, "config": {"thread_id": "new"}}), + ] + ) + websocket = MagicMock() + websocket.send_text = AsyncMock() + skipped = await graph_router._next_ws_input(frames, websocket, 1_000_000, {}) + assert skipped is graph_router._WS_SKIP + assert "thread_id" in websocket.send_text.await_args.args[0] + ok = await graph_router._next_ws_input(frames, websocket, 1_000_000, {}) + assert ok.config["thread_id"] == "new" diff --git a/tests/unit_tests/test_group3a_storage.py b/tests/unit_tests/test_group3a_storage.py new file mode 100644 index 00000000..21137395 --- /dev/null +++ b/tests/unit_tests/test_group3a_storage.py @@ -0,0 +1,308 @@ +"""Group 3a (storage) API fixes: G-97 write guard, G-123 partial delete, G-40 keys, G-42 boot.""" + +# ruff: noqa: S101 + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException +from tenxgraph.core.state import ImageBlock, MediaRef, Message + +from tenxgraph_api.src.app import main as app_main +from tenxgraph_api.src.app.core import redis_client +from tenxgraph_api.src.app.core.auth.ownership_resolver import ThreadOwnershipResolver +from tenxgraph_api.src.app.core.auth.revocation import RedisTokenRevocation +from tenxgraph_api.src.app.core.concurrency import MemoryConcurrencyLimiter, RedisConcurrencyLimiter +from tenxgraph_api.src.app.routers.checkpointer import router as cp_router +from tenxgraph_api.src.app.routers.checkpointer.services.checkpointer_service import ( + CheckpointerService, +) +from tenxgraph_api.src.app.routers.graph import thread_lock +from tenxgraph_api.src.app.routers.graph.router import fix_graph +from tenxgraph_api.src.app.routers.graph.run_limits import run_slot_key +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import FixGraphRequestSchema + +from .test_concurrency_limiter import _FakeRedis + + +USER = {"user_id": "alice"} + + +@pytest.fixture(autouse=True) +def fresh_locks(monkeypatch): + limiter = MemoryConcurrencyLimiter() + monkeypatch.setattr(thread_lock, "concurrency_limiter", lambda: limiter) + return limiter + + +async def _hold(limiter, thread_id): + """What a running graph holds on the thread.""" + return await limiter.acquire(thread_lock.thread_lock_key(thread_id), 1) + + +# --------------------------------------------------------------------------- +# G-97: state-changing REST routes take the run lock +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_guard_refuses_with_409_while_a_run_holds_the_thread(fresh_locks): + lease = await _hold(fresh_locks, "t1") + with pytest.raises(HTTPException) as exc: + async with thread_lock.thread_write_guard("t1"): + pytest.fail("the body must not run while a run holds the thread") + assert exc.value.status_code == 409 + lease.release() + async with thread_lock.thread_write_guard("t1"): + pass + # and it lets go afterwards, so a run can start + again = await _hold(fresh_locks, "t1") + assert again is not None + + +@pytest.mark.asyncio +async def test_guard_is_released_when_the_body_raises(fresh_locks): + with pytest.raises(RuntimeError): + async with thread_lock.thread_write_guard("t1"): + raise RuntimeError("boom") + assert await _hold(fresh_locks, "t1") is not None + + +def _routes(service): + from tenxgraph_api.src.app.routers.checkpointer.schemas.checkpointer_schemas import ( + ConfigSchema, + PutMessagesSchema, + StateSchema, + ) + + request = MagicMock() + return { + "put_state": lambda: cp_router.put_state( + request, "t1", StateSchema(state={}), service, USER + ), + "clear_state": lambda: cp_router.clear_state(request, "t1", service, USER), + "put_messages": lambda: cp_router.put_messages( + request, + "t1", + PutMessagesSchema(messages=[Message.text_message("x")]), + service, + USER, + ), + "delete_message": lambda: cp_router.delete_message( + request, "m1", "t1", ConfigSchema(), service, USER + ), + "delete_thread": lambda: cp_router.delete_thread( + request, "t1", ConfigSchema(), service, USER + ), + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "name", ["put_state", "clear_state", "put_messages", "delete_message", "delete_thread"] +) +async def test_checkpointer_write_routes_answer_409_during_a_run(fresh_locks, name): + service = MagicMock() + lease = await _hold(fresh_locks, "t1") + with pytest.raises(HTTPException) as exc: + await _routes(service)[name]() + assert exc.value.status_code == 409 + for method in ("put_state", "clear_state", "put_messages", "delete_message", "delete_thread"): + getattr(service, method).assert_not_called() + lease.release() + + +@pytest.mark.asyncio +async def test_fix_route_answers_409_during_a_run(fresh_locks): + service = MagicMock() + service.fix_graph = AsyncMock(return_value={}) + lease = await _hold(fresh_locks, "t1") + with pytest.raises(HTTPException) as exc: + await fix_graph(MagicMock(), FixGraphRequestSchema(thread_id="t1"), service, USER) + assert exc.value.status_code == 409 + service.fix_graph.assert_not_called() + lease.release() + + +# --------------------------------------------------------------------------- +# G-40: deployment prefix and colon-safe keys +# --------------------------------------------------------------------------- + + +def test_key_part_keeps_plain_ids_and_separates_colons(): + assert redis_client.key_part("abc-123_x@y.com") == "abc-123_x@y.com" + assert redis_client.key_part("a:b") != redis_client.key_part("a%3Ab") + assert ":" not in redis_client.key_part("a:b") + + +def test_run_slot_key_is_unambiguous(): + assert run_slot_key("a:b") != run_slot_key("a") + ":b" + assert thread_lock.thread_lock_key("a:b") == "runs:thread:a%3Ab" + + +@pytest.fixture +def key_prefix(monkeypatch): + from tenxgraph_api.src.app.core.config.settings import get_settings + + monkeypatch.setattr(get_settings(), "REDIS_KEY_PREFIX", "prod-eu") + + +def test_prefix_is_applied_to_every_api_redis_key(key_prefix): + resolver = ThreadOwnershipResolver(AsyncMock(), redis=MagicMock()) + assert resolver._redis_key("t:1").startswith("prod-eu:af:authz:owner:id:") + assert ":" not in resolver._redis_key("t:1").split(":id:", 1)[1] + assert resolver._tombstone_key("t1").startswith("prod-eu:af:authz:owner:tomb:") + revocation = RedisTokenRevocation(client=MagicMock()) + assert revocation._jti_key("j").startswith("prod-eu:10xgraph:revoked:jti:") + assert revocation._user_key("u:1") == "prod-eu:10xgraph:revoked:user:u%3A1" + limiter = RedisConcurrencyLimiter(_FakeRedis()) + assert limiter._key("runs:thread:t1").startswith("prod-eu:10xgraph:concurrency:") + + +def test_no_prefix_keeps_the_existing_keys(): + resolver = ThreadOwnershipResolver(AsyncMock(), redis=MagicMock()) + assert resolver._redis_key("t1") == "af:authz:owner:id:t1" + assert RedisTokenRevocation(client=MagicMock())._jti_key("j") == "10xgraph:revoked:jti:j" + + +# --------------------------------------------------------------------------- +# G-123: partial failure on thread delete, media linked through PUT state +# --------------------------------------------------------------------------- + + +def _service(state_context=None, messages=None): + service = CheckpointerService.__new__(CheckpointerService) + cp = MagicMock() + cp.aget_state = AsyncMock(return_value=SimpleNamespace(context=state_context or [])) + cp.alist_messages = AsyncMock(return_value=messages or []) + cp.aclean_thread = AsyncMock(return_value=None) + cp.adelete_message = AsyncMock(return_value=None) + cp.aget_message = AsyncMock(return_value=None) + service.checkpointer = cp + service.settings = MagicMock() + return service + + +def _img(key, message_id="m1"): + return Message( + message_id=message_id, + role="user", + content=[ImageBlock(media=MediaRef(kind="url", url=f"graph://media/{key}"))], + ) + + +KEY = "a" * 32 + + +@pytest.mark.asyncio +async def test_thread_delete_releases_media_linked_through_put_state(monkeypatch): + media = MagicMock(supports_links=True) + media.unlink_threads = AsyncMock(return_value=[]) + media.release_thread = AsyncMock(return_value=[]) + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: media)) + service = _service(state_context=[_img(KEY)]) + res = await service.delete_thread({"thread_id": "t1"}, USER, "t1") + media.unlink_threads.assert_awaited_once_with([KEY], "t1") + assert res.success is True + + +@pytest.mark.asyncio +async def test_thread_delete_reports_a_release_that_failed(monkeypatch): + media = MagicMock(supports_links=True) + media.unlink_threads = AsyncMock(return_value=[KEY]) + media.release_thread = AsyncMock(return_value=[]) + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: media)) + service = _service(state_context=[_img(KEY)]) + res = await service.delete_thread({"thread_id": "t1"}, USER, "t1") + assert res.success is False + assert res.data["deleted"] is True and res.data["cleanup_failed"] == ["files"] + assert "files" in res.message + + +@pytest.mark.asyncio +async def test_thread_delete_reports_a_trace_cleanup_failure(monkeypatch): + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: None)) + store = MagicMock() + store.delete_thread.side_effect = RuntimeError("disk full") + container = MagicMock() + container.try_get.return_value = store + monkeypatch.setattr("injectq.InjectQ.get_instance", lambda: container) + res = await _service().delete_thread({"thread_id": "t1"}, USER, "t1") + assert res.success is False and "traces" in res.data["cleanup_failed"] + + +@pytest.mark.asyncio +async def test_clean_thread_delete_is_still_a_plain_success(monkeypatch): + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: None)) + container = MagicMock() + container.try_get.return_value = None + monkeypatch.setattr("injectq.InjectQ.get_instance", lambda: container) + res = await _service().delete_thread({"thread_id": "t1"}, USER, "t1") + assert res.success is True + + +# --------------------------------------------------------------------------- +# G-11: files go only after the core delete has removed the message from state +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_message_media_is_released_only_after_the_core_delete(monkeypatch): + order: list[str] = [] + message = _img(KEY, "m1") + media = MagicMock(supports_links=True) + media.unlink_threads = AsyncMock(side_effect=lambda *a: order.append("unlink") or []) + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: media)) + service = _service(state_context=[message], messages=[message]) + service.checkpointer.aget_message = AsyncMock(return_value=message) + service.checkpointer.adelete_message = AsyncMock(side_effect=lambda *a: order.append("delete")) + await service.delete_message({"thread_id": "t1"}, USER, "m1") + assert order == ["delete", "unlink"] + service.checkpointer.adelete_message.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_message_media_still_in_state_through_another_message_is_kept(monkeypatch): + gone, other = _img(KEY, "m1"), _img(KEY, "m2") + media = MagicMock(supports_links=True) + media.unlink_threads = AsyncMock(return_value=[]) + monkeypatch.setattr(CheckpointerService, "_media_service", staticmethod(lambda: media)) + service = _service(state_context=[gone, other], messages=[]) + service.checkpointer.aget_message = AsyncMock(return_value=gone) + await service.delete_message({"thread_id": "t1"}, USER, "m1") + media.unlink_threads.assert_not_awaited() + + +# --------------------------------------------------------------------------- +# G-42: the checkpointer schema is prepared at boot +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_boot_runs_the_checkpointer_setup(monkeypatch): + checkpointer = MagicMock() + checkpointer.asetup = AsyncMock(return_value=True) + container = MagicMock() + container.try_get.side_effect = lambda name: checkpointer if name == "checkpointer" else None + monkeypatch.setattr(app_main, "container", container) + await app_main._prepare_checkpointer() + checkpointer.asetup.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_boot_survives_a_failing_setup(monkeypatch): + checkpointer = MagicMock() + checkpointer.asetup = AsyncMock(side_effect=ConnectionError("postgres down")) + container = MagicMock() + container.try_get.return_value = checkpointer + monkeypatch.setattr(app_main, "container", container) + await app_main._prepare_checkpointer() # logs, does not raise + + +@pytest.mark.asyncio +async def test_boot_skips_a_checkpointer_without_setup(monkeypatch): + container = MagicMock() + container.try_get.return_value = SimpleNamespace() + monkeypatch.setattr(app_main, "container", container) + await app_main._prepare_checkpointer() diff --git a/tests/unit_tests/test_group6_nesting_depth.py b/tests/unit_tests/test_group6_nesting_depth.py new file mode 100644 index 00000000..b7100abb --- /dev/null +++ b/tests/unit_tests/test_group6_nesting_depth.py @@ -0,0 +1,65 @@ +"""G-128: deeply nested client state is refused with VALIDATION_ERROR before the run starts.""" + +# ruff: noqa: S101 + +from __future__ import annotations + +from typing import Any + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from pydantic import ValidationError + +from tenxgraph_api.src.app.core.auth.request_config import MAX_NESTING_DEPTH, nesting_depth_error +from tenxgraph_api.src.app.core.config.setup_middleware import RequestIDMiddleware +from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler +from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import GraphInputSchema + + +def _chain(depth: int) -> dict[str, Any]: + value: dict[str, Any] = {} + for _ in range(depth): + value = {"k": value} + return value + + +MSG = [{"role": "user", "content": [{"type": "text", "text": "hi"}]}] + + +def test_depth_check_does_not_recurse() -> None: + assert nesting_depth_error(_chain(50_000), "initial_state") is not None + + +def test_depth_at_limit_is_accepted() -> None: + assert nesting_depth_error(_chain(MAX_NESTING_DEPTH - 1), "x") is None + assert nesting_depth_error(_chain(MAX_NESTING_DEPTH), "x") is not None + + +@pytest.mark.parametrize("field", ["initial_state", "config"]) +def test_schema_refuses_deep_dicts(field: str) -> None: + GraphInputSchema(messages=MSG, **{field: _chain(10)}) + with pytest.raises(ValidationError, match="levels deep"): + GraphInputSchema(messages=MSG, **{field: _chain(5000)}) + + +def test_schema_refuses_deep_resume() -> None: + with pytest.raises(ValidationError, match="levels deep"): + GraphInputSchema(resume=[_chain(5000)]) + + +def test_route_answers_422_validation_error() -> None: + app = FastAPI() + app.add_middleware(RequestIDMiddleware) + init_errors_handler(app) + + @app.post("/t") + async def t(body: GraphInputSchema): + return {"ok": True} + + client = TestClient(app) + ok = client.post("/t", json={"messages": MSG, "initial_state": _chain(10)}) + assert ok.status_code == 200 + resp = client.post("/t", json={"messages": MSG, "initial_state": _chain(500)}) + assert resp.status_code == 422 + assert resp.json()["error"]["code"] == "VALIDATION_ERROR" diff --git a/tests/unit_tests/test_handle_errors.py b/tests/unit_tests/test_handle_errors.py index c00c0c19..d63a4030 100644 --- a/tests/unit_tests/test_handle_errors.py +++ b/tests/unit_tests/test_handle_errors.py @@ -477,3 +477,65 @@ async def run(payload: GraphInputSchema): assert r.json()["error"]["message"] == "The request data is invalid. Please check your input." cleanup_env() + + +NODE_MISSING_MESSAGE = ( + "This thread was saved at a step that no longer exists in the graph. " + "Start a new thread, or ask the operator to map the old step." +) +NEWER_STATE_MESSAGE = ( + "This thread was saved by a newer server version. The server must be upgraded to read it." +) + + +def _raise_app(mode, exc): + app = setup_app(mode) + + @app.get("/boom") + def boom(): + raise exc + + try: + return TestClient(app).get("/boom") + finally: + cleanup_env() + + +def test_stored_node_missing_is_409_with_detail_in_development(): + from tenxgraph.core.exceptions import StoredNodeMissingError + + r = _raise_app("development", StoredNodeMissingError("old_node", thread_id="t1")) + assert r.status_code == 409 + error = r.json()["error"] + assert error["code"] == "GRAPH_NODE_MISSING_001" + assert "old_node" in error["message"] + + +def test_stored_node_missing_is_409_with_safe_message_in_production(): + from tenxgraph.core.exceptions import StoredNodeMissingError + + r = _raise_app("production", StoredNodeMissingError("old_node", thread_id="t1")) + assert r.status_code == 409 + error = r.json()["error"] + assert error["code"] == "GRAPH_NODE_MISSING_001" + assert error["message"] == NODE_MISSING_MESSAGE + + +def test_state_format_error_is_500_with_detail_in_development(): + from tenxgraph.core.exceptions import StateFormatVersionError + + r = _raise_app("development", StateFormatVersionError(found=9, supported=2, thread_id="t1")) + assert r.status_code == 500 + error = r.json()["error"] + assert error["code"] == "STATE_FORMAT_001" + assert "format" in error["message"] + + +def test_state_format_error_is_500_with_clear_message_in_production(): + from tenxgraph.core.exceptions import StateFormatVersionError + + r = _raise_app("production", StateFormatVersionError(found=9, supported=2, thread_id="t1")) + assert r.status_code == 500 + error = r.json()["error"] + assert error["code"] == "STATE_FORMAT_001" + assert error["message"] == NEWER_STATE_MESSAGE diff --git a/tests/unit_tests/test_loader.py b/tests/unit_tests/test_loader.py index d4cfa1e1..1bab730b 100644 --- a/tests/unit_tests/test_loader.py +++ b/tests/unit_tests/test_loader.py @@ -455,3 +455,59 @@ class _Bare(BaseCheckpointer): # type: ignore[misc] # Auth off: no owner checks ever run, so nothing to refuse. await attach_all_modules(config, container) assert isinstance(container.get(AuthorizationBackend), OwnershipAuthorizationBackend) + + +# --------------------------------------------------------------------------- +# G-28: per-process state warning +# --------------------------------------------------------------------------- + + +def _warn_container(checkpointer): + from unittest.mock import MagicMock + + container = MagicMock() + container.try_get.return_value = checkpointer + return container + + +def test_warns_for_in_memory_checkpointer_and_missing_redis_in_production(monkeypatch, caplog): + import logging + + from tenxgraph.storage.checkpointer import InMemoryCheckpointer + + from tenxgraph_api.src.app import loader + + monkeypatch.setattr(loader, "_is_production", lambda: True) + with caplog.at_level(logging.WARNING): + loader.warn_per_process_state(_warn_container(InMemoryCheckpointer()), None) + text = " ".join(r.getMessage() for r in caplog.records) + assert "InMemoryCheckpointer" in text + assert "No Redis configured" in text + + +def test_no_warning_in_single_process_development(monkeypatch, caplog): + import logging + + from tenxgraph.storage.checkpointer import InMemoryCheckpointer + + from tenxgraph_api.src.app import loader + + monkeypatch.setattr(loader, "_is_production", lambda: False) + monkeypatch.delenv("WEB_CONCURRENCY", raising=False) + with caplog.at_level(logging.WARNING): + loader.warn_per_process_state(_warn_container(InMemoryCheckpointer()), None) + assert not caplog.records + + +def test_web_concurrency_above_one_warns_even_in_development(monkeypatch, caplog): + import logging + + from tenxgraph_api.src.app import loader + + monkeypatch.setattr(loader, "_is_production", lambda: False) + monkeypatch.setenv("WEB_CONCURRENCY", "4") + with caplog.at_level(logging.WARNING): + loader.warn_per_process_state(_warn_container(None), "redis://x") + assert not caplog.records + loader.warn_per_process_state(_warn_container(None), None) + assert "No Redis configured" in caplog.text diff --git a/tests/unit_tests/test_media_g123_g125.py b/tests/unit_tests/test_media_g123_g125.py new file mode 100644 index 00000000..2d37edca --- /dev/null +++ b/tests/unit_tests/test_media_g123_g125.py @@ -0,0 +1,152 @@ +"""G-123 thread links lose no update; G-125 listing and quota work through an owner index.""" + +# ruff: noqa: S101, PLR2004, SLF001 + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from tenxgraph.storage.media.storage.base import BaseMediaStore +from tenxgraph.storage.media.storage.memory_store import InMemoryMediaStore + +from tenxgraph_api.src.app.core.auth.request_config import ClientInputError +from tenxgraph_api.src.app.core.config.media_settings import MediaSettings +from tenxgraph_api.src.app.routers.media import MediaService, _BoundedCache + + +class _SlowStore(InMemoryMediaStore): + """Metadata reads go stale while other tasks run, as they do over a network.""" + + async def get_metadata(self, storage_key): + meta = await super().get_metadata(storage_key) + await asyncio.sleep(0) + return meta + + +class _OldCoreStore(_SlowStore): + """A store from a core without modify_metadata.""" + + modify_metadata = None + + +class _IndexedStore(BaseMediaStore): + """Like the cloud store: no list_keys, an owner index instead.""" + + def __init__(self) -> None: + self.inner = InMemoryMediaStore() + + async def store(self, data, mime_type, metadata=None): + return await self.inner.store(data, mime_type, metadata) + + async def retrieve(self, storage_key): + return await self.inner.retrieve(storage_key) + + async def delete(self, storage_key): + return await self.inner.delete(storage_key) + + async def exists(self, storage_key): + return await self.inner.exists(storage_key) + + async def get_metadata(self, storage_key): + return await self.inner.get_metadata(storage_key) + + async def update_metadata(self, storage_key, updates): + return await self.inner.update_metadata(storage_key, updates) + + async def list_keys_for_owner(self, owner_id): + return await self.inner.list_keys_for_owner(owner_id) + + +def _media(store: Any, **settings: Any) -> MediaService: + svc = MediaService.__new__(MediaService) + svc._settings = MediaSettings(_env_file=None, MEDIA_REQUIRE_OWNER=True, **settings) + svc._checkpointer = MagicMock( + aget_cache_value=AsyncMock(return_value=None), aput_cache_value=AsyncMock() + ) + svc._store = store + svc._extraction_cache = _BoundedCache(8) + svc._signed_url_cache = _BoundedCache(8) + svc._links_notice_logged = False + svc._pipeline = SimpleNamespace(handling="skip") + return svc + + +async def _upload(media: MediaService, owner: str = "alice") -> str: + result = await media.upload_file(b"\x89PNG\r\n\x1a\n", "a.png", "image/png", user_id=owner) + return result["file_id"] + + +@pytest.mark.asyncio +async def test_concurrent_links_to_one_file_are_all_kept(): + media = _media(_SlowStore()) + key = await _upload(media) + await asyncio.gather(*(media.link_threads([key], f"t{i}", "alice") for i in range(15))) + assert sorted((await media.store.get_metadata(key))["threads"]) == sorted( + f"t{i}" for i in range(15) + ) + + +@pytest.mark.asyncio +async def test_concurrent_unlinks_keep_the_rest_and_delete_at_the_end(monkeypatch): + media = _media(_SlowStore()) + monkeypatch.setattr(media, "delete_with_thread", lambda: True) + key = await _upload(media) + for i in range(6): + await media.link_threads([key], f"t{i}", "alice") + await asyncio.gather(*(media.unlink_threads([key], f"t{i}") for i in range(5))) + assert (await media.store.get_metadata(key))["threads"] == ["t5"] + await media.unlink_threads([key], "t5") + assert not await media.store.exists(key) + + +@pytest.mark.asyncio +async def test_store_without_modify_metadata_still_links(): + media = _media(_OldCoreStore()) + key = await _upload(media) + await media.link_threads([key], "t1", "alice") + assert (await media.store.get_metadata(key))["threads"] == ["t1"] + + +@pytest.mark.asyncio +async def test_without_an_atomic_store_concurrent_links_do_lose_updates(): + # Documents the failure the store-side modify_metadata prevents + media = _media(_OldCoreStore()) + key = await _upload(media) + await asyncio.gather(*(media.link_threads([key], f"t{i}", "alice") for i in range(5))) + assert len((await media.store.get_metadata(key))["threads"]) < 5 + + +@pytest.mark.asyncio +async def test_list_files_works_through_the_owner_index(): + media = _media(_IndexedStore()) + a = await _upload(media, "alice") + await _upload(media, "bob") + items, more = await media.list_files("alice", 0, 10) + assert [i["file_id"] for i in items] == [a] + assert more is False + # With no user there is no owner to index by: still not listable + with pytest.raises(NotImplementedError): + await media.list_files(None, 0, 10) + + +@pytest.mark.asyncio +async def test_quota_is_enforced_through_the_owner_index(): + media = _media(_IndexedStore(), MEDIA_MAX_FILES_PER_USER=1) + await _upload(media, "alice") + with pytest.raises(ClientInputError): + await _upload(media, "alice") + await _upload(media, "bob") # another user is unaffected + + +@pytest.mark.asyncio +async def test_release_thread_finds_linked_files_through_the_owner_index(monkeypatch): + media = _media(_IndexedStore()) + monkeypatch.setattr(media, "delete_with_thread", lambda: False) + key = await _upload(media) + await media.link_threads([key], "t1", "alice") + assert await media.release_thread("t1", "alice") == [] + assert (await media.store.get_metadata(key))["threads"] == [] diff --git a/tests/unit_tests/test_media_group3b.py b/tests/unit_tests/test_media_group3b.py new file mode 100644 index 00000000..25ef3261 --- /dev/null +++ b/tests/unit_tests/test_media_group3b.py @@ -0,0 +1,365 @@ +"""Group 3b API rows: G-124, G-126, G-130, G-38, G-89 wiring and J-5.""" + +# ruff: noqa: S101, PLR2004, SLF001 + +from __future__ import annotations + +import asyncio +import base64 +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from tenxgraph.core.state import Message +from tenxgraph.core.state.message_block import ImageBlock, MediaRef +from tenxgraph.storage.media.storage.base import MediaTooLargeError +from tenxgraph.storage.media.storage.memory_store import InMemoryMediaStore + +from tenxgraph_api.src.app import main as main_module +from tenxgraph_api.src.app.core.auth.request_config import ClientInputError +from tenxgraph_api.src.app.core.config.media_settings import MediaSettings, MediaStorageType +from tenxgraph_api.src.app.routers.graph.services.multimodal_preprocessor import ( + internal_media_keys, + preprocess_multimodal_messages, +) +from tenxgraph_api.src.app.routers.media import MediaService, _BoundedCache, _create_media_store +from tenxgraph_api.src.app.routers.media.router import upload_file +from tenxgraph_api.src.app.routers.store.schemas.store_schemas import ListMemoriesSchema +from tenxgraph_api.src.app.routers.store.services.store_service import StoreService +from tenxgraph_api.src.app.utils.media.extractor import DocumentExtractor + + +PNG = b"\x89PNG\r\n\x1a\n" + + +class ListStore(InMemoryMediaStore): + async def update_metadata(self, storage_key: str, updates: dict[str, Any]) -> bool: + data, mime, meta = self._data[storage_key] + self._data[storage_key] = (data, mime, {**meta, **updates}) + return True + + +def _media(store: Any = None, **settings: Any) -> MediaService: + svc = MediaService.__new__(MediaService) + svc._settings = MediaSettings(_env_file=None, MEDIA_REQUIRE_OWNER=True, **settings) + svc._checkpointer = MagicMock( + aget_cache_value=AsyncMock(return_value=None), + aput_cache_value=AsyncMock(), + aclear_cache_value=AsyncMock(), + alist_cache_keys=AsyncMock(return_value=[]), + ) + svc._store = store if store is not None else ListStore() + svc._extraction_cache = _BoundedCache(8) + svc._signed_url_cache = _BoundedCache(8) + svc._links_notice_logged = False + svc._pipeline = SimpleNamespace(handling="skip") + return svc + + +# --- G-124 ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_delete_clears_shared_extraction_and_signed_url_cache(): + svc = _media() + svc._checkpointer.alist_cache_keys.return_value = ["f:image/png:60", "f:image/png:3600"] + key = await svc.store.store(PNG, "image/png", metadata={"owner_id": "a"}) + svc._checkpointer.alist_cache_keys.return_value = [f"{key}:image/png:60"] + svc._signed_url_cache[f"{key}:image/png:3600"] = {"url": "u"} + svc._extraction_cache[key] = "secret text" + + assert await svc.delete_file(key) is True + + cleared = {c.args for c in svc._checkpointer.aclear_cache_value.await_args_list} + assert ("media:extraction", key) in cleared + assert ("media:signed-url", f"{key}:image/png:60") in cleared + assert ("media:signed-url", f"{key}:image/png:3600") in cleared + assert key not in svc._extraction_cache + assert not list(svc._signed_url_cache) + + +# --- G-126 ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_list_memories_pages_with_offset_and_has_more(): + rows = [f"m{i}" for i in range(7)] + + async def aget_all(cfg, limit, **kw): + return rows[:limit] + + store = MagicMock(aget_all=aget_all) + svc = StoreService(store=store) + svc._config = lambda config, user: {} + with patch( + "tenxgraph_api.src.app.routers.store.services.store_service.MemoryListResponseSchema", + lambda **kw: SimpleNamespace(**kw), + ): + first = await svc.list_memories({}, {}, limit=3, offset=0) + second = await svc.list_memories({}, {}, limit=3, offset=3) + last = await svc.list_memories({}, {}, limit=3, offset=6) + assert (first.memories, first.has_more, first.next_offset) == (rows[0:3], True, 3) + assert (second.memories, second.has_more, second.next_offset) == (rows[3:6], True, 6) + assert (last.memories, last.has_more, last.next_offset) == (rows[6:], False, None) + + +def test_list_memories_schema_offset_defaults_to_zero_and_rejects_negative(): + assert ListMemoriesSchema().offset == 0 + assert ListMemoriesSchema(offset=5).offset == 5 + with pytest.raises(ValueError): + ListMemoriesSchema(offset=-1) + + +# --- G-130 ------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_file_count_quota_blocks_the_next_upload_for_that_user_only(): + svc = _media(MEDIA_MAX_FILES_PER_USER=2) + await svc.upload_file(PNG, "a.png", "image/png", user_id="alice") + await svc.upload_file(PNG, "b.png", "image/png", user_id="alice") + with pytest.raises(ClientInputError, match="at most 2 files"): + await svc.upload_file(PNG, "c.png", "image/png", user_id="alice") + await svc.upload_file(PNG, "c.png", "image/png", user_id="bob") + + +@pytest.mark.asyncio +async def test_storage_quota_counts_bytes(): + svc = _media(MEDIA_MAX_USER_STORAGE_MB=0.001) # about 1048 bytes + await svc.upload_file(b"\x89PNG\r\n\x1a\n" + b"x" * 600, "a.png", "image/png", user_id="a") + with pytest.raises(ClientInputError, match="Storage limit"): + await svc.upload_file(b"\x89PNG\r\n\x1a\n" + b"x" * 600, "b.png", "image/png", user_id="a") + + +@pytest.mark.asyncio +async def test_quota_off_by_default(): + svc = _media() + for i in range(5): + await svc.upload_file(PNG, f"{i}.png", "image/png", user_id="alice") + + +@pytest.mark.asyncio +async def test_cleanup_uses_the_media_service_store(): + """The temp-cache cleanup used to get media_store=None because nothing bound that name.""" + store = object() + cache = AsyncMock() + cache.cleanup.return_value = 0 + container = MagicMock() + container.try_get.side_effect = lambda key: ( + SimpleNamespace(store=store) + if key is MediaService + else (object() if key in ("checkpointer",) else None) + ) + with ( + patch.object(main_module, "container", container), + patch("tenxgraph.storage.media.temp_cache.TemporaryMediaCache", return_value=cache), + ): + await main_module._cleanup_temp_media_cache() + assert cache.cleanup.await_args.args[1] is store + + +@pytest.mark.asyncio +async def test_sweeper_calls_sweep_orphans_and_survives_errors(): + calls: list[float] = [] + + class Store: + async def sweep_orphans(self, min_age: float) -> int: + calls.append(min_age) + if len(calls) == 1: + raise OSError("boom") + return 1 + + container = MagicMock() + container.try_get.return_value = SimpleNamespace(store=Store()) + with patch.object(main_module, "container", container): + task = asyncio.create_task(main_module._sweep_media_loop(0.01)) + await asyncio.sleep(0.1) + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert len(calls) >= 2 + assert calls[0] == main_module.MEDIA_SWEEP_MIN_AGE_SECONDS + + +@pytest.mark.asyncio +async def test_sweeper_returns_for_a_store_without_sweep_orphans(): + container = MagicMock() + container.try_get.return_value = SimpleNamespace(store=object()) + with patch.object(main_module, "container", container): + await asyncio.wait_for(main_module._sweep_media_loop(0.01), 1) + + +# --- G-38 -------------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_image_pixel_limit_is_enforced_on_upload(): + svc = _media(MEDIA_MAX_IMAGE_PIXELS=123) + seen: list[Any] = [] + + def fake(data: bytes, max_pixels: Any) -> None: + seen.append(max_pixels) + raise ValueError("Image has 999 pixels; the limit is 123 pixels") + + with patch("tenxgraph_api.src.app.routers.media.check_image_pixels", fake): + with pytest.raises(ClientInputError, match="pixels"): + await svc.upload_file(PNG, "a.png", "image/png", user_id="a") + assert seen == [123] + assert not await svc.store.list_keys() + + +@pytest.mark.asyncio +async def test_pixel_check_skips_non_images(): + svc = _media(MEDIA_MAX_IMAGE_PIXELS=1) + with patch("tenxgraph_api.src.app.routers.media.check_image_pixels") as check: + await svc.upload_file(b"hello", "a.txt", "text/plain", user_id="a") + check.assert_not_called() + + +@pytest.mark.parametrize( + "stype", [MediaStorageType.MEMORY, MediaStorageType.LOCAL], ids=["memory", "local"] +) +@pytest.mark.asyncio +async def test_media_store_gets_the_size_cap(stype, tmp_path): + settings = MediaSettings( + _env_file=None, + MEDIA_STORAGE_TYPE=stype, + MEDIA_STORAGE_PATH=str(tmp_path), + MEDIA_MAX_SIZE_MB=0.001, + ) + store = _create_media_store(settings) + with pytest.raises(MediaTooLargeError): + await store.store(b"x" * 2000, "application/octet-stream") + + +# --- J-5 --------------------------------------------------------------------------------- + + +def _pdf(pages: int, pad: int = 0) -> bytes: + body = b"".join(b"%d 0 obj << /Type /Page >> endobj\n" % i for i in range(pages)) + return b"%PDF-1.4\n" + body + b"/Type /Pages /Count %d\n" % pages + b"x" * pad + + +def _extracting_media(**settings: Any) -> MediaService: + svc = _media(**settings) + svc._pipeline = SimpleNamespace( + handling="extract_text", + extractor=DocumentExtractor( + extractor=SimpleNamespace(extract=AsyncMock(return_value="text")), + max_pdf_bytes=int(svc._settings.MEDIA_MAX_PDF_MB * 1024 * 1024), + max_pdf_pages=svc._settings.MEDIA_MAX_PDF_PAGES, + ), + ) + from tenxgraph.storage.media.config import DocumentHandling + + svc._pipeline.handling = DocumentHandling.EXTRACT_TEXT + return svc + + +@pytest.mark.asyncio +async def test_oversize_pdf_is_refused_before_it_is_stored(): + svc = _extracting_media(MEDIA_MAX_PDF_MB=0.001) + with pytest.raises(ClientInputError, match="PDF is"): + await svc.upload_file(_pdf(1, pad=5000), "a.pdf", "application/pdf", user_id="a") + assert not await svc.store.list_keys() + + +@pytest.mark.asyncio +async def test_pdf_with_too_many_pages_is_refused(): + svc = _extracting_media(MEDIA_MAX_PDF_PAGES=10) + with pytest.raises(ClientInputError, match="pages"): + await svc.upload_file(_pdf(50), "a.pdf", "application/pdf", user_id="a") + # page count hidden in a Pages node alone is caught too + big_tree = b"%PDF-1.4\n/Type /Pages /Count 100000\n" + with pytest.raises(ClientInputError, match="pages"): + await svc.upload_file(big_tree, "b.pdf", "application/pdf", user_id="a") + + +@pytest.mark.asyncio +async def test_small_pdf_is_still_extracted(): + svc = _extracting_media() + result = await svc.upload_file(_pdf(3), "a.pdf", "application/pdf", user_id="a") + assert result["extracted_text"] == "text" + + +@pytest.mark.asyncio +async def test_extractor_itself_refuses_oversize_pdf(): + from tenxgraph_api.src.app.utils.media.extractor import DocumentTooLargeError + + fake = SimpleNamespace(extract=AsyncMock(return_value="x")) + extractor = DocumentExtractor(extractor=fake, max_pdf_pages=5) + with pytest.raises(DocumentTooLargeError): + await extractor.extract(_pdf(20), "a.pdf", "application/pdf") + fake.extract.assert_not_called() + + +@pytest.mark.asyncio +async def test_pdf_cap_is_a_413_on_the_upload_route(): + from fastapi import HTTPException + + service = AsyncMock() + service.upload_file.side_effect = ClientInputError("PDF is 99 bytes; the limit is 1 bytes") + file = MagicMock() + file.filename = "a.pdf" + file.content_type = "application/pdf" + file.size = 12 + file.read = AsyncMock(return_value=b"%PDF-1.4\nabcd") + with pytest.raises(HTTPException) as exc: + await upload_file(MagicMock(), file, service, {"user_id": "a"}) + assert exc.value.status_code == 413 + + +# --- G-89 wiring ------------------------------------------------------------------------- + + +def _inline_message(size: int) -> Message: + data = base64.b64encode(b"\x89PNG\r\n\x1a\n" + b"x" * size).decode() + return Message( + message_id="m1", + role="user", + content=[ImageBlock(media=MediaRef(kind="data", data_base64=data, mime_type="image/png"))], + ) + + +@pytest.mark.asyncio +async def test_large_inline_media_is_offloaded_with_owner_and_thread_tracking(): + svc = _media(MEDIA_INLINE_MAX_BYTES=100) + out = await preprocess_multimodal_messages([_inline_message(500)], _as_service(svc), "alice") + keys = internal_media_keys(out[0]) + assert len(keys) == 1 + assert out[0].content[0].media.kind == "url" + meta = await svc.store.get_metadata(keys[0]) + assert meta["owner_id"] == "alice" + assert meta["threads"] == [] + await svc.ensure_can_access(keys[0], "alice") + + +@pytest.mark.asyncio +async def test_small_inline_media_stays_inline_and_zero_turns_it_off(): + svc = _media(MEDIA_INLINE_MAX_BYTES=10_000) + out = await preprocess_multimodal_messages([_inline_message(50)], _as_service(svc), "a") + assert out[0].content[0].media.kind == "data" + off = _media(MEDIA_INLINE_MAX_BYTES=0) + out = await preprocess_multimodal_messages([_inline_message(500)], _as_service(off), "a") + assert out[0].content[0].media.kind == "data" + + +@pytest.mark.asyncio +async def test_offload_rejects_bad_base64_as_client_error(): + svc = _media(MEDIA_INLINE_MAX_BYTES=1) + msg = Message( + message_id="m1", + role="user", + content=[ + ImageBlock( + media=MediaRef(kind="data", data_base64="!!!notbase64!!!", mime_type="image/png") + ) + ], + ) + with pytest.raises(ClientInputError): + await svc.offload_inline([msg], "a") + + +def _as_service(svc: MediaService) -> MediaService: + # preprocess reads extraction via these; the stub checkpointer returns nothing + return svc diff --git a/tests/unit_tests/test_media_lifecycle.py b/tests/unit_tests/test_media_lifecycle.py index d12a9c83..e91bceec 100644 --- a/tests/unit_tests/test_media_lifecycle.py +++ b/tests/unit_tests/test_media_lifecycle.py @@ -448,6 +448,7 @@ async def adelete(cfg, message_id): async def aclean(cfg): state["messages"] = [] + cp.aget_state = AsyncMock(return_value=None) cp.alist_messages = AsyncMock(side_effect=alist) cp.aget_message = AsyncMock(side_effect=aget) cp.adelete_message = AsyncMock(side_effect=adelete) @@ -471,6 +472,34 @@ async def test_thread_delete_removes_files_with_no_thread_left(monkeypatch, dele assert not await media.store.exists(key) +@pytest.mark.asyncio +async def test_thread_delete_releases_files_no_message_references(monkeypatch, delete_setting): + media = _media() + key = await _upload(media) + other = await _upload(media, owner="bob") + await media.link_threads([key], "t1", "alice") + # No message references the file any more (cleared state), so only its own link finds it. + service = _thread_service(monkeypatch, media, []) + await service.delete_thread({"thread_id": "t1"}, _USER, "t1") + assert not await media.store.exists(key) + assert await media.store.exists(other) + + +@pytest.mark.asyncio +async def test_release_thread_skips_other_owners_and_stores_without_list_keys(delete_setting): + media = _media() + key = await _upload(media, owner="bob") + await media.link_threads([key], "t1", "bob") + assert await media.release_thread("t1", "alice") == [] + assert await media.store.exists(key) + + class NoList(LinkStore): + async def list_keys(self): + raise NotImplementedError + + assert await _media(NoList()).release_thread("t1", "alice") == [] + + @pytest.mark.asyncio async def test_shared_file_stays_until_the_last_thread_goes(monkeypatch, delete_setting): media = _media() @@ -541,10 +570,12 @@ async def test_cleanup_failures_never_fail_the_delete(monkeypatch, delete_settin service = _thread_service(monkeypatch, media, [_img(f"graph://media/{key}")]) media.store.get_metadata = AsyncMock(side_effect=RuntimeError("store down")) res = await service.delete_thread({"thread_id": "t1"}, _USER, "t1") - assert res.success is True + # The delete itself went through, but a cleanup that failed is no longer reported as success. + assert res.success is False and res.data["deleted"] is True + assert "files" in res.data["cleanup_failed"] service.checkpointer.alist_messages = AsyncMock(side_effect=RuntimeError("db down")) res = await service.delete_thread({"thread_id": "t1"}, _USER, "t1") - assert res.success is True + assert res.success is False and "files" in res.data["cleanup_failed"] res = await service.delete_message({"thread_id": "t1"}, _USER, "m1") assert res.success is True diff --git a/tests/unit_tests/test_nc23_media_ownership.py b/tests/unit_tests/test_nc23_media_ownership.py new file mode 100644 index 00000000..86a43cb7 --- /dev/null +++ b/tests/unit_tests/test_nc23_media_ownership.py @@ -0,0 +1,535 @@ +"""NC-23: user B cannot read or resolve user A's ``graph://media/`` through any path. + +The core resolves ``graph://media/`` for any key (LLM-MED-04); the API's ownership check +is what protects it. This drives every route that reads a stored file or lets a message carry +a media reference, with auth on and the ownership check on, through the real routers and a +real ``MediaService``: + +- the media routes (download, info, signed URL, delete, list); +- invoke, stream, the ``/v1/graph/ws`` WebSocket and AG-UI, all of which reach one shared + ``GraphService._prepare_input`` -> ``preprocess_multimodal_messages``; +- the checkpointer routes that write messages or state (the next run resolves them); +- the preprocessor and ``MediaService`` called directly; +- ``/v1/graph/live``, which accepts no message or media reference at all. + +The graph is real: a node resolves every ``graph://media`` reference with the core +``resolve_media_refs`` and echoes the bytes, so "B cannot resolve A's key" is checked on what +the run actually produced, not only on a status code. +""" + +# ruff: noqa: S101, PLR2004 + +from __future__ import annotations + +import base64 +import json +from pathlib import Path +from typing import Any + +import pytest +from fastapi.testclient import TestClient +from injectq import InjectQ +from tenxgraph.core.graph import CompiledGraph, StateGraph +from tenxgraph.core.state import AgentState, Message, TextBlock +from tenxgraph.core.state.message_block import ( + DocumentBlock, + ImageBlock, + MediaRef, + ToolResultBlock, +) +from tenxgraph.storage.checkpointer import BaseCheckpointer, InMemoryCheckpointer +from tenxgraph.storage.media.resolver import MediaRefResolver +from tenxgraph.utils.constants import END +from tenxgraph.utils.converter import resolve_media_refs + +from tenxgraph_api.src.app.core.auth.authorization import ( + AuthorizationBackend, + DefaultAuthorizationBackend, +) +from tenxgraph_api.src.app.core.auth.base_auth import BaseAuth +from tenxgraph_api.src.app.core.auth.request_config import ClientInputError +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig +from tenxgraph_api.src.app.core.config.media_settings import MediaSettings, MediaStorageType +from tenxgraph_api.src.app.routers.checkpointer.router import router as checkpointer_router +from tenxgraph_api.src.app.routers.graph.router import router as graph_router +from tenxgraph_api.src.app.routers.graph.services.multimodal_preprocessor import ( + check_media_references, + internal_media_keys, + preprocess_multimodal_messages, +) +from tenxgraph_api.src.app.routers.media import MediaService +from tenxgraph_api.src.app.routers.media.router import router as media_router +from tests.integration_tests.conftest import HeaderAuth, build_app, make_client, user_headers + + +SECRET_A = b"secret-of-alice-4f1c" +SECRET_B = b"secret-of-bob-77aa" +ALICE, BOB = "alice", "bob" +LEAK = SECRET_A.decode() +# POST /v1/threads/{id}/messages is admin-only (checkpointer:import). +IMPORTER = {**user_headers(BOB), "X-Test-Scopes": "checkpointer:import"} + + +# --------------------------------------------------------------------------- the app + + +class _Auth(HeaderAuth): + """``X-Test-User`` for HTTP; a bearer token (the user id) also works, which the + WebSocket pre-run credential check needs.""" + + def authenticate(self, request, response, credential): # type: ignore[override] + if credential is not None and not request.headers.get("X-Test-User"): + return {"user_id": credential.credentials} + return super().authenticate(request, response, credential) + + +class _Harness: + """One app with every route that can carry or resolve a media reference.""" + + def __init__(self, tmp_path: Path, with_ag_ui: bool) -> None: + self.resolved: list[tuple[str, str]] = [] # (user_id, key) of every resolution + settings = MediaSettings( + MEDIA_STORAGE_TYPE=MediaStorageType.MEMORY, MEDIA_REQUIRE_OWNER=True + ) # type: ignore[call-arg] + self.media = MediaService(settings=settings) + self.checkpointer = InMemoryCheckpointer() + + config_file = tmp_path / "10xgraph.json" + config_file.write_text( + json.dumps( + { + "agent": "x:y", + "auth": {"method": "custom", "path": "x:y"}, + "ag_ui": {"enabled": with_ag_ui}, + "runs": {"thread_ids": "client"}, + } + ) + ) + self.config = GraphConfig(str(config_file)) + + graph = StateGraph(AgentState()) + graph.add_node("MAIN", self._resolve_node) + graph.set_entry_point("MAIN") + graph.add_edge("MAIN", END) + self.graph = graph.compile(checkpointer=self.checkpointer) + + # GraphService, CheckpointerService and the WebSocket handshake look services up in the + # active container (as in production, where the loader fills it): bind them there too. + active = InjectQ.get_instance() + for kind, value in ( + (MediaService, self.media), + (BaseAuth, _Auth()), + (AuthorizationBackend, DefaultAuthorizationBackend()), + (GraphConfig, self.config), + (BaseCheckpointer, self.checkpointer), + (CompiledGraph, self.graph), + ): + active.bind_instance(kind, value, allow_none=True) + + routers = [media_router, graph_router, checkpointer_router] + if with_ag_ui: + from tenxgraph_api.src.app.routers.ag_ui import router as ag_ui_router + + routers.append(ag_ui_router) + self.app = build_app( + routers=routers, + checkpointer=self.checkpointer, + extra_bindings={ + MediaService: self.media, + CompiledGraph: self.graph, + GraphConfig: self.config, + BaseCheckpointer: self.checkpointer, + }, + ) + self.client: TestClient = make_client(self.app) + + async def _resolve_node(self, state: AgentState, config: dict[str, Any]) -> Message: + """Resolve every internal media reference of the context, as the LLM call would.""" + user = config.get("user_id", "?") + context = [m.model_copy(deep=True) for m in state.context] + for message in context: + for key in internal_media_keys(message): + self.resolved.append((user, key)) + resolver = MediaRefResolver(media_store=self.media.store) + resolved = await resolve_media_refs(context, resolver) + found: list[str] = [] + for message in resolved: + for block in message.content: + media = getattr(block, "media", None) + if media is not None and media.kind == "data" and media.data_base64: + found.append(base64.b64decode(media.data_base64).decode()) + return Message(role="assistant", content=[TextBlock(text="resolved:" + "|".join(found))]) + + def upload(self, user: str, data: bytes) -> str: + resp = self.client.post( + "/v1/files/upload", + files={"file": ("note.txt", data, "text/plain")}, + headers=user_headers(user), + ) + assert resp.status_code == 200, resp.text + return resp.json()["data"]["file_id"] + + +@pytest.fixture +def h(tmp_path): + pytest.importorskip("ag_ui") + with InjectQ.test_mode(): + yield _Harness(tmp_path, with_ag_ui=True) + + +@pytest.fixture +def keys(h): + """Alice's file, Bob's own file.""" + return h.upload(ALICE, SECRET_A), h.upload(BOB, SECRET_B) + + +def _image(url: str | None = None, *, file_id: str | None = None) -> dict: + media: dict[str, Any] = {"kind": "file_id", "file_id": file_id} if file_id else {"kind": "url"} + if url: + media["url"] = url + return {"role": "user", "content": [{"type": "image", "media": media}]} + + +def _victim_variants(key: str) -> dict[str, dict]: + """Every way a client can point a message at Alice's file.""" + return { + "graph url": _image(f"graph://media/{key}"), + "legacy url": _image(f"agentflow://media/{key}"), + "file_id": _image(file_id=key), + "file_id with a url": _image(f"graph://media/{key}", file_id="not-mine"), + "document file_id": { + "role": "user", + "content": [{"type": "document", "media": {"kind": "file_id", "file_id": key}}], + }, + "nested in a tool result": { + "role": "tool", + "content": [ + { + "type": "tool_result", + "call_id": "c1", + "output": [ + {"type": "image", "media": {"kind": "url", "url": f"graph://media/{key}"}} + ], + } + ], + }, + } + + +def _never_alice(h: _Harness, key: str, who: str) -> None: + """The run did not resolve Alice's key for ``who``.""" + assert (who, key) not in h.resolved, f"{who} resolved {key}" + + +# --------------------------------------------------------------------------- media routes + + +def test_media_routes_refuse_another_users_file(h, keys): + alice_key, bob_key = keys + bob = user_headers(BOB) + + for path in ( + f"/v1/files/{alice_key}", + f"/v1/files/{alice_key}/info", + f"/v1/files/{alice_key}/url", + ): + resp = h.client.get(path, headers=bob) + assert resp.status_code == 404, f"{path} -> {resp.status_code}" + assert LEAK not in resp.text + + # B cannot delete it, and cannot tell it from a missing file. + assert h.client.delete(f"/v1/files/{alice_key}", headers=bob).status_code == 404 + missing = h.client.get("/v1/files/does-not-exist", headers=bob) + assert missing.status_code == 404 + + # B's listing holds B's file only. + listed = h.client.get("/v1/files", headers=bob) + assert listed.status_code == 200 + ids = [f["file_id"] for f in listed.json()["data"]["files"]] + assert ids == [bob_key] + + # Alice still has her file (the delete did nothing) and can read it. + own = h.client.get(f"/v1/files/{alice_key}", headers=user_headers(ALICE)) + assert own.status_code == 200 and own.content == SECRET_A + + +@pytest.mark.asyncio +async def test_media_service_denies_another_user_on_every_resolve_path(h, keys): + alice_key, _ = keys + service = h.media + with pytest.raises(PermissionError): + await service.ensure_can_access(alice_key, BOB) + with pytest.raises(PermissionError): + await service.get_file(alice_key, BOB) + with pytest.raises(PermissionError): + await service.get_file_info(alice_key, BOB) + with pytest.raises(KeyError): + await service.ensure_can_access("nope", BOB) + # The owner passes. + await service.ensure_can_access(alice_key, ALICE) + assert (await service.get_file(alice_key, ALICE))[0] == SECRET_A + + +@pytest.mark.asyncio +async def test_a_file_with_no_owner_is_denied_when_owners_are_required(h): + key = await h.media.store.store(b"legacy", "text/plain", metadata={}) + with pytest.raises(PermissionError): + await h.media.ensure_can_access(key, BOB) + + +# --------------------------------------------------------------------------- preprocessor + + +@pytest.mark.asyncio +async def test_preprocessor_rejects_every_variant(h, keys): + alice_key, bob_key = keys + for raw in _victim_variants(alice_key).values(): + message = Message.model_validate(raw) + with pytest.raises(ClientInputError, match="not found"): + await preprocess_multimodal_messages([message], h.media, BOB) + with pytest.raises(ClientInputError, match="not found"): + await check_media_references([message], h.media, BOB) + # Missing and foreign give the same error text, so existence is not confirmed. + for key in (alice_key, "missing-key"): + with pytest.raises(ClientInputError) as exc: + await preprocess_multimodal_messages( + [Message.model_validate(_image(f"graph://media/{key}"))], h.media, BOB + ) + assert str(exc.value).replace(key, "K") == "File not found: K" + # Own file is allowed, and the owner can use hers. + own = Message.model_validate(_image(f"graph://media/{bob_key}")) + assert await preprocess_multimodal_messages([own], h.media, BOB) + assert await preprocess_multimodal_messages( + [Message.model_validate(_image(f"graph://media/{alice_key}"))], h.media, ALICE + ) + + +# --------------------------------------------------------------------------- invoke / stream + + +@pytest.mark.parametrize("variant", list(_victim_variants("K"))) +def test_invoke_cannot_resolve_another_users_file(h, keys, variant): + alice_key, _ = keys + body = {"messages": [_victim_variants(alice_key)[variant]]} + resp = h.client.post("/v1/graph/invoke", json=body, headers=user_headers(BOB)) + assert resp.status_code in (400, 422), f"{variant}: {resp.status_code} {resp.text[:200]}" + assert LEAK not in resp.text + _never_alice(h, alice_key, BOB) + + +@pytest.mark.parametrize("variant", list(_victim_variants("K"))) +def test_stream_cannot_resolve_another_users_file(h, keys, variant): + alice_key, _ = keys + body = {"messages": [_victim_variants(alice_key)[variant]]} + resp = h.client.post("/v1/graph/stream", json=body, headers=user_headers(BOB)) + assert LEAK not in resp.text + assert "resolved:" not in resp.text + _never_alice(h, alice_key, BOB) + + +def test_invoke_and_stream_resolve_the_callers_own_file(h, keys): + """Control: the same request shape works for the owner, so the denials above are real.""" + alice_key, bob_key = keys + ref = {"messages": [_image(f"graph://media/{bob_key}")]} + invoked = h.client.post("/v1/graph/invoke", json=ref, headers=user_headers(BOB)) + assert invoked.status_code == 200, invoked.text + assert SECRET_B.decode() in invoked.text + streamed = h.client.post("/v1/graph/stream", json=ref, headers=user_headers(BOB)) + assert streamed.status_code == 200 and SECRET_B.decode() in streamed.text + assert (BOB, bob_key) in h.resolved + + own = {"messages": [_image(f"graph://media/{alice_key}")]} + assert LEAK in h.client.post("/v1/graph/invoke", json=own, headers=user_headers(ALICE)).text + + +def test_resume_tool_result_messages_are_checked_too(h, keys): + """The tool_result of a resume reaches the graph through the same input path.""" + alice_key, _ = keys + victim = _victim_variants(alice_key)["nested in a tool result"] + body = {"messages": [], "resume": None, "tool_result": [victim]} + resp = h.client.post("/v1/graph/invoke", json=body, headers=user_headers(BOB)) + assert LEAK not in resp.text + _never_alice(h, alice_key, BOB) + + +# --------------------------------------------------------------------------- WebSocket + + +def _ws_run(h: _Harness, user: str, message: dict) -> list[str]: + frames: list[str] = [] + headers = {**user_headers(user), "Authorization": f"Bearer {user}"} + with h.client.websocket_connect("/v1/graph/ws", headers=headers) as ws: + ws.send_json({"invoke_type": "fresh", "messages": [message]}) + for _ in range(60): + frame = ws.receive_text() # the run ends with an error frame or status "done" + frames.append(frame) + if '"event":"error"' in frame or '"status":"done"' in frame: + break + return frames + + +@pytest.mark.parametrize("variant", ["graph url", "legacy url", "file_id"]) +def test_websocket_run_cannot_resolve_another_users_file(h, keys, variant): + alice_key, _ = keys + frames = _ws_run(h, BOB, _victim_variants(alice_key)[variant]) + assert LEAK not in "".join(frames) + _never_alice(h, alice_key, BOB) + + +def test_websocket_run_resolves_the_callers_own_file(h, keys): + _, bob_key = keys + frames = _ws_run(h, BOB, _image(f"graph://media/{bob_key}")) + assert SECRET_B.decode() in "".join(frames) + + +def test_live_endpoint_accepts_no_message_or_media_reference(): + """``/v1/graph/live`` takes audio bytes and plain text controls; its init frame cannot + carry messages, state or a media reference, so nothing there resolves a stored file.""" + from tenxgraph_api.src.app.routers.graph.services.graph_service import REALTIME_INIT_FIELDS + + assert set(REALTIME_INIT_FIELDS) == { + "model", + "voice", + "modalities", + "vad", + "system_prompt", + "tools_tags", + } + assert not {"messages", "state", "context", "media", "file_id"} & set(REALTIME_INIT_FIELDS) + + +# --------------------------------------------------------------------------- AG-UI + + +def _ag_ui_body(thread_id: str, part: dict) -> dict: + return { + "threadId": thread_id, + "runId": "r1", + "state": {}, + "tools": [], + "context": [], + "forwardedProps": {}, + "messages": [{"id": "u1", "role": "user", "content": [part]}], + } + + +@pytest.mark.parametrize("source_kind", ["url-graph", "url-legacy", "file"]) +def test_ag_ui_cannot_resolve_another_users_file(h, keys, source_kind): + alice_key, _ = keys + source = { + "url-graph": {"type": "url", "value": f"graph://media/{alice_key}"}, + "url-legacy": {"type": "url", "value": f"agentflow://media/{alice_key}"}, + "file": {"type": "file", "value": alice_key}, + }[source_kind] + resp = h.client.post( + "/v1/ag-ui", + json=_ag_ui_body("t-bob-1", {"type": "image", "source": source}), + headers=user_headers(BOB), + ) + assert LEAK not in resp.text + assert "resolved:" not in resp.text + _never_alice(h, alice_key, BOB) + + +def test_ag_ui_resolves_the_callers_own_file(h, keys): + _, bob_key = keys + resp = h.client.post( + "/v1/ag-ui", + json=_ag_ui_body( + "t-bob-2", + {"type": "image", "source": {"type": "url", "value": f"graph://media/{bob_key}"}}, + ), + headers=user_headers(BOB), + ) + assert resp.status_code == 200 + assert SECRET_B.decode() in resp.text + + +# --------------------------------------------------------------------------- checkpointer + + +@pytest.mark.parametrize( + "variant", ["graph url", "legacy url", "file_id", "document file_id", "nested in a tool result"] +) +def test_writing_messages_into_a_thread_is_checked(h, keys, variant): + """A message written into a thread is resolved by the next run, so it is checked now.""" + alice_key, _ = keys + message = {**_victim_variants(alice_key)[variant], "message_id": "m1"} + resp = h.client.post( + "/v1/threads/bob-thread/messages", + json={"messages": [message]}, + headers=IMPORTER, + ) + assert resp.status_code in (400, 422), f"{variant}: {resp.status_code} {resp.text[:200]}" + assert LEAK not in resp.text + + +def test_writing_state_into_a_thread_is_checked(h, keys): + alice_key, _ = keys + message = {**_victim_variants(alice_key)["graph url"], "message_id": "m1"} + resp = h.client.put( + "/v1/threads/bob-thread/state", + json={"state": {"context": [message]}, "config": {}}, + headers=user_headers(BOB), + ) + assert resp.status_code in (400, 422), f"{resp.status_code} {resp.text[:200]}" + assert LEAK not in resp.text + + +def test_writing_the_callers_own_file_into_a_thread_is_allowed(h, keys): + _, bob_key = keys + message = {**_image(f"graph://media/{bob_key}"), "message_id": "m1"} + resp = h.client.post( + "/v1/threads/bob-thread/messages", + json={"messages": [message]}, + headers=IMPORTER, + ) + assert resp.status_code == 200, resp.text + + +# --------------------------------------------------------------------------- no media service + + +@pytest.mark.asyncio +async def test_references_are_refused_when_ownership_cannot_be_checked(): + message = Message( + role="user", content=[ImageBlock(media=MediaRef(kind="url", url="graph://media/x"))] + ) + with pytest.raises(ClientInputError, match="cannot be verified"): + await preprocess_multimodal_messages([message], None, BOB) + doc = Message( + role="user", content=[DocumentBlock(media=MediaRef(kind="url", url="agentflow://media/x"))] + ) + with pytest.raises(ClientInputError, match="cannot be verified"): + await preprocess_multimodal_messages([doc], None, BOB) + tool = Message( + role="tool", + content=[ + ToolResultBlock( + call_id="1", output=[{"type": "image", "media": {"url": "graph://media/x"}}] + ) + ], + ) + with pytest.raises(ClientInputError, match="cannot be verified"): + await preprocess_multimodal_messages([tool], None, BOB) + + +@pytest.mark.parametrize("variant", ["file_id", "document file_id"]) +def test_writing_state_with_a_foreign_file_id_is_refused(h, keys, variant): + alice_key, _ = keys + message = {**_victim_variants(alice_key)[variant], "message_id": "m1"} + resp = h.client.put( + "/v1/threads/bob-thread/state", + json={"state": {"context": [message]}, "config": {}}, + headers=user_headers(BOB), + ) + assert resp.status_code in (400, 422), resp.text[:200] + + +def test_writing_the_callers_own_file_id_is_allowed(h, keys): + _, bob_key = keys + message = {**_image(file_id=bob_key), "message_id": "m1"} + resp = h.client.post( + "/v1/threads/bob-thread/messages", json={"messages": [message]}, headers=IMPORTER + ) + assert resp.status_code == 200, resp.text diff --git a/tests/unit_tests/test_ownership_authorization.py b/tests/unit_tests/test_ownership_authorization.py index d1bb53cf..a34424b4 100644 --- a/tests/unit_tests/test_ownership_authorization.py +++ b/tests/unit_tests/test_ownership_authorization.py @@ -124,19 +124,24 @@ async def aget_thread_owner(self, thread_id): @pytest.mark.asyncio -async def test_no_checkpointer_fails_closed(): +async def test_no_checkpointer_fails_closed(monkeypatch): + from injectq import InjectQ + + # The property falls back to the global DI container, which other tests in a long run may + # have bound a checkpointer into. Point it at an empty container so "none configured" holds. + monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: InjectQ())) b = OwnershipAuthorizationBackend(checkpointer=None) - # Property tries the DI container; in this unit context nothing is bound -> None. - b._checkpointer = None assert b.checkpointer is None assert await b.authorize(ALICE, "checkpointer", "read", resource_id="t1") is False assert await b.authorize(ALICE, "graph", "invoke", resource_id="t1") is False @pytest.mark.asyncio -async def test_no_checkpointer_still_allows_requests_without_a_thread(): +async def test_no_checkpointer_still_allows_requests_without_a_thread(monkeypatch): + from injectq import InjectQ + + monkeypatch.setattr(InjectQ, "get_instance", classmethod(lambda cls: InjectQ())) b = OwnershipAuthorizationBackend(checkpointer=None) - b._checkpointer = None assert await b.authorize(ALICE, "checkpointer", "read", resource_id=None) is True diff --git a/tests/unit_tests/test_prod_template_validators.py b/tests/unit_tests/test_prod_template_validators.py new file mode 100644 index 00000000..c7b72de6 --- /dev/null +++ b/tests/unit_tests/test_prod_template_validators.py @@ -0,0 +1,33 @@ +"""The production template validator does not block ordinary text (G-56).""" + +import importlib.util +from pathlib import Path + +import pytest +from tenxgraph.core.state import Message + + +TEMPLATE = ( + Path(__file__).resolve().parents[2] + / "tenxgraph_api/cli/templates/prod/graph/validators/validators.py" +) + + +def _validator(): + spec = importlib.util.spec_from_file_location("prod_validators_template", TEMPLATE) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod.prompt_validator + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "text", + [ + "Is the free plan limited? I need a token for the API, and how do I disable alerts?", + "How do I override a default setting or bypass the proxy?", + "word " * 400, + ], +) +async def test_ordinary_text_passes(text): + assert await _validator().validate([Message.text_message(text, role="user")]) is True diff --git a/tests/unit_tests/test_readiness.py b/tests/unit_tests/test_readiness.py index 93acf86b..59ca212e 100644 --- a/tests/unit_tests/test_readiness.py +++ b/tests/unit_tests/test_readiness.py @@ -224,3 +224,34 @@ async def readiness_check(self): monkeypatch.setattr(readiness, "CACHE_SECONDS", 0.0) await readiness.cached_readiness(probe, None, None) assert len(calls) == 2 + + +# --------------------------------------------------------------------------- +# schema version (G-164) +# --------------------------------------------------------------------------- + + +class _PgWithVersion(_PgLike): + def __init__(self, found, expected=4): + super().__init__() + self._found, self._expected = found, expected + + async def aget_schema_version(self): + return self._found + + def _get_current_schema_version(self): + return self._expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("found", [0, 3]) +async def test_old_or_missing_schema_is_not_ready(found): + checks = await readiness.readiness(_PgWithVersion(found), None, None) + assert checks["checkpointer"] == "fail" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("found", [4, 5]) +async def test_current_schema_is_ready(found): + checks = await readiness.readiness(_PgWithVersion(found), None, None) + assert checks["checkpointer"] == "ok" diff --git a/tests/unit_tests/test_resource_id_rule.py b/tests/unit_tests/test_resource_id_rule.py new file mode 100644 index 00000000..153c0f64 --- /dev/null +++ b/tests/unit_tests/test_resource_id_rule.py @@ -0,0 +1,114 @@ +"""One rule for client-named ids (thread, message, memory): G-127.""" + +# ruff: noqa: S101 + +from typing import Any + +import pytest +from fastapi import Depends, FastAPI +from fastapi.testclient import TestClient +from injectq import InjectQ +from injectq.integrations.fastapi import setup_fastapi + +from tenxgraph_api.src.app.core.auth.authorization import ( + AuthorizationBackend, + DefaultAuthorizationBackend, +) +from tenxgraph_api.src.app.core.auth.base_auth import BaseAuth +from tenxgraph_api.src.app.core.auth.permissions import RequirePermission +from tenxgraph_api.src.app.core.auth.request_config import resource_id_error +from tenxgraph_api.src.app.core.config.graph_config import GraphConfig +from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler + + +class _Auth(BaseAuth): + def authenticate(self, request, response, credential): + return {"user_id": "alice"} + + +class _AuthOn: + introspection_scoped = False + + def auth_config(self): + return {"method": "custom", "path": "x:y"} + + +def _client() -> TestClient: + container = InjectQ() + container.bind_instance(GraphConfig, _AuthOn()) + container.bind_instance(BaseAuth, _Auth()) + container.bind_instance(AuthorizationBackend, DefaultAuthorizationBackend()) + app = FastAPI() + setup_fastapi(container, app) + init_errors_handler(app) + guard = Depends(RequirePermission("checkpointer", "read")) + + @app.get("/v1/threads/{thread_id}/messages/{message_id}") + async def message(thread_id: str, message_id: str, user: dict[str, Any] = guard): + return {"ok": True} + + @app.get("/v1/store/memories/{memory_id}") + async def memory(memory_id: str, user: dict[str, Any] = guard): + return {"ok": True} + + @app.post("/v1/graph/stop") + async def stop(body: dict[str, Any], user: dict[str, Any] = guard): + return {"ok": True} + + return TestClient(app) + + +_H = {"Authorization": "Bearer t"} +BAD_IDS = ["a" * 129, "a b", "a/b", "a\n", "é"] +GOOD_IDS = [ + "1234567890123456789", # Snowflake + "3f2b8c1e-9d4a-4c1b-8f6e-0a1b2c3d4e5f", # UUID + "mem0:user@example.com_1.2-3", + "a" * 128, +] + + +@pytest.mark.parametrize("value", GOOD_IDS) +def test_api_made_ids_pass(value): + assert resource_id_error(value, "thread_id") is None + assert resource_id_error(int(value) if value.isdigit() else value, "thread_id") is None + + +@pytest.mark.parametrize("value", [*BAD_IDS, "", ["x"], {"a": 1}]) +def test_odd_ids_fail(value): + assert resource_id_error(value, "thread_id") + + +def test_missing_id_is_not_this_rules_business(): + assert resource_id_error(None, "thread_id") is None + + +@pytest.mark.parametrize("value", ["a" * 129, "a b", "a;b"]) +def test_bad_path_ids_are_422_validation_error(value): + client = _client() + for url in ( + f"/v1/threads/{value}/messages/m1", + f"/v1/threads/t1/messages/{value}", + f"/v1/store/memories/{value}", + ): + res = client.get(url, headers=_H) + assert res.status_code == 422, url + assert res.json()["error"]["code"] == "VALIDATION_ERROR" + + +def test_good_path_ids_reach_the_route(): + client = _client() + for value in GOOD_IDS: + assert client.get(f"/v1/threads/{value}/messages/{value}", headers=_H).status_code == 200 + assert client.get(f"/v1/store/memories/{value}", headers=_H).status_code == 200 + + +def test_bad_body_thread_id_is_422(): + client = _client() + res = client.post("/v1/graph/stop", json={"thread_id": "x" * 5000}, headers=_H) + assert res.status_code == 422 + assert res.json()["error"]["code"] == "VALIDATION_ERROR" + res = client.post("/v1/graph/stop", json={"config": {"thread_id": "a b"}}, headers=_H) + assert res.status_code == 422 + ok = client.post("/v1/graph/stop", json={"thread_id": GOOD_IDS[0]}, headers=_H) + assert ok.status_code == 200 diff --git a/tests/unit_tests/test_shutdown_wiring.py b/tests/unit_tests/test_shutdown_wiring.py new file mode 100644 index 00000000..8ab81d24 --- /dev/null +++ b/tests/unit_tests/test_shutdown_wiring.py @@ -0,0 +1,92 @@ +"""The lifespan hands shutdown to core ``graph.aclose`` (plan Batch F).""" + +# ruff: noqa: S101 + +import asyncio +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient +from tenxgraph.core.exceptions import GraphClosedError + +from tenxgraph_api.src.app import main as app_main +from tenxgraph_api.src.app.core.exceptions.handle_errors import init_errors_handler + + +def _graph(order: list[str]) -> AsyncMock: + graph = AsyncMock() + + async def aclose(**kwargs): + order.append("aclose") + graph.kwargs = kwargs + return {"runs": {}} + + graph.aclose = AsyncMock(side_effect=aclose) + return graph + + +@pytest.mark.asyncio +async def test_lifespan_calls_aclose_with_the_configured_times(): + order: list[str] = [] + graph = _graph(order) + with ( + patch.object(app_main, "attach_all_modules", AsyncMock(return_value=graph)), + patch.object(app_main, "_cleanup_temp_media_cache", AsyncMock()), + patch.object(app_main.settings, "SHUTDOWN_GRACE_SECONDS", 3.0), + patch.object(app_main.settings, "SHUTDOWN_STOP_SECONDS", 2.0), + patch.object(app_main.settings, "SHUTDOWN_TIMEOUT_SECONDS", 7.0), + ): + app = FastAPI() + async with app_main.lifespan(app): + pass + graph.aclose.assert_awaited_once_with(grace=3.0, stop_timeout=2.0, timeout=7.0) + assert app.state.draining is True + + +@pytest.mark.asyncio +async def test_sweeper_is_cancelled_before_the_graph_closes(): + order: list[str] = [] + graph = _graph(order) + + async def sweep_loop(interval): + try: + await asyncio.sleep(3600) + except asyncio.CancelledError: + order.append("sweeper_cancelled") + raise + + with ( + patch.object(app_main, "attach_all_modules", AsyncMock(return_value=graph)), + patch.object(app_main, "_cleanup_temp_media_cache", AsyncMock()), + patch.object(app_main, "_sweep_media_loop", sweep_loop), + ): + async with app_main.lifespan(FastAPI()): + await asyncio.sleep(0) # let the sweeper start + assert order == ["sweeper_cancelled", "aclose"] + + +@pytest.mark.asyncio +async def test_a_failing_close_does_not_stop_the_rest_of_shutdown(): + graph = AsyncMock() + graph.aclose = AsyncMock(side_effect=RuntimeError("boom")) + with ( + patch.object(app_main, "attach_all_modules", AsyncMock(return_value=graph)), + patch.object(app_main, "_cleanup_temp_media_cache", AsyncMock()), + ): + async with app_main.lifespan(FastAPI()): + pass # exiting must not raise + + +def test_graph_closed_error_is_503_with_retry_after(): + app = FastAPI() + init_errors_handler(app) + + @app.get("/run") + async def run(): + raise GraphClosedError() + + res = TestClient(app).get("/run") + assert res.status_code == 503 + assert res.json()["error"]["code"] == "SERVICE_UNAVAILABLE" + assert int(res.headers["Retry-After"]) >= 1 diff --git a/tests/unit_tests/test_static_pause_resume.py b/tests/unit_tests/test_static_pause_resume.py index feaa63b2..d8f71ac4 100644 --- a/tests/unit_tests/test_static_pause_resume.py +++ b/tests/unit_tests/test_static_pause_resume.py @@ -64,3 +64,16 @@ async def test_remote_tool_pause_keeps_the_resume_key(): state.execution_meta.interrupt_reason = "remote_tool_call" input_data = await _prepare(state, resume=None) assert "resume" in input_data + + +def test_core_resume_errors_answer_409(): + from tenxgraph.core.exceptions import InterruptPendingError, NothingToResumeError + + from tenxgraph_api.src.app.routers.graph.services.graph_service import ( + _resume_error_types, + _resume_http_error, + ) + + for exc in (NothingToResumeError(), InterruptPendingError("paused at i1")): + assert isinstance(exc, _resume_error_types()) + assert _resume_http_error(exc).status_code == 409 diff --git a/tests/unit_tests/test_token_revocation.py b/tests/unit_tests/test_token_revocation.py index edead7dc..f95f4903 100644 --- a/tests/unit_tests/test_token_revocation.py +++ b/tests/unit_tests/test_token_revocation.py @@ -20,9 +20,9 @@ ) +# Reference time for issued-at values only; token expiry is computed in each test. NOW = time.time() - # --------------------------------------------------------------------------- # in-memory backend # --------------------------------------------------------------------------- @@ -31,7 +31,7 @@ @pytest.mark.asyncio async def test_memory_revokes_one_token_by_jti(): backend = InMemoryTokenRevocation() - backend.revoke_token("t-1", expires_at=NOW + 60) + backend.revoke_token("t-1", expires_at=time.time() + 60) assert await backend.is_revoked({"user_id": "a", "jti": "t-1"}) is True assert await backend.is_revoked({"user_id": "a", "jti": "t-2"}) is False assert await backend.is_revoked({"user_id": "a"}) is False @@ -40,7 +40,7 @@ async def test_memory_revokes_one_token_by_jti(): @pytest.mark.asyncio async def test_memory_drops_entries_once_the_token_expired(): backend = InMemoryTokenRevocation() - backend.revoke_token("old", expires_at=NOW - 1) + backend.revoke_token("old", expires_at=time.time() - 1) assert await backend.is_revoked({"user_id": "a", "jti": "old"}) is False assert backend._tokens == {} diff --git a/tests/unit_tests/test_websocket_realtime.py b/tests/unit_tests/test_websocket_realtime.py index 75e3a3b0..6ce2519a 100644 --- a/tests/unit_tests/test_websocket_realtime.py +++ b/tests/unit_tests/test_websocket_realtime.py @@ -56,6 +56,7 @@ def _make_service(events: list): service = MagicMock() service.is_live_agent = True # live graph: allowed on /v1/graph/live service.runs_policy.return_value = RunsConfig() + service.new_thread_id = AsyncMock(return_value="srv-1") captured = {} async def _gen(input_queue, init, user): @@ -160,6 +161,7 @@ async def test_downstream_error_closes_with_1011(self): ws = _make_websocket([WebSocketDisconnect()], init={"model": "gemini-2.5-flash-live"}) service = MagicMock() service.runs_policy.return_value = RunsConfig() + service.new_thread_id = AsyncMock(return_value="srv-1") async def _boom(input_queue, init, user): raise RuntimeError("graph has no LiveAgent") @@ -234,6 +236,7 @@ async def _slow_gen(input_queue, init, user): service = MagicMock() service.runs_policy.return_value = RunsConfig() + service.new_thread_id = AsyncMock(return_value="srv-1") service.realtime_graph = _slow_gen ws = _make_websocket( @@ -259,6 +262,7 @@ async def _boom(input_queue, init, user): service = MagicMock() service.runs_policy.return_value = RunsConfig() + service.new_thread_id = AsyncMock(return_value="srv-1") service.realtime_graph = _boom ws = _make_websocket([WebSocketDisconnect()], init={"model": "gemini-x"}) @@ -271,6 +275,76 @@ async def _boom(input_queue, init, user): assert errors[0]["code"] == "invalid_config" +class TestLiveThreadIds: + """G-113/R-113: /v1/graph/live follows ``runs.thread_ids`` and returns the thread id.""" + + @staticmethod + def _authz(monkeypatch, *, thread_ids): + from injectq import InjectQ + from tenxgraph.storage.checkpointer import InMemoryCheckpointer + + from tenxgraph_api.src.app.core.auth.authorization import OwnershipAuthorizationBackend + + container = MagicMock() + container.get.return_value = SimpleNamespace(auth_config=lambda: "jwt") + monkeypatch.setattr(InjectQ, "get_instance", staticmethod(lambda: container)) + runs = RunsConfig(thread_ids=thread_ids) + return OwnershipAuthorizationBackend( + checkpointer=InMemoryCheckpointer(), server_thread_ids=runs.server_thread_ids + ) + + @staticmethod + def _sent(ws): + return [json.loads(c.args[0]) for c in ws.send_text.call_args_list] + + @pytest.mark.asyncio + @pytest.mark.parametrize("init", [{}, {"thread_id": ""}, {"thread_id": "new"}]) + async def test_no_id_gets_a_server_id_returned_first(self, monkeypatch, init): + authz = self._authz(monkeypatch, thread_ids="server") + ws = _make_websocket([WebSocketDisconnect()], init={"model": "m", **init}) + service = _make_service([_json_event("turn_complete")]) + + await realtime_graph_ws( + websocket=ws, service=service, user={"user_id": "u1"}, authz=authz + ) + + service.new_thread_id.assert_awaited_once() + assert service._captured["init"]["thread_id"] == "srv-1" + sent = self._sent(ws) + assert sent[0] == {"type": "session_started", "thread_id": "srv-1"} + assert not any(e.get("code") == "not_authorized" for e in sent) + + @pytest.mark.asyncio + async def test_unknown_id_is_refused_under_server(self, monkeypatch): + authz = self._authz(monkeypatch, thread_ids="server") + ws = _make_websocket([WebSocketDisconnect()], init={"model": "m", "thread_id": "made-up"}) + service = _make_service([]) + + await realtime_graph_ws( + websocket=ws, service=service, user={"user_id": "u1"}, authz=authz + ) + + sent = self._sent(ws) + assert any(e.get("code") == "not_authorized" for e in sent) + assert not any(e.get("type") == "session_started" for e in sent) + assert any(c.kwargs.get("code") == 1008 for c in ws.close.call_args_list) + assert "init" not in service._captured + + @pytest.mark.asyncio + async def test_unknown_id_is_allowed_under_client(self, monkeypatch): + authz = self._authz(monkeypatch, thread_ids="client") + ws = _make_websocket([WebSocketDisconnect()], init={"model": "m", "thread_id": "mine"}) + service = _make_service([]) + + await realtime_graph_ws( + websocket=ws, service=service, user={"user_id": "u1"}, authz=authz + ) + + service.new_thread_id.assert_not_awaited() + assert service._captured["init"]["thread_id"] == "mine" + assert self._sent(ws)[0] == {"type": "session_started", "thread_id": "mine"} + + class TestIsLiveAgent: """GraphService.is_live_agent resolves live detection across core versions.""" @@ -623,3 +697,71 @@ async def test_consumed_bytes_free_the_budget(self): assert [i.kind for i in got] == ["audio"] q.send_audio(b"c") # closed queue drops and does not leak budget assert q._buffered == 0 + + +class TestConcurrentLiveSessions: + """Two live connections on one worker's graph must not share provider state (G-10).""" + + @pytest.mark.asyncio + async def test_concurrent_sessions_get_their_own_provider_client(self): + import asyncio + + from tenxgraph.core.realtime.base import AudioDeltaEvent + from tenxgraph.prebuilt.agent import AudioAgent + + from tenxgraph_api.src.app.routers.graph.services.graph_service import GraphService + + class Client: + def __init__(self, tag): + self.tag, self.audio, self.closed = tag, [], False + self.connected = asyncio.Event() + + async def connect(self, config, resume_handle=None): + self.connected.set() + + async def send_audio(self, pcm, sample_rate): + self.audio.append(pcm) + + async def receive(self): + yield AudioDeltaEvent(data=self.tag) + await asyncio.Event().wait() + + async def close(self): + self.closed = True + + clients = [Client(b"A"), Client(b"B")] + pending = list(clients) + app = AudioAgent( + "gemini-2.5-flash-live", realtime_client_factory=lambda: pending.pop(0) + ).compile() + svc = GraphService(graph=app, checkpointer=_checkpointer(), config=_config()) + queues = {} + + async def session(name): + from tenxgraph_api.src.app.routers.graph.router import _ByteBoundedLiveInputQueue + + queues[name] = _ByteBoundedLiveInputQueue() + return [ + e + async for e in svc.realtime_graph( + queues[name], {"thread_id": name}, {"user_id": name} + ) + ] + + ta = asyncio.create_task(session("ta")) + await clients[0].connected.wait() + tb = asyncio.create_task(session("tb")) + await clients[1].connected.wait() + + queues["ta"].send_audio(b"1") + queues["tb"].send_audio(b"2") + await asyncio.sleep(0.1) + queues["ta"].close() + await asyncio.wait_for(ta, 2) + + assert clients[0].audio == [b"1"] + assert clients[1].audio == [b"2"] + assert clients[0].closed is True + assert clients[1].closed is False + queues["tb"].close() + await asyncio.wait_for(tb, 2) diff --git a/uv.lock b/uv.lock index ed18791b..1750985e 100644 --- a/uv.lock +++ b/uv.lock @@ -41,6 +41,7 @@ dependencies = [ { name = "python-multipart" }, { name = "questionary" }, { name = "rich" }, + { name = "starlette" }, { name = "typer" }, { name = "uvicorn" }, ] @@ -113,20 +114,21 @@ requires-dist = [ { name = "firebase-admin", marker = "extra == 'firebase'", specifier = ">=6.5.0" }, { name = "google-cloud-logging", marker = "extra == 'gcloud'" }, { name = "oauth2client", marker = "extra == 'firebase'", specifier = ">=4.1.3" }, - { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.20.0" }, + { name = "opentelemetry-api", marker = "extra == 'otel'", specifier = ">=1.45.1" }, { name = "opentelemetry-exporter-otlp", marker = "extra == 'otel'", specifier = ">=1.20.0" }, { name = "opentelemetry-instrumentation-fastapi", marker = "extra == 'otel'", specifier = ">=0.40b0" }, - { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.20.0" }, + { name = "opentelemetry-sdk", marker = "extra == 'otel'", specifier = ">=1.45.1" }, { name = "pydantic", specifier = ">=2.13,<3" }, { name = "pydantic-settings", specifier = ">=2.3,<3" }, - { name = "pyjwt", marker = "extra == 'jwt'", specifier = ">=2.8,<3" }, + { name = "pyjwt", marker = "extra == 'jwt'", specifier = ">=2.15.0,<3" }, { name = "python-dotenv", specifier = ">=1.0,<2" }, - { name = "python-multipart", specifier = ">=0.0.19,<0.1" }, + { name = "python-multipart", specifier = ">=0.0.31,<0.1" }, { name = "questionary", specifier = ">=2.0,<3" }, { name = "redis", marker = "extra == 'redis'", specifier = ">=5.0.7,<7" }, { name = "rich", specifier = ">=14.0,<15" }, { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.10.0" }, { name = "snowflakekit", marker = "extra == 'snowflakekit'", specifier = ">=0.1.1,<1.0" }, + { name = "starlette", specifier = ">=1.3.1" }, { name = "textxtract", extras = ["pdf", "docx", "html", "xml", "md"], marker = "extra == 'media'", specifier = ">=0.2.0" }, { name = "typer", specifier = ">=0.17,<1.0" }, { name = "uvicorn", specifier = ">=0.36,<1.0" }, @@ -1313,80 +1315,112 @@ wheels = [ [[package]] name = "opentelemetry-api" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + +[[package]] +name = "opentelemetry-exporter-http-transport" +version = "0.66b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/62/0c/e3ebdb4b507f66afcc905e6885a4946969bd75b45988492643356fbbdc63/opentelemetry_exporter_http_transport-0.66b1.tar.gz", hash = "sha256:443080203bf52586ce0b2ad901e8951c61833eab1aa539ae6f1f16fe9e8e7952", size = 11693, upload-time = "2026-10-06T17:32:59.65Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/69/6af86ff66492b481c6a4c05dcfd68beb47ed8ba046440a26a2aac76b95c7/opentelemetry_exporter_http_transport-0.66b1-py3-none-any.whl", hash = "sha256:2f95404bdee7f9d2d529c7de56c7bd86d014d774d8fbf137810e0167f8a492bf", size = 12155, upload-time = "2026-10-06T17:32:35.454Z" }, +] + +[package.optional-dependencies] +requests = [ + { name = "requests" }, ] [[package]] name = "opentelemetry-exporter-otlp" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-exporter-otlp-proto-grpc" }, { name = "opentelemetry-exporter-otlp-proto-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f2/45/7af37fe54e5d3e66e7dcd7ba8b8aeee73f202bfac909cc94b8c4e428f9ac/opentelemetry_exporter_otlp-1.44.0.tar.gz", hash = "sha256:af1cde7c33ea8ed624bf04ac49a885730fe44c1f1ad698656e592c38f70ce106", size = 6090, upload-time = "2026-07-16T15:25:34.585Z" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/6f/5a561048ea372894f22f58f70e8478e4cd97d9a4e2d6f014559584a29bc5/opentelemetry_exporter_otlp-1.45.1.tar.gz", hash = "sha256:d0ac35592e77663a9fabf2740b4818c57196b1b764e0c8449d0d7bb2c7b2bc67", size = 6094, upload-time = "2026-10-06T17:33:00.966Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/e1/68c28d7da4482ce882a640b2c61405a78284c0a3a17a4db125647189b0b7/opentelemetry_exporter_otlp-1.45.1-py3-none-any.whl", hash = "sha256:ef3910d32b36ccbaf62390189759bd43a8109885e2d50b738ed9c9b255bf5cb5", size = 6730, upload-time = "2026-10-06T17:32:37.236Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-common" +version = "0.66b1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-sdk" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cb/19/41de712173f43057e4532d42ece7d0c6d4210d353e5752433cb14987643f/opentelemetry_exporter_otlp_common-0.66b1.tar.gz", hash = "sha256:6b1403487a2185ac1feb45fd5546fdf8630ce71c36bcefaadf51e2130e9e23f9", size = 14325, upload-time = "2026-10-06T17:33:01.725Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/18/c3/7b466a9463944e70b37b744072a0c1b88a425dade3fff0631adec66c9bcc/opentelemetry_exporter_otlp-1.44.0-py3-none-any.whl", hash = "sha256:4a498fa8d8fd8be9e8e2d175fe5524a3fe581ccffadd8509db86526a5fb97051", size = 6727, upload-time = "2026-07-16T15:25:14.445Z" }, + { url = "https://files.pythonhosted.org/packages/fc/39/8c23d67665c762aa51840fa06f86e902e8f6f1693bc8d7e3d98cd6e2f753/opentelemetry_exporter_otlp_common-0.66b1-py3-none-any.whl", hash = "sha256:00ff8592c3a7cb729ff3fdc7ffa12372c243bdf2163e80c180994d0c7bd83ee9", size = 12385, upload-time = "2026-10-06T17:32:38.177Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-common" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-proto" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c1/8e/65e85e5137991a3c493b11682151d198638a5bc1dd4b4c5f67e013c57d7c/opentelemetry_exporter_otlp_proto_common-1.45.1.tar.gz", hash = "sha256:2e4adcc3a67bcf57804fc49514f0ef64974ca7590aa3491da389852b4a0628f6", size = 18873, upload-time = "2026-10-06T17:33:04.471Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/aa/92f225d353904e7f70b8b3e3c1b02db0cf56f744c2e83c581dc372e78873/opentelemetry_exporter_otlp_proto_common-1.45.1-py3-none-any.whl", hash = "sha256:2f446183ae7047b036226f1d846c41a834b0e8755ad13b51a51dd38952eb466c", size = 15393, upload-time = "2026-10-06T17:32:41.911Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-grpc" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, { name = "grpcio" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-proto-common" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/47/80d9e9d468dc5de3af5096f5ccdb065fa4dd1470f74495cc53e59e397f47/opentelemetry_exporter_otlp_proto_grpc-1.44.0.tar.gz", hash = "sha256:40d1ae9e03fcc36de3cbac610cc99f35894938bff9cfd90fc4ec68bd85448463", size = 27225, upload-time = "2026-07-16T15:25:38.308Z" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/00/a82af0be959dc58495740b169c6669a86e0811f6cd353a01eda34d255db3/opentelemetry_exporter_otlp_proto_grpc-1.45.1.tar.gz", hash = "sha256:3b3dcfbfdcb4e35149fcf309972282054b45228f5c10547d0095d6578510a9a0", size = 25997, upload-time = "2026-10-06T17:33:05.114Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/54/29/6ae42ba32b153ae0a44ae125f0caff2188bbe62d99c82d1768da30864e72/opentelemetry_exporter_otlp_proto_grpc-1.44.0-py3-none-any.whl", hash = "sha256:6a1a645ea182a2f59440c51fa8301d309f3324a8f9d65f8395584b064b67ee4e", size = 19624, upload-time = "2026-07-16T15:25:19.096Z" }, + { url = "https://files.pythonhosted.org/packages/4d/46/2d1da202f1e17c81aae7efcf702898d524b46709e4d3e2bf1f7f8ca8fbc6/opentelemetry_exporter_otlp_proto_grpc-1.45.1-py3-none-any.whl", hash = "sha256:e42ecb789d2fc5d8145e3dadc3e2991c9f18cd166d7c7514e234702540274b76", size = 19492, upload-time = "2026-10-06T17:32:42.838Z" }, ] [[package]] name = "opentelemetry-exporter-otlp-proto-http" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos" }, { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-http-transport", extra = ["requests"] }, + { name = "opentelemetry-exporter-otlp-common" }, { name = "opentelemetry-exporter-otlp-proto-common" }, { name = "opentelemetry-proto" }, { name = "opentelemetry-sdk" }, { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1b/17/26487707ea4caa97b17e6e4b5fa72133a53512ffa2f5cf7a49ef284b29cb/opentelemetry_exporter_otlp_proto_http-1.45.1.tar.gz", hash = "sha256:45c218405ce3fd879596924b1874bf9a8f6880206d61065c5a912c8e5c297fb7", size = 28839, upload-time = "2026-10-06T17:33:05.713Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, + { url = "https://files.pythonhosted.org/packages/aa/1f/517eaa0187ba106a9da97160ce2add3a371812681dc440930b267f714e42/opentelemetry_exporter_otlp_proto_http-1.45.1-py3-none-any.whl", hash = "sha256:24a97cf3753c7fb52fad44a696e452ff371686339e2acf3309e2eda3d0230700", size = 22180, upload-time = "2026-10-06T17:32:43.946Z" }, ] [[package]] name = "opentelemetry-instrumentation" -version = "0.65b0" +version = "0.66b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -1394,14 +1428,14 @@ dependencies = [ { name = "packaging" }, { name = "wrapt" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/13/91/3c58961cb0360cd60509064734f0be4275383c8681d73c580a40ca83ddce/opentelemetry_instrumentation-0.65b0.tar.gz", hash = "sha256:071d9d9eced9bd6460444ec3b0c77229870ed05a881c22c84fdede58e4eed09b", size = 42689, upload-time = "2026-07-16T15:25:50.275Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a5/03/89e47ff8d52a4f83b343e6eb9ef1698ff45357216e5b6b2b21e0da5c5c7d/opentelemetry_instrumentation-0.66b1.tar.gz", hash = "sha256:e79a510f7d87c72d95e964ddb42193a0d9a75668c027d980eab032ea1322a5ce", size = 43308, upload-time = "2026-10-06T17:36:10.703Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/40/7b/85eab1215f72adf0e68d3dc4a679b9bff993fa679ff34cd8dd378e2659fd/opentelemetry_instrumentation-0.65b0-py3-none-any.whl", hash = "sha256:ea967a72b9939b5fcfdad572753b4306c59dcb99e3f382d95dae04286805e137", size = 36717, upload-time = "2026-07-16T15:24:51.424Z" }, + { url = "https://files.pythonhosted.org/packages/da/b2/d1413681ff43e13ac9860df27e1226d3199ab0b97b352ceea41abcc660a5/opentelemetry_instrumentation-0.66b1-py3-none-any.whl", hash = "sha256:4c4aa14dc9a24a02325a9d4c42c4d0208dbb1374c2b1b8fe6c9392d59f3e1008", size = 36904, upload-time = "2026-10-06T17:35:11.663Z" }, ] [[package]] name = "opentelemetry-instrumentation-asgi" -version = "0.65b0" +version = "0.66b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "asgiref" }, @@ -1410,14 +1444,14 @@ dependencies = [ { name = "opentelemetry-semantic-conventions" }, { name = "opentelemetry-util-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/17/83/8e8e83b7ac285281687c7be2fd305213ccccbb8c0a2dd4fb45a8ccaf12c7/opentelemetry_instrumentation_asgi-0.65b0.tar.gz", hash = "sha256:892bca67c56522ffa85a8a83cf934d7b50b3be2132e45cbee705825f0a5ba426", size = 26140, upload-time = "2026-07-16T15:25:54.544Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/d9/ff522f5c3e340e9007554923b1a4d2ac451676f8757bafb3d0057f68b5c3/opentelemetry_instrumentation_asgi-0.66b1.tar.gz", hash = "sha256:78cdc5e45e897e16a8dac9d282e8d5bdf9af2d58e1313fa0bdd4a134c6f9dafc", size = 25992, upload-time = "2026-10-06T17:36:14.593Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/9c/376962840b619d2d55fe8ee2285f8c70971c090e5fff614516fc654a6f3a/opentelemetry_instrumentation_asgi-0.65b0-py3-none-any.whl", hash = "sha256:3a845a8ebd1c4ef0d8263401e6545f5b219b2feee612090d50f578a87e71fd65", size = 15903, upload-time = "2026-07-16T15:24:57.198Z" }, + { url = "https://files.pythonhosted.org/packages/67/ea/10ba99110bf3c9fb736af39c96ca8f3668b988cabb6b59309e058c44461c/opentelemetry_instrumentation_asgi-0.66b1-py3-none-any.whl", hash = "sha256:78b3f9bdf0fa38c65935a2ab46d59e0f9de873a51e0c95b0329f106e2ccb5274", size = 15812, upload-time = "2026-10-06T17:35:17.638Z" }, ] [[package]] name = "opentelemetry-instrumentation-fastapi" -version = "0.65b0" +version = "0.66b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, @@ -1426,57 +1460,57 @@ dependencies = [ { name = "opentelemetry-semantic-conventions" }, { name = "opentelemetry-util-http" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/30/23/b057f8196d06efdc1b50e3ff11fbc499a7d96b35c87f217eb7885542f4ea/opentelemetry_instrumentation_fastapi-0.65b0.tar.gz", hash = "sha256:10a3a95486036230413a58fe4fdf4a83fa6bba46918407e527476994bd92bd97", size = 26236, upload-time = "2026-07-16T15:26:05.954Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5d/2a/cd4125b7acbea2ed17f1d31b58c184cb0a79fcb5541ceb4de90ffc6d8c01/opentelemetry_instrumentation_fastapi-0.66b1.tar.gz", hash = "sha256:584cf9d2c4417ff8b2d6ff2bc606bfe13c8b3456018bf94f50f2cf658492505b", size = 26634, upload-time = "2026-10-06T17:36:25.157Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fa/b0/c9b0300d33349ecc3dfd2362516eaffc44877e90970e6a52178ff953fec3/opentelemetry_instrumentation_fastapi-0.65b0-py3-none-any.whl", hash = "sha256:cda2610a0ec1b22d19886f33e4d861e9f5dbb886aeaa3a1263b47aff82c36943", size = 13261, upload-time = "2026-07-16T15:25:12.429Z" }, + { url = "https://files.pythonhosted.org/packages/75/70/676928d537978acc7bff2ac8657bd0836ba608ffc8f65455238f1fa2bd0f/opentelemetry_instrumentation_fastapi-0.66b1-py3-none-any.whl", hash = "sha256:97f8ac8fd7537517f9e6988bd0aca04bfa5aad564bcd46c245530739e2be72d1", size = 14216, upload-time = "2026-10-06T17:35:32.827Z" }, ] [[package]] name = "opentelemetry-proto" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4b/7f/15f014fb195da6c2dbb6c71399b8e76824878718e94de6454038488eed28/opentelemetry_proto-1.45.1.tar.gz", hash = "sha256:79e0fb95e4616691a469439238aa9224d75779b3e108e895d1aa125ab29ca77c", size = 46488, upload-time = "2026-10-06T17:33:11.49Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, + { url = "https://files.pythonhosted.org/packages/ab/9a/42ec8180a769516ae757e893b69736826efceac7332553915b4528a91c6d/opentelemetry_proto-1.45.1-py3-none-any.whl", hash = "sha256:f38e2a8413053c180cd3d2637fbb279673ec2f6a6e09c995aafa2f452c52b46e", size = 72488, upload-time = "2026-10-06T17:32:53.057Z" }, ] [[package]] name = "opentelemetry-sdk" -version = "1.44.0" +version = "1.45.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "opentelemetry-semantic-conventions" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/79/7392e21a1c8f0c61d90b223e31c7e48cb9d452e91a6b820ad24cca5f23c4/opentelemetry_sdk-1.45.1.tar.gz", hash = "sha256:63d24a6ca645019a631e6a51999c73e93adcac1196ca640b8ae78a7cc4762bf3", size = 218324, upload-time = "2026-10-06T17:33:13.26Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, + { url = "https://files.pythonhosted.org/packages/95/3c/87c42b4bd6dd297536f04cd9383d212ac557ecd49f2cbdcd46da1c9ef5c8/opentelemetry_sdk-1.45.1-py3-none-any.whl", hash = "sha256:c604c11dc429810812348989115fa44bd558772a3d7442afc43d024f2c250ca4", size = 140063, upload-time = "2026-10-06T17:32:55.04Z" }, ] [[package]] name = "opentelemetry-semantic-conventions" -version = "0.65b0" +version = "0.66b1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "opentelemetry-api" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +sdist = { url = "https://files.pythonhosted.org/packages/46/e4/dbbfb2a010c4db2224a5114638acede6fe563d33cc20fb1752cebcbe6298/opentelemetry_semantic_conventions-0.66b1.tar.gz", hash = "sha256:497ca63bf383723411e8eaf60c8779e9877633c936bb641080adab59d0eb6ec8", size = 150250, upload-time = "2026-10-06T17:33:14.073Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, + { url = "https://files.pythonhosted.org/packages/bc/14/67f8aa798857f8cf686f515bf93d9bb877ce952ddc8efae0fa25b45ce0d6/opentelemetry_semantic_conventions-0.66b1-py3-none-any.whl", hash = "sha256:d4cddeb4315490b35213f55e2bdc9ac54bb1e4d318927475bed62b35545e581b", size = 206279, upload-time = "2026-10-06T17:32:56.103Z" }, ] [[package]] name = "opentelemetry-util-http" -version = "0.65b0" +version = "0.66b1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/32/a9/d7525a59fdd240e69b5af4a6338e78fafa1b4203394122cbd6701fb5f84a/opentelemetry_util_http-0.65b0.tar.gz", hash = "sha256:84f82d826978bba416ab453460ff6a7391cdc3534c93a786595e4068680016b7", size = 11243, upload-time = "2026-07-16T15:26:27.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7c/b5/df4b61da899f6ebdffdbdf0c8b0f3189ee57151694ccd5b7d50ee2906241/opentelemetry_util_http-0.66b1.tar.gz", hash = "sha256:047dea1a628031f857a5a32261dc0e955bc162d39993ed1cffb8f2cff5ba8a62", size = 11586, upload-time = "2026-10-06T17:36:46.572Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/23/3f/ab8d29df207ce5f470a07fa96ebb48af4e95b7fab7e7635311b9a32f2fab/opentelemetry_util_http-0.65b0-py3-none-any.whl", hash = "sha256:7553b606f963097cb190536dc30556cce85090692e471a422fff30ca29b04348", size = 8245, upload-time = "2026-07-16T15:25:46.482Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9b/c77ecaea79ba0de1a11e7f06a7f5eea7043ec23f1860dcf5f03536698e4c/opentelemetry_util_http-0.66b1-py3-none-any.whl", hash = "sha256:8f443d7abcaf29c4a07b373bbd31b5b39132c0ed3c27d015a59dc0323d5b1c58", size = 8387, upload-time = "2026-10-06T17:36:06.984Z" }, ] [[package]] @@ -1786,11 +1820,11 @@ wheels = [ [[package]] name = "pyjwt" -version = "2.13.0" +version = "2.15.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, ] [package.optional-dependencies] From 3a3c51eec11810a61264d65ecd9b3ac02dee9845 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 21:12:01 +0600 Subject: [PATCH 07/16] fix: improve exception handling and comments in init, test, and server error modules --- tenxgraph_api/cli/commands/init.py | 2 +- tenxgraph_api/cli/commands/test.py | 2 +- tenxgraph_api/src/app/core/exceptions/server_errors.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tenxgraph_api/cli/commands/init.py b/tenxgraph_api/cli/commands/init.py index 2b6b3a08..a4925e1b 100644 --- a/tenxgraph_api/cli/commands/init.py +++ b/tenxgraph_api/cli/commands/init.py @@ -608,7 +608,7 @@ def _build_config(self, context: dict, is_prod: bool) -> dict: if auth in ("jwt", "custom"): # Redis is configured, so a leaked or logged-out token can be revoked # for every worker. See `10xgraph revoke` and the token_revocation docs. - config["token_revocation"] = "redis" # noqa: S105 + config["token_revocation"] = "redis" # noqa: S105 - a backend name, not a secret # nosec B105 return config diff --git a/tenxgraph_api/cli/commands/test.py b/tenxgraph_api/cli/commands/test.py index 50719788..336f9a3a 100644 --- a/tenxgraph_api/cli/commands/test.py +++ b/tenxgraph_api/cli/commands/test.py @@ -45,7 +45,7 @@ def execute( config_manager.load_config(str(discovered)) cfg = config_manager.get_test_config() step.detail(str(discovered)) - except Exception: # nosec: B110 + except Exception: self.logger.warning("Failed to load test configuration from %s", discovered) step.detail(f"ignored unreadable config at {discovered}") else: diff --git a/tenxgraph_api/src/app/core/exceptions/server_errors.py b/tenxgraph_api/src/app/core/exceptions/server_errors.py index 9b0d87f0..b4b2552f 100644 --- a/tenxgraph_api/src/app/core/exceptions/server_errors.py +++ b/tenxgraph_api/src/app/core/exceptions/server_errors.py @@ -121,11 +121,12 @@ def report(request: Request, exc: BaseException) -> None: log_safe(request.url.path), exc_info=exc, ) + # Sentry is optional and must never break the error path try: import sentry_sdk sentry_sdk.capture_exception(exc) - except Exception: # noqa: S110 - Sentry is optional and must never break the error path + except Exception: # noqa: S110 # nosec B110 pass From ba7311f46a9e08ce112a06674c5b835091bf01d1 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 23:20:15 +0600 Subject: [PATCH 08/16] feat: remove IS_DEBUG configuration and update related documentation and tests --- CHANGELOG.md | 8 +++++-- CLAUDE.md | 4 ++-- README.md | 1 - SECURITY.md | 2 +- examples/security/production_config/README.md | 1 - .../production_config/docker-compose.yml | 1 - tenxgraph_api/cli/config_editor/schema.py | 4 ---- tenxgraph_api/cli/templates/defaults.py | 4 ---- .../references/api-settings-and-middleware.md | 1 - tenxgraph_api/src/app/core/config/settings.py | 14 +++++------ tenxgraph_api/src/app/main.py | 2 +- tests/cli/test_utils_parse_and_callable.py | 16 ++++++++----- tests/test_utils_parse_and_callable.py | 16 ++++++++----- tests/unit_tests/test_group6_small_fixes.py | 5 ++-- tests/unit_tests/test_parse_output.py | 16 ++++--------- tests/unit_tests/test_security_config.py | 24 ++++++++++--------- 16 files changed, 56 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 028a5d7b..7a0b540e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -187,6 +187,8 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Changed +- **The local media store no longer creates `./uploads` at server startup.** The folder is + created on the first upload (a `10xgraph` core change). - **`10xgraph init --template production` scaffolds the recommended production setup.** Auth defaults to JWT (with ownership authorization) and the rate limiter to Redis, in both the prompts and `--non-interactive` (the rate-limit question is now asked whatever the auth @@ -233,8 +235,7 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig redacts `sk-` and `AIza` secrets as well as Bearer and JWT values, keeps UUID ids readable, redacts keys by whole name, and replaces CR and LF. Validation errors on WebSocket and AG-UI no longer echo pydantic `input_value` text, and the access log does not print `?token=`. -- **`ALLOWED_HOST` is stripped** before the middleware compares it, `IS_DEBUG=true` is reported - as ignored in production, and `10xgraph dev --host 0.0.0.0` and `--reload` in production log +- **`ALLOWED_HOST` is stripped** before the middleware compares it, and `10xgraph dev --host 0.0.0.0` and `--reload` in production log a warning. - **Token revocation.** A `revoke_user` cutoff is never replaced by an older one, entries expire (30 days by default), and a token without `iat` is dated from its `exp` when you give @@ -272,6 +273,9 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Removed +- **`IS_DEBUG` is removed.** Debug now follows `MODE`: on in development, off in production. A + leftover `IS_DEBUG` in the environment is ignored and logs one WARNING at startup; delete it. + The unused `IS_PRODUCTION` constant in `settings.py` is gone too. - Removed the `/v1/evals/runs` endpoints; eval reports are JSON files opened in the playground. `10xgraph eval` now writes only the JSON report and no longer has `--open`. - The `graph:setup` scope is removed from the config editor and the docs (see Breaking). diff --git a/CLAUDE.md b/CLAUDE.md index 828b11aa..24d5a707 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -227,13 +227,13 @@ were removed in the 0.5.0 readiness pass; they were fully commented out and neve `core/config/settings.py` is a `pydantic-settings` `Settings` (with `extra="allow"`, so unknown env vars are tolerated). Notable vars: `APP_NAME`, `APP_VERSION`, `MODE` (`production` | `prod` | `development` | `dev` | `local`, case-insensitive, normalised to `production` or `development`; -anything else refuses to start, unset is `development`), `LOG_LEVEL`, `IS_DEBUG`, `MAX_REQUEST_SIZE` (10MB default), security headers +anything else refuses to start, unset is `development`), `LOG_LEVEL`, `MAX_REQUEST_SIZE` (10MB default), security headers (`SECURITY_HEADERS_ENABLED`, `HSTS_*`, `FRAME_OPTIONS`, `CSP_POLICY`, ...), `ORIGINS` (CORS, default `*` with a wildcard warning), `ALLOWED_HOST`, `ROOT_PATH`/`DOCS_PATH`/`REDOCS_PATH`, `REDIS_URL`, `SENTRY_DSN`, `SNOWFLAKE_*` (epoch/node/worker/bit layout), `JWT_SECRET_KEY`/ `JWT_ALGORITHM`, `OTEL_ENABLED`/`OTEL_SERVICE_NAME`/`OTEL_EXPORTER_OTLP_ENDPOINT`/`OTEL_LEVEL`, `STRICT_PRODUCTION_CHECKS` (production refuses wildcard `ALLOWED_HOST`/`ORIGINS` or enabled -docs). `IS_DEBUG` is forced off in production. Media: `MEDIA_REQUIRE_OWNER` defaults on in +docs). Debug follows `MODE` (on in development, off in production); a leftover `IS_DEBUG` is ignored with a warning. Media: `MEDIA_REQUIRE_OWNER` defaults on in production. In production: set `MODE=production`, a non-`*` `ORIGINS`, and a strong `JWT_SECRET_KEY`; see `SECURITY.md` for the full checklist. diff --git a/README.md b/README.md index d0153526..37bce0b5 100644 --- a/README.md +++ b/README.md @@ -447,7 +447,6 @@ Security features of the server: ```bash MODE=production # production mode JWT_SECRET_KEY=<32+ chars> # strong secret (secrets.token_urlsafe(32)) -IS_DEBUG=false # disable debug ORIGINS=https://yourdomain.com # specific CORS origins (never *) ALLOWED_HOST=yourdomain.com # specific allowed hosts (never *) DOCS_PATH= # recommended: disable API docs diff --git a/SECURITY.md b/SECURITY.md index 3617394a..9d649043 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -69,7 +69,7 @@ Out of scope: The API refuses to start on some misconfigurations rather than warning - notably wildcard CORS combined with credentials in production. For anything else, in production set: -- `MODE=production` and `IS_DEBUG=false` +- `MODE=production` (debug is off in production) - `ORIGINS` to an explicit comma-separated list, never `*` - `JWT_SECRET_KEY` to a random value of 32 characters or more - An `authorization` backend (it defaults to `ownership` whenever auth is on; do not weaken diff --git a/examples/security/production_config/README.md b/examples/security/production_config/README.md index 52abcf95..d19c4cf0 100644 --- a/examples/security/production_config/README.md +++ b/examples/security/production_config/README.md @@ -118,7 +118,6 @@ Before going to production, verify: ### Required Configuration - [ ] `MODE=production` in .env.production - [ ] Strong `JWT_SECRET_KEY` (32+ chars, random) -- [ ] `IS_DEBUG=false` - [ ] Specific `ORIGINS` (not `*`) - [ ] Specific `ALLOWED_HOST` (not `*`) - [ ] `DOCS_PATH` and `REDOCS_PATH` are empty diff --git a/examples/security/production_config/docker-compose.yml b/examples/security/production_config/docker-compose.yml index 4682fc7a..87e57690 100644 --- a/examples/security/production_config/docker-compose.yml +++ b/examples/security/production_config/docker-compose.yml @@ -14,7 +14,6 @@ services: - JWT_SECRET_KEY=${JWT_SECRET_KEY} - ORIGINS=${ORIGINS} - ALLOWED_HOST=${ALLOWED_HOST} - - IS_DEBUG=false - DOCS_PATH= - REDOCS_PATH= - MAX_REQUEST_SIZE=10485760 diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index 792d58da..66ab3238 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -933,10 +933,6 @@ "including the API docs being off." ), }, - { - "name": "IS_DEBUG", - "purpose": "Debug output. Always forced off when MODE is production.", - }, {"name": "LOG_LEVEL", "purpose": "Python log level such as INFO or DEBUG."}, { "name": "STRICT_PRODUCTION_CHECKS", diff --git a/tenxgraph_api/cli/templates/defaults.py b/tenxgraph_api/cli/templates/defaults.py index 23a5eb1d..a0e8e9f5 100644 --- a/tenxgraph_api/cli/templates/defaults.py +++ b/tenxgraph_api/cli/templates/defaults.py @@ -376,7 +376,6 @@ def generate_dockerfile_content( "# In production the local in-memory telemetry store stays disabled", "# (use OTEL / a publisher for observability instead).", "ENV MODE=production", - "ENV IS_DEBUG=false", "# Gunicorn worker count. Gunicorn reads WEB_CONCURRENCY natively; a sane default", "# here beats gunicorn's built-in default of 1 worker. Tune to your CPU/memory", "# at deploy time, e.g. `docker run -e WEB_CONCURRENCY=8 ...`.", @@ -527,8 +526,6 @@ def generate_k8s_manifest_content(service_name: str, port: int) -> str: " env:", " - name: MODE", ' value: "production"', - " - name: IS_DEBUG", - ' value: "false"', # ORIGINS and ALLOWED_HOST come from the Secret (an env entry here would # override it). Production refuses to start while either is missing or a wildcard. " lifecycle:", @@ -610,7 +607,6 @@ def generate_docker_compose_content(service_name: str, port: int) -> str: " - PYTHONDONTWRITEBYTECODE=1", # Default to production; local in-memory telemetry stays disabled. " - MODE=production", - " - IS_DEBUG=false", " # ORIGINS and ALLOWED_HOST come from .env (a value here would override it).", " # Production refuses to start while either is missing or a wildcard.", " # Loopback only: put a TLS reverse proxy (Caddy, nginx, a cloud load", diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md index 9e34ec1a..4fc58702 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/api-settings-and-middleware.md @@ -12,7 +12,6 @@ Important variables: - `APP_VERSION` - `MODE` - `LOG_LEVEL` -- `IS_DEBUG` - `MAX_REQUEST_SIZE` - `ORIGINS` - `ALLOWED_HOST` diff --git a/tenxgraph_api/src/app/core/config/settings.py b/tenxgraph_api/src/app/core/config/settings.py index 6f3f5670..060fd8bd 100644 --- a/tenxgraph_api/src/app/core/config/settings.py +++ b/tenxgraph_api/src/app/core/config/settings.py @@ -6,7 +6,6 @@ from pydantic_settings import BaseSettings -IS_PRODUCTION = False LOGGER_NAME = os.getenv("LOGGER_NAME", "10xgraph-api") logger = logging.getLogger(LOGGER_NAME) @@ -92,8 +91,6 @@ class Settings(BaseSettings): # DEBUG = 10 # NOTSET = 0 LOG_LEVEL: str = "INFO" - # Forced off when MODE=production, whatever the environment says. - IS_DEBUG: bool = True # When true, MODE=production refuses to start on the settings it otherwise only warns # about: wildcard ALLOWED_HOST, wildcard ORIGINS, and enabled API docs. Unset means on in # production and off in development; STRICT_PRODUCTION_CHECKS=false opts out. @@ -257,6 +254,12 @@ def check_production_security(self): """Check for insecure configurations in production mode.""" if self.STRICT_PRODUCTION_CHECKS is None: self.STRICT_PRODUCTION_CHECKS = self.MODE == "production" + # IS_DEBUG used to switch debug on or off; debug now follows MODE. Never fail on it. + if any(key.upper() == "IS_DEBUG" for key in (*os.environ, *(self.model_extra or {}))): + logger.warning( + "IS_DEBUG is no longer read; debug follows MODE " + "(on in development, off in production)." + ) if self.MODE == "production": # Docs (and the OpenAPI schema behind them) are off in production unless a path # was set explicitly. @@ -264,11 +267,6 @@ def check_production_security(self): if field not in self.model_fields_set: setattr(self, field, "") - # Debug output is never allowed in production: enforce it rather than warn. - if self.IS_DEBUG: - self.IS_DEBUG = False - logger.warning("IS_DEBUG=true is ignored in production; debug is forced off.") - warnings = [] if self.DOCS_PATH or self.REDOCS_PATH: diff --git a/tenxgraph_api/src/app/main.py b/tenxgraph_api/src/app/main.py index 1780888c..eec3b85e 100644 --- a/tenxgraph_api/src/app/main.py +++ b/tenxgraph_api/src/app/main.py @@ -225,7 +225,7 @@ async def lifespan(app: FastAPI): app = FastAPI( title=settings.APP_NAME, version=settings.APP_VERSION, - debug=settings.IS_DEBUG, + debug=settings.MODE != "production", summary=settings.SUMMARY, docs_url=settings.DOCS_PATH if settings.DOCS_PATH else None, redoc_url=settings.REDOCS_PATH if settings.REDOCS_PATH else None, diff --git a/tests/cli/test_utils_parse_and_callable.py b/tests/cli/test_utils_parse_and_callable.py index e10374f6..55becc99 100644 --- a/tests/cli/test_utils_parse_and_callable.py +++ b/tests/cli/test_utils_parse_and_callable.py @@ -23,18 +23,22 @@ class _MessageModel(BaseModel): raw: dict[str, Any] | None = None -@pytest.mark.parametrize("is_debug", [True, False]) -def test_parse_state_output(is_debug: bool): - settings = Settings(IS_DEBUG=is_debug) +@pytest.mark.parametrize("mode", ["development", "production"]) +def test_parse_state_output(mode: str): + settings = Settings( + MODE=mode, ALLOWED_HOST="api.example.com", ORIGINS="https://app.example.com" + ) model = _StateModel(a=1, b="x", execution_meta={"duration": 123}) out = parse_state_output(settings, model) assert out["execution_meta"] == {"duration": 123} assert out["a"] == 1 and out["b"] == "x" -@pytest.mark.parametrize("is_debug", [True, False]) -def test_parse_message_output(is_debug: bool): - settings = Settings(IS_DEBUG=is_debug) +@pytest.mark.parametrize("mode", ["development", "production"]) +def test_parse_message_output(mode: str): + settings = Settings( + MODE=mode, ALLOWED_HOST="api.example.com", ORIGINS="https://app.example.com" + ) model = _MessageModel(content="hello", raw={"tokens": 5}) out = parse_message_output(settings, model) assert "raw" not in out diff --git a/tests/test_utils_parse_and_callable.py b/tests/test_utils_parse_and_callable.py index 32c223d0..5812687a 100644 --- a/tests/test_utils_parse_and_callable.py +++ b/tests/test_utils_parse_and_callable.py @@ -23,9 +23,11 @@ class _MessageModel(BaseModel): raw: dict[str, Any] | None = None -@pytest.mark.parametrize("is_debug", [True, False]) -def test_parse_state_output(is_debug: bool): - settings = Settings(IS_DEBUG=is_debug) +@pytest.mark.parametrize("mode", ["development", "production"]) +def test_parse_state_output(mode: str): + settings = Settings( + MODE=mode, ALLOWED_HOST="api.example.com", ORIGINS="https://app.example.com" + ) model = _StateModel(a=1, b="x", execution_meta={"duration": 123}) out = parse_state_output(settings, model) # Current implementation always includes execution_meta regardless of debug @@ -33,9 +35,11 @@ def test_parse_state_output(is_debug: bool): assert out["a"] == 1 and out["b"] == "x" -@pytest.mark.parametrize("is_debug", [True, False]) -def test_parse_message_output(is_debug: bool): - settings = Settings(IS_DEBUG=is_debug) +@pytest.mark.parametrize("mode", ["development", "production"]) +def test_parse_message_output(mode: str): + settings = Settings( + MODE=mode, ALLOWED_HOST="api.example.com", ORIGINS="https://app.example.com" + ) model = _MessageModel(content="hello", raw={"tokens": 5}) out = parse_message_output(settings, model) # raw is never returned to clients, whatever the debug flag diff --git a/tests/unit_tests/test_group6_small_fixes.py b/tests/unit_tests/test_group6_small_fixes.py index ffa2e88f..564b023a 100644 --- a/tests/unit_tests/test_group6_small_fixes.py +++ b/tests/unit_tests/test_group6_small_fixes.py @@ -67,12 +67,13 @@ def x(): assert TestClient(app, base_url="http://example.com").get("/x").status_code == 200 -def test_is_debug_reaches_the_app(): +def test_mode_sets_app_debug(): import inspect from tenxgraph_api.src.app import main - assert "debug=settings.IS_DEBUG" in inspect.getsource(main) + assert 'debug=settings.MODE != "production"' in inspect.getsource(main) + assert main.app.debug is (main.settings.MODE != "production") def test_production_warns_about_debug_logging_and_full_otel(caplog): diff --git a/tests/unit_tests/test_parse_output.py b/tests/unit_tests/test_parse_output.py index 45a588c3..cba9fb6e 100644 --- a/tests/unit_tests/test_parse_output.py +++ b/tests/unit_tests/test_parse_output.py @@ -18,9 +18,7 @@ class MessageModel(BaseModel): def test_parse_state_output_debug_true(monkeypatch): monkeypatch.setenv("JWT_SECRET_KEY", "test") monkeypatch.setenv("JWT_ALGORITHM", "HS256") - settings = Settings( - IS_DEBUG=True, - ) + settings = Settings() model = StateModel(a=1, b=2, execution_meta="meta") out = parse_state_output(settings, model) assert out == {"a": 1, "b": 2, "execution_meta": "meta"} @@ -29,9 +27,7 @@ def test_parse_state_output_debug_true(monkeypatch): def test_parse_state_output_debug_false(monkeypatch): monkeypatch.setenv("JWT_SECRET_KEY", "test") monkeypatch.setenv("JWT_ALGORITHM", "HS256") - settings = Settings( - IS_DEBUG=False, - ) + settings = Settings() model = StateModel(a=1, b=2, execution_meta="meta") out = parse_state_output(settings, model) assert out == {"a": 1, "b": 2, "execution_meta": "meta"} @@ -40,9 +36,7 @@ def test_parse_state_output_debug_false(monkeypatch): def test_parse_message_output_debug_true(monkeypatch): monkeypatch.setenv("JWT_SECRET_KEY", "test") monkeypatch.setenv("JWT_ALGORITHM", "HS256") - settings = Settings( - IS_DEBUG=True, - ) + settings = Settings() model = MessageModel(text="hello", raw={"tokens": 3}) out = parse_message_output(settings, model) assert out == {"text": "hello"} @@ -51,9 +45,7 @@ def test_parse_message_output_debug_true(monkeypatch): def test_parse_message_output_debug_false(monkeypatch): monkeypatch.setenv("JWT_SECRET_KEY", "test") monkeypatch.setenv("JWT_ALGORITHM", "HS256") - settings = Settings( - IS_DEBUG=False, - ) + settings = Settings() model = MessageModel(text="hello", raw={"tokens": 3}) out = parse_message_output(settings, model) assert out == {"text": "hello"} diff --git a/tests/unit_tests/test_security_config.py b/tests/unit_tests/test_security_config.py index f0a2fbc2..2e8da5e5 100644 --- a/tests/unit_tests/test_security_config.py +++ b/tests/unit_tests/test_security_config.py @@ -41,17 +41,21 @@ def test_cors_wildcard_no_warning_in_development(caplog): assert "CORS ORIGINS" not in caplog.text or "production" not in caplog.text -def test_debug_mode_is_forced_off_in_production(caplog): - """IS_DEBUG=true is overridden in production, not just warned about.""" - with patch.dict(os.environ, {"MODE": "production", "IS_DEBUG": "true", **SAFE_HOSTS}): - settings = Settings() - assert settings.IS_DEBUG is False - assert "IS_DEBUG=true is ignored in production" in caplog.text +def test_leftover_is_debug_warns_and_is_ignored(caplog): + """A stale IS_DEBUG warns once and never stops startup.""" + for mode in ("development", "production"): + caplog.clear() + with patch.dict(os.environ, {"MODE": mode, "IS_DEBUG": "true", **SAFE_HOSTS}): + settings = Settings() + assert mode == settings.MODE + assert caplog.text.count("IS_DEBUG is no longer read") == 1 -def test_debug_mode_kept_in_development(): - with patch.dict(os.environ, {"MODE": "development", "IS_DEBUG": "true"}): - assert Settings().IS_DEBUG is True +def test_no_is_debug_warning_when_unset(caplog): + with patch.dict(os.environ, {"MODE": "development"}): + os.environ.pop("IS_DEBUG", None) + Settings() + assert "IS_DEBUG" not in caplog.text def test_docs_enabled_warning_in_production(caplog): @@ -93,7 +97,6 @@ def test_multiple_warnings_in_production(caplog): { "MODE": "production", "ORIGINS": "*", - "IS_DEBUG": "true", "ALLOWED_HOST": "*", "STRICT_PRODUCTION_CHECKS": "false", }, @@ -102,7 +105,6 @@ def test_multiple_warnings_in_production(caplog): log_text = caplog.text assert "CORS ORIGINS='*'" in log_text - assert "IS_DEBUG=true is ignored" in log_text assert "ALLOWED_HOST='*'" in log_text From a24aa39d2d1614ac2031afc0ed3946d63f65df10 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Thu, 29 Oct 2026 23:58:53 +0600 Subject: [PATCH 09/16] feat: remove IS_DEBUG configuration from production environment template --- tenxgraph_api/cli/templates/prod/.env.example | 1 - 1 file changed, 1 deletion(-) diff --git a/tenxgraph_api/cli/templates/prod/.env.example b/tenxgraph_api/cli/templates/prod/.env.example index 301c0184..d9c95005 100644 --- a/tenxgraph_api/cli/templates/prod/.env.example +++ b/tenxgraph_api/cli/templates/prod/.env.example @@ -5,7 +5,6 @@ APP_NAME="$agent_name" APP_VERSION="0.1.0" MODE="production" LOG_LEVEL="INFO" -IS_DEBUG="false" # In production the app refuses to start on wildcard ALLOWED_HOST/ORIGINS or enabled docs. # Set STRICT_PRODUCTION_CHECKS="false" to downgrade these to warnings. # STRICT_PRODUCTION_CHECKS="true" From fd31a510fc32e56f11c0c0c50c18f0d14b55ffca Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 00:36:05 +0600 Subject: [PATCH 10/16] feat(eval): enhance evaluation command with HTML report generation and browser opening - Updated `10xgraph eval` command to generate both JSON and HTML reports. - Added `--open` option to open the HTML report in the browser by default. - Introduced configuration option `open_report` in `10xgraph.json` to control HTML report opening. - Enhanced progress reporting during evaluations to include detailed step messages. - Updated evaluation templates to ensure compatibility with new reporting features. - Added tests to verify the new functionality and ensure expected behavior during evaluations. --- CHANGELOG.md | 22 ++ CLAUDE.md | 2 +- README.md | 2 +- tenxgraph_api/cli/commands/eval.py | 133 ++++++- tenxgraph_api/cli/config_editor/schema.py | 8 + tenxgraph_api/cli/config_editor/validation.py | 2 + tenxgraph_api/cli/core/steps.py | 36 ++ tenxgraph_api/cli/main.py | 15 +- .../prod/evals/user_simulator_eval.py | 20 +- .../prod/evals/weather_agents_eval.py | 42 ++- .../10xgraph/references/cli-commands.md | 2 +- .../skills/10xgraph/references/evaluation.md | 2 +- tests/cli/test_eval_command.py | 13 +- tests/cli/test_eval_progress_and_report.py | 332 ++++++++++++++++++ tests/cli/test_eval_templates.py | 35 ++ 15 files changed, 621 insertions(+), 45 deletions(-) create mode 100644 tests/cli/test_eval_progress_and_report.py create mode 100644 tests/cli/test_eval_templates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a0b540e..6ef654b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -183,6 +183,18 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig - **`session_started` event on `/v1/graph/live`.** The first server event of every live session is `{"type": "session_started", "thread_id": ""}`, with the server-made id when the init frame had none. Reconnect with that id to continue the thread. +- **`10xgraph eval` writes an HTML report and opens it.** A run now writes the HTML report next + to the JSON one in `eval_reports/`, prints both paths, and opens the HTML report in the default + browser when it finishes. Pass `--no-open` (or set `evaluation.open_report` to `false` in + `10xgraph.json`) to skip it. It is never opened with `--no-report`, `-q`, `--format json` or + `jsonl`, `--non-interactive`, when `CI` is set, or when stdout is not a terminal. A failure to + open the browser only prints a warning. The playground can still open the JSON report. +- **Live progress lines for `10xgraph eval`.** Each case prints a muted `running` line, and a + user-simulation scenario prints a line per step (simulated user message, agent reply, newly + achieved goals, criteria being judged, agent errors) so a long scenario no longer looks + frozen. Needs the matching `10xgraph` core release; with an older core the simulation still runs + and only the per-step lines are missing. `-q` hides them; `--format jsonl` emits `eval_step` + events. `ProgressRun` gains `step(label, message)`. ### Changed @@ -282,6 +294,16 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Fixed +- **`10xgraph eval` reports carry each case's expectations.** The report's case results now + include `expected` (response, tool calls, node order, input). For a user simulation, `input` + is the starting prompt and `response` is the goals joined with `; `. Needs `10xgraph` 1.0. +- **The `init` eval templates check what they declare and can pass.** `weather_agents_eval.py` + now scores the expected response (ROUGE-1, no LLM), the expected node order and the tool + calls; the greeting case no longer expects the placeholder `assistant`. + `user_simulator_eval.py` replaces the flight booking scenario, which a weather-only agent + could never complete, with a two-city weather comparison. +- **`10xgraph eval` showed `0.00s` for user-simulation scenarios.** The CLI now times each + scenario and sets its duration. - A thread that was cancelled, disconnected or cut by a restart is usable on the next request (the core fix), and a client disconnect during a stream no longer leaves the thread lock held. - `POST /v1/threads/{thread_id}/messages` and the other write routes no longer race an active diff --git a/CLAUDE.md b/CLAUDE.md index 24d5a707..38669a8f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -54,7 +54,7 @@ Public exports from the package root (`from tenxgraph_api import ...`): `BaseAut | `10xgraph dev` | The only server command: local Uvicorn dev server (reload on) via `commands/api.py::APICommand`, opens the hosted playground when ready. `api` and `play` were removed (use `dev --no-open` / `dev`); production runs Gunicorn in the `10xgraph build` image | `--config/-c` (default `10xgraph.json`), `--host/-H`, `--port/-p` (8000), `--reload/--no-reload`, `--open/--no-open`, `-v/-q` | | `10xgraph init` | Interactively scaffold a project (guided prompts pick dev vs production, auth, rate limit) | `--path/-p`, `--force/-f`, `--name`, `--template` (quick-start\|production), `--auth` (none\|jwt\|custom), `--rate-limit` (none\|memory\|redis), `--provider` (google\|openai\|anthropic), `--websocket/--no-websocket` (default off; writes `websocket.enabled`). Prod `pyproject.toml` depends on `10xgraph[]` and `10xgraph-api[jwt,redis,websocket]` as chosen, `--yes/-y`, `--non-interactive`, `--dry-run`. There is **no `--prod` flag** | | `10xgraph build` | Generate a `Dockerfile` (and optionally `docker-compose.yml` / `k8s.yaml`). The builder stage strips CLI assets, runs `tenxgraph_api/_slim_image.py` (uninstalls typer/rich/questionary and what only they need, unless another package needs them) and removes pip, so the image carries no CLI code paths | `--output/-o`, `--force/-f`, `--python-version` (3.13), `--port`, `--docker-compose/--no-docker-compose`, `--k8s/--no-k8s`, `--service-name` | -| `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes a JSON report to `eval_reports/` | `--output/-o`, `--no-report`, `--threshold/-t`, `--parallel/-p`, `--max-concurrency/-c` | +| `10xgraph eval` | Run agent evaluations; discovers `*_eval.py`/`eval_*.py`, runs cases (optionally `--parallel`), writes JSON and HTML reports to `eval_reports/` and opens the HTML report in the browser (only at an interactive terminal: skipped with `--no-open`, `--no-report`, `-q`, json/jsonl, `--non-interactive`, `CI`, non-TTY; `evaluation.open_report` in `10xgraph.json` sets the default). Simulations print per-turn step lines via `ProgressRun.step` | `--output/-o`, `--no-report`, `--threshold/-t`, `--parallel/-p`, `--max-concurrency/-c`, `--open/--no-open` | | `10xgraph test` | Run project tests via pytest (args after `--` forwarded verbatim) | `--coverage/-C`, `--html`, `-k`, path arg | | `10xgraph skills` | Install the bundled 10xGraph skill (Agent Skills spec) for Codex/Claude/GitHub, or validate skills | `--agent/-a`, `--path/-p`, `--force/-f`, `--all`, `--list/-l`, `--validate PATH` | | `10xgraph version` | Show CLI + core framework version | both resolve from installed distribution metadata | diff --git a/README.md b/README.md index 37bce0b5..554ae5de 100644 --- a/README.md +++ b/README.md @@ -217,7 +217,7 @@ Generate production Docker files. ### `10xgraph eval` / `10xgraph test` -Run agent evaluations (discovers `*_eval.py` / `eval_*.py`, writes a JSON report to `eval_reports/`) and project tests (pytest). +Run agent evaluations (discovers `*_eval.py` / `eval_*.py`, writes JSON and HTML reports to `eval_reports/` and opens the HTML report; `--no-open` skips that) and project tests (pytest). ```bash 10xgraph eval --parallel --threshold 0.8 diff --git a/tenxgraph_api/cli/commands/eval.py b/tenxgraph_api/cli/commands/eval.py index 2702ae02..7a7434e3 100644 --- a/tenxgraph_api/cli/commands/eval.py +++ b/tenxgraph_api/cli/commands/eval.py @@ -7,7 +7,9 @@ import importlib.util import inspect import sys +import time import typing +import webbrowser from collections import defaultdict from dataclasses import dataclass from pathlib import Path @@ -26,8 +28,10 @@ from tenxgraph.qa.evaluation.reporters.manager import ReporterManager from tenxgraph.utils.callbacks import CallbackManager +from tenxgraph_api.cli.capabilities import OutputFormat from tenxgraph_api.cli.commands import BaseCommand from tenxgraph_api.cli.core.config import ConfigManager +from tenxgraph_api.cli.core.steps import ProgressRun if TYPE_CHECKING: @@ -539,11 +543,53 @@ def _case_detail(result: EvalCaseResult) -> str: detail += f" in={tok.input_tokens} out={tok.output_tokens}" return detail + @staticmethod + def _accepts_on_event(run: Any) -> bool: + try: + params = inspect.signature(run).parameters + except (TypeError, ValueError): + return False + return "on_event" in params or any( + p.kind is inspect.Parameter.VAR_KEYWORD for p in params.values() + ) + + @staticmethod + def _simulation_events(progress: ProgressRun, label: str) -> Any: + """Build the ``on_event`` callback that turns simulator events into progress lines.""" + + max_turns: list[int] = [] + + def on_event(name: str, data: dict[str, Any]) -> None: + if "max_turns" in data: + max_turns[:] = [data["max_turns"]] + turn = "" + if "turn" in data: + of = f"/{max_turns[0]}" if max_turns else "" + turn = f"turn {data['turn']}{of} " + if name == "turn_started": + progress.step(label, f"{turn}user: {data.get('user_message', '')}") + elif name == "agent_replied": + progress.step(label, f"{turn}agent: {data.get('response', '')}") + elif name == "goals_checked": + gained = data.get("newly_achieved") or [] + if gained: + progress.step( + label, + f"{turn}goals {data.get('achieved', 0)}/{data.get('total', 0)}" + f" (+ {', '.join(str(g) for g in gained)})", + ) + elif name == "evaluating": + progress.step(label, f"evaluating criteria: {', '.join(data.get('criteria', []))}") + elif name == "agent_error": + progress.step(label, f"{turn}agent error: {data.get('error', '')}") + + return on_event + # ------------------------------------------------------------------ # Flat pool execution — single asyncio event loop for all cases # ------------------------------------------------------------------ - async def _run_flat_pool( + async def _run_flat_pool( # noqa: PLR0915 self, pending: list[_PendingCase | _PendingSimulation], max_concurrency: int, @@ -555,7 +601,12 @@ async def _run_flat_pool( """ total = len(pending) - async def _run_case(pc: _PendingCase) -> tuple[str, str, str, EvalCaseResult]: + async def _run_case( + pc: _PendingCase, progress: ProgressRun + ) -> tuple[str, str, str, EvalCaseResult]: + label = f"{pc.file_name}::{pc.case.name or pc.case.eval_id}" + progress.step(label, "running") + started = time.perf_counter() local_collector = TrajectoryCollector( capture_all_events=pc.evaluator.collector.capture_all_events, ) @@ -575,15 +626,34 @@ async def _run_case(pc: _PendingCase) -> tuple[str, str, str, EvalCaseResult]: eval_id=pc.case.eval_id, error=str(exc), name=pc.case.name, + duration_seconds=time.perf_counter() - started, + expected=pc.case.expected_summary(), ) + if not result.duration_seconds: + result.duration_seconds = time.perf_counter() - started return (pc.file_name, pc.eval_set_id, pc.eval_set_name, result) - async def _run_simulation(ps: _PendingSimulation) -> tuple[str, str, str, EvalCaseResult]: + async def _run_simulation( + ps: _PendingSimulation, progress: ProgressRun + ) -> tuple[str, str, str, EvalCaseResult]: from tenxgraph.qa.evaluation.eval_result import CriterionResult from tenxgraph.qa.evaluation.token_usage import TokenUsage + sim_name = ps.scenario.description or ps.scenario.scenario_id + label = f"{ps.file_name}::{sim_name}" + run_kwargs: dict[str, Any] = {} + # Older core releases have no on_event hook; run without step lines there. + if self._accepts_on_event(ps.simulator.run): + run_kwargs["on_event"] = self._simulation_events(progress, label) + started = time.perf_counter() + expected = { + "input": getattr(ps.scenario, "starting_prompt", "") or None, + "response": "; ".join(getattr(ps.scenario, "goals", None) or []) or None, + "tool_calls": None, + "node_order": None, + } try: - sim_result = await ps.simulator.run(ps.graph, ps.scenario) + sim_result = await ps.simulator.run(ps.graph, ps.scenario, **run_kwargs) # Use the full CriterionResult objects (with per-criterion token_usage) # if available; fall back to rebuilding from scores dict. criterion_results = sim_result.criterion_results or [ @@ -626,21 +696,25 @@ async def _run_simulation(ps: _PendingSimulation) -> tuple[str, str, str, EvalCa "goals_achieved": sim_result.goals_achieved, "completed": sim_result.completed, }, + duration_seconds=time.perf_counter() - started, + expected=expected, ) except Exception as exc: result = EvalCaseResult.failure( eval_id=ps.scenario.scenario_id, error=str(exc), name=ps.scenario.description or ps.scenario.scenario_id, + duration_seconds=time.perf_counter() - started, + expected=expected, ) return (ps.file_name, ps.eval_set_id, ps.eval_set_name, result) async def _dispatch( - item: _PendingCase | _PendingSimulation, + item: _PendingCase | _PendingSimulation, progress: ProgressRun ) -> tuple[str, str, str, EvalCaseResult]: if isinstance(item, _PendingSimulation): - return await _run_simulation(item) - return await _run_case(item) + return await _run_simulation(item, progress) + return await _run_case(item, progress) def _record(progress: Any, quad: tuple[str, str, str, EvalCaseResult]) -> None: file_name, _, _, result = quad @@ -656,7 +730,7 @@ def _record(progress: Any, quad: tuple[str, str, str, EvalCaseResult]) -> None: results: list[tuple[str, str, str, EvalCaseResult]] = [] with self.output.progress_run(title, total=total) as progress: for item in pending: - quad = await _dispatch(item) + quad = await _dispatch(item, progress) _record(progress, quad) results.append(quad) return results @@ -664,14 +738,14 @@ def _record(progress: Any, quad: tuple[str, str, str, EvalCaseResult]) -> None: semaphore = asyncio.Semaphore(max_concurrency) async def _run_one( - item: _PendingCase | _PendingSimulation, + item: _PendingCase | _PendingSimulation, progress: ProgressRun ) -> tuple[str, str, str, EvalCaseResult]: async with semaphore: - return await _dispatch(item) + return await _dispatch(item, progress) output_results: list[tuple[str, str, str, EvalCaseResult]] = [] with self.output.progress_run(title, total=total) as progress: - tasks = [asyncio.create_task(_run_one(item)) for item in pending] + tasks = [asyncio.create_task(_run_one(item, progress)) for item in pending] for coro in asyncio.as_completed(tasks): quad = await coro _record(progress, quad) @@ -716,6 +790,26 @@ def _resolve_eval_dir(self) -> Path: ) return Path.cwd() / directory + def _should_open_report( + self, open_report: bool | None, *, quiet: bool, non_interactive: bool + ) -> bool: + """Open the HTML report only for a person at an interactive terminal.""" + if open_report is False or quiet or non_interactive or self.output.quiet: + return False + # `interactive` is false when stdout is not a TTY or CI is set; structured + # formats (json/jsonl) never run with an interactive human format. + caps = self.output.capabilities + return caps.interactive and caps.output_format == OutputFormat.HUMAN + + def _open_report(self, path: str) -> None: + try: + opened = webbrowser.open(Path(path).resolve().as_uri()) + except Exception as exc: + self.output.warning(f"Could not open the HTML report: {exc}") + return + if not opened: + self.output.warning("Could not open the HTML report in a browser; open the file above.") + @staticmethod def _exit_code(pass_rate: float, threshold: float | None) -> int: """0 when the pass rate meets ``threshold``; without one, only a clean run passes.""" @@ -726,7 +820,7 @@ def _exit_code(pass_rate: float, threshold: float | None) -> int: # Main entry point # ------------------------------------------------------------------ - def execute( # noqa: PLR0912, PLR0915 + def execute( # noqa: PLR0912, PLR0913, PLR0915 self, target: str | None = None, output_dir: str = "eval_reports", @@ -736,6 +830,8 @@ def execute( # noqa: PLR0912, PLR0915 max_concurrency: int = 4, verbose: bool = False, quiet: bool = False, + open_report: bool | None = None, + non_interactive: bool = False, **kwargs: Any, ) -> int: # 1. Load infra settings from 10xgraph.json — no criteria here. @@ -760,6 +856,8 @@ def execute( # noqa: PLR0912, PLR0915 "max_concurrency", DEFAULT_CONCURRENCY ) json_directory = global_eval_cfg.get("directory", json_directory) + if open_report is None and isinstance(global_eval_cfg.get("open_report"), bool): + open_report = global_eval_cfg["open_report"] except Exception: self.logger.warning("Failed to load eval config from 10xgraph.json; using defaults") @@ -877,7 +975,7 @@ def execute( # noqa: PLR0912, PLR0915 manager = ReporterManager( ReporterConfig( output_dir=output_dir, - html=False, + html=True, json_report=True, console=False, timestamp_files=True, @@ -885,14 +983,21 @@ def execute( # noqa: PLR0912, PLR0915 ) report_result = manager.run_all(merged) + if report_result.html_path: + self.output.info(f"HTML report: {report_result.html_path}", emoji=False) if report_result.json_path: self.output.info(f"JSON report: {report_result.json_path}", emoji=False) self.output.info( - "Open it in the playground: Evals > Open report file.", emoji=False + "Also viewable in the playground: Evals > Open report file (pick the JSON).", + emoji=False, ) if report_result.has_errors: for name, err in report_result.errors: self.output.warning(f"Reporter error [{name}]: {err}") + if report_result.html_path and self._should_open_report( + open_report, quiet=quiet, non_interactive=non_interactive + ): + self._open_report(report_result.html_path) summary = merged.summary self.output.info( diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index 66ab3238..e24e5dc0 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -892,6 +892,7 @@ "threshold": 0.75, "parallel": False, "max_concurrency": 4, + "open_report": True, }, "fields": [ {"path": ["evaluation", "directory"], "label": "Eval directory", "type": "text"}, @@ -918,6 +919,13 @@ "step": 1, "show_if": {"path": ["evaluation", "parallel"], "equals": True}, }, + { + "path": ["evaluation", "open_report"], + "label": "Open HTML report", + "type": "bool", + "default": True, + "help": "Open the HTML report in your browser when 10xgraph eval finishes.", + }, ], }, ] diff --git a/tenxgraph_api/cli/config_editor/validation.py b/tenxgraph_api/cli/config_editor/validation.py index 495720c2..ce3c7f9f 100644 --- a/tenxgraph_api/cli/config_editor/validation.py +++ b/tenxgraph_api/cli/config_editor/validation.py @@ -432,6 +432,8 @@ def _check_evaluation(config: dict, out: _Collector) -> None: out.error("evaluation.threshold", "threshold must be a number between 0 and 1.") if "parallel" in data and not isinstance(data["parallel"], bool): out.error("evaluation.parallel", "parallel must be true or false.") + if "open_report" in data and not isinstance(data["open_report"], bool): + out.error("evaluation.open_report", "open_report must be true or false.") concurrency = data.get("max_concurrency") if concurrency is not None and not (_is_int(concurrency) and concurrency >= 1): out.error("evaluation.max_concurrency", "max_concurrency must be a whole number >= 1.") diff --git a/tenxgraph_api/cli/core/steps.py b/tenxgraph_api/cli/core/steps.py index a7c8dc95..c301b946 100644 --- a/tenxgraph_api/cli/core/steps.py +++ b/tenxgraph_api/cli/core/steps.py @@ -15,6 +15,7 @@ from __future__ import annotations +import re import time from collections.abc import Callable, Iterator, Sequence from contextlib import contextmanager @@ -126,6 +127,20 @@ def __exit__( def record(self, label: str, *, status: str, detail: str = "") -> None: ... + def step(self, label: str, message: str) -> None: ... + + +_STEP_MESSAGE_LIMIT = 100 +_LINE_BREAKS = re.compile(r"\s*[\r\n\t]+\s*") + + +def one_line(text: str, limit: int = _STEP_MESSAGE_LIMIT) -> str: + """Collapse whitespace and newlines and cut ``text`` to at most ``limit`` characters.""" + collapsed = _LINE_BREAKS.sub(" ", str(text)).strip() + if len(collapsed) <= limit: + return collapsed + return collapsed[: limit - 3].rstrip() + "..." + def format_elapsed(seconds: float) -> str: """Render a duration compactly enough to sit in a fixed-width column.""" @@ -459,9 +474,16 @@ def record(self, label: str, *, status: str, detail: str = "") -> None: self.tally[status] += 1 self._render(label, status, detail) + def step(self, label: str, message: str) -> None: + """Show an in-progress line for ``label`` without advancing the bar or tally.""" + self._render_step(label, one_line(message)) + def _render(self, label: str, status: str, detail: str) -> None: """Emit whatever this output mode shows for one finished item.""" + def _render_step(self, label: str, message: str) -> None: + """Emit whatever this output mode shows for one in-progress line.""" + class LiveProgressRun(_BaseProgressRun): """Animated bar with per-item result lines printed above it.""" @@ -544,6 +566,14 @@ def _render(self, label: str, status: str, detail: str) -> None: self._progress.console.print(line) self._progress.update(self._task, completed=self.completed, tally=self._tally_text()) + def _render_step(self, label: str, message: str) -> None: + if self._progress is None: + return + line = Text(" ") + line.append(f"{label} ", style="tenxgraph.muted") + line.append(message, style="tenxgraph.muted") + self._progress.console.print(line) + def _tally_text(self) -> Text: glyphs = self._glyphs text = Text() @@ -578,6 +608,9 @@ def _render(self, label: str, status: str, detail: str) -> None: suffix = f" {detail}" if detail else "" self._emit(f"[{self.completed:>3}/{self.total}] {symbol} {label} {status.upper()}{suffix}") + def _render_step(self, label: str, message: str) -> None: + self._emit(f" {label} {message}") + class StructuredProgressRun(_BaseProgressRun): """Versioned per-item events for ``--format json`` and ``--format jsonl``.""" @@ -616,6 +649,9 @@ def _render(self, label: str, status: str, detail: str) -> None: }, ) + def _render_step(self, label: str, message: str) -> None: + self._emit("eval_step", label, {"message": message}) + class QuietProgressRun(_BaseProgressRun): """No-op progress used when output is suppressed.""" diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index 8cc0d1d3..8403e09f 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -769,6 +769,7 @@ def test( @app.command(name="eval") def eval_cmd( + ctx: typer.Context, target: str | None = typer.Argument( None, help="File or directory to evaluate (default: evals/ from 10xgraph.json or cwd)", @@ -802,6 +803,11 @@ def eval_cmd( "-c", help="Max cases running concurrently when --parallel is set (global semaphore)", ), + open_report: bool | None = typer.Option( + None, + "--open/--no-open", + help="Open the HTML report in your browser when the run finishes (default: open).", + ), verbose: bool = typer.Option(False, "--verbose", "-v", help="Enable verbose output"), quiet: bool = typer.Option(False, "--quiet", "-q", help="Suppress all output except errors"), ) -> None: @@ -809,8 +815,9 @@ def eval_cmd( Discovers *_eval.py / eval_*.py files in the target directory (default: evals/). Collects all cases from all files into a flat pool, then runs them under a single - event loop throttled by --max-concurrency. Always writes a JSON report - in eval_reports/ unless --no-report is set. + event loop throttled by --max-concurrency. Writes JSON and HTML reports + to eval_reports/ unless --no-report is set, and opens the HTML report in your + browser when the run ends at an interactive terminal (--no-open to skip). Each eval file must expose one of: get_eval_set() + get_eval_config() # CLI loads agent from 10xgraph.json @@ -818,6 +825,8 @@ def eval_cmd( any function returning EvalSet # auto-discovered, pytest-style """ _configure_command(verbose=verbose, quiet=quiet) + runtime = ctx.find_root().obj + non_interactive = isinstance(runtime, CLIContext) and runtime.non_interactive try: command = EvalCommand(output) @@ -830,6 +839,8 @@ def eval_cmd( max_concurrency=max_concurrency, verbose=verbose, quiet=quiet, + open_report=open_report, + non_interactive=non_interactive, ) sys.exit(exit_code) except Exception as e: diff --git a/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py b/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py index 3502cd90..c4c02862 100644 --- a/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py +++ b/tenxgraph_api/cli/templates/prod/evals/user_simulator_eval.py @@ -59,19 +59,19 @@ def get_scenarios() -> list[ConversationScenario]: max_turns=8, ), ConversationScenario( - scenario_id="flight_booking_assistance", - description="User wants help finding a flight from London to New York", - starting_prompt="I need to fly from London to New York next Friday. Can you help?", + scenario_id="compare_two_cities", + description="User compares the weather in two cities to choose a destination", + starting_prompt="I can't decide between Rome and Berlin for a weekend trip.", conversation_plan=( - "1. Share travel dates and seat preferences\n" - "2. Ask about available flights and prices\n" - "3. Confirm booking details" + "1. Ask for the current weather in Rome\n" + "2. Ask for the current weather in Berlin\n" + "3. Ask which city has the nicer weather" ), goals=[ - "User receives flight options for the requested route", - "User gets pricing information", - "Booking process is initiated or completed", + "User receives weather information for Rome", + "User receives weather information for Berlin", + "User is told which of the two cities has the nicer weather", ], - max_turns=10, + max_turns=8, ), ] diff --git a/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py b/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py index 07c46e5e..4921323d 100644 --- a/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +++ b/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py @@ -1,4 +1,11 @@ -from tenxgraph.qa.evaluation import EvalConfig, EvalSet, EvalSetBuilder +from tenxgraph.qa.evaluation import ( + CriteriaConfig, + CriterionConfig, + EvalConfig, + EvalSet, + EvalSetBuilder, + MatchType, +) from tenxgraph.qa.evaluation.config.presets import EvalPresets @@ -9,13 +16,26 @@ def get_eval_config() -> EvalConfig: what your agent does, or call EvalPresets.combine() to mix them. Available presets: - EvalPresets.response_quality() — LLM judge on response accuracy - EvalPresets.tool_usage() — checks tool calls + response quality - EvalPresets.conversation_flow() — multi-turn conversation evaluation - EvalPresets.comprehensive() — all of the above combined - EvalPresets.quick_check() — fast ROUGE-based check, no LLM cost + EvalPresets.response_quality() - LLM judge on response accuracy + EvalPresets.tool_usage() - checks tool names + trajectory (no LLM) + EvalPresets.conversation_flow() - multi-turn conversation evaluation + EvalPresets.comprehensive() - all of the above combined + EvalPresets.quick_check() - fast ROUGE-based response check, no LLM cost + + This file checks, without any LLM call, every expectation the cases below set: + - tool names and arguments (tool_usage) + - the expected final response (quick_check, ROUGE-1 word overlap) + - the expected node order (node_order, extra nodes allowed) """ - return EvalPresets.tool_usage(threshold=0.6) + return EvalPresets.combine( + EvalPresets.tool_usage(threshold=0.6), + EvalPresets.quick_check(), + EvalConfig( + criteria=CriteriaConfig( + node_order=CriterionConfig.node_order(match_type=MatchType.IN_ORDER), + ) + ), + ) def get_eval_set() -> EvalSet: @@ -23,7 +43,7 @@ def get_eval_set() -> EvalSet: EvalSetBuilder(name="weather-agent-regression") .add_case( query="Hi", - expected="assistant", + expected="Hello! How can I help you today?", expected_node_order=["MAIN"], name="greeting_response", description="The agent responds to a greeting.", @@ -32,21 +52,21 @@ def get_eval_set() -> EvalSet: query="What is the weather in London?", tool_name="get_weather", tool_args={"location": "London"}, - expected_response="London", + expected_response="The weather in London is sunny.", case_id="weather_london", ) .add_tool_test( query="Tell me the current weather in New York", tool_name="get_weather", tool_args={"location": "New York"}, - expected_response="New York", + expected_response="The weather in New York is sunny.", case_id="weather_new_york", ) .add_tool_test( query="How is the weather in Tokyo today?", tool_name="get_weather", tool_args={"location": "Tokyo"}, - expected_response="Tokyo", + expected_response="The weather in Tokyo is sunny.", case_id="weather_tokyo", ) .build() diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md index dc5f1f66..109cc2ce 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/cli-commands.md @@ -52,7 +52,7 @@ agentflow = tenxgraph_api.cli.main:legacy_main # deprecated alias, removed in `10xgraph eval [TARGET]` - Runs agent evaluations: discovers `*_eval.py` and `eval_*.py` files in `TARGET` (default `evals/`) and runs their cases. -- Options include `--output/-o` (default `eval_reports`), `--no-report`, `--threshold/-t` (fail below this pass rate), `--parallel/-p`, `--max-concurrency/-c` (default 4), `--verbose/-v`, and `--quiet/-q`. +- Options include `--output/-o` (default `eval_reports`), `--no-report`, `--threshold/-t` (fail below this pass rate), `--parallel/-p`, `--max-concurrency/-c` (default 4), `--verbose/-v`, and `--quiet/-q`. Writes JSON and HTML reports and opens the HTML report in the browser at an interactive terminal; `--open/--no-open` (or `evaluation.open_report` in `10xgraph.json`) controls it. `10xgraph test [PATH]` diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md index 2581eb52..76abaf37 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/evaluation.md @@ -323,7 +323,7 @@ JUnit XML — for CI test summary tabs (GitHub Actions, Jenkins). Auto-discovers `*_eval.py` / `eval_*.py` in `evals/` (or configured directory). Each file must expose `get_eval_set()`, optionally `get_eval_config()` or `EVAL_CONFIG`, or a full `run()` function. The CLI writes a JSON report only (no HTML, no browser); open it in the playground with Evals > Open report file. ```bash -10xgraph eval # scan evals/, write a JSON report to eval_reports/ +10xgraph eval # scan evals/, write JSON + HTML reports to eval_reports/, open the HTML (--no-open to skip) 10xgraph eval evals/my_eval.py # single file 10xgraph eval evals/regression/ # subdirectory 10xgraph eval --threshold 0.8 # exit non-zero if pass rate < 0.8 diff --git a/tests/cli/test_eval_command.py b/tests/cli/test_eval_command.py index d396a3cc..b1b9c10c 100644 --- a/tests/cli/test_eval_command.py +++ b/tests/cli/test_eval_command.py @@ -167,6 +167,7 @@ async def test_run_flat_pool_case_error(cmd): case1 = MagicMock() case1.eval_id = "c1" case1.name = "case_name" + case1.expected_summary.return_value = {"response": "r", "input": "q"} evaluator = MagicMock() async def mock_evaluate_case(case, collector_override): @@ -192,6 +193,8 @@ async def mock_evaluate_case(case, collector_override): assert len(results) == 1 assert results[0][3].passed is False assert "evaluation failed" in results[0][3].error + assert results[0][3].expected == {"response": "r", "input": "q"} + assert results[0][3].duration_seconds > 0 @pytest.mark.asyncio @@ -214,7 +217,7 @@ async def test_run_flat_pool_simulation(cmd): sim_result.turns = 2 sim_result.goals_achieved = 1 - async def mock_simulator_run(graph, scenario): + async def mock_simulator_run(graph, scenario, on_event=None): return sim_result simulator.run = mock_simulator_run @@ -241,7 +244,7 @@ async def test_run_flat_pool_simulation_error(cmd): mock_criterion.threshold = 0.7 simulator.criteria = [mock_criterion] - async def mock_simulator_run(graph, scenario): + async def mock_simulator_run(graph, scenario, on_event=None): raise ValueError("simulation failed") simulator.run = mock_simulator_run @@ -298,6 +301,7 @@ def test_execute_success(cmd, tmp_path): mock_rep_mgr_res = MagicMock() mock_rep_mgr_res.json_path = "path/to/report.json" + mock_rep_mgr_res.html_path = "path/to/report.html" mock_rep_mgr_res.has_errors = False mock_rep_mgr = MagicMock() @@ -325,7 +329,8 @@ def test_execute_success(cmd, tmp_path): ): code = cmd.execute(target=str(tmp_path)) assert code == 0 - assert any("Open it in the playground" in m for m in cmd.output.infos) + assert any("Evals > Open report file" in m for m in cmd.output.infos) + assert any(m.startswith("HTML report:") for m in cmd.output.infos) def test_execute_below_threshold(cmd, tmp_path): @@ -543,7 +548,7 @@ async def test_run_flat_pool_simulation_no_criterion(cmd): sim_result.turns = 2 sim_result.goals_achieved = 1 - async def mock_simulator_run(graph, scenario): + async def mock_simulator_run(graph, scenario, on_event=None): return sim_result simulator.run = mock_simulator_run diff --git a/tests/cli/test_eval_progress_and_report.py b/tests/cli/test_eval_progress_and_report.py new file mode 100644 index 00000000..a390cceb --- /dev/null +++ b/tests/cli/test_eval_progress_and_report.py @@ -0,0 +1,332 @@ +"""Eval progress steps, simulation duration, HTML report and browser opening.""" + +import asyncio +import dataclasses +import io +import types +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from rich.console import Console +from tenxgraph.qa.evaluation import EvalConfig +from tenxgraph.qa.evaluation.eval_result import EvalCaseResult +from tenxgraph.qa.evaluation.token_usage import TokenUsage + +from tenxgraph_api.cli.capabilities import OutputFormat +from tenxgraph_api.cli.commands.eval import EvalCommand, _PendingSimulation +from tenxgraph_api.cli.core.output import OutputFormatter +from tenxgraph_api.cli.core.steps import ( + LiveProgressRun, + QuietProgressRun, + StaticProgressRun, + StructuredProgressRun, + one_line, +) +from tenxgraph_api.cli.core.theme import UNICODE_GLYPHS + + +EvalCommand.__test__ = False + + +class _RecordingProgress: + def __init__(self) -> None: + self.steps: list[tuple[str, str]] = [] + self.records: list[tuple[str, str]] = [] + + def __enter__(self): + return self + + def __exit__(self, *exc): + return None + + def record(self, label, *, status, detail=""): + self.records.append((label, status)) + + def step(self, label, message): + self.steps.append((label, message)) + + +class _Out(OutputFormatter): + def __init__(self, progress=None) -> None: + super().__init__() + self.progress = progress + self.infos: list[str] = [] + self.warnings: list[str] = [] + + def progress_run(self, title, *, total): + return self.progress + + def info(self, message, emoji=True): + self.infos.append(message) + + def warning(self, message, emoji=True): + self.warnings.append(message) + + def error(self, message, emoji=True): + pass + + def success(self, message, emoji=True): + pass + + def print_banner(self, *args, **kwargs): + pass + + +class _FakeSimulator: + criteria: list = [] + + async def run(self, graph, scenario, *, on_event=None): + await asyncio.sleep(0.01) + on_event("turn_started", {"turn": 1, "max_turns": 10, "user_message": "hi\nthere"}) + on_event("agent_replied", {"turn": 1, "response": "x" * 300}) + on_event( + "goals_checked", + {"turn": 1, "newly_achieved": ["greet"], "achieved": 1, "total": 2}, + ) + on_event("goals_checked", {"turn": 1, "newly_achieved": [], "achieved": 1, "total": 2}) + on_event("agent_error", {"turn": 2, "error": "boom"}) + on_event("evaluating", {"criteria": ["a", "b"]}) + return types.SimpleNamespace( + criterion_results=[], + criterion_scores={}, + criterion_details={}, + completed=True, + conversation=[{"role": "user", "content": "hi"}], + simulator_token_usage=TokenUsage(), + turns=1, + goals_achieved=["greet"], + ) + + +def _simulation() -> _PendingSimulation: + scenario = types.SimpleNamespace(scenario_id="sim1", description="Greeting flow") + return _PendingSimulation( + scenario=scenario, + graph=object(), + simulator=_FakeSimulator(), + file_name="f_eval.py", + eval_set_id="s", + eval_set_name="S", + ) + + +def test_simulation_steps_and_duration() -> None: + progress = _RecordingProgress() + cmd = EvalCommand(output=_Out(progress)) + quads = asyncio.run(cmd._run_flat_pool([_simulation()], 1, False)) + result = quads[0][3] + assert result.duration_seconds > 0 + messages = [m for _, m in progress.steps] + assert messages[0] == "turn 1/10 user: hi\nthere" # raw; truncation is in the renderer + assert any(m.startswith("turn 1/10 agent: ") for m in messages) + assert "turn 1/10 goals 1/2 (+ greet)" in messages + assert len([m for m in messages if "goals" in m]) == 1 + assert "turn 2/10 agent error: boom" in messages + assert "evaluating criteria: a, b" in messages + assert all(label == "f_eval.py::Greeting flow" for label, _ in progress.steps) + + +def test_case_gets_running_step_and_duration() -> None: + progress = _RecordingProgress() + cmd = EvalCommand(output=_Out(progress)) + case = types.SimpleNamespace(eval_id="c1", name="case one") + + async def _evaluate(case, collector_override=None): + await asyncio.sleep(0.01) + return EvalCaseResult.success( + eval_id="c1", name="case one", criterion_results=[], actual_response="" + ) + + evaluator = MagicMock() + evaluator._evaluate_case = _evaluate + evaluator.collector.capture_all_events = False + pc = types.SimpleNamespace( + case=case, evaluator=evaluator, file_name="a_eval.py", eval_set_id="s", eval_set_name="S" + ) + with ( + patch("tenxgraph_api.cli.commands.eval._PendingCase", types.SimpleNamespace), + patch("tenxgraph_api.cli.commands.eval.override_dependency", MagicMock()), + patch("tenxgraph_api.cli.commands.eval.make_trajectory_callback") as mk, + ): + mk.return_value = (None, None) + quads = asyncio.run(cmd._run_flat_pool([pc], 1, False)) + assert progress.steps == [("a_eval.py::case one", "running")] + assert quads[0][3].duration_seconds > 0 + + +def test_one_line_collapses_and_truncates() -> None: + assert one_line("a\n b\tc") == "a b c" + out = one_line("x" * 300) + assert len(out) == 100 and out.endswith("...") + + +def test_static_step_line() -> None: + emitted: list[str] = [] + run = StaticProgressRun(total=1, title="t", glyphs=UNICODE_GLYPHS, emit=emitted.append) + with run: + run.step("f::s", "turn 1/3 user: hello\nworld") + assert emitted[-1].strip() == "f::s turn 1/3 user: hello world" + assert run.completed == 0 + + +def test_structured_step_event() -> None: + events: list[tuple[str, str, dict]] = [] + run = StructuredProgressRun(total=1, title="t", emit=lambda *a: events.append(a)) + with run: + run.step("f::s", "running") + assert ("eval_step", "f::s", {"message": "running"}) in events + assert run.completed == 0 + + +def test_live_step_line_does_not_advance() -> None: + stream = io.StringIO() + console = Console(file=stream, force_terminal=True, width=120, legacy_windows=False) + with LiveProgressRun(console, total=2, title="t", glyphs=UNICODE_GLYPHS) as run: + run.step("f::s", "turn 1/3 agent: hi") + assert "f::s" in stream.getvalue() and "agent: hi" in stream.getvalue() + assert run.completed == 0 and run.tally["passed"] == 0 + + +def test_quiet_step_is_noop() -> None: + run = QuietProgressRun(total=1, title="t") + with run: + run.step("a", "b") + assert run.completed == 0 + + +# --------------------------------------------------------------------------- +# Report + browser +# --------------------------------------------------------------------------- + + +def _execute(tmp_path: Path, *, caps: dict | None = None, env_ci: bool = False, **kwargs): + out = _Out() + out.capabilities = dataclasses.replace( + out.capabilities, interactive=True, output_format=OutputFormat.HUMAN, is_ci=False + ) + if caps: + out.capabilities = dataclasses.replace(out.capabilities, **caps) + cmd = EvalCommand(output=out) + + mock_cm = MagicMock() + mock_cm.auto_discover_config.return_value = "10xgraph.json" + mock_cm.get_evaluation_config.return_value = kwargs.pop("eval_cfg", {}) + report = MagicMock() + report.summary.pass_rate = 1.0 + report.summary.passed_cases = 1 + report.summary.total_cases = 1 + report.summary.total_token_usage = TokenUsage() + res = MagicMock() + res.json_path = str(tmp_path / "r.json") + res.html_path = str(tmp_path / "r.html") + res.has_errors = False + mgr = MagicMock() + mgr.run_all.return_value = res + case = EvalCaseResult.success(eval_id="c", name="c", criterion_results=[], actual_response="") + fake = MagicMock() + fake.config = EvalConfig() + with ( + patch("tenxgraph_api.cli.commands.eval.ConfigManager", return_value=mock_cm), + patch.object(cmd, "_discover", return_value=[Path("t_eval.py")]), + patch.object(cmd, "_load_confeval", return_value=None), + patch.object(cmd, "_collect_from_file", return_value=[fake]), + patch.object(cmd, "_print_criteria_per_file"), + patch.object(cmd, "_run_flat_pool", return_value=[("t", "id", "n", case)]), + patch.object(cmd, "_merge_reports", return_value=report), + patch("tenxgraph_api.cli.commands.eval.ReporterManager", return_value=mgr) as rm, + patch("tenxgraph_api.cli.commands.eval.webbrowser.open", return_value=True) as wb, + ): + code = cmd.execute(target=str(tmp_path), **kwargs) + return code, out, wb, rm + + +def test_html_and_json_requested_and_listed(tmp_path: Path) -> None: + _, out, _, rm = _execute(tmp_path) + cfg = rm.call_args.args[0] + assert cfg.html is True and cfg.json_report is True and cfg.timestamp_files is True + assert f"HTML report: {tmp_path / 'r.html'}" in out.infos + assert f"JSON report: {tmp_path / 'r.json'}" in out.infos + + +def test_opens_html_once_by_default(tmp_path: Path) -> None: + _, _, wb, _ = _execute(tmp_path) + wb.assert_called_once_with((tmp_path / "r.html").resolve().as_uri()) + + +@pytest.mark.parametrize( + ("caps", "kwargs"), + [ + (None, {"open_report": False}), + (None, {"quiet": True}), + (None, {"no_report": True}), + (None, {"non_interactive": True}), + ({"output_format": OutputFormat.JSON}, {}), + ({"output_format": OutputFormat.JSONL}, {}), + ({"interactive": False}, {}), + ({"interactive": False, "is_ci": True}, {}), + (None, {"eval_cfg": {"open_report": False}}), + ], +) +def test_does_not_open(tmp_path: Path, caps, kwargs) -> None: + _, _, wb, _ = _execute(tmp_path, caps=caps, **kwargs) + wb.assert_not_called() + + +def test_cli_flag_beats_config(tmp_path: Path) -> None: + _, _, wb, _ = _execute(tmp_path, open_report=True, eval_cfg={"open_report": False}) + wb.assert_called_once() + + +def test_open_failure_only_warns(tmp_path: Path) -> None: + out = _Out() + cmd = EvalCommand(output=out) + with patch( + "tenxgraph_api.cli.commands.eval.webbrowser.open", side_effect=RuntimeError("no display") + ): + cmd._open_report(str(tmp_path / "r.html")) + assert out.warnings and "no display" in out.warnings[0] + + +def test_simulator_without_on_event_still_runs() -> None: + class _Old(_FakeSimulator): + async def run(self, graph, scenario): + return await _FakeSimulator.run(self, graph, scenario, on_event=lambda *a: None) + + sim = _simulation() + sim.simulator = _Old() + progress = _RecordingProgress() + quads = asyncio.run(EvalCommand(output=_Out(progress))._run_flat_pool([sim], 1, False)) + assert quads[0][3].error is None + assert progress.steps == [] + + +def test_simulation_result_carries_expected_start_and_goals() -> None: + sim = _simulation() + sim.scenario.starting_prompt = "Plan a trip" + sim.scenario.goals = ["Get weather", "Get advice"] + cmd = EvalCommand(output=_Out(_RecordingProgress())) + result = asyncio.run(cmd._run_flat_pool([sim], 1, False))[0][3] + assert result.expected == { + "input": "Plan a trip", + "response": "Get weather; Get advice", + "tool_calls": None, + "node_order": None, + } + + +def test_failed_simulation_and_case_keep_expected_and_duration() -> None: + class _Boom(_FakeSimulator): + async def run(self, graph, scenario, *, on_event=None): + await asyncio.sleep(0.01) + raise RuntimeError("down") + + sim = _simulation() + sim.simulator = _Boom() + sim.scenario.starting_prompt = "Hi" + sim.scenario.goals = ["g"] + cmd = EvalCommand(output=_Out(_RecordingProgress())) + failed = asyncio.run(cmd._run_flat_pool([sim], 1, False))[0][3] + assert failed.error == "down" + assert failed.expected["response"] == "g" + assert failed.duration_seconds > 0 diff --git a/tests/cli/test_eval_templates.py b/tests/cli/test_eval_templates.py new file mode 100644 index 00000000..5459b8ec --- /dev/null +++ b/tests/cli/test_eval_templates.py @@ -0,0 +1,35 @@ +"""The scaffolded eval files check the expectations they declare and are satisfiable.""" + +import importlib.util +from pathlib import Path + +import tenxgraph_api + + +EVALS = Path(tenxgraph_api.__file__).parent / "cli" / "templates" / "prod" / "evals" + + +def _load(name: str): + spec = importlib.util.spec_from_file_location(name, EVALS / f"{name}.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_weather_eval_checks_response_tools_and_node_order() -> None: + module = _load("weather_agents_eval") + criteria = module.get_eval_config().criteria + assert criteria.tool_name_match is not None + assert criteria.trajectory is not None + assert criteria.rouge_match is not None + assert criteria.node_order is not None + cases = module.get_eval_set().eval_cases + for case in cases: + assert case.expected_summary()["response"] != "assistant" + + +def test_simulator_scenarios_fit_a_weather_only_agent() -> None: + module = _load("user_simulator_eval") + ids = [s.scenario_id for s in module.get_scenarios()] + assert "flight_booking_assistance" not in ids + assert len(ids) == 2 From a9bb26ba9d7e0c14df0a7545001691170a7fd59b Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 01:12:00 +0600 Subject: [PATCH 11/16] feat(eval): record pass-rate gate in evaluation reports metadata --- CHANGELOG.md | 6 ++++++ tenxgraph_api/cli/commands/eval.py | 7 +++++++ tests/cli/test_eval_progress_and_report.py | 19 +++++++++++++++++++ 3 files changed, 32 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ef654b4..41930777 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -199,6 +199,12 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Changed +- **`10xgraph eval` records the pass-rate gate in the report.** The merged report's metadata + carries `pass_rate_threshold` (the `--threshold` value, or 1.0 without one, which is what the + exit code uses) and `pass_rate_threshold_source` (`"threshold"` or `"default"`). The JSON + report includes both, and the new HTML scorecard (a `10xgraph` core change) uses them for its + verdict. + - **The local media store no longer creates `./uploads` at server startup.** The folder is created on the first upload (a `10xgraph` core change). - **`10xgraph init --template production` scaffolds the recommended production setup.** Auth diff --git a/tenxgraph_api/cli/commands/eval.py b/tenxgraph_api/cli/commands/eval.py index 7a7434e3..ebc06a3b 100644 --- a/tenxgraph_api/cli/commands/eval.py +++ b/tenxgraph_api/cli/commands/eval.py @@ -963,6 +963,13 @@ def execute( # noqa: PLR0912, PLR0913, PLR0915 if isinstance(getattr(merged, "metadata", None), dict): merged.metadata["criteria_by_file"] = self._criteria_by_file(pending) + # Record the pass-rate gate the exit code uses so the reports can state it. + if isinstance(getattr(merged, "metadata", None), dict): + merged.metadata["pass_rate_threshold"] = 1.0 if threshold is None else threshold + merged.metadata["pass_rate_threshold_source"] = ( + "default" if threshold is None else "threshold" + ) + # 9. Determine exit code return_code = self._exit_code(merged.summary.pass_rate, threshold) if threshold is not None and return_code: diff --git a/tests/cli/test_eval_progress_and_report.py b/tests/cli/test_eval_progress_and_report.py index a390cceb..7c06395a 100644 --- a/tests/cli/test_eval_progress_and_report.py +++ b/tests/cli/test_eval_progress_and_report.py @@ -213,6 +213,7 @@ def _execute(tmp_path: Path, *, caps: dict | None = None, env_ci: bool = False, mock_cm.auto_discover_config.return_value = "10xgraph.json" mock_cm.get_evaluation_config.return_value = kwargs.pop("eval_cfg", {}) report = MagicMock() + report.metadata = {} report.summary.pass_rate = 1.0 report.summary.passed_cases = 1 report.summary.total_cases = 1 @@ -249,6 +250,24 @@ def test_html_and_json_requested_and_listed(tmp_path: Path) -> None: assert f"JSON report: {tmp_path / 'r.json'}" in out.infos +def _recorded_metadata(rm) -> dict: + return rm.return_value.run_all.call_args.args[0].metadata + + +def test_report_metadata_records_explicit_gate(tmp_path: Path) -> None: + _, _, _, rm = _execute(tmp_path, threshold=0.62) + meta = _recorded_metadata(rm) + assert meta["pass_rate_threshold"] == 0.62 + assert meta["pass_rate_threshold_source"] == "threshold" + + +def test_report_metadata_records_default_gate(tmp_path: Path) -> None: + _, _, _, rm = _execute(tmp_path) + meta = _recorded_metadata(rm) + assert meta["pass_rate_threshold"] == 1.0 + assert meta["pass_rate_threshold_source"] == "default" + + def test_opens_html_once_by_default(tmp_path: Path) -> None: _, _, wb, _ = _execute(tmp_path) wb.assert_called_once_with((tmp_path / "r.html").resolve().as_uri()) From 38f034953be48bd15ebbce70f61dd3c915f4a491 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 09:36:46 +0600 Subject: [PATCH 12/16] feat(shutdown): ensure thread lock is released only after run save during cancellation --- CHANGELOG.md | 5 + tenxgraph_api/src/app/core/shutdown.py | 67 +++++- .../routers/graph/services/graph_service.py | 11 +- tests/unit_tests/test_run_cleanup_order.py | 216 ++++++++++++++++++ 4 files changed, 284 insertions(+), 15 deletions(-) create mode 100644 tests/unit_tests/test_run_cleanup_order.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 41930777..8a6fe208 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -300,6 +300,11 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Fixed +- **A disconnected stream frees its thread only after the run is saved.** `end_on_shutdown` + and the `runs.timeout` wrapper cancelled the run task and then awaited it, which under + Starlette's cancel scope cancelled the run on every loop turn and returned before core's + cancel save finished, so the thread lock was free while the old run still wrote. They now + cancel once and wait (bounded, 15 s) in a shielded scope. Needs `10xgraph` 1.0. - **`10xgraph eval` reports carry each case's expectations.** The report's case results now include `expected` (response, tool calls, node order, input). For a user simulation, `input` is the starting prompt and `response` is the goals joined with `; `. Needs `10xgraph` 1.0. diff --git a/tenxgraph_api/src/app/core/shutdown.py b/tenxgraph_api/src/app/core/shutdown.py index 47c9445c..4f3837fe 100644 --- a/tenxgraph_api/src/app/core/shutdown.py +++ b/tenxgraph_api/src/app/core/shutdown.py @@ -11,11 +11,21 @@ import asyncio import contextlib +import logging from collections.abc import AsyncIterator, Callable +import anyio from tenxgraph.core.state import StreamChunk, StreamEvent +try: + from tenxgraph.core.graph.utils.utils import FINALIZE_TIMEOUT_SECONDS +except ImportError: # an older core; same value + FINALIZE_TIMEOUT_SECONDS = 10.0 + +logger = logging.getLogger("tenxgraph_api.shutdown") + + INTERRUPTED_REASON = ( "The server is shutting down and this run was interrupted. " "The thread is saved: send a new request on the same thread_id to resume." @@ -27,6 +37,10 @@ KEEPALIVE_LINE = "\n" KEEPALIVE_SECONDS = 15.0 +# How long a cancelled run task is waited for: core saves a cancelled run within +# ``FINALIZE_TIMEOUT_SECONDS``, then releases its run marker. +RUN_CLEANUP_WAIT_SECONDS = FINALIZE_TIMEOUT_SECONDS + 5.0 + _interrupted = False _waiters: set[asyncio.Future[None]] = set() @@ -60,6 +74,39 @@ async def _next(stream: AsyncIterator[str]) -> str: return await anext(stream) +async def cancel_and_wait(task: asyncio.Future, timeout: float = RUN_CLEANUP_WAIT_SECONDS) -> None: + """Cancel ``task`` once and wait until it is done, up to ``timeout`` seconds. + + For a task that runs a graph: core saves a cancelled run before the task ends, and a lock + the caller releases afterwards (the thread lock) must not be free before that save is. + ``await task`` would forward every cancel the caller gets to ``task``; under a cancelled + anyio scope (Starlette on a client disconnect) that is one cancel per event-loop turn. The + wait runs in a shielded anyio scope instead, and a plain cancel that still reaches the + caller is raised once ``task`` is done or the time is up. The task's own result or error + is not raised. + """ + if task.done(): + return + task.cancel() + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout + cancel: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while not task.done(): + remaining = deadline - loop.time() + if remaining <= 0: + logger.warning("A cancelled run did not finish within %ss", timeout) + break + try: + await asyncio.wait({task}, timeout=remaining) + except asyncio.CancelledError as e: + cancel = e + if task.done() and not task.cancelled(): + task.exception() # retrieved: it is the run's own error, reported by the run + if cancel is not None: + raise cancel + + async def end_on_shutdown( stream: AsyncIterator[str], final: Callable[[], str] = interrupted_line, @@ -89,9 +136,7 @@ async def end_on_shutdown( if not pending.done() and not waiter.done(): yield KEEPALIVE_LINE if not pending.done(): - pending.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await pending + await cancel_and_wait(pending) pending = None yield final() return @@ -104,11 +149,11 @@ async def end_on_shutdown( yield item finally: _waiters.discard(waiter) - if pending is not None and not pending.done(): - pending.cancel() - with contextlib.suppress(asyncio.CancelledError, Exception): - await pending - aclose = getattr(stream, "aclose", None) - if callable(aclose): - with contextlib.suppress(Exception): - await aclose() + try: + if pending is not None and not pending.done(): + await cancel_and_wait(pending) + finally: + aclose = getattr(stream, "aclose", None) + if callable(aclose): + with contextlib.suppress(Exception): + await aclose() diff --git a/tenxgraph_api/src/app/routers/graph/services/graph_service.py b/tenxgraph_api/src/app/routers/graph/services/graph_service.py index 44919bb4..537cd449 100644 --- a/tenxgraph_api/src/app/routers/graph/services/graph_service.py +++ b/tenxgraph_api/src/app/routers/graph/services/graph_service.py @@ -35,7 +35,11 @@ ) from tenxgraph_api.src.app.core.concurrency import Lease from tenxgraph_api.src.app.core.config.graph_config import GraphConfig, RunsConfig -from tenxgraph_api.src.app.core.shutdown import KEEPALIVE_SECONDS, end_on_shutdown +from tenxgraph_api.src.app.core.shutdown import ( + KEEPALIVE_SECONDS, + cancel_and_wait, + end_on_shutdown, +) from tenxgraph_api.src.app.core.utils.log_sanitizer import log_safe, sanitize_for_logging from tenxgraph_api.src.app.routers.graph.schemas.graph_schemas import ( GraphInputSchema, @@ -324,9 +328,8 @@ async def produce() -> None: consumed.set() finally: if not producer.done(): - producer.cancel() - with contextlib.suppress(asyncio.CancelledError): - await producer + # Waits for the run's cancel save before the caller frees the thread lock. + await cancel_and_wait(producer) # Realtime init-frame key -> RealtimeConfig field name. diff --git a/tests/unit_tests/test_run_cleanup_order.py b/tests/unit_tests/test_run_cleanup_order.py new file mode 100644 index 00000000..26e6d9f6 --- /dev/null +++ b/tests/unit_tests/test_run_cleanup_order.py @@ -0,0 +1,216 @@ +"""G-171: the API frees a run's thread only after core saved the cancelled run. + +A client that disconnects cancels the response task through a level-triggered anyio scope +(Starlette). The wrappers around the graph stream (``end_on_shutdown``, ``_bounded_stream``) +run the graph in a task of their own; they must wait for that task, not forward a cancel to it +on every loop turn and return early. The store here is slow, so returning early shows up as a +state that is not saved yet. +""" + +# ruff: noqa: S101 + +import asyncio +import time + +import anyio +import pytest +from injectq import InjectQ +from injectq.core.context import ContainerContext +from starlette.responses import StreamingResponse +from tenxgraph.core.graph import StateGraph +from tenxgraph.core.state import AgentState, ExecutionStatus, Message +from tenxgraph.storage.checkpointer import InMemoryCheckpointer +from tenxgraph.storage.checkpointer.base_checkpointer import ACTIVE_RUN_NAMESPACE +from tenxgraph.utils.constants import END + +from tenxgraph_api.src.app.core import shutdown +from tenxgraph_api.src.app.core.concurrency import Lease +from tenxgraph_api.src.app.routers.graph.run_limits import release_when_done +from tenxgraph_api.src.app.routers.graph.services.graph_service import _bounded_stream + + +SAVE_DELAY = 0.3 +CFG = {"thread_id": "t1", "user_id": "u1", "run_id": "r1"} + + +@pytest.fixture(autouse=True) +def _own_container(): + """Each graph here binds its checkpointer; a leftover binding would take the saves.""" + ContainerContext.clear_current() + InjectQ.reset_instance() + yield + ContainerContext.clear_current() + InjectQ.reset_instance() + + +class SlowCheckpointer(InMemoryCheckpointer): + def __init__(self): + super().__init__() + self.log: list[str] = [] + + async def aput_checkpoint(self, config, state, messages=None, metadata=None): + status = state.execution_meta.status.value + await asyncio.sleep(SAVE_DELAY) + result = await super().aput_checkpoint(config, state, messages, metadata) + self.log.append(f"saved {status}") + return result + + async def aend_run(self, config): + self.log.append("run ended") + return await super().aend_run(config) + + +def build(): + cp = SlowCheckpointer() + started = asyncio.Event() + + async def node(state: AgentState, config: dict): + started.set() + await asyncio.sleep(5) + return [Message.text_message("answer", role="assistant")] + + graph = StateGraph() + graph.add_node("N", node) + graph.set_entry_point("N") + graph.add_edge("N", END) + return graph.compile(checkpointer=cp), cp, started + + +def chunks(app): + return app.astream({"messages": [Message.text_message("hi")]}, dict(CFG)) + + +async def lines(app): + async for chunk in chunks(app): + yield chunk.model_dump_json() + "\n" + + +async def assert_saved(cp): + state = await cp.aget_state(CFG) + assert state is not None + assert state.execution_meta.status == ExecutionStatus.INTERRUPTED_AFTER + marker = await cp.aget_cache_value(ACTIVE_RUN_NAMESPACE, cp._stop_request_key(CFG)) + assert marker is None + + +async def cancel_host(host, started) -> tuple[float, float]: + """Run ``host`` in an anyio task group, cancel it once the run started; (wall, cpu).""" + async with anyio.create_task_group() as tg: + tg.start_soon(host) + await asyncio.wait_for(started.wait(), 5) + cpu, wall = time.process_time(), time.monotonic() + tg.cancel_scope.cancel() + return time.monotonic() - wall, time.process_time() - cpu + + +@pytest.mark.asyncio +async def test_end_on_shutdown_returns_only_after_the_core_save(): + shutdown.reset() + app, cp, started = build() + + async def host(): + async for _ in shutdown.end_on_shutdown(lines(app), keepalive_seconds=15): + pass + + wall, cpu = await cancel_host(host, started) + + await assert_saved(cp) # at once: nothing left to finish in the background + assert cp.log[-2:] == ["saved interrupted_after", "run ended"] + assert wall >= SAVE_DELAY * 0.8 + assert cpu < wall / 2 # no cancel forwarded to the run on every loop turn + await app.aclose() + + +@pytest.mark.asyncio +async def test_bounded_stream_returns_only_after_the_core_save(): + app, cp, started = build() + + async def host(): + async for _ in _bounded_stream(chunks(app), 10): + pass + + wall, cpu = await cancel_host(host, started) + + await assert_saved(cp) + assert cp.log[-2:] == ["saved interrupted_after", "run ended"] + assert cpu < wall / 2 + await app.aclose() + + +@pytest.mark.asyncio +async def test_thread_lock_is_released_only_after_the_save(): + """The REST stream shape: StreamingResponse over release_when_done(end_on_shutdown(...)).""" + shutdown.reset() + app, cp, started = build() + at_release: list[list[str]] = [] + released = asyncio.Event() + + def on_release(): + at_release.append(list(cp.log)) + released.set() + + async def stream_graph(): + async for line in shutdown.end_on_shutdown(lines(app), keepalive_seconds=15): + yield line + + response = StreamingResponse( + release_when_done(stream_graph(), Lease(on_release)), media_type="application/x-ndjson" + ) + + async def receive(): + await started.wait() + return {"type": "http.disconnect"} + + async def send(message): + pass + + scope = {"type": "http", "asgi": {"version": "3.0", "spec_version": "2.3"}} + await response(scope, receive, send) + await asyncio.wait_for(released.wait(), 5) + + assert len(at_release) == 1 + assert at_release[0][-2:] == ["saved interrupted_after", "run ended"] + await assert_saved(cp) + await app.aclose() + + +@pytest.mark.asyncio +async def test_cancel_and_wait_raises_a_cancel_of_the_caller_once_the_task_is_done(): + job_done = asyncio.Event() + + async def job(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await asyncio.sleep(0.1) # its own cleanup + job_done.set() + raise + + task = asyncio.ensure_future(job()) + await asyncio.sleep(0) + waiter = asyncio.ensure_future(shutdown.cancel_and_wait(task, timeout=5)) + await asyncio.sleep(0.02) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + assert job_done.is_set() + + +@pytest.mark.asyncio +async def test_cancel_and_wait_gives_up_after_the_timeout(): + release = asyncio.Event() + + async def slow_cleanup(): + try: + await asyncio.sleep(5) + except asyncio.CancelledError: + await release.wait() # cleanup that outlasts the wait + raise + + task = asyncio.ensure_future(slow_cleanup()) + await asyncio.sleep(0) + await shutdown.cancel_and_wait(task, timeout=0.05) + assert not task.done() + release.set() + await asyncio.wait({task}, timeout=1) + assert task.cancelled() From 9c6b337b923ed9cab614d2e45741bdad6949486f Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 15:49:27 +0600 Subject: [PATCH 13/16] feat: update version to 1.0.0rc2 and enhance weather tool functionality --- CHANGELOG.md | 19 ++++- CLAUDE.md | 4 +- README.md | 2 +- RELEASE_NOTES.md | 65 ++++++++++++++--- SECURITY.md | 2 +- pyproject.toml | 8 +- tenxgraph_api/cli/commands/eval.py | 2 +- tenxgraph_api/cli/config_editor/schema.py | 2 +- tenxgraph_api/cli/main.py | 2 +- .../cli/templates/dev/graph/agent.py | 24 +++++- .../prod/evals/weather_agents_eval.py | 6 +- .../prod/graph/tools/weather_tool.py | 24 +++++- .../prod/tests/test_catalog_tools.py | 8 ++ .../references/prebuilt-agents-and-tools.md | 4 +- .../skills/10xgraph/references/realtime.md | 6 +- .../src/app/core/config/graph_config.py | 2 +- tenxgraph_api/src/app/core/shutdown.py | 8 +- tests/unit_tests/test_run_cleanup_order.py | 73 +++++++++++++++++++ 18 files changed, 224 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a6fe208..fdb5202b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,10 +27,10 @@ project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). --- -## [1.0.0] - Unreleased +## [1.0.0rc2] - 2026-10-30 -The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is aligned with -`10xgraph-client` 1.0. It is the `1.0.0rc1` code plus the changes below. If you upgrade from +The second release candidate of `10xgraph-api` 1.0. It requires `10xgraph` 1.0.0rc2 and is aligned +with `10xgraph-client` 1.0.0-rc.2. It is the `1.0.0rc1` code plus the changes below. If you upgrade from 0.9.x, the `1.0.0rc1` breaking list further down applies to you as well. ### Breaking @@ -94,7 +94,7 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig of rotation. See the `10xgraph` changelog for the full note. - **Dependency minimums:** `python-multipart>=0.0.31`, `starlette>=1.3.1`, `PyJWT>=2.15.0` (the `jwt` extra) and `opentelemetry-api` and `opentelemetry-sdk` `>=1.45.1` (the `otel` - extra). `10xgraph>=1.0.0rc1,<2.0` is the core floor in `pyproject.toml`. + extra). `10xgraph>=1.0.0rc2,<2.0` is the core floor in `pyproject.toml`. - **Shutdown timing:** the generated Gunicorn command uses a new worker class, `tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker`. Regenerate Docker and Kubernetes files with `10xgraph build`, or point your own command at it. Keep Gunicorn @@ -300,6 +300,17 @@ The stable 1.0 release of `10xgraph-api`. It requires `10xgraph` 1.0 and is alig ### Fixed +- **Stop and shutdown reach a REST stream after its first chunk.** `end_on_shutdown` ran each + step of the stream in a new task with a fresh context, so the graph's current run (which + carries the in-process stop flag and the shutdown reason) was lost after the first chunk. All + steps now share one context. +- **Template weather tool answers per city.** The weather tool in the `dev` and `prod` templates + always said "sunny", so an eval goal such as "which city has nicer weather" could never pass. + It now returns fixed weather for a few known cities and a stable answer for any other name + (no randomness), with the same signature and reply format. +- **Current Gemini Live model in the defaults.** The `websocket.realtime_models` default and the + config editor placeholder named `gemini-2.5-flash-live`, which is not a current model; they + now name `gemini-3.8-live`. - **A disconnected stream frees its thread only after the run is saved.** `end_on_shutdown` and the `runs.timeout` wrapper cancelled the run task and then awaited it, which under Starlette's cancel scope cancelled the run on every loop turn and returned before core's diff --git a/CLAUDE.md b/CLAUDE.md index 38669a8f..b4f81948 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,13 +5,13 @@ core framework see `agentflow/CLAUDE.md`; for the TS client, docs, or playground for the monorepo overview see the workspace-root `CLAUDE.md`. - Package name (PyPI): `10xgraph-api` (formerly `10xscale-agentflow-cli`, last release 0.6.x) -- Version: `1.0.0rc1` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` +- Version: `1.0.0rc2` (`pyproject.toml`; 0.7.0 was the first release under the 10xGraph name). `CLI_VERSION` and `tenxgraph_api.__version__` are single-sourced from the installed distribution metadata (falling back to `pyproject.toml` only for a non-installed source checkout). - Requires: Python >= 3.12 · Status: `5 - Production/Stable` - Console entry points: `10xgraph = tenxgraph_api.cli.main:main`, plus the deprecated alias `agentflow = tenxgraph_api.cli.main:legacy_main` (prints a notice to stderr; removed in 2.0) -- Depends on the core framework: `10xgraph>=1.0.0rc1,<2.0` (import `tenxgraph`). Must not be +- Depends on the core framework: `10xgraph>=1.0.0rc2,<2.0` (import `tenxgraph`). Must not be installed alongside the old `10xscale-agentflow`: both provide an `agentflow` module. ## Rename compatibility (kept until 2.0) diff --git a/README.md b/README.md index 554ae5de..c9536538 100644 --- a/README.md +++ b/README.md @@ -527,7 +527,7 @@ sdist + wheel, checks the distribution metadata, and creates a GitHub Release wi notes and the artifacts attached. PyPI publishing is manual (`make publish`). ```bash -git tag v1.0.0rc1 && git push origin v1.0.0rc1 +git tag v1.0.0rc2 && git push origin v1.0.0rc2 ``` --- diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index e4954be8..738cb7bf 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,21 +1,66 @@ -# 10xgraph-api 1.0.0rc1 +# 10xgraph-api 1.0.0rc2 -The 1.0 release candidate: secure-by-default threads, production checks and error handling, plus resume, run-busy and file-lifecycle features. +The second 1.0 release candidate: it is the `1.0.0rc1` code plus graceful shutdown, stricter request validation, 409 answers for writes during a run, idempotency keys, JWT key rotation and a redesigned eval and config experience. -**This is a release candidate.** Pip skips it unless you pass `--pre`. It requires `10xgraph` 1.0, and its version is aligned with core `10xgraph` and `10xgraph-client`. This release has **breaking changes**. Read "Before you upgrade" first. +**This is a release candidate.** Pip skips it unless you pass `--pre`. It requires `10xgraph` 1.0.0rc2, and its version is aligned with core `10xgraph` and `10xgraph-client` (1.0.0-rc.2). This release has **breaking changes**. Read "Before you upgrade" first. ```bash -pip install -U --pre "10xgraph-api==1.0.0rc1" +pip install -U --pre "10xgraph-api==1.0.0rc2" ``` -This pulls `10xgraph` 1.0.0rc1. +This pulls `10xgraph` 1.0.0rc2. -## Before you upgrade +## Before you upgrade (changes in rc2) + +If you are on `1.0.0rc1`, this section is all that is new. If you are coming from 0.9.x, read it and then "Upgrading from 0.9.x" below. + +- **AG-UI and `/v1/graph/live` now follow `runs.thread_ids` like REST.** Under the default `"server"`, a thread id the server never issued gets 403 on `POST /v1/ag-ui`, and an error event (`not_authorized`, close code 1008) on live. A blank or `"new"` id, or none, makes the server create the thread: AG-UI returns it in `RUN_STARTED`, and live sends `{"type": "session_started", "thread_id": ""}` first. Frontends that make their own ids (CopilotKit does by default) must set `"runs": {"thread_ids": "client"}` in `10xgraph.json`. +- **Ids must match `^[A-Za-z0-9._:@-]{1,128}$`** (`thread_id`, `message_id`, memory ids, WebSocket and AG-UI thread ids). Others get 422 `VALIDATION_ERROR`. Snowflake integers and UUIDs fit. +- **Nested `initial_state`, `config` and `resume` deeper than 64 levels get 422**, and client dicts and lists are size-capped (1000 keys per object, 10000 items per list, 200000 values in all). +- **Resume errors answer 409 or 422.** A wrong interrupt id, a resume sent to a thread that is not paused, or a plain message sent to a thread paused at `interrupt()` returns 409 (before the stream starts on `/v1/graph/stream`). A resume value that fails the interrupt's `response_schema` returns 422. +- **Writes during a run return 409.** `PUT`/`DELETE` thread state, `POST` messages, `DELETE` of a message or thread, and `POST /v1/graph/fix` are refused with 409 while a run holds the thread. Retry after the run ends or stop it first. +- **Thread delete can return `success=false`** with `data.cleanup_failed` (`files`, `traces`, `cache`) when a cleanup step failed. The thread rows are deleted; treat it as "retry the cleanup". +- **`JWT_ALGORITHM` must be on an allowlist** (HS/RS/ES/PS 256/384/512 and `EdDSA`), PyJWT must be installed when `"auth": "jwt"`, and `auth` must be exactly `"jwt"`. +- **Other error and format changes:** the `graph:setup` scope is gone; the duplicate `AGENTFLOW_VALIDATION_ERROR` code is no longer sent (use `VALIDATION_ERROR`); a recursion limit answers 422 `GRAPH_RECURSION_LIMIT`, not 500; `POST /v1/graph/stream` is labelled `application/x-ndjson` and sends a blank keep-alive line every 15 s (update proxy rules that matched `text/event-stream`). +- **Uploads:** images over 50,000,000 pixels and PDFs over 10 MB or 500 pages get 413. +- **`10xgraph build`** refuses `--docker-compose` together with `--k8s`; run it once per target. +- **`IS_DEBUG` is removed.** Debug follows `MODE`. The `/v1/evals/runs` endpoints are removed, and `10xgraph eval` writes the JSON report (and now an HTML report, see below) with no `--open` flag. +- **Postgres:** `10xgraph` 1.0 runs schema migration v4 on first use. Stop all old pods, run the migration (one new pod), then start the rest. A rollback needs a database restore. `/health/ready` fails while the schema is older than the code expects. +- **Dependency minimums:** `python-multipart>=0.0.31`, `starlette>=1.3.1`, `PyJWT>=2.15.0` (`jwt` extra), `opentelemetry-api` and `opentelemetry-sdk` `>=1.45.1` (`otel` extra). The core floor is `10xgraph>=1.0.0rc2,<2.0`. +- **Gunicorn and shutdown:** the generated command uses the worker class `tenxgraph_api.src.app.uvicorn_worker.TenxGraphUvicornWorker`. Regenerate Docker and Kubernetes files with `10xgraph build`, or point your own command at it. Keep Gunicorn `--graceful-timeout` above the sum of the three `SHUTDOWN_*_SECONDS` settings (50 s by default) and the pod grace period above that. +- **Alerting:** handled client errors log one WARNING line, not an ERROR with a traceback. Alerts that matched ERROR for 4xx must match WARNING. + +## What changed in rc2 + +**Shutdown and reliability** +- Graceful shutdown: `SHUTDOWN_GRACE_SECONDS` (10), `SHUTDOWN_STOP_SECONDS` (10) and `SHUTDOWN_TIMEOUT_SECONDS` (30). Cut runs are saved INTERRUPTED and the thread resumes on the next request. While closing, runs answer 503 `SERVICE_UNAVAILABLE` with `Retry-After`, `/health/ready` reports `draining`, and an open HTTP stream ends with a chunk that has `data.interrupted: true`. +- HTTP run streams end with `{"event": "updates", "data": {"status": "done"}}`, so a client can tell a finished stream from a cut one. +- `Retry-After` on 429 and 409, typed error envelopes for unhandled errors, 503 for database or Redis connection errors, and 429 `PROVIDER_RATE_LIMITED` for model provider rate limits. +- Redis timeouts and pool settings (`REDIS_SOCKET_TIMEOUT` and others) and `REDIS_KEY_PREFIX` for deployments that share a Redis. Leases and revocation use the Redis clock. +- A stored thread at a missing node answers 409 (`GRAPH_NODE_MISSING_001`); state from a newer `10xgraph` answers 500 (`STATE_FORMAT_001`). +- Fixes: stop and shutdown now reach a REST stream after its first chunk, a disconnected stream frees its thread only after the run is saved, each realtime connection gets its own `LiveAgent`, and resuming a thread paused at `interrupt()` works on Postgres. + +**New and opt-in** +- Idempotency keys (`Idempotency-Key` header, or `idempotency_key` in the WebSocket run frame) so a gateway retry does not run tools twice. +- JWT key rotation (`JWT_SECRET_KEYS`) and a maximum token age (`JWT_MAX_TOKEN_AGE_SECONDS`). +- `10xgraph revoke` revokes one token or all of a user's tokens without a restart. +- Media limits (per-user files and storage, image pixels, PDF size and pages, inline media size), `GET /v1/files`, paging fields on lists, a run trace span tree, and `realtime_options` in graph details. +- `10xgraph eval` writes an HTML scorecard next to the JSON report and opens it (use `--no-open` to skip), prints live progress lines, and `--threshold` now exits 1 only when the pass rate is below it. +- `10xgraph init --template production` scaffolds JWT, ownership authorization, Redis rate limiting and a `PgCheckpointer`; `10xgraph build`, `10xgraph config` and `10xgraph audit` warn about missing auth, authorization or rate limiting. The `10xgraph config` editor is redesigned. + +**Security** +- Ownership checks on `file_id` media blocks for message and state writes, percent-encoded Redis keys, secrets redacted from logs and validation errors, and uploads authenticated before the body is parsed. + +## Upgrading from 0.9.x (changes since rc1 also apply) + +Everything below was the `1.0.0rc1` guidance and still applies when you jump from 0.9.x to rc2. Where rc2 changes an rc1 item, the rc2 section above wins. + +### Breaking changes from 0.9.x - **`runs.thread_ids` defaults to `"server"`** (was `"client"`). - REST and WebSocket clients that invent a thread id for a new thread now get 403 on `/v1/graph/invoke`, `/v1/graph/stream`, `/v1/graph/ws` and the checkpointer write routes. - Omit the id on the first request and use the `meta.thread_id` the server returns from then on (`"new"` over `/v1/graph/ws`), or set `"runs": {"thread_ids": "client"}` in `10xgraph.json`. - - AG-UI and `/v1/graph/live` need no change. Only the built-in ownership and rbac backends refuse. + - AG-UI and `/v1/graph/live` need no change in rc1, but rc2 enforces this setting there too. Only the built-in ownership and rbac backends refuse. - **Production deployments with the default or wildcard `ALLOWED_HOST` or `ORIGINS` refuse to start.** - Set explicit values, or `STRICT_PRODUCTION_CHECKS=false`. - A public, token-less API that runs in production with `ORIGINS=*` and `CORS_ALLOW_CREDENTIALS=false` also needs the opt-out. @@ -32,7 +77,7 @@ This pulls `10xgraph` 1.0.0rc1. - A `scope` / `scopes` claim that is not a string or a list of strings is rejected. Fix the issuer. - A string `scopes` is now split on spaces instead of matched by substring. -## New +### New - **`observability.user_id_mode`** in `10xgraph.json` (`"raw"` default, `"hash"`, `"mask"`) controls how the user id appears in traces. - **`on_busy` request option** (`"wait"` default, or `"cancel"`) on invoke, stream and `WS /v1/graph/ws`. With `"cancel"`, a run that finds its thread busy asks the running run to stop, then runs. @@ -42,7 +87,7 @@ This pulls `10xgraph` 1.0.0rc1. - **`interrupt_id` on invoke, stream and WebSocket input**, so a resume can name the interrupt it answers. WebSocket input also accepts `resume` again. - **`RequirePermission(..., explicit_scope=True)`** and the `checkpointer:import` scope. -## Security +### Security - Handled client errors log one WARNING line instead of an ERROR with a traceback, and Sentry no longer sends request bodies. Alerts that matched ERROR for 4xx must match WARNING instead. - AG-UI context, tools and forwarded props are server config, and `ag_ui` is a reserved key that other routes drop. @@ -55,7 +100,7 @@ This pulls `10xgraph` 1.0.0rc1. - Production hides raw error text on state, stream, WebSocket and AG-UI, and OTEL no longer prints spans to stdout. - Generated deployment files are locked down by default, and `10xgraph init` writes a `.gitignore` that keeps `.env` and credentials out of git. -## Fixed +### Fixed - Client mistakes on thread routes are a 404 or 422, not a 500, and state and message responses no longer carry provider or framework internals. - Development error responses for node and graph errors are no longer a 422 `VALIDATION_ERROR`. diff --git a/SECURITY.md b/SECURITY.md index 9d649043..2e56f013 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -7,7 +7,7 @@ | `1.0.x` (`10xgraph-api`) | Yes | | `< 1.0` | No | -`10xgraph-api` is at 1.0 (release candidate `1.0.0rc1`). Security fixes are applied to the +`10xgraph-api` is at 1.0 (release candidate `1.0.0rc2`). Security fixes are applied to the latest published release only; there are no long-term support branches. The old `10xscale-agentflow-cli` package is no longer published; upgrade to `10xgraph-api` to receive fixes. diff --git a/pyproject.toml b/pyproject.toml index e1e3934a..df8701c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "10xgraph-api" -version = "1.0.0rc1" +version = "1.0.0rc2" description = "Production server for 10xGraph agents (formerly 10xscale-agentflow-cli). Serves a compiled graph over REST, SSE, WebSocket, realtime audio and AG-UI with JWT or custom auth, scoped authorization on every endpoint, owner-only threads and rate limiting, and generates Docker Compose and Kubernetes files. Includes the 10xgraph CLI. MIT licensed, self-hosted." readme = "README.md" license = "MIT" @@ -60,7 +60,7 @@ classifiers = [ dependencies = [ # This release needs core 1.0 (import `tenxgraph`). The explicit rc in the # specifier lets pip and uv pick the core pre-release until 1.0.0 is out. - "10xgraph>=1.0.0rc1,<2.0", + "10xgraph>=1.0.0rc2,<2.0", # Pre-1.0 and major-version-risky deps are capped; floors are the versions # the suite is verified against. "fastapi>=0.116,<1.0", @@ -105,7 +105,7 @@ media = [ # Pillow for image resize / EXIF / format conversion in core's media processor. # Without it, uploaded images pass through unchanged. images = [ - "10xgraph[images]>=1.0.0rc1,<2.0", + "10xgraph[images]>=1.0.0rc2,<2.0", ] gcloud = [ "google-cloud-logging", @@ -373,7 +373,7 @@ ENVIRONMENT = "pytest" [dependency-groups] dev = [ - "10xgraph>=1.0.0rc1", + "10xgraph>=1.0.0rc2", "snowflakekit", "pytest>=8.4.2", "pytest-asyncio>=1.2.0", diff --git a/tenxgraph_api/cli/commands/eval.py b/tenxgraph_api/cli/commands/eval.py index ebc06a3b..cf71aa6f 100644 --- a/tenxgraph_api/cli/commands/eval.py +++ b/tenxgraph_api/cli/commands/eval.py @@ -820,7 +820,7 @@ def _exit_code(pass_rate: float, threshold: float | None) -> int: # Main entry point # ------------------------------------------------------------------ - def execute( # noqa: PLR0912, PLR0913, PLR0915 + def execute( # noqa: PLR0912, PLR0915 self, target: str | None = None, output_dir: str = "eval_reports", diff --git a/tenxgraph_api/cli/config_editor/schema.py b/tenxgraph_api/cli/config_editor/schema.py index e24e5dc0..ee2de8bf 100644 --- a/tenxgraph_api/cli/config_editor/schema.py +++ b/tenxgraph_api/cli/config_editor/schema.py @@ -413,7 +413,7 @@ "path": ["websocket", "realtime_models"], "label": "Realtime models clients may pick", "type": "list", - "placeholder": "gemini-2.5-flash-live", + "placeholder": "gemini-3.8-live", "help": ( "Models a /v1/graph/live client may request. Any other request uses the " "agent's own model. Empty means clients cannot choose." diff --git a/tenxgraph_api/cli/main.py b/tenxgraph_api/cli/main.py index 8403e09f..4cddc660 100644 --- a/tenxgraph_api/cli/main.py +++ b/tenxgraph_api/cli/main.py @@ -49,7 +49,7 @@ def execute(self, *args: Any, **kwargs: Any) -> int: hints=( "Run `10xgraph audit` to inspect installed package compatibility.", "Upgrade with `python -m pip install --upgrade " - '"10xgraph>=1.0.0rc1,<2"`.', + '"10xgraph>=1.0.0rc2,<2"`.', ), ) from exc implementation = getattr(module, class_name) diff --git a/tenxgraph_api/cli/templates/dev/graph/agent.py b/tenxgraph_api/cli/templates/dev/graph/agent.py index 46250a47..3b98610c 100644 --- a/tenxgraph_api/cli/templates/dev/graph/agent.py +++ b/tenxgraph_api/cli/templates/dev/graph/agent.py @@ -1,5 +1,6 @@ import logging from datetime import datetime +import zlib from tenxgraph.core.state import AgentState, StreamEmitter from tenxgraph.prebuilt.agent import ReactAgent @@ -14,8 +15,29 @@ logger = logging.getLogger(__name__) +_CITY_WEATHER = { + "london": "cloudy, 14C", + "new york": "partly cloudy, 18C", + "tokyo": "sunny, 24C", + "paris": "rainy, 12C", + "rome": "sunny, 27C", + "berlin": "overcast, 10C", + "sydney": "clear, 22C", + "cairo": "hot and dry, 35C", +} +_FALLBACK_WEATHER = ("sunny, 25C", "cloudy, 15C", "rainy, 11C", "windy, 17C", "foggy, 9C") + + +def _weather_for(location: str) -> str: + """Deterministic weather: known cities from a table, others from a stable hash.""" + key = location.strip().lower() + if key in _CITY_WEATHER: + return _CITY_WEATHER[key] + return _FALLBACK_WEATHER[zlib.crc32(key.encode("utf-8")) % len(_FALLBACK_WEATHER)] + + def call_weather_api(location: str) -> str: - return f"The weather in {location} is sunny" + return f"The weather in {location} is {_weather_for(location)}" def get_weather( diff --git a/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py b/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py index 4921323d..fffbad79 100644 --- a/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py +++ b/tenxgraph_api/cli/templates/prod/evals/weather_agents_eval.py @@ -52,21 +52,21 @@ def get_eval_set() -> EvalSet: query="What is the weather in London?", tool_name="get_weather", tool_args={"location": "London"}, - expected_response="The weather in London is sunny.", + expected_response="The weather in London is cloudy, 14C.", case_id="weather_london", ) .add_tool_test( query="Tell me the current weather in New York", tool_name="get_weather", tool_args={"location": "New York"}, - expected_response="The weather in New York is sunny.", + expected_response="The weather in New York is partly cloudy, 18C.", case_id="weather_new_york", ) .add_tool_test( query="How is the weather in Tokyo today?", tool_name="get_weather", tool_args={"location": "Tokyo"}, - expected_response="The weather in Tokyo is sunny.", + expected_response="The weather in Tokyo is sunny, 24C.", case_id="weather_tokyo", ) .build() diff --git a/tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py b/tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py index cd8b1322..209503b9 100644 --- a/tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py +++ b/tenxgraph_api/cli/templates/prod/graph/tools/weather_tool.py @@ -1,5 +1,6 @@ import logging import random +import zlib from tenxgraph.core.state.stream_emitter import StreamEmitter @@ -11,11 +12,32 @@ MAX_RETRIES = 3 +_CITY_WEATHER = { + "london": "cloudy, 14C", + "new york": "partly cloudy, 18C", + "tokyo": "sunny, 24C", + "paris": "rainy, 12C", + "rome": "sunny, 27C", + "berlin": "overcast, 10C", + "sydney": "clear, 22C", + "cairo": "hot and dry, 35C", +} +_FALLBACK_WEATHER = ("sunny, 25C", "cloudy, 15C", "rainy, 11C", "windy, 17C", "foggy, 9C") + + +def _weather_for(location: str) -> str: + """Deterministic weather: known cities from a table, others from a stable hash.""" + key = location.strip().lower() + if key in _CITY_WEATHER: + return _CITY_WEATHER[key] + return _FALLBACK_WEATHER[zlib.crc32(key.encode("utf-8")) % len(_FALLBACK_WEATHER)] + + def call_weather_api(location: str) -> str: is_failed = random.choice([True, False]) # noqa: S311 if is_failed: raise Exception("Failed to fetch weather data due to a simulated API error.") - return f"The weather in {location} is sunny" + return f"The weather in {location} is {_weather_for(location)}" def get_weather( diff --git a/tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py b/tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py index 5efbd5de..2ebd1544 100644 --- a/tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py +++ b/tenxgraph_api/cli/templates/prod/tests/test_catalog_tools.py @@ -45,3 +45,11 @@ def test_get_weather_accepts_injected_tool_call_id_and_state() -> None: result = get_weather(location="Berlin", tool_call_id="call_1", state=state) assert "Berlin" in result + + +def test_weather_is_deterministic_and_differs_between_cities() -> None: + from graph.tools.weather_tool import _weather_for + + assert _weather_for("Rome") == _weather_for("rome ") + assert _weather_for("Rome") != _weather_for("Berlin") + assert _weather_for("Atlantis") == _weather_for("Atlantis") diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md index c82164cc..842ccc48 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/prebuilt-agents-and-tools.md @@ -267,8 +267,8 @@ from tenxgraph.prebuilt.agent import AudioAgent from tenxgraph.core.realtime import RealtimeConfig, LiveInputQueue app = AudioAgent( - "gemini-live-2.5-flash-preview", - realtime_config=RealtimeConfig(model="gemini-live-2.5-flash-preview", voice="Puck"), + "gemini-3.8-live", + realtime_config=RealtimeConfig(model="gemini-3.8-live", voice="Puck"), tools=[my_tool], # React-style tool calling, including barge-in ).compile() diff --git a/tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md b/tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md index d3ba1d0e..194b0035 100644 --- a/tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md +++ b/tenxgraph_api/cli/templates/skills/10xgraph/references/realtime.md @@ -9,7 +9,7 @@ Live is implemented. ## Install and environment - `pip install "10xgraph[realtime]"` (pulls `google-genai>=1.56.0`). -- Set `GEMINI_API_KEY`; optionally `GEMINI_LIVE_MODEL` (default `gemini-live-2.5-flash-preview`). +- Set `GEMINI_API_KEY`; optionally `GEMINI_LIVE_MODEL` (default `gemini-3.8-live`). - Provider SDK imports are lazy: importing `tenxgraph.core.realtime` never requires the extra. ## Audio and media formats @@ -31,8 +31,8 @@ from tenxgraph.prebuilt.agent import AudioAgent from tenxgraph.core.realtime import RealtimeConfig app = AudioAgent( - "gemini-live-2.5-flash-preview", - realtime_config=RealtimeConfig(model="gemini-live-2.5-flash-preview", voice="Puck"), + "gemini-3.8-live", + realtime_config=RealtimeConfig(model="gemini-3.8-live", voice="Puck"), tools=[my_tool], # advertised to the model automatically; runs React-style ).compile() ``` diff --git a/tenxgraph_api/src/app/core/config/graph_config.py b/tenxgraph_api/src/app/core/config/graph_config.py index bb00c8df..eba5ebd2 100644 --- a/tenxgraph_api/src/app/core/config/graph_config.py +++ b/tenxgraph_api/src/app/core/config/graph_config.py @@ -307,7 +307,7 @@ class WebSocketConfig: "enabled": true, "max_connections": 1000, "max_connections_per_user": 10, - "realtime_models": ["gemini-2.5-flash-live"], + "realtime_models": ["gemini-3.8-live"], "realtime_client_overrides": ["voice", "modalities", "vad"] } diff --git a/tenxgraph_api/src/app/core/shutdown.py b/tenxgraph_api/src/app/core/shutdown.py index 4f3837fe..0c547f09 100644 --- a/tenxgraph_api/src/app/core/shutdown.py +++ b/tenxgraph_api/src/app/core/shutdown.py @@ -11,6 +11,7 @@ import asyncio import contextlib +import contextvars import logging from collections.abc import AsyncIterator, Callable @@ -116,8 +117,13 @@ async def end_on_shutdown( With ``keepalive_seconds``, a blank line is yielded each time the stream has been silent that long, so idle proxies keep the connection open. + + Each step of ``stream`` runs in a task of its own, all in one shared context: a context + variable the stream sets (the graph's current run, which carries its stop flag and + shutdown reason) stays set for every later step. Only one step runs at a time. """ loop = asyncio.get_running_loop() + context = contextvars.copy_context() waiter: asyncio.Future[None] = loop.create_future() if _interrupted: waiter.set_result(None) @@ -126,7 +132,7 @@ async def end_on_shutdown( try: while True: # Eager: a chunk that is ready without waiting costs no event-loop turn. - pending = asyncio.Task(_next(stream), loop=loop, eager_start=True) + pending = asyncio.Task(_next(stream), loop=loop, eager_start=True, context=context) while not pending.done() and not waiter.done(): await asyncio.wait( {pending, waiter}, diff --git a/tests/unit_tests/test_run_cleanup_order.py b/tests/unit_tests/test_run_cleanup_order.py index 26e6d9f6..0d83dae3 100644 --- a/tests/unit_tests/test_run_cleanup_order.py +++ b/tests/unit_tests/test_run_cleanup_order.py @@ -18,6 +18,7 @@ from injectq.core.context import ContainerContext from starlette.responses import StreamingResponse from tenxgraph.core.graph import StateGraph +from tenxgraph.core.graph.utils.run_registry import current_run from tenxgraph.core.state import AgentState, ExecutionStatus, Message from tenxgraph.storage.checkpointer import InMemoryCheckpointer from tenxgraph.storage.checkpointer.base_checkpointer import ACTIVE_RUN_NAMESPACE @@ -214,3 +215,75 @@ async def slow_cleanup(): release.set() await asyncio.wait({task}, timeout=1) assert task.cancelled() + + +# --- the run's context across stream chunks --------------------------------------------------- + + +class CacheDown(InMemoryCheckpointer): + """The stop key is unreachable: a stop reaches the run only through its in-process flag.""" + + async def arequest_stop(self, config): + raise ConnectionError("cache down") + + async def ais_stop_requested(self, config): + raise ConnectionError("cache down") + + +def build_chain(seen: dict, stop_in_b: bool = False): + """A -> B -> C; B records the run handle it sees and may stop the run.""" + holder: dict = {} + + async def a(state: AgentState, config: dict): + return [Message.text_message("a", role="assistant")] + + async def b(state: AgentState, config: dict): + seen["b"] = current_run.get() + if seen["b"] is not None: + # The task a hard cancel targets is the one running this step, not a finished one. + seen["b_task_done"] = seen["b"].task is None or seen["b"].task.done() + if stop_in_b: + await holder["app"].astop(config) + return [Message.text_message("b", role="assistant")] + + async def c(state: AgentState, config: dict): + seen["c"] = True + return [Message.text_message("c", role="assistant")] + + graph = StateGraph() + graph.add_node("A", a) + graph.add_node("B", b) + graph.add_node("C", c) + graph.set_entry_point("A") + graph.add_edge("A", "B") + graph.add_edge("B", "C") + graph.add_edge("C", END) + app = holder["app"] = graph.compile(checkpointer=CacheDown()) + return app + + +@pytest.mark.asyncio +async def test_end_on_shutdown_keeps_the_run_context_after_the_first_chunk(): + shutdown.reset() + seen: dict = {} + app = build_chain(seen) + + async for _ in shutdown.end_on_shutdown(lines(app), keepalive_seconds=15): + pass + + assert seen["b"] is not None # the run's handle, not lost with the first chunk's task + assert seen["b_task_done"] is False + await app.aclose() + + +@pytest.mark.asyncio +async def test_in_process_stop_reaches_a_rest_stream_after_the_first_chunk(): + shutdown.reset() + seen: dict = {} + app = build_chain(seen, stop_in_b=True) + + async for _ in shutdown.end_on_shutdown(lines(app), keepalive_seconds=15): + pass + + assert "c" not in seen # stopped after B + await app.aclose() From 35b53498ec7234684381ff0059063b03bef51c3a Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 15:58:23 +0600 Subject: [PATCH 14/16] feat(dependencies): add gunicorn and uvicorn-worker for production worker class and shutdown tests --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index df8701c1..6a372b78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -390,4 +390,8 @@ dev = [ "textxtract[pdf,docx,html,xml,md]>=0.2.0", # The ag-ui extra, so the AG-UI router tests run in CI instead of failing. "ag-ui-protocol>=1.0,<2", + # The production worker class (uvicorn_worker.py) and its shutdown tests import these; + # same ranges the generated Dockerfile installs. + "gunicorn>=23,<27", + "uvicorn-worker>=0.4,<1", ] From f51eae90d017f691e863f26cfbce53ea282cbd30 Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 16:03:10 +0600 Subject: [PATCH 15/16] Refactor code structure for improved readability and maintainability --- tests/unit_tests/test_config_editor.py | 9 +- uv.lock | 179 +++++++++++++++---------- 2 files changed, 111 insertions(+), 77 deletions(-) diff --git a/tests/unit_tests/test_config_editor.py b/tests/unit_tests/test_config_editor.py index ddf2a94d..76d0e9b4 100644 --- a/tests/unit_tests/test_config_editor.py +++ b/tests/unit_tests/test_config_editor.py @@ -3,6 +3,7 @@ from __future__ import annotations import http.client +import importlib.util import json import re from pathlib import Path @@ -89,9 +90,11 @@ def test_every_env_name_exists_in_the_source_tree() -> None: sources = [ p for p in root.rglob("*.py") if p.name != "schema.py" or "config_editor" not in str(p) ] - core = Path(__file__).resolve().parents[3] / "agentflow" / "tenxgraph" - if core.is_dir(): - sources += list(core.rglob("*.py")) + # Core env names live in the installed tenxgraph package (a sibling checkout only exists in + # the monorepo, not in this repo's CI). + core_spec = importlib.util.find_spec("tenxgraph") + for location in (core_spec.submodule_search_locations or []) if core_spec else []: + sources += list(Path(location).rglob("*.py")) text = "\n".join(p.read_text(encoding="utf-8", errors="ignore") for p in sources) missing = sorted({e["name"] for e in _env_entries() if e["name"] not in text}) assert not missing diff --git a/uv.lock b/uv.lock index 1750985e..1d95f40e 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ [[package]] name = "10xgraph" -version = "1.0.0rc1" +version = "1.0.0rc2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "injectq" }, @@ -18,9 +18,9 @@ dependencies = [ { name = "python-dotenv" }, { name = "pyyaml" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/79/481682c7bbb18a109f71fe1417d55685470a35f2b8b28130c4c906144a8a/10xgraph-1.0.0rc1.tar.gz", hash = "sha256:7c8ed7c56f163f25f692f428647a8b7102c4791609a19beddf17c93f8b89eb9e", size = 592179, upload-time = "2026-10-28T07:37:57.755Z" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/99/3c2e5c419d4630688996f1598ad026f8da1d2eee1ee635d43183deedd6ab/10xgraph-1.0.0rc2.tar.gz", hash = "sha256:948a1c0f8ebe18689f0cd1a9c6832e3bfe708f12f15c1f8bf20a3832bc9fc72b", size = 685744, upload-time = "2026-10-30T09:54:04.821Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/70/e4/e1dcd822ff4a97273f63d840b7e150359c7ef2c2fa84499b787a744193d8/10xgraph-1.0.0rc1-py3-none-any.whl", hash = "sha256:65ede5bec44abc9af46b19e7d43eb8bde83d6111f7f59279120cc8892d099615", size = 645183, upload-time = "2026-10-28T07:37:55.589Z" }, + { url = "https://files.pythonhosted.org/packages/5c/33/13b8713f1e254dd2e70ad7b0481535ed6f899e607eddaddb04b56d77c493/10xgraph-1.0.0rc2-py3-none-any.whl", hash = "sha256:bb5d713b9e97355b6281f346da446f694edb8cf2dd352f3653cf56f186193c2e", size = 742180, upload-time = "2026-10-30T09:54:01.963Z" }, ] [package.optional-dependencies] @@ -30,7 +30,7 @@ images = [ [[package]] name = "10xgraph-api" -version = "1.0.0rc1" +version = "1.0.0rc2" source = { editable = "." } dependencies = [ { name = "10xgraph" }, @@ -89,6 +89,7 @@ websocket = [ dev = [ { name = "10xgraph" }, { name = "ag-ui-protocol" }, + { name = "gunicorn" }, { name = "httpx" }, { name = "mypy" }, { name = "pre-commit" }, @@ -103,12 +104,13 @@ dev = [ { name = "ruff" }, { name = "snowflakekit" }, { name = "textxtract", extra = ["docx", "html", "md", "pdf", "xml"] }, + { name = "uvicorn-worker" }, ] [package.metadata] requires-dist = [ - { name = "10xgraph", specifier = ">=1.0.0rc1,<2.0" }, - { name = "10xgraph", extras = ["images"], marker = "extra == 'images'", specifier = ">=1.0.0rc1,<2.0" }, + { name = "10xgraph", specifier = ">=1.0.0rc2,<2.0" }, + { name = "10xgraph", extras = ["images"], marker = "extra == 'images'", specifier = ">=1.0.0rc2,<2.0" }, { name = "ag-ui-protocol", marker = "extra == 'ag-ui'", specifier = ">=1.0,<2" }, { name = "fastapi", specifier = ">=0.116,<1.0" }, { name = "firebase-admin", marker = "extra == 'firebase'", specifier = ">=6.5.0" }, @@ -138,8 +140,9 @@ provides-extras = ["sentry", "firebase", "snowflakekit", "redis", "jwt", "media" [package.metadata.requires-dev] dev = [ - { name = "10xgraph", specifier = ">=1.0.0rc1" }, + { name = "10xgraph", specifier = ">=1.0.0rc2" }, { name = "ag-ui-protocol", specifier = ">=1.0,<2" }, + { name = "gunicorn", specifier = ">=23,<27" }, { name = "httpx", specifier = "==0.27.0" }, { name = "mypy", specifier = ">=1.11.0" }, { name = "pre-commit", specifier = ">=3.8.0" }, @@ -154,6 +157,7 @@ dev = [ { name = "ruff", specifier = "==0.5.2" }, { name = "snowflakekit" }, { name = "textxtract", extras = ["pdf", "docx", "html", "xml", "md"], specifier = ">=0.2.0" }, + { name = "uvicorn-worker", specifier = ">=0.4,<1" }, ] [[package]] @@ -921,6 +925,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/46/5c/2f6c7e24b99dbaf5f8d7e5b1413fc9fc23360cdeb7f290b49a1c87b49560/grpcio_status-1.82.1-py3-none-any.whl", hash = "sha256:71c7f2bea725c0027fa396b77a55d4e9d90591bab90de4c1c03d4df9a56552f0", size = 14636, upload-time = "2026-07-08T12:39:23.113Z" }, ] +[[package]] +name = "gunicorn" +version = "26.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/8a/e4ef6ee11701b6cd64702848415ffb69eeff85cb388a3c6c7fe86f22f3f8/gunicorn-26.2.0.tar.gz", hash = "sha256:62b864895d9ebff0b2f9867ba04fe811c93121596540830c9c916d0769668447", size = 787921, upload-time = "2026-08-24T15:05:59.3Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/85/7522a52e5e2f42faf1a129113ab63e548c42e103e9af395b7bfe65e403e2/gunicorn-26.2.0-py3-none-any.whl", hash = "sha256:bd249d0b3f7972f7432f0a6b6ff3b3ee2d129f70cd1ff6c09a9dd9e29a2b88e3", size = 228389, upload-time = "2026-08-24T15:05:57.67Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -1533,73 +1546,78 @@ wheels = [ [[package]] name = "pillow" -version = "12.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/bf/fb3ebff8ddcb76aac5a01389251bbbb9519922a9b520d8247c1ca864a25d/pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965", size = 5345969, upload-time = "2026-07-01T11:54:06.397Z" }, - { url = "https://files.pythonhosted.org/packages/d8/66/9a386a92561f402389a4fc70c18838bf6d35eb5eb5c6850b4b2dc64f5048/pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7", size = 4780323, upload-time = "2026-07-01T11:54:09.351Z" }, - { url = "https://files.pythonhosted.org/packages/25/27/ac8f99618ffd3dde21db0f4d4b1d2ab00c0880595bfd17df103f7f39fd0c/pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9", size = 6266838, upload-time = "2026-07-01T11:54:11.71Z" }, - { url = "https://files.pythonhosted.org/packages/84/21/a35af28dcc61f37ed850a2d64c65c701321dfbf25085e469d5559360cbbf/pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91", size = 6940830, upload-time = "2026-07-01T11:54:13.732Z" }, - { url = "https://files.pythonhosted.org/packages/eb/51/8b08617af3ad95e33ce6d7dd2c99ed6c8298f7fb131636303956be022e25/pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c", size = 6344383, upload-time = "2026-07-01T11:54:15.756Z" }, - { url = "https://files.pythonhosted.org/packages/1d/72/cf78ac9780bb93c28328f408973845a309d4d145041665f734572ced1b52/pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df", size = 7052934, upload-time = "2026-07-01T11:54:17.721Z" }, - { url = "https://files.pythonhosted.org/packages/20/20/25e0f4dc178a6bc0696793720055519a0de89e7661dae886992decbd2f81/pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f", size = 6472684, upload-time = "2026-07-01T11:54:19.839Z" }, - { url = "https://files.pythonhosted.org/packages/45/89/da2f7971a317f83d807fdd4065c0af40208e59e692cc43d315a71a0e96d1/pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09", size = 7227137, upload-time = "2026-07-01T11:54:22.025Z" }, - { url = "https://files.pythonhosted.org/packages/de/47/4845a0a6c0dbf1db8456bd9fc791f13c5ced7ced20606d08a0aacfd25b49/pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510", size = 2568267, upload-time = "2026-07-01T11:54:24.051Z" }, - { url = "https://files.pythonhosted.org/packages/9d/ac/31fb64e1e7efb5a4b50cd3d92049ba89ac6e4d8d3bb6a74e15048ca3353e/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89", size = 4161684, upload-time = "2026-07-01T11:54:25.934Z" }, - { url = "https://files.pythonhosted.org/packages/87/b4/9805e23d2b4d77842b468513841fda254ee42f0289d25088340e4ff46e2d/pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace", size = 4255487, upload-time = "2026-07-01T11:54:27.935Z" }, - { url = "https://files.pythonhosted.org/packages/df/39/ecf519435a200c693fe053a6ee4d835b41cf963a4dfc2551c4e637cb2a71/pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec", size = 3696433, upload-time = "2026-07-01T11:54:29.813Z" }, - { url = "https://files.pythonhosted.org/packages/42/92/2fc3ffad878ae8dd5469ec1bc8eb83b71f48e13efdf68f02709003982a32/pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66", size = 5345889, upload-time = "2026-07-01T11:54:31.97Z" }, - { url = "https://files.pythonhosted.org/packages/10/76/8803c13605b763d33d156c4678fc77f8443389c0c51c8aef707bb02015f4/pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35", size = 4780109, upload-time = "2026-07-01T11:54:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/e18aff37cb0b4aac47ac90f016d347a49aca667ef97f190b06ac2aabc928/pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65", size = 6263736, upload-time = "2026-07-01T11:54:36.131Z" }, - { url = "https://files.pythonhosted.org/packages/f7/62/de5bdd77d935331f4f802edc11e4d82950f642caad6cb2f949837b8560e2/pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3", size = 6937129, upload-time = "2026-07-01T11:54:38.216Z" }, - { url = "https://files.pythonhosted.org/packages/70/4d/105627a13300c5e0df1d174230b32fd1273062c96f7745fd552b945d1e1d/pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a", size = 6339562, upload-time = "2026-07-01T11:54:40.354Z" }, - { url = "https://files.pythonhosted.org/packages/6b/1d/f13de01a553988ab895ba1c722e06cf3144d4f57656fd5b81b6d881f1179/pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e", size = 7049439, upload-time = "2026-07-01T11:54:42.489Z" }, - { url = "https://files.pythonhosted.org/packages/c9/f9/066794cca041b969964f779ee5fa66a9498bbf34248ac39c5d7954e4198f/pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f", size = 6473287, upload-time = "2026-07-01T11:54:44.9Z" }, - { url = "https://files.pythonhosted.org/packages/a6/9b/7a58e61d62be561da3a356fe2384d4059a6345fc130e23ef1c36a5b81d24/pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8", size = 7239691, upload-time = "2026-07-01T11:54:47.141Z" }, - { url = "https://files.pythonhosted.org/packages/aa/b0/c4ed4f0ef8f8fa5ee8351537db6650bb8189f7e118842978dd6589065692/pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b", size = 2568185, upload-time = "2026-07-01T11:54:49.137Z" }, - { url = "https://files.pythonhosted.org/packages/dc/01/001f65b68192f0228cc1dbbc8d2530ab5d58b61037ba0587f946fea607cd/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330", size = 4161736, upload-time = "2026-07-01T11:54:51.156Z" }, - { url = "https://files.pythonhosted.org/packages/1a/d2/0219746d0fd16fc8a84498e79452375be3797d3ce4044596ce565164b84f/pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217", size = 4255435, upload-time = "2026-07-01T11:54:53.414Z" }, - { url = "https://files.pythonhosted.org/packages/c8/02/8d0bc62ef0302318c46ff2a512822d2610e81c7aa46c9b3abe6cbaca5ad0/pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930", size = 3696262, upload-time = "2026-07-01T11:54:55.739Z" }, - { url = "https://files.pythonhosted.org/packages/85/e2/73c77d218410b14f5f2d565e8a998d5317b7b9c75368d29985139f7a46f0/pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8", size = 5350344, upload-time = "2026-07-01T11:54:57.657Z" }, - { url = "https://files.pythonhosted.org/packages/c7/da/32c752228ae345f489e3a42499d817b6c3996da7e8a3bc7a04fc806b243b/pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0", size = 4780131, upload-time = "2026-07-01T11:54:59.713Z" }, - { url = "https://files.pythonhosted.org/packages/b1/9d/8b2c807dbef61a5197c047afe99823787eb66f63daf9fb2432f91d6f0462/pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321", size = 6263757, upload-time = "2026-07-01T11:55:01.778Z" }, - { url = "https://files.pythonhosted.org/packages/5c/44/c85361f65dbe00eea8576ee467c768d25129989efb76e94f205e9ca9bb46/pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b", size = 6936962, upload-time = "2026-07-01T11:55:03.93Z" }, - { url = "https://files.pythonhosted.org/packages/18/7e/e483414b35800b86b6f08dbbc7803fb5cd52c4d6f897f47d53ea2c7e6f65/pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198", size = 6339171, upload-time = "2026-07-01T11:55:05.989Z" }, - { url = "https://files.pythonhosted.org/packages/f0/f4/68c491844841ede6bed70189546b3ee9731cf9f2cbad396faff5e1ccba45/pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130", size = 7048116, upload-time = "2026-07-01T11:55:08.131Z" }, - { url = "https://files.pythonhosted.org/packages/a3/34/77f3f793fed8efc7d243f21b33c5a3f0d1c97ee70346d3db855587e155ff/pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a", size = 6467209, upload-time = "2026-07-01T11:55:10.408Z" }, - { url = "https://files.pythonhosted.org/packages/f1/e0/492879f69d94f91f60fc8cd05ba03650e9520afebb2fb7aa12777d7c7f38/pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d", size = 7237707, upload-time = "2026-07-01T11:55:12.745Z" }, - { url = "https://files.pythonhosted.org/packages/c9/ac/6b11f2875f1c2ac040d84e1bbf9cf22a88038f901ca1037898b280b38365/pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838", size = 2565995, upload-time = "2026-07-01T11:55:14.736Z" }, - { url = "https://files.pythonhosted.org/packages/52/69/c2208e56af9bfc1913afb24020297a691eb1d4ef688474c8a04913f65e04/pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e", size = 5352503, upload-time = "2026-07-01T11:55:17.076Z" }, - { url = "https://files.pythonhosted.org/packages/07/70/e5686d753e898a45d778ff1718dba8516ead6ab6b95d85fc8c4b70650cf2/pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17", size = 4782956, upload-time = "2026-07-01T11:55:19.448Z" }, - { url = "https://files.pythonhosted.org/packages/d5/37/25c6692f06927ee973ff18c8d9ee98ad0b4d84ee67a09610c2dd1447958e/pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385", size = 6322855, upload-time = "2026-07-01T11:55:21.613Z" }, - { url = "https://files.pythonhosted.org/packages/cc/91/420637fcb8f1bc11029e403b4538e6694744428d8246118e45719f944556/pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c", size = 6989642, upload-time = "2026-07-01T11:55:24.006Z" }, - { url = "https://files.pythonhosted.org/packages/10/08/b94d7811281ccf0d143a1cf768d1c49e1e54af63e7b708ab2ee3eb87face/pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d", size = 6391281, upload-time = "2026-07-01T11:55:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/d2/87/24233f785f55474dc02ce3e739c5528a77e3a862e9333d1dd7a25cc31f70/pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931", size = 7096716, upload-time = "2026-07-01T11:55:28.318Z" }, - { url = "https://files.pythonhosted.org/packages/23/26/fcb2f6e37175b04f53570b59937867e2b80ee1685e744023153028fc14f9/pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7", size = 6474125, upload-time = "2026-07-01T11:55:30.956Z" }, - { url = "https://files.pythonhosted.org/packages/90/de/3634abee5f1c9e13c56787b7d5517b0ba8d6de51700b95578cf338349c9f/pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c", size = 7242939, upload-time = "2026-07-01T11:55:34.044Z" }, - { url = "https://files.pythonhosted.org/packages/ce/2a/fd13f8eb24de5714a6eb444a3d67e2842c6c576e159a43793adf23051351/pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45", size = 2567506, upload-time = "2026-07-01T11:55:35.988Z" }, - { url = "https://files.pythonhosted.org/packages/5d/dc/8fdce34ec725a33c81c6ba122b904d6b9024e50ea9ac7bede62fab54506c/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139", size = 4162063, upload-time = "2026-07-01T11:55:37.941Z" }, - { url = "https://files.pythonhosted.org/packages/76/66/2044b9a63d3b84ff048228dfcb7cd9bf0df983e8470971bf7d4c57b693de/pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402", size = 4255549, upload-time = "2026-07-01T11:55:40.022Z" }, - { url = "https://files.pythonhosted.org/packages/52/7e/1f67e6f4ece6b582ee4b539decbcc9f848dc245a93ed8cd7338bafef72f1/pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c", size = 3696331, upload-time = "2026-07-01T11:55:41.98Z" }, - { url = "https://files.pythonhosted.org/packages/12/40/d306fc2c8e4d45d7f175c77edca7063be7b86fe7fe6e68f4353bf71d808c/pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f", size = 5350370, upload-time = "2026-07-01T11:55:44.028Z" }, - { url = "https://files.pythonhosted.org/packages/dd/44/668fb1437e8ce420f62d6106eb66e44a5971602a4d794615bdf79315d82d/pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701", size = 4780147, upload-time = "2026-07-01T11:55:46.073Z" }, - { url = "https://files.pythonhosted.org/packages/0c/08/93fa2e70e30a2d81547e481b6ee2bb9522117221fb1e0ce4b5df70967677/pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace", size = 6273659, upload-time = "2026-07-01T11:55:48.264Z" }, - { url = "https://files.pythonhosted.org/packages/f8/6d/043e96ff814fc31a33077e4cba86082167db520c93632afdf2042febbb0c/pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4", size = 6947439, upload-time = "2026-07-01T11:55:50.503Z" }, - { url = "https://files.pythonhosted.org/packages/af/92/ba71d2ee2ac0edf3fa33bd9d5ee9ee080da70b1766f3ca3934f9938ddac9/pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39", size = 6353577, upload-time = "2026-07-01T11:55:52.697Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ce/e63064e2122923ff687c8ad792d0d736a7b3920a56a46982e81a7fdd25d6/pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71", size = 7060394, upload-time = "2026-07-01T11:55:55.149Z" }, - { url = "https://files.pythonhosted.org/packages/54/76/a09cc3ccc8d773a7283d34c38bec1708f9e3cc932093cbc4c5e71ac4060b/pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827", size = 6467375, upload-time = "2026-07-01T11:55:57.769Z" }, - { url = "https://files.pythonhosted.org/packages/3e/03/1846c49ba3b1d5550392a4bbd06d6fb4578e1cd91a803198b5c90f5f7d53/pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5", size = 7237048, upload-time = "2026-07-01T11:55:59.975Z" }, - { url = "https://files.pythonhosted.org/packages/fb/bb/89f35dcc79610423f9f195504d7def7f0d1416a711541b42867e25fe3412/pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658", size = 2566006, upload-time = "2026-07-01T11:56:02.143Z" }, - { url = "https://files.pythonhosted.org/packages/30/88/707027ba09942dfa2c28759b5c222d769290a41c6d20ea60ec250801941f/pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf", size = 5352509, upload-time = "2026-07-01T11:56:04.2Z" }, - { url = "https://files.pythonhosted.org/packages/b0/6d/00352fa25332c2569cd387851f568cc5a4b75a9adbfb37ac4fbce4c02eec/pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64", size = 4783167, upload-time = "2026-07-01T11:56:06.631Z" }, - { url = "https://files.pythonhosted.org/packages/13/4f/9e049dfa21af7c22427275720e2490267ba8138120add5c4c574deb69782/pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e", size = 6329237, upload-time = "2026-07-01T11:56:08.868Z" }, - { url = "https://files.pythonhosted.org/packages/36/16/cf6eeaae8d0fce8dd390a33437cf68c5d5bd73834a2bc6e2f14efda0ab45/pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777", size = 6997047, upload-time = "2026-07-01T11:56:11.379Z" }, - { url = "https://files.pythonhosted.org/packages/1e/69/dbf769bdd55f48bf5733cac28edc6364ffaa072ec9ba336266e4fe66be55/pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1", size = 6400440, upload-time = "2026-07-01T11:56:13.908Z" }, - { url = "https://files.pythonhosted.org/packages/a0/e1/ffc9cfc2eea0d178da8018e18e959301ad9d6bc9f3edb7181e748a474b97/pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9", size = 7105895, upload-time = "2026-07-01T11:56:16.575Z" }, - { url = "https://files.pythonhosted.org/packages/18/f0/a5595c1e8c3ae44b9828cb2f0fa8155e5095ef04d6327b8f61cf44a3df85/pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8", size = 6474384, upload-time = "2026-07-01T11:56:18.855Z" }, - { url = "https://files.pythonhosted.org/packages/e4/04/62bcd9f844984c5938d3b05264a61d797a29d3e0812341a8204af70bbdee/pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418", size = 7243537, upload-time = "2026-07-01T11:56:21.214Z" }, - { url = "https://files.pythonhosted.org/packages/3d/68/1f3066acedf37673694a7141381d8f811ae97f30d34413d236abe7d489f1/pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59", size = 2567491, upload-time = "2026-07-01T11:56:23.506Z" }, +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/6e/eb/e86a41962a3a31ad495a3d6a912d9002b01bba4a6fa71c28703abdca7f69/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7d5e097eabd5d4915c27ac9f6e9325bb1e01a7c94d7348b5c4451d3fc30e8521", size = 4191213, upload-time = "2026-10-16T22:17:09.449Z" }, + { url = "https://files.pythonhosted.org/packages/dd/04/7e7445e706afec84ef06dc25b2c77f9cecff242c23730c62928ad1089d4a/pillow-13.0.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:51ebb529fc5e8abdd55302f6c3f8943037348ec2384329a3e55677f69ae2fdfc", size = 4290650, upload-time = "2026-10-16T22:17:11.196Z" }, + { url = "https://files.pythonhosted.org/packages/46/85/d534089068ac40c0f61f650389a296c812eab745a8a733f81feec39ae955/pillow-13.0.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:c87a04e7f152ee4db5dbe15f0bfa3fe628a1aeedbe66b81c43b87f52c813f9ac", size = 3723078, upload-time = "2026-10-16T22:17:12.772Z" }, + { url = "https://files.pythonhosted.org/packages/ce/2b/00b5c3b401df538b5dbe311d3899730fae5ba1e21cb8b80a128887ac6747/pillow-13.0.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:20722cd5375533ad754053363a80b8fb30c79d37930a6f7228030e237ce3bd5d", size = 5572292, upload-time = "2026-10-16T22:17:14.539Z" }, + { url = "https://files.pythonhosted.org/packages/72/f5/a6b0610f946e9b54ae888ba7ad29d6e2f8d9b11f69e212d70416b7186c09/pillow-13.0.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:04a63720f79e22fd54d897c233a68314c50ca0603360f54ce2bd5db8f0f43782", size = 4852222, upload-time = "2026-10-16T22:17:16.521Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d0/c0ff25bb0427ec70f32aac995453321fc3c4494a46c7292ff02d9d260b04/pillow-13.0.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4f8db54177035a2b5f717f94475b74907e12a96dc74f6d487f3966b0a4694071", size = 6527349, upload-time = "2026-10-16T22:17:18.534Z" }, + { url = "https://files.pythonhosted.org/packages/85/11/e46fa0bcc0fc117f1eacc206b3d51f04d583fab328b4b740672d0438d83f/pillow-13.0.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8f36aa77f74b14a2103622b196402c71330e4e7cf4122ee45c854e9a01e0eecf", size = 7152873, upload-time = "2026-10-16T22:17:20.415Z" }, + { url = "https://files.pythonhosted.org/packages/7c/53/2bdeca705f3a58679fa9201233193c86a30238340ca29b3a152c3fcc719b/pillow-13.0.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:149a1c323ba366f90b48e6f1a6db6582eb50512df374588de56fc909abc0f0f3", size = 6626871, upload-time = "2026-10-16T22:17:22.361Z" }, + { url = "https://files.pythonhosted.org/packages/14/de/3829030f209722ade3753bb456b805a0e7f885cf885b483407365c82e9ca/pillow-13.0.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e892354a424b1e82a78ba16785822bdd40de06f19446a3d4a28047df1a4f8932", size = 7236439, upload-time = "2026-10-16T22:17:24.352Z" }, + { url = "https://files.pythonhosted.org/packages/44/b1/67fc37388382bc036a74f04da5bb6fcbf90ec3ac1f45893e06176c373fd3/pillow-13.0.0-cp315-cp315-win32.whl", hash = "sha256:8e459b613cf31f8980f584387b4fe47bf08e076e288d61aa8eba1f7fd4ddbb81", size = 6508688, upload-time = "2026-10-16T22:17:26.314Z" }, + { url = "https://files.pythonhosted.org/packages/44/05/fdb6a79aee10861165526b41b7b440e36ee1ff58613b2fbefe3acbf06d4f/pillow-13.0.0-cp315-cp315-win_amd64.whl", hash = "sha256:d4737f50d23ddd11d6685faacbb9252887dde0bc7997127757247983d74399d6", size = 7281107, upload-time = "2026-10-16T22:17:28.198Z" }, + { url = "https://files.pythonhosted.org/packages/e6/9c/6c6f7dc2a566467baf121394e83de25ac287cffe34cc030726beb4be0224/pillow-13.0.0-cp315-cp315-win_arm64.whl", hash = "sha256:2310e2adfc13c6d457d7d0f2ffdd5c56a74a317650cf765887df8d515a647ac0", size = 2695513, upload-time = "2026-10-16T22:17:30.013Z" }, + { url = "https://files.pythonhosted.org/packages/23/16/e9120defdb8bff693b582d035312acc6f530cdc2bc5238a87f410efccd82/pillow-13.0.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:21c32061a5512dc13f87cc7303e120e4d77c2f70df89a35af0e3ce246e5ac57f", size = 5578129, upload-time = "2026-10-16T22:17:31.642Z" }, + { url = "https://files.pythonhosted.org/packages/47/22/76faf2bb2448f48a84cf894c1e2bb1090d7d2b1aa521ece131cb2f05d631/pillow-13.0.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:1c774b9c0ba0d0b356f71530677b39028878a2cae5a93e0a13779cfd17120f88", size = 4857428, upload-time = "2026-10-16T22:17:33.43Z" }, + { url = "https://files.pythonhosted.org/packages/b3/64/0861ea4e03c50b14af9ed9dd8b9f388ab42f445c623a9fcbab19aef7aa55/pillow-13.0.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:20822140c9a2ba9c5e42df342b0a892ab2e102fc0e3db967a133bb9ce3f898bb", size = 6612582, upload-time = "2026-10-16T22:17:35.415Z" }, + { url = "https://files.pythonhosted.org/packages/5c/20/cd1e9b3f60a9f3d5ebe46da36955c26004092a501f7f2c42f9fcc26611bc/pillow-13.0.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ffea6b2986a797c84b6d1d27d4f43e0b83c73fc16281a930891cd237cf0d08fe", size = 7230176, upload-time = "2026-10-16T22:17:37.362Z" }, + { url = "https://files.pythonhosted.org/packages/b6/ef/ffb0a2fc85690714fb7ac662956949645d57f15bf4426f520da361e7b6e3/pillow-13.0.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:05cd2cd39055d0b9650060e4513ef4a9a0aea870083950bd046a0d6490f93a22", size = 6703640, upload-time = "2026-10-16T22:17:39.313Z" }, + { url = "https://files.pythonhosted.org/packages/e0/c2/6402ee3ccb3363aeed8fdb3f9fe56703ac5377a539f0b81b53999d7e121f/pillow-13.0.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c6fd031333dedcad7ce3e03d244093f43504f209e86925ebdb2b868fa4037630", size = 7309231, upload-time = "2026-10-16T22:17:41.021Z" }, + { url = "https://files.pythonhosted.org/packages/fc/96/d6b82e8b8c06ad143b66b12e0b31051e4bf489b4fabca1f390a14fe1ebde/pillow-13.0.0-cp315-cp315t-win32.whl", hash = "sha256:c50c80e3591e5cf9a8dfa6007d4f10732f02e32263850999286e56ccd650cfb2", size = 6517200, upload-time = "2026-10-16T22:17:42.886Z" }, + { url = "https://files.pythonhosted.org/packages/60/d4/0062e6005e9b270b165015aafd343364ac378ec5a685431ea42baad33357/pillow-13.0.0-cp315-cp315t-win_amd64.whl", hash = "sha256:adb3c2124c68e480f4632d06cb7caa3b48a51678475f483b4edb1c6dee29ea43", size = 7287488, upload-time = "2026-10-16T22:17:45.56Z" }, + { url = "https://files.pythonhosted.org/packages/e4/81/b8eb349746be7bce47dd95337c6634e84910af91b59aaa9159c034c614fa/pillow-13.0.0-cp315-cp315t-win_arm64.whl", hash = "sha256:4a25756244bdcce3b50580baadf320fa48297f0eeb8e114ba64139a40d4d0fc8", size = 2698073, upload-time = "2026-10-16T22:17:47.556Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, ] [[package]] @@ -2268,6 +2286,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/ec/dbb7e5a6b91f86bfb9eb7d2988a2730907b6a729875b949c7f022e8b88fa/uvicorn-0.51.0-py3-none-any.whl", hash = "sha256:5d38af6cd620f2ae3849fb44fd4879e0890aa1febe8d47eb355fb45d93fe6a5b", size = 73219, upload-time = "2026-07-08T10:59:04.44Z" }, ] +[[package]] +name = "uvicorn-worker" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gunicorn" }, + { name = "uvicorn" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/59/9101b9c0680fd80e9d26c07deb822a5d18a324339fcf9cd017885ee808ad/uvicorn_worker-0.4.0.tar.gz", hash = "sha256:8ee5306070d8f38dce124adce488c3c0b50f20cf0c0222b12c66188da7214493", size = 9361, upload-time = "2025-09-20T10:47:01.218Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/25/09cd7a90c8bb7fb693be0d6704fccd5f9778d5513214b7a01cc4a94ff314/uvicorn_worker-0.4.0-py3-none-any.whl", hash = "sha256:e2ed952cef976f5e9e429d7269640bbcafbd36c80aa80f1003c8c77a6797abde", size = 5364, upload-time = "2025-09-20T10:46:59.776Z" }, +] + [[package]] name = "virtualenv" version = "21.6.1" From 79c144e8dd7ee88077f2bfb951ee90877449bdfd Mon Sep 17 00:00:00 2001 From: Shudipto Trafder Date: Fri, 30 Oct 2026 16:07:58 +0600 Subject: [PATCH 16/16] feat: enhance type hinting and improve code clarity across multiple modules --- pyproject.toml | 1 + tenxgraph_api/cli/commands/init.py | 4 +++- tenxgraph_api/src/app/core/shutdown.py | 4 +++- .../src/app/routers/media/__init__.py | 10 +++++--- tenxgraph_api/src/app/utils/run_trace.py | 24 +++++++++---------- .../src/app/utils/snowflake_id_generator.py | 5 ++++ 6 files changed, 31 insertions(+), 17 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6a372b78..71d8f48b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -252,6 +252,7 @@ docstring-code-format = true [tool.mypy] python_version = "3.12" +plugins = ["pydantic.mypy"] files = ["tenxgraph_api"] # Templates are emitted code, not library code - they reference modules that only # exist in a scaffolded project. diff --git a/tenxgraph_api/cli/commands/init.py b/tenxgraph_api/cli/commands/init.py index a4925e1b..d148c7d2 100644 --- a/tenxgraph_api/cli/commands/init.py +++ b/tenxgraph_api/cli/commands/init.py @@ -537,7 +537,9 @@ def _print_summary(self, context: dict) -> None: if rl == "none": rows.append(("Rate limit", "None")) else: - rl_by = {"global": "Global", "user": "Per user"}.get(context.get("rl_by"), "Per IP") + rl_by = {"global": "Global", "user": "Per user"}.get( + str(context.get("rl_by")), "Per IP" + ) proxy = " · proxy headers on" if context.get("rl_trusted_proxy") else "" rows.append( ( diff --git a/tenxgraph_api/src/app/core/shutdown.py b/tenxgraph_api/src/app/core/shutdown.py index 0c547f09..48f9be50 100644 --- a/tenxgraph_api/src/app/core/shutdown.py +++ b/tenxgraph_api/src/app/core/shutdown.py @@ -14,6 +14,7 @@ import contextvars import logging from collections.abc import AsyncIterator, Callable +from typing import Any import anyio from tenxgraph.core.state import StreamChunk, StreamEvent @@ -134,8 +135,9 @@ async def end_on_shutdown( # Eager: a chunk that is ready without waiting costs no event-loop turn. pending = asyncio.Task(_next(stream), loop=loop, eager_start=True, context=context) while not pending.done() and not waiter.done(): + watched: set[asyncio.Future[Any]] = {pending, waiter} await asyncio.wait( - {pending, waiter}, + watched, timeout=keepalive_seconds, return_when=asyncio.FIRST_COMPLETED, ) diff --git a/tenxgraph_api/src/app/routers/media/__init__.py b/tenxgraph_api/src/app/routers/media/__init__.py index 571d3764..1a750244 100644 --- a/tenxgraph_api/src/app/routers/media/__init__.py +++ b/tenxgraph_api/src/app/routers/media/__init__.py @@ -9,7 +9,7 @@ from collections import OrderedDict from collections.abc import Callable from pathlib import Path -from typing import Any +from typing import Any, cast from injectq import InjectQ, inject, singleton from tenxgraph.storage.checkpointer import BaseCheckpointer @@ -454,7 +454,10 @@ async def offload_inline(self, messages: list[Any], user_id: str | None = None) try: for message in messages: await ensure_media_offloaded( - message, store, MediaOffloadPolicy.THRESHOLD, max_inline_bytes=threshold + message, + cast("BaseMediaStore", store), + MediaOffloadPolicy.THRESHOLD, + max_inline_bytes=threshold, ) except ValueError as exc: # bad base64 (binascii.Error) or MediaTooLargeError raise ClientInputError(f"Inline media rejected: {exc}") from exc @@ -619,7 +622,8 @@ async def release_thread(self, thread_id: str, user_id: str | None) -> list[str] continue threads = metadata.get(THREADS_KEY) if metadata else None if ( - isinstance(threads, list) + metadata is not None + and isinstance(threads, list) and thread_id in threads and str(metadata.get("owner_id")) == str(user_id) ): diff --git a/tenxgraph_api/src/app/utils/run_trace.py b/tenxgraph_api/src/app/utils/run_trace.py index caeb56c4..db0ad678 100644 --- a/tenxgraph_api/src/app/utils/run_trace.py +++ b/tenxgraph_api/src/app/utils/run_trace.py @@ -135,7 +135,7 @@ def telemetry_record(chunk: Any, ts: float | None = None) -> dict[str, Any]: elif btype == "error" and _get(block, "message"): block_errors.append(str(_get(block, "message"))) - event = getattr(chunk, "event", None) + event: Any = getattr(chunk, "event", None) event_name = str(event.value) if hasattr(event, "value") else str(event or "") is_error = event_name == "error" or bool(getattr(chunk, "is_error", False)) error = None @@ -364,12 +364,12 @@ def parent_id() -> str: # Tool results close the tool span (create one for runs without tool updates). for res in rec.get("tool_results") or []: cid = res.get("call_id") - tspan = tool_spans.get(cid) if cid else None - if tspan is None and cid: + result_span = tool_spans.get(cid) if cid else None + if result_span is None and cid: call = pending_calls.get(cid) or {} name = call.get("name") or rec.get("tool_name") or "tool" start = call.get("ts", ts) - tspan = ObsSpanSchema( + result_span = ObsSpanSchema( id=new_span_id(), name=f"tool: {name}", kind="tool", @@ -380,16 +380,16 @@ def parent_id() -> str: tool_name=name, tool_call_id=cid, ) - tool_spans[cid] = tspan - spans.append(tspan) - if tspan is not None and tspan.outputs is None: - tspan.duration_ms = max(0.0, off_ms(ts) - tspan.start_ms) - tspan.outputs = {"result": res.get("output")} + tool_spans[cid] = result_span + spans.append(result_span) + if result_span is not None and result_span.outputs is None: + result_span.duration_ms = max(0.0, off_ms(ts) - result_span.start_ms) + result_span.outputs = {"result": res.get("output")} if res.get("is_error"): - tspan.status = "error" - tspan.error = tspan.error or clip(res.get("output")) or "tool error" + result_span.status = "error" + result_span.error = result_span.error or clip(res.get("output")) or "tool error" else: - tspan.status = "ok" + result_span.status = "ok" if cur is not None: cur.boundary = max(cur.boundary, ts) diff --git a/tenxgraph_api/src/app/utils/snowflake_id_generator.py b/tenxgraph_api/src/app/utils/snowflake_id_generator.py index a4599555..ecb201ea 100644 --- a/tenxgraph_api/src/app/utils/snowflake_id_generator.py +++ b/tenxgraph_api/src/app/utils/snowflake_id_generator.py @@ -3,6 +3,7 @@ import socket import zlib from importlib.util import find_spec +from typing import overload from tenxgraph.utils.id_generator import BaseIDGenerator, IDType @@ -13,6 +14,10 @@ logger = logging.getLogger("tenxgraph_api.snowflake") +@overload +def _setting(value: int | None, env_name: str, default: int) -> int: ... +@overload +def _setting(value: int | None, env_name: str, default: None) -> int | None: ... def _setting(value: int | None, env_name: str, default: int | None) -> int | None: """``value`` when given, else the environment variable, else ``default``.""" if value is not None: