From 4152a9d8584d148af304e8bb68dd8c76247ac6b8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 15:40:22 -0500 Subject: [PATCH 001/204] chore: open 0.14.0-alpha.1 on develop --- plugins/dw/.claude-plugin/plugin.json | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/dw/.claude-plugin/plugin.json b/plugins/dw/.claude-plugin/plugin.json index 9e4b6aa5..18c809cd 100644 --- a/plugins/dw/.claude-plugin/plugin.json +++ b/plugins/dw/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "dw", "description": "Compose MiniMax H3 video, MiniMax Music 3, LTX-2.5 and Kandinsky 6.0 workflows over a dw MCP server, and cut a multi-episode series from them: which template fits which shape, the hard rules, cost, and how to judge the output. Prompt format comes from the vendors' own guides.", - "version": "0.13.0", + "version": "0.14.0-alpha.1", "author": { "name": "Don Kackman" }, diff --git a/pyproject.toml b/pyproject.toml index 223607f3..b517db58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ name = "diffusers-workflow" # runtime (static in TOML because importing dw at build time would drag # torch into the build environment). Release tags must match it - see # docs/RELEASING.md -version = "0.13.0" +version = "0.14.0-alpha.1" description = "Declarative workflow engine and web UI for Hugging Face Diffusers" readme = "README.md" requires-python = ">=3.10" From f651fe6f114d82a81acbb91fb6a159e9e44a751a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 21:34:33 -0500 Subject: [PATCH 002/204] docs(spec): roles and notes for runs, outputs, assets and workspaces Co-Authored-By: Claude Opus 5.5 --- .../2026-10-12-roles-and-notes-design.md | 343 ++++++++++++++++++ 1 file changed, 343 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-12-roles-and-notes-design.md diff --git a/docs/superpowers/specs/2026-10-12-roles-and-notes-design.md b/docs/superpowers/specs/2026-10-12-roles-and-notes-design.md new file mode 100644 index 00000000..0e775d1d --- /dev/null +++ b/docs/superpowers/specs/2026-10-12-roles-and-notes-design.md @@ -0,0 +1,343 @@ +# Roles and notes for runs, outputs, assets and workspaces + +Date: 2026-10-12. Status: approved in conversation, awaiting spec review. + +## Goal + +The gallery is hard to navigate because nothing tells the deliverable from +the scratch files. The `final` / `intermediate` subfolder convention works +for the catalog templates (121 and 37 uses across all 65), but the engine +treats no folder name specially. As a result, a hand-built workflow lands +its files flat, or in whatever folder the agent picked. And nothing records +*why* a file exists: an agent asked to "experiment with x, y, z and keep a +note of the settings with each output" has nowhere to put the note. + +This adds two things: + +1. **Roles.** Every file in a run is `final` or `intermediate`, and the + files a run read are its `input`s. A role is decided by default, with + nothing for the agent to do. In the UI roles are the main way to + navigate, and folders become a detail. +2. **Notes.** Optional plain text on a run, on a file in a run, on an asset + and on a workspace. It can be set at submission or afterwards, by the UI + or by an agent. + +Free-form tags are deliberately out of scope. They will be revisited if +roles turn out not to be enough categorization. + +## Build order + +1. Roles: engine recording, server inference for older runs, API and MCP + fields, gallery and job UI. +2. Notes: the four stores, submission and edit paths, API and MCP, UI. + +Each stage ships on its own and passes the full suite. + +## 1. Roles + +### Values + +- `final`: a file the run produced as a deliverable. +- `intermediate`: a file the run produced so that a later step could + consume it. +- `input`: an `asset:` or `output:` file the run's arguments resolved to. + An input is never copied. It is a reference recorded in the run's + manifest. + +### Deciding a step's role + +The role is decided per step, and every file the step wrote gets it. The +first rule that holds wins: + +1. **Explicit.** The step's `result.role` is `"final"` or + `"intermediate"`. This is a new optional property in + `dw/workflow_schema.json` under the result block, next to `save` and + `subfolder`, and it may be a `variable:` reference like `subfolder`. +2. **Convention.** The step's resolved `result.subfolder` is `final` or + `intermediate`, or its first path segment is one of them + (`final/act-1`). +3. **Inferred.** + - A step whose files a later step *that writes media* consumes is + `intermediate`. Media means a gallery media kind: image, video or + audio. + - Every other step is `final`. + - Consuming means a `previous_result:` reference anywhere in the + later step's arguments, or in its task or pipeline inputs. + - A consumer that writes only text or JSON does not demote what it + reads. This covers transcripts, `check_script`, `plan_cuts` and + assessment probes. It is why `check_script` on a finished video + leaves the video `final`. + +A flat-layout file with no manifest has no role. It is listed with +`role: null` and counted as final by the UI filter, so nothing disappears +from view. + +### Sub-workflows and `for_each` + +- A composing step's role applies to the files the composed child wrote + into the parent's run. +- A `for_each` step has one role for all its expansions. +- The child's own manifest, if it has its own run, keeps its own roles. + +### Inputs + +As a run opens, the engine records every `asset:` and `output:` reference +the substituted arguments resolved to, as `inputs: [{reference, kind}]` in +`manifest.json`. Duplicates are removed and the order follows first use. + +### Where roles are stored + +- **New runs.** `write_run_manifest` (`dw/workflow_run.py:209`) adds + `role` to each `steps[]` entry when it writes the terminal manifest, + alongside `inputs`. +- **Older runs.** A manifest whose steps carry no `role` is inferred on the + server from the realized `workflow.json` beside it, using the same + function. The rules live in one module (`dw/roles.py`) that both the + engine and the server import, and that module names no model. The result + is cached per run directory, keyed by the manifest's mtime, next to the + gallery's existing run metadata lookups. + +### API and MCP + +- `GalleryFile` (`dw/server/api_models.py:781`) gains `role`. +- `GalleryMetadata` (`:843`) gains `role` and `inputs`. +- `GET /api/gallery` gains an optional `role` query parameter, repeatable, + taking `final` and `intermediate`. Leaving it out lists every file. +- MCP `list_gallery` gains the same `role` filter, and its compact + projection gains `role`. MCP's default stays every role, because + diagnosis needs the intermediates. +- `get_gallery_metadata` returns `role` and `inputs`. +- The OpenAPI document is regenerated (`scripts/dump_openapi.py`, then + `npm run gen:api`), and `scripts/surface_snapshot.py` is updated. + +### UI + +**Gallery** (`ui/src/lib/pages/GalleryPage.svelte`): + +- **Role chips are the main filter,** in the top bar beside the workflow + filter: **Final** and **Intermediate**, multi-select. Final is on by + default. The UI's default differs from MCP's on purpose. +- **Every tile carries its role as a coloured tag,** ahead of the file + name and visible at thumbnail size. Final is prominent and intermediate + is muted. +- **The subfolder filter leaves the top bar.** The folder path moves to a + small "Location" line at the bottom of the detail panel. +- Inside a run, finals sort before intermediates. + +**Job results** (`ui/src/lib/pages/JobPage.svelte`): + +- Grouped into sections: **Final** first and expanded, then + **Intermediate**, collapsed when the run has any finals, then **Made + from**, the input references tagged `input`. + +**Detail panel** (`ui/src/lib/gallery/GalleryDetail.svelte`), top to +bottom: + +1. Role tag. +2. Note (section 2). +3. A collapsed **Settings** line with the run's recorded arguments and + seed, which the manifest already holds. +4. **Made from**, the inputs, each one clickable. +5. Location. + +**Assets page:** every asset is an input, so assets get no role tag. A kept +file shows a "kept from v" line from the provenance +`record_kept_shots` already writes (`dw/runs.py:784`, read by +`kept_provenance` `:872`). + +## 2. Notes + +### What a note is + +- Optional plain text, at most 4,000 characters. Longer is refused with a + 400. +- Rendered as text, never as HTML or markdown. +- An empty string or null clears the note. + +### Stores + +| On | Stored in | Shape | +|---|---|---| +| Run | `notes.json` in the run directory | `{"run": "..."}` | +| File in a run | the same `notes.json` | `{"files": {"": "..."}}` | +| Asset | the existing directory sidecar `manifest.json` that `record_kept_shots` writes | a top-level `notes` map keyed by file name | +| Workspace | new `workspace.json` at the workspace root | `{"notes": "..."}` | + +Why `notes.json` and not `manifest.json`: the engine rewrites the whole +manifest on every write while a run is going (`write_run_manifest`), so a +note written into it by the server could be lost. A separate file keeps the +engine's manifest single-writer. + +Every write is atomic, using a temp file and replace, as `write_manifest` +does (`dw/runs.py:683`). Concurrent edits to the same `notes.json` are +serialized by a per-path lock in the server. The engine writes +`notes.json` only once, when the run opens, before the server can address +the run directory. + +### The effective note + +A file's effective note is its own entry in `files`, or else the run's +note. + +### Bookkeeping + +- `notes.json` joins `RUN_SIDECARS` (`dw/server/routes/gallery.py:277`) and + `RUN_BOOKKEEPING_FILES` (`dw/server/outputs.py:54`). As a result: + - a run left holding only notes is pruned when its last output is + deleted; + - it is still detected as an orphan; + - `notes.json` is never listed as media; + - `dw/server/exports.py` copies it into the export zip beside the + manifest. +- Deleting one output removes that file's entry from `notes.json`. +- `workspace.json` joins the entries `delete_workspace` tolerates + (`_foreign_entries`, `dw/workspace.py:374`). +- Deleting an asset (`dw/server/routes/assets.py:470`) removes that file's + entries from its directory sidecar: its notes, provenance and shots. This + fixes an existing leak, where a later upload under the same name + inherited stale provenance. + +### Setting a note + +**At submission.** `JobRequest` (`dw/server/admission.py:383`) and MCP +`run_workflow` gain an optional `note`. + +- It is stored in a new `note` column on the sqlite `jobs` table, added by + the existing `ALTER TABLE ADD COLUMN` migration path + (`dw/server/job_history.py:61-133`). +- It is passed to the worker in the job spec, and the engine writes it to + `notes.json` when the run opens. +- `note` joins `RERUN_SPEC_KEYS` (`dw/server/job_record.py:31`), so + `rerun_job` carries it forward. + +**At upload or keep.** + +- `POST /api/uploads` and MCP `upload_asset` gain `note`. Upload writes + the directory sidecar when a note is given. +- `KeepRequest` and MCP `keep_output` gain `note`. Without one, keep copies + the source file's effective note. + +**Afterwards.** One route, `PUT /api/notes`, with body +`{target, note}`. `target` is one of: + +- a gallery file name: sets that file's note; +- a run directory, `/`: sets the run note; +- `asset:`; +- `workspace:`. + +The target is resolved by the same validated resolution the delete and +metadata routes use (`dw/server/routes/gallery.py:346`, +`dw/server/routes/media.py:69`, `validate_asset_reference`). No path +handling is new, and every write lands on a path that has been through +`validate_output_path` against its root. + +The route answers with the target's effective note and `note_source`. +Notes are free metadata, so they need no `acknowledged_cost`. + +MCP gains one tool, `set_note(target, note)`, over that route. + +### Reading notes + +- `GalleryFile` gains `note`, the effective one. +- `GalleryMetadata` gains `note`, `run_note` and `note_source`, which is + `"file"` or `"run"`. +- `AssetFile` (`api_models.py:864`) and `WorkspaceInfo` (`:627`) gain + `note`. +- `JobSummary` (`:345`) and `JobDetail` (`:419`) gain `note`. +- In MCP, the compact projections of `list_gallery`, `list_assets`, + `list_workspaces` (`WORKSPACE_SUMMARY_FIELDS`, `dw_mcp/workspaces.py:21`) + and `list_jobs` / `get_job` include `note`. + - In a compact projection the note is clipped to 200 characters with + `note_truncated: true`. + - The detail tools (`get_gallery_metadata`, `get_job`) return it in full. +- New response keys are declared with `sometimes(...)` where optional, as + strict responses require. + +### UI + +**Workspace overview** (`ui/src/lib/pages/OverviewPage.svelte:128`): + +- The note is shown under the heading. It is edited in place with a + pencil, a textarea and Save/Cancel. +- With no note, the place shows a muted "Add a note". +- `loadWorkspaces` (`ui/src/lib/workspace.svelte.ts:72`) keeps `note`. + +**Gallery and asset detail:** + +- The note sits under the role tag, edited the same way. +- When the note shown is the run's, it is labelled "run note", and the + editor offers "edit for this file" or "edit run note". + +**Gallery tiles:** a small note icon when the effective note is set, +showing the first line on hover. + +**Job page:** the run note sits under the job header, and can be edited. + +**One editor component.** `NoteEditor.svelte` is shared by all four +places. It renders the note as text, never with `{@html}`. + +### Agent guidance + +A short paragraph goes in the MCP server instructions +(`dw_mcp`) and in `docs/WORKFLOW_GUIDE.md`'s *Authoring a workflow from an +agent* section, saying: + +- When running a set of variations, pass `note` on each `run_workflow` + saying what that run tests. +- Use `set_note` to mark a keeper. +- `result.role` is available when the default inference is wrong for a + step. + +`docs/SERVER.md` and `docs/MCP.md` document the new route, fields and tool. +`docs/ARCHITECTURE.md` gains a row for `dw/roles.py`. + +## Testing + +**Python:** + +- Role inference: + - each of the three rules and their precedence; + - the text and JSON consumer exception; + - `for_each`; + - sub-workflow composing steps; + - a `variable:` role or subfolder; + - older-run inference matches what a new run records for the same + workflow; + - a flat file gives `null`. +- `inputs` recording for `asset:` and `output:` references, with + duplicates removed. +- Each note store: write, read, clear, the 4,000-character limit, atomic + writes, and the effective-note fallback. +- Bookkeeping: + - prune when only notes remain; + - orphan detection; + - the export includes `notes.json`; + - deleting a single output removes its note entry; + - deleting an asset clears its sidecar entry; + - a workspace holding `workspace.json` deletes. +- `PUT /api/notes` security: + - traversal in each target form is refused; + - an unknown target gives a 404; + - a non-writable library gives a 409, as keep and delete answer. +- Submission: `note` reaches the sqlite row and `notes.json`, and survives + a rerun. +- MCP: `set_note`, the `note` parameters, the compact clipping and + `note_truncated`. + +**UI (Vitest):** + +- Role chips, the default Final filter and tile tags. +- Job grouping. +- `NoteEditor`: save, cancel, clear, file versus run, and text-only + rendering. +- The note in each of the four places. + +**Snapshots:** OpenAPI regenerated, and `scripts/surface_snapshot.py` +updated for the MCP tool and parameters. + +## Out of scope + +- Free-form tags. +- Search over notes. +- Notes on prompts and saved workflows, which already have `description`. +- Moving notes with a cross-workspace move (#853). From 1654d1960d707176dca11b4c87484c0abfe74da6 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 21:46:50 -0500 Subject: [PATCH 003/204] docs(plan): roles and notes implementation plan Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-10-12-roles-and-notes.md | 2238 +++++++++++++++++ 1 file changed, 2238 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-12-roles-and-notes.md diff --git a/docs/superpowers/plans/2026-10-12-roles-and-notes.md b/docs/superpowers/plans/2026-10-12-roles-and-notes.md new file mode 100644 index 00000000..ea856954 --- /dev/null +++ b/docs/superpowers/plans/2026-10-12-roles-and-notes.md @@ -0,0 +1,2238 @@ +# Roles and Notes Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** +- Every file a run writes carries a role, `final` or `intermediate`, and + every run records the files it read as `input`s. +- Runs, run files, assets and workspaces can carry an optional plain-text + note. +- Roles and notes are surfaced through the HTTP API, MCP and the web UI, + and roles become the gallery's main way to navigate. + +**Architecture:** +- **`dw/roles.py`** decides a step's role from: + - its `result.role`; + - its `result.subfolder` convention; + - or the data flow, i.e. which later step consumes it and whether that + consumer writes media. + + The engine writes the role into each manifest step entry when the run + ends. The server infers it, with the same function, for runs that + predate this change. +- **`dw/notes.py`** owns two stores: + - a `notes.json` in a directory, which holds a run note and per-file + notes, and serves both run directories and asset-library directories; + - a `workspace.json` at a workspace root. + + The server writes a submitted run note into `notes.json` when the + worker's `run_start` event names the run directory, so the engine never + touches notes. One `PUT /api/notes` route edits any note. MCP gains + `set_note` and a `note` parameter on `run_workflow`, `upload_asset` and + `keep_output`. + +**Tech Stack:** +- Python 3 (FastAPI, pydantic, pytest with xdist); +- Svelte 5 with runes and Vitest; +- openapi-typescript codegen; +- sqlite job history. + +**Spec:** `docs/superpowers/specs/2026-10-12-roles-and-notes-design.md` + +**Deviation from spec:** asset notes go in the asset directory's +`notes.json`, the same format as a run directory's, not in the +`manifest.json` sidecar. `record_kept_shots` deletes that sidecar whenever +its last shots or provenance entry goes, which would take the notes with +it. One format for both stores is also simpler. Task 5 updates the spec +table. + +## Global Constraints + +- Note: plain text. At most `4000` characters, otherwise HTTP 400. An + empty or whitespace-only string, or null, clears the note. Never + rendered as HTML or markdown, so never `{@html}` in the UI. +- Role values: `"final"`, `"intermediate"`, `"input"`. Only `final` and + `intermediate` are valid in `result.role`. +- Sidecar file names: + - `notes.json` in a run directory or an asset directory; + - `workspace.json` at a workspace root. +- Every `dw/` module stays at or under 1,100 lines. `dw/workflow_run.py` + is already at 1,013, so keep additions there minimal. +- The architecture ratchet (`scripts/arch_metrics.py --check + docs/stabilization/baseline.json`) must pass. This plan adds exactly two + modules (`dw/roles.py` and `dw/notes.py`), so `modules` goes from 208 to + 210. That rise needs Don's approval, and Task 1's commit names it + (`modules 208 -> 210: dw/roles.py, dw/notes.py, roles-and-notes spec`). + Add no other new Python module: routes go in existing route files, and + MCP code goes in existing `dw_mcp` files. +- Every JSON route the UI calls declares a response model in + `dw/server/api_models.py`, with `response_model_exclude_unset`. Keys + sent only sometimes use `sometimes(...)`. +- After any API model change: + - run `python scripts/dump_openapi.py`, under the `constraints-openapi.txt` + pins; + - run `cd ui && npm run gen:api`; + - commit both generated files. + + `ui/src/lib/types.ts` only re-exports. +- A new JSON route that `ui/src/lib/api.ts` calls must be added to + `tests/test_api_contract.py`'s `UI_READ_ROUTES`, or + `tests/test_ui_twins.py` fails. +- Bits UI is imported only under `ui/src/lib/ui/`. +- Security: + - every filesystem write lands on a path checked by + `validate_output_path(path, root)` against its root, with a non-None + root; + - no `eval`, `exec` or `shell=True`; + - targets are resolved through the existing validated resolvers. +- Run Python tests with `DW_DEVICE=cpu venv/bin/python -m pytest ...`. The + local venv has torch. +- Never `git add -A` under `ui/`, because Don edits `ui/package.json`. Add + files by name. +- Commit messages end with `Co-Authored-By: Claude Opus 5.5 `. +- Work on branch `feat/roles-and-notes`, cut from `develop`. + +## Review Focus + +These failure modes have no spec line but are likely to bite. Each has a +test in the task that owns it. + +1. **A note edited while its run is still going.** The engine never writes + `notes.json` and the server serializes its own writes, so an edit + during a run must survive the run's final manifest write. Pinned in + Task 6: `test_note_set_during_run_survives_manifest_rewrite`. +2. **Deleting the last output of a run that has a note.** The run + directory must still be pruned, and must not be left behind holding + only `notes.json`. Pinned in Task 5: `test_run_with_only_notes_is_pruned_and_orphan`. +3. **Re-uploading an asset under the name of a deleted one.** It must not + inherit the old note or provenance. Pinned in Task 6: + `test_deleted_asset_leaves_no_note_or_provenance`. +4. **A `PUT /api/notes` target with traversal, an absolute path, or a + `workspace:` name that does not exist.** Expect 400 or 404, with + nothing written outside the root. Pinned in Task 6: + `test_put_note_refuses_traversal_targets`. +5. **A run whose consumer step saved nothing** (`save: false`) between two + media steps. The upstream step must still be `intermediate`. Pinned in + Task 1: `test_unsaved_consumer_still_demotes`. + +--- + +## Stage 1: Roles + +### Task 1: `dw/roles.py` and the `result.role` schema field + +**Files:** +- Create: `dw/roles.py` +- Modify: `dw/workflow_schema.json`. The result block's properties sit + near `save` and `subfolder` (around line 1230). Add `role`. +- Modify: `docs/ARCHITECTURE.md`. Add one row for roles. +- Modify: `docs/stabilization/baseline.json`. `modules` goes from 208 to + 210. Task 5 adds the second module, so raise it once, here, and name + both in the commit message. +- Test: `tests/test_roles.py` + +**Interfaces:** +- Produces: + - `FINAL = "final"`, `INTERMEDIATE = "intermediate"`, `INPUT = "input"`, + `ROLES = (FINAL, INTERMEDIATE)` + - `step_roles(steps: list[dict], manifest_steps: list[dict]) -> dict[str, str]`. + `steps` is the *expanded* definition's `"steps"`. The result maps step + name to role, for every step named in `steps` or in a manifest + entry's `step`. A manifest entry carrying `parent_step` takes its + parent step's role. + - `annotate_roles(steps: list[dict], manifest_steps: list[dict]) -> list[dict]` + returns copies of `manifest_steps`, each with `"role"` set. An entry + whose `step` is unknown gets `FINAL`. + - `run_inputs(steps: list[dict]) -> list[dict]` returns + `[{"reference": "asset:x.png", "kind": "asset"}, {"reference": "output:...", "kind": "output"}]`, + with duplicates removed and in first-use order. It is read from the + expanded steps' arguments. + - `entry_role(entry: dict) -> str | None` returns a manifest entry's + recorded role, or None. + +- [ ] **Step 1: Write the failing tests** + +```python +# tests/test_roles.py +from dw.roles import FINAL, INTERMEDIATE, annotate_roles, run_inputs, step_roles + + +def _step(name, result=None, args=None): + step = {"name": name, "task": {"command": "noop", "arguments": args or {}}} + if result is not None: + step["result"] = result + return step + + +def _entry(step, *files, **extra): + return {"step": step, "files": list(files), **extra} + + +def test_leaf_media_step_is_final_and_consumed_one_intermediate(): + steps = [ + _step("image"), + _step("video", args={"image": "previous_result:image"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "b.mp4")] + assert step_roles(steps, manifest) == {"image": INTERMEDIATE, "video": FINAL} + + +def test_property_reference_counts_as_consumption(): + steps = [ + _step("stems"), + _step("mix", args={"audio": "previous_result:stems.vocals"}), + ] + manifest = [_entry("stems", "v.wav"), _entry("mix", "m.wav")] + assert step_roles(steps, manifest)["stems"] == INTERMEDIATE + + +def test_text_only_consumer_does_not_demote(): + steps = [ + _step("video"), + _step("check", args={"video": "previous_result:video"}), + ] + manifest = [_entry("video", "v.mp4"), _entry("check", "report.json")] + roles = step_roles(steps, manifest) + assert roles == {"video": FINAL, "check": FINAL} + + +def test_unsaved_consumer_still_demotes(): + steps = [ + _step("image"), + _step("latent", args={"image": "previous_result:image"}), + _step("video", args={"latent": "previous_result:latent"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "v.mp4")] + roles = step_roles(steps, manifest) + assert roles["image"] == INTERMEDIATE + assert roles["latent"] == INTERMEDIATE + assert roles["video"] == FINAL + + +def test_explicit_role_wins_over_inference_and_subfolder(): + steps = [ + _step("image", result={"role": "final", "subfolder": "intermediate"}), + _step("video", args={"image": "previous_result:image"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "v.mp4")] + assert step_roles(steps, manifest)["image"] == FINAL + + +def test_subfolder_convention_including_nested(): + steps = [ + _step("a", result={"subfolder": "intermediate/shots"}), + _step("b", result={"subfolder": "final/act-1"}), + ] + manifest = [_entry("a", "x.png"), _entry("b", "y.png")] + assert step_roles(steps, manifest) == {"a": INTERMEDIATE, "b": FINAL} + + +def test_other_subfolder_name_falls_through_to_inference(): + steps = [_step("a", result={"subfolder": "shots"})] + assert step_roles(steps, [_entry("a", "x.png")]) == {"a": FINAL} + + +def test_from_previous_result_key_counts(): + steps = [ + _step("seg"), + _step("inpaint", args={"mask": {"from_previous_result": "seg"}}), + ] + manifest = [_entry("seg", "m.png"), _entry("inpaint", "o.png")] + assert step_roles(steps, manifest)["seg"] == INTERMEDIATE + + +def test_dotted_step_name_resolves_by_longest_match(): + steps = [ + _step("v1.0"), + _step("up", args={"image": "previous_result:v1.0"}), + ] + manifest = [_entry("v1.0", "a.png"), _entry("up", "b.png")] + assert step_roles(steps, manifest)["v1.0"] == INTERMEDIATE + + +def test_for_each_members_are_steps_of_their_own(): + steps = [ + _step("shot@0"), + _step("shot@1"), + _step("join", args={"videos": ["previous_result:shot@0", "previous_result:shot@1"]}), + ] + manifest = [_entry("shot@0", "0.mp4"), _entry("shot@1", "1.mp4"), _entry("join", "j.mp4")] + roles = step_roles(steps, manifest) + assert roles["shot@0"] == roles["shot@1"] == INTERMEDIATE + assert roles["join"] == FINAL + + +def test_child_entries_take_parent_step_role(): + steps = [ + _step("child", args={}), + _step("score", args={"video": "previous_result:child"}), + ] + manifest = [ + _entry("render", "r.mp4", parent_step="child"), + _entry("score", "s.mp4"), + ] + annotated = annotate_roles(steps, manifest) + assert annotated[0]["role"] == INTERMEDIATE + assert annotated[1]["role"] == FINAL + + +def test_annotate_roles_copies_and_defaults_unknown_to_final(): + manifest = [_entry("ghost", "g.png")] + annotated = annotate_roles([], manifest) + assert annotated[0]["role"] == FINAL + assert "role" not in manifest[0] + + +def test_run_inputs_dedups_in_first_use_order(): + steps = [ + _step("a", args={"image": "asset:cast/iris.png", "mask": "output:w/r/m.png"}), + _step("b", args={"images": ["asset:cast/iris.png", "asset:bg.png"]}), + ] + assert run_inputs(steps) == [ + {"reference": "asset:cast/iris.png", "kind": "asset"}, + {"reference": "output:w/r/m.png", "kind": "output"}, + {"reference": "asset:bg.png", "kind": "asset"}, + ] +``` + +- [ ] **Step 2: Run the tests to verify they fail** + +Run: `DW_DEVICE=cpu venv/bin/python -m pytest tests/test_roles.py -v -n0` +Expected: FAIL with `ModuleNotFoundError: No module named 'dw.roles'`. + +- [ ] **Step 3: Write `dw/roles.py`** + +```python +"""The role of each file a run wrote - final or intermediate - and the +inputs it read. + +The gallery mixes a run's deliverable with the scratch files that went +into it. The 'final'/'intermediate' result subfolder convention separates +them for the catalog templates, but the engine treats no folder name +specially and a hand-built workflow rarely sets one. A role is decided per +step, first rule wins: + +1. the step's `result.role` ('final' or 'intermediate'); +2. its `result.subfolder`, when that is or starts with one of those names; +3. the data flow: a step a later step consumes (a previous_result + reference) is intermediate when that consumer writes media or saves + nothing at all, and final otherwise - a consumer that writes only text + or JSON (a transcript, check_script, plan_cuts) reads a deliverable, it + does not make one out of it. + +A composed child's rolled-up manifest entries carry `parent_step` and take +that step's role. The engine records roles when a run ends; the server +infers them with the same function for a run that predates this. Inputs +are the asset: and output: references the expanded steps carry - a run's +'made from' list. This module names no model. +""" + +from .references import ASSET, OUTPUT, is_ref, iter_previous_result_references +from .security import ( + ALLOWED_AUDIO_EXTENSIONS, + ALLOWED_IMAGE_EXTENSIONS, + ALLOWED_VIDEO_EXTENSIONS, +) +from .step_cache import reference_resolves_to + +FINAL = "final" +INTERMEDIATE = "intermediate" +INPUT = "input" +ROLES = (FINAL, INTERMEDIATE) +ROLE_KEY = "role" + +_MEDIA_EXTENSIONS = frozenset( + ALLOWED_IMAGE_EXTENSIONS | ALLOWED_VIDEO_EXTENSIONS | ALLOWED_AUDIO_EXTENSIONS +) +_INPUT_KINDS = ((ASSET, "asset"), (OUTPUT, "output")) + + +def _declared_role(step): + """Rules 1 and 2: the role a step's result block states, or None.""" + result = step.get("result") + if not isinstance(result, dict): + return None + role = result.get(ROLE_KEY) + if role in ROLES: + return role + subfolder = result.get("subfolder") + if isinstance(subfolder, str): + head = subfolder.split("/", 1)[0] + if head in ROLES: + return head + return None + + +def _writes_only_text(files): + """Whether a consumer saved files and none of them is media.""" + if not files: + return False + return not any( + str(name).lower().endswith(tuple(_MEDIA_EXTENSIONS)) for name in files + ) + + +def _files_by_step(manifest_steps): + files = {} + for entry in manifest_steps or []: + if not isinstance(entry, dict): + continue + # A child's entry is its composing step's output in the parent + name = entry.get("parent_step") or entry.get("step") + files.setdefault(name, []).extend(entry.get("files") or []) + return files + + +def step_roles(steps, manifest_steps): + """Each step's role, by name - see the module docstring for the rules.""" + steps = [s for s in steps or [] if isinstance(s, dict) and "name" in s] + files = _files_by_step(manifest_steps) + names = [s["name"] for s in steps] + roles = {} + for step in steps: + declared = _declared_role(step) + if declared: + roles[step["name"]] = declared + for consumer in steps: + if _writes_only_text(files.get(consumer["name"])): + continue + for _path, reference, _via in iter_previous_result_references( + consumer, descend_into_from=True + ): + if not isinstance(reference, str): + continue + target = max( + (n for n in names if reference_resolves_to(reference, n)), + key=len, + default=None, + ) + if target is not None and target not in roles: + roles[target] = INTERMEDIATE + for name in list(names) + list(files): + roles.setdefault(name, FINAL) + return roles + + +def annotate_roles(steps, manifest_steps): + """Copies of `manifest_steps`, each with its step's `role`.""" + roles = step_roles(steps, manifest_steps) + annotated = [] + for entry in manifest_steps or []: + if not isinstance(entry, dict): + annotated.append(entry) + continue + name = entry.get("parent_step") or entry.get("step") + annotated.append({**entry, ROLE_KEY: roles.get(name, FINAL)}) + return annotated + + +def entry_role(entry): + """A manifest entry's recorded role, or None when it predates roles.""" + role = entry.get(ROLE_KEY) if isinstance(entry, dict) else None + return role if role in ROLES else None + + +def _walk_strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _walk_strings(item) + elif isinstance(value, list): + for item in value: + yield from _walk_strings(item) + + +def run_inputs(steps): + """The asset: and output: references the steps read, in first-use order.""" + seen = [] + for step in steps or []: + if not isinstance(step, dict): + continue + for value in _walk_strings(step): + for prefix, kind in _INPUT_KINDS: + if is_ref(prefix, value) and value not in (s["reference"] for s in seen): + seen.append({"reference": value, "kind": kind}) + return seen +``` + +There are two judgment points here. In `step_roles`, a declared role is +never overridden by inference: that is the "explicit wins" rule. Steps +that are not declared get `INTERMEDIATE` from any consumer that is not +text-only. + +`dw/references.py` is the only module allowed to spell a reference +prefix (`PREFIX_OWNERS` in `scripts/arch_metrics.py`). That is why this +module imports `ASSET` and `OUTPUT` and never writes `"asset:"`. + +- [ ] **Step 4: Add `role` to the schema** + +In `dw/workflow_schema.json`, next to `"save"` in the result block's +`properties`, add: + +```json +"role": { + "description": "What this step's files are to whoever looks at the run: 'final' (a deliverable) or 'intermediate' (made for a later step). Optional: without it a subfolder named 'final' or 'intermediate' decides, else a step a later media-writing step consumes is intermediate and every other step is final. May be a 'variable:' reference.", + "anyOf": [ + {"enum": ["final", "intermediate"]}, + {"type": "string", "pattern": "^variable:"} + ] +} +``` + +Check how `subfolder` declares its `variable:` allowance in the same +block, and mirror that exactly if it differs. + +- [ ] **Step 5: Add a schema test** + +Append to `tests/test_roles.py`: + +```python +import pytest +from dw.validation import validate_workflow_definition # use the name tests/test_schema*.py use + + +@pytest.mark.parametrize("role,ok", [("final", True), ("intermediate", True), ("input", False), ("keeper", False)]) +def test_schema_role_values(role, ok): + workflow = { + "id": "t", + "steps": [{"name": "s", "task": {"command": "noop", "arguments": {}}, "result": {"role": role}}], + } + errors = validate_workflow_definition(workflow) + assert (not errors) == ok +``` + +Before writing this, `grep -rn "def validate" dw/validation.py`, and use +the schema-validation entry point that the existing schema tests import. +Adapt the call and the shape of its return value to match. + +- [ ] **Step 6: Run the tests** + +Run: `DW_DEVICE=cpu venv/bin/python -m pytest tests/test_roles.py -v -n0` +Expected: all PASS. + +- [ ] **Step 7: Add the architecture map row and raise the baseline** + +In `docs/ARCHITECTURE.md`, add a row in the table that holds the run and +manifest rows, using the same column shape: + +`| A file's role (final / intermediate / input) | dw/roles.py | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server infers it identically for older runs. | tests/test_roles.py |` + +Set `"modules": 210` in `docs/stabilization/baseline.json`. + +Run: `venv/bin/python scripts/arch_metrics.py --check docs/stabilization/baseline.json` +Expected: pass. The current count is 209 with `dw/roles.py`, which is +under 210. + +Run: `venv/bin/python -m pytest tests/test_architecture_map.py -n0 -q` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add dw/roles.py dw/workflow_schema.json tests/test_roles.py docs/ARCHITECTURE.md docs/stabilization/baseline.json +git commit -m "feat(roles): decide each step's role - final, intermediate - and a run's inputs + +modules 208 -> 210: dw/roles.py (this commit) and dw/notes.py (notes +stage), per the roles-and-notes spec, approved by Don. + +Co-Authored-By: Claude Opus 5.5 " +``` + +### Task 2: The engine records roles and inputs in the manifest + +**Files:** +- Modify: `dw/workflow_run.py`: + - `RunRecord`, around line 95: add a field; + - `open_run`, line 616: keep the expanded steps; + - `write_run_manifest`, line 209: annotate. +- Test: `tests/test_run_manifest_roles.py` + +**Interfaces:** +- Consumes: `annotate_roles(steps, manifest_steps)` and `run_inputs(steps)` + from Task 1. +- Produces: a terminal `manifest.json` has: + - `steps[i]["role"]` on every entry; + - a top-level `"inputs": [{"reference", "kind"}]`. + + A `"running"` manifest carries `inputs` but no roles. + +- [ ] **Step 1: Write the failing test** + +Find an existing test that runs a small workflow end to end on the CPU +and reads its `manifest.json`: `grep -rln "manifest.json" tests/ | head`, +and prefer the one using noop or utility tasks. Copy its fixture pattern. +The test: + +```python +# tests/test_run_manifest_roles.py +import json, os +# reuse the fixture/helper the chosen existing test uses to run a workflow +# into a tmp output dir and return the run directory + + +def test_terminal_manifest_records_roles_and_inputs(run_workflow_into, tmp_path, asset_png): + workflow = { + "id": "roles-e2e", + "steps": [ + # two utility steps that write images, the second reading the first, + # e.g. an image-processor task over an asset - pick tasks the chosen + # fixture's workflows already use so no model is downloaded + ], + } + run_dir = run_workflow_into(workflow) + manifest = json.load(open(os.path.join(run_dir, "manifest.json"))) + roles = {e["step"]: e["role"] for e in manifest["steps"]} + assert roles == {"first": "intermediate", "second": "final"} + assert manifest["inputs"] == [{"reference": f"asset:{asset_png}", "kind": "asset"}] +``` + +Fill the step list with two real, model-free image tasks from +`docs/TASKS.md`, for example the resize or crop tasks the +image-processors template uses. Check that the chosen fixture can serve +an `asset:` file. If no end-to-end fixture fits, unit-test +`write_run_manifest` directly instead: +- construct a `RunRecord` and a minimal stand-in `workflow` with + `manifest`, `_run_dir`, `_run_version`, `_elided_steps`, `name` and + `file_spec`; +- set `record.expanded_steps`; +- assert the written file. + +- [ ] **Step 2: Run it to verify it fails** + +Run: `DW_DEVICE=cpu venv/bin/python -m pytest tests/test_run_manifest_roles.py -v -n0` +Expected: FAIL with `KeyError: 'role'`. + +- [ ] **Step 3: Implement it** + +In `dw/workflow_run.py`: + +```python +from .roles import annotate_roles, run_inputs +``` + +Add to `RunRecord`, after `annotations`: + +```python + # The expanded steps the run executed - what roles and inputs are read from + expanded_steps: list = field(default_factory=list) +``` + +In `open_run`, at the top, after `record.run_id = None`: + +```python + record.expanded_steps = prepared.workflow_def.get("steps") or [] +``` + +In `write_run_manifest`, replace the `"steps": [...]` value with a local +computed before the `write_manifest(...)` call, and add `"inputs"`: + +```python + steps = [ + { + **entry, + "files": manifest_relative_files(entry.get("files"), workflow._run_dir), + **_relative_shots(entry, workflow._run_dir), + } + for entry in workflow.manifest + ] + # Roles once the run has ended - a running manifest would show the + # newest step as final until its consumer runs (dw/roles.py) + if status != "running": + steps = annotate_roles(record.expanded_steps, steps) +``` + +Then use `"steps": steps,` and add `"inputs": run_inputs(record.expanded_steps),` +after `"arguments"`. + +Keep the net addition to roughly 12 lines. + +Run: `wc -l dw/workflow_run.py` +Expected: at or under 1,030. + +- [ ] **Step 4: Run the test and the run and manifest suites** + +Run: `DW_DEVICE=cpu venv/bin/python -m pytest tests/test_run_manifest_roles.py tests/ -k "manifest or run_dir or runs" -q` +Expected: PASS. If an existing test compares the full manifest dict for +equality, update its expectation to include `role` and `inputs`. Those +fields are the intended change. + +- [ ] **Step 5: Commit** + +```bash +git add dw/workflow_run.py tests/test_run_manifest_roles.py +git commit -m "feat(roles): the engine records each step's role and the run's inputs in its manifest + +Co-Authored-By: Claude Opus 5.5 " +``` + +Also `git add` any existing test files whose expectations you updated, +naming each one. + +### Task 3: Server and MCP surface roles + +**Files:** +- Modify: `dw/server/routes/gallery.py`: + - `_gallery_entries`, line 57: add `role`; + - `GET /api/gallery`, around line 163: add the `role` filter. +- Modify: `dw/server/routes/media.py`. In `GET /api/gallery/{name}/metadata`, + around line 69, add `role` and `inputs`. +- Modify: `dw/server/api_models.py`: + - `GalleryFile`, line 781; + - `GalleryMetadata`, line 843. +- Modify: `dw_mcp/catalog.py`: + - `list_gallery` logic, around line 229, and its compact projection; + - `get_gallery_metadata`, line 285. +- Modify: `dw_mcp/tools_catalog.py`. `list_gallery` at line 255 gains a + `role` parameter. +- Modify: `ui/src/lib/generated/openapi.json` and + `ui/src/lib/generated/api-schema.ts`, regenerated. +- Modify: `docs/SERVER.md` and `docs/MCP.md`, gallery sections, one + paragraph each. +- Test: `tests/test_gallery_roles.py` + +**Interfaces:** +- Consumes: `annotate_roles`, `entry_role` and `ROLES` from Task 1, and + the manifests Task 2 writes. +- Produces: + - `run_roles(run_dir: str) -> dict[str, str]`, a module-level function + in `dw/server/routes/gallery.py`. It maps a file's path relative to + the run directory, `/`-separated, to its role. It reads + `manifest.json` and uses recorded roles where they are present. + Otherwise it expands the realized `workflow.json` and calls + `annotate_roles`. If neither file is readable it returns `{}`. Cache + it in a module dict keyed by `(run_dir, manifest mtime)`, bounded at + 512 entries and evicting oldest first. + - `GalleryFile.role: Literal["final","intermediate"] | None` + - `GalleryMetadata.role` (the same type), and + `GalleryMetadata.inputs: list[GalleryInput]` with + `GalleryInput{reference: str, kind: Literal["asset","output"]}`, both + via `sometimes()`. + - `GET /api/gallery?role=final&role=intermediate`, a repeatable query. + Leaving it out returns everything. A file with `role` None counts as + final for filtering. + - MCP `list_gallery(..., role: list[str] | None = None)`. The compact + projection gains `role`. + +- [ ] **Step 1: Write the failing tests** + +```python +# tests/test_gallery_roles.py +import json, os +# use the TestClient / app fixture tests/test_server.py uses for /api/gallery +# (grep "api/gallery" tests/test_server.py for the fixture names) + + +def _write_run(root, identity, run_id, steps, realized=None, files=()): + run_dir = os.path.join(root, identity, run_id) + os.makedirs(run_dir, exist_ok=True) + for f in files: + path = os.path.join(run_dir, f) + os.makedirs(os.path.dirname(path), exist_ok=True) + open(path, "wb").write(b"\x89PNG\r\n\x1a\n") + json.dump({"run_id": run_id, "version": 1, "status": "succeeded", "steps": steps}, + open(os.path.join(run_dir, "manifest.json"), "w")) + if realized is not None: + json.dump(realized, open(os.path.join(run_dir, "workflow.json"), "w")) + return run_dir + + +def test_recorded_roles_reach_the_listing(client, output_root): + _write_run(output_root, "wf", "20261012-000000-aaaa", + [{"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["b.png"], "role": "final"}], + files=["a.png", "b.png"]) + files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} + assert files["a.png"]["role"] == "intermediate" + assert files["b.png"]["role"] == "final" + + +def test_older_run_roles_inferred_from_realized_workflow(client, output_root): + realized = {"id": "wf", "steps": [ + {"name": "a", "task": {"command": "noop", "arguments": {}}}, + {"name": "b", "task": {"command": "noop", "arguments": {"image": "previous_result:a"}}}, + ]} + _write_run(output_root, "wf", "20261012-000001-bbbb", + [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}], + realized=realized, files=["a.png", "b.png"]) + files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} + assert files["a.png"]["role"] == "intermediate" + assert files["b.png"]["role"] == "final" + + +def test_role_filter(client, output_root): + _write_run(output_root, "wf", "20261012-000002-cccc", + [{"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["final/b.png"], "role": "final"}], + files=["a.png", "final/b.png"]) + finals = client.get("/api/gallery", params={"role": "final"}).json()["files"] + assert [f["label"] for f in finals] == ["b.png"] + + +def test_flat_file_has_null_role_and_passes_final_filter(client, output_root): + os.makedirs(os.path.join(output_root, "loose"), exist_ok=True) + open(os.path.join(output_root, "loose", "x.png"), "wb").write(b"\x89PNG\r\n\x1a\n") + entries = client.get("/api/gallery", params={"role": "final"}).json()["files"] + loose = [f for f in entries if f["label"] == "x.png"] + assert loose and loose[0].get("role") is None + + +def test_metadata_carries_role_and_inputs(client, output_root): + run_dir = _write_run(output_root, "wf", "20261012-000003-dddd", + [{"step": "b", "files": ["b.png"], "role": "final"}], files=["b.png"]) + manifest_path = os.path.join(run_dir, "manifest.json") + manifest = json.load(open(manifest_path)) + manifest["inputs"] = [{"reference": "asset:iris.png", "kind": "asset"}] + json.dump(manifest, open(manifest_path, "w")) + meta = client.get("/api/gallery/wf/20261012-000003-dddd/b.png/metadata").json() + assert meta["role"] == "final" + assert meta["inputs"] == [{"reference": "asset:iris.png", "kind": "asset"}] +``` + +Match the run-id format to `is_run_id` in `dw/runs.py`, and match the +fixture names (`client`, `output_root`) to what `tests/test_server.py` +already uses, renaming where needed. Add an MCP test in the existing MCP +catalog test file (`grep -ln "list_gallery" tests/`) asserting that +`role` appears in the compact projection and that `role=["final"]` is +forwarded as a query parameter. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `DW_DEVICE=cpu venv/bin/python -m pytest tests/test_gallery_roles.py -v -n0` +Expected: FAIL, because `role` is missing from the entries. + +- [ ] **Step 3: Implement `run_roles` and the listing** + +In `dw/server/routes/gallery.py`: + +```python +from ...roles import ROLES, annotate_roles, entry_role +from ...runs import _read_manifest # already used nearby? reuse the existing import + +_ROLE_CACHE = {} +_ROLE_CACHE_LIMIT = 512 + + +def _realized_steps(run_dir): + """The realized workflow's expanded steps, or None.""" + path = os.path.join(run_dir, REALIZED_FILE_NAME) + try: + with open(path) as handle: + definition = json.load(handle) + return Workflow(definition, None, path).expanded_definition().get("steps") or [] + except Exception: # an unreadable or unexpandable record leaves roles unknown + return None + + +def run_roles(run_dir): + """Each file of a run, relative to the run directory, mapped to its role.""" + manifest_path = os.path.join(run_dir, MANIFEST_FILE_NAME) + try: + key = (run_dir, os.stat(manifest_path).st_mtime_ns) + except OSError: + return {} + if key in _ROLE_CACHE: + return _ROLE_CACHE[key] + manifest = _read_manifest(run_dir) or {} + steps = [e for e in manifest.get("steps") or [] if isinstance(e, dict)] + if steps and any(entry_role(e) is None for e in steps): + expanded = _realized_steps(run_dir) + if expanded is not None: + steps = annotate_roles(expanded, steps) + roles = {} + for entry in steps: + role = entry_role(entry) + if role: + for name in entry.get("files") or []: + roles[str(name).replace(os.sep, "/")] = role + if len(_ROLE_CACHE) >= _ROLE_CACHE_LIMIT: + _ROLE_CACHE.pop(next(iter(_ROLE_CACHE))) + _ROLE_CACHE[key] = roles + return roles +``` + +`Workflow` comes from `...workflow`. Check that +`Workflow(definition, output_dir=None, file_spec=path)` constructs +without touching the filesystem, as `dw/server/routes/library.py:303` +does, and copy that call's arguments. + +In `_gallery_entries`, add `role` to each entry. Compute the run +directory as `os.path.join(root, folder, run_id)` when `run_id` is set. +The relative name inside it is `relative_name` with the +`f"{folder}/{run_id}/"` prefix removed. Then: + +```python + entry["role"] = ( + run_roles(os.path.join(root, folder, run_id)).get(inner_name) + if run_id + else None + ) +``` + +Add a `role: list[str] | None = Query(None)` parameter to the gallery +route. After building the entries, filter: + +```python + if role: + wanted = set(role) + files = [f for f in files if (f.get("role") or "final") in wanted] +``` + +Refuse any value outside `ROLES` with a 400. + +In `api_models.py`: +- `GalleryFile` gains `role: Literal["final", "intermediate"] | None = sometimes("...")`; +- add the `GalleryInput` model; +- `GalleryMetadata` gains `role` and `inputs`. + +In `media.py`'s metadata route, for an output name (not `asset:`): +- set `role` from `run_roles(run_dir).get(inner_name)`; +- set `inputs` from the manifest's `inputs` when present. + +Reuse the route's existing run-directory resolution. It already finds +the manifest for `job`, `shots` and so on. + +- [ ] **Step 4: MCP** + +In `dw_mcp/tools_catalog.py`, `list_gallery` gains: + +```python + role: Annotated[ + list[Literal["final", "intermediate"]] | None, + Field(description="Only files of these roles. A deliverable is 'final', a file a later step consumed is 'intermediate'. Omit for every file - intermediates matter when diagnosing a run."), + ] = None, +``` + +Forward it as the repeated `role` query parameter in `dw_mcp/catalog.py`, +and add `"role"` to the gallery compact projection tuple. Match the +`Annotated` and `Field` style the neighbouring parameters use. + +- [ ] **Step 5: Regenerate the contract and run the tests** + +```bash +venv/bin/python scripts/dump_openapi.py +(cd ui && npm run gen:api) +DW_DEVICE=cpu venv/bin/python -m pytest tests/test_gallery_roles.py tests/test_server.py tests/test_api_contract.py tests/test_ui_twins.py -q +venv/bin/python scripts/surface_snapshot.py --check || venv/bin/python scripts/surface_snapshot.py +``` + +Expected: PASS. Run `scripts/surface_snapshot.py --help` first to learn +its update flag, then update the snapshot. + +- [ ] **Step 6: Docs** + +`docs/SERVER.md`, gallery section: + +> Each entry carries `role`, `final` or `intermediate` (null for a flat-layout file with no run), and `GET /api/gallery?role=final` lists only those. See `dw/roles.py` for how a role is decided. + +`docs/MCP.md`, `list_gallery`: the same sentence, plus "the `role` filter +defaults to every file". + +- [ ] **Step 7: Commit** + +```bash +git add dw/server/routes/gallery.py dw/server/routes/media.py dw/server/api_models.py dw_mcp/catalog.py dw_mcp/tools_catalog.py tests/test_gallery_roles.py ui/src/lib/generated/openapi.json ui/src/lib/generated/api-schema.ts docs/SERVER.md docs/MCP.md +git commit -m "feat(roles): gallery entries and metadata carry role and inputs; role filter on the listing and list_gallery + +Co-Authored-By: Claude Opus 5.5 " +``` + +Also add the surface snapshot file and any MCP test file you touched, by +name. + +### Task 4: UI roles + +**Files:** +- Create: `ui/src/lib/gallery/RoleTag.svelte` +- Modify: `ui/src/lib/pages/GalleryPage.svelte`: + - the `subfolder` state at line 22 is replaced by role chips; + - the `subfolders` derivation at line 53 is removed from the top bar. +- Modify: the gallery tile component. `grep -n "label" ui/src/lib/pages/GalleryPage.svelte` + finds where tiles render. Add the `RoleTag` ahead of the label. +- Modify: `ui/src/lib/gallery/GalleryDetail.svelte`, the meta block at + lines 173-225. The order becomes role tag, (the note slot comes in + Task 8), Settings collapsed, Made from, and Location last. +- Modify: `ui/src/lib/pages/JobPage.svelte`. Group results as Final + (expanded), then Intermediate (collapsed when there are finals), then + Made from. +- Modify: `ui/src/lib/assets/AssetDetail.svelte`. Add a "kept from + v" line when the metadata's `job` or provenance is + present. +- Modify: `ui/src/lib/api.ts`. `gallery()` gains an optional + `roles?: string[]` that it serializes as repeated `role` parameters. +- Test: `ui/src/lib/gallery/RoleTag.test.ts`, `ui/src/lib/pages/GalleryPage.test.ts` + (extend it if it exists, otherwise create it), and `ui/src/lib/pages/JobPage.test.ts` + (extend it). + +**Interfaces:** +- Consumes: `GalleryFile.role`, `GalleryMetadata.role` and + `GalleryMetadata.inputs`, from the regenerated `api-schema.ts`, through + `types.ts` re-exports. Add the re-exports for `GalleryInput`. +- Produces: `RoleTag` with props `{ role: 'final' | 'intermediate' | 'input' | null }`. + It renders nothing for null. Task 8 places notes next to it. + +- [ ] **Step 1: Write the failing component test** + +```ts +// ui/src/lib/gallery/RoleTag.test.ts +import { render, screen } from '@testing-library/svelte' +import { describe, expect, it } from 'vitest' +import RoleTag from './RoleTag.svelte' + +describe('RoleTag', () => { + it.each([ + ['final', 'Final'], + ['intermediate', 'Intermediate'], + ['input', 'Input'], + ])('renders %s', (role, text) => { + render(RoleTag, { role }) + const tag = screen.getByText(text) + expect(tag.dataset.role).toBe(role) + }) + + it('renders nothing for a null role', () => { + const { container } = render(RoleTag, { role: null }) + expect(container.textContent?.trim()).toBe('') + }) +}) +``` + +Check the existing component tests for the render helper this repo uses +(`grep -rln "@testing-library/svelte" ui/src | head -3`), and match their +imports. + +- [ ] **Step 2: Run it to verify it fails** + +Run: `cd ui && npx vitest run src/lib/gallery/RoleTag.test.ts` +Expected: FAIL, because the module is not found. + +- [ ] **Step 3: Implement `RoleTag.svelte`** + +```svelte + + +{#if role} + {text[role]} +{/if} + + +``` + +Use the colour tokens `ui/src/app.css` actually defines. Grep it for the +accent, muted-text, surface and border variable names, and substitute +them for the ones above. + +- [ ] **Step 4: Gallery page tests, then implementation** + +Add these tests to the GalleryPage test, mocking `api.gallery` the way +existing page tests mock api calls: + +```ts +it('opens on finals only and shows each tile role', async () => { + // mock gallery() to return one final and one intermediate entry + // render GalleryPage; expect the Final chip pressed, Intermediate not + // expect only the final tile rendered, carrying a RoleTag "Final" +}) + +it('toggling Intermediate shows intermediates too', async () => { + // click the Intermediate chip; expect both tiles +}) + +it('has no subfolder filter in the top bar', async () => { + // expect no element with the subfolder filter's label/test id +}) +``` + +Write these out as real assertions against the mock data, following the +nearest existing page test for its mocking and querying idiom. + +Implementation in `GalleryPage.svelte`: + +```ts +let roles = $state>(new Set(['final'])) +const visible = $derived( + files.filter((f) => roles.has((f.role ?? 'final') as 'final' | 'intermediate')), +) +function toggleRole(role: 'final' | 'intermediate') { + const next = new Set(roles) + next.has(role) ? next.delete(role) : next.add(role) + roles = next.size ? next : new Set([role === 'final' ? 'intermediate' : 'final']) +} +``` + +The chips are two ` + + + {#if error}{/if} + {:else} +
+ {label} + {#if shown} +

{shown}

+ {:else} +

{placeholder}

+ {/if} + +
+ {/if} + + + +``` + +Match the button classes and colour tokens to `ui/src/app.css` and the +existing small-button style, for example by copying from +`ConfirmDialog`'s buttons. Check how `api.ts` surfaces server error +detail, and make `setNote` throw an `Error` carrying the server's +`detail` the way the other `api.ts` write calls do. + +- [ ] **Step 4: Place it in the four views and on the tile** + +- **Overview:** `` + under the `

`. +- **Gallery detail:** if `meta.note_source === 'run'`, show + `` + plus a "Note this file" button. That button swaps it for + ``. + Otherwise show ``. + `runDir` is the file name's `/` prefix. Derive it + from `folder` and `run_id` on the gallery entry. After a save, + refetch the metadata and refresh the listing entry's `note`. +- **Asset detail:** `` +- **Job page:** `` + when `job.run_dir` is set. +- **Tile:** `{#if file.note}✎{/if}` + +Add one assertion per page test that the note text renders, extending +the page tests Task 4 touched. + +- [ ] **Step 5: Run the UI checks** + +```bash +cd ui && npx vitest run && npm run lint && npm run check && npm run metrics -- --check ../docs/stabilization/ui/baseline.json +``` + +Expected: all pass. Then run the twins test from the repo root: +`DW_DEVICE=cpu venv/bin/python -m pytest tests/test_ui_twins.py tests/test_api_contract.py -q`. + +- [ ] **Step 6: Commit** + +```bash +git add ui/src/lib/ui/NoteEditor.svelte ui/src/lib/ui/NoteEditor.test.ts ui/src/lib/api.ts ui/src/lib/types.ts ui/src/lib/pages/OverviewPage.svelte ui/src/lib/workspace.svelte.ts ui/src/lib/gallery/GalleryDetail.svelte ui/src/lib/assets/AssetDetail.svelte ui/src/lib/pages/JobPage.svelte ui/src/lib/pages/GalleryPage.svelte +git commit -m "feat(ui): notes on workspaces, runs, files and assets, edited in place + +Co-Authored-By: Claude Opus 5.5 " +``` + +Also add the page test files you touched, by name. + +### Task 9: Whole-branch verification + +**Files:** none new. This task only fixes what the checks turn up. + +- [ ] **Step 1: Run the full Python suite, lint and ratchet** + +```bash +DW_DEVICE=cpu venv/bin/python -m pytest tests/ -q +venv/bin/ruff check . && venv/bin/ruff format --check . +venv/bin/python scripts/arch_metrics.py --check docs/stabilization/baseline.json +venv/bin/python scripts/dump_openapi.py --check +``` + +Expected: all pass, with `modules` at 210. + +- [ ] **Step 2: Run the full UI checks and e2e** + +```bash +cd ui && npx vitest run && npm run lint && npm run check && npm run gen:api && git diff --exit-code src/lib/generated && npm run metrics -- --check ../docs/stabilization/ui/baseline.json +cd ui && npx playwright test +``` + +Expected: all pass. If an e2e gallery spec relied on the subfolder +filter or on intermediates being visible by default, update the spec to +click the Intermediate chip. That is the intended behaviour change. + +- [ ] **Step 3: Check the app by hand** + +Start `python -m dw.serve` against a scratch `DIFFUSERS_HELPER_ROOT`. Run +`workflows/templates/text-to-image.json` with +`note="first look"`. Then check: +- the gallery shows it with a Final tag; +- the run note appears in the detail panel; +- editing a file note works; +- the workspace overview note saves. + +Take one screenshot each of the gallery, the detail panel and the +overview for the release note. + +- [ ] **Step 4: Add a release note line** + +Following the convention in `docs/RELEASING.md`, add to the +unreleased-notes section: +"Gallery files are tagged final or intermediate (inferred when a +workflow doesn't say) and the gallery opens on finals; runs, files, +assets and workspaces take an optional note (`note` on `run_workflow`, +MCP `set_note`)." + +- [ ] **Step 5: Commit** + +```bash +git add docs/RELEASING.md +git commit -m "docs(release): note roles and notes for 0.14.0 + +Co-Authored-By: Claude Opus 5.5 " +``` + +Also add any e2e specs you updated, by name. From 3bcf9f552dde2fdb532cac1d60bba8ba0efe8b7b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 22:40:27 -0500 Subject: [PATCH 004/204] feat(roles): decide each step's role - final, intermediate - and a run's inputs modules 208 -> 210: dw/roles.py (this commit) and dw/notes.py (notes stage), per the roles-and-notes spec, approved by Don. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 1 + docs/stabilization/baseline.json | 2 +- dw/roles.py | 153 +++++++++++++++++++++++++++ dw/workflow_schema.json | 7 ++ tests/test_roles.py | 175 +++++++++++++++++++++++++++++++ 5 files changed, 337 insertions(+), 1 deletion(-) create mode 100644 dw/roles.py create mode 100644 tests/test_roles.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 43f13588..af4ae2ea 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -64,6 +64,7 @@ one to open. | Concept | Owner | Rule | Enforced by | | --- | --- | --- | --- | | Run directories | `dw/runs.py`: `open_run`, `workflow_identity`, `new_run_id` | Each execution writes into `///` with a `manifest.json`, unless the flat layout is chosen (the test pins the flat-layout exception). | `tests/test_runs.py::TestRunDirectories::test_the_flat_layout_writes_where_it_always_did` | +| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server infers it identically for older runs. | `tests/test_roles.py` | | Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions` | A run takes its number once, when it opens, so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | | Result subfolders | `dw/subfolders.py`: `subfolder_errors`, `dw/security.py`: `SUBFOLDER_PATTERN` | A step's `result.subfolder` is checked for shape at validation. | `tests/test_subfolders.py::TestValidationErrorsIntegration::test_validation_errors_reports_a_bad_subfolder` | diff --git a/docs/stabilization/baseline.json b/docs/stabilization/baseline.json index 303dabdb..021a3180 100644 --- a/docs/stabilization/baseline.json +++ b/docs/stabilization/baseline.json @@ -1,5 +1,5 @@ { - "modules": 208, + "modules": 210, "modules_over_size_ceiling": 0, "functions_over_150_lines": 0, "prefix_literals": 0, diff --git a/dw/roles.py b/dw/roles.py new file mode 100644 index 00000000..60dcf1bb --- /dev/null +++ b/dw/roles.py @@ -0,0 +1,153 @@ +"""The role of each file a run wrote - final or intermediate - and the +inputs it read. + +The gallery mixes a run's deliverable with the scratch files that went +into it. The 'final'/'intermediate' result subfolder convention separates +them for the catalog templates, but the engine treats no folder name +specially and a hand-built workflow rarely sets one. A role is decided per +step, first rule wins: + +1. the step's `result.role` ('final' or 'intermediate'); +2. its `result.subfolder`, when that is or starts with one of those names; +3. the data flow: a step a later step consumes (a previous_result + reference) is intermediate when that consumer writes media or saves + nothing at all, and final otherwise - a consumer that writes only text + or JSON (a transcript, check_script, plan_cuts) reads a deliverable, it + does not make one out of it. + +A composed child's rolled-up manifest entries carry `parent_step` and take +that step's role. The engine records roles when a run ends; the server +infers them with the same function for a run that predates this. Inputs +are the asset: and output: references the expanded steps carry - a run's +'made from' list. This module names no model. +""" + +from .references import ASSET, OUTPUT, is_ref, iter_previous_result_references +from .security import ( + ALLOWED_AUDIO_EXTENSIONS, + ALLOWED_IMAGE_EXTENSIONS, + ALLOWED_VIDEO_EXTENSIONS, +) +from .step_cache import reference_resolves_to + +FINAL = "final" +INTERMEDIATE = "intermediate" +INPUT = "input" +ROLES = (FINAL, INTERMEDIATE) +ROLE_KEY = "role" + +_MEDIA_EXTENSIONS = frozenset( + ALLOWED_IMAGE_EXTENSIONS | ALLOWED_VIDEO_EXTENSIONS | ALLOWED_AUDIO_EXTENSIONS +) + + +def _declared_role(step): + """Rules 1 and 2: the role a step's result block states, or None.""" + result = step.get("result") + if not isinstance(result, dict): + return None + role = result.get(ROLE_KEY) + if role in ROLES: + return role + subfolder = result.get("subfolder") + if isinstance(subfolder, str): + head = subfolder.split("/", 1)[0] + if head in ROLES: + return head + return None + + +def _writes_only_text(files): + """Whether a consumer saved files and none of them is media.""" + if not files: + return False + return not any( + str(name).lower().endswith(tuple(_MEDIA_EXTENSIONS)) for name in files + ) + + +def _files_by_step(manifest_steps): + files = {} + for entry in manifest_steps or []: + if not isinstance(entry, dict): + continue + # A child's entry is its composing step's output in the parent + name = entry.get("parent_step") or entry.get("step") + files.setdefault(name, []).extend(entry.get("files") or []) + return files + + +def step_roles(steps, manifest_steps): + """Each step's role, by name - see the module docstring for the rules.""" + steps = [s for s in steps or [] if isinstance(s, dict) and "name" in s] + files = _files_by_step(manifest_steps) + names = [s["name"] for s in steps] + roles = {} + for step in steps: + declared = _declared_role(step) + if declared: + roles[step["name"]] = declared + for consumer in steps: + if _writes_only_text(files.get(consumer["name"])): + continue + for _path, reference, _via in iter_previous_result_references( + consumer, descend_into_from=True + ): + if not isinstance(reference, str): + continue + target = max( + (n for n in names if reference_resolves_to(reference, n)), + key=len, + default=None, + ) + if target is not None and target not in roles: + roles[target] = INTERMEDIATE + for name in list(names) + list(files): + roles.setdefault(name, FINAL) + return roles + + +def annotate_roles(steps, manifest_steps): + """Copies of `manifest_steps`, each with its step's `role`.""" + roles = step_roles(steps, manifest_steps) + annotated = [] + for entry in manifest_steps or []: + if not isinstance(entry, dict): + annotated.append(entry) + continue + name = entry.get("parent_step") or entry.get("step") + annotated.append({**entry, ROLE_KEY: roles.get(name, FINAL)}) + return annotated + + +def entry_role(entry): + """A manifest entry's recorded role, or None when it predates roles.""" + role = entry.get(ROLE_KEY) if isinstance(entry, dict) else None + return role if role in ROLES else None + + +def _walk_strings(value): + if isinstance(value, str): + yield value + elif isinstance(value, dict): + for item in value.values(): + yield from _walk_strings(item) + elif isinstance(value, list): + for item in value: + yield from _walk_strings(item) + + +def run_inputs(steps): + """The asset: and output: references the steps read, in first-use order.""" + kinds = ((ASSET, "asset"), (OUTPUT, "output")) + seen = [] + for step in steps or []: + if not isinstance(step, dict): + continue + for value in _walk_strings(step): + for prefix, kind in kinds: + if is_ref(prefix, value) and value not in ( + s["reference"] for s in seen + ): + seen.append({"reference": value, "kind": kind}) + return seen diff --git a/dw/workflow_schema.json b/dw/workflow_schema.json index 837d44c4..abff7b82 100644 --- a/dw/workflow_schema.json +++ b/dw/workflow_schema.json @@ -1240,6 +1240,13 @@ "description": "A subfolder of the run directory this step's files are written into, as a relative path like 'final' or 'shots/act-1'. Optional: without it the files land at the root of the run directory as they always have. By convention a step whose output the user will be shown is 'final' and everything else is 'intermediate', which is how a consumer tells the deliverable from the scratch files without knowing the workflow; the engine treats no name specially. May be a 'variable:' or, inside a for_each step, an 'item:' reference. Each segment must start with a letter, digit or underscore; '..' and '\\' are refused.", "type": "string" }, + "role": { + "description": "What this step's files are to whoever looks at the run: 'final' (a deliverable) or 'intermediate' (made for a later step). Optional: without it a subfolder named 'final' or 'intermediate' decides, else a step a later media-writing step consumes is intermediate and every other step is final. May be a 'variable:' reference.", + "anyOf": [ + {"enum": ["final", "intermediate"]}, + {"type": "string", "pattern": "^variable:"} + ] + }, "fps": { "description": "Frames per second - only used when output is video. Defaults to the rate the frames themselves carry (a chain's 'fps', a join's own rate, the rate of the file a task read) and to 8 only when nothing knows better. Set it to write at a rate other than the source's - a deliberate slow motion - which the run then warns about. May be a 'variable:' or, inside a for_each step, an 'item:' reference, resolved the same way 'subfolder' is - the resolved value must be a positive whole number; a fractional rate is refused, since the video writer would silently truncate rather than honor it.", "type": ["integer", "number", "string"], diff --git a/tests/test_roles.py b/tests/test_roles.py new file mode 100644 index 00000000..51baf31d --- /dev/null +++ b/tests/test_roles.py @@ -0,0 +1,175 @@ +import pytest + +from dw.roles import FINAL, INTERMEDIATE, annotate_roles, run_inputs, step_roles +from dw.schema import load_schema, validate_data_all + + +def _step(name, result=None, args=None): + step = {"name": name, "task": {"command": "noop", "arguments": args or {}}} + if result is not None: + step["result"] = result + return step + + +def _entry(step, *files, **extra): + return {"step": step, "files": list(files), **extra} + + +def test_leaf_media_step_is_final_and_consumed_one_intermediate(): + steps = [ + _step("image"), + _step("video", args={"image": "previous_result:image"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "b.mp4")] + assert step_roles(steps, manifest) == {"image": INTERMEDIATE, "video": FINAL} + + +def test_property_reference_counts_as_consumption(): + steps = [ + _step("stems"), + _step("mix", args={"audio": "previous_result:stems.vocals"}), + ] + manifest = [_entry("stems", "v.wav"), _entry("mix", "m.wav")] + assert step_roles(steps, manifest)["stems"] == INTERMEDIATE + + +def test_text_only_consumer_does_not_demote(): + steps = [ + _step("video"), + _step("check", args={"video": "previous_result:video"}), + ] + manifest = [_entry("video", "v.mp4"), _entry("check", "report.json")] + roles = step_roles(steps, manifest) + assert roles == {"video": FINAL, "check": FINAL} + + +def test_unsaved_consumer_still_demotes(): + steps = [ + _step("image"), + _step("latent", args={"image": "previous_result:image"}), + _step("video", args={"latent": "previous_result:latent"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "v.mp4")] + roles = step_roles(steps, manifest) + assert roles["image"] == INTERMEDIATE + assert roles["latent"] == INTERMEDIATE + assert roles["video"] == FINAL + + +def test_explicit_role_wins_over_inference_and_subfolder(): + steps = [ + _step("image", result={"role": "final", "subfolder": "intermediate"}), + _step("video", args={"image": "previous_result:image"}), + ] + manifest = [_entry("image", "a.png"), _entry("video", "v.mp4")] + assert step_roles(steps, manifest)["image"] == FINAL + + +def test_subfolder_convention_including_nested(): + steps = [ + _step("a", result={"subfolder": "intermediate/shots"}), + _step("b", result={"subfolder": "final/act-1"}), + ] + manifest = [_entry("a", "x.png"), _entry("b", "y.png")] + assert step_roles(steps, manifest) == {"a": INTERMEDIATE, "b": FINAL} + + +def test_other_subfolder_name_falls_through_to_inference(): + steps = [_step("a", result={"subfolder": "shots"})] + assert step_roles(steps, [_entry("a", "x.png")]) == {"a": FINAL} + + +def test_from_previous_result_key_counts(): + steps = [ + _step("seg"), + _step("inpaint", args={"mask": {"from_previous_result": "seg"}}), + ] + manifest = [_entry("seg", "m.png"), _entry("inpaint", "o.png")] + assert step_roles(steps, manifest)["seg"] == INTERMEDIATE + + +def test_dotted_step_name_resolves_by_longest_match(): + steps = [ + _step("v1.0"), + _step("up", args={"image": "previous_result:v1.0"}), + ] + manifest = [_entry("v1.0", "a.png"), _entry("up", "b.png")] + assert step_roles(steps, manifest)["v1.0"] == INTERMEDIATE + + +def test_for_each_members_are_steps_of_their_own(): + steps = [ + _step("shot@0"), + _step("shot@1"), + _step( + "join", + args={"videos": ["previous_result:shot@0", "previous_result:shot@1"]}, + ), + ] + manifest = [ + _entry("shot@0", "0.mp4"), + _entry("shot@1", "1.mp4"), + _entry("join", "j.mp4"), + ] + roles = step_roles(steps, manifest) + assert roles["shot@0"] == roles["shot@1"] == INTERMEDIATE + assert roles["join"] == FINAL + + +def test_child_entries_take_parent_step_role(): + steps = [ + _step("child", args={}), + _step("score", args={"video": "previous_result:child"}), + ] + manifest = [ + _entry("render", "r.mp4", parent_step="child"), + _entry("score", "s.mp4"), + ] + annotated = annotate_roles(steps, manifest) + assert annotated[0]["role"] == INTERMEDIATE + assert annotated[1]["role"] == FINAL + + +def test_annotate_roles_copies_and_defaults_unknown_to_final(): + manifest = [_entry("ghost", "g.png")] + annotated = annotate_roles([], manifest) + assert annotated[0]["role"] == FINAL + assert "role" not in manifest[0] + + +def test_run_inputs_dedups_in_first_use_order(): + steps = [ + _step("a", args={"image": "asset:cast/iris.png", "mask": "output:w/r/m.png"}), + _step("b", args={"images": ["asset:cast/iris.png", "asset:bg.png"]}), + ] + assert run_inputs(steps) == [ + {"reference": "asset:cast/iris.png", "kind": "asset"}, + {"reference": "output:w/r/m.png", "kind": "output"}, + {"reference": "asset:bg.png", "kind": "asset"}, + ] + + +@pytest.mark.parametrize( + "role,ok", + [ + ("final", True), + ("intermediate", True), + ("variable:r", True), + ("input", False), + ("keeper", False), + ], +) +def test_schema_role_values(role, ok): + workflow = { + "id": "t", + "steps": [ + { + "name": "s", + "task": {"command": "noop", "arguments": {}}, + "result": {"role": role}, + } + ], + } + errors = validate_data_all(workflow, load_schema("workflow")) + role_errors = [e for e in errors if "role" in (e["path"] or "")] + assert (not role_errors) == ok From e77a3f6776185797b50426b425d534725321884d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 22:44:40 -0500 Subject: [PATCH 005/204] feat(roles): the engine records each step's role and the run's inputs in its manifest Co-Authored-By: Claude Opus 5.5 --- dw/workflow_run.py | 28 +++++++++----- tests/test_run_manifest_roles.py | 66 ++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 10 deletions(-) create mode 100644 tests/test_run_manifest_roles.py diff --git a/dw/workflow_run.py b/dw/workflow_run.py index ca913a05..0b3ce6df 100644 --- a/dw/workflow_run.py +++ b/dw/workflow_run.py @@ -33,6 +33,7 @@ from .pipeline_ownership import allocated_mb, finish_release from .previous_results import StepResults from .realize import realize_workflow +from .roles import annotate_roles, run_inputs from .runs import ( FLAT_LAYOUT, REALIZED_FILE_NAME, @@ -104,6 +105,8 @@ class RunRecord: seed: object = None realized_name: object = None annotations: dict = field(default_factory=_empty_annotations) + # The expanded steps the run executed - what roles and inputs are read from + expanded_steps: list = field(default_factory=list) @dataclass @@ -220,6 +223,18 @@ def write_run_manifest(workflow, record, status=None): """ status = status or record.status annotations = record.annotations + steps = [ + { + **entry, + "files": manifest_relative_files(entry.get("files"), workflow._run_dir), + **_relative_shots(entry, workflow._run_dir), + } + for entry in workflow.manifest + ] + # Roles once the run has ended - a running manifest would show the + # newest step as final until its consumer runs (dw/roles.py) + if status != "running": + steps = annotate_roles(record.expanded_steps, steps) write_manifest( workflow._run_dir, { @@ -252,16 +267,8 @@ def write_run_manifest(workflow, record, status=None): # What did not run, and why - a run says what it did not do # as well as what it did (#122) "elided_steps": workflow._elided_steps, - "steps": [ - { - **entry, - "files": manifest_relative_files( - entry.get("files"), workflow._run_dir - ), - **_relative_shots(entry, workflow._run_dir), - } - for entry in workflow.manifest - ], + "inputs": run_inputs(record.expanded_steps), + "steps": steps, }, ) @@ -620,6 +627,7 @@ def open_run(workflow, prepared, record, run_context): inherits the parent's and never opens its own.""" record.started_at = _now() record.run_id = None + record.expanded_steps = prepared.workflow_def.get("steps") or [] if not workflow._run_dir_inherited: _claim_run_dir(workflow, prepared, record) diff --git a/tests/test_run_manifest_roles.py b/tests/test_run_manifest_roles.py new file mode 100644 index 00000000..6a3b056c --- /dev/null +++ b/tests/test_run_manifest_roles.py @@ -0,0 +1,66 @@ +"""The engine records each step's role and the run's inputs in manifest.json.""" + +import copy +import json +from types import SimpleNamespace + +from dw.workflow_run import RunRecord, write_run_manifest + +from .test_runs import _workflow_definition, fake_pipeline # noqa: F401 + + +def _two_step_definition(): + definition = _workflow_definition() + second = copy.deepcopy(definition["steps"][0]) + second["name"] = "second" + second["pipeline"]["arguments"]["image"] = "previous_result:gen0" + definition["steps"].append(second) + return definition + + +def _stand_in(tmp_path, manifest): + return SimpleNamespace( + manifest=manifest, + _run_dir=str(tmp_path), + _run_version=1, + _elided_steps=[], + name="roles-test", + file_spec=None, + ) + + +def test_terminal_manifest_records_roles_and_inputs(tmp_path, fake_pipeline): # noqa: F811 + from dw.workflow import Workflow + + definition = _two_step_definition() + Workflow(definition, str(tmp_path), "/w/workflows/Roles.json").run({}) + + run_dir = next((tmp_path / "Roles").iterdir()) + manifest = json.loads((run_dir / "manifest.json").read_text()) + assert {e["step"]: e["role"] for e in manifest["steps"]} == { + "gen0": "intermediate", + "second": "final", + } + assert manifest["inputs"] == [] + + +def test_a_terminal_manifest_lists_the_assets_the_steps_read(tmp_path): + steps = [{"name": "a", "x": "asset:cat.png"}, {"name": "b"}] + record = RunRecord(status="completed", expanded_steps=steps) + workflow = _stand_in(tmp_path, [{"step": "a", "files": []}]) + write_run_manifest(workflow, record) + + manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert manifest["inputs"] == [{"reference": "asset:cat.png", "kind": "asset"}] + assert manifest["steps"][0]["role"] == "final" + + +def test_a_running_manifest_carries_inputs_but_no_roles(tmp_path): + steps = [{"name": "a", "x": "asset:cat.png"}, {"name": "b"}] + record = RunRecord(status="running", expanded_steps=steps) + workflow = _stand_in(tmp_path, [{"step": "a", "files": []}]) + write_run_manifest(workflow, record) + + manifest = json.loads((tmp_path / "manifest.json").read_text()) + assert manifest["inputs"] == [{"reference": "asset:cat.png", "kind": "asset"}] + assert all("role" not in e for e in manifest["steps"]) From 40024dd6b5100241c9e8f10db17e2874ac50628a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 22:50:39 -0500 Subject: [PATCH 006/204] feat(roles): gallery entries and metadata carry role and inputs; role filter on the listing and list_gallery Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 2 +- docs/SERVER.md | 6 +- dw/server/api_models.py | 16 ++++ dw/server/outputs.py | 74 +++++++++++++++++++ dw/server/routes/gallery.py | 25 ++++++- dw/server/routes/media.py | 10 +++ dw_mcp/catalog.py | 8 +- dw_mcp/tools_catalog.py | 27 ++++--- tests/conftest.py | 67 +++++++++++++++++ tests/test_api_contract.py | 2 + tests/test_gallery_roles.py | 113 +++++++++++++++++++++++++++++ tests/test_mcp_catalog.py | 18 +++++ ui/src/lib/generated/api-schema.ts | 30 ++++++++ ui/src/lib/generated/openapi.json | 91 ++++++++++++++++++++++- 14 files changed, 469 insertions(+), 20 deletions(-) create mode 100644 tests/test_gallery_roles.py diff --git a/docs/MCP.md b/docs/MCP.md index 820c4831..ccf5b4a3 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -231,7 +231,7 @@ when no single workflow covers it. | `get_health()` | — | Check that the server is alive, and which machine answered: `version`, `device`, whether any model process is resident (`worker_alive`), and the queue depth. The engine runs one job per GPU: `workers` has one entry per card - `device`, `name`, `vram_gb`, `alive`, its own `current_job`, and the worker's `host_memory_rss_mb` while it runs - and the top-level `current_job` is the longest-running job. `worker_alive: false` is the normal idle state on a server that has not run a job since startup or the last memory clear - not a fault - the on-demand worker starts with the next job (#206) | | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server. `runtime` (#222) reports the Python version, torch version and the CUDA version torch was built against, the NVIDIA driver version (when `nvidia-smi` is reachable), and the installed versions of diffusers, transformers, accelerate, bitsandbytes, peft, safetensors and sentencepiece (`null` for one not installed) - for diagnosing an environment mismatch between boxes without shelling in. `caches` lists dw's disk caches (`inductor`, `triton`, `quantized`), each with `name`, `bytes`, `files`, `enabled`, `clearable` and `disk_free`, plus `entry_count` for a cache with entries, and `cache_entries=true` lists them (`id`, `model`, `subfolder`, `component`, `bytes`, `files`, `unreadable`, `created`, `last_hit`; the quantizer and dtype are not in `/api/caches`); it is omitted when the server has no `/api/caches`. There is no MCP tool to clear one - that is `DELETE /api/caches/{name}` or `.../entries/{entry}` ([SERVER.md](SERVER.md)) | | `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first**. Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | -| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | +| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_gallery_metadata(name, envelope=False, workspace=None, interval_seconds=None)` | `name`, `workspace` | Get the metadata embedded in a generated file — or, when `name` is an `asset:` reference, what an *input* asset holds (`source` says which; `job` is null for an asset). Reading an input's duration, frame count, fps and sample rate before a run is how a caller learns the `total_frames`, `fps` and `sample_rate` a workflow expects it to supply: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) and `findings` - each level problem the server measured against `dw/audio_qc.py`'s thresholds (`full_scale`, `near_silent`), with the fix. Only an image (PNG/JPEG/WebP) embeds `metadata` this way — it is always null for audio and video, and `next` then points at `get_job_workflow(job_id)` when `job` is known, or says a kept asset carries no provenance at all when it isn't. `envelope=true` adds `media.envelope` — `rms_dbfs` and `peak_dbfs` one entry per second, or per `interval_seconds` (default 1.0, 0.05 to 3600; 0.1 places a vocal onset; a finer-than-default width that would need over 2400 bins is refused with the interval that fits) — which is what locates something in a track rather than measuring the whole of it. `media.shots` is set on an output joined from shots: one `{name, start_frame, num_frames, start_sample, num_samples}` per shot, as the join measured them (see the run manifest in docs/WORKFLOW_GUIDE.md), else null. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | ### Media diff --git a/docs/SERVER.md b/docs/SERVER.md index 9e403bcb..b3356311 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -533,7 +533,11 @@ The editor's forms come from these; they are just as usable from scripts: `?subfolder=` filter independently (`?version=` too - with `?folder=`, the one run the gallery labels `v4`), and the reply's `folders` and `subfolders` list every distinct value over the whole tree, `''` always a - member of each so root-level files stay selectable + member of each so root-level files stay selectable. Each entry also + carries `role`, `final` or `intermediate` (null for a flat-layout file with + no run), and `GET /api/gallery?role=final` lists only those (repeatable; a + null role counts as final). See `dw/roles.py` for how a role is decided; + the metadata route reports the file's `role` and the run's `inputs` too - `GET /api/gallery/{name:path}/download` — download an output file - `GET /api/builtin-workflows` — the packaged fragments a sub-workflow step names as `builtin:.json` (`dw/workflows/`), which no workflow listing diff --git a/dw/server/api_models.py b/dw/server/api_models.py index ceaf9087..469eb1dd 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -802,6 +802,10 @@ class GalleryFile(ApiModel): duration_seconds: int | float | None = sometimes( "An audio or video file's length, with `media=true`." ) + role: Literal["final", "intermediate"] | None = sometimes( + "`final` for a deliverable, `intermediate` for a file a later step " + "consumed; null under the flat layout or when no manifest lists the file." + ) class GalleryList(ApiModel): @@ -840,6 +844,11 @@ class LevelFinding(ApiModel): says: str +class GalleryInput(ApiModel): + reference: str + kind: Literal["asset", "output"] + + class GalleryMetadata(ApiModel): name: str source: Literal["output", "asset"] @@ -853,6 +862,13 @@ class GalleryMetadata(ApiModel): description="The probe of an audio or video file; null for an image." ) findings: list[LevelFinding] + role: Literal["final", "intermediate"] | None = sometimes( + "The file's role in its run; null when the run does not say." + ) + inputs: list[GalleryInput] | None = sometimes( + "The `asset:` and `output:` references the run read; null for a run " + "that recorded none." + ) class AssetLibraryRoot(ApiModel): diff --git a/dw/server/outputs.py b/dw/server/outputs.py index 6d2835a7..0c164301 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -6,6 +6,7 @@ app's `state` explicitly, so two apps in one process resolve independently. """ +import json import logging import os import shutil @@ -30,8 +31,13 @@ LibraryRoot, library_path, ) +from ..roles import annotate_roles, entry_role from ..runs import ( + MANIFEST_FILE_NAME, + REALIZED_FILE_NAME, + _read_manifest, is_output_reference, + is_run_id, record_run_versions, run_lock_path, run_versions, @@ -48,6 +54,7 @@ validate_asset_reference, validate_path, ) +from ..workflow import Workflow from ..workspace import forget_workspace_usage logger = logging.getLogger("dw") @@ -463,3 +470,70 @@ def delete_run_directory(run_dir, root): logger.info(f"Deleted run directory {os.path.relpath(run_dir, root)}") forget_workspace_usage() return os.path.basename(run_dir) + + +def split_run_name(root, relative_name): + """A gallery file name's run directory and its path inside that run: + 'wf/20261012-000003-dddddddd/final/b.png' -> (/wf/, + 'final/b.png'). None for a flat-layout file, which has no run.""" + parts = [part for part in (relative_name or "").split("/") if part] + for position, segment in enumerate(parts[:-1]): + if is_run_id(segment): + run_dir = os.path.join(root, *parts[: position + 1]) + return run_dir, "/".join(parts[position + 1 :]) + return None + + +# Roles by (run directory, manifest mtime): a listing asks for every file +# of a run, and the manifest it came from has not changed since the last ask +_ROLE_CACHE = {} +_ROLE_CACHE_LIMIT = 512 + + +def _realized_steps(run_dir): + """The realized workflow's expanded steps, or None when unreadable.""" + path = os.path.join(run_dir, REALIZED_FILE_NAME) + try: + with open(path) as handle: + definition = json.load(handle) + return Workflow(definition, None, path).expanded_definition().get("steps") or [] + except Exception: # an unreadable or unexpandable record leaves roles unknown + return None + + +def run_roles(run_dir): + """Each file of a run, by path relative to the run directory ('/' + separated), mapped to its role. Recorded roles win; a run that predates + them is inferred from its realized workflow. {} when neither is + readable.""" + try: + key = ( + run_dir, + os.stat(os.path.join(run_dir, MANIFEST_FILE_NAME)).st_mtime_ns, + ) + except OSError: + return {} + if key in _ROLE_CACHE: + return _ROLE_CACHE[key] + manifest = _read_manifest(run_dir) or {} + steps = [e for e in manifest.get("steps") or [] if isinstance(e, dict)] + if steps and any(entry_role(e) is None for e in steps): + expanded = _realized_steps(run_dir) + if expanded is not None: + steps = annotate_roles(expanded, steps) + roles = {} + for entry in steps: + role = entry_role(entry) + if role: + for name in entry.get("files") or []: + roles[str(name).replace(os.sep, "/")] = role + if len(_ROLE_CACHE) >= _ROLE_CACHE_LIMIT: + _ROLE_CACHE.pop(next(iter(_ROLE_CACHE))) + _ROLE_CACHE[key] = roles + return roles + + +def run_inputs(run_dir): + """The inputs a run's manifest recorded, or None when it has none.""" + inputs = (_read_manifest(run_dir) or {}).get("inputs") + return inputs if isinstance(inputs, list) else None diff --git a/dw/server/routes/gallery.py b/dw/server/routes/gallery.py index eb7778f4..233bf339 100644 --- a/dw/server/routes/gallery.py +++ b/dw/server/routes/gallery.py @@ -12,7 +12,7 @@ from typing import Optional from urllib.parse import quote -from fastapi import APIRouter, Depends, Request +from fastapi import APIRouter, Depends, HTTPException, Query, Request from fastapi.responses import JSONResponse from ..api_models import ( @@ -20,6 +20,7 @@ OutputDeleted, ) from ...media import probe_media +from ...roles import ROLES from ...runs import ( MANIFEST_FILE_NAME, REALIZED_FILE_NAME, @@ -39,7 +40,9 @@ archive_selection, iter_gallery_files, resolve_output_file, + run_roles, served_url, + split_run_name, strip_output_prefix, ) @@ -111,6 +114,9 @@ def _version(folder, run_id): "mtime": stat.st_mtime, "label": label, } + # None under the flat layout, and for a file no manifest lists + split = split_run_name(root, relative_name) if run_id else None + entry["role"] = run_roles(split[0]).get(split[1]) if split else None absolute_url = absolute_served_url(output_path, ws, int(stat.st_mtime)) if absolute_url is not None: entry["absolute_url"] = absolute_url @@ -171,6 +177,7 @@ def gallery( only_orphans: bool = False, version: Optional[int] = None, media: bool = False, + role: Optional[list[str]] = Query(None), ws: Workspace = Depends(selected_workspace), ): """A page of media files in the output directory, newest first. @@ -205,7 +212,18 @@ def gallery( by, since size and mtime are misleading proxies for length (#356). Off by default and bounded by `limit`: only the page actually returned is probed, not the whole listing, so the cost of asking - stays proportional to the page size rather than the library size.""" + stays proportional to the page size rather than the library size. + + Each entry carries `role`, `final` or `intermediate` (null for a + flat-layout file or one no manifest lists). `role` is a repeatable + filter; a null role counts as final. Omitted, every file is listed.""" + if role: + unknown = sorted(set(role) - set(ROLES)) + if unknown: + raise HTTPException( + status_code=400, + detail=f"unknown role {unknown}; roles are {list(ROLES)}", + ) if only_orphans: entries = _orphan_entries(ws.outputs) offset = max(0, offset) @@ -231,6 +249,9 @@ def gallery( entries = [e for e in entries if e["subfolder"] == subfolder] if version is not None: entries = [e for e in entries if e["version"] == version] + if role: + wanted = set(role) + entries = [e for e in entries if (e.get("role") or "final") in wanted] offset = max(0, offset) limit = max(0, limit) page = entries[offset : offset + limit] diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index b42b68d8..562205b3 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -57,6 +57,9 @@ asset_file, job_provenance, resolve_output_file, + run_inputs, + run_roles, + split_run_name, strip_output_prefix, ) @@ -116,6 +119,7 @@ def gallery_metadata( pass `get_job_workflow`, since a pruned run leaves no `workflow.json` of its own to reopen.""" run_id, version = "", None + role, inputs = None, None name = strip_output_prefix(name) if is_asset_reference(name): path = asset_file(request.app.state, name, ws) @@ -133,6 +137,10 @@ def gallery_metadata( # call, so it confirms the right file was reached rather than # sending the caller back to the listing job, run_id, version = job_provenance(request.app.state, name, ws) + split = split_run_name(ws.outputs, name) + if split: + role = run_roles(split[0]).get(split[1]) + inputs = run_inputs(split[0]) metadata = read_embedded_metadata(path) extension = os.path.splitext(path)[1].lower() try: @@ -161,6 +169,8 @@ def gallery_metadata( "version": version, "media": media, "findings": level_findings(media), + "role": role, + "inputs": inputs, } diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index c6a2d184..232eb4c2 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -235,6 +235,7 @@ def list_gallery( folder=None, version=None, media=False, + role=None, ): """Generated media in the output directory, newest first. `subfolder` narrows to one in-run subfolder ('final', 'intermediate', '' for files @@ -267,8 +268,13 @@ def list_gallery( `media=True` adds `duration_seconds` to each audio/video entry in the page returned, probed the way `get_gallery_metadata` measures a file - enough to pick between two takes without one metadata call per - candidate. Off by default; a plain call carries no `duration_seconds`.""" + candidate. Off by default; a plain call carries no `duration_seconds`. + + Each file carries `role` (`final`, `intermediate`, or null with no run); + `role=[...]` keeps only those, null counting as final.""" params = {"limit": limit} + if role: + params["role"] = list(role) if subfolder is not None: params["subfolder"] = subfolder if folder is not None: diff --git a/dw_mcp/tools_catalog.py b/dw_mcp/tools_catalog.py index eceedfdf..774de83c 100644 --- a/dw_mcp/tools_catalog.py +++ b/dw_mcp/tools_catalog.py @@ -261,6 +261,7 @@ def list_gallery( folder: str | None = None, version: int | None = None, media: bool = False, + role: list[Literal["final", "intermediate"]] | None = None, ) -> dict: """List generated output files, newest first. A name is //, where may itself sit in a @@ -268,12 +269,12 @@ def list_gallery( `get_output_image`, `get_output_text`, `download_output`, `keep_output` and `delete_output` all take, and the form an `output:` reference in a later workflow is built from. Each entry - carries `folder` (the workflow) and `subfolder` (the part of the run: - by convention `final` is the deliverable and `intermediate` the - scratch work, '' when the step chose none); `subfolder=` filters on - the latter, so `subfolder="final"` is "what did these runs - deliver". Each entry's `url` is already scoped to its workspace; - use it as given rather than composing one from the name. + carries `folder` (the workflow), `subfolder` (the step's chosen + subfolder, '' for none; `subfolder=` filters on it) and `role`: + `final` is a deliverable, `intermediate` a file a later step + consumed, null with no run. `role=` filters; omit it for every + file. Each entry's `url` is already scoped to its workspace; use + it as given. Entries also carry `run_id` and `version`, the run's stable ordinal (the web UI shows `v5`) - quote the version to a person. `folder=` @@ -283,14 +284,11 @@ def list_gallery( `only_orphans=True` inverts the call: instead of files, it returns run directories holding nothing but their own bookkeeping (manifest.json, workflow.json, job.json) as `runs`, each - `{name, mtime}` - a run whose output was deleted before - `delete_output` could remove it by name, or one that failed before - writing anything. `subfolder` does not apply in this mode. `name` is - exactly what `delete_output` accepts, so clearing the backlog is - list, then delete each name. A run that wrote any file at - all - a text-shape prompt, a utility's side output - is not listed; - this call only lists, so deciding whether a listed entry is actually - junk before calling `delete_output` on it is still yours to make. + `{name, mtime}` - a run whose output was deleted, or one that failed + before writing anything. `subfolder` does not apply in this mode. + `name` is exactly what `delete_output` accepts. A run that wrote any + file is not listed; whether a listed entry is junk is still yours + to decide. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` @@ -309,6 +307,7 @@ def list_gallery( folder=folder, version=version, media=media, + role=role, ) def get_gallery_metadata( diff --git a/tests/conftest.py b/tests/conftest.py index 8fcb8dfc..07150124 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,6 +5,7 @@ _os.environ.setdefault("DW_STRICT_RESPONSES", "1") import gc +import json import pytest import os import tempfile @@ -190,3 +191,69 @@ def __call__(self, **kwargs): return type("MockOutput", (), {"images": ["mock_image"]})() return MockPipeline() + + +@pytest.fixture +def write_run(): + """Build a fake run directory: `write_run(root, identity, run_id, steps, + realized=None, files=(), inputs=None)` writes the media `files` (paths + relative to the run), a manifest.json carrying `steps` and, when given, + `inputs`, and a workflow.json from `realized`. Returns the run directory.""" + + def build(root, identity, run_id, steps, realized=None, files=(), inputs=None): + run_dir = os.path.join(root, identity, run_id) + os.makedirs(run_dir, exist_ok=True) + for name in files: + path = os.path.join(run_dir, name) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as handle: + handle.write(b"\x89PNG\r\n\x1a\n") + manifest = { + "run_id": run_id, + "version": 1, + "status": "succeeded", + "steps": steps, + } + if inputs is not None: + manifest["inputs"] = inputs + with open(os.path.join(run_dir, "manifest.json"), "w") as handle: + json.dump(manifest, handle) + if realized is not None: + with open(os.path.join(run_dir, "workflow.json"), "w") as handle: + json.dump(realized, handle) + return run_dir + + return build + + +@pytest.fixture +def output_root(tmp_path): + """The output directory `gallery_client` serves.""" + return str(tmp_path / "outputs") + + +@pytest.fixture +def gallery_client(tmp_path, output_root): + """A TestClient on an app with no jobs, over `output_root`.""" + from fastapi.testclient import TestClient + + from dw.server.app import create_app + from dw.server.jobs import JobManager + + from .test_server import ScriptedWorkerManager, success_script + + os.makedirs(output_root, exist_ok=True) + for sub in ("workflows", "prompts"): + os.makedirs(tmp_path / sub, exist_ok=True) + manager = JobManager( + output_root, + worker_manager=ScriptedWorkerManager(success_script), + history_path=str(tmp_path / "jobs.sqlite"), + ) + app = create_app( + workflow_dir=str(tmp_path / "workflows"), + output_dir=output_root, + job_manager=manager, + prompt_dir=str(tmp_path / "prompts"), + ) + return TestClient(app, base_url="http://localhost") diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py index 8cd912c3..0e33ab6c 100644 --- a/tests/test_api_contract.py +++ b/tests/test_api_contract.py @@ -533,6 +533,8 @@ def test_gallery_and_asset_answers_keep_their_keys(server, tmp_path): "version", "media", "findings", + "role", + "inputs", } assert metadata["job"] is None and metadata["findings"] == [] assert set(assets) == {"workspace", "libraries", "assets", "folders", "shadowed"} diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py new file mode 100644 index 00000000..51417615 --- /dev/null +++ b/tests/test_gallery_roles.py @@ -0,0 +1,113 @@ +"""The gallery says which files of a run are its deliverable and which are +scratch: each entry's `role`, the listing's `role` filter, and the metadata's +`role` and `inputs` (dw/roles.py decides the roles).""" + +import os + +from dw.server.outputs import run_roles, split_run_name + + +def test_recorded_roles_reach_the_listing(gallery_client, output_root, write_run): + write_run( + output_root, + "wf", + "20261012-000000-aaaaaaaa", + [ + {"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["b.png"], "role": "final"}, + ], + files=["a.png", "b.png"], + ) + files = {f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"]} + assert files["a.png"]["role"] == "intermediate" + assert files["b.png"]["role"] == "final" + + +def test_older_run_roles_inferred_from_realized_workflow( + gallery_client, output_root, write_run +): + realized = { + "id": "wf", + "steps": [ + {"name": "a", "task": {"command": "noop", "arguments": {}}}, + { + "name": "b", + "task": { + "command": "noop", + "arguments": {"image": "previous_result:a"}, + }, + }, + ], + } + write_run( + output_root, + "wf", + "20261012-000001-bbbbbbbb", + [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}], + realized=realized, + files=["a.png", "b.png"], + ) + files = {f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"]} + assert files["a.png"]["role"] == "intermediate" + assert files["b.png"]["role"] == "final" + + +def test_role_filter(gallery_client, output_root, write_run): + write_run( + output_root, + "wf", + "20261012-000002-cccccccc", + [ + {"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["final/b.png"], "role": "final"}, + ], + files=["a.png", "final/b.png"], + ) + finals = gallery_client.get("/api/gallery", params={"role": "final"}).json()[ + "files" + ] + assert [f["label"] for f in finals] == ["b.png"] + both = gallery_client.get( + "/api/gallery", params={"role": ["final", "intermediate"]} + ).json()["files"] + assert len(both) == 2 + assert ( + gallery_client.get("/api/gallery", params={"role": "bogus"}).status_code == 400 + ) + + +def test_flat_file_has_null_role_and_passes_final_filter(gallery_client, output_root): + os.makedirs(os.path.join(output_root, "loose"), exist_ok=True) + with open(os.path.join(output_root, "loose", "x.png"), "wb") as handle: + handle.write(b"\x89PNG\r\n\x1a\n") + entries = gallery_client.get("/api/gallery", params={"role": "final"}).json()[ + "files" + ] + loose = [f for f in entries if f["label"] == "x.png"] + assert loose and loose[0].get("role") is None + + +def test_metadata_carries_role_and_inputs(gallery_client, output_root, write_run): + write_run( + output_root, + "wf", + "20261012-000003-dddddddd", + [{"step": "b", "files": ["b.png"], "role": "final"}], + files=["b.png"], + inputs=[{"reference": "asset:iris.png", "kind": "asset"}], + ) + meta = gallery_client.get( + "/api/gallery/wf/20261012-000003-dddddddd/b.png/metadata" + ).json() + assert meta["role"] == "final" + assert meta["inputs"] == [{"reference": "asset:iris.png", "kind": "asset"}] + + +def test_split_run_name(tmp_path): + root = str(tmp_path) + assert split_run_name(root, "wf/20261012-000003-dddddddd/final/b.png") == ( + os.path.join(root, "wf", "20261012-000003-dddddddd"), + "final/b.png", + ) + assert split_run_name(root, "loose/x.png") is None + assert run_roles(os.path.join(root, "missing")) == {} diff --git a/tests/test_mcp_catalog.py b/tests/test_mcp_catalog.py index c84fab86..b75defb6 100644 --- a/tests/test_mcp_catalog.py +++ b/tests/test_mcp_catalog.py @@ -412,3 +412,21 @@ def test_full_get_workflow_without_history_has_no_observed_key(): ) assert "observed" not in catalog.get_workflow(client, "w") + + +def test_list_gallery_forwards_role_as_a_repeated_parameter(): + seen = {} + + def handler(request): + seen["roles"] = request.url.params.get_list("role") + return httpx.Response(200, json={"files": []}) + + client = DwClient(transport=httpx.MockTransport(handler)) + catalog.list_gallery(client) + assert seen["roles"] == [] + + catalog.list_gallery(client, role=["final"]) + assert seen["roles"] == ["final"] + + catalog.list_gallery(client, role=["final", "intermediate"]) + assert seen["roles"] == ["final", "intermediate"] diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 835f49c5..6c1d9701 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -339,6 +339,10 @@ export interface paths { * Off by default and bounded by `limit`: only the page actually * returned is probed, not the whole listing, so the cost of asking * stays proportional to the page size rather than the library size. + * + * Each entry carries `role`, `final` or `intermediate` (null for a + * flat-layout file or one no manifest lists). `role` is a repeatable + * filter; a null role counts as final. Omitted, every file is listed. */ get: operations["gallery_api_gallery_get"]; put?: never; @@ -2182,6 +2186,11 @@ export interface components { mtime: number; /** Name */ name: string; + /** + * Role + * @description `final` for a deliverable, `intermediate` for a file a later step consumed; null under the flat layout or when no manifest lists the file. + */ + role?: ("final" | "intermediate") | null; /** * Run Id * @description The run that wrote the file, `''` under the flat layout. @@ -2202,6 +2211,16 @@ export interface components { */ version: number | null; }; + /** GalleryInput */ + GalleryInput: { + /** + * Kind + * @enum {string} + */ + kind: "asset" | "output"; + /** Reference */ + reference: string; + }; /** GalleryList */ GalleryList: { /** @@ -2226,6 +2245,11 @@ export interface components { GalleryMetadata: { /** Findings */ findings: components["schemas"]["LevelFinding"][]; + /** + * Inputs + * @description The `asset:` and `output:` references the run read; null for a run that recorded none. + */ + inputs?: components["schemas"]["GalleryInput"][] | null; job: components["schemas"]["JobRef"] | null; /** * Media @@ -2243,6 +2267,11 @@ export interface components { } | null; /** Name */ name: string; + /** + * Role + * @description The file's role in its run; null when the run does not say. + */ + role?: ("final" | "intermediate") | null; /** Run Id */ run_id: string; /** @@ -3937,6 +3966,7 @@ export interface operations { only_orphans?: boolean; version?: number | null; media?: boolean; + role?: string[] | null; workspace?: string | null; }; header?: never; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 95b66e1e..9a1d05a1 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -1130,6 +1130,22 @@ "title": "Name", "type": "string" }, + "role": { + "anyOf": [ + { + "enum": [ + "final", + "intermediate" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "`final` for a deliverable, `intermediate` for a file a later step consumed; null under the flat layout or when no manifest lists the file.", + "title": "Role" + }, "run_id": { "description": "The run that wrote the file, `''` under the flat layout.", "title": "Run Id", @@ -1176,6 +1192,29 @@ "title": "GalleryFile", "type": "object" }, + "GalleryInput": { + "additionalProperties": false, + "properties": { + "kind": { + "enum": [ + "asset", + "output" + ], + "title": "Kind", + "type": "string" + }, + "reference": { + "title": "Reference", + "type": "string" + } + }, + "required": [ + "reference", + "kind" + ], + "title": "GalleryInput", + "type": "object" + }, "GalleryList": { "additionalProperties": false, "properties": { @@ -1240,6 +1279,21 @@ "title": "Findings", "type": "array" }, + "inputs": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/GalleryInput" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The `asset:` and `output:` references the run read; null for a run that recorded none.", + "title": "Inputs" + }, "job": { "anyOf": [ { @@ -1280,6 +1334,22 @@ "title": "Name", "type": "string" }, + "role": { + "anyOf": [ + { + "enum": [ + "final", + "intermediate" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "The file's role in its run; null when the run does not say.", + "title": "Role" + }, "run_id": { "title": "Run Id", "type": "string" @@ -5266,7 +5336,7 @@ }, "/api/gallery": { "get": { - "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.", + "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists). `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", "operationId": "gallery_api_gallery_get", "parameters": [ { @@ -5357,6 +5427,25 @@ "type": "boolean" } }, + { + "in": "query", + "name": "role", + "required": false, + "schema": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Role" + } + }, { "in": "query", "name": "workspace", From af8ae46e1b744ca0a2932e385b8be26febe27bca Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 22:54:20 -0500 Subject: [PATCH 007/204] fix(roles): read run roles and inputs from the validated output path Co-Authored-By: Claude Opus 5.5 --- dw/server/routes/media.py | 6 +++++- tests/test_gallery_roles.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index 562205b3..716dea00 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -137,7 +137,11 @@ def gallery_metadata( # call, so it confirms the right file was reached rather than # sending the caller back to the listing job, run_id, version = job_provenance(request.app.state, name, ws) - split = split_run_name(ws.outputs, name) + # Split the validated path, not the raw route parameter, so the + # run directory the manifest is read from is a validator's output + split = split_run_name( + ws.outputs, os.path.relpath(path, ws.outputs).replace(os.sep, "/") + ) if split: role = run_roles(split[0]).get(split[1]) inputs = run_inputs(split[0]) diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 51417615..edc90897 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -111,3 +111,21 @@ def test_split_run_name(tmp_path): ) assert split_run_name(root, "loose/x.png") is None assert run_roles(os.path.join(root, "missing")) == {} + + +def test_metadata_for_a_subfolder_file_of_a_nested_identity_run( + gallery_client, output_root, write_run +): + write_run( + output_root, + "series/ep1", + "20261012-000004-eeeeeeee", + [{"step": "b", "files": ["final/b.png"], "role": "final"}], + files=["final/b.png"], + inputs=[{"reference": "output:wf/v1/a.png", "kind": "output"}], + ) + meta = gallery_client.get( + "/api/gallery/series/ep1/20261012-000004-eeeeeeee/final/b.png/metadata" + ).json() + assert meta["role"] == "final" + assert meta["inputs"] == [{"reference": "output:wf/v1/a.png", "kind": "output"}] From 517fb8708cdc3d1e1539e183a2438589ab2d73b2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 22:59:31 -0500 Subject: [PATCH 008/204] feat(ui): roles lead the gallery - role chips, tile tags, grouped job results, made-from list Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/api.ts | 9 +- ui/src/lib/assets/AssetDetail.svelte | 25 ++++ ui/src/lib/assets/AssetDetail.test.ts | 56 +++++++ ui/src/lib/gallery/GalleryDetail.svelte | 152 ++++++++++++++----- ui/src/lib/gallery/GalleryDetail.test.ts | 70 +++++++++ ui/src/lib/gallery/RoleTag.svelte | 38 +++++ ui/src/lib/gallery/RoleTag.test.ts | 22 +++ ui/src/lib/job/JobResults.svelte | 83 ++++++++-- ui/src/lib/pages/AssetsPage.test.ts | 1 + ui/src/lib/pages/GalleryPage.svelte | 131 +++++++++------- ui/src/lib/pages/GalleryPage.test.ts | 183 +++++++++-------------- ui/src/lib/pages/JobPage.test.ts | 52 +++++++ ui/src/lib/results.test.ts | 53 ++++++- ui/src/lib/results.ts | 25 ++++ ui/src/lib/types.ts | 3 + 15 files changed, 683 insertions(+), 220 deletions(-) create mode 100644 ui/src/lib/assets/AssetDetail.test.ts create mode 100644 ui/src/lib/gallery/GalleryDetail.test.ts create mode 100644 ui/src/lib/gallery/RoleTag.svelte create mode 100644 ui/src/lib/gallery/RoleTag.test.ts diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index bdc551e2..eeac76df 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -276,7 +276,14 @@ export const api = { ), // Loads the whole gallery in one request, like listWorkflows/listPrompts - // the limit just needs to exceed any real output directory's file count - gallery: () => request('/api/gallery?limit=100000'), + // `roles` narrows to those roles server-side (repeated `role` parameters; + // a null role counts as final there). The page filters on the client, so + // it asks for none + gallery: (roles?: string[]) => { + const params = new URLSearchParams({ limit: '100000' }) + for (const role of roles ?? []) params.append('role', role) + return request(`/api/gallery?${params}`) + }, // `workspace`, when given, names the file's own workspace and wins over // whatever is currently selected in the picker - mirrors `outputUrl`, since // a job page must read its own files from where they were written diff --git a/ui/src/lib/assets/AssetDetail.svelte b/ui/src/lib/assets/AssetDetail.svelte index 49d5cc68..411d4ace 100644 --- a/ui/src/lib/assets/AssetDetail.svelte +++ b/ui/src/lib/assets/AssetDetail.svelte @@ -1,6 +1,7 @@
@@ -63,6 +85,9 @@ aria-label="close details">
+ {#if keptFrom} +

{keptFrom}

+ {/if} {#if asset.used_by?.length}

used by diff --git a/ui/src/lib/assets/AssetDetail.test.ts b/ui/src/lib/assets/AssetDetail.test.ts new file mode 100644 index 00000000..f59df2eb --- /dev/null +++ b/ui/src/lib/assets/AssetDetail.test.ts @@ -0,0 +1,56 @@ +import { cleanup, render, screen, waitFor } from '@testing-library/svelte' +import { afterEach, expect, it, vi } from 'vitest' +import AssetDetail from './AssetDetail.svelte' +import type { AssetFile } from '../types' + +const provenance = vi.hoisted(() => ({ + response: null as Record | null, +})) +const galleryMetadata = vi.hoisted(() => + vi.fn<(name: string) => Promise | null>>(() => + Promise.resolve(provenance.response), + ), +) +vi.mock('../api', () => ({ + api: { galleryMetadata: (name: string) => galleryMetadata(name) }, +})) + +afterEach(() => { + cleanup() + galleryMetadata.mockClear() +}) + +const asset: AssetFile = { + name: 'face.png', + folder: '', + reference: 'asset:face.png', + kind: 'image', + size: 10, + mtime: 1, + url: '/assets/face.png', + origin: 'workspace', + writable: true, +} + +const props = { asset, busy: false, onremove: () => {}, onclose: () => {} } + +it('says which workflow and version a kept asset came from', async () => { + provenance.response = { + job: { id: 'j1', status: 'succeeded', workflow: 'portrait' }, + version: 3, + } + render(AssetDetail, props) + await waitFor(() => expect(screen.getByText(/kept from/)).toBeTruthy()) + expect(screen.getByText(/kept from/).textContent?.trim()).toBe( + 'kept from portrait v3', + ) + expect(galleryMetadata).toHaveBeenCalledWith('asset:face.png') +}) + +it('says nothing for an asset with no recorded provenance', async () => { + provenance.response = { job: null, version: null } + render(AssetDetail, props) + await waitFor(() => expect(galleryMetadata).toHaveBeenCalled()) + await new Promise((r) => setTimeout(r, 0)) + expect(screen.queryByText(/kept from/)).toBeNull() +}) diff --git a/ui/src/lib/gallery/GalleryDetail.svelte b/ui/src/lib/gallery/GalleryDetail.svelte index f170f9b8..a717b9d1 100644 --- a/ui/src/lib/gallery/GalleryDetail.svelte +++ b/ui/src/lib/gallery/GalleryDetail.svelte @@ -6,7 +6,12 @@ import { notify } from '../toast' import { confirmDialog } from '../confirm.svelte' import { describeOutputMeta } from '../outputMeta' - import type { GalleryFile } from '../types' + import type { GalleryFile, GalleryInput, OutputRole } from '../types' + import RoleTag from './RoleTag.svelte' + import { ASSET, OUTPUT, referenceName } from '../references' + import { outputUrl } from '../apiUrls' + import { wsHref } from '../routes' + import { workspace } from '../workspace.svelte' import { formatBytes, formatMtime } from '../format' // The detail panel under the grid for the one file picked. The page owns @@ -17,6 +22,8 @@ metadata, metadataLoading, sourceJob, + role = null, + inputs = [], onremove, onclose, }: { @@ -24,6 +31,10 @@ metadata: Record | null metadataLoading: boolean sourceJob: { id: string; status: string } | null + /** The file's role in its run; null when the run does not say */ + role?: OutputRole | null + /** The assets and outputs the run read */ + inputs?: GalleryInput[] onremove: () => void onclose: () => void } = $props() @@ -93,6 +104,21 @@ const negativePrompt = $derived(described.negativePrompt) const seed = $derived(described.seed) + // Where a run input leads: an output opens its file, an asset the library + const inputName = (input: GalleryInput) => + referenceName(input.reference, input.kind === 'asset' ? ASSET : OUTPUT) ?? + input.reference + const inputHref = (input: GalleryInput) => + input.kind === 'asset' + ? wsHref(workspace.current, 'assets') + : outputUrl(inputName(input)) + // The folder the file sits in, as the engine wrote it + const location = $derived( + file.name.includes('/') + ? file.name.slice(0, file.name.lastIndexOf('/')) + : '', + ) + function openAsWorkflow() { if (!embeddedWorkflow) return // Pin the run's seed into the definition so reopening reproduces this @@ -170,49 +196,84 @@ {:else if file.kind === 'text'} 

{textBody ?? 'loading…'}
{/if} - {#if metadata} + {#if metadata || role || inputs.length}
- {#if metadata.step_name}
- step - {metadata.step_name} -
{/if} - {#if metadata.model_name}
- model - {metadata.model_name} -
{/if} - {#if seed !== undefined} -
- seed {seed} -
- {/if} - {#if prompt} -
- prompt -

{prompt}

-
- {/if} - {#if negativePrompt} -
- negative prompt -

{negativePrompt}

-
+
+ {#if metadata} +
+ Settings + {#if metadata.step_name}
+ step + {metadata.step_name} +
{/if} + {#if metadata.model_name}
+ model + {metadata.model_name} +
{/if} + {#if seed !== undefined} +
+ seed {seed} +
+ {/if} + {#if prompt} +
+ prompt +

{prompt}

+
+ {/if} + {#if negativePrompt} +
+ negative prompt +

{negativePrompt}

+
+ {/if} + {#if sourceJob} + + {/if} + {#if embeddedWorkflow} +
+ workflow + {embeddedWorkflow.id} +
+ {:else} +
+ no embedded workflow - enable embed_metadata in the step's + result +
+ {/if} +
{/if} - {#if sourceJob} -
- job - - {sourceJob.id} - + {#if inputs.length} +
+ Made from +
    + {#each inputs as input (input.kind + input.reference)} +
  • + {inputName(input)} + {input.kind} +
    • + {/each} +
{/if} - {#if embeddedWorkflow} -
workflow {embeddedWorkflow.id}
- {:else} -
- no embedded workflow - enable embed_metadata in the step's result + {#if location} +
+ Location + {location}/
{/if}
@@ -276,6 +337,17 @@ .meta .muted { margin-right: 0.4rem; } + .settings summary { + cursor: pointer; + color: var(--muted); + } + .madefrom ul { + margin: 0.15rem 0 0; + padding-left: 1.1rem; + } + .location code { + overflow-wrap: anywhere; + } .prompt p { margin: 0.15rem 0 0; white-space: pre-wrap; diff --git a/ui/src/lib/gallery/GalleryDetail.test.ts b/ui/src/lib/gallery/GalleryDetail.test.ts new file mode 100644 index 00000000..cbd49515 --- /dev/null +++ b/ui/src/lib/gallery/GalleryDetail.test.ts @@ -0,0 +1,70 @@ +import { cleanup, render } from '@testing-library/svelte' +import { afterEach, expect, it, vi } from 'vitest' +import GalleryDetail from './GalleryDetail.svelte' +import type { GalleryFile } from '../types' + +vi.mock('../api', () => ({ + api: { outputDownloadUrl: (name: string) => `/download/${name}` }, +})) + +afterEach(cleanup) + +const file: GalleryFile = { + name: 'wf/run-1/final/a.png', + folder: 'wf', + subfolder: 'final', + run_id: 'run-1', + version: 1, + url: '/outputs/wf/run-1/final/a.png', + kind: 'image', + size: 10, + mtime: 1, + label: 'a.png', + role: 'final', +} + +const props = { + file, + metadata: { model_name: 'org/model', seed: 7 }, + metadataLoading: false, + sourceJob: null, + role: 'final' as const, + inputs: [ + { reference: 'asset:face.png', kind: 'asset' as const }, + { reference: 'output:wf/run-0/b.png', kind: 'output' as const }, + ], + onremove: () => {}, + onclose: () => {}, +} + +it('orders the panel role, Settings, Made from, Location', () => { + const { container } = render(GalleryDetail, props) + const meta = container.querySelector('.meta') as HTMLElement + const order = [...meta.children].map( + (el) => + (el.querySelector('.role-tag') && 'role') || + (el.matches('details') && el.querySelector('summary')?.textContent) || + el.querySelector('.muted')?.textContent, + ) + expect(order).toEqual(['role', 'Settings', 'Made from', 'Location']) + // Settings starts collapsed + expect((meta.querySelector('details') as HTMLDetailsElement).open).toBe(false) + expect(meta.querySelector('.location code')?.textContent).toBe( + 'wf/run-1/final/', + ) +}) + +it('lists each run input as a link', () => { + const { container } = render(GalleryDetail, props) + const links = [...container.querySelectorAll('.madefrom a')] + expect(links.map((a) => a.textContent)).toEqual([ + 'face.png', + 'wf/run-0/b.png', + ]) + expect(links[1].getAttribute('href')).toContain('wf/run-0/b.png') +}) + +it('omits Made from when the run read nothing', () => { + const { container } = render(GalleryDetail, { ...props, inputs: [] }) + expect(container.querySelector('.madefrom')).toBeNull() +}) diff --git a/ui/src/lib/gallery/RoleTag.svelte b/ui/src/lib/gallery/RoleTag.svelte new file mode 100644 index 00000000..c7ff1b02 --- /dev/null +++ b/ui/src/lib/gallery/RoleTag.svelte @@ -0,0 +1,38 @@ + + +{#if role} + {text[role]} +{/if} + + diff --git a/ui/src/lib/gallery/RoleTag.test.ts b/ui/src/lib/gallery/RoleTag.test.ts new file mode 100644 index 00000000..df5eb882 --- /dev/null +++ b/ui/src/lib/gallery/RoleTag.test.ts @@ -0,0 +1,22 @@ +import { cleanup, render, screen } from '@testing-library/svelte' +import { afterEach, describe, expect, it } from 'vitest' +import RoleTag from './RoleTag.svelte' + +afterEach(cleanup) + +describe('RoleTag', () => { + it.each([ + ['final', 'Final'], + ['intermediate', 'Intermediate'], + ['input', 'Input'], + ] as const)('renders %s', (role, text) => { + render(RoleTag, { role }) + const tag = screen.getByText(text) + expect(tag.dataset.role).toBe(role) + }) + + it('renders nothing for a null role', () => { + const { container } = render(RoleTag, { role: null }) + expect(container.textContent?.trim()).toBe('') + }) +}) diff --git a/ui/src/lib/job/JobResults.svelte b/ui/src/lib/job/JobResults.svelte index b1ed62ee..a320e2e1 100644 --- a/ui/src/lib/job/JobResults.svelte +++ b/ui/src/lib/job/JobResults.svelte @@ -3,12 +3,15 @@ import { api, outputUrl } from '../api' import { describeOutputMeta } from '../outputMeta' import { + sectionByRole, sectionBySubfolder, type StepGroup, type UnsavedStep, } from '../results' import DownloadLink from '../DownloadLink.svelte' - import type { JobDetail } from '../types' + import type { GalleryInput, JobDetail } from '../types' + import { ASSET, OUTPUT, referenceName } from '../references' + import { wsHref } from '../routes' // The Results panel's body: each output beside what made it, sectioned by // subfolder and grouped by step, then the steps that wrote nothing. The @@ -34,6 +37,7 @@ // seed. Images only: the metadata route decodes an audio or video file // whole to probe it, and nothing it would report is shown here. A key // present with null is a lookup already in flight or failed + let runInputs = $state([]) let fileMeta = $state | null>>({}) $effect(() => { const workspace = job.workspace @@ -51,6 +55,8 @@ .galleryMetadata(file, workspace) .then((r) => { fileMeta[file] = r.metadata + // The run's inputs are the same whichever output asks + if (r.inputs?.length && !runInputs.length) runInputs = r.inputs }) .catch(() => {}) } @@ -66,6 +72,12 @@ // Sections by result.subfolder - one root section, no heading, when no // step chose one, so an older run renders as it always did const sections = $derived(sectionBySubfolder(fileGroups)) + // By role when the manifest states one: Final, Intermediate, Made from. + // Null for a run that predates roles, which keeps the subfolder layout + const byRole = $derived(sectionByRole(fileGroups)) + const inputName = (input: GalleryInput) => + referenceName(input.reference, input.kind === 'asset' ? ASSET : OUTPUT) ?? + input.reference const sectioned = $derived( sections.length > 1 || sections[0].subfolder !== '', ) @@ -83,15 +95,48 @@ Use New seed for a different image.{/if}

{/if} -{#each sections as section (section.subfolder)} - {#if sectioned} -

- {section.subfolder === '' ? '(run root)' : `${section.subfolder}/`} -

- {/if} - {#each section.groups as group (group.step)} +{#if byRole} +
+

Final

+ {@render stepGroups(byRole.final, true)} + {#if byRole.intermediate.length} +
+ Intermediate + {@render stepGroups(byRole.intermediate, true)} +
+ {/if} + {#if runInputs.length} +

Made from

+
    + {#each runInputs as input (input.kind + input.reference)} +
  • + {inputName(input)} + {input.kind} +
    • + {/each} +
+ {/if} +
+{:else} + {#each sections as section (section.subfolder)} + {#if sectioned} +

+ {section.subfolder === '' ? '(run root)' : `${section.subfolder}/`} +

+ {/if} + {@render stepGroups(section.groups, sectioned)} + {/each} +{/if} +{#snippet stepGroups(groups: StepGroup[], nested: boolean)} + {#each groups as group (group.step)} {#if fileGroups.length > 1} - + {group.step} {#if group.reused && !allReused} {/each} {/each} -{/each} +{/snippet} {#if unsaved.length}

Steps that wrote nothing

    @@ -186,6 +231,24 @@ {/if} diff --git a/ui/src/lib/ui/NoteEditor.test.ts b/ui/src/lib/ui/NoteEditor.test.ts new file mode 100644 index 00000000..4dd97888 --- /dev/null +++ b/ui/src/lib/ui/NoteEditor.test.ts @@ -0,0 +1,73 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte' +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as api from '../api' +import NoteEditor from './NoteEditor.svelte' + +describe('NoteEditor', () => { + beforeEach(() => vi.restoreAllMocks()) + afterEach(cleanup) + + it('shows the note as text, never markup', () => { + render(NoteEditor, { + props: { note: 'bold', target: 'workspace:default' }, + }) + expect(screen.getByText('bold')).toBeTruthy() + expect(document.querySelector('b')).toBeNull() + }) + + it('shows the placeholder when empty', () => { + render(NoteEditor, { props: { note: null, target: 'workspace:default' } }) + expect(screen.getByText('Add a note')).toBeTruthy() + }) + + it('saves through setNote and reports the stored note', async () => { + const spy = vi + .spyOn(api, 'setNote') + .mockResolvedValue({ target: 't', note: 'new', note_source: 'run' }) + const onsaved = vi.fn() + render(NoteEditor, { props: { note: 'old', target: 't', onsaved } }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'new' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + expect(spy).toHaveBeenCalledWith('t', 'new') + expect(onsaved).toHaveBeenCalledWith('new') + }) + + it('cancel restores the old note without calling the api', async () => { + const spy = vi.spyOn(api, 'setNote') + render(NoteEditor, { props: { note: 'old', target: 't' } }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'changed' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /cancel/i })) + expect(spy).not.toHaveBeenCalled() + expect(screen.getByText('old')).toBeTruthy() + }) + + it('saving an empty box clears the note', async () => { + const spy = vi + .spyOn(api, 'setNote') + .mockResolvedValue({ target: 't', note: null, note_source: null }) + render(NoteEditor, { props: { note: 'old', target: 't' } }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: '' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + expect(spy).toHaveBeenCalledWith('t', null) + }) + + it('shows the server error and stays in edit mode', async () => { + vi.spyOn(api, 'setNote').mockRejectedValue( + new Error('a note is at most 4000 characters'), + ) + render(NoteEditor, { props: { note: 'old', target: 't' } }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + expect(await screen.findByText(/at most 4000/)).toBeTruthy() + expect(screen.getByRole('textbox')).toBeTruthy() + }) +}) diff --git a/ui/src/lib/workspace.svelte.ts b/ui/src/lib/workspace.svelte.ts index 040c94e9..f21aa1a2 100644 --- a/ui/src/lib/workspace.svelte.ts +++ b/ui/src/lib/workspace.svelte.ts @@ -86,6 +86,11 @@ export function loadWorkspaces(): Promise { .filter((entry) => entry.usage) .map((entry) => [entry.name, entry.usage!]), ) + workspace.notes = Object.fromEntries( + result.workspaces + .filter((entry) => entry.note) + .map((entry) => [entry.name, entry.note!]), + ) // A route naming a workspace that has gone away (deleted elsewhere, // or a stale bookmark) is caught here for the route the app opened // on; `applyRouteWorkspace` catches every one after diff --git a/ui/src/lib/workspaceState.svelte.ts b/ui/src/lib/workspaceState.svelte.ts index 14a51872..4ccf89b2 100644 --- a/ui/src/lib/workspaceState.svelte.ts +++ b/ui/src/lib/workspaceState.svelte.ts @@ -18,4 +18,12 @@ export const workspace = $state<{ * computes it per listing and caches it briefly, so it is a glance * rather than a live figure. Absent for a server that does not send it. */ usage: Record -}>({ current: DEFAULT_WORKSPACE, names: undefined, root: null, usage: {} }) + /** The note each workspace carries, by name; a workspace without one is absent */ + notes: Record +}>({ + current: DEFAULT_WORKSPACE, + names: undefined, + root: null, + usage: {}, + notes: {}, +}) From 694346bd44bdc864f774637ce29720046b93ce42 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 23:41:07 -0500 Subject: [PATCH 015/204] fix(ui): notes refresh after a save, file note falls back to the run note, draft survives a refetch Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/assets/AssetDetail.svelte | 9 +++- ui/src/lib/gallery/GalleryDetail.svelte | 15 +++++-- ui/src/lib/gallery/GalleryDetail.test.ts | 52 +++++++++++++++++++++++- ui/src/lib/pages/AssetsPage.svelte | 12 ++++++ ui/src/lib/pages/AssetsPage.test.ts | 25 ++++++++++++ ui/src/lib/pages/GalleryPage.svelte | 16 +++++--- ui/src/lib/pages/GalleryPage.test.ts | 49 +++++++++++++++++++++- ui/src/lib/ui/NoteEditor.svelte | 11 +++-- ui/src/lib/ui/NoteEditor.test.ts | 14 +++++++ 9 files changed, 187 insertions(+), 16 deletions(-) diff --git a/ui/src/lib/assets/AssetDetail.svelte b/ui/src/lib/assets/AssetDetail.svelte index e4c56a12..8dc824dd 100644 --- a/ui/src/lib/assets/AssetDetail.svelte +++ b/ui/src/lib/assets/AssetDetail.svelte @@ -16,11 +16,14 @@ busy, onremove, onclose, + onnotesaved, }: { asset: AssetFile busy: boolean onremove: (asset: AssetFile) => void onclose: () => void + /** The note the server stored, so the page can carry it on the asset */ + onnotesaved?: (note: string | null) => void } = $props() // Where a kept asset came from, when keep_output recorded it: the @@ -99,7 +102,11 @@ >{/each}

    {/if} - + onnotesaved?.(note)} + />
    {#if asset.kind === 'image'} {asset.name} diff --git a/ui/src/lib/gallery/GalleryDetail.svelte b/ui/src/lib/gallery/GalleryDetail.svelte index 04d4263e..5263aea3 100644 --- a/ui/src/lib/gallery/GalleryDetail.svelte +++ b/ui/src/lib/gallery/GalleryDetail.svelte @@ -45,7 +45,7 @@ runNote?: string | null noteSource?: 'file' | 'run' | null /** A note changed, so the page can refetch what it shows */ - onnotesaved?: () => void + onnotesaved?: (kind: 'run' | 'file') => void | Promise onremove: () => void onclose: () => void } = $props() @@ -141,6 +141,13 @@ }) const inherited = $derived(noteSource === 'run' && !ownNote && !!runDir) + // Back to what the server says once it has answered: a cleared file note + // returns the run's, a saved one is the file's own + async function saveFileNote() { + await onnotesaved?.('file') + ownNote = false + } + function openAsWorkflow() { if (!embeddedWorkflow) return // Pin the run's seed into the definition so reopening reproduces this @@ -226,7 +233,7 @@ note={runNote} target={runDir} label="Run note" - onsaved={() => onnotesaved?.()} + onsaved={() => onnotesaved?.('run')} /> onnotesaved?.()} + label={ownNote || noteSource === 'file' ? 'File note' : 'Note'} + onsaved={saveFileNote} /> {/if} {#if metadata} diff --git a/ui/src/lib/gallery/GalleryDetail.test.ts b/ui/src/lib/gallery/GalleryDetail.test.ts index 317011bd..4346325d 100644 --- a/ui/src/lib/gallery/GalleryDetail.test.ts +++ b/ui/src/lib/gallery/GalleryDetail.test.ts @@ -1,9 +1,17 @@ -import { cleanup, fireEvent, render, screen } from '@testing-library/svelte' +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from '@testing-library/svelte' import { afterEach, expect, it, vi } from 'vitest' import GalleryDetail from './GalleryDetail.svelte' import type { GalleryFile } from '../types' +const setNote = vi.hoisted(() => vi.fn()) vi.mock('../api', () => ({ + setNote: (t: string, n: string | null) => setNote(t, n), api: { outputDownloadUrl: (name: string) => `/download/${name}` }, })) @@ -98,3 +106,45 @@ it("shows the file's own note", () => { expect(screen.getByText('keeper')).toBeTruthy() expect(screen.queryByRole('button', { name: 'Note this file' })).toBeNull() }) + +it("labels a saved file note as the file's own, and a cleared one returns the run note", async () => { + const run = { + ...props, + note: 'for the client', + runNote: 'for the client', + noteSource: 'run' as const, + } + const spy = setNote + const onnotesaved = vi.fn(async () => {}) + const { rerender } = render(GalleryDetail, { ...run, onnotesaved }) + await fireEvent.click(screen.getByRole('button', { name: 'Note this file' })) + spy.mockResolvedValue({ + target: file.name, + note: 'mine', + note_source: 'file', + }) + onnotesaved.mockImplementation(async () => { + await rerender({ ...run, note: 'mine', noteSource: 'file', onnotesaved }) + }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'mine' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + await waitFor(() => expect(screen.getByText('File note')).toBeTruthy()) + expect(screen.getByText('mine')).toBeTruthy() + + spy.mockResolvedValue({ + target: file.name, + note: 'for the client', + note_source: 'run', + }) + onnotesaved.mockImplementation(async () => { + await rerender({ ...run, onnotesaved }) + }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { target: { value: '' } }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + await waitFor(() => expect(screen.getByText('Run note')).toBeTruthy()) + expect(screen.getByText('for the client')).toBeTruthy() +}) diff --git a/ui/src/lib/pages/AssetsPage.svelte b/ui/src/lib/pages/AssetsPage.svelte index d745ba84..e5f3705c 100644 --- a/ui/src/lib/pages/AssetsPage.svelte +++ b/ui/src/lib/pages/AssetsPage.svelte @@ -62,6 +62,17 @@ storageGet(COLLAPSE_KEY, {} as Record), ) + // Carry a saved note onto the listing and the pick, so selecting the asset + // again shows it rather than what was fetched before the edit + function saveNote(note: string | null) { + if (!selected) return + const name = selected.name + const patch = (a: AssetFile) => + a.name === name ? { ...a, note: note ?? undefined } : a + assets = assets.map(patch) + selected = patch(selected) + } + function load() { return api .listAssets() @@ -418,6 +429,7 @@ {busy} onremove={remove} onclose={() => (selected = null)} + onnotesaved={saveNote} /> {/if} diff --git a/ui/src/lib/pages/AssetsPage.test.ts b/ui/src/lib/pages/AssetsPage.test.ts index f51e14e8..ead259ae 100644 --- a/ui/src/lib/pages/AssetsPage.test.ts +++ b/ui/src/lib/pages/AssetsPage.test.ts @@ -1,5 +1,6 @@ import { cleanup, + fireEvent, render, screen, waitFor, @@ -87,7 +88,9 @@ const uploadMedia = vi.hoisted(() => () => Promise.resolve({ reference: 'asset:uploads/x.png' }), ), ) +const setNote = vi.hoisted(() => vi.fn()) vi.mock('../api', () => ({ + setNote: (t: string, n: string | null) => setNote(t, n), api: { listAssets: () => listAssets(), galleryMetadata: () => new Promise(() => {}), @@ -633,3 +636,25 @@ it('names the workflows that use an asset in its details', async () => { const link = await waitFor(() => screen.getByRole('link', { name: 'ep/two' })) expect(link.getAttribute('href')).toContain('workflows/ep/two') }) + +it('keeps a saved note on the asset when it is selected again', async () => { + setNote.mockResolvedValue({ + target: 'asset:iris.png', + note: 'hero shot', + note_source: null, + }) + await renderAssets() + screen.getByLabelText('show details for iris.png').click() + await fireEvent.click( + await screen.findByRole('button', { name: /edit note/i }), + ) + await fireEvent.input(screen.getByLabelText('Note'), { + target: { value: 'hero shot' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + await waitFor(() => expect(screen.getByText('hero shot')).toBeTruthy()) + screen.getByLabelText('show details for cast/priya.jpg').click() + await waitFor(() => expect(screen.queryByText('hero shot')).toBeNull()) + screen.getByLabelText('show details for iris.png').click() + await waitFor(() => expect(screen.getByText('hero shot')).toBeTruthy()) +}) diff --git a/ui/src/lib/pages/GalleryPage.svelte b/ui/src/lib/pages/GalleryPage.svelte index 7ec37286..222988fa 100644 --- a/ui/src/lib/pages/GalleryPage.svelte +++ b/ui/src/lib/pages/GalleryPage.svelte @@ -37,10 +37,8 @@ let role = $state(null) let inputs = $state([]) - $effect(() => { - // Read inside the effect so switching workspaces refetches the gallery - void workspace.current - api + function loadListing() { + return api .gallery() .then((result) => { files = result.files @@ -52,6 +50,12 @@ }) .catch((e) => (error = e.message)) .finally(() => (loaded = true)) + } + + $effect(() => { + // Read inside the effect so switching workspaces refetches the gallery + void workspace.current + void loadListing() }) // A file whose run does not say its role counts as final @@ -155,9 +159,11 @@ // A note changed: refetch what the panel shows and carry the file's // effective note to its tile, which is what the grid marks - async function refreshNote() { + async function refreshNote(kind: 'run' | 'file') { const name = selected?.name if (!name) return + // A run's note shows on every tile of the run that inherits it + if (kind === 'run') await loadListing() try { const r = await api.galleryMetadata(name) if (selected?.name !== name) return diff --git a/ui/src/lib/pages/GalleryPage.test.ts b/ui/src/lib/pages/GalleryPage.test.ts index f781fd36..1fcbdf06 100644 --- a/ui/src/lib/pages/GalleryPage.test.ts +++ b/ui/src/lib/pages/GalleryPage.test.ts @@ -51,7 +51,12 @@ const deleteOutput = vi.hoisted(() => const archiveOutputs = vi.hoisted(() => vi.fn<(names: string[]) => Promise>(() => Promise.resolve()), ) +const setNote = vi.hoisted(() => vi.fn()) +const galleryMetadata = vi.hoisted(() => + vi.fn<(name: string) => Promise>(() => new Promise(() => {})), +) vi.mock('../api', () => ({ + setNote: (t: string, n: string | null) => setNote(t, n), api: { gallery: () => gallery(), galleryThumbnailUrl: (name: string) => `/thumb/${name}`, @@ -59,7 +64,7 @@ vi.mock('../api', () => ({ outputDownloadUrl: (name: string) => `/download/${name}`, deleteOutput: (name: string) => deleteOutput(name), archiveOutputs: (names: string[]) => archiveOutputs(names), - galleryMetadata: vi.fn(() => new Promise(() => {})), + galleryMetadata: (name: string) => galleryMetadata(name), }, })) @@ -77,6 +82,8 @@ afterEach(() => { deleteOutput.mockClear() archiveOutputs.mockClear() notifyError.mockClear() + galleryMetadata.mockReset() + galleryMetadata.mockImplementation(() => new Promise(() => {})) vi.unstubAllGlobals() workspace.current = DEFAULT_WORKSPACE }) @@ -473,3 +480,43 @@ it('marks a tile that carries a note, titled with its first line', async () => { expect(marks).toHaveLength(1) expect(marks[0].getAttribute('title')).toBe('first line') }) + +it('marks every tile of a run after its run note is saved', async () => { + const inRun = (name: string, note?: string): GalleryFile => ({ + ...file(`wf/r1/${name}`), + folder: 'wf', + run_id: 'r1', + note, + }) + listing.files = [inRun('a.png'), inRun('b.png')] + galleryMetadata.mockResolvedValue({ + metadata: {}, + job: null, + inputs: [], + role: null, + note: 'old', + run_note: 'old', + note_source: 'run', + }) + await renderGallery('wf/r1/a.png') + screen + .getByLabelText('select wf/r1/a.png') + .closest('.cellwrap')! + .querySelector('button.cell')! + .click() + await fireEvent.click( + await screen.findByRole('button', { name: /edit note/i }), + ) + setNote.mockImplementation(async () => { + listing.files = [inRun('a.png', 'fresh'), inRun('b.png', 'fresh')] + return { target: 'wf/r1', note: 'fresh', note_source: 'run' } + }) + await fireEvent.input(screen.getByLabelText('Run note'), { + target: { value: 'fresh' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + await waitFor(() => + expect(document.querySelectorAll('.note-mark')).toHaveLength(2), + ) + expect(setNote).toHaveBeenCalledWith('wf/r1', 'fresh') +}) diff --git a/ui/src/lib/ui/NoteEditor.svelte b/ui/src/lib/ui/NoteEditor.svelte index 3a3408b9..7e0798e1 100644 --- a/ui/src/lib/ui/NoteEditor.svelte +++ b/ui/src/lib/ui/NoteEditor.svelte @@ -24,11 +24,14 @@ let saving = $state(false) let error = $state(null) // What the box last stored, until the parent hands down a newer note - let shown = $state(null) + let shown = $derived(note ?? null) + // Another target is another note, even when the text is the same. Only a + // changed target ends an edit: a refetch landing mid-edit keeps the draft + // svelte-ignore state_referenced_locally + let editedTarget = target $effect(() => { - // Another target is another note, even when the text is the same - void target - shown = note ?? null + if (target === editedTarget) return + editedTarget = target editing = false }) diff --git a/ui/src/lib/ui/NoteEditor.test.ts b/ui/src/lib/ui/NoteEditor.test.ts index 4dd97888..1c03d595 100644 --- a/ui/src/lib/ui/NoteEditor.test.ts +++ b/ui/src/lib/ui/NoteEditor.test.ts @@ -70,4 +70,18 @@ describe('NoteEditor', () => { expect(await screen.findByText(/at most 4000/)).toBeTruthy() expect(screen.getByRole('textbox')).toBeTruthy() }) + + it('keeps the draft when the note prop changes mid-edit', async () => { + const { rerender } = render(NoteEditor, { + props: { note: 'old', target: 't' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'half typed' }, + }) + await rerender({ note: 'refetched', target: 't' }) + expect((screen.getByRole('textbox') as HTMLTextAreaElement).value).toBe( + 'half typed', + ) + }) }) From 848f8d0823c12afa3aa3e3346e4a9dd3ea77e576 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Mon, 12 Oct 2026 23:49:36 -0500 Subject: [PATCH 016/204] docs(release): note roles and notes for 0.14.0 Also formats three plan docs whose python code blocks ruff format flagged. Co-Authored-By: Claude Opus 5.5 --- docs/RELEASING.md | 7 + .../plans/2026-10-07-develop-review-fixes.md | 224 ++++++++++-------- .../plans/2026-10-10-workflow-speed.md | 141 ++++++----- .../plans/2026-10-12-roles-and-notes.md | 167 +++++++++---- 4 files changed, 336 insertions(+), 203 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index e1fcf87c..f2ddb571 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -7,6 +7,13 @@ notes from commits at tag time (see below). This section is a scratch pad for items a branch's author wants the next release note to name; clear it when a release ships. +### 0.14.0 + +- Gallery files are tagged final or intermediate (inferred when a + workflow doesn't say) and the gallery opens on finals; runs, files, + assets and workspaces take an optional note (`note` on `run_workflow`, + MCP `set_note`). + ### 0.13.0 diff --git a/docs/superpowers/plans/2026-10-07-develop-review-fixes.md b/docs/superpowers/plans/2026-10-07-develop-review-fixes.md index 41d8e876..ac9c712f 100644 --- a/docs/superpowers/plans/2026-10-07-develop-review-fixes.md +++ b/docs/superpowers/plans/2026-10-07-develop-review-fixes.md @@ -74,7 +74,9 @@ class TestOwnGuidesPlusTheChain: assert "4" in error["message"] def test_three_own_guides_leave_room(self): - assert guide_chain_errors(definition(prompt="a cat", guides=self.guides(3))) == [] + assert ( + guide_chain_errors(definition(prompt="a cat", guides=self.guides(3))) == [] + ) def test_an_unresolved_guides_value_is_left_to_the_run(self): assert ( @@ -85,7 +87,9 @@ class TestOwnGuidesPlusTheChain: def test_last_frame_continuity_takes_the_full_four(self): chain = {"segments": 2, "continuity": "last_frame"} assert ( - guide_chain_errors(definition(chain=chain, prompt="a cat", guides=self.guides(4))) + guide_chain_errors( + definition(chain=chain, prompt="a cat", guides=self.guides(4)) + ) == [] ) ``` @@ -190,7 +194,9 @@ def test_the_context_defaults_to_the_devices_capacity(tmp_path, monkeypatch): def test_the_context_takes_a_pool_ceiling_instead(tmp_path, monkeypatch): monkeypatch.setattr("dw.validation.device_capacity_gb", lambda: 12.0) - assert workflow_context(empty_workflow(tmp_path), capacity_gb=24.0).capacity_gb == 24.0 + assert ( + workflow_context(empty_workflow(tmp_path), capacity_gb=24.0).capacity_gb == 24.0 + ) def _patched_admit_for(monkeypatch, state): @@ -225,18 +231,19 @@ def test_a_state_without_a_manager_falls_back_to_the_device(monkeypatch): Append to `class TestFitAtSubmit` in `tests/test_worker_pool.py`: ```python - def test_the_largest_card_is_the_ceiling_admission_checks(self, pool): - manager = pool( - card(success_script, "cuda:0", 12, name="Small GPU"), - card(success_script, "cuda:1", 24, name="Large GPU"), - ) - assert manager.largest_ceiling_gb() == 24 +def test_the_largest_card_is_the_ceiling_admission_checks(self, pool): + manager = pool( + card(success_script, "cuda:0", 12, name="Small GPU"), + card(success_script, "cuda:1", 24, name="Large GPU"), + ) + assert manager.largest_ceiling_gb() == 24 + - def test_an_unread_card_has_no_ceiling(self, pool): - worker = ScriptedWorkerManager(success_script) - worker.device = "cuda:0" - manager = pool(worker) - assert manager.largest_ceiling_gb() is None +def test_an_unread_card_has_no_ceiling(self, pool): + worker = ScriptedWorkerManager(success_script) + worker.device = "cuda:0" + manager = pool(worker) + assert manager.largest_ceiling_gb() is None ``` Note for the implementer: `card()` sets `_capacity_read = True` and `_capacity_gb`; a bare `ScriptedWorkerManager` leaves them unread, and `ceiling_gb()` falls back to `capacity_gb()`, which is `None` until read. If `ScriptedWorkerManager` reads a real device on first call, patch `dw.worker_manager.device_capacity_gb` to return `None` in the second test, as `test_a_declared_need_is_held_to_the_ceiling_admission_uses` patches it. @@ -261,7 +268,7 @@ Add to the docstring, after "the device the request is checked against": `(its o Replace `capacity_gb=device_capacity_gb(),` with: ```python - capacity_gb=device_capacity_gb() if capacity_gb is None else capacity_gb, +capacity_gb = (device_capacity_gb() if capacity_gb is None else capacity_gb,) ``` - [ ] **Step 4: Add `JobManager.largest_ceiling_gb`** @@ -519,47 +526,50 @@ class TestSafeRequest: Then change `test_an_untrusted_run_withholds_the_token_from_a_third_party` so it patches the shared path instead of `remote.requests.post`. Replace its `_Response`/`_post`/`with` block with: ```python - sent = {} +sent = {} - class _Response: - ok = True - is_redirect = False - headers = {"Content-Type": "application/octet-stream"} - status_code = 200 - _content = b"" - def raise_for_status(self): - pass +class _Response: + ok = True + is_redirect = False + headers = {"Content-Type": "application/octet-stream"} + status_code = 200 + _content = b"" - def iter_content(self, chunk_size): - return iter([b""]) + def raise_for_status(self): + pass - def close(self): - pass + def iter_content(self, chunk_size): + return iter([b""]) - class _Session: - def request(self, method, url, **kwargs): - sent["method"] = method - sent["headers"] = kwargs.get("headers") - sent["timeout"] = kwargs.get("timeout") - return _Response() + def close(self): + pass - def close(self): - pass - with ( - patch("dw.locations._pinned_session", lambda url, address: _Session()), - patch.object(remote.torch, "load", return_value=_Embeds()), - patch( - "dw.locations.socket.getaddrinfo", - return_value=[(None, None, None, "", ("93.184.216.34", 443))], - ), - ): - remote.remote_text_encoder(["a"], "https://evil.example.com/encode", "cpu") - - assert "Authorization" not in sent["headers"] - assert sent["method"] == "POST" - assert sent["timeout"] == remote.REMOTE_ENCODER_TIMEOUT +class _Session: + def request(self, method, url, **kwargs): + sent["method"] = method + sent["headers"] = kwargs.get("headers") + sent["timeout"] = kwargs.get("timeout") + return _Response() + + def close(self): + pass + + +with ( + patch("dw.locations._pinned_session", lambda url, address: _Session()), + patch.object(remote.torch, "load", return_value=_Embeds()), + patch( + "dw.locations.socket.getaddrinfo", + return_value=[(None, None, None, "", ("93.184.216.34", 443))], + ), +): + remote.remote_text_encoder(["a"], "https://evil.example.com/encode", "cpu") + +assert "Authorization" not in sent["headers"] +assert sent["method"] == "POST" +assert sent["timeout"] == remote.REMOTE_ENCODER_TIMEOUT ``` - [ ] **Step 2: Run them to verify they fail** @@ -677,7 +687,13 @@ def _read_capped(response, max_bytes, what, url): def _safe_request( - method, url, what, timeout, validate=validate_media_url, max_bytes=MAX_MEDIA_BYTES, **kwargs + method, + url, + what, + timeout, + validate=validate_media_url, + max_bytes=MAX_MEDIA_BYTES, + **kwargs, ): """One outbound request on the workflow's behalf: `validate` on the URL and on every redirect target, each hop dialed at the address the policy @@ -733,7 +749,12 @@ def safe_get(url, what="a media argument", timeout=60, max_bytes=MAX_MEDIA_BYTES def safe_post( - url, what, timeout=120, max_bytes=MAX_MEDIA_BYTES, validate=validate_media_url, **kwargs + url, + what, + timeout=120, + max_bytes=MAX_MEDIA_BYTES, + validate=validate_media_url, + **kwargs, ): """POST on a workflow's behalf, the way safe_get fetches: `kwargs` are requests' own (`json=`, `headers=`).""" @@ -877,39 +898,41 @@ Context: `cancel()` on a RUNNING job calls `slot.manager.cancel()` immediately. Append to `class TestCancel` in `tests/test_worker_pool.py`: ```python - def test_a_cancel_before_the_execute_is_sent_still_lands(self, pool): - """The dispatcher marks RUNNING before the job's thread takes the - slot; a cancel in that window used to reach an idle worker, which - ignores it, and the job ran anyway.""" - gate = Gate() - manager = pool(card(gate, "cuda:0", 24)) - slot = manager.slots[0] - with slot.lock: # a cache probe holds the card between dispatch and Execute - job = submit(manager, "late-cancel") - assert wait_until(lambda: job.status == "running") - assert manager.cancel(job.id) == "running" - wait_for(job, {"cancelled"}) - assert not gate.started.is_set(), "the worker was still sent the job" - - def test_a_cancel_after_the_job_finished_is_a_no_op(self, pool): - manager = pool(card(success_script, "cuda:0", 24)) - job = submit(manager, "done") - wait_for(job, {"succeeded"}) - assert manager.cancel(job.id) == "succeeded" - assert job.status == "succeeded" - - def test_cancelling_a_queued_job_drops_its_dispatch_bookkeeping(self, pool): - gate = Gate() - manager = pool(card(gate, "cuda:0", 24)) - first = submit(manager, "first") - gate.wait_started() - second = submit(manager, "second", vram_need=(8, True)) - assert second.id in manager._needs - assert manager.cancel(second.id) == "cancelled" - assert second.id not in manager._needs - assert second.id not in manager._preferred - gate.release.set() - wait_for(first, {"succeeded"}) +def test_a_cancel_before_the_execute_is_sent_still_lands(self, pool): + """The dispatcher marks RUNNING before the job's thread takes the + slot; a cancel in that window used to reach an idle worker, which + ignores it, and the job ran anyway.""" + gate = Gate() + manager = pool(card(gate, "cuda:0", 24)) + slot = manager.slots[0] + with slot.lock: # a cache probe holds the card between dispatch and Execute + job = submit(manager, "late-cancel") + assert wait_until(lambda: job.status == "running") + assert manager.cancel(job.id) == "running" + wait_for(job, {"cancelled"}) + assert not gate.started.is_set(), "the worker was still sent the job" + + +def test_a_cancel_after_the_job_finished_is_a_no_op(self, pool): + manager = pool(card(success_script, "cuda:0", 24)) + job = submit(manager, "done") + wait_for(job, {"succeeded"}) + assert manager.cancel(job.id) == "succeeded" + assert job.status == "succeeded" + + +def test_cancelling_a_queued_job_drops_its_dispatch_bookkeeping(self, pool): + gate = Gate() + manager = pool(card(gate, "cuda:0", 24)) + first = submit(manager, "first") + gate.wait_started() + second = submit(manager, "second", vram_need=(8, True)) + assert second.id in manager._needs + assert manager.cancel(second.id) == "cancelled" + assert second.id not in manager._needs + assert second.id not in manager._preferred + gate.release.set() + wait_for(first, {"succeeded"}) ``` - [ ] **Step 2: Run them to verify they fail** @@ -1026,27 +1049,28 @@ Context: the held-track outputs are added only when `output` is a list containin Append to `class TestWithHeldAudio` in `tests/test_h3_hold_audio.py`: ```python - def test_a_string_output_naming_audio_gets_the_held_keys_too(self): - pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() - insert_audio_hold(pipeline) - reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) +def test_a_string_output_naming_audio_gets_the_held_keys_too(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) - result = Pipeline._with_held_audio( - ns(pipeline), {"hold_audio": reference, "output": "audio"} - ) + result = Pipeline._with_held_audio( + ns(pipeline), {"hold_audio": reference, "output": "audio"} + ) - assert result["output"] == ["audio", HELD_AUDIO_OUTPUT, HELD_AUDIO_RATE_OUTPUT] + assert result["output"] == ["audio", HELD_AUDIO_OUTPUT, HELD_AUDIO_RATE_OUTPUT] - def test_a_string_output_not_naming_audio_is_left_as_written(self): - pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() - insert_audio_hold(pipeline) - reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) - result = Pipeline._with_held_audio( - ns(pipeline), {"hold_audio": reference, "output": "videos"} - ) +def test_a_string_output_not_naming_audio_is_left_as_written(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) + + result = Pipeline._with_held_audio( + ns(pipeline), {"hold_audio": reference, "output": "videos"} + ) - assert result["output"] == "videos" + assert result["output"] == "videos" ``` - [ ] **Step 2: Run them to verify they fail** diff --git a/docs/superpowers/plans/2026-10-10-workflow-speed.md b/docs/superpowers/plans/2026-10-10-workflow-speed.md index 77651954..a367f002 100644 --- a/docs/superpowers/plans/2026-10-10-workflow-speed.md +++ b/docs/superpowers/plans/2026-10-10-workflow-speed.md @@ -61,7 +61,10 @@ import pytest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TEMPLATES = sorted( - glob.glob(os.path.join(REPO_ROOT, "workflows", "templates", "**", "*.json"), recursive=True) + glob.glob( + os.path.join(REPO_ROOT, "workflows", "templates", "**", "*.json"), + recursive=True, + ) ) @@ -100,9 +103,13 @@ def test_turbo_schedules_carry_no_block_cache(path): def test_compose_workflows_video_steps_match_the_turbo_default(): - compose = _load(os.path.join(REPO_ROOT, "workflows", "templates", "compose-workflows.json")) + compose = _load( + os.path.join(REPO_ROOT, "workflows", "templates", "compose-workflows.json") + ) i2v = _load( - os.path.join(REPO_ROOT, "workflows", "templates", "minimax", "image-to-video.json") + os.path.join( + REPO_ROOT, "workflows", "templates", "minimax", "image-to-video.json" + ) ) assert _default(compose, "video_num_inference_steps") == _default( i2v, "num_inference_steps" @@ -220,7 +227,8 @@ def test_two_frames_share_one_helper(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" + "dw.tasks.upscale.resolve_model_path", + lambda model_name, filename: "weights.pth", ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) @@ -238,15 +246,14 @@ def test_helper_state_is_cleared_before_each_frame(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" + "dw.tasks.upscale.resolve_model_path", + lambda model_name, filename: "weights.pth", ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) module.restore_faces(frame, "m", device="cpu") module.restore_faces(frame, "m", device="cpu") - helper = model_cache._cache[ - ("restore_faces_helper", 1, 512, True, "cpu") - ] + helper = model_cache._cache[("restore_faces_helper", 1, 512, True, "cpu")] assert helper.cleaned == 2 @@ -256,7 +263,8 @@ def test_different_settings_build_different_helpers(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" + "dw.tasks.upscale.resolve_model_path", + lambda model_name, filename: "weights.pth", ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) module.restore_faces(frame, "m", device="cpu", face_size=512) @@ -276,25 +284,26 @@ Expected: `test_two_frames_share_one_helper` fails with 2 constructions; the cac Replace the helper construction at `dw/tasks/restore_faces.py:70-77` with: ```python - # facexlib's helper loads RetinaFace (and ParseNet with use_parse) in - # its constructor. per_frame calls this once per frame, so a video was - # reloading detector weights for every frame. The helper keeps per-image - # state between calls; clean_all() resets it before each read - def build_helper(): - return FaceRestoreHelper( - upscale_factor=upscale_factor, - face_size=face_size, - crop_ratio=(1, 1), - det_model="retinaface_resnet50", - use_parse=use_parse, - device=torch.device(device), - ) - - face_helper = cached_model( - ("restore_faces_helper", upscale_factor, face_size, use_parse, str(device)), - build_helper, +# facexlib's helper loads RetinaFace (and ParseNet with use_parse) in +# its constructor. per_frame calls this once per frame, so a video was +# reloading detector weights for every frame. The helper keeps per-image +# state between calls; clean_all() resets it before each read +def build_helper(): + return FaceRestoreHelper( + upscale_factor=upscale_factor, + face_size=face_size, + crop_ratio=(1, 1), + det_model="retinaface_resnet50", + use_parse=use_parse, + device=torch.device(device), ) - face_helper.clean_all() + + +face_helper = cached_model( + ("restore_faces_helper", upscale_factor, face_size, use_parse, str(device)), + build_helper, +) +face_helper.clean_all() ``` Keep the existing `read_image` call that follows. @@ -431,6 +440,7 @@ git commit -m "perf(media): decode picture frames with libav AUTO threading" ```python RUNTIME_LORA_KEYS = ("scale", "alpha") + def weights_identity(pipeline_definition): """The pipeline definition with every runtime-mutable setting removed: a LoRA's scale and alpha (set_adapters / set_adapter_alpha) and a @@ -447,6 +457,7 @@ def adapter_settings(loras): `loras`, naming an adapter by its own adapter_name or its ORIGINAL index (a nulled entry keeps its slot). Reads, never pops.""" + def apply_adapter_settings(loras, pipeline): """Set each active LoRA's alpha then every scale on a pipeline that already holds them. No-op with no active entry.""" @@ -487,7 +498,12 @@ def _definition(**overrides): "configuration": {"component_type": "FluxPipeline"}, "from_pretrained_arguments": {"model_name": "black-forest-labs/FLUX.1-dev"}, "loras": [ - {"model_name": "a/b", "weight_name": "x.safetensors", "scale": 1.0, "alpha": 16}, + { + "model_name": "a/b", + "weight_name": "x.safetensors", + "scale": 1.0, + "alpha": 16, + }, ], "scheduler": {"shift": 6.0}, "audio_scheduler": {"shift": 3.0}, @@ -499,7 +515,16 @@ def _definition(**overrides): def test_lora_scale_and_alpha_do_not_change_the_pipeline_key(): a = _definition() - b = _definition(loras=[{"model_name": "a/b", "weight_name": "x.safetensors", "scale": 0.5, "alpha": 8}]) + b = _definition( + loras=[ + { + "model_name": "a/b", + "weight_name": "x.safetensors", + "scale": 0.5, + "alpha": 8, + } + ] + ) assert pipeline_cache_key(a) == pipeline_cache_key(b) @@ -511,7 +536,9 @@ def test_scheduler_shift_does_not_change_the_pipeline_key(): def test_lora_model_name_still_changes_the_pipeline_key(): a = _definition() - b = _definition(loras=[{"model_name": "a/c", "weight_name": "x.safetensors", "scale": 1.0}]) + b = _definition( + loras=[{"model_name": "a/c", "weight_name": "x.safetensors", "scale": 1.0}] + ) assert pipeline_cache_key(a) != pipeline_cache_key(b) @@ -978,40 +1005,36 @@ Leave `clear_model_cache()` and `step_cache.clear()` as they are. Append `f", ke `_prepare_workflow`: ```python - if identity != self.workflow_identity: - if self.workflow_identity is not None: - keep = self._keys_worth_keeping(command, job.workflow) - if keep: - self._reply( - Output( - message=f"Workflow changed - keeping {len(keep)} warm " - "model(s) the new workflow loads, releasing the rest..." - ) - ) - else: - self._reply( - Output(message="Workflow changed - releasing cached models...") - ) - self._reply(Output(message=self._cleanup_all(keep=keep))) - self.workflow_identity = identity +if identity != self.workflow_identity: + if self.workflow_identity is not None: + keep = self._keys_worth_keeping(command, job.workflow) + if keep: + self._reply( + Output( + message=f"Workflow changed - keeping {len(keep)} warm " + "model(s) the new workflow loads, releasing the rest..." + ) + ) + else: + self._reply(Output(message="Workflow changed - releasing cached models...")) + self._reply(Output(message=self._cleanup_all(keep=keep))) + self.workflow_identity = identity ``` with ```python - def _keys_worth_keeping(self, command, workflow): - """The resident pipeline keys the next workflow loads anyway. Any - failure to prepare the definition answers the empty set - the run - itself will report what is wrong, and a full release is the safe - default on a card that cannot hold two stacks.""" - try: - wanted = workflow_run.pipeline_keys( - workflow, command.get("arguments") or {} - ) - except Exception as e: - logger.debug(f"Could not take the next workflow's pipeline keys: {e}") - return set() - return wanted & set(self.loaded_pipelines) +def _keys_worth_keeping(self, command, workflow): + """The resident pipeline keys the next workflow loads anyway. Any + failure to prepare the definition answers the empty set - the run + itself will report what is wrong, and a full release is the safe + default on a card that cannot hold two stacks.""" + try: + wanted = workflow_run.pipeline_keys(workflow, command.get("arguments") or {}) + except Exception as e: + logger.debug(f"Could not take the next workflow's pipeline keys: {e}") + return set() + return wanted & set(self.loaded_pipelines) ``` Check whether `_prepare_workflow` runs with the job's asset dir already active (`_activate_job` is phase 1, so yes); `pipeline_keys` may realize `asset:` variables and needs it. diff --git a/docs/superpowers/plans/2026-10-12-roles-and-notes.md b/docs/superpowers/plans/2026-10-12-roles-and-notes.md index ea856954..02ce353c 100644 --- a/docs/superpowers/plans/2026-10-12-roles-and-notes.md +++ b/docs/superpowers/plans/2026-10-12-roles-and-notes.md @@ -254,9 +254,16 @@ def test_for_each_members_are_steps_of_their_own(): steps = [ _step("shot@0"), _step("shot@1"), - _step("join", args={"videos": ["previous_result:shot@0", "previous_result:shot@1"]}), + _step( + "join", + args={"videos": ["previous_result:shot@0", "previous_result:shot@1"]}, + ), + ] + manifest = [ + _entry("shot@0", "0.mp4"), + _entry("shot@1", "1.mp4"), + _entry("join", "j.mp4"), ] - manifest = [_entry("shot@0", "0.mp4"), _entry("shot@1", "1.mp4"), _entry("join", "j.mp4")] roles = step_roles(steps, manifest) assert roles["shot@0"] == roles["shot@1"] == INTERMEDIATE assert roles["join"] == FINAL @@ -451,7 +458,9 @@ def run_inputs(steps): continue for value in _walk_strings(step): for prefix, kind in _INPUT_KINDS: - if is_ref(prefix, value) and value not in (s["reference"] for s in seen): + if is_ref(prefix, value) and value not in ( + s["reference"] for s in seen + ): seen.append({"reference": value, "kind": kind}) return seen ``` @@ -489,14 +498,25 @@ Append to `tests/test_roles.py`: ```python import pytest -from dw.validation import validate_workflow_definition # use the name tests/test_schema*.py use +from dw.validation import ( + validate_workflow_definition, +) # use the name tests/test_schema*.py use -@pytest.mark.parametrize("role,ok", [("final", True), ("intermediate", True), ("input", False), ("keeper", False)]) +@pytest.mark.parametrize( + "role,ok", + [("final", True), ("intermediate", True), ("input", False), ("keeper", False)], +) def test_schema_role_values(role, ok): workflow = { "id": "t", - "steps": [{"name": "s", "task": {"command": "noop", "arguments": {}}, "result": {"role": role}}], + "steps": [ + { + "name": "s", + "task": {"command": "noop", "arguments": {}}, + "result": {"role": role}, + } + ], } errors = validate_workflow_definition(workflow) assert (not errors) == ok @@ -571,7 +591,9 @@ import json, os # into a tmp output dir and return the run directory -def test_terminal_manifest_records_roles_and_inputs(run_workflow_into, tmp_path, asset_png): +def test_terminal_manifest_records_roles_and_inputs( + run_workflow_into, tmp_path, asset_png +): workflow = { "id": "roles-e2e", "steps": [ @@ -730,41 +752,69 @@ def _write_run(root, identity, run_id, steps, realized=None, files=()): path = os.path.join(run_dir, f) os.makedirs(os.path.dirname(path), exist_ok=True) open(path, "wb").write(b"\x89PNG\r\n\x1a\n") - json.dump({"run_id": run_id, "version": 1, "status": "succeeded", "steps": steps}, - open(os.path.join(run_dir, "manifest.json"), "w")) + json.dump( + {"run_id": run_id, "version": 1, "status": "succeeded", "steps": steps}, + open(os.path.join(run_dir, "manifest.json"), "w"), + ) if realized is not None: json.dump(realized, open(os.path.join(run_dir, "workflow.json"), "w")) return run_dir def test_recorded_roles_reach_the_listing(client, output_root): - _write_run(output_root, "wf", "20261012-000000-aaaa", - [{"step": "a", "files": ["a.png"], "role": "intermediate"}, - {"step": "b", "files": ["b.png"], "role": "final"}], - files=["a.png", "b.png"]) + _write_run( + output_root, + "wf", + "20261012-000000-aaaa", + [ + {"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["b.png"], "role": "final"}, + ], + files=["a.png", "b.png"], + ) files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} assert files["a.png"]["role"] == "intermediate" assert files["b.png"]["role"] == "final" def test_older_run_roles_inferred_from_realized_workflow(client, output_root): - realized = {"id": "wf", "steps": [ - {"name": "a", "task": {"command": "noop", "arguments": {}}}, - {"name": "b", "task": {"command": "noop", "arguments": {"image": "previous_result:a"}}}, - ]} - _write_run(output_root, "wf", "20261012-000001-bbbb", - [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}], - realized=realized, files=["a.png", "b.png"]) + realized = { + "id": "wf", + "steps": [ + {"name": "a", "task": {"command": "noop", "arguments": {}}}, + { + "name": "b", + "task": { + "command": "noop", + "arguments": {"image": "previous_result:a"}, + }, + }, + ], + } + _write_run( + output_root, + "wf", + "20261012-000001-bbbb", + [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}], + realized=realized, + files=["a.png", "b.png"], + ) files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} assert files["a.png"]["role"] == "intermediate" assert files["b.png"]["role"] == "final" def test_role_filter(client, output_root): - _write_run(output_root, "wf", "20261012-000002-cccc", - [{"step": "a", "files": ["a.png"], "role": "intermediate"}, - {"step": "b", "files": ["final/b.png"], "role": "final"}], - files=["a.png", "final/b.png"]) + _write_run( + output_root, + "wf", + "20261012-000002-cccc", + [ + {"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["final/b.png"], "role": "final"}, + ], + files=["a.png", "final/b.png"], + ) finals = client.get("/api/gallery", params={"role": "final"}).json()["files"] assert [f["label"] for f in finals] == ["b.png"] @@ -778,8 +828,13 @@ def test_flat_file_has_null_role_and_passes_final_filter(client, output_root): def test_metadata_carries_role_and_inputs(client, output_root): - run_dir = _write_run(output_root, "wf", "20261012-000003-dddd", - [{"step": "b", "files": ["b.png"], "role": "final"}], files=["b.png"]) + run_dir = _write_run( + output_root, + "wf", + "20261012-000003-dddd", + [{"step": "b", "files": ["b.png"], "role": "final"}], + files=["b.png"], + ) manifest_path = os.path.join(run_dir, "manifest.json") manifest = json.load(open(manifest_path)) manifest["inputs"] = [{"reference": "asset:iris.png", "kind": "asset"}] @@ -862,11 +917,9 @@ The relative name inside it is `relative_name` with the `f"{folder}/{run_id}/"` prefix removed. Then: ```python - entry["role"] = ( - run_roles(os.path.join(root, folder, run_id)).get(inner_name) - if run_id - else None - ) +entry["role"] = ( + run_roles(os.path.join(root, folder, run_id)).get(inner_name) if run_id else None +) ``` Add a `role: list[str] | None = Query(None)` parameter to the gallery @@ -897,10 +950,12 @@ the manifest for `job`, `shots` and so on. In `dw_mcp/tools_catalog.py`, `list_gallery` gains: ```python - role: Annotated[ - list[Literal["final", "intermediate"]] | None, - Field(description="Only files of these roles. A deliverable is 'final', a file a later step consumed is 'intermediate'. Omit for every file - intermediates matter when diagnosing a run."), - ] = None, +role: Annotated[ + list[Literal["final", "intermediate"]] | None, + Field( + description="Only files of these roles. A deliverable is 'final', a file a later step consumed is 'intermediate'. Omit for every file - intermediates matter when diagnosing a run." + ), +] = (None,) ``` Forward it as the repeated `role` query parameter in `dw_mcp/catalog.py`, @@ -1243,7 +1298,10 @@ def test_corrupt_file_reads_empty(tmp_path): def test_concurrent_file_notes_all_land(tmp_path): d = str(tmp_path) - threads = [threading.Thread(target=N.set_file_note, args=(d, f"{i}.png", str(i))) for i in range(20)] + threads = [ + threading.Thread(target=N.set_file_note, args=(d, f"{i}.png", str(i))) + for i in range(20) + ] [t.start() for t in threads] [t.join() for t in threads] assert len(N.read_notes(d)["files"]) == 20 @@ -1254,7 +1312,9 @@ def test_workspace_note(tmp_path): assert N.read_workspace_note(root) is None N.set_workspace_note(root, "the anthem project") assert N.read_workspace_note(root) == "the anthem project" - assert json.load(open(tmp_path / N.WORKSPACE_FILE_NAME)) == {"notes": "the anthem project"} + assert json.load(open(tmp_path / N.WORKSPACE_FILE_NAME)) == { + "notes": "the anthem project" + } N.set_workspace_note(root, None) assert not os.path.exists(tmp_path / N.WORKSPACE_FILE_NAME) @@ -1372,7 +1432,9 @@ def clean_note(note): if not note.strip(): return None if len(note) > MAX_NOTE_LENGTH: - raise NoteTooLong(f"a note is at most {MAX_NOTE_LENGTH} characters, got {len(note)}") + raise NoteTooLong( + f"a note is at most {MAX_NOTE_LENGTH} characters, got {len(note)}" + ) return note @@ -1395,7 +1457,9 @@ def read_notes(directory): data = _read_json(os.path.join(directory, NOTES_FILE_NAME)) run = data.get("run") if isinstance(data.get("run"), str) else None files = data.get("files") if isinstance(data.get("files"), dict) else {} - files = {k: v for k, v in files.items() if isinstance(k, str) and isinstance(v, str)} + files = { + k: v for k, v in files.items() if isinstance(k, str) and isinstance(v, str) + } return {"run": run, "files": files} @@ -1688,13 +1752,17 @@ Expected: FAIL with 404 or 405 on `/api/notes`. In `dw/server/routes/media.py`: ```python -class NoteRequest(BaseModel): # or define in api_models.py next to the response, per file convention +class NoteRequest( + BaseModel +): # or define in api_models.py next to the response, per file convention target: str note: Optional[str] = None @router.put("/api/notes", response_model=NoteSet, response_model_exclude_unset=True) -def put_note(request: Request, body: NoteRequest, ws: Workspace = Depends(selected_workspace)): +def put_note( + request: Request, body: NoteRequest, ws: Workspace = Depends(selected_workspace) +): """Set or clear the note on a run, a file, an asset or a workspace.""" try: note = clean_note(body.note) @@ -1712,7 +1780,11 @@ def put_note(request: Request, body: NoteRequest, ws: Workspace = Depends(select stored, source = effective_note(read_notes(directory), name) except OSError as e: raise HTTPException(status_code=500, detail=f"Could not write the note: {e}") - return {"target": body.target, "note": stored, "note_source": source if stored else None} + return { + "target": body.target, + "note": stored, + "note_source": source if stored else None, + } ``` Write `resolve_note_target(state, ws, target) -> (directory, name, kind)` @@ -1847,11 +1919,18 @@ def test_set_note_puts_target_and_note(fake_server): result = set_note(target="wf/20261012-000000-aaaa", note="shift 5") assert fake_server.last_request.method == "PUT" assert fake_server.last_request.path == "/api/notes" - assert fake_server.last_request.json == {"target": "wf/20261012-000000-aaaa", "note": "shift 5"} + assert fake_server.last_request.json == { + "target": "wf/20261012-000000-aaaa", + "note": "shift 5", + } def test_run_workflow_forwards_note(fake_server): - run_workflow(workflow_path="templates/text-to-image", note="seed sweep 1/4", acknowledged_cost=True) + run_workflow( + workflow_path="templates/text-to-image", + note="seed sweep 1/4", + acknowledged_cost=True, + ) assert fake_server.last_job_body["note"] == "seed sweep 1/4" From 33a591b14de8f81d17a26a23921ad25c7f3e3afb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:03:50 -0500 Subject: [PATCH 017/204] fix(roles): take the role basis and inputs as the run opens, before realization rewrites the steps The terminal manifest's inputs were always [] because RunRecord aliased the live step list, which realize_args rewrites in place. open_run now records role_basis (name, declared role, previous_result references) and the asset:/output: inputs - including those a variable carries - up front. roles.py gains fallback_roles for a run whose steps cannot be read. Co-Authored-By: Claude Opus 5.5 --- dw/roles.py | 89 +++++++++++++++++++++++--------- dw/workflow_run.py | 19 ++++--- tests/test_roles.py | 25 +++++++++ tests/test_run_manifest_roles.py | 60 ++++++++++++++++++++- 4 files changed, 160 insertions(+), 33 deletions(-) diff --git a/dw/roles.py b/dw/roles.py index 60dcf1bb..b21d7fb7 100644 --- a/dw/roles.py +++ b/dw/roles.py @@ -16,8 +16,10 @@ does not make one out of it. A composed child's rolled-up manifest entries carry `parent_step` and take -that step's role. The engine records roles when a run ends; the server -infers them with the same function for a run that predates this. Inputs +that step's role. The engine records roles when a run ends, from the +`role_basis` it took as the run opened; the server infers them with the +same function for a run that predates this, and falls back to +`fallback_roles` when the run's steps cannot be read. Inputs are the asset: and output: references the expanded steps carry - a run's 'made from' list. This module names no model. """ @@ -41,6 +43,15 @@ ) +def _subfolder_role(subfolder): + """Rule 2: the role a subfolder's head names, or None.""" + if isinstance(subfolder, str): + head = subfolder.split("/", 1)[0] + if head in ROLES: + return head + return None + + def _declared_role(step): """Rules 1 and 2: the role a step's result block states, or None.""" result = step.get("result") @@ -49,12 +60,7 @@ def _declared_role(step): role = result.get(ROLE_KEY) if role in ROLES: return role - subfolder = result.get("subfolder") - if isinstance(subfolder, str): - head = subfolder.split("/", 1)[0] - if head in ROLES: - return head - return None + return _subfolder_role(result.get("subfolder")) def _writes_only_text(files): @@ -77,24 +83,43 @@ def _files_by_step(manifest_steps): return files -def step_roles(steps, manifest_steps): - """Each step's role, by name - see the module docstring for the rules.""" - steps = [s for s in steps or [] if isinstance(s, dict) and "name" in s] +def role_basis(steps): + """What the rules read from expanded steps: each step's name, its + declared role (rules 1 and 2) and the previous_result references it + carries. A run rewrites its steps' references in place as they resolve, + so the engine takes this as the run opens, before any step runs.""" + basis = [] + for step in steps or []: + if not isinstance(step, dict) or "name" not in step: + continue + consumes = [ + reference + for _path, reference, _via in iter_previous_result_references( + step, descend_into_from=True + ) + if isinstance(reference, str) + ] + basis.append( + { + "name": step["name"], + "declared": _declared_role(step), + "consumes": consumes, + } + ) + return basis + + +def step_roles(steps, manifest_steps, basis=None): + """Each step's role, by name - see the module docstring for the rules. + `basis` is `role_basis(steps)` taken earlier, in place of `steps`.""" + basis = role_basis(steps) if basis is None else basis files = _files_by_step(manifest_steps) - names = [s["name"] for s in steps] - roles = {} - for step in steps: - declared = _declared_role(step) - if declared: - roles[step["name"]] = declared - for consumer in steps: + names = [b["name"] for b in basis] + roles = {b["name"]: b["declared"] for b in basis if b["declared"]} + for consumer in basis: if _writes_only_text(files.get(consumer["name"])): continue - for _path, reference, _via in iter_previous_result_references( - consumer, descend_into_from=True - ): - if not isinstance(reference, str): - continue + for reference in consumer["consumes"]: target = max( (n for n in names if reference_resolves_to(reference, n)), key=len, @@ -107,9 +132,9 @@ def step_roles(steps, manifest_steps): return roles -def annotate_roles(steps, manifest_steps): +def annotate_roles(steps, manifest_steps, basis=None): """Copies of `manifest_steps`, each with its step's `role`.""" - roles = step_roles(steps, manifest_steps) + roles = step_roles(steps, manifest_steps, basis) annotated = [] for entry in manifest_steps or []: if not isinstance(entry, dict): @@ -120,6 +145,20 @@ def annotate_roles(steps, manifest_steps): return annotated +def fallback_roles(manifest_steps): + """Copies of `manifest_steps` for a run whose steps cannot be read (no + realized workflow, or one that no longer expands): rule 2 against each + entry's recorded subfolder, and a run of one step is final. Any other + entry is left without a role - unknown, not guessed.""" + entries = [e for e in manifest_steps or [] if isinstance(e, dict)] + single = len({e.get("parent_step") or e.get("step") for e in entries}) == 1 + annotated = [] + for entry in entries: + role = _subfolder_role(entry.get("subfolder")) or (FINAL if single else None) + annotated.append({**entry, ROLE_KEY: role} if role else dict(entry)) + return annotated + + def entry_role(entry): """A manifest entry's recorded role, or None when it predates roles.""" role = entry.get(ROLE_KEY) if isinstance(entry, dict) else None diff --git a/dw/workflow_run.py b/dw/workflow_run.py index 0b3ce6df..7d85ee8a 100644 --- a/dw/workflow_run.py +++ b/dw/workflow_run.py @@ -33,7 +33,7 @@ from .pipeline_ownership import allocated_mb, finish_release from .previous_results import StepResults from .realize import realize_workflow -from .roles import annotate_roles, run_inputs +from .roles import annotate_roles, role_basis, run_inputs from .runs import ( FLAT_LAYOUT, REALIZED_FILE_NAME, @@ -105,8 +105,11 @@ class RunRecord: seed: object = None realized_name: object = None annotations: dict = field(default_factory=_empty_annotations) - # The expanded steps the run executed - what roles and inputs are read from - expanded_steps: list = field(default_factory=list) + # Taken from the expanded steps as the run opens, before realization + # rewrites their references in place: what roles are decided from + # (dw/roles.py role_basis) and the asset:/output: inputs it read + role_basis: list = field(default_factory=list) + inputs: list = field(default_factory=list) @dataclass @@ -234,7 +237,7 @@ def write_run_manifest(workflow, record, status=None): # Roles once the run has ended - a running manifest would show the # newest step as final until its consumer runs (dw/roles.py) if status != "running": - steps = annotate_roles(record.expanded_steps, steps) + steps = annotate_roles(None, steps, record.role_basis) write_manifest( workflow._run_dir, { @@ -267,7 +270,7 @@ def write_run_manifest(workflow, record, status=None): # What did not run, and why - a run says what it did not do # as well as what it did (#122) "elided_steps": workflow._elided_steps, - "inputs": run_inputs(record.expanded_steps), + "inputs": record.inputs, "steps": steps, }, ) @@ -627,7 +630,11 @@ def open_run(workflow, prepared, record, run_context): inherits the parent's and never opens its own.""" record.started_at = _now() record.run_id = None - record.expanded_steps = prepared.workflow_def.get("steps") or [] + expanded = prepared.workflow_def.get("steps") or [] + record.role_basis = role_basis(expanded) + # The variables as folded, before their assets loaded: a template's + # input image is usually a variable, gone from the steps by now + record.inputs = run_inputs([*expanded, prepared.recorded_variables]) if not workflow._run_dir_inherited: _claim_run_dir(workflow, prepared, record) diff --git a/tests/test_roles.py b/tests/test_roles.py index 51baf31d..caf0650c 100644 --- a/tests/test_roles.py +++ b/tests/test_roles.py @@ -173,3 +173,28 @@ def test_schema_role_values(role, ok): errors = validate_data_all(workflow, load_schema("workflow")) role_errors = [e for e in errors if "role" in (e["path"] or "")] assert (not role_errors) == ok + + +def test_role_basis_survives_realization_of_the_steps(): + from dw.roles import role_basis + + steps = [ + {"name": "image", "result": {"subfolder": "x"}}, + {"name": "video", "arguments": {"image": "previous_result:image"}}, + ] + basis = role_basis(steps) + steps[1]["arguments"]["image"] = object() # what a run does in place + manifest = [_entry("image", "a.png"), _entry("video", "b.mp4")] + assert step_roles(None, manifest, basis) == {"image": INTERMEDIATE, "video": FINAL} + + +def test_fallback_roles(): + from dw.roles import fallback_roles + + entries = [ + {"step": "a", "files": ["a.png"], "subfolder": "intermediate/x"}, + {"step": "b", "files": ["b.png"], "subfolder": ""}, + ] + assert [e.get("role") for e in fallback_roles(entries)] == [INTERMEDIATE, None] + assert fallback_roles([{"step": "a", "files": ["a.png"]}])[0]["role"] == FINAL + assert fallback_roles([]) == [] diff --git a/tests/test_run_manifest_roles.py b/tests/test_run_manifest_roles.py index 6a3b056c..b3f582bd 100644 --- a/tests/test_run_manifest_roles.py +++ b/tests/test_run_manifest_roles.py @@ -4,6 +4,7 @@ import json from types import SimpleNamespace +from dw.roles import role_basis, run_inputs from dw.workflow_run import RunRecord, write_run_manifest from .test_runs import _workflow_definition, fake_pipeline # noqa: F401 @@ -46,7 +47,9 @@ def test_terminal_manifest_records_roles_and_inputs(tmp_path, fake_pipeline): # def test_a_terminal_manifest_lists_the_assets_the_steps_read(tmp_path): steps = [{"name": "a", "x": "asset:cat.png"}, {"name": "b"}] - record = RunRecord(status="completed", expanded_steps=steps) + record = RunRecord( + status="completed", role_basis=role_basis(steps), inputs=run_inputs(steps) + ) workflow = _stand_in(tmp_path, [{"step": "a", "files": []}]) write_run_manifest(workflow, record) @@ -57,10 +60,63 @@ def test_a_terminal_manifest_lists_the_assets_the_steps_read(tmp_path): def test_a_running_manifest_carries_inputs_but_no_roles(tmp_path): steps = [{"name": "a", "x": "asset:cat.png"}, {"name": "b"}] - record = RunRecord(status="running", expanded_steps=steps) + record = RunRecord( + status="running", role_basis=role_basis(steps), inputs=run_inputs(steps) + ) workflow = _stand_in(tmp_path, [{"step": "a", "files": []}]) write_run_manifest(workflow, record) manifest = json.loads((tmp_path / "manifest.json").read_text()) assert manifest["inputs"] == [{"reference": "asset:cat.png", "kind": "asset"}] assert all("role" not in e for e in manifest["steps"]) + + +def _source_run(tmp_path): + """A finished run of the one-step workflow, and the output: reference + naming the image it wrote.""" + from dw.workflow import Workflow + + Workflow(_workflow_definition(), str(tmp_path), "/w/workflows/Src.json").run({}) + run_dir = next((tmp_path / "Src").iterdir()) + image = next(p.name for p in run_dir.iterdir() if p.suffix == ".png") + return f"output:Src/{run_dir.name}/{image}" + + +def _terminal_manifest(tmp_path, identity): + run_dir = next((tmp_path / identity).iterdir()) + return json.loads((run_dir / "manifest.json").read_text()) + + +def test_a_real_run_records_the_output_a_step_read(tmp_path, fake_pipeline): # noqa: F811 + # The step's output: string is realized in place as the step runs; the + # terminal manifest must still name it (it was always [] before) + from dw.workflow import Workflow + + reference = _source_run(tmp_path) + definition = _two_step_definition() + definition["id"] = "Dst" + definition["steps"][0]["pipeline"]["arguments"]["image"] = reference + Workflow(definition, str(tmp_path), "/w/workflows/Dst.json").run({}) + + manifest = _terminal_manifest(tmp_path, "Dst") + assert manifest["status"] == "completed" + assert manifest["inputs"] == [{"reference": reference, "kind": "output"}] + # and roles still see the previous_result reference realization replaced + assert {e["step"]: e["role"] for e in manifest["steps"]} == { + "gen0": "intermediate", + "second": "final", + } + + +def test_a_real_run_records_an_output_carried_by_a_variable(tmp_path, fake_pipeline): # noqa: F811 + from dw.workflow import Workflow + + reference = _source_run(tmp_path) + definition = _workflow_definition() + definition["id"] = "Var" + definition["variables"] = {"image": reference} + definition["steps"][0]["pipeline"]["arguments"]["image"] = "variable:image" + Workflow(definition, str(tmp_path), "/w/workflows/Var.json").run({}) + + manifest = _terminal_manifest(tmp_path, "Var") + assert manifest["inputs"] == [{"reference": reference, "kind": "output"}] From 4617776b479459b7bfe95aecb1a111df102cc125 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:04:02 -0500 Subject: [PATCH 018/204] fix(server): job detail carries the run's roles and current note; role cache is a locked LRU The worker reports the manifest it held in memory, which has no roles, so the job page never grouped by role. get_job now fills each entry's role from run_roles(), which also infers them for older jobs, and the job detail and list take the note from the run's notes.json when the run directory is there. run_roles lets recorded roles win per entry, falls back to subfolders and single-step runs when the workflow is unreadable, logs why, and caches in a locked OrderedDict LRU of 8192. run_inputs keeps only well-formed entries. Co-Authored-By: Claude Opus 5.5 --- dw/server/api_models.py | 4 + dw/server/job_history.py | 6 +- dw/server/jobs.py | 16 ++- dw/server/outputs.py | 135 +++++++++++++++++--- dw/server/routes/jobs.py | 8 ++ tests/test_gallery_roles.py | 198 +++++++++++++++++++++++++++++ tests/test_notes_api.py | 35 +++++ ui/src/lib/generated/api-schema.ts | 6 + ui/src/lib/generated/openapi.json | 9 ++ 9 files changed, 393 insertions(+), 24 deletions(-) diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 8510aed5..bfe9d84d 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -396,6 +396,10 @@ class ManifestEntry(ApiModel): "earlier run's, republished, and nothing was generated for them this time." ) parent_step: str = sometimes("The composing step a rolled-up entry came from.") + role: Literal["final", "intermediate"] = sometimes( + "The step's role in its run, from the run's manifest.json or inferred " + "for an older run. Absent while the run is going, or when unknown." + ) class JobProgress(ApiModel): diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 6ec6d3de..0b1957d9 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -215,7 +215,7 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): query = ( "SELECT id, workflow, status, created_at, started_at, finished_at," " workspace, workflow_name, run_id, acknowledged, run_version," - " device, note FROM jobs" + " device, note, run_dir, json_extract(spec, '$.output_dir') FROM jobs" ) params = [] clauses = [] @@ -250,6 +250,10 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): "historical": True, # Only when set: JobSummary.note is absent, never null **({"note": row[12]} if row[12] else {}), + # Where the run is, for the list to read its current note + # from; JobManager.list takes these off before serving + "_run_dir": row[13], + "_output_dir": row[14], } for row in rows ] diff --git a/dw/server/jobs.py b/dw/server/jobs.py index 33cc5090..a9c8b582 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -58,6 +58,7 @@ from ..workspace import DEFAULT_WORKSPACE_NAME from . import job_results, pool, worker_memory from .job_history import JobHistory +from .outputs import job_run_directory, with_run_note from .job_record import ( ACK_NONE, CANCELLED, @@ -602,15 +603,26 @@ def list(self, workspace=None, statuses=None): continue if job.id in positions: summary["queue_position"] = positions[job.id] - summaries.append(summary) + output_dir = job.spec.get("output_dir") + summaries.append(self._current_note(summary, output_dir, job.run_dir)) for historical in self.history.recent_summaries( workspace=workspace, statuses=statuses ): + run_dir = historical.pop("_run_dir", None) + output_dir = historical.pop("_output_dir", None) if historical["id"] not in live_ids: - summaries.append(historical) + summaries.append(self._current_note(historical, output_dir, run_dir)) summaries.sort(key=lambda summary: summary["created_at"] or 0) return summaries + def _current_note(self, summary, output_dir, run_dir): + """The summary with its run's current note (outputs.with_run_note), + when the run directory is still there.""" + if not run_dir: + return summary + root = validate_output_path(output_dir or self.output_dir, None) + return with_run_note(summary, job_run_directory(root, run_dir)) + # ------------------------------------------------------------ cancel/stop def cancel(self, job_id): diff --git a/dw/server/outputs.py b/dw/server/outputs.py index 78b3e522..792e1869 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -11,7 +11,9 @@ import os import shutil import tempfile +import threading import zipfile +from collections import OrderedDict from datetime import datetime from urllib.parse import quote @@ -31,7 +33,8 @@ LibraryRoot, library_path, ) -from ..roles import annotate_roles, entry_role +from ..notes import read_notes +from ..roles import annotate_roles, entry_role, fallback_roles from ..runs import ( MANIFEST_FILE_NAME, REALIZED_FILE_NAME, @@ -521,9 +524,14 @@ def run_directory(name, root): # Roles by (run directory, manifest mtime): a listing asks for every file -# of a run, and the manifest it came from has not changed since the last ask -_ROLE_CACHE = {} -_ROLE_CACHE_LIMIT = 512 +# of a run, and the manifest it came from has not changed since the last ask. +# Least recently used goes first; an entry is a small dict, so the limit +# sits well above the runs one workspace holds. Listings run on the +# threadpool, so every touch holds the lock +_ROLE_CACHE = OrderedDict() +_ROLE_CACHE_LIMIT = 8192 +_ROLE_CACHE_LOCK = threading.Lock() +_INPUT_KINDS = ("asset", "output") def _realized_steps(run_dir): @@ -533,15 +541,25 @@ def _realized_steps(run_dir): with open(path) as handle: definition = json.load(handle) return Workflow(definition, None, path).expanded_definition().get("steps") or [] - except Exception: # an unreadable or unexpandable record leaves roles unknown + except Exception as e: # an unreadable or unexpandable record + logger.debug(f"No expanded steps for the roles of {run_dir}: {e}") return None +def _inferred_roles(run_dir, steps): + """Manifest entries annotated from the realized workflow, or by the + subfolder and single-step fallback when it cannot be read.""" + expanded = _realized_steps(run_dir) + if expanded is None: + return fallback_roles(steps) + return annotate_roles(expanded, steps) + + def run_roles(run_dir): """Each file of a run, by path relative to the run directory ('/' - separated), mapped to its role. Recorded roles win; a run that predates - them is inferred from its realized workflow. {} when neither is - readable.""" + separated), mapped to its role. Recorded roles win; an entry without one + (a run that predates them) is inferred from the realized workflow, or + from its subfolder. {} when the manifest is unreadable.""" try: key = ( run_dir, @@ -549,27 +567,102 @@ def run_roles(run_dir): ) except OSError: return {} - if key in _ROLE_CACHE: - return _ROLE_CACHE[key] + with _ROLE_CACHE_LOCK: + if key in _ROLE_CACHE: + _ROLE_CACHE.move_to_end(key) + return _ROLE_CACHE[key] manifest = _read_manifest(run_dir) or {} steps = [e for e in manifest.get("steps") or [] if isinstance(e, dict)] - if steps and any(entry_role(e) is None for e in steps): - expanded = _realized_steps(run_dir) - if expanded is not None: - steps = annotate_roles(expanded, steps) + inferred = steps + if any(entry_role(e) is None for e in steps): + inferred = _inferred_roles(run_dir, steps) roles = {} - for entry in steps: - role = entry_role(entry) + for entry, guess in zip(steps, inferred): + role = entry_role(entry) or entry_role(guess) if role: for name in entry.get("files") or []: roles[str(name).replace(os.sep, "/")] = role - if len(_ROLE_CACHE) >= _ROLE_CACHE_LIMIT: - _ROLE_CACHE.pop(next(iter(_ROLE_CACHE))) - _ROLE_CACHE[key] = roles + with _ROLE_CACHE_LOCK: + _ROLE_CACHE[key] = roles + _ROLE_CACHE.move_to_end(key) + while len(_ROLE_CACHE) > _ROLE_CACHE_LIMIT: + _ROLE_CACHE.popitem(last=False) return roles def run_inputs(run_dir): - """The inputs a run's manifest recorded, or None when it has none.""" + """The inputs a run's manifest recorded, or None when it has none. Only + well-formed entries: a str reference of a kind the engine records.""" inputs = (_read_manifest(run_dir) or {}).get("inputs") - return inputs if isinstance(inputs, list) else None + if not isinstance(inputs, list): + return None + return [ + {"reference": entry["reference"], "kind": entry["kind"]} + for entry in inputs + if isinstance(entry, dict) + and isinstance(entry.get("reference"), str) + and entry.get("kind") in _INPUT_KINDS + ] + + +def job_run_directory(root, run_dir): + """A job's run directory under its validated output root, or None when + the job never opened one or the directory is gone.""" + if not run_dir: + return None + try: + path = validate_path(os.path.join(root, run_dir), root, allow_create=False) + except SecurityError: + return None + return path if os.path.isdir(path) else None + + +def with_run_note(summary, path): + """`summary` with the run's current note from its notes.json - a note + edited after submission replaces the one the job was submitted with, + and a cleared one removes it. Unchanged when `path` is None.""" + if path is None: + return summary + summary = {key: value for key, value in summary.items() if key != "note"} + note = read_notes(path)["run"] + return {**summary, "note": note} if note else summary + + +def with_run_state(detail, root, run_dir, roles=True): + """A job's detail with what its run directory says now: the run note, + and each manifest entry's role. The worker reports the manifest it held + in memory, which carries no roles, and a job recorded before roles has + none - so each entry takes the role `run_roles` gives its files, a + recorded one first. `roles` false leaves the manifest alone.""" + path = job_run_directory(root, run_dir) + if path is None: + return detail + detail = with_run_note(detail, path) + roles = run_roles(path) if roles else None + manifest = detail.get("manifest") + if not roles or not isinstance(manifest, list): + return detail + entries = [] + for entry in manifest: + if isinstance(entry, dict) and entry_role(entry) is None: + role = next( + ( + roles[name] + for name in ( + _run_relative(root, path, f) for f in entry.get("files") or [] + ) + if name in roles + ), + None, + ) + if role: + entry = {**entry, "role": role} + entries.append(entry) + return {**detail, "manifest": entries} + + +def _run_relative(root, run_path, name): + """A job manifest's file name (relative to the output root) as the run + directory's manifest names it, '/' separated.""" + full = os.path.normpath(os.path.join(root, str(name))) + return os.path.relpath(full, run_path).replace(os.sep, "/") diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index b4a99b8d..ae1b0bfd 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -75,6 +75,7 @@ delete_run_directory, output_kinds, served_url, + with_run_state, ) logger = logging.getLogger("dw") @@ -263,6 +264,13 @@ def get_job(request: Request, job_id: str): # field of its own, or get_job_workflow's, so it is not served detail = job if isinstance(job, dict) else manager.describe(job) detail = {key: value for key, value in detail.items() if key != "spec"} + # The run as it stands now: its current note, and once it has ended + # each entry's role - the engine records none on a running manifest + location = manager.run_location(job_id) + if location is not None: + detail = with_run_state( + detail, *location, roles=detail.get("status") in TERMINAL_STATES + ) return {**detail, "output_kinds": output_kinds(detail.get("manifest"))} diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index edc90897..571c9c83 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -2,6 +2,7 @@ scratch: each entry's `role`, the listing's `role` filter, and the metadata's `role` and `inputs` (dw/roles.py decides the roles).""" +import json import os from dw.server.outputs import run_roles, split_run_name @@ -129,3 +130,200 @@ def test_metadata_for_a_subfolder_file_of_a_nested_identity_run( ).json() assert meta["role"] == "final" assert meta["inputs"] == [{"reference": "output:wf/v1/a.png", "kind": "output"}] + + +def _role_less_run_script(output_dir): + """A worker that writes a two-step run - b reads a's result - whose + manifest.json predates roles, and reports the manifest it held in + memory, which never carries them.""" + from dw.runs import new_run_id + + def script(command): + run_id = new_run_id({"command": 1}) + relative = f"wf/{run_id}" + run_dir = os.path.join(output_dir, "wf", run_id) + os.makedirs(run_dir, exist_ok=True) + realized = { + "id": "wf", + "steps": [ + {"name": "a", "task": {"command": "noop", "arguments": {}}}, + { + "name": "b", + "task": { + "command": "noop", + "arguments": {"image": "previous_result:a"}, + }, + }, + ], + } + steps = [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}] + for name in ("a.png", "b.png"): + with open(os.path.join(run_dir, name), "wb") as handle: + handle.write(b"\x89PNG\r\n\x1a\n") + with open(os.path.join(run_dir, "manifest.json"), "w") as handle: + json.dump({"run_id": run_id, "status": "succeeded", "steps": steps}, handle) + with open(os.path.join(run_dir, "workflow.json"), "w") as handle: + json.dump(realized, handle) + yield { + "type": "progress", + "event": "run_start", + "run_id": run_id, + "identity": "wf", + "run_dir": relative, + "version": 1, + } + manifest = [ + {"step": e["step"], "files": [os.path.join(run_dir, e["files"][0])]} + for e in steps + ] + yield {"type": "success", "message": "ok", "run_count": 1, "manifest": manifest} + + return script + + +def test_job_detail_entries_carry_the_run_roles(tmp_path): + """The worker reports a role-less manifest; the job detail fills each + entry's role from the run directory, live and from history.""" + from fastapi.testclient import TestClient + + from dw.server.app import create_app + from dw.server.jobs import JobManager + from dw.workspace import Workspace + + from .test_server import ScriptedWorkerManager, valid_workflow, wait_for_status + + studio = Workspace(tmp_path / "studio", "flag").ensure() + manager = JobManager( + studio.outputs, + worker_manager=ScriptedWorkerManager(_role_less_run_script(studio.outputs)), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=studio.workflows, + ) + app = create_app( + workflow_dir=studio.workflows, + output_dir=studio.outputs, + job_manager=manager, + prompt_dir=studio.prompts, + asset_dir=studio.assets, + workspace=studio.root, + ) + with TestClient(app, base_url="http://localhost") as client: + job = client.post("/api/jobs", json={"workflow": valid_workflow()}).json() + detail = wait_for_status(client, job["id"], ["succeeded"]) + assert all("role" not in e for e in manager.jobs[job["id"]].manifest) + assert {e["step"]: e["role"] for e in detail["manifest"]} == { + "a": "intermediate", + "b": "final", + } + manager.jobs.pop(job["id"]) + again = client.get(f"/api/jobs/{job['id']}").json() + assert {e["step"]: e["role"] for e in again["manifest"]} == { + "a": "intermediate", + "b": "final", + } + + +def test_recorded_roles_win_in_a_mixed_manifest(tmp_path, write_run): + # a says final although b reads it; inference would call it intermediate + realized = { + "id": "wf", + "steps": [ + {"name": "a", "task": {"command": "noop", "arguments": {}}}, + { + "name": "b", + "task": {"command": "noop", "arguments": {"x": "previous_result:a"}}, + }, + ], + } + run_dir = write_run( + str(tmp_path), + "wf", + "20261012-000010-aaaaaaaa", + [ + {"step": "a", "files": ["a.png"], "role": "final"}, + {"step": "b", "files": ["b.png"]}, + ], + realized=realized, + ) + assert run_roles(run_dir) == {"a.png": "final", "b.png": "final"} + + +def test_a_run_without_its_workflow_falls_back_to_subfolders(tmp_path, write_run): + run_dir = write_run( + str(tmp_path), + "wf", + "20261012-000011-aaaaaaaa", + [ + {"step": "a", "files": ["intermediate/a.png"], "subfolder": "intermediate"}, + {"step": "b", "files": ["final/x/b.png"], "subfolder": "final/x"}, + {"step": "c", "files": ["c.png"], "subfolder": ""}, + ], + ) + assert run_roles(run_dir) == { + "intermediate/a.png": "intermediate", + "final/x/b.png": "final", + } + + +def test_a_single_step_run_without_its_workflow_is_final(tmp_path, write_run): + run_dir = write_run( + str(tmp_path), + "wf", + "20261012-000012-aaaaaaaa", + [{"step": "a", "files": ["a.png", "a2.png"]}], + realized={"not": "a workflow"}, + ) + assert run_roles(run_dir) == {"a.png": "final", "a2.png": "final"} + + +def test_run_inputs_keeps_only_well_formed_entries(tmp_path, write_run): + from dw.server.outputs import run_inputs + + run_dir = write_run( + str(tmp_path), + "wf", + "20261012-000013-aaaaaaaa", + [], + inputs=[ + {"reference": "asset:a.png", "kind": "asset"}, + {"reference": 3, "kind": "asset"}, + {"reference": "x", "kind": "variable"}, + "asset:b.png", + {"reference": "output:wf/v1/a.png", "kind": "output", "extra": 1}, + ], + ) + assert run_inputs(run_dir) == [ + {"reference": "asset:a.png", "kind": "asset"}, + {"reference": "output:wf/v1/a.png", "kind": "output"}, + ] + + +def test_role_cache_is_lru_and_safe_under_threads(tmp_path, write_run, monkeypatch): + from concurrent.futures import ThreadPoolExecutor + + from dw.server import outputs + + monkeypatch.setattr(outputs, "_ROLE_CACHE", outputs.OrderedDict()) + monkeypatch.setattr(outputs, "_ROLE_CACHE_LIMIT", 4) + runs = [ + write_run( + str(tmp_path), + "wf", + f"20261012-0001{i:02d}-aaaaaaaa", + [{"step": "a", "files": ["a.png"], "role": "final"}], + ) + for i in range(12) + ] + with ThreadPoolExecutor(8) as pool: + results = list(pool.map(run_roles, runs * 20)) + assert all(r == {"a.png": "final"} for r in results) + assert len(outputs._ROLE_CACHE) == 4 + + # the most recently used survives a newcomer; the oldest goes + outputs._ROLE_CACHE.clear() + for run in runs[:4]: + run_roles(run) + run_roles(runs[0]) + run_roles(runs[4]) + kept = {key[0] for key in outputs._ROLE_CACHE} + assert runs[0] in kept and runs[1] not in kept diff --git a/tests/test_notes_api.py b/tests/test_notes_api.py index e1ec8d38..0dc65c87 100644 --- a/tests/test_notes_api.py +++ b/tests/test_notes_api.py @@ -507,3 +507,38 @@ def test_note_set_during_run_survives_manifest_rewrite(tmp_path): assert N.read_notes(d)["run"] == "mid-run" with open(os.path.join(d, "manifest.json")) as handle: assert "mid-run" not in json.dumps(json.load(handle)) + + +def test_job_detail_and_list_show_the_current_run_note(client, output_root): + """A run note edited after submission replaces the submitted one on + the job, live and from history, and a cleared one removes it.""" + job = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "note": "A"} + ).json() + detail = wait_for_status(client, job["id"], ["succeeded"]) + r = client.put("/api/notes", json={"target": detail["run_dir"], "note": "B"}) + assert r.status_code == 200 + + def shown(): + listed = {j["id"]: j for j in client.get("/api/jobs").json()["jobs"]} + return ( + client.get(f"/api/jobs/{job['id']}").json().get("note"), + listed[job["id"]].get("note"), + ) + + assert shown() == ("B", "B") + client.app.state.job_manager.jobs.pop(job["id"]) # from sqlite now + assert shown() == ("B", "B") + client.put("/api/notes", json={"target": detail["run_dir"], "note": None}) + assert shown() == (None, None) + + +def test_a_job_whose_run_is_gone_keeps_its_submitted_note(client, output_root): + import shutil + + job = client.post( + "/api/jobs", json={"workflow": valid_workflow(), "note": "A"} + ).json() + detail = wait_for_status(client, job["id"], ["succeeded"]) + shutil.rmtree(os.path.join(output_root, *detail["run_dir"].split("/"))) + assert client.get(f"/api/jobs/{job['id']}").json()["note"] == "A" diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 3dc95514..f07b26ad 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -2809,6 +2809,12 @@ export interface components { * @description The step was served from the step cache: these files are an earlier run's, republished, and nothing was generated for them this time. */ reused?: boolean; + /** + * Role + * @description The step's role in its run, from the run's manifest.json or inferred for an older run. Absent while the run is going, or when unknown. + * @enum {string} + */ + role?: "final" | "intermediate"; /** Step */ step: string; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 0344ed61..5b42e948 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -2551,6 +2551,15 @@ "title": "Reused", "type": "boolean" }, + "role": { + "description": "The step's role in its run, from the run's manifest.json or inferred for an older run. Absent while the run is going, or when unknown.", + "enum": [ + "final", + "intermediate" + ], + "title": "Role", + "type": "string" + }, "step": { "title": "Step", "type": "string" From 4d72c1a74fc3d1120f60eeb1b356aaafb11dba7b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:04:59 -0500 Subject: [PATCH 019/204] fix(assets): keep with an empty note keeps none; an upload clears a kept record only when it replaced a file Co-Authored-By: Claude Opus 5.5 --- dw/server/routes/assets.py | 22 +++++++----- tests/test_notes_api.py | 58 ++++++++++++++++++++++++++++++ ui/src/lib/generated/api-schema.ts | 2 +- ui/src/lib/generated/openapi.json | 2 +- 4 files changed, 74 insertions(+), 10 deletions(-) diff --git a/dw/server/routes/assets.py b/dw/server/routes/assets.py index 9669fefe..92946741 100644 --- a/dw/server/routes/assets.py +++ b/dw/server/routes/assets.py @@ -221,17 +221,21 @@ async def upload_media( except SecurityError as e: raise HTTPException(status_code=400, detail=str(e)) os.makedirs(os.path.dirname(dest), exist_ok=True) + # Taken before the write: only a file this one replaces can have left a + # kept record behind, and a fresh name leaves the directory's alone + replaced = os.path.exists(dest) # Off the event loop: a 200 MB write would otherwise stall every SSE # stream and poll for its duration await run_in_threadpool(_write_bytes, dest, body) logger.info(f"Saved upload {filename!r} -> {dest}") _describe_new_asset(dest, note) - try: - # A kept file this one replaced left its provenance and shots behind - record_kept_shots(os.path.dirname(dest), os.path.basename(dest), None, None) - except OSError as e: - logger.warning(f"Could not clear the kept record beside {dest}: {e}") + if replaced: + try: + # A kept file this one replaced left its provenance and shots + record_kept_shots(os.path.dirname(dest), os.path.basename(dest), None, None) + except OSError as e: + logger.warning(f"Could not clear the kept record beside {dest}: {e}") path = f"/inputs/{UPLOADS_SUBDIR}/{quote(name)}" result = { "reference": make_ref(ASSET, f"{UPLOADS_SUBDIR}/{name}"), @@ -388,7 +392,8 @@ class KeepRequest(BaseModel): note: Optional[str] = Field( default=None, description=f"A note on the kept asset, at most {MAX_NOTE_LENGTH} " - "characters; the source file's effective note when omitted", + "characters; the source file's effective note when omitted, and " + "none when empty", ) @@ -420,8 +425,9 @@ def keep_output_as_asset( kept_name = strip_output_prefix(body.name) source = resolve_output_file(request.app.state, kept_name, ws.outputs) - if note is None: - # The note the file carried where it was made: its own, else its run's + if body.note is None: + # The note the file carried where it was made: its own, else its + # run's. Only when no note was given - an empty one keeps none split = validated_run_split(ws.outputs, source) if split: note = effective_note(read_notes(split[0]), split[1])[0] diff --git a/tests/test_notes_api.py b/tests/test_notes_api.py index 0dc65c87..45ccd329 100644 --- a/tests/test_notes_api.py +++ b/tests/test_notes_api.py @@ -542,3 +542,61 @@ def test_a_job_whose_run_is_gone_keeps_its_submitted_note(client, output_root): detail = wait_for_status(client, job["id"], ["succeeded"]) shutil.rmtree(os.path.join(output_root, *detail["run_dir"].split("/"))) assert client.get(f"/api/jobs/{job['id']}").json()["note"] == "A" + + +def test_keep_with_an_empty_note_keeps_none(client, output_root, make_run): + run = run_name(make_run("wf", ["a.png"]), output_root) + client.put("/api/notes", json={"target": run, "note": "the run note"}) + kept = client.post("/api/assets/keep", json={"name": f"{run}/a.png", "note": ""}) + assert kept.status_code == 201 + assert "note" not in asset_entry(client, "a.png") + + +def _keep_with_provenance(client, run, asset_name): + history = client.app.state.job_manager.history + history.job_for_file = lambda name, workspace=None: { + "id": "abc", + "status": "succeeded", + } + kept = client.post( + "/api/assets/keep", json={"name": f"{run}/x.png", "asset_name": asset_name} + ) + assert kept.status_code == 201 + meta = client.get(f"/api/gallery/asset:{asset_name}/metadata").json() + assert meta["job"]["id"] == "abc" + + +def test_an_upload_over_a_kept_name_clears_its_provenance( + client, output_root, make_run +): + run = run_name(make_run("wf", ["x.png"]), output_root) + _keep_with_provenance(client, run, "uploads/x.png") + up = client.post( + "/api/uploads", params={"filename": "x.png", "asset_name": "x.png"}, content=PNG + ) + assert up.status_code == 201 + assert client.get("/api/gallery/asset:uploads/x.png/metadata").json()["job"] is None + + +def test_an_upload_under_a_fresh_name_leaves_other_provenance( + client, studio, output_root, make_run +): + run = run_name(make_run("wf", ["x.png"]), output_root) + _keep_with_provenance(client, run, "uploads/x.png") + up = client.post( + "/api/uploads", params={"filename": "y.png", "asset_name": "y.png"}, content=PNG + ) + assert up.status_code == 201 + meta = client.get("/api/gallery/asset:uploads/x.png/metadata").json() + assert meta["job"]["id"] == "abc" + + # a kept record caught mid-write is not the upload's to rewrite + manifest = os.path.join(studio.assets, "uploads", "manifest.json") + with open(manifest, "w") as handle: + handle.write('{"steps": [') + up = client.post( + "/api/uploads", params={"filename": "z.png", "asset_name": "z.png"}, content=PNG + ) + assert up.status_code == 201 + with open(manifest) as handle: + assert handle.read() == '{"steps": [' diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index f07b26ad..3156f09c 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -2722,7 +2722,7 @@ export interface components { name: string; /** * Note - * @description A note on the kept asset, at most 4000 characters; the source file's effective note when omitted + * @description A note on the kept asset, at most 4000 characters; the source file's effective note when omitted, and none when empty */ note?: string | null; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 5b42e948..5b5956fd 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -2392,7 +2392,7 @@ "type": "null" } ], - "description": "A note on the kept asset, at most 4000 characters; the source file's effective note when omitted", + "description": "A note on the kept asset, at most 4000 characters; the source file's effective note when omitted, and none when empty", "title": "Note" }, "overwrite": { From cfd0711308e6f30a85fa32baa6e79c0f76a8e012 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:06:15 -0500 Subject: [PATCH 020/204] fix(mcp): set_note takes the per-call workspace pin; MCP.md run_workflow row and the only_orphans sentence restored Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 4 ++-- dw_mcp/assets.py | 10 +++++++--- dw_mcp/tools_catalog.py | 7 +++---- dw_mcp/tools_media.py | 9 ++++++--- tests/test_mcp_diagnose.py | 10 ++++++++++ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index 0f8a1a74..920f1c59 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -281,7 +281,7 @@ tools) rather than relying on the session pin. | `delete_workspace(name, acknowledged_cost=False)` | `name`, `acknowledged_cost` | Permanently delete a workspace and everything in it. Refuses without the acknowledgement, reporting what it would remove | | `list_assets(limit=50, prefix)` | `limit`, `prefix` | The input media on the server, each with the `asset:` reference a workflow argument carries. Look here before asking for a file - what a workflow needs may already be there. `libraries` names the roots searched and which are writable; `shadowed` lists names a nearer library hides. Newest first; `prefix` keeps names starting with it (e.g. `cast/`), `limit` bounds the answer, `total` is the count matched before the cut | | `keep_output(name, asset_name=None, overwrite=False, shared=False, workspace=None, note=None)` | `name`, optional `asset_name`, `overwrite`, `shared`, `workspace`, `note` | Keep a generated file as an input asset under a stable `asset:` name, so a later workflow can rely on it. The copy happens on the server: nothing is downloaded or re-uploaded. `note` is kept with the asset; left out, the source file's note is copied. `asset_name` may name a folder and takes the kept file's extension when it has none; `shared=true` keeps it in the library every workspace shares, which is where a recurring cast belongs. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | -| `set_note(target, note=None)` | `target` (a run `/`, a gallery file name, `asset:` or `workspace:`), `note` | Set the note on a run, an output file, an asset or a workspace; null or `""` clears it. Plain text up to 4000 characters saying what the thing is for - what a run tested, why a file is the keeper. Free: no `acknowledged_cost`. Returns `{target, note, note_source}`. `run_workflow(note=...)` sets a run's note at submission and it shows on every file the run wrote; `list_gallery`, `list_assets`, `list_workspaces` and `list_jobs` carry each note clipped to 200 characters (`note_truncated: true` when cut), while `get_job` and `get_gallery_metadata` return it whole | +| `set_note(target, note=None, workspace=None)` | `target` (a run `/`, a gallery file name, `asset:` or `workspace:`), `note`, `workspace` | Set the note on a run, an output file, an asset or a workspace; null or `""` clears it. Plain text up to 4000 characters saying what the thing is for - what a run tested, why a file is the keeper. Free: no `acknowledged_cost`. Returns `{target, note, note_source}`. `run_workflow(note=...)` sets a run's note at submission and it shows on every file the run wrote; `list_gallery`, `list_assets`, `list_workspaces` and `list_jobs` carry each note clipped to 200 characters (`note_truncated: true` when cut), while `get_job` and `get_gallery_metadata` return it whole. `get_job` shows the run's current note, so one set here after submission replaces the submitted one. `workspace` names the workspace a run or file name is read in for this one call without switching the session to it - the same pin `run_workflow` takes | | `upload_asset(file_path=None, content=None, asset_name=None, shared=False, note=None)` | exactly one of `file_path` or `content` (base64), optional `asset_name`, `shared`, `note` (kept with the asset) | Push an image, video or audio file, or a `.cube` 3D LUT for `apply_lut`, into the server's asset library and get back its `asset:` reference. `file_path` is read from the machine the MCP server runs on, so this is how an input reaches a dw.serve running somewhere else - but only when the caller shares a filesystem with that machine. `content` is the alternative for an agent that does not: the bytes travel inline, base64-encoded, in the tool call itself, capped at 4MB (well under `file_path`'s 200MB) because inline bytes compete with the calling agent's own context budget rather than being a bulk-transfer path (#203). `asset_name` is required with `content` (there is no file name to infer one from) and, either way, stores it under a readable name (`cast/priya-voice.wav`) instead of a random one; `shared=true` puts it in the library every workspace shares | | `delete_asset(name)` | `name` | Permanently remove one file from the asset library, by the name `list_assets` reports. Deletes from whichever library holds it - this workspace's own before the shared one; one from a read-only examples library is refused. Any workflow still carrying that `asset:` reference stops loading | | `save_workflow(name, workflow)` | `name`, `workflow` | Save a workflow into the server's writable workflow directory, overwriting any existing workflow of that name there. A name that currently resolves to a read-only source (an examples directory) is not overwritten - the copy lands in the writable directory and shadows it | @@ -311,7 +311,7 @@ references written in the same session. | Tool | Arguments | Purpose | | --- | --- | --- | -| `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False, workspace=None, wait_seconds=0, note=None)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost`, `workspace`, `wait_seconds` | Queue a workflow for generation. Returns as soon as the job is queued - unless `wait_seconds` is above 0, in which case the call then waits on the queued job exactly as `wait_for_job(job_id, timeout_seconds=wait_seconds)` would (same per-call cap, clamped not honoured) and the result carries the queued-job fields plus that wait's (`status`, `still_running`, `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds`, `timeout_capped`, the slim `job`); when the cap covers the job's runtime one call is the run and the wait, and a `still_running: true` result is followed with `wait_for_job` as before. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate, and nothing is waited on `note` says what the run is for (see `set_note`) | +| `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False, workspace=None, wait_seconds=0, note=None)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost`, `workspace`, `wait_seconds`, `note` | Queue a workflow for generation. Returns as soon as the job is queued - unless `wait_seconds` is above 0, in which case the call then waits on the queued job exactly as `wait_for_job(job_id, timeout_seconds=wait_seconds)` would (same per-call cap, clamped not honoured) and the result carries the queued-job fields plus that wait's (`status`, `still_running`, `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds`, `timeout_capped`, the slim `job`); when the cap covers the job's runtime one call is the run and the wait, and a `still_running: true` result is followed with `wait_for_job` as before. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate, and nothing is waited on. `note` says what the run is for (see `set_note`) | | `get_job(job_id)` | `job_id` | Get a job's status, warnings, output manifest, error and traceback; each manifest entry's `subfolder` is the in-run subfolder the step declared - by convention `final` for the deliverable, `intermediate` for scratch, `''` for none. A running job also carries `progress` (below) | | `get_job_workflow(job_id)` | `job_id` | The REST equivalent is `GET /api/jobs/{id}/workflow` (see [SERVER.md](SERVER.md#jobs-api)). The workflow the job actually ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, `output:latest`); `false` means the job predates run tracking and this is the definition as submitted. Pass it to `save_workflow` to keep it under a name | | `export_job(job_id, overwrite=False)` | `job_id`, `overwrite` | Gather one finished job into `/exports//` on the server: the realized workflow, the run's manifest, the job row, a README, and copies of the assets, earlier-run inputs and outputs. Returns the directory, a zip URL, the file list with sizes and the total. The three JSON files are in the zip, not repeated here - get_job_workflow and get_job serve them individually. **The directory is on the machine running the server**, like `download_output`'s destination. The zip needs no token (`/exports/*.zip` is ungated like `/outputs`, #592), so `auth_required` is false and the agent fetches `open_url` itself (prefixing a relative one with the server address) and unpacks it into `exports/` under the session's working directory (a deliverable, not a temp file) - the archive already unpacks into one folder named after the job id, so do not create that folder first; only an agent without HTTP gives the user `open_url`. `true` is kept as a forward guard for a gated zip, when `open_url` is handed to the person instead | diff --git a/dw_mcp/assets.py b/dw_mcp/assets.py index 628ae2b4..81d9066f 100644 --- a/dw_mcp/assets.py +++ b/dw_mcp/assets.py @@ -228,10 +228,14 @@ def delete_asset(client, name, workspace=None): return client.delete_json(api_path("api", "assets", name), workspace=workspace) -def set_note(client, target, note): +def set_note(client, target, note, workspace=None): """Set (or clear, with None or '') the note on a run, an output file, an - asset ('asset:') or a workspace ('workspace:').""" - return client.put_json("/api/notes", {"target": target, "note": note}) + asset ('asset:') or a workspace ('workspace:'). `workspace` + pins this one call, as `delete_asset`'s does: a run or file name is read + in that workspace rather than the session's.""" + return client.put_json( + "/api/notes", {"target": target, "note": note}, workspace=workspace + ) def keep_output( diff --git a/dw_mcp/tools_catalog.py b/dw_mcp/tools_catalog.py index 68fe780e..d42b4a6d 100644 --- a/dw_mcp/tools_catalog.py +++ b/dw_mcp/tools_catalog.py @@ -282,11 +282,10 @@ def list_gallery( only bookkeeping (manifest.json, workflow.json, job.json) as `runs`, each `{name, mtime}` - output deleted, or a run that failed before writing anything; `name` is what `delete_output` accepts. - `subfolder` does not apply. Whether an entry is junk is yours to - decide. + A run that wrote any file is not listed. `subfolder` does not + apply. Whether an entry is junk is yours to decide. - `workspace` names the workspace for this one call without - switching the session, as `run_workflow` takes it. + `workspace` scopes one call without switching the session. `media=True` adds `duration_seconds` to audio/video entries, to tell takes apart.""" diff --git a/dw_mcp/tools_media.py b/dw_mcp/tools_media.py index ac8980df..c66b2833 100644 --- a/dw_mcp/tools_media.py +++ b/dw_mcp/tools_media.py @@ -384,12 +384,15 @@ def keep_output( note=note, ) - def set_note(self, target: str, note: str | None = None) -> dict: + def set_note( + self, target: str, note: str | None = None, workspace: str | None = None + ) -> dict: """Set or clear (note null or '') the note on a run ('/'), an output file (gallery name), 'asset:' or 'workspace:'. Plain text, 4000 chars max: what it is for - - what a run tested, why a file is the keeper. Free.""" - return assets.set_note(self.client, target, note) + what a run tested, why a file is the keeper. Free. `workspace` scopes + one call.""" + return assets.set_note(self.client, target, note, workspace=workspace) def delete_asset(self, name: str, workspace: str | None = None) -> dict: """Permanently remove one file from the asset library, by the name diff --git a/tests/test_mcp_diagnose.py b/tests/test_mcp_diagnose.py index 3cf3f03b..ad8721b3 100644 --- a/tests/test_mcp_diagnose.py +++ b/tests/test_mcp_diagnose.py @@ -846,6 +846,16 @@ def test_set_note_none_clears(): assert json.loads(seen[0]["body"]) == {"target": "asset:a.png", "note": None} +def test_set_note_workspace_pins_the_call(): + from dw_mcp import assets + + client, seen = scripted({("PUT", "/api/notes"): (200, {})}) + assets.set_note(client, "wf/run", "n", workspace="music-video") + assert seen[0]["params"]["workspace"] == "music-video" + assets.set_note(client, "wf/run", "n") + assert "workspace" not in seen[1]["params"] + + def test_list_jobs_clips_long_notes(): body = {"jobs": [{"id": "j", "note": "x" * 500}, {"id": "k", "note": "hi"}]} client, _ = scripted({("GET", "/api/jobs"): (200, body)}) From 94f7337d5ad2f067de908b68f21125ea759eeaf7 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:07:25 -0500 Subject: [PATCH 021/204] fix(ui): the job page saves its run note in the job's own workspace setNote takes an optional workspace, scoped the way galleryMetadata is, and NoteEditor passes it through; JobPage hands it job.workspace. Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/api.test.ts | 21 ++++++++++++++++++++- ui/src/lib/api.ts | 24 +++++++++++++++++------- ui/src/lib/pages/JobPage.svelte | 7 ++++++- ui/src/lib/pages/JobPage.test.ts | 26 +++++++++++++++++++++++++- ui/src/lib/ui/NoteEditor.svelte | 10 +++++++++- ui/src/lib/ui/NoteEditor.test.ts | 19 +++++++++++++++++-- 6 files changed, 94 insertions(+), 13 deletions(-) diff --git a/ui/src/lib/api.test.ts b/ui/src/lib/api.test.ts index 447ba5eb..ec070afb 100644 --- a/ui/src/lib/api.test.ts +++ b/ui/src/lib/api.test.ts @@ -1,5 +1,5 @@ import { afterEach, describe, expect, it, vi } from 'vitest' -import { api, fetchOutputText } from './api' +import { api, fetchOutputText, setNote } from './api' type Stub = { ok: boolean @@ -289,3 +289,22 @@ describe('job export', () => { expect(api.exportZipUrl('/exports/j1.zip')).toBe('/exports/j1.zip') }) }) + +describe('setNote', () => { + it('scopes to a named workspace, not the picker', async () => { + const calls = stubFetch({ ok: true, body: {} }) + await setNote('wf/run-1', 'n', 'studio') + expect(calls[0][0]).toBe('/api/notes?workspace=studio') + expect(calls[0][1]?.method).toBe('PUT') + expect(JSON.parse(String(calls[0][1]?.body))).toEqual({ + target: 'wf/run-1', + note: 'n', + }) + }) + + it("uses the picker's workspace when none is named", async () => { + const calls = stubFetch({ ok: true, body: {} }) + await setNote('wf/run-1', null) + expect(calls[0][0]).toBe('/api/notes') + }) +}) diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 844f617b..81e7296f 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -442,13 +442,23 @@ export async function fetchOutputText( } /** Set (or, with null, clear) the note on a workspace, asset, run or file. - * A file name or run directory is read in the picked workspace. */ -export function setNote(target: string, note: string | null): Promise { - return request('/api/notes', { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ target, note }), - }) + * A file name or run directory is read in the picked workspace, unless + * `workspace` names another - a job page edits the run of the job's own + * workspace, wherever the picker has since moved. */ +export function setNote( + target: string, + note: string | null, + workspace?: string, +): Promise { + return request( + workspace === undefined ? '/api/notes' : scopeTo('/api/notes', workspace), + { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ target, note }), + }, + { scope: workspace === undefined }, + ) } export const TERMINAL_STATUSES = ['succeeded', 'failed', 'cancelled'] diff --git a/ui/src/lib/pages/JobPage.svelte b/ui/src/lib/pages/JobPage.svelte index 15bcc409..c17f6e6e 100644 --- a/ui/src/lib/pages/JobPage.svelte +++ b/ui/src/lib/pages/JobPage.svelte @@ -232,7 +232,12 @@ {/if} {#if job.run_dir} - + {/if} {#if definition} diff --git a/ui/src/lib/pages/JobPage.test.ts b/ui/src/lib/pages/JobPage.test.ts index fa5f8761..8297889e 100644 --- a/ui/src/lib/pages/JobPage.test.ts +++ b/ui/src/lib/pages/JobPage.test.ts @@ -8,7 +8,7 @@ import { import { afterEach, beforeEach, expect, it, vi } from 'vitest' import JobPage from './JobPage.svelte' import '../router.svelte' -import { api } from '../api' +import { api, setNote } from '../api' import type { JobDetail, JobEvent } from '../types' const detail = vi.hoisted(() => ({ job: null as JobDetail | null })) @@ -58,6 +58,9 @@ vi.mock('../api', () => ({ outputDownloadUrl: (name: string) => `/api/gallery/${name}/download`, }, outputUrl: (path: string) => `/outputs/${path}`, + setNote: vi.fn((target: string, note: string | null) => + Promise.resolve({ target, note, note_source: note ? 'run' : null }), + ), streamJobEvents: ( _jobId: string, _after: number, @@ -586,3 +589,24 @@ it('shows the run note when the job has a run', async () => { await waitFor(() => expect(screen.getByText('second pass')).toBeTruthy()) expect(screen.getByText('Run note')).toBeTruthy() }) + +it("saves the run note in the job's own workspace", async () => { + detail.job = { + ...job([]), + workspace: 'studio', + run_dir: 'wf/run-1', + note: 'second pass', + } + render(JobPage, { jobId: 'j1' }) + await waitFor(() => expect(screen.getByText('second pass')).toBeTruthy()) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'third pass' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + expect(vi.mocked(setNote)).toHaveBeenCalledWith( + 'wf/run-1', + 'third pass', + 'studio', + ) +}) diff --git a/ui/src/lib/ui/NoteEditor.svelte b/ui/src/lib/ui/NoteEditor.svelte index 7e0798e1..5a46636f 100644 --- a/ui/src/lib/ui/NoteEditor.svelte +++ b/ui/src/lib/ui/NoteEditor.svelte @@ -10,10 +10,14 @@ target, label = 'Note', placeholder = 'Add a note', + workspace, onsaved, }: { note: string | null | undefined target: string + /** The workspace a run or file target is read in; the picker's when + * omitted */ + workspace?: string label?: string placeholder?: string onsaved?: (note: string | null) => void @@ -45,7 +49,11 @@ saving = true error = null try { - const stored = await setNote(target, draft.trim() ? draft : null) + const stored = await setNote( + target, + draft.trim() ? draft : null, + workspace, + ) shown = stored.note ?? null editing = false onsaved?.(shown) diff --git a/ui/src/lib/ui/NoteEditor.test.ts b/ui/src/lib/ui/NoteEditor.test.ts index 1c03d595..8f0686c5 100644 --- a/ui/src/lib/ui/NoteEditor.test.ts +++ b/ui/src/lib/ui/NoteEditor.test.ts @@ -31,7 +31,7 @@ describe('NoteEditor', () => { target: { value: 'new' }, }) await fireEvent.click(screen.getByRole('button', { name: /save/i })) - expect(spy).toHaveBeenCalledWith('t', 'new') + expect(spy).toHaveBeenCalledWith('t', 'new', undefined) expect(onsaved).toHaveBeenCalledWith('new') }) @@ -57,7 +57,7 @@ describe('NoteEditor', () => { target: { value: '' }, }) await fireEvent.click(screen.getByRole('button', { name: /save/i })) - expect(spy).toHaveBeenCalledWith('t', null) + expect(spy).toHaveBeenCalledWith('t', null, undefined) }) it('shows the server error and stays in edit mode', async () => { @@ -71,6 +71,21 @@ describe('NoteEditor', () => { expect(screen.getByRole('textbox')).toBeTruthy() }) + it('saves in the workspace it was given', async () => { + const spy = vi + .spyOn(api, 'setNote') + .mockResolvedValue({ target: 't', note: 'new', note_source: 'run' }) + render(NoteEditor, { + props: { note: 'old', target: 't', workspace: 'ws2' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /edit note/i })) + await fireEvent.input(screen.getByRole('textbox'), { + target: { value: 'new' }, + }) + await fireEvent.click(screen.getByRole('button', { name: /save/i })) + expect(spy).toHaveBeenCalledWith('t', 'new', 'ws2') + }) + it('keeps the draft when the note prop changes mid-edit', async () => { const { rerender } = render(NoteEditor, { props: { note: 'old', target: 't' }, From 5dd2d5e29720a6739c0ce8093eb656baf961e34b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:07:36 -0500 Subject: [PATCH 022/204] docs: restore two plan docs to their content before the release-note commit reformatted them Co-Authored-By: Claude Opus 5.5 --- .../plans/2026-10-07-develop-review-fixes.md | 224 ++++++++---------- .../plans/2026-10-10-workflow-speed.md | 141 +++++------ 2 files changed, 159 insertions(+), 206 deletions(-) diff --git a/docs/superpowers/plans/2026-10-07-develop-review-fixes.md b/docs/superpowers/plans/2026-10-07-develop-review-fixes.md index ac9c712f..41d8e876 100644 --- a/docs/superpowers/plans/2026-10-07-develop-review-fixes.md +++ b/docs/superpowers/plans/2026-10-07-develop-review-fixes.md @@ -74,9 +74,7 @@ class TestOwnGuidesPlusTheChain: assert "4" in error["message"] def test_three_own_guides_leave_room(self): - assert ( - guide_chain_errors(definition(prompt="a cat", guides=self.guides(3))) == [] - ) + assert guide_chain_errors(definition(prompt="a cat", guides=self.guides(3))) == [] def test_an_unresolved_guides_value_is_left_to_the_run(self): assert ( @@ -87,9 +85,7 @@ class TestOwnGuidesPlusTheChain: def test_last_frame_continuity_takes_the_full_four(self): chain = {"segments": 2, "continuity": "last_frame"} assert ( - guide_chain_errors( - definition(chain=chain, prompt="a cat", guides=self.guides(4)) - ) + guide_chain_errors(definition(chain=chain, prompt="a cat", guides=self.guides(4))) == [] ) ``` @@ -194,9 +190,7 @@ def test_the_context_defaults_to_the_devices_capacity(tmp_path, monkeypatch): def test_the_context_takes_a_pool_ceiling_instead(tmp_path, monkeypatch): monkeypatch.setattr("dw.validation.device_capacity_gb", lambda: 12.0) - assert ( - workflow_context(empty_workflow(tmp_path), capacity_gb=24.0).capacity_gb == 24.0 - ) + assert workflow_context(empty_workflow(tmp_path), capacity_gb=24.0).capacity_gb == 24.0 def _patched_admit_for(monkeypatch, state): @@ -231,19 +225,18 @@ def test_a_state_without_a_manager_falls_back_to_the_device(monkeypatch): Append to `class TestFitAtSubmit` in `tests/test_worker_pool.py`: ```python -def test_the_largest_card_is_the_ceiling_admission_checks(self, pool): - manager = pool( - card(success_script, "cuda:0", 12, name="Small GPU"), - card(success_script, "cuda:1", 24, name="Large GPU"), - ) - assert manager.largest_ceiling_gb() == 24 - + def test_the_largest_card_is_the_ceiling_admission_checks(self, pool): + manager = pool( + card(success_script, "cuda:0", 12, name="Small GPU"), + card(success_script, "cuda:1", 24, name="Large GPU"), + ) + assert manager.largest_ceiling_gb() == 24 -def test_an_unread_card_has_no_ceiling(self, pool): - worker = ScriptedWorkerManager(success_script) - worker.device = "cuda:0" - manager = pool(worker) - assert manager.largest_ceiling_gb() is None + def test_an_unread_card_has_no_ceiling(self, pool): + worker = ScriptedWorkerManager(success_script) + worker.device = "cuda:0" + manager = pool(worker) + assert manager.largest_ceiling_gb() is None ``` Note for the implementer: `card()` sets `_capacity_read = True` and `_capacity_gb`; a bare `ScriptedWorkerManager` leaves them unread, and `ceiling_gb()` falls back to `capacity_gb()`, which is `None` until read. If `ScriptedWorkerManager` reads a real device on first call, patch `dw.worker_manager.device_capacity_gb` to return `None` in the second test, as `test_a_declared_need_is_held_to_the_ceiling_admission_uses` patches it. @@ -268,7 +261,7 @@ Add to the docstring, after "the device the request is checked against": `(its o Replace `capacity_gb=device_capacity_gb(),` with: ```python -capacity_gb = (device_capacity_gb() if capacity_gb is None else capacity_gb,) + capacity_gb=device_capacity_gb() if capacity_gb is None else capacity_gb, ``` - [ ] **Step 4: Add `JobManager.largest_ceiling_gb`** @@ -526,50 +519,47 @@ class TestSafeRequest: Then change `test_an_untrusted_run_withholds_the_token_from_a_third_party` so it patches the shared path instead of `remote.requests.post`. Replace its `_Response`/`_post`/`with` block with: ```python -sent = {} + sent = {} + class _Response: + ok = True + is_redirect = False + headers = {"Content-Type": "application/octet-stream"} + status_code = 200 + _content = b"" -class _Response: - ok = True - is_redirect = False - headers = {"Content-Type": "application/octet-stream"} - status_code = 200 - _content = b"" - - def raise_for_status(self): - pass - - def iter_content(self, chunk_size): - return iter([b""]) - - def close(self): - pass - + def raise_for_status(self): + pass -class _Session: - def request(self, method, url, **kwargs): - sent["method"] = method - sent["headers"] = kwargs.get("headers") - sent["timeout"] = kwargs.get("timeout") - return _Response() + def iter_content(self, chunk_size): + return iter([b""]) - def close(self): - pass + def close(self): + pass + class _Session: + def request(self, method, url, **kwargs): + sent["method"] = method + sent["headers"] = kwargs.get("headers") + sent["timeout"] = kwargs.get("timeout") + return _Response() -with ( - patch("dw.locations._pinned_session", lambda url, address: _Session()), - patch.object(remote.torch, "load", return_value=_Embeds()), - patch( - "dw.locations.socket.getaddrinfo", - return_value=[(None, None, None, "", ("93.184.216.34", 443))], - ), -): - remote.remote_text_encoder(["a"], "https://evil.example.com/encode", "cpu") + def close(self): + pass -assert "Authorization" not in sent["headers"] -assert sent["method"] == "POST" -assert sent["timeout"] == remote.REMOTE_ENCODER_TIMEOUT + with ( + patch("dw.locations._pinned_session", lambda url, address: _Session()), + patch.object(remote.torch, "load", return_value=_Embeds()), + patch( + "dw.locations.socket.getaddrinfo", + return_value=[(None, None, None, "", ("93.184.216.34", 443))], + ), + ): + remote.remote_text_encoder(["a"], "https://evil.example.com/encode", "cpu") + + assert "Authorization" not in sent["headers"] + assert sent["method"] == "POST" + assert sent["timeout"] == remote.REMOTE_ENCODER_TIMEOUT ``` - [ ] **Step 2: Run them to verify they fail** @@ -687,13 +677,7 @@ def _read_capped(response, max_bytes, what, url): def _safe_request( - method, - url, - what, - timeout, - validate=validate_media_url, - max_bytes=MAX_MEDIA_BYTES, - **kwargs, + method, url, what, timeout, validate=validate_media_url, max_bytes=MAX_MEDIA_BYTES, **kwargs ): """One outbound request on the workflow's behalf: `validate` on the URL and on every redirect target, each hop dialed at the address the policy @@ -749,12 +733,7 @@ def safe_get(url, what="a media argument", timeout=60, max_bytes=MAX_MEDIA_BYTES def safe_post( - url, - what, - timeout=120, - max_bytes=MAX_MEDIA_BYTES, - validate=validate_media_url, - **kwargs, + url, what, timeout=120, max_bytes=MAX_MEDIA_BYTES, validate=validate_media_url, **kwargs ): """POST on a workflow's behalf, the way safe_get fetches: `kwargs` are requests' own (`json=`, `headers=`).""" @@ -898,41 +877,39 @@ Context: `cancel()` on a RUNNING job calls `slot.manager.cancel()` immediately. Append to `class TestCancel` in `tests/test_worker_pool.py`: ```python -def test_a_cancel_before_the_execute_is_sent_still_lands(self, pool): - """The dispatcher marks RUNNING before the job's thread takes the - slot; a cancel in that window used to reach an idle worker, which - ignores it, and the job ran anyway.""" - gate = Gate() - manager = pool(card(gate, "cuda:0", 24)) - slot = manager.slots[0] - with slot.lock: # a cache probe holds the card between dispatch and Execute - job = submit(manager, "late-cancel") - assert wait_until(lambda: job.status == "running") - assert manager.cancel(job.id) == "running" - wait_for(job, {"cancelled"}) - assert not gate.started.is_set(), "the worker was still sent the job" - - -def test_a_cancel_after_the_job_finished_is_a_no_op(self, pool): - manager = pool(card(success_script, "cuda:0", 24)) - job = submit(manager, "done") - wait_for(job, {"succeeded"}) - assert manager.cancel(job.id) == "succeeded" - assert job.status == "succeeded" - - -def test_cancelling_a_queued_job_drops_its_dispatch_bookkeeping(self, pool): - gate = Gate() - manager = pool(card(gate, "cuda:0", 24)) - first = submit(manager, "first") - gate.wait_started() - second = submit(manager, "second", vram_need=(8, True)) - assert second.id in manager._needs - assert manager.cancel(second.id) == "cancelled" - assert second.id not in manager._needs - assert second.id not in manager._preferred - gate.release.set() - wait_for(first, {"succeeded"}) + def test_a_cancel_before_the_execute_is_sent_still_lands(self, pool): + """The dispatcher marks RUNNING before the job's thread takes the + slot; a cancel in that window used to reach an idle worker, which + ignores it, and the job ran anyway.""" + gate = Gate() + manager = pool(card(gate, "cuda:0", 24)) + slot = manager.slots[0] + with slot.lock: # a cache probe holds the card between dispatch and Execute + job = submit(manager, "late-cancel") + assert wait_until(lambda: job.status == "running") + assert manager.cancel(job.id) == "running" + wait_for(job, {"cancelled"}) + assert not gate.started.is_set(), "the worker was still sent the job" + + def test_a_cancel_after_the_job_finished_is_a_no_op(self, pool): + manager = pool(card(success_script, "cuda:0", 24)) + job = submit(manager, "done") + wait_for(job, {"succeeded"}) + assert manager.cancel(job.id) == "succeeded" + assert job.status == "succeeded" + + def test_cancelling_a_queued_job_drops_its_dispatch_bookkeeping(self, pool): + gate = Gate() + manager = pool(card(gate, "cuda:0", 24)) + first = submit(manager, "first") + gate.wait_started() + second = submit(manager, "second", vram_need=(8, True)) + assert second.id in manager._needs + assert manager.cancel(second.id) == "cancelled" + assert second.id not in manager._needs + assert second.id not in manager._preferred + gate.release.set() + wait_for(first, {"succeeded"}) ``` - [ ] **Step 2: Run them to verify they fail** @@ -1049,28 +1026,27 @@ Context: the held-track outputs are added only when `output` is a list containin Append to `class TestWithHeldAudio` in `tests/test_h3_hold_audio.py`: ```python -def test_a_string_output_naming_audio_gets_the_held_keys_too(self): - pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() - insert_audio_hold(pipeline) - reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) - - result = Pipeline._with_held_audio( - ns(pipeline), {"hold_audio": reference, "output": "audio"} - ) + def test_a_string_output_naming_audio_gets_the_held_keys_too(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) - assert result["output"] == ["audio", HELD_AUDIO_OUTPUT, HELD_AUDIO_RATE_OUTPUT] + result = Pipeline._with_held_audio( + ns(pipeline), {"hold_audio": reference, "output": "audio"} + ) + assert result["output"] == ["audio", HELD_AUDIO_OUTPUT, HELD_AUDIO_RATE_OUTPUT] -def test_a_string_output_not_naming_audio_is_left_as_written(self): - pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() - insert_audio_hold(pipeline) - reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) + def test_a_string_output_not_naming_audio_is_left_as_written(self): + pipeline = minimax.MiniMaxH3Blocks().get_workflow("t2va").init_pipeline() + insert_audio_hold(pipeline) + reference = MiniMaxH3AudioReference(audio=torch.zeros(2, 10), sample_rate=8000) - result = Pipeline._with_held_audio( - ns(pipeline), {"hold_audio": reference, "output": "videos"} - ) + result = Pipeline._with_held_audio( + ns(pipeline), {"hold_audio": reference, "output": "videos"} + ) - assert result["output"] == "videos" + assert result["output"] == "videos" ``` - [ ] **Step 2: Run them to verify they fail** diff --git a/docs/superpowers/plans/2026-10-10-workflow-speed.md b/docs/superpowers/plans/2026-10-10-workflow-speed.md index a367f002..77651954 100644 --- a/docs/superpowers/plans/2026-10-10-workflow-speed.md +++ b/docs/superpowers/plans/2026-10-10-workflow-speed.md @@ -61,10 +61,7 @@ import pytest REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TEMPLATES = sorted( - glob.glob( - os.path.join(REPO_ROOT, "workflows", "templates", "**", "*.json"), - recursive=True, - ) + glob.glob(os.path.join(REPO_ROOT, "workflows", "templates", "**", "*.json"), recursive=True) ) @@ -103,13 +100,9 @@ def test_turbo_schedules_carry_no_block_cache(path): def test_compose_workflows_video_steps_match_the_turbo_default(): - compose = _load( - os.path.join(REPO_ROOT, "workflows", "templates", "compose-workflows.json") - ) + compose = _load(os.path.join(REPO_ROOT, "workflows", "templates", "compose-workflows.json")) i2v = _load( - os.path.join( - REPO_ROOT, "workflows", "templates", "minimax", "image-to-video.json" - ) + os.path.join(REPO_ROOT, "workflows", "templates", "minimax", "image-to-video.json") ) assert _default(compose, "video_num_inference_steps") == _default( i2v, "num_inference_steps" @@ -227,8 +220,7 @@ def test_two_frames_share_one_helper(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", - lambda model_name, filename: "weights.pth", + "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) @@ -246,14 +238,15 @@ def test_helper_state_is_cleared_before_each_frame(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", - lambda model_name, filename: "weights.pth", + "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) module.restore_faces(frame, "m", device="cpu") module.restore_faces(frame, "m", device="cpu") - helper = model_cache._cache[("restore_faces_helper", 1, 512, True, "cpu")] + helper = model_cache._cache[ + ("restore_faces_helper", 1, 512, True, "cpu") + ] assert helper.cleaned == 2 @@ -263,8 +256,7 @@ def test_different_settings_build_different_helpers(fake_facexlib, monkeypatch): descriptor = MagicMock(supports_half=False) monkeypatch.setattr(module, "_load_face_model", lambda path, device: descriptor) monkeypatch.setattr( - "dw.tasks.upscale.resolve_model_path", - lambda model_name, filename: "weights.pth", + "dw.tasks.upscale.resolve_model_path", lambda model_name, filename: "weights.pth" ) frame = Image.fromarray(np.zeros((8, 8, 3), dtype=np.uint8)) module.restore_faces(frame, "m", device="cpu", face_size=512) @@ -284,26 +276,25 @@ Expected: `test_two_frames_share_one_helper` fails with 2 constructions; the cac Replace the helper construction at `dw/tasks/restore_faces.py:70-77` with: ```python -# facexlib's helper loads RetinaFace (and ParseNet with use_parse) in -# its constructor. per_frame calls this once per frame, so a video was -# reloading detector weights for every frame. The helper keeps per-image -# state between calls; clean_all() resets it before each read -def build_helper(): - return FaceRestoreHelper( - upscale_factor=upscale_factor, - face_size=face_size, - crop_ratio=(1, 1), - det_model="retinaface_resnet50", - use_parse=use_parse, - device=torch.device(device), - ) - + # facexlib's helper loads RetinaFace (and ParseNet with use_parse) in + # its constructor. per_frame calls this once per frame, so a video was + # reloading detector weights for every frame. The helper keeps per-image + # state between calls; clean_all() resets it before each read + def build_helper(): + return FaceRestoreHelper( + upscale_factor=upscale_factor, + face_size=face_size, + crop_ratio=(1, 1), + det_model="retinaface_resnet50", + use_parse=use_parse, + device=torch.device(device), + ) -face_helper = cached_model( - ("restore_faces_helper", upscale_factor, face_size, use_parse, str(device)), - build_helper, -) -face_helper.clean_all() + face_helper = cached_model( + ("restore_faces_helper", upscale_factor, face_size, use_parse, str(device)), + build_helper, + ) + face_helper.clean_all() ``` Keep the existing `read_image` call that follows. @@ -440,7 +431,6 @@ git commit -m "perf(media): decode picture frames with libav AUTO threading" ```python RUNTIME_LORA_KEYS = ("scale", "alpha") - def weights_identity(pipeline_definition): """The pipeline definition with every runtime-mutable setting removed: a LoRA's scale and alpha (set_adapters / set_adapter_alpha) and a @@ -457,7 +447,6 @@ def adapter_settings(loras): `loras`, naming an adapter by its own adapter_name or its ORIGINAL index (a nulled entry keeps its slot). Reads, never pops.""" - def apply_adapter_settings(loras, pipeline): """Set each active LoRA's alpha then every scale on a pipeline that already holds them. No-op with no active entry.""" @@ -498,12 +487,7 @@ def _definition(**overrides): "configuration": {"component_type": "FluxPipeline"}, "from_pretrained_arguments": {"model_name": "black-forest-labs/FLUX.1-dev"}, "loras": [ - { - "model_name": "a/b", - "weight_name": "x.safetensors", - "scale": 1.0, - "alpha": 16, - }, + {"model_name": "a/b", "weight_name": "x.safetensors", "scale": 1.0, "alpha": 16}, ], "scheduler": {"shift": 6.0}, "audio_scheduler": {"shift": 3.0}, @@ -515,16 +499,7 @@ def _definition(**overrides): def test_lora_scale_and_alpha_do_not_change_the_pipeline_key(): a = _definition() - b = _definition( - loras=[ - { - "model_name": "a/b", - "weight_name": "x.safetensors", - "scale": 0.5, - "alpha": 8, - } - ] - ) + b = _definition(loras=[{"model_name": "a/b", "weight_name": "x.safetensors", "scale": 0.5, "alpha": 8}]) assert pipeline_cache_key(a) == pipeline_cache_key(b) @@ -536,9 +511,7 @@ def test_scheduler_shift_does_not_change_the_pipeline_key(): def test_lora_model_name_still_changes_the_pipeline_key(): a = _definition() - b = _definition( - loras=[{"model_name": "a/c", "weight_name": "x.safetensors", "scale": 1.0}] - ) + b = _definition(loras=[{"model_name": "a/c", "weight_name": "x.safetensors", "scale": 1.0}]) assert pipeline_cache_key(a) != pipeline_cache_key(b) @@ -1005,36 +978,40 @@ Leave `clear_model_cache()` and `step_cache.clear()` as they are. Append `f", ke `_prepare_workflow`: ```python -if identity != self.workflow_identity: - if self.workflow_identity is not None: - keep = self._keys_worth_keeping(command, job.workflow) - if keep: - self._reply( - Output( - message=f"Workflow changed - keeping {len(keep)} warm " - "model(s) the new workflow loads, releasing the rest..." - ) - ) - else: - self._reply(Output(message="Workflow changed - releasing cached models...")) - self._reply(Output(message=self._cleanup_all(keep=keep))) - self.workflow_identity = identity + if identity != self.workflow_identity: + if self.workflow_identity is not None: + keep = self._keys_worth_keeping(command, job.workflow) + if keep: + self._reply( + Output( + message=f"Workflow changed - keeping {len(keep)} warm " + "model(s) the new workflow loads, releasing the rest..." + ) + ) + else: + self._reply( + Output(message="Workflow changed - releasing cached models...") + ) + self._reply(Output(message=self._cleanup_all(keep=keep))) + self.workflow_identity = identity ``` with ```python -def _keys_worth_keeping(self, command, workflow): - """The resident pipeline keys the next workflow loads anyway. Any - failure to prepare the definition answers the empty set - the run - itself will report what is wrong, and a full release is the safe - default on a card that cannot hold two stacks.""" - try: - wanted = workflow_run.pipeline_keys(workflow, command.get("arguments") or {}) - except Exception as e: - logger.debug(f"Could not take the next workflow's pipeline keys: {e}") - return set() - return wanted & set(self.loaded_pipelines) + def _keys_worth_keeping(self, command, workflow): + """The resident pipeline keys the next workflow loads anyway. Any + failure to prepare the definition answers the empty set - the run + itself will report what is wrong, and a full release is the safe + default on a card that cannot hold two stacks.""" + try: + wanted = workflow_run.pipeline_keys( + workflow, command.get("arguments") or {} + ) + except Exception as e: + logger.debug(f"Could not take the next workflow's pipeline keys: {e}") + return set() + return wanted & set(self.loaded_pipelines) ``` Check whether `_prepare_workflow` runs with the job's asset dir already active (`_activate_job` is phase 1, so yes); `pipeline_keys` may realize `asset:` variables and needs it. From 5092126d07fe50c88b39438224dfd0eae3637592 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 00:09:32 -0500 Subject: [PATCH 023/204] fix(ui): spell setNote's /api/notes once so the route-contract check reads its PUT Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/api.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ui/src/lib/api.ts b/ui/src/lib/api.ts index 81e7296f..2c296304 100644 --- a/ui/src/lib/api.ts +++ b/ui/src/lib/api.ts @@ -450,8 +450,9 @@ export function setNote( note: string | null, workspace?: string, ): Promise { + const path = '/api/notes' return request( - workspace === undefined ? '/api/notes' : scopeTo('/api/notes', workspace), + workspace === undefined ? path : scopeTo(path, workspace), { method: 'PUT', headers: { 'Content-Type': 'application/json' }, From 15fa576ea4214abd708555210bcbdcaa7d277c0d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 05:35:49 -0500 Subject: [PATCH 024/204] docs(plugin): script-to-video and series-episodes pass a note per run and set_note the keeper Co-Authored-By: Claude Opus 5.5 --- plugins/dw/skills/script-to-video/SKILL.md | 3 +++ plugins/dw/skills/series-episodes/SKILL.md | 1 + 2 files changed, 4 insertions(+) diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index dfe20694..b62a08dc 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -86,6 +86,9 @@ human (or an explicit pre-authorization) has signed off - this is a real gate, not a formality, and skipping it here is the one thing this skill must not do even when running unattended. +Pass `note` on each `run_workflow` naming the shot and what the run tests, +and `set_note` the take you keep: the note shows on every file it made. + ## Run and judge Wait with `wait_for_job`, asking for the plan's estimate plus a margin as diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index cac9696f..f615f1f5 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -88,6 +88,7 @@ every episode; the same palette gives the same look. An uploaded `.cube` as `lut` works too. The graded clips are then the recut's `shots`. The recipe and its workflow are in `references/look.md`. +Give each `run_workflow` a `note` (episode, shot); `set_note` the keeper. Once every shot for the episode exists (freshly generated, or promoted from a run with `keep_output`), cut and score them with `templates/assemble-and-score` - it is the five-beat pipeline this skill From 960bf69eb610f0b1437b606fac3a9d6c4a344983 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 07:37:39 -0500 Subject: [PATCH 025/204] feat(plan): #851 - inherited estimate re-priced by step and frame count, with scaled reason and low_confidence Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/RELEASING.md | 7 ++-- dw/plan.py | 5 +++ dw/server/api_models.py | 5 +++ dw/server/deps.py | 62 +++++++++++++++++++++++------ dw/server/routes/jobs.py | 1 + dw/vram_inheritance.py | 20 ++++++++++ tests/test_inherited_cost.py | 64 +++++++++++++++++++++++++----- ui/src/lib/generated/api-schema.ts | 5 +++ ui/src/lib/generated/openapi.json | 5 +++ 10 files changed, 149 insertions(+), 27 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index db18a2de..80875260 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -100,7 +100,7 @@ one to open. | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | -| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization and frame count set differently from the template. It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns` | +| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization and frame count set differently from the template. More pipeline steps or frames than the template run re-price its minutes linearly (steps x mean frames per step) with `scaled` naming why and `low_confidence` set (#851). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns` | | H3 adapter partition | `dw/adapter_compatibility.py` | An FL2VA LoRA on a reference step is refused, and a LoRA whose file name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal` | | H3 audio hold, refine and guide layout | Five modules, imported one way: `dw/pipeline_processors/h3_rules.py` holds the names and rules, torch- and diffusers-free (`refine_problems`, the per-argument refine rules asked before validation and before the call; `not_h3`, the step rule; `snap_guide_length`, `guide_frame_problem`, `guide_chain_problems`, `guide_end_problem`; `CHAIN_CONTINUITY_MODES`; the chunk constants), `dw/pipeline_processors/h3_hold.py` puts the hold and refine blocks in (`insert_audio_hold`, `core_denoise_sequences`, `refines`), and `dw/pipeline_processors/h3_guides.py` the guide layout (`LAYOUT_ANCHORS`, `insert_guides`), which imports `h3_hold` and never the reverse; the blocks themselves, and the arithmetic they run, are `dw/pipeline_processors/h3_hold_steps.py` (`encode_audio_span`, `refine_sigmas`) and `dw/pipeline_processors/h3_guide_steps.py` (`splice_guide_rows`), which import diffusers when they load, so the two owners import them only inside `blocks()` and `guide_blocks()` (#790), and `h3_guide_steps` imports `h3_hold_steps` and never the reverse (`_with_guides` in `dw/pipeline_processors/pipeline.py` adds its blocks for a `guides` call), `dw/guides.py`: the `guides` validation check and the chain `continuity: "guide"` check, `guide_chain_errors`, which asks the `guides` check's own step rule (`_takes_guides_problem`, built on `h3_rules.not_h3` and `REFERENCE_WORKFLOWS`) and reads the mode names from `h3_rules.CHAIN_CONTINUITY_MODES`, the names `chain.py`'s `CONTINUITY_MODES` is zipped against (both registered in `dw/validation.py`; a `previous_result:` guide is checked at run time), `dw/hold_audio.py`: `hold_audio_errors`, `refine_strength_errors` | The one place dw modifies a diffusers modular pipeline's block graph: three blocks go in, before `set_timesteps`, `denoise` and `after_denoise`, in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio` / `refine_strength`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. `dw/introspection.py` builds the same graph without weights, hold included, so `get_pipeline_signature` lists a modular pipeline's block inputs. The copies of diffusers behaviour are pinned to the installed diffusers: `shifted_sigma_grid` (the spacing `refine_sigmas` rests on) to stock `set_timesteps`, `GUIDE_FRAMES_PER_CHUNK`/`GUIDE_LATENTS_PER_CHUNK` to the H3 video VAE's `clip_length` and latents per clip, and every name in `LAYOUT_ANCHORS` (`_fill_audio_positions` among them) to `before_denoise`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused`, `tests/test_h3_hold_audio.py::TestSignature::test_the_h3_signature_shows_hold_audio`, `tests/test_h3_refine.py::TestAnchors::test_inserted_before_denoise_in_every_shape`, `tests/test_h3_refine.py::TestSigmaGridDrift::test_the_full_grid_is_stock_set_timesteps`, `tests/test_h3_guides.py::TestChunkDrift::test_frames_per_chunk_is_the_vae_clip_length`, `tests/test_h3_guides.py::TestChunkDrift::test_latents_per_chunk_is_the_vae_tokens_per_clip`, `tests/test_h3_guides.py::TestAnchors::test_installed_diffusers_has_every_anchor`, `tests/test_guide_chain_validation.py::TestRefused::test_the_step_rule_is_the_guides_rule`, `tests/test_guide_chain_validation.py::TestRefused::test_every_run_mode_is_known_to_validate`, `tests/test_h3_rules.py::test_h3_rules_imports_neither_torch_nor_diffusers`, `tests/test_h3_rules.py::test_the_block_owners_import_no_diffusers` | | IC-LoRA reference scale | `workflows/templates/ltx2/` | The three conditioning templates run at `reference_downscale_factor: 1`, and the two upscalers at 2. | `tests/test_ltx2_ic_loras.py::TestEachTemplateMatchesItsCard::test_the_reference_is_encoded_at_the_output_resolution`, `tests/test_ltx2_ic_loras.py::TestTheGenerativeUpscaleMatchesTheSameCard::test_it_loads_the_upscaler_at_factor_two` | diff --git a/docs/RELEASING.md b/docs/RELEASING.md index f2ddb571..9ce20675 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -147,9 +147,10 @@ hallucinations as lyrics. **Fixes** -- The inherited cost estimate is withheld when an inline workflow runs - more steps or frames than the template it came from, instead of - quoting the template's number (#821: a 7x under-quote). +- The inherited cost estimate is re-priced linearly when an inline + workflow runs more steps or frames than the template it came from, + instead of quoting the template's number (#821: a 7x under-quote); the + estimate carries `scaled` (why) and `low_confidence` (#851). - `assess_output` shot findings carry `index` and `start_frame`, so a repeated shot name is unambiguous (#835). - `dialogue-short` and `music-video` expose `match_levels_dbfs` (#836). diff --git a/dw/plan.py b/dw/plan.py index c26c1b55..5ce2202d 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -753,6 +753,11 @@ def _observed(observed, device): estimate["basis"] = INHERITED estimate["inherited_from"] = inherited_from estimate["differs"] = list(observed.get("differs") or []) + if observed.get("scaled"): + # Re-priced linearly from the template's size (#851): an + # approximation, not a measurement + estimate["scaled"] = observed["scaled"] + estimate["low_confidence"] = True return estimate diff --git a/dw/server/api_models.py b/dw/server/api_models.py index bfe9d84d..146a934e 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -564,6 +564,11 @@ class PlanEstimate(ApiModel): "`inherited_from` among offload, quantization and frame count; empty " "when nothing does." ) + scaled: str = sometimes( + "With basis `inherited`: why the template's minutes were re-priced " + "linearly (more pipeline steps or frames than the template run measured) " + "- an approximation, with `low_confidence` true." + ) priced_for: str = sometimes( "The card the run would be dispatched to, whose history the figure is - " "as `cuda:1 NVIDIA GeForce RTX 3090`." diff --git a/dw/server/deps.py b/dw/server/deps.py index ef9a6efb..28478e14 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -15,7 +15,12 @@ from .. import references from ..realize import strings_with_prefix from ..security import SecurityError, validate_asset_reference -from ..vram_inheritance import build_index, inherited_differences, template_identity +from ..vram_inheritance import ( + build_index, + inherited_differences, + pipeline_frames, + template_identity, +) from .observed_cost import declared_drivers from ..library import ASSETS_KIND, LORAS_KIND, PROMPTS_KIND, library_path from ..workspace import ( @@ -211,14 +216,39 @@ def _pipeline_steps(definition): ) -def _scales_past_template(definition, template, drivers, arguments): - """Whether `definition` runs the template's pipeline more times, or for - more frames, than the template run the bucket was measured on (#821).""" - if _pipeline_steps(definition) != _pipeline_steps(template): - return True +def _scale_past_template(definition, template, drivers, arguments): + """How the workflow's size compares to the template run the bucket was + measured on (#821, #851): (factor, reason). The factor is the ratio of + pipeline steps times the ratio of mean frames per step - linear, as + `derived` re-prices a list of another length - and the reason names + each axis that moved. (1, None) when the workflow is the template's + size; (None, reason) when its size cannot be compared.""" + steps, template_steps = _pipeline_steps(definition), _pipeline_steps(template) # The template as the bucket's drivers set it bucket = {**template, "variables": {**(template.get("variables") or {}), **drivers}} - return "frame count" in inherited_differences(definition, bucket, arguments) + factor, reasons = 1.0, [] + if steps != template_steps: + factor *= steps / template_steps + reasons.append( + f"runs the pipeline in {steps} steps, template measured {template_steps}" + ) + # `pipeline_frames` takes the template's variables as the bucket sets them + mine = [f for f in pipeline_frames(definition, arguments) if f] + theirs = [f for f in pipeline_frames(bucket) if f] + if mine and theirs: + mean_mine, mean_theirs = sum(mine) / len(mine), sum(theirs) / len(theirs) + if mean_mine != mean_theirs: + factor *= mean_mine / mean_theirs + reasons.append( + f"frame count {_number(mean_mine)} vs {_number(mean_theirs)} per step" + ) + elif "frame count" in inherited_differences(definition, bucket, arguments): + return None, "frame count differs and cannot be compared" + return factor, "; ".join(reasons) or None + + +def _number(value): + return int(value) if float(value).is_integer() else round(value, 1) def inherited_observed(state, ws, definition, arguments=None, *, card=None): @@ -261,13 +291,19 @@ def inherited_observed(state, ws, definition, arguments=None, *, card=None): ) if not block or not block.get("cold_runs"): continue - # A figure is quoted only for a run of the template's own size: the - # bucket above prices the drivers the inline workflow shares by - # name, and anything else that moves the frame count or repeats the - # pipeline in more steps (#821) is a multiple the template's - # minutes do not carry - withheld rather than quoted unchanged - if _scales_past_template(definition, template, drivers, arguments): + # The bucket above prices the drivers the inline workflow shares by + # name; more pipeline steps or frames than that (#821) are a + # multiple the template's minutes do not carry, so they re-price it + # linearly (#851). A size that cannot be compared is withheld + factor, reason = _scale_past_template(definition, template, drivers, arguments) + if factor is None: continue + if reason: + block = { + **block, + "cold_minutes": block["cold_minutes"] * factor, + "scaled": f"scaled from {name}: {reason}", + } key = (-block["cold_runs"], name) if best is None or key < best[0]: best = (key, name, template, block) diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index ae1b0bfd..d506bf7a 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -948,6 +948,7 @@ def _inherited_cost_warnings(estimate): return [ f"cost estimate is inherited from this box's runs of {estimate.get('inherited_from')} " "(same pipeline), not of this workflow" + + (f"; {estimate['scaled']}" if estimate.get("scaled") else "") + ( f"; its {', '.join(differs)} differ from that template, so the real cost may too" if differs diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 7d62ed1a..a113f38f 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -93,6 +93,26 @@ def inherited_differences(definition, template, arguments=None): return differs +def pipeline_frames(definition, arguments=None): + """The frame count each pipeline step sets (`num_frames`, a `variable:` + reference resolved; `arguments` override the variables), None for a + step that sets none or a non-numeric one - for scaling an inherited + price by frame count (#851).""" + variables = dict(definition.get("variables") or {}) + variables.update(arguments or {}) + frames = [] + for step in definition.get("steps") or []: + if not isinstance(step, dict) or not isinstance(step.get("pipeline"), dict): + continue + counts = [ + value + for value in map(json.loads, _found(step, "num_frames", variables)) + if isinstance(value, (int, float)) and not isinstance(value, bool) + ] + frames.append(max(counts) if counts else None) + return frames + + def declarations(catalog): """Every (identity, name, definition) a catalog of (name, definition) pairs offers the index: a declared `vram_estimate` on a single-identity diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 249f37af..92227a84 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -195,25 +195,69 @@ def test_an_observed_estimate_does_not_warn(self): assert _inherited_cost_warnings({"basis": "observed"}) == [] -class TestNotQuotedUnchangedWhenItScales: - """#821: the template's minutes are for one template-sized run.""" +class TestScaledWhenItScales: + """#821/#851: the template's minutes are for one template-sized run, so a + bigger workflow is re-priced linearly and says why.""" HISTORY = {"templates/t": [run(8 * MINUTE, {"num_frames": 124})]} + CATALOG = [("templates/t", definition("t"))] - def test_several_steps_of_one_pipeline_are_not_priced_as_one_run(self): + def test_several_steps_scale_by_step_count(self): inline = definition("inline") inline["steps"] = [{**inline["steps"][0], "id": f"shot{n}"} for n in range(3)] - assert ( - inherited(inline, [("templates/t", definition("t"))], self.HISTORY) is None - ) + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 24.0 + assert block["inherited_from"] == "templates/t" + assert "3 steps, template measured 1" in block["scaled"] - def test_a_literal_longer_frame_count_is_not_priced_as_the_templates(self): + def test_a_longer_frame_count_scales_by_frames(self): inline = definition("inline") inline["variables"] = {} - inline["steps"][0]["run"] = {"num_frames": 899} - assert ( - inherited(inline, [("templates/t", definition("t"))], self.HISTORY) is None + inline["steps"][0]["run"] = {"num_frames": 248} + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 16.0 + assert "frame count 248 vs 124" in block["scaled"] + assert "frame count" in block["differs"] + + def test_steps_and_frames_multiply(self): + inline = definition("inline") + inline["variables"] = {} + inline["steps"] = [ + {**inline["steps"][0], "id": f"shot{n}", "run": {"num_frames": 62}} + for n in range(4) + ] + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 16.0 # 4 steps x half the frames + assert "4 steps" in block["scaled"] and "62 vs 124" in block["scaled"] + + def test_a_template_sized_workflow_is_not_scaled(self): + block = inherited(definition("inline"), self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 8.0 + assert "scaled" not in block + + def test_the_plan_reports_the_reason_and_low_confidence(self, tmp_path): + inline = definition("inline") + inline["steps"] = [{**inline["steps"][0], "id": f"shot{n}"} for n in range(3)] + block = inherited( + inline, self.CATALOG, {"templates/t": self.HISTORY["templates/t"] * 3} + ) + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + plan = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: block, + ) + estimate = plan["estimate"] + assert estimate["basis"] == "inherited" + assert estimate["minutes"] == 24.0 + assert estimate["low_confidence"] is True + assert "3 steps, template measured 1" in estimate["scaled"] + assert "3 steps" in _inherited_cost_warnings(estimate)[0] def test_a_shared_driver_name_still_prices_its_bucket(self): block = inherited( diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 3156f09c..1ee18c11 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3158,6 +3158,11 @@ export interface components { priced_for?: string; /** Runs */ runs: number | null; + /** + * Scaled + * @description With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true. + */ + scaled?: string; /** Tempered */ tempered?: boolean; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 5b5956fd..2df381f7 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3501,6 +3501,11 @@ ], "title": "Runs" }, + "scaled": { + "description": "With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true.", + "title": "Scaled", + "type": "string" + }, "tempered": { "title": "Tempered", "type": "boolean" From e2a979f35d4b35dbb8d8395b06fb69cc2cf1dd32 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 07:44:27 -0500 Subject: [PATCH 026/204] fix(server): #855 #856 - jobs list survives a malformed spec row and reads run notes only for the page Co-Authored-By: Claude Sonnet 5.5 --- dw/server/job_history.py | 4 +++- dw/server/jobs.py | 21 +++++++++++++++++---- dw/server/routes/jobs.py | 14 +++----------- tests/test_jobs_listing.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 59 insertions(+), 16 deletions(-) diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 0b1957d9..26e32121 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -215,7 +215,9 @@ def recent_summaries(self, limit=200, workspace=None, statuses=None): query = ( "SELECT id, workflow, status, created_at, started_at, finished_at," " workspace, workflow_name, run_id, acknowledged, run_version," - " device, note, run_dir, json_extract(spec, '$.output_dir') FROM jobs" + " device, note, run_dir," + " CASE WHEN json_valid(spec) THEN json_extract(spec, '$.output_dir') END" + " FROM jobs" ) params = [] clauses = [] diff --git a/dw/server/jobs.py b/dw/server/jobs.py index a9c8b582..2593f186 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -589,6 +589,13 @@ def list(self, workspace=None, statuses=None): workspaces existed. `statuses` filters to a set of job states ('queued', 'running', 'succeeded', 'failed', 'cancelled'); omitted, every state is listed.""" + return self.list_page(workspace, statuses)[0] + + def list_page(self, workspace=None, statuses=None, limit=None): + """`list`'s jobs cut to the newest `limit` (every one when None), + and how many matched before the cut. Each run's current note is + read after the cut, so a small page does not read the notes of + the whole history.""" statuses = set(statuses) if statuses else None with self._lock: live = sorted(self.jobs.values(), key=lambda j: j.created_at) @@ -604,16 +611,22 @@ def list(self, workspace=None, statuses=None): if job.id in positions: summary["queue_position"] = positions[job.id] output_dir = job.spec.get("output_dir") - summaries.append(self._current_note(summary, output_dir, job.run_dir)) + summaries.append((summary, output_dir, job.run_dir)) for historical in self.history.recent_summaries( workspace=workspace, statuses=statuses ): run_dir = historical.pop("_run_dir", None) output_dir = historical.pop("_output_dir", None) if historical["id"] not in live_ids: - summaries.append(self._current_note(historical, output_dir, run_dir)) - summaries.sort(key=lambda summary: summary["created_at"] or 0) - return summaries + summaries.append((historical, output_dir, run_dir)) + summaries.sort(key=lambda entry: entry[0]["created_at"] or 0) + total = len(summaries) + if limit is not None: + # the newest are the interesting ones and the list is oldest + # first, so the cut comes off the front; max(0, ...) because a + # bare negative start would be read from the end instead + summaries = summaries[max(0, total - limit) :] if limit else [] + return [self._current_note(*entry) for entry in summaries], total def _current_note(self, summary, output_dir, run_dir): """The summary with its run's current note (outputs.with_run_note), diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index d506bf7a..8413dd70 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -235,17 +235,9 @@ def list_jobs( detail=f"Unknown job status {', '.join(unknown)} - one of " f"{', '.join((QUEUED, RUNNING, *TERMINAL_STATES))}", ) - jobs = manager.list(workspace=workspace, statuses=statuses) - total = len(jobs) - if limit is not None: - if limit < 0: - raise HTTPException(status_code=400, detail="limit must not be negative") - # the newest are the interesting ones, and the list is oldest - # first - so the cut comes off the front, not the back. max(0, ...) - # because a limit above what matched is no cut at all: a bare - # negative start would be read from the end instead, and answer a - # limit of 12 against 9 matching jobs with the last 3 of them - jobs = jobs[max(0, len(jobs) - limit) :] if limit else [] + if limit is not None and limit < 0: + raise HTTPException(status_code=400, detail="limit must not be negative") + jobs, total = manager.list_page(workspace=workspace, statuses=statuses, limit=limit) return {"jobs": jobs, "total": total} diff --git a/tests/test_jobs_listing.py b/tests/test_jobs_listing.py index bd3b7f83..36ff5f5c 100644 --- a/tests/test_jobs_listing.py +++ b/tests/test_jobs_listing.py @@ -200,3 +200,39 @@ def test_limit_larger_than_the_match_under_a_status_filter(server): assert body["total"] == 2 assert [entry["id"] for entry in body["jobs"]] == ids + + +def test_history_summaries_survive_a_malformed_spec_row(history): + stored_job(history, "good", "succeeded", 1) + stored_job(history, "bad", "succeeded", 2) + with history._connect() as conn: + conn.execute("UPDATE jobs SET spec = '{not json' WHERE id = 'bad'") + + rows = history.recent_summaries() + + assert {row["id"] for row in rows} == {"good", "bad"} + assert next(row for row in rows if row["id"] == "bad")["_output_dir"] is None + + +def test_list_page_reads_notes_only_for_the_page(tmp_path): + from dw.server.jobs import JobManager + + manager = JobManager.__new__(JobManager) + import threading + + manager._lock = threading.RLock() + manager.jobs = {} + manager._pending = [] + manager.history = JobHistory(str(tmp_path / "jobs.sqlite")) + for index in range(6): + stored_job(manager.history, f"j{index}", "succeeded", index) + seen = [] + manager._current_note = lambda summary, out, run: ( + seen.append(summary["id"]) or summary + ) + + jobs, total = manager.list_page(limit=2) + + assert total == 6 + assert [j["id"] for j in jobs] == ["j4", "j5"] + assert sorted(seen) == ["j4", "j5"] From e14b244fa29d8a14a8c0e83f0ae538bd885514dd Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 07:48:59 -0500 Subject: [PATCH 027/204] fix(run): #857 #858 - Made from lists only variables a surviving step read, and pins output: latest/vN Co-Authored-By: Claude Sonnet 5.5 --- dw/realize.py | 5 +++ dw/workflow.py | 3 ++ dw/workflow_run.py | 70 ++++++++++++++++++++++++++++++-- tests/test_run_manifest_roles.py | 37 +++++++++++++++++ 4 files changed, 112 insertions(+), 3 deletions(-) diff --git a/dw/realize.py b/dw/realize.py index e191e963..17d44ba1 100644 --- a/dw/realize.py +++ b/dw/realize.py @@ -202,6 +202,11 @@ def _pin_output(reference, output_root): return references.make_ref(references.OUTPUT, relative) +def pin_output_reference(reference, output_root): + """`_pin_output` for a caller outside this module (a run's Made from).""" + return _pin_output(reference, output_root) + + def _record_sub_workflows(steps, annotations, base_dir, workflow_dir): """Digest every sub-workflow a step names by local path. diff --git a/dw/workflow.py b/dw/workflow.py index 2e7e6cfe..e23980d1 100644 --- a/dw/workflow.py +++ b/dw/workflow.py @@ -550,6 +550,9 @@ def run( # What elision dropped this run, filled by prepare_definition and # read by the warning pass and the manifest (#122) self._elided_steps = [] + # The variables the surviving steps referenced, filled by + # prepare_definition for the run's Made from (#857) + self._used_variables = [] # A reused Workflow (the persistent worker's) still holds the last # run's directory: a run that fails before open_run would otherwise # rewrite that run's manifest. A composed child's values were set by diff --git a/dw/workflow_run.py b/dw/workflow_run.py index 7d85ee8a..8eb36ba2 100644 --- a/dw/workflow_run.py +++ b/dw/workflow_run.py @@ -9,7 +9,7 @@ with the run, so the probe answers for the run that would happen. Every phase takes the Workflow and writes what others read back onto it: -`manifest`, `_run_dir`, `_run_version`, `_elided_steps` and +`manifest`, `_run_dir`, `_run_version`, `_elided_steps`, `_used_variables` and `_cache_enabled_this_run`. A composed child is recognized as an instance of the running workflow's own type. @@ -32,7 +32,7 @@ from .media import probe_metadata from .pipeline_ownership import allocated_mb, finish_release from .previous_results import StepResults -from .realize import realize_workflow +from .realize import pin_output_reference, realize_workflow from .roles import annotate_roles, role_basis, run_inputs from .runs import ( FLAT_LAYOUT, @@ -122,6 +122,7 @@ class PreparedRun: default_seed: int recorded_variables: object cache_enabled: bool + used_variables: list = field(default_factory=list) @dataclass @@ -322,6 +323,7 @@ def prepare_definition(workflow, workflow_def, arguments, base_dir): realize_args(variables, base_dir, apply_key_conventions=False) ## then replace any variable references in the workflow definition # with the actual values, and expand + authored_steps = workflow_def.get("steps") workflow_def = workflow._expand(workflow_def, variables) # A step a declared vram_estimate projects past the card 'cost' was @@ -351,6 +353,9 @@ def prepare_definition(workflow, workflow_def, arguments, base_dir): workflow._elided_steps = elide_definition( workflow_def, workflow.workflow_definition ) + workflow._used_variables = _surviving_variables( + authored_steps, workflow_def.get("steps") + ) # Set up random seed for reproducibility. Resolved lazily - as a # dict.get default, torch.seed() would run on every call and reseed @@ -372,6 +377,53 @@ def prepare_definition(workflow, workflow_def, arguments, base_dir): return workflow_def, default_seed, recorded_variables +def _variable_names(value): + """Every 'variable:' name a nested value references, in first-use order.""" + names = [] + if isinstance(value, str): + name = references.ref_name(references.VARIABLE, value) + if name is not None: + names.append(name) + elif isinstance(value, dict): + for item in value.values(): + names.extend(_variable_names(item)) + elif isinstance(value, list): + for item in value: + names.extend(_variable_names(item)) + return names + + +def _surviving_variables(authored_steps, expanded_steps): + """The variable names the steps that survived expansion and elision + referenced before substitution (#857). A for_each member + ('@') stands for the step it was expanded from.""" + groups = { + str(step.get("name")).split(references.MEMBER_SEPARATOR)[0] + for step in expanded_steps or [] + if isinstance(step, dict) + } + used = [] + for step in authored_steps or []: + if isinstance(step, dict) and step.get("name") in groups: + for name in _variable_names(step): + if name not in used: + used.append(name) + return used + + +def _pin_inputs(inputs, output_root): + """`inputs` with each output: reference pinned as the realized workflow + pins it, duplicates merged.""" + pinned = [] + for entry in inputs: + reference = entry["reference"] + if entry["kind"] == "output": + reference = pin_output_reference(reference, output_root) + if not any(p["reference"] == reference for p in pinned): + pinned.append({**entry, "reference": reference}) + return pinned + + def cache_lookup( workflow, workflow_id, @@ -596,6 +648,7 @@ def prepare_run(workflow, record): default_seed, recorded_variables, cache_enabled, + list(workflow._used_variables), ) @@ -634,7 +687,15 @@ def open_run(workflow, prepared, record, run_context): record.role_basis = role_basis(expanded) # The variables as folded, before their assets loaded: a template's # input image is usually a variable, gone from the steps by now - record.inputs = run_inputs([*expanded, prepared.recorded_variables]) + # Only the variables a surviving step referenced: a default nothing + # read is not something the run was made from (#857) + recorded = prepared.recorded_variables + read = ( + {k: v for k, v in recorded.items() if k in prepared.used_variables} + if isinstance(recorded, dict) + else None + ) + record.inputs = run_inputs([*expanded, read]) if not workflow._run_dir_inherited: _claim_run_dir(workflow, prepared, record) @@ -655,6 +716,9 @@ def open_run(workflow, prepared, record, run_context): ) if write_realized_workflow(workflow._run_dir, realized): record.realized_name = REALIZED_FILE_NAME + # Made from names the run realize_workflow pinned, not 'latest' + # as read when the page is viewed (#858) + record.inputs = _pin_inputs(record.inputs, workflow.output_dir) except Exception as e: # Never fatal: the record is worth less than the run logger.warning(f"Could not realize workflow {prepared.workflow_id}: {e}") diff --git a/tests/test_run_manifest_roles.py b/tests/test_run_manifest_roles.py index b3f582bd..8f97bc03 100644 --- a/tests/test_run_manifest_roles.py +++ b/tests/test_run_manifest_roles.py @@ -120,3 +120,40 @@ def test_a_real_run_records_an_output_carried_by_a_variable(tmp_path, fake_pipel manifest = _terminal_manifest(tmp_path, "Var") assert manifest["inputs"] == [{"reference": reference, "kind": "output"}] + + +def test_a_real_run_omits_a_variable_default_no_step_read(tmp_path, fake_pipeline): # noqa: F811 + # #857: an unused variable's asset:/output: default is not an input + from dw.workflow import Workflow + + reference = _source_run(tmp_path) + definition = _workflow_definition() + definition["id"] = "Unused" + definition["variables"] = { + "image": reference, + "spare": "output:Src/latest/" + reference.rsplit("/", 1)[1], + "prompt": "a cat", + } + definition["steps"][0]["pipeline"]["arguments"]["prompt"] = "variable:prompt" + definition["steps"][0]["pipeline"]["arguments"]["image"] = "variable:image" + Workflow(definition, str(tmp_path), "/w/workflows/Unused.json").run({}) + + manifest = _terminal_manifest(tmp_path, "Unused") + assert manifest["inputs"] == [{"reference": reference, "kind": "output"}] + + +def test_a_real_run_pins_a_latest_output_reference(tmp_path, fake_pipeline): # noqa: F811 + # #858: Made from names the run 'latest' resolved to, not 'latest' + from dw.workflow import Workflow + + pinned = _source_run(tmp_path) + _, run_id, image = pinned.removeprefix("output:").split("/") + definition = _workflow_definition() + definition["id"] = "Pin" + definition["steps"][0]["pipeline"]["arguments"]["image"] = ( + f"output:Src/latest/{image}" + ) + Workflow(definition, str(tmp_path), "/w/workflows/Pin.json").run({}) + + manifest = _terminal_manifest(tmp_path, "Pin") + assert manifest["inputs"] == [{"reference": pinned, "kind": "output"}] From d0565f832b7fef3c795b0c83cb6d203527674663 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 16:31:33 -0500 Subject: [PATCH 028/204] fix(plan): #851 - variable num_frames prices from the template's default bucket; a withheld inherited figure names the template and reason Co-Authored-By: Claude Sonnet 5.5 --- dw/plan.py | 9 +++++- dw/server/api_models.py | 5 +++ dw/server/deps.py | 24 +++++++++++++- dw/server/routes/jobs.py | 2 ++ tests/test_inherited_cost.py | 50 ++++++++++++++++++++++++++++++ ui/src/lib/generated/api-schema.ts | 5 +++ ui/src/lib/generated/openapi.json | 5 +++ 7 files changed, 98 insertions(+), 2 deletions(-) diff --git a/dw/plan.py b/dw/plan.py index 5ce2202d..32490e15 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -716,7 +716,14 @@ def estimate( child_definition, child_cost, step_arguments, device ) totals.add(path, child, is_observed) - return _rolled_up_estimate(own, totals, device, cached_steps, total_steps) + result = _rolled_up_estimate(own, totals, device, cached_steps, total_steps) + withheld = observed.get("withheld") if isinstance(observed, dict) else None + if withheld and result["basis"] == UNKNOWN and result["minutes"] is None: + # History exists but could not be re-priced for this workflow's size: + # say which template and why, so it reads differently from no history + result["inherited_from"] = observed.get("inherited_from") + result["withheld"] = withheld + return result def _observed(observed, device): diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 146a934e..7544b4bc 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -569,6 +569,11 @@ class PlanEstimate(ApiModel): "linearly (more pipeline steps or frames than the template run measured) " "- an approximation, with `low_confidence` true." ) + withheld: str = sometimes( + "With basis `unknown` and `inherited_from` set: why the template's runs " + "could not be re-priced for this workflow (its size cannot be compared " + "to the template's)." + ) priced_for: str = sometimes( "The card the run would be dispatched to, whose history the figure is - " "as `cuda:1 NVIDIA GeForce RTX 3090`." diff --git a/dw/server/deps.py b/dw/server/deps.py index 28478e14..11d58a9b 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -270,6 +270,7 @@ def inherited_observed(state, ws, definition, arguments=None, *, card=None): variables = definition.get("variables") variables = variables if isinstance(variables, dict) else {} best = None + withheld = None for name, template, writable in _catalog_listing(state, ws): if not isinstance(template, dict) or template_identity(template) != identity: continue @@ -289,14 +290,28 @@ def inherited_observed(state, ws, definition, arguments=None, *, card=None): workspace=ws.name if writable else None, card=card, ) + if not (block and block.get("cold_runs")) and drivers: + # No run in that bucket (a frame count the template never ran at): + # price from the template's own default bucket and scale from it + drivers = {} + block = observed_for_name( + state, + name, + template, + drivers, + workspace=ws.name if writable else None, + card=card, + ) if not block or not block.get("cold_runs"): continue # The bucket above prices the drivers the inline workflow shares by # name; more pipeline steps or frames than that (#821) are a # multiple the template's minutes do not carry, so they re-price it - # linearly (#851). A size that cannot be compared is withheld + # linearly (#851). A size that cannot be compared is withheld, with + # the template and the reason named factor, reason = _scale_past_template(definition, template, drivers, arguments) if factor is None: + withheld = withheld or (name, reason) continue if reason: block = { @@ -308,6 +323,13 @@ def inherited_observed(state, ws, definition, arguments=None, *, card=None): if best is None or key < best[0]: best = (key, name, template, block) if best is None: + if withheld: + # Not priced, but the plan says which template had history and why + # it was not used + return { + "inherited_from": withheld[0], + "withheld": f"not priced from {withheld[0]}: {withheld[1]}", + } return None _, name, template, block = best return { diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index 8413dd70..8ea14d23 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -934,6 +934,8 @@ def _inherited_cost_warnings(estimate): """A price inherited from a catalog template's runs (#797) is an approximation; say so, and name what this workflow sets differently from the template. Warns and never refuses.""" + if isinstance(estimate, dict) and estimate.get("withheld"): + return [f"cost estimate withheld: {estimate['withheld']}"] if not isinstance(estimate, dict) or estimate.get("basis") != "inherited": return [] differs = estimate.get("differs") or [] diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 92227a84..64b07a3f 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -267,3 +267,53 @@ def test_a_shared_driver_name_still_prices_its_bucket(self): {"num_frames": 124}, ) assert block["cold_minutes"] == 8.0 + + +class TestVariableFrameCountAndWithheldReason: + """#851 bounce: a frame count written as `variable:num_frames` that the + template never ran at still prices from the template's default bucket, and + a figure still withheld names the template and why.""" + + HISTORY = {"templates/t": [run(8 * MINUTE, {"num_frames": 124})]} + CATALOG = [("templates/t", definition("t"))] + + def test_a_variable_frame_count_scales_from_the_default_bucket(self): + inline = definition("inline", num_frames=248) + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 16.0 + assert "frame count 248 vs 124" in block["scaled"] + + def test_variable_frames_and_steps_multiply(self): + inline = definition("inline", num_frames=248) + inline["steps"] = [{**inline["steps"][0], "id": f"s{n}"} for n in range(2)] + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 32.0 + + def test_a_withheld_figure_names_template_and_reason(self): + inline = definition("inline") + inline["variables"] = {} + inline["steps"][0]["run"] = {"num_frames": "variable:missing"} + block = inherited(inline, self.CATALOG, self.HISTORY) + assert block["inherited_from"] == "templates/t" + assert "cannot be compared" in block["withheld"] + assert "cold_minutes" not in block + + def test_the_plan_carries_the_withheld_reason(self, tmp_path): + inline = definition("inline") + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) + ) + withheld = {"inherited_from": "templates/t", "withheld": "not priced: why"} + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + plan = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: withheld, + ) + estimate = plan["estimate"] + assert estimate["basis"] == "unknown" and estimate["minutes"] is None + assert estimate["inherited_from"] == "templates/t" + assert estimate["withheld"] == "not priced: why" + assert "withheld" in _inherited_cost_warnings(estimate)[0] diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 1ee18c11..4e1d97d0 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3163,6 +3163,11 @@ export interface components { * @description With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true. */ scaled?: string; + /** + * Withheld + * @description With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's). + */ + withheld?: string; /** Tempered */ tempered?: boolean; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 2df381f7..44eb3e43 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3506,6 +3506,11 @@ "title": "Scaled", "type": "string" }, + "withheld": { + "description": "With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's).", + "title": "Withheld", + "type": "string" + }, "tempered": { "title": "Tempered", "type": "boolean" From 61c688c0c5cbe3dbae6d3c5994cd0e2697aaca0e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 16:34:38 -0500 Subject: [PATCH 029/204] refactor(run): #857 - for_each.group_of owns the member-to-group inverse; map row names it Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/for_each.py | 8 ++++++++ dw/workflow_run.py | 3 ++- tests/test_for_each.py | 6 ++++++ 4 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 80875260..1983fa66 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -36,7 +36,7 @@ one to open. | Task argument domains | `dw/task_domains.py`: `check_arguments`, `task_argument_errors`; `dw/task_problems.py`: `face_track_errors`, `beats_errors`, `cuts_errors`, `lut_errors` | A numeric domain that a signature cannot express is declared on the command's `@register_command(domains=...)`, and it is checked at validation and again at run time. A numeric argument is read by `whole_number` or `real_number`, the only numeric coercion in `dw/tasks/`, and validation applies the same rule (`number_problem`), so a value is refused at both or at neither: `"3.0"` is a whole number, `"3.5"` for one of `whole_numbers` is not, and `true` is no number (#774). `Task.run` coerces every declared argument through `coerce_arguments` before dispatch, so a handler sees a number, and an argument that has no range besides being a number declares `finite`. A command's cross-argument rules are its registered `static_check`, which `task_argument_errors` runs at validation (`attribute_voices`' `voices_argument_errors`, kept in `dw/tasks/voice_attribution.py`, among them). A literal enum choice (`choice_errors`) is refused at validation by the same module. The per-family rules past a single domain live in `dw/task_problems.py`, which reads `task_domains`' helpers and is never imported by it (`task_argument_errors` reaches them only through the registered `static_check`, so `import_cycles` stays 0; #790): the `ingredients_grid` image count against `max_images` (`ingredients_grid_errors`), and `crop_face_track`'s cross-argument rules (`face_track_problems`, `face_track_errors`: crop size a multiple of the step's `multiple` (32 by default), `remainder` below `modulus`, `gate_zero` above `gate_full`, padding and confidence limits), which the task checks again at run time, as `analyze_beats`' are too (`beats_problems`, `beats_errors`: `min_bpm` below `max_bpm`, and anchors that are one kind per list, ascending, with whole beat indexes; `tests/test_analyze_beats.py`), and `plan_cuts`' (`cuts_problems`, `cuts_errors`: a bare-string transcript (`transcript_problem`, which `plan_cuts` also calls at run time), `min_scene_s` above `max_scene_s`, `segment_by: "beat"` without beats), and `apply_lut`'s (`lut_source_problem`, `palette_problem`, `lut_errors`: exactly one of `lut` or `palette`, and a palette of 2..16 `#rrggbb` colours, re-checked at run time by `check_lut_source`), and `check_face_detector_source` vets its Hub repo and `.onnx` file before any download. `select`'s index stays in `task_domains` with the other media-timing rules (`select_index_problem`, `select_rule_problems`: a whole number, at least 0, and below the candidate count when the count is known, i.e. at run time or when `candidates` is a written list). | `tests/test_task_domains.py::TestTheStaticPass::test_a_negative_frame_count_is_refused_at_its_path`, `tests/test_numeric_coercion.py::TestGradeExposure::test_a_string_number_grades_like_the_number`, `tests/test_task_domains.py::TestAtRunTime::test_slice_audio_refuses_a_negative_count`, `tests/test_ingredients_grid.py::TestStaticValidation::test_bad_layout_and_fit`, `tests/test_face_track.py::TestStaticValidation::test_bad_literal_is_an_error_at_its_argument`, `tests/test_face_track.py::TestCropFaceTrack::test_bad_arguments_are_refused_at_run_time`, `tests/test_face_track.py::TestStaticValidation::test_a_declared_grid_is_accepted`, `tests/test_face_track.py::TestDetectorSource::test_traversing_repo_is_refused`, `tests/test_select_validation.py::TestSelectErrors::test_an_out_of_range_literal_index_names_both_ends`, `tests/test_lut.py::TestPaletteAtValidate::test_both_refuse_naming_the_two`, `tests/test_lut.py::TestPalette::test_a_bad_palette_refuses_naming_the_entry` | | Variables | `dw/variables.py`: `set_variables`, `argument_errors`, `resolve_variable_values` | `argument_errors` folds a caller's arguments in exactly as `set_variables` does when a run starts, so a bad name or value is refused before anything is queued. | `tests/test_variables.py::test_argument_errors_reports_a_dict_passed_for_a_string_variable` | | List variables from the CLI | `dw/run.py`, `dw/variables.py`: `get_value` | A `name=value` string given for a list variable is split on commas, so a list of objects can only be passed as JSON over the API or MCP. | `tests/test_variables.py::test_set_variables_list_default_splits_on_comma` | -| `for_each` expansion | `dw/for_each.py`: `expand_for_each` | Each entry becomes an ordinary step named `@`, after substitution and before the reference check. | `tests/test_for_each.py::TestNaming::test_member_name_joins_with_at` | +| `for_each` expansion | `dw/for_each.py`: `expand_for_each`, `group_of` | Each entry becomes an ordinary step named `@`, after substitution and before the reference check. | `tests/test_for_each.py::TestNaming::test_member_name_joins_with_at` | | Optional entry field | `dw/for_each.py`: `_optional_item`, `_item_fields` | `{"item": "field", "default": X}` reads an entry field with a fallback; without `default` it is the plain required read, and `list_fields` / `entry_field_warnings` count the field as one entries take. The editor's string scan needs no twin code, only the shared case file. | `tests/test_for_each.py::TestOptionalItem::test_a_present_field_wins_and_an_absent_one_takes_the_default`, `tests/test_ui_twins.py::test_the_engine_decides_each_shared_reference_case` | | Previous results | `dw/previous_results.py`: `get_iterations`, `previous_result_reference_errors` | Several `previous_result:` references in one step multiply into a cartesian product. | `tests/test_previous_results.py::TestGetIterations::test_multiple_references_create_cartesian_product` | | Previous-result check | `dw/previous_results.py`: `previous_result_reference_errors` | A literal reference to a step that is not earlier in the workflow is a validation error. | `tests/test_previous_results.py::TestStaticReferenceChecking::test_a_step_cannot_reference_itself_or_a_later_one` | diff --git a/dw/for_each.py b/dw/for_each.py index 6ddc6cc4..161aad76 100644 --- a/dw/for_each.py +++ b/dw/for_each.py @@ -69,6 +69,14 @@ def is_member_name(name): return isinstance(name, str) and MEMBER_SEPARATOR in name +def group_of(name): + """The authored step a name stands for: a member's group, else the name + itself (the inverse of `member_name`; a step name never holds the + separator, so the first split is the group).""" + name = str(name) + return name.split(MEMBER_SEPARATOR)[0] if is_member_name(name) else name + + def expand_for_each(definition, source_indices=None): """The definition with every 'for_each' step replaced by its members. diff --git a/dw/workflow_run.py b/dw/workflow_run.py index 8eb36ba2..e32691da 100644 --- a/dw/workflow_run.py +++ b/dw/workflow_run.py @@ -29,6 +29,7 @@ from .arguments import realize_args from .elision import elide_definition, warn_elided from .events import emit_warning +from .for_each import group_of from .media import probe_metadata from .pipeline_ownership import allocated_mb, finish_release from .previous_results import StepResults @@ -398,7 +399,7 @@ def _surviving_variables(authored_steps, expanded_steps): referenced before substitution (#857). A for_each member ('@') stands for the step it was expanded from.""" groups = { - str(step.get("name")).split(references.MEMBER_SEPARATOR)[0] + group_of(step.get("name")) for step in expanded_steps or [] if isinstance(step, dict) } diff --git a/tests/test_for_each.py b/tests/test_for_each.py index a8f030fa..d0b0b354 100644 --- a/tests/test_for_each.py +++ b/tests/test_for_each.py @@ -46,6 +46,12 @@ class TestNaming: def test_member_name_joins_with_at(self): assert member_name("shot", "wide_open") == "shot@wide_open" + def test_group_of_inverts_member_name(self): + from dw.for_each import group_of + + assert group_of(member_name("shot", "wide_open")) == "shot" + assert group_of("plain") == "plain" + def test_is_member_name_round_trips_member_name(self): assert is_member_name(member_name("shot", "wide_open")) assert not is_member_name("shot") From 2dca6d27aa47c6d8bbbf1d8262679f6fcd6897cb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 16:47:53 -0500 Subject: [PATCH 030/204] docs(assets): #854 - upload asset_name is confined like keep_output's but placed under uploads/ Co-Authored-By: Claude Sonnet 5.5 --- dw/server/routes/assets.py | 3 ++- dw_mcp/assets.py | 4 +++- dw_mcp/tools_media.py | 4 ++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dw/server/routes/assets.py b/dw/server/routes/assets.py index 92946741..7fce782d 100644 --- a/dw/server/routes/assets.py +++ b/dw/server/routes/assets.py @@ -152,7 +152,8 @@ async def upload_media( 'cast/priya-voice.wav' rather than the random one a browser upload gets - which is what makes a recurring cast's references readable in every workflow that carries them. It may name a folder, is - confined to the library the way `keep_output`'s is, and takes the + confined the way `keep_output`'s is but placed under `uploads/` + (not at the library root, as `keep_output`'s is), and takes the uploaded file's extension when it has none of its own. Without it the name stays random, so two uploads of the same file never collide. diff --git a/dw_mcp/assets.py b/dw_mcp/assets.py index 81d9066f..f018859a 100644 --- a/dw_mcp/assets.py +++ b/dw_mcp/assets.py @@ -309,7 +309,9 @@ def upload_asset( required so the upload has a name and extension to be validated against. `asset_name` is the name it is stored under - 'cast/priya-voice.wav' - rather than the random one an upload gets by default. A recurring cast + rather than the random one an upload gets by default. It lands under + `uploads/` ('asset:uploads/cast/priya-voice.wav'), not at the library + root as `keep_output`'s `asset_name` does. A recurring cast referenced as 'asset:uploads/084eaecc....wav' in every workflow cannot be told apart without opening each file, which is the whole reason to name one (2026-09-11). With `file_path`, the extension comes from the diff --git a/dw_mcp/tools_media.py b/dw_mcp/tools_media.py index c66b2833..497294f9 100644 --- a/dw_mcp/tools_media.py +++ b/dw_mcp/tools_media.py @@ -338,8 +338,8 @@ def upload_asset( Accepts the usual image, video and audio extensions; reference the result, not a path. Pass `asset_name` for a readable stored name - ("cast/priya-voice.wav", folders allowed) - otherwise (with - `file_path`) the name is random. Pass `shared=true` to put it in + ("cast/priya-voice.wav", folders allowed; under `uploads/`) - + otherwise (with `file_path`) it is random. Pass `shared=true` to put it in the library every workspace shares - where a recurring cast belongs, since a workspace's own assets are invisible from the next. From dfb3b91d30e8639cbd2f24e4892435d17ffa8c8a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 17:17:33 -0500 Subject: [PATCH 031/204] feat(workspace): #859 - copy runs between workspaces: engine, route and copy_to_workspace Co-Authored-By: Claude Opus 5.5 --- dw/server/api_models.py | 26 +++ dw/server/routes/library.py | 88 ++++++++ dw/workspace_copy.py | 422 ++++++++++++++++++++++++++++++++++++ dw_mcp/server.py | 1 + dw_mcp/tools_authoring.py | 15 ++ dw_mcp/workspaces.py | 12 + 6 files changed, 564 insertions(+) create mode 100644 dw/workspace_copy.py diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 7544b4bc..b79ffe31 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -664,6 +664,32 @@ class WorkspaceDeleted(ApiModel): contents: dict[str, DiskUsage] = Field(description="What it held, per folder.") +class CopiedItems(ApiModel): + runs: list[str] = Field(description="`/` of each run copied.") + assets: list[str] = Field(description="Always empty until assets can be copied.") + + +class WorkspaceCopied(ApiModel): + """What a copy between workspaces did (`POST /api/workspaces/{name}/copy`).""" + + source: str + destination: str + copied: CopiedItems + already_present: list[str] = Field( + description="Runs the destination already held, by run id - skipped." + ) + renumbered: dict[str, dict[str, int]] = Field( + description="Copied runs whose `vN` changed, keyed `/`: " + "`{from, to}`." + ) + warnings: dict[str, Any] = Field( + description="`unresolved_output_refs`, `absolute_paths` and `no_jobs`, " + "each only when it applies." + ) + bytes_linked: int + bytes_copied: int + + class WorkflowCost(ApiModel): """A measured run, as the workflow's own `cost` block declares it.""" diff --git a/dw/server/routes/library.py b/dw/server/routes/library.py index 091c563c..16c7e327 100644 --- a/dw/server/routes/library.py +++ b/dw/server/routes/library.py @@ -27,6 +27,7 @@ WorkflowDeleted, WorkflowList, WorkflowSaved, + WorkspaceCopied, WorkspaceDeleted, WorkspaceInfo, WorkspaceList, @@ -36,6 +37,7 @@ from ... import references, validation from ...security import InvalidInputError, SecurityError, validate_prompt_reference from ...workflow import Workflow +from ...workspace_copy import CopyRefused, copy_runs from ...notes import read_workspace_note from ...library import ( builtin_root, @@ -148,6 +150,92 @@ def add_workspace(http_request: Request, request: WorkspaceRequest): return created.describe() +class WorkspaceCopyRequest(BaseModel): + source: Optional[str] = Field( + default=None, description="Workspace to copy from; the default when omitted" + ) + runs: list[str] = Field( + default_factory=list, + description="`/` entries, or a bare `` for " + "all its runs", + ) + assets: list[str] = Field( + default_factory=list, description="Not supported yet - leave empty" + ) + + +def _copy_side(state, root, name): + """A workspace a copy names, by name: the default when None.""" + if name is None or name == DEFAULT_WORKSPACE_NAME: + return named_workspace( + root, DEFAULT_WORKSPACE_NAME, prompts_root=state.prompt_dir + ) + try: + known = name in workspace_names(root) + except SecurityError as e: + raise HTTPException(status_code=400, detail=str(e)) + if not known: + raise HTTPException(status_code=404, detail=f"No such workspace: {name}") + try: + return named_workspace(root, name, prompts_root=state.prompt_dir) + except SecurityError as e: + raise HTTPException(status_code=400, detail=str(e)) + + +@router.post( + "/api/workspaces/{destination}/copy", + response_model=WorkspaceCopied, + response_model_exclude_unset=True, +) +def copy_into_workspace( + http_request: Request, destination: str, request: WorkspaceCopyRequest +): + """Copy runs from another workspace into this one. + + Media is hard-linked where the filesystem allows, so a copy costs + little disk; sidecars are copied. All or nothing: a run already here is + skipped as `already_present`, a run whose job is still running refuses + the request. A move is a copy, then `DELETE` of the source runs. + """ + state = http_request.app.state + manager = state.job_manager + root = workspace_root(state) + if request.assets: + raise HTTPException( + status_code=400, + detail="Copying assets is not supported yet - copy runs, and " + "re-upload the assets they need", + ) + if not request.runs: + raise HTTPException(status_code=400, detail="Name at least one run to copy") + source_name = request.source or DEFAULT_WORKSPACE_NAME + if source_name == destination: + raise HTTPException( + status_code=400, + detail=f"Source and destination are both '{destination}' - name " + f"another workspace to copy into", + ) + target = _copy_side(state, root, destination) + source = _copy_side(state, root, source_name) + with manager._lock: + active = [ + job.run_dir + for job in manager.jobs.values() + if job.status not in TERMINAL_STATES + and (job.spec.get("workspace") or DEFAULT_WORKSPACE_NAME) == source_name + and getattr(job, "run_dir", None) + ] + try: + return copy_runs(source, target, request.runs, active_runs=active) + except CopyRefused as e: + detail = ( + {"message": str(e), "conflicts": e.conflicts} if e.conflicts else str(e) + ) + raise HTTPException(status_code=e.status, detail=detail) + except (SecurityError, InvalidInputError, ValueError) as e: + raise HTTPException(status_code=400, detail=str(e)) + + @router.delete( "/api/workspaces/{name}", response_model=WorkspaceDeleted, diff --git a/dw/workspace_copy.py b/dw/workspace_copy.py new file mode 100644 index 00000000..249fe197 --- /dev/null +++ b/dw/workspace_copy.py @@ -0,0 +1,422 @@ +"""Copying runs from one workspace to another under the same root (#853). + +A run is self-contained on disk - `latest`, `vN`, `output:` resolution and +the gallery all read the run folder and its manifests, never the jobs +database - so copying the folder into the destination's outputs is the whole +of making it resolvable there. What this module adds is the care around that: + +- Confinement. Both ends are workspaces under one root, named and validated + (`named_workspace`); every path is checked with `validate_path` against + its own side's outputs folder; a source that is, passes through or holds a + symlink is refused, and nothing here ever creates one - output references + refuse linked run folders (tests/test_security_symlinks.py). +- All or nothing. Every entry is checked before anything is written. A run + folder already in the destination is the same run (same run id), so it is + `already_present` and skipped, which makes a repeated copy idempotent; a + run whose job is still running is a conflict and refuses the request. +- Files. Media is hard-linked, falling back to `copy2` across devices, as + `keep_output` does - dw never edits media in place. JSON sidecars + (manifest, workflow.json, notes.json, job.json, any `.json`) are always + copied: they *are* edited in place (`record_kept_shots`, + `write_realized_workflow`, the notes), and a link would carry an edit in + one workspace into the other. +- Versions. A copied run keeps its `vN` unless the destination's identity + already uses that number; then every run this request copies into that + identity is renumbered after the destination's highest, in run-id order, + under the identity's run lock so it cannot race `open_run`. The version is + the only manifest field a copy edits; pinned `output:` refs name run ids, + so renumbering breaks none of them. +- Warnings, never refusals: `output:` inputs that do not resolve in the + destination, manifest paths that still point into the source workspace + (reported relative to its root - no reply carries an absolute server + path), and that a copied run has no job row (it appears in the gallery, + not in the jobs list). + +Copying assets is stage B of #853. +""" + +import logging +import os +import re +import shutil + +from filelock import FileLock + +from .runs import ( + MANIFEST_FILE_NAME, + RUN_VERSION_KEY, + _read_manifest, + _run_ids, + is_run_id, + resolve_output_reference, + run_id_sort_key, + run_lock_path, + run_versions, + write_json_atomic, +) +from .security import InvalidInputError, PathTraversalError, validate_path +from .workspace import ConfiguredWorkspace, forget_workspace_usage + +logger = logging.getLogger("dw") + +# One segment of a run entry: the segment rule `output:` names follow +# (security.OUTPUT_REFERENCE_PATTERN), so whatever can be copied can be named +_SEGMENT = re.compile(r"^[\w][\w.@-]*\Z") +_MAX_SEGMENTS = 7 + +# The sidecars a copy writes fresh rather than links: every JSON file +SIDECAR_SUFFIX = ".json" + +NO_JOBS_NOTE = ( + "Copied runs appear in list_gallery and resolve through output: " + "references, but not in list_jobs - a copy carries no job record." +) + + +class CopyRefused(Exception): + """The request cannot be carried out as asked; nothing was written. + + `status` is the HTTP status the route answers with: 404 for something + that is not there, 409 for a conflict with what is.""" + + def __init__(self, message, status=409, conflicts=None): + super().__init__(message) + self.status = status + self.conflicts = conflicts or [] + + +def _run_entry(entry): + """A `runs` entry as its segments: `/` or a bare + ``. An `output:` prefix is accepted, as the gallery quotes it. + + Raises: + PathTraversalError: For an absolute path or a `..` + InvalidInputError: For anything else that is not a run name + """ + if not isinstance(entry, str) or not entry.strip(): + raise InvalidInputError("A runs entry must be a non-empty string") + name = entry.strip() + if name.startswith("output:"): + name = name[len("output:") :] + if name.startswith(("/", "\\")) or os.path.isabs(name): + raise PathTraversalError(f"'{entry}' is an absolute path - name a run") + parts = name.strip("/").split("/") + if any(part in ("..", ".") for part in parts): + raise PathTraversalError(f"'{entry}' leaves the outputs folder") + if len(parts) > _MAX_SEGMENTS or not all(_SEGMENT.match(p) for p in parts): + raise InvalidInputError( + f"'{entry}' is not a run - name one as '/', as " + f"the gallery lists it, or a bare '' for all its runs" + ) + return parts + + +def _symlink_on_the_way(base, path): + """The first symlink between `base` (exclusive) and `path` (inclusive), + or None. validate_path resolves links, so a link that stays inside the + base passes it; this refuses those too.""" + relative = os.path.relpath(path, base) + current = base + for part in relative.split(os.sep): + current = os.path.join(current, part) + if os.path.islink(current): + return current + return None + + +def _symlink_inside(directory): + """The first symlink anywhere under a run folder, or None.""" + for current, dirs, files in os.walk(directory): + for name in dirs + files: + path = os.path.join(current, name) + if os.path.islink(path): + return path + return None + + +def _source_runs(source, entry): + """[(identity, run id)] that one entry names in the source. + + Raises: + PathTraversalError: When the entry escapes, or reaches a symlink + CopyRefused (404): When there is no such run or workflow + """ + parts = _run_entry(entry) + outputs = os.path.realpath(source.outputs) + if is_run_id(parts[-1]) and len(parts) > 1: + identity, run_ids = "/".join(parts[:-1]), [parts[-1]] + else: + identity, run_ids = "/".join(parts), None + identity_dir = validate_path(os.path.join(outputs, *identity.split("/")), outputs) + if run_ids is None: + run_ids = _run_ids(identity_dir) if os.path.isdir(identity_dir) else [] + if not run_ids: + raise CopyRefused( + f"No runs of '{identity}' in workspace '{source.name}'", status=404 + ) + found = [] + for run_id in run_ids: + run_dir = os.path.join(outputs, *identity.split("/"), run_id) + link = _symlink_on_the_way(outputs, run_dir) + if link: + raise PathTraversalError( + f"'{entry}' passes through a symlink - a copy never follows one" + ) + run_dir = validate_path(run_dir, outputs, allow_create=False) + if not os.path.isdir(run_dir): + raise CopyRefused( + f"No run '{identity}/{run_id}' in workspace '{source.name}'", + status=404, + ) + if _symlink_inside(run_dir): + raise PathTraversalError( + f"Run '{identity}/{run_id}' holds a symlink - a copy never " + f"follows or creates one" + ) + found.append((identity, run_id)) + return found + + +def _copy_file(source, target, totals): + """Hard-link a media file, falling back to a copy; copy a sidecar.""" + size = os.path.getsize(source) + if not source.endswith(SIDECAR_SUFFIX): + try: + os.link(source, target) + totals["bytes_linked"] += size + return + except OSError: + pass # another device, or a filesystem without hard links + shutil.copy2(source, target) + totals["bytes_copied"] += size + + +def _copy_tree(source_dir, target_dir, totals): + """Recreate a run folder file by file - no symlinks, ever.""" + os.makedirs(target_dir) + for current, dirs, files in os.walk(source_dir): + relative = os.path.relpath(current, source_dir) + here = ( + target_dir if relative == os.curdir else os.path.join(target_dir, relative) + ) + for name in dirs: + os.makedirs(os.path.join(here, name), exist_ok=True) + for name in files: + _copy_file(os.path.join(current, name), os.path.join(here, name), totals) + + +def _place_run(source_dir, identity_dir, run_id, totals): + """Copy one run folder in under a name no listing reads, then rename it + into place - a reader never sees half a run.""" + staging = os.path.join(identity_dir, f".copy-{run_id}") + if os.path.exists(staging): + shutil.rmtree(staging) + try: + _copy_tree(source_dir, staging, totals) + os.rename(staging, os.path.join(identity_dir, run_id)) + except BaseException: + shutil.rmtree(staging, ignore_errors=True) + raise + + +def _set_version(run_dir, version): + """Record a copied run's ordinal in its own (copied, never linked) + manifest. A run with no readable manifest is left alone.""" + manifest = _read_manifest(run_dir) + if manifest is None or manifest.get(RUN_VERSION_KEY) == version: + return + manifest[RUN_VERSION_KEY] = version + write_json_atomic(os.path.join(run_dir, MANIFEST_FILE_NAME), manifest) + + +def _copy_identity(source, destination, identity, run_ids, totals): + """Copy one identity's runs under its run lock and settle their + versions. Returns {"/": {"from", "to"}} for each run whose + number changed.""" + source_identity = os.path.join( + os.path.realpath(source.outputs), *identity.split("/") + ) + outputs = os.path.realpath(destination.outputs) + identity_dir = validate_path(os.path.join(outputs, *identity.split("/")), outputs) + source_versions = run_versions(source_identity) + run_ids = sorted(run_ids, key=run_id_sort_key) + renumbered = {} + with FileLock(run_lock_path(identity_dir)): + os.makedirs(identity_dir, exist_ok=True) + held = run_versions(identity_dir) + wanted = [source_versions.get(run_id) for run_id in run_ids] + collides = any(v is not None and v in held.values() for v in wanted) + next_number = max(held.values(), default=0) + 1 + for run_id, version in zip(run_ids, wanted): + _place_run( + os.path.join(source_identity, run_id), identity_dir, run_id, totals + ) + if version is None: + continue + new = version + if collides: + new, next_number = next_number, next_number + 1 + # Pinned even when unchanged: a source run that recorded no + # number was ranked, and its rank in the destination is not + # the one it had + _set_version(os.path.join(identity_dir, run_id), new) + if new != version: + renumbered[f"{identity}/{run_id}"] = {"from": version, "to": new} + return renumbered + + +def _absolute_paths(manifest, source_root): + """Paths in a manifest that still point into the source workspace, + relative to its root: a file a step cache hit republished from an + earlier run, and the workflow file the run was started from.""" + found = [] + + def visit(value): + if isinstance(value, dict): + for item in value.values(): + visit(item) + elif isinstance(value, list): + for item in value: + visit(item) + elif isinstance(value, str) and os.path.isabs(value): + try: + inside = ( + os.path.commonpath([os.path.realpath(value), source_root]) + == source_root + ) + except ValueError: + inside = False + if inside: + relative = os.path.relpath(os.path.realpath(value), source_root) + found.append(relative.replace(os.sep, "/")) + + visit({key: manifest.get(key) for key in ("workflow", "steps")}) + return sorted(set(found)) + + +def _unresolved_inputs(manifest, outputs): + """The `output:` inputs a copied run's manifest records that name + nothing in the destination.""" + unresolved = [] + for entry in manifest.get("inputs") or []: + if not isinstance(entry, dict) or entry.get("kind") != "output": + continue + reference = entry.get("reference") + if not isinstance(reference, str): + continue + try: + resolve_output_reference(reference, outputs) + except (ValueError, OSError, InvalidInputError, PathTraversalError): + unresolved.append(reference) + return unresolved + + +def copy_runs(source, destination, runs, active_runs=()): + """Copy runs from one workspace into another. + + Args: + source, destination: Workspaces under one root, already resolved + and known to exist + runs: `/` or bare `` entries + active_runs: `/` of the source's runs whose job + is still queued or running - their manifest is not final + + Returns: + {copied: {runs, assets}, already_present, renumbered, warnings, + bytes_linked, bytes_copied} + + Raises: + InvalidInputError, PathTraversalError: A malformed or escaping entry + CopyRefused: Anything else that stops the copy; nothing was written + """ + for side in (source, destination): + if isinstance(side, ConfiguredWorkspace): + raise CopyRefused( + "This server has no workspace root - it was started with " + "individual directory overrides, so there is no other " + "workspace to copy to or from" + ) + if os.path.realpath(source.root) == os.path.realpath(destination.root): + raise InvalidInputError( + f"Source and destination are both '{source.name}' - name another " + f"workspace to copy into" + ) + if not runs: + raise InvalidInputError("Name at least one run to copy") + os.makedirs(destination.outputs, exist_ok=True) + if not os.access(destination.outputs, os.W_OK): + raise CopyRefused(f"Workspace '{destination.name}' is not writable") + + named = [] + for entry in runs: + for found in _source_runs(source, entry): + if found not in named: + named.append(found) + + outputs = os.path.realpath(destination.outputs) + active = set(active_runs) + conflicts = [] + already_present = [] + to_copy = {} + for identity, run_id in named: + name = f"{identity}/{run_id}" + if name in active: + conflicts.append(name) + continue + target = validate_path( + os.path.join(outputs, *identity.split("/"), run_id), outputs + ) + if os.path.lexists(target): + already_present.append(name) + else: + to_copy.setdefault(identity, []).append(run_id) + if conflicts: + raise CopyRefused( + "These runs' jobs are still queued or running, so their manifests " + "are not final - wait for them, then copy: " + ", ".join(conflicts), + conflicts=conflicts, + ) + + totals = {"bytes_linked": 0, "bytes_copied": 0} + renumbered = {} + copied = [] + for identity, run_ids in to_copy.items(): + renumbered.update( + _copy_identity(source, destination, identity, run_ids, totals) + ) + copied.extend(f"{identity}/{run_id}" for run_id in run_ids) + forget_workspace_usage() + logger.info( + f"Copied {len(copied)} run(s) from workspace {source.name} to " + f"{destination.name}" + ) + + warnings = {} + source_root = os.path.realpath(source.root) + unresolved = {} + absolute = {} + for name in copied: + run_dir = os.path.join(outputs, *name.split("/")) + manifest = _read_manifest(run_dir) or {} + refs = _unresolved_inputs(manifest, outputs) + if refs: + unresolved[name] = refs + paths = _absolute_paths(manifest, source_root) + if paths: + absolute[name] = paths + if unresolved: + warnings["unresolved_output_refs"] = unresolved + if absolute: + warnings["absolute_paths"] = absolute + if copied: + warnings["no_jobs"] = NO_JOBS_NOTE + + return { + "source": source.name, + "destination": destination.name, + "copied": {"runs": copied, "assets": []}, + "already_present": already_present, + "renumbered": renumbered, + "warnings": warnings, + **totals, + } diff --git a/dw_mcp/server.py b/dw_mcp/server.py index fc316475..0042d681 100644 --- a/dw_mcp/server.py +++ b/dw_mcp/server.py @@ -171,6 +171,7 @@ def tool(fn, annotations): tool(wks.list_workspaces, READ_ONLY) tool(wks.use_workspace, WRITES) tool(wks.create_workspace, WRITES) + tool(wks.copy_to_workspace, WRITES) tool(wks.delete_workspace, DELETES) tool(aut.validate_workflow, READ_ONLY) diff --git a/dw_mcp/tools_authoring.py b/dw_mcp/tools_authoring.py index e5c4318f..139fdb5b 100644 --- a/dw_mcp/tools_authoring.py +++ b/dw_mcp/tools_authoring.py @@ -38,6 +38,21 @@ def create_workspace(self, name: str, use: bool = False) -> dict: the result says so.""" return workspaces.create_workspace(self.client, name, use=use) + def copy_to_workspace( + self, destination: str, runs: list[str], source: str | None = None + ) -> dict: + """Copy runs from one workspace into another: each entry is + '/' as list_gallery names it, or a bare + '' for all its runs. source defaults to this session's + workspace. Media is hard-linked, so a copy costs little disk. Runs + the destination already has are skipped; a vN it already uses is + renumbered and reported. Copies get no job record: they show in + list_gallery and resolve as output: refs, not in list_jobs. To move, + copy then delete_output the source runs.""" + return workspaces.copy_to_workspace( + self.client, destination, runs, source=source + ) + def delete_workspace(self, name: str, acknowledged_cost: bool = False) -> dict: """Permanently delete a workspace and every workflow, asset and generated file in it. Refuses without acknowledged_cost=True, and diff --git a/dw_mcp/workspaces.py b/dw_mcp/workspaces.py index c54cc5c8..754cf7a8 100644 --- a/dw_mcp/workspaces.py +++ b/dw_mcp/workspaces.py @@ -151,6 +151,18 @@ def create_workspace(client, name, use=False): return result +def copy_to_workspace(client, destination, runs, source=None): + """Copy runs into another workspace under the same root. `source` + defaults to this session's workspace. The server hard-links media and + copies sidecars, skips runs the destination already holds + (`already_present`), and reports renumbered `vN` and warnings.""" + source = source or client.workspace + return client.post_json( + api_path("api", "workspaces", destination, "copy"), + {"source": source, "runs": list(runs or [])}, + ) + + def delete_workspace(client, name, acknowledged_cost=False): """Delete a workspace and every workflow, asset and generated file in it. From 3f03c9fd750d962721d4077929c33b1220f8722a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 17:23:23 -0500 Subject: [PATCH 032/204] feat(workspace): #859 - copy tests, docs, pinned tool lists, surface budget and OpenAPI A named run missing from the source is a 404, not a 400. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/ARCHITECTURE.md | 1 + docs/MCP.md | 3 +- docs/SERVER.md | 12 + docs/WORKSPACES.md | 36 ++ dw/server/guides.py | 2 +- dw/workspace_copy.py | 2 +- tests/test_mcp_server.py | 13 +- tests/test_workspace_copy.py | 550 +++++++++++++++++++++++++++++ ui/src/lib/generated/api-schema.ts | 140 +++++++- ui/src/lib/generated/openapi.json | 185 +++++++++- 11 files changed, 929 insertions(+), 17 deletions(-) create mode 100644 tests/test_workspace_copy.py diff --git a/README.md b/README.md index 3ef0fca6..e7526730 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ once; without it the run fails partway through with a 401/403 from the Hub. ## Drive it from an agent -Then just ask. The agent has 62 tools covering the whole surface — the +Then just ask. The agent has 63 tools covering the whole surface — the workflow catalog, the real diffusers pipeline signatures, the job queue, the gallery, the model cache: diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1983fa66..23c5a9db 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -57,6 +57,7 @@ one to open. | Packaged builtins | `dw/library.py`: `builtin_root`, `resolve_sub_workflow_reference` | `builtin:` names the packaged `dw/workflows/`, not the top-level `workflows/` examples. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_builtin_resolves_in_the_packaged_root` | | Sub-workflow paths | `dw/library.py`: `resolve_sub_workflow_reference`, `resolve_sub_workflow` | A sub-workflow path is confined to the root it resolves in. The search order is in the docstring of `resolve_sub_workflow`. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_path_escaping_its_root_is_refused` | | Workspace | `dw/workspace.py`: `resolve_workspace`, `set_workspace` | `--workspace` beats `DW_WORKSPACE`, which beats the setting, which beats a working directory that looks like a workspace, which beats `~/diffusers-workspace`. | `tests/test_workspace.py::TestResolution::test_a_flag_wins_over_everything`, `tests/test_workspace.py::TestResolution::test_a_bare_working_directory_falls_back_to_the_home_workspace` | +| Workspace copy | `dw/workspace_copy.py`: `copy_runs` | Copies runs between workspaces under one root: media hard-linked, sidecars always copied, never a symlink; all or nothing; `vN` renumbered under the run lock. | `tests/test_workspace_copy.py` | | Realized workflow | `dw/realize.py`: `realize_workflow`, `dw/runs.py`: `write_realized_workflow` | Each run directory holds `workflow.json`, with the run's arguments, seed and stored prompt text pinned. | `tests/test_realize.py::TestVariablesAndSeed::test_arguments_become_the_variable_defaults`, `tests/test_runs.py::TestRealizedWorkflow::test_the_run_directory_holds_a_realized_copy` | ## Runs and outputs diff --git a/docs/MCP.md b/docs/MCP.md index 920f1c59..df53d504 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -187,7 +187,7 @@ Nothing in this sequence costs GPU time. ## Tool reference -62 tools in six groups. Names and arguments below are transcribed from +63 tools in six groups. Names and arguments below are transcribed from `dw_mcp/tools_*.py` — nothing here is renamed or reshaped for the docs. ### Catalog (read-only) @@ -278,6 +278,7 @@ tools) rather than relying on the session pin. | `list_workspaces()` | — | The server's workspaces and which one this session is using. Each has its own workflows, assets and outputs; the prompt library is shared by all of them | | `use_workspace(name)` | `name` | Work in that workspace for the rest of the session - every later call reads and writes there. This is how to keep your work out of another agent's namespace rather than sharing the default one. Checked against the server, so a typo fails here rather than scoping every later call to nothing. On a `dw.serve --mcp` endpoint the pin is shared by every connected client (#298, see the note above the table) - a `warning` field appears when this call overwrote a pin already pointed away from `default` | | `create_workspace(name, use=False)` | `name`, `use` | Create a workspace. Pass use=true to switch this session to it as well; otherwise the session stays where it was and the result says so. The result names the workspace (`name`, `default`, `current`, `next`), not its folders - `list_workspaces(detail=true)` is the opt-in for those. `use=true` carries the same mounted-server caveat and `warning` field as `use_workspace` | +| `copy_to_workspace(destination, runs, source=None)` | `destination`, `runs`, `source` | Copy runs from one workspace into another. Each `runs` entry is `/` as `list_gallery` names it, or a bare `` for all its runs; `source` defaults to this session's workspace. Free: no `acknowledged_cost`. Media is hard-linked (a copy across devices falls back to a real copy) and JSON sidecars are always copied, so an edit in one workspace never reaches the other. All or nothing; a run the destination already has is skipped as `already_present`, so a repeat is idempotent, and a `vN` it already uses is renumbered after its highest and reported in `renumbered`. Warnings: `unresolved_output_refs`, `absolute_paths`, `no_jobs`. A copied run has no job record - it shows in `list_gallery` and resolves as an `output:` reference, not in `list_jobs`. A run whose job is still queued or running is refused. To move, copy then `delete_output` the source runs. See [Workspaces](WORKSPACES.md#copying-between-workspaces) | | `delete_workspace(name, acknowledged_cost=False)` | `name`, `acknowledged_cost` | Permanently delete a workspace and everything in it. Refuses without the acknowledgement, reporting what it would remove | | `list_assets(limit=50, prefix)` | `limit`, `prefix` | The input media on the server, each with the `asset:` reference a workflow argument carries. Look here before asking for a file - what a workflow needs may already be there. `libraries` names the roots searched and which are writable; `shadowed` lists names a nearer library hides. Newest first; `prefix` keeps names starting with it (e.g. `cast/`), `limit` bounds the answer, `total` is the count matched before the cut | | `keep_output(name, asset_name=None, overwrite=False, shared=False, workspace=None, note=None)` | `name`, optional `asset_name`, `overwrite`, `shared`, `workspace`, `note` | Keep a generated file as an input asset under a stable `asset:` name, so a later workflow can rely on it. The copy happens on the server: nothing is downloaded or re-uploaded. `note` is kept with the asset; left out, the source file's note is copied. `asset_name` may name a folder and takes the kept file's extension when it has none; `shared=true` keeps it in the library every workspace shares, which is where a recurring cast belongs. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | diff --git a/docs/SERVER.md b/docs/SERVER.md index 886f8584..e647c64b 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -577,6 +577,18 @@ The editor's forms come from these; they are just as usable from scripts: `exports/` sits beside the workspace's own folders, holding one directory per exported job. It is a reserved name: no workspace can be called `exports`, and the folder is never listed as one. +- `POST /api/workspaces/{destination}/copy` (`{"source": ..., "runs": [...], + "assets": []}`) — copy runs from one workspace into another. A `runs` entry + is `/`, or a bare `` for all its runs. A + non-empty `assets` is a 400: copying assets is not supported yet. An empty + `runs` or a source equal to the destination is a 400, an unknown workspace + or run a 404, and a server without a workspace root, a run whose job is still + queued or running, or a destination that is not writable a 409. All or + nothing. The reply is `{source, destination, copied: {runs, assets}, + already_present, renumbered, warnings: {unresolved_output_refs, + absolute_paths, no_jobs}, bytes_linked, bytes_copied}`; `renumbered` maps + `"/"` to `{"from": 3, "to": 7}`. What is copied and how is + in [WORKSPACES.md](WORKSPACES.md#copying-between-workspaces). - `GET /api/assets` — the asset library: input media, each with the `asset:` reference a workflow carries rather than a path, since a path only means something on the server's own machine. Empty rather than an diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index f7964b40..843a9aa3 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -324,6 +324,42 @@ A workspace is a **namespace, not a security boundary**. The API token is all-or-nothing: anything that can reach the server can name any workspace on it. Use them to keep work apart, not to keep it private. +## Copying between workspaces + +A run is self-contained on disk, so copying its folder into another +workspace's `outputs/` is what makes it resolvable there. `copy_to_workspace` +(MCP) and `POST /api/workspaces/{destination}/copy` ([SERVER.md](SERVER.md)) +do that for runs, named `/` or, for all of a workflow's runs, +a bare ``. Only runs are copied for now; assets are not, and a +request naming some is refused. + +Media is hard-linked, so a copy costs almost no disk, falling back to a real +copy across devices. JSON sidecars (the manifest, `workflow.json`, notes, +any `.json`) are always copied, never linked: they are edited in place, and a +link would carry an edit in one workspace into the other. Nothing follows or +creates a symlink. + +The copy is all or nothing: every entry is checked before anything is written. +A run whose run id the destination already has is the same run, so it is +skipped and listed as `already_present`, and repeating a copy changes nothing. +A run whose job is still queued or running is refused, since its manifest is +not final. + +A run keeps its `vN` unless the destination already uses that number for the +workflow. Then this request's runs of that workflow are renumbered after the +destination's highest, in run-id order, under the run lock so a concurrent run +cannot take the same number. `renumbered` reports each `from` and `to`. Pinned +`output:` references name run ids, so none of them break. + +The reply carries warnings, not refusals: `unresolved_output_refs` (a run's +`output:` input that does not resolve in the destination), `absolute_paths` +(manifest paths that still point into the source workspace, reported relative +to the source workspace's root) and `no_jobs`. A copied run has no job record, so it appears in +the gallery and resolves as an `output:` reference, but not in `list_jobs`. + +A move is a copy, then a delete of the source runs (`delete_output`, or +`DELETE` on the gallery runs); there is no move that could lose a run halfway. + ## Where this is going The resolver, the workflow search path with writes confined to the writable diff --git a/dw/server/guides.py b/dw/server/guides.py index ddbbebee..2ce08aa1 100644 --- a/dw/server/guides.py +++ b/dw/server/guides.py @@ -240,7 +240,7 @@ def get_guide(name, section=None): Without a `section` the answer is the *index*: the guide's preamble, its first section, and the headings of the rest - not the whole file. A full WORKFLOW_GUIDE.md is ~19.6k tokens and TASKS.md ~16k, which is - more in one call than the entire 62-tool MCP surface costs to connect, + more in one call than the entire 63-tool MCP surface costs to connect, and an agent can make that call twice before noticing (#101). Every withheld section is named in `sections` and fetched by name, so nothing is unreachable - only unspent by accident. diff --git a/dw/workspace_copy.py b/dw/workspace_copy.py index 249fe197..9a3751e4 100644 --- a/dw/workspace_copy.py +++ b/dw/workspace_copy.py @@ -162,7 +162,7 @@ def _source_runs(source, entry): raise PathTraversalError( f"'{entry}' passes through a symlink - a copy never follows one" ) - run_dir = validate_path(run_dir, outputs, allow_create=False) + run_dir = validate_path(run_dir, outputs) if not os.path.isdir(run_dir): raise CopyRefused( f"No run '{identity}/{run_id}' in workspace '{source.name}'", diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 9f16d234..9fcd5fc3 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -76,6 +76,7 @@ "delete_asset", "list_workspaces", "use_workspace", + "copy_to_workspace", "create_workspace", "delete_workspace", } @@ -103,6 +104,7 @@ "keep_output", "delete_asset", "use_workspace", + "copy_to_workspace", "create_workspace", "delete_workspace", } @@ -704,6 +706,12 @@ def _a_file_to_upload(): ("list_workspaces", {}, "GET", "/api/workspaces"), ("use_workspace", {"name": "default"}, "GET", "/api/workspaces"), ("create_workspace", {"name": "shots"}, "POST", "/api/workspaces"), + ( + "copy_to_workspace", + {"destination": "shots", "runs": ["wf/20261013-120000-0123abcd"]}, + "POST", + "/api/workspaces/shots/copy", + ), ( "delete_workspace", {"name": "shots", "acknowledged_cost": True}, @@ -1569,7 +1577,10 @@ def test_the_stated_tool_count_is_the_registered_one(): # tokens) - measured 14_246.25 before and 14_262.25 after (9_281.75 / # 4_470.25 / 510.25). Text trimmed since 10-07 left room, so the ceiling # stays, 2.75 of headroom. -SURFACE_BUDGET = 14_265 +# 2026-10-13 (#859, #853 stage A): copy_to_workspace (121.25 description + +# 84.75 schema) - measured at 14_470.25 (9_276.25 / 4_683.75 / 510.25); the +# ceiling moves to 14_471 with 0.75 of headroom. +SURFACE_BUDGET = 14_471 @pytest.mark.asyncio diff --git a/tests/test_workspace_copy.py b/tests/test_workspace_copy.py new file mode 100644 index 00000000..b84c79b9 --- /dev/null +++ b/tests/test_workspace_copy.py @@ -0,0 +1,550 @@ +"""Copying runs between workspaces (#853): the engine, the route and the MCP tool.""" + +import json +import os +import re +from types import SimpleNamespace + +import httpx +import pytest +from fastapi.testclient import TestClient + +from dw.runs import RUN_ID_PATTERN, resolve_output_reference, run_versions +from dw.security import InvalidInputError, PathTraversalError, SecurityError +from dw.server.app import create_app +from dw.server.jobs import JobManager +from dw.workspace import ( + ConfiguredWorkspace, + Workspace, + create_workspace, +) +from dw.workspace_copy import CopyRefused, copy_runs +from dw_mcp.client import DwClient +from dw_mcp.workspaces import copy_to_workspace + +from .test_server import ScriptedWorkerManager, success_script + +R1 = "20260101-000000-aaaaaaaa" +R2 = "20260102-000000-bbbbbbbb" +R3 = "20260103-000000-cccccccc" + + +def test_run_ids_match_the_pattern(): + for run_id in (R1, R2, R3): + assert RUN_ID_PATTERN.match(run_id) + assert re.match(r"\d{8}-\d{6}-[0-9a-f]{8}$", run_id) + + +@pytest.fixture +def root(tmp_path): + return Workspace(tmp_path / "studio", "flag").ensure() + + +@pytest.fixture +def src(root): + return create_workspace(root, "src") + + +@pytest.fixture +def dst(root): + return create_workspace(root, "dst") + + +def make_run( + ws, + identity, + run_id, + version=1, + inputs=(), + workflow_file=None, + steps=None, + notes=None, + content=b"png-bytes", +): + run_dir = os.path.join(ws.outputs, *identity.split("/"), run_id) + os.makedirs(run_dir) + with open(os.path.join(run_dir, "img.png"), "wb") as f: + f.write(content) + manifest = { + "run_id": run_id, + "version": version, + "inputs": [{"reference": r, "kind": "output"} for r in inputs], + "workflow": {"file": workflow_file or "relative.json"}, + } + if steps is not None: + manifest["steps"] = steps + with open(os.path.join(run_dir, "manifest.json"), "w") as f: + json.dump(manifest, f) + if notes is not None: + with open(os.path.join(run_dir, "notes.json"), "w") as f: + json.dump(notes, f) + return run_dir + + +def read_json(path): + with open(path) as f: + return json.load(f) + + +def nothing_written(ws): + return not os.path.exists(ws.outputs) or not any(os.scandir(ws.outputs)) + + +class TestHappyPath: + def test_one_run_is_copied(self, src, dst): + make_run(src, "wf", R1) + reply = copy_runs(src, dst, [f"wf/{R1}"]) + assert set(reply) == { + "source", + "destination", + "copied", + "already_present", + "renumbered", + "warnings", + "bytes_linked", + "bytes_copied", + } + assert reply["source"] == "src" + assert reply["destination"] == "dst" + assert reply["copied"] == {"runs": [f"wf/{R1}"], "assets": []} + assert reply["already_present"] == [] + assert reply["renumbered"] == {} + for name in ("img.png", "manifest.json"): + assert os.path.isfile(os.path.join(dst.outputs, "wf", R1, name)) + + def test_a_bare_workflow_copies_all_its_runs(self, src, dst): + make_run(src, "wf", R1, 1) + make_run(src, "wf", R2, 2) + reply = copy_runs(src, dst, ["wf"]) + assert reply["copied"]["runs"] == [f"wf/{R1}", f"wf/{R2}"] + + def test_a_multi_segment_identity(self, src, dst): + make_run(src, "ltx2/Gyre", R1) + reply = copy_runs(src, dst, [f"ltx2/Gyre/{R1}"]) + assert reply["copied"]["runs"] == [f"ltx2/Gyre/{R1}"] + assert os.path.isfile(os.path.join(dst.outputs, "ltx2", "Gyre", R1, "img.png")) + + def test_a_bare_multi_segment_identity(self, src, dst): + make_run(src, "ltx2/Gyre", R1) + reply = copy_runs(src, dst, ["ltx2/Gyre"]) + assert reply["copied"]["runs"] == [f"ltx2/Gyre/{R1}"] + + def test_an_output_prefix_is_accepted(self, src, dst): + make_run(src, "wf", R1) + reply = copy_runs(src, dst, [f"output:wf/{R1}"]) + assert reply["copied"]["runs"] == [f"wf/{R1}"] + + def test_repeating_a_copy_is_idempotent(self, src, dst): + make_run(src, "wf", R1) + copy_runs(src, dst, [f"wf/{R1}"]) + again = copy_runs(src, dst, [f"wf/{R1}"]) + assert again["already_present"] == [f"wf/{R1}"] + assert again["copied"]["runs"] == [] + assert "no_jobs" not in again["warnings"] + + +class TestRenumbering: + def test_a_collision_renumbers_in_run_id_order(self, src, dst): + make_run(dst, "wf", "20251231-000000-dddddddd", 1) + make_run(src, "wf", R2, 2) + make_run(src, "wf", R1, 1) + make_run(src, "wf", R3, 3) + # v1 collides, so every copied run of the identity is renumbered + reply = copy_runs(src, dst, ["wf"]) + assert reply["renumbered"] == { + f"wf/{R1}": {"from": 1, "to": 2}, + f"wf/{R2}": {"from": 2, "to": 3}, + f"wf/{R3}": {"from": 3, "to": 4}, + } + copied = read_json(os.path.join(dst.outputs, "wf", R1, "manifest.json")) + assert copied["version"] == 2 + versions = run_versions(os.path.join(dst.outputs, "wf")) + assert versions[R1] == 2 and versions[R2] == 3 and versions[R3] == 4 + + def test_two_runs_renumber_after_one(self, src, dst): + make_run(dst, "wf", "20251231-000000-dddddddd", 1) + make_run(src, "wf", R1, 1) + make_run(src, "wf", R2, 2) + reply = copy_runs(src, dst, ["wf"]) + assert reply["renumbered"] == { + f"wf/{R1}": {"from": 1, "to": 2}, + f"wf/{R2}": {"from": 2, "to": 3}, + } + + def test_no_collision_keeps_numbers(self, src, dst): + make_run(dst, "wf", "20251231-000000-dddddddd", 1) + make_run(src, "wf", R2, 2) + reply = copy_runs(src, dst, [f"wf/{R2}"]) + assert reply["renumbered"] == {} + copied = read_json(os.path.join(dst.outputs, "wf", R2, "manifest.json")) + assert copied["version"] == 2 + + +class TestWarnings: + def test_unresolved_output_refs(self, src, dst): + make_run(dst, "other", R1) + make_run( + src, + "wf", + R2, + inputs=["output:other/latest/img.png", "output:gone/latest/img.png"], + ) + reply = copy_runs(src, dst, [f"wf/{R2}"]) + unresolved = reply["warnings"]["unresolved_output_refs"] + assert unresolved == {f"wf/{R2}": ["output:gone/latest/img.png"]} + + def test_all_refs_resolving_gives_no_warning(self, src, dst): + make_run(dst, "other", R1) + make_run(src, "wf", R2, inputs=["output:other/latest/img.png"]) + reply = copy_runs(src, dst, [f"wf/{R2}"]) + assert "unresolved_output_refs" not in reply["warnings"] + + def test_absolute_paths_are_relative_to_the_source_root(self, src, dst, tmp_path): + inside = os.path.join(src.outputs, "wf", "earlier", "img.png") + workflow = os.path.join(src.workflows, "wf.json") + make_run( + src, + "wf", + R1, + workflow_file=workflow, + steps=[{"step": "s", "files": [inside, "relative/img.png"]}], + ) + reply = copy_runs(src, dst, [f"wf/{R1}"]) + paths = reply["warnings"]["absolute_paths"][f"wf/{R1}"] + assert paths == sorted(["outputs/wf/earlier/img.png", "workflows/wf.json"]) + assert str(tmp_path) not in json.dumps(reply) + + def test_a_path_outside_the_source_root_is_not_reported(self, src, dst): + make_run(src, "wf", R1, workflow_file="/somewhere/else/wf.json") + reply = copy_runs(src, dst, [f"wf/{R1}"]) + assert "absolute_paths" not in reply["warnings"] + + def test_no_jobs_note_after_a_copy(self, src, dst): + make_run(src, "wf", R1) + reply = copy_runs(src, dst, [f"wf/{R1}"]) + assert "no_jobs" in reply["warnings"] + + +class TestRefusals: + @pytest.mark.parametrize("entry", ["../x", "/abs", "wf/../../x", "wf/.."]) + def test_traversal(self, src, dst, entry): + make_run(src, "wf", R1) + with pytest.raises(SecurityError): + copy_runs(src, dst, [entry]) + assert nothing_written(dst) + + def test_a_symlinked_run_folder(self, src, dst, tmp_path): + make_run(src, "elsewhere", R1) + os.makedirs(os.path.join(src.outputs, "wf")) + os.symlink( + os.path.join(src.outputs, "elsewhere", R1), + os.path.join(src.outputs, "wf", R1), + ) + with pytest.raises(PathTraversalError): + copy_runs(src, dst, [f"wf/{R1}"]) + assert nothing_written(dst) + + def test_a_symlinked_identity_folder(self, src, dst): + make_run(src, "real", R1) + os.symlink( + os.path.join(src.outputs, "real"), os.path.join(src.outputs, "linked") + ) + with pytest.raises(PathTraversalError): + copy_runs(src, dst, [f"linked/{R1}"]) + assert nothing_written(dst) + + def test_a_symlink_inside_a_run(self, src, dst, tmp_path): + run_dir = make_run(src, "wf", R1) + os.symlink(os.path.join(run_dir, "img.png"), os.path.join(run_dir, "link.png")) + with pytest.raises(PathTraversalError): + copy_runs(src, dst, [f"wf/{R1}"]) + assert nothing_written(dst) + + def test_a_missing_run(self, src, dst): + make_run(src, "wf", R1) + with pytest.raises(CopyRefused) as raised: + copy_runs(src, dst, [f"wf/{R2}"]) + assert raised.value.status == 404 + assert nothing_written(dst) + + def test_a_missing_workflow(self, src, dst): + with pytest.raises(CopyRefused) as raised: + copy_runs(src, dst, ["nope"]) + assert raised.value.status == 404 + + def test_same_workspace(self, src): + make_run(src, "wf", R1) + with pytest.raises(InvalidInputError): + copy_runs(src, src, [f"wf/{R1}"]) + + def test_empty_runs(self, src, dst): + with pytest.raises(InvalidInputError): + copy_runs(src, dst, []) + + def configured(self, ws): + return ConfiguredWorkspace(ws.workflows, ws.assets, ws.outputs, ws.prompts) + + def test_configured_source(self, src, dst): + make_run(src, "wf", R1) + with pytest.raises(CopyRefused) as raised: + copy_runs(self.configured(src), dst, [f"wf/{R1}"]) + assert raised.value.status == 409 + assert nothing_written(dst) + + def test_configured_destination(self, src, dst): + make_run(src, "wf", R1) + with pytest.raises(CopyRefused) as raised: + copy_runs(src, self.configured(dst), [f"wf/{R1}"]) + assert raised.value.status == 409 + assert nothing_written(dst) + + def test_an_active_run_refuses_all_or_nothing(self, src, dst): + make_run(src, "wf", R1, 1) + make_run(src, "wf", R2, 2) + with pytest.raises(CopyRefused) as raised: + copy_runs(src, dst, ["wf"], active_runs=[f"wf/{R2}"]) + assert raised.value.status == 409 + assert raised.value.conflicts == [f"wf/{R2}"] + assert nothing_written(dst) + assert not os.path.exists(os.path.join(dst.outputs, "wf", R1)) + + +class TestResolution: + def test_output_references_resolve_in_the_destination(self, src, dst): + make_run(src, "wf", R1) + copy_runs(src, dst, ["wf"]) + latest = resolve_output_reference("output:wf/latest/img.png", dst.outputs) + pinned = resolve_output_reference(f"output:wf/{R1}/img.png", dst.outputs) + expected = os.path.join(os.path.realpath(dst.outputs), "wf", R1, "img.png") + assert latest == expected + assert pinned == expected + assert os.path.isfile(pinned) + + +class TestIndependence: + def test_notes_are_not_shared(self, src, dst): + source_run = make_run(src, "wf", R1, notes={"note": "original"}) + copy_runs(src, dst, [f"wf/{R1}"]) + copied = os.path.join(dst.outputs, "wf", R1, "notes.json") + original = os.path.join(source_run, "notes.json") + assert os.stat(copied).st_ino != os.stat(original).st_ino + with open(copied, "w") as f: + json.dump({"note": "edited"}, f) + assert read_json(original) == {"note": "original"} + + def test_a_renumbered_manifest_leaves_the_source_alone(self, src, dst): + make_run(dst, "wf", "20251231-000000-dddddddd", 1) + source_run = make_run(src, "wf", R1, 1) + copy_runs(src, dst, [f"wf/{R1}"]) + copied = os.path.join(dst.outputs, "wf", R1, "manifest.json") + original = os.path.join(source_run, "manifest.json") + assert read_json(copied)["version"] == 2 + assert read_json(original)["version"] == 1 + assert os.stat(copied).st_ino != os.stat(original).st_ino + + +class TestMediaLinking: + def test_media_is_hard_linked(self, src, dst): + source_run = make_run(src, "wf", R1, content=b"x" * 100) + reply = copy_runs(src, dst, [f"wf/{R1}"]) + a = os.stat(os.path.join(source_run, "img.png")) + b = os.stat(os.path.join(dst.outputs, "wf", R1, "img.png")) + assert a.st_ino == b.st_ino + assert reply["bytes_linked"] >= 100 + + def test_a_failed_link_falls_back_to_a_copy(self, src, dst, monkeypatch): + source_run = make_run(src, "wf", R1, content=b"x" * 100) + + def refuse(*args, **kwargs): + raise OSError("cross-device") + + monkeypatch.setattr("dw.workspace_copy.os.link", refuse) + reply = copy_runs(src, dst, [f"wf/{R1}"]) + target = os.path.join(dst.outputs, "wf", R1, "img.png") + source_file = os.path.join(source_run, "img.png") + assert os.stat(target).st_ino != os.stat(source_file).st_ino + with open(target, "rb") as a, open(source_file, "rb") as b: + assert a.read() == b.read() + assert reply["bytes_linked"] == 0 + assert reply["bytes_copied"] >= 100 + + +@pytest.fixture +def server(root, tmp_path): + def make(with_root=True): + manager = JobManager( + root.outputs, + worker_manager=ScriptedWorkerManager(success_script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=root.workflows, + ) + kwargs = {"workspace": root.root} if with_root else {} + app = create_app( + workflow_dir=root.workflows, + output_dir=root.outputs, + job_manager=manager, + prompt_dir=root.prompts, + asset_dir=root.assets, + **kwargs, + ) + return TestClient(app, base_url="http://localhost") + + return make + + +class TestRoute: + def test_happy_path(self, server, root, src, dst): + make_run(src, "wf", R1) + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["copied"]["runs"] == [f"wf/{R1}"] + assert os.path.isfile(os.path.join(dst.outputs, "wf", R1, "img.png")) + + def test_assets_are_not_supported(self, server, src, dst): + make_run(src, "wf", R1) + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"], "assets": ["a.png"]}, + ) + assert response.status_code == 400 + assert "not supported" in response.text + assert nothing_written(dst) + + def test_empty_runs(self, server, src, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", json={"source": "src", "runs": []} + ) + assert response.status_code == 400 + + def test_same_workspace(self, server, src): + with server() as client: + response = client.post( + "/api/workspaces/src/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 400 + + def test_unknown_destination(self, server, src): + make_run(src, "wf", R1) + with server() as client: + response = client.post( + "/api/workspaces/nope/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 404 + + def test_unknown_source(self, server, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "nope", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 404 + + def test_a_missing_run_is_404(self, server, src, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 404 + + def test_traversal_is_400(self, server, src, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": ["../x"]}, + ) + assert response.status_code == 400 + + def test_an_active_job_conflicts(self, server, src, dst): + make_run(src, "wf", R1) + with server() as client: + job = SimpleNamespace( + status="running", + run_dir=f"wf/{R1}", + spec={"workspace": "src"}, + ) + client.app.state.job_manager.jobs["j1"] = job + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 409 + assert response.json()["detail"]["conflicts"] == [f"wf/{R1}"] + assert nothing_written(dst) + + def test_a_finished_job_does_not_conflict(self, server, src, dst): + make_run(src, "wf", R1) + with server() as client: + client.app.state.job_manager.jobs["j1"] = SimpleNamespace( + status="succeeded", run_dir=f"wf/{R1}", spec={"workspace": "src"} + ) + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 200 + + def test_no_workspace_root(self, server): + with server(with_root=False) as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 409 + + def test_the_default_is_the_source_when_omitted(self, server, root, dst): + make_run(root, "wf", R1) + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", json={"runs": [f"wf/{R1}"]} + ) + assert response.status_code == 200, response.text + assert response.json()["source"] == "default" + assert os.path.isfile(os.path.join(dst.outputs, "wf", R1, "img.png")) + + def test_the_default_can_be_the_destination(self, server, root, src): + make_run(src, "wf", R1) + with server() as client: + response = client.post( + "/api/workspaces/default/copy", + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 200, response.text + assert os.path.isfile(os.path.join(root.outputs, "wf", R1, "img.png")) + + +class TestMcp: + def recording(self, workspace=None): + seen = [] + + def handler(request): + seen.append(request) + return httpx.Response(200, json={"ok": True}) + + client = DwClient( + transport=httpx.MockTransport(handler), + **({"workspace": workspace} if workspace else {}), + ) + return client, seen + + def test_source_defaults_to_the_session_workspace(self): + client, seen = self.recording("shots") + copy_to_workspace(client, "final", ["wf/x"]) + assert seen[0].method == "POST" + assert seen[0].url.path == "/api/workspaces/final/copy" + body = json.loads(seen[0].content) + assert body == {"source": "shots", "runs": ["wf/x"]} + + def test_an_explicit_source_passes_through(self): + client, seen = self.recording("shots") + copy_to_workspace(client, "final", ["wf/x"], source="other") + assert json.loads(seen[0].content)["source"] == "other" diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 4e1d97d0..a01bc09b 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -1491,7 +1491,8 @@ export interface paths { * 'cast/priya-voice.wav' rather than the random one a browser upload * gets - which is what makes a recurring cast's references readable * in every workflow that carries them. It may name a folder, is - * confined to the library the way `keep_output`'s is, and takes the + * confined the way `keep_output`'s is but placed under `uploads/` + * (not at the library root, as `keep_output`'s is), and takes the * uploaded file's extension when it has none of its own. Without it * the name stays random, so two uploads of the same file never * collide. @@ -1692,6 +1693,31 @@ export interface paths { patch?: never; trace?: never; }; + "/api/workspaces/{destination}/copy": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Copy Into Workspace + * @description Copy runs from another workspace into this one. + * + * Media is hard-linked where the filesystem allows, so a copy costs + * little disk; sidecars are copied. All or nothing: a run already here is + * skipped as `already_present`, a run whose job is still running refuses + * the request. A move is a copy, then `DELETE` of the source runs. + */ + post: operations["copy_into_workspace_api_workspaces__destination__copy_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/api/workspaces/{name}": { parameters: { query?: never; @@ -1974,6 +2000,19 @@ export interface components { /** Kind */ kind: string; }; + /** CopiedItems */ + CopiedItems: { + /** + * Assets + * @description Always empty until assets can be copied. + */ + assets: string[]; + /** + * Runs + * @description `/` of each run copied. + */ + runs: string[]; + }; /** Deleted */ Deleted: { /** Deleted */ @@ -3163,11 +3202,6 @@ export interface components { * @description With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true. */ scaled?: string; - /** - * Withheld - * @description With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's). - */ - withheld?: string; /** Tempered */ tempered?: boolean; /** @@ -3175,6 +3209,11 @@ export interface components { * @description What contributed nothing to `minutes` when `partial` is true - the workflow's own id when its own steps went unpriced, else the path of each composed child with no cost block or observed history. When `minutes` is null it still names the composed children that left it unpriced (`partial` is then false); empty when nothing composed went unpriced. */ unpriced: string[]; + /** + * Withheld + * @description With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's). + */ + withheld?: string; }; /** PromptCard */ PromptCard: { @@ -3655,6 +3694,60 @@ export interface components { /** Workspace */ workspace: string; }; + /** + * WorkspaceCopied + * @description What a copy between workspaces did (`POST /api/workspaces/{name}/copy`). + */ + WorkspaceCopied: { + /** + * Already Present + * @description Runs the destination already held, by run id - skipped. + */ + already_present: string[]; + /** Bytes Copied */ + bytes_copied: number; + /** Bytes Linked */ + bytes_linked: number; + copied: components["schemas"]["CopiedItems"]; + /** Destination */ + destination: string; + /** + * Renumbered + * @description Copied runs whose `vN` changed, keyed `/`: `{from, to}`. + */ + renumbered: { + [key: string]: { + [key: string]: number; + }; + }; + /** Source */ + source: string; + /** + * Warnings + * @description `unresolved_output_refs`, `absolute_paths` and `no_jobs`, each only when it applies. + */ + warnings: { + [key: string]: unknown; + }; + }; + /** WorkspaceCopyRequest */ + WorkspaceCopyRequest: { + /** + * Assets + * @description Not supported yet - leave empty + */ + assets?: string[]; + /** + * Runs + * @description `/` entries, or a bare `` for all its runs + */ + runs?: string[]; + /** + * Source + * @description Workspace to copy from; the default when omitted + */ + source?: string | null; + }; /** WorkspaceDeleted */ WorkspaceDeleted: { /** @@ -6063,6 +6156,41 @@ export interface operations { }; }; }; + copy_into_workspace_api_workspaces__destination__copy_post: { + parameters: { + query?: never; + header?: never; + path: { + destination: string; + }; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["WorkspaceCopyRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["WorkspaceCopied"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; remove_workspace_api_workspaces__name__delete: { parameters: { query?: { diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 44eb3e43..61d5ab6a 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -422,6 +422,33 @@ "title": "ClassList", "type": "object" }, + "CopiedItems": { + "additionalProperties": false, + "properties": { + "assets": { + "description": "Always empty until assets can be copied.", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "runs": { + "description": "`/` of each run copied.", + "items": { + "type": "string" + }, + "title": "Runs", + "type": "array" + } + }, + "required": [ + "runs", + "assets" + ], + "title": "CopiedItems", + "type": "object" + }, "Deleted": { "additionalProperties": false, "properties": { @@ -3506,11 +3533,6 @@ "title": "Scaled", "type": "string" }, - "withheld": { - "description": "With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's).", - "title": "Withheld", - "type": "string" - }, "tempered": { "title": "Tempered", "type": "boolean" @@ -3522,6 +3544,11 @@ }, "title": "Unpriced", "type": "array" + }, + "withheld": { + "description": "With basis `unknown` and `inherited_from` set: why the template's runs could not be re-priced for this workflow (its size cannot be compared to the template's).", + "title": "Withheld", + "type": "string" } }, "required": [ @@ -4776,6 +4803,102 @@ "title": "WorkflowSaved", "type": "object" }, + "WorkspaceCopied": { + "additionalProperties": false, + "description": "What a copy between workspaces did (`POST /api/workspaces/{name}/copy`).", + "properties": { + "already_present": { + "description": "Runs the destination already held, by run id - skipped.", + "items": { + "type": "string" + }, + "title": "Already Present", + "type": "array" + }, + "bytes_copied": { + "title": "Bytes Copied", + "type": "integer" + }, + "bytes_linked": { + "title": "Bytes Linked", + "type": "integer" + }, + "copied": { + "$ref": "#/components/schemas/CopiedItems" + }, + "destination": { + "title": "Destination", + "type": "string" + }, + "renumbered": { + "additionalProperties": { + "additionalProperties": { + "type": "integer" + }, + "type": "object" + }, + "description": "Copied runs whose `vN` changed, keyed `/`: `{from, to}`.", + "title": "Renumbered", + "type": "object" + }, + "source": { + "title": "Source", + "type": "string" + }, + "warnings": { + "additionalProperties": true, + "description": "`unresolved_output_refs`, `absolute_paths` and `no_jobs`, each only when it applies.", + "title": "Warnings", + "type": "object" + } + }, + "required": [ + "source", + "destination", + "copied", + "already_present", + "renumbered", + "warnings", + "bytes_linked", + "bytes_copied" + ], + "title": "WorkspaceCopied", + "type": "object" + }, + "WorkspaceCopyRequest": { + "properties": { + "assets": { + "description": "Not supported yet - leave empty", + "items": { + "type": "string" + }, + "title": "Assets", + "type": "array" + }, + "runs": { + "description": "`/` entries, or a bare `` for all its runs", + "items": { + "type": "string" + }, + "title": "Runs", + "type": "array" + }, + "source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Workspace to copy from; the default when omitted", + "title": "Source" + } + }, + "title": "WorkspaceCopyRequest", + "type": "object" + }, "WorkspaceDeleted": { "additionalProperties": false, "properties": { @@ -8221,7 +8344,7 @@ }, "/api/uploads": { "post": { - "description": "Save a browser-picked image, video or audio file into the asset library's\nuploads/ subfolder and hand back the reference a workflow argument\ncan carry.\n\nAn upload is input, so it belongs in the asset library rather than\namong generated output, and the reference handed back is\n'asset:uploads/' - portable, and meaningful in a workflow that\nis saved and rerun later. A server with no asset library configured\nanswers 409. The body is the raw file bytes: no multipart parser\ndependency needed for a single-file upload.\n\n`asset_name` stores it under a name of the caller's choosing -\n'cast/priya-voice.wav' rather than the random one a browser upload\ngets - which is what makes a recurring cast's references readable\nin every workflow that carries them. It may name a folder, is\nconfined to the library the way `keep_output`'s is, and takes the\nuploaded file's extension when it has none of its own. Without it\nthe name stays random, so two uploads of the same file never\ncollide.\n\n`shared` puts it in the library every workspace under this root\nshares rather than in this workspace's own - a recurring cast that\nepisode four, in a workspace of its own, still has to reach.\n\n`note` is kept beside it (the directory's notes.json). An upload over an\nexisting name takes its own note, or none: not the old file's, and not\nthe old file's kept provenance either.", + "description": "Save a browser-picked image, video or audio file into the asset library's\nuploads/ subfolder and hand back the reference a workflow argument\ncan carry.\n\nAn upload is input, so it belongs in the asset library rather than\namong generated output, and the reference handed back is\n'asset:uploads/' - portable, and meaningful in a workflow that\nis saved and rerun later. A server with no asset library configured\nanswers 409. The body is the raw file bytes: no multipart parser\ndependency needed for a single-file upload.\n\n`asset_name` stores it under a name of the caller's choosing -\n'cast/priya-voice.wav' rather than the random one a browser upload\ngets - which is what makes a recurring cast's references readable\nin every workflow that carries them. It may name a folder, is\nconfined the way `keep_output`'s is but placed under `uploads/`\n(not at the library root, as `keep_output`'s is), and takes the\nuploaded file's extension when it has none of its own. Without it\nthe name stays random, so two uploads of the same file never\ncollide.\n\n`shared` puts it in the library every workspace under this root\nshares rather than in this workspace's own - a recurring cast that\nepisode four, in a workspace of its own, still has to reach.\n\n`note` is kept beside it (the directory's notes.json). An upload over an\nexisting name takes its own note, or none: not the old file's, and not\nthe old file's kept provenance either.", "operationId": "upload_media_api_uploads_post", "parameters": [ { @@ -8920,6 +9043,56 @@ "summary": "Add Workspace" } }, + "/api/workspaces/{destination}/copy": { + "post": { + "description": "Copy runs from another workspace into this one.\n\nMedia is hard-linked where the filesystem allows, so a copy costs\nlittle disk; sidecars are copied. All or nothing: a run already here is\nskipped as `already_present`, a run whose job is still running refuses\nthe request. A move is a copy, then `DELETE` of the source runs.", + "operationId": "copy_into_workspace_api_workspaces__destination__copy_post", + "parameters": [ + { + "in": "path", + "name": "destination", + "required": true, + "schema": { + "title": "Destination", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceCopyRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceCopied" + } + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Copy Into Workspace" + } + }, "/api/workspaces/{name}": { "delete": { "description": "Delete a workspace and everything in it.\n\nAnswers what it would remove and refuses until `acknowledged=true`:\nthis deletes generated work, and a count is what makes it an\ninformed choice rather than a surprise. The unacknowledged message\nnames only what would be removed - how to proceed is left to the\ncaller, since the MCP surface tells its own callers to acknowledge\nthrough a differently-named parameter (`acknowledged_cost`).", From fecd361f5de409c0d2a22ceb84f54193608732d0 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 17:37:38 -0500 Subject: [PATCH 033/204] refactor(workspace): #859 - copy calls its owners: prefixes, run numbering, output name shape Architecture review of #859: - `output:` is stripped with references.ref_name(OUTPUT, ...), not spelled in workspace_copy (prefix_literals back to 0). - Run numbering moves to dw/runs.py: copy_runs_in places the folders under the identity's run lock, pins the destination's unrecorded runs first, and writes the version; identity_run_ids replaces the private _run_ids. - The entry shape is security.validate_output_prefix, built from the same segment as OUTPUT_REFERENCE_PATTERN (pattern string unchanged). - copy_runs split into _refuse_unless_copyable, _plan_copy and _copy_warnings (C901 back under 15). Baseline rise: modules 210 -> 211, for dw/workspace_copy.py, the new module plan v2 of #853 calls for. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 4 +- docs/stabilization/baseline.json | 2 +- dw/runs.py | 59 ++++++++++ dw/security.py | 27 ++++- dw/workspace_copy.py | 180 +++++++++++++------------------ tests/test_workspace_copy.py | 44 ++++++++ 6 files changed, 209 insertions(+), 107 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 23c5a9db..ecd3fb50 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -57,7 +57,7 @@ one to open. | Packaged builtins | `dw/library.py`: `builtin_root`, `resolve_sub_workflow_reference` | `builtin:` names the packaged `dw/workflows/`, not the top-level `workflows/` examples. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_builtin_resolves_in_the_packaged_root` | | Sub-workflow paths | `dw/library.py`: `resolve_sub_workflow_reference`, `resolve_sub_workflow` | A sub-workflow path is confined to the root it resolves in. The search order is in the docstring of `resolve_sub_workflow`. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_path_escaping_its_root_is_refused` | | Workspace | `dw/workspace.py`: `resolve_workspace`, `set_workspace` | `--workspace` beats `DW_WORKSPACE`, which beats the setting, which beats a working directory that looks like a workspace, which beats `~/diffusers-workspace`. | `tests/test_workspace.py::TestResolution::test_a_flag_wins_over_everything`, `tests/test_workspace.py::TestResolution::test_a_bare_working_directory_falls_back_to_the_home_workspace` | -| Workspace copy | `dw/workspace_copy.py`: `copy_runs` | Copies runs between workspaces under one root: media hard-linked, sidecars always copied, never a symlink; all or nothing; `vN` renumbered under the run lock. | `tests/test_workspace_copy.py` | +| Workspace copy | `dw/workspace_copy.py`: `copy_runs` | Copies runs between workspaces under one root: media hard-linked, sidecars always copied, never a symlink; all or nothing; `vN` renumbered by `runs.copy_runs_in` under the run lock; entries checked by `security.validate_output_prefix`. | `tests/test_workspace_copy.py` | | Realized workflow | `dw/realize.py`: `realize_workflow`, `dw/runs.py`: `write_realized_workflow` | Each run directory holds `workflow.json`, with the run's arguments, seed and stored prompt text pinned. | `tests/test_realize.py::TestVariablesAndSeed::test_arguments_become_the_variable_defaults`, `tests/test_runs.py::TestRealizedWorkflow::test_the_run_directory_holds_a_realized_copy` | ## Runs and outputs @@ -67,7 +67,7 @@ one to open. | Run directories | `dw/runs.py`: `open_run`, `workflow_identity`, `new_run_id` | Each execution writes into `///` with a `manifest.json`, unless the flat layout is chosen (the test pins the flat-layout exception). | `tests/test_runs.py::TestRunDirectories::test_the_flat_layout_writes_where_it_always_did` | | A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server infers it identically for older runs. | `tests/test_roles.py` | | Notes on runs, files, assets, workspaces | dw/notes.py | notes.json beside what it describes (run dir or asset dir; a file's own note else the run's), workspace.json at a workspace root; only the server writes them, never the engine; ≤4000 chars, plain text. | tests/test_notes.py | -| Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions` | A run takes its number once, when it opens, so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | +| Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions`, `copy_runs_in` | A run takes its number once, when it opens (or, copied in from another workspace, when `copy_runs_in` places it), so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | | Result subfolders | `dw/subfolders.py`: `subfolder_errors`, `dw/security.py`: `SUBFOLDER_PATTERN` | A step's `result.subfolder` is checked for shape at validation. | `tests/test_subfolders.py::TestValidationErrorsIntegration::test_validation_errors_reports_a_bad_subfolder` | | Template subfolder roles | `workflows/templates/`, `dw/workflows/` | Every saving step of a template is marked `final` or `intermediate`, with at least one `final`, and packaged builtins stay unmarked. | `tests/test_template_subfolders.py::test_every_saving_step_of_a_template_names_its_role`, `tests/test_template_subfolders.py::test_the_packaged_builtins_stay_unmarked` | diff --git a/docs/stabilization/baseline.json b/docs/stabilization/baseline.json index 021a3180..9d02b7fe 100644 --- a/docs/stabilization/baseline.json +++ b/docs/stabilization/baseline.json @@ -1,5 +1,5 @@ { - "modules": 210, + "modules": 211, "modules_over_size_ceiling": 0, "functions_over_150_lines": 0, "prefix_literals": 0, diff --git a/dw/runs.py b/dw/runs.py index da7e5bac..72600fda 100644 --- a/dw/runs.py +++ b/dw/runs.py @@ -680,6 +680,65 @@ def open_run(output_dir, file_spec, workflow_id, run_id): return candidate, version +def copy_runs_in(source_identity_dir, identity_dir, run_ids, place): + """Bring runs of one identity in from another workspace and settle their + numbers - the copy's counterpart of `open_run` (dw/workspace_copy.py). + + `place(run_id)` puts one run folder at `/`. It is + called under the identity's run lock, oldest run first, so no run can + open between a copy landing and its number being written down. + + A copied run keeps the version it had in the source unless the + destination already holds that number; then every run of this call is + numbered after the destination's highest, in run-id order. The + destination's unrecorded runs are pinned first (`record_run_versions`): + a copied run sorting among them would otherwise move their ranked + numbers. The number is written into the copied manifest - a copy, never + a link - so the source's stays as it was. + + Returns: + {run id: {"from": old, "to": new}} for each run whose number changed + """ + source_versions = run_versions(source_identity_dir) + run_ids = sorted(run_ids, key=run_id_sort_key) + renumbered = {} + with FileLock(run_lock_path(identity_dir)): + os.makedirs(identity_dir, exist_ok=True) + held = set(record_run_versions(identity_dir).values()) + wanted = [source_versions.get(run_id) for run_id in run_ids] + collides = any(v is not None and v in held for v in wanted) + next_number = max(held, default=0) + 1 + for run_id, version in zip(run_ids, wanted): + place(run_id) + if version is None: + continue + new = version + if collides: + new, next_number = next_number, next_number + 1 + # Pinned even when unchanged: a source run that recorded no + # number was ranked, and its rank in the destination is not + # the one it had + _pin_version(os.path.join(identity_dir, run_id), new) + if new != version: + renumbered[run_id] = {"from": version, "to": new} + return renumbered + + +def _pin_version(run_dir, version): + """Record a run's ordinal in its manifest. A run with no readable + manifest is left alone, and stays ranked.""" + manifest = _read_manifest(run_dir) + if manifest is None or manifest.get(RUN_VERSION_KEY) == version: + return + manifest[RUN_VERSION_KEY] = version + write_json_atomic(os.path.join(run_dir, MANIFEST_FILE_NAME), manifest) + + +def identity_run_ids(identity_dir): + """Every run id under one workflow identity, oldest first.""" + return _run_ids(identity_dir) + + def write_json_atomic(path, data): """Write `data` as JSON to `path` by a temp file and a rename, so a reader never sees half a file. Raises OSError.""" diff --git a/dw/security.py b/dw/security.py index df01c4f0..4fa506de 100644 --- a/dw/security.py +++ b/dw/security.py @@ -492,8 +492,12 @@ def validate_asset_reference(name: str) -> str: # checked by the validate_path that joins the name onto the output root - # and a name still may not *start* with it. OUTPUT_REFERENCE_CHARACTERS = r"[\w.@-]" -OUTPUT_REFERENCE_PATTERN = r"^[\w][\w.@-]*(/[\w][\w.@-]*){1,6}\Z" +_OUTPUT_SEGMENT = r"[\w][\w.@-]*" +OUTPUT_REFERENCE_PATTERN = rf"^{_OUTPUT_SEGMENT}(/{_OUTPUT_SEGMENT}){{1,6}}\Z" MAX_OUTPUT_REFERENCE_LENGTH = 500 +# A run, or every run of one workflow: an output name cut short of its +# file, so whatever a copy between workspaces takes can be named after it +OUTPUT_PREFIX_PATTERN = rf"^{_OUTPUT_SEGMENT}(/{_OUTPUT_SEGMENT}){{0,6}}\Z" def validate_output_reference(name: str) -> str: @@ -525,6 +529,27 @@ def validate_output_reference(name: str) -> str: ) +def validate_output_prefix(name: str) -> str: + """ + Validate the leading segments of an output name: `/` + or a bare ``, as a copy between workspaces names runs. The + segment rule is validate_output_reference's; containment is checked by + the validate_path that joins the name. + + Raises: + InvalidInputError: If name is invalid + """ + return _validate_name( + name, + OUTPUT_PREFIX_PATTERN, + MAX_OUTPUT_REFERENCE_LENGTH, + "Run name", + "name a run as '/', as the gallery lists it, or a " + "bare '' for all its runs", + allowed=OUTPUT_REFERENCE_CHARACTERS, + ) + + # A step's result 'subfolder': a relative path under the run directory. The # segment rule is OUTPUT_REFERENCE_PATTERN's, so every subfolder the engine # writes is one a later workflow can name with 'output:'. It also refuses a diff --git a/dw/workspace_copy.py b/dw/workspace_copy.py index 9a3751e4..83cca460 100644 --- a/dw/workspace_copy.py +++ b/dw/workspace_copy.py @@ -23,9 +23,10 @@ - Versions. A copied run keeps its `vN` unless the destination's identity already uses that number; then every run this request copies into that identity is renumbered after the destination's highest, in run-id order, - under the identity's run lock so it cannot race `open_run`. The version is - the only manifest field a copy edits; pinned `output:` refs name run ids, - so renumbering breaks none of them. + under the identity's run lock so it cannot race `open_run`. dw/runs.py + owns the numbering (`copy_runs_in`); this module only places the folders. + The version is the only manifest field a copy edits; pinned `output:` refs + name run ids, so renumbering breaks none of them. - Warnings, never refusals: `output:` inputs that do not resolve in the destination, manifest paths that still point into the source workspace (reported relative to its root - no reply carries an absolute server @@ -37,33 +38,26 @@ import logging import os -import re import shutil -from filelock import FileLock - +from .references import OUTPUT, ref_name from .runs import ( - MANIFEST_FILE_NAME, - RUN_VERSION_KEY, _read_manifest, - _run_ids, + copy_runs_in, + identity_run_ids, is_run_id, resolve_output_reference, - run_id_sort_key, - run_lock_path, - run_versions, - write_json_atomic, ) -from .security import InvalidInputError, PathTraversalError, validate_path +from .security import ( + InvalidInputError, + PathTraversalError, + validate_output_prefix, + validate_path, +) from .workspace import ConfiguredWorkspace, forget_workspace_usage logger = logging.getLogger("dw") -# One segment of a run entry: the segment rule `output:` names follow -# (security.OUTPUT_REFERENCE_PATTERN), so whatever can be copied can be named -_SEGMENT = re.compile(r"^[\w][\w.@-]*\Z") -_MAX_SEGMENTS = 7 - # The sidecars a copy writes fresh rather than links: every JSON file SIDECAR_SUFFIX = ".json" @@ -96,18 +90,14 @@ def _run_entry(entry): if not isinstance(entry, str) or not entry.strip(): raise InvalidInputError("A runs entry must be a non-empty string") name = entry.strip() - if name.startswith("output:"): - name = name[len("output:") :] + name = ref_name(OUTPUT, name) or name if name.startswith(("/", "\\")) or os.path.isabs(name): raise PathTraversalError(f"'{entry}' is an absolute path - name a run") - parts = name.strip("/").split("/") + name = name.strip("/") + parts = name.split("/") if any(part in ("..", ".") for part in parts): raise PathTraversalError(f"'{entry}' leaves the outputs folder") - if len(parts) > _MAX_SEGMENTS or not all(_SEGMENT.match(p) for p in parts): - raise InvalidInputError( - f"'{entry}' is not a run - name one as '/', as " - f"the gallery lists it, or a bare '' for all its runs" - ) + validate_output_prefix(name) return parts @@ -149,7 +139,7 @@ def _source_runs(source, entry): identity, run_ids = "/".join(parts), None identity_dir = validate_path(os.path.join(outputs, *identity.split("/")), outputs) if run_ids is None: - run_ids = _run_ids(identity_dir) if os.path.isdir(identity_dir) else [] + run_ids = identity_run_ids(identity_dir) if not run_ids: raise CopyRefused( f"No runs of '{identity}' in workspace '{source.name}'", status=404 @@ -219,50 +209,21 @@ def _place_run(source_dir, identity_dir, run_id, totals): raise -def _set_version(run_dir, version): - """Record a copied run's ordinal in its own (copied, never linked) - manifest. A run with no readable manifest is left alone.""" - manifest = _read_manifest(run_dir) - if manifest is None or manifest.get(RUN_VERSION_KEY) == version: - return - manifest[RUN_VERSION_KEY] = version - write_json_atomic(os.path.join(run_dir, MANIFEST_FILE_NAME), manifest) - - def _copy_identity(source, destination, identity, run_ids, totals): - """Copy one identity's runs under its run lock and settle their - versions. Returns {"/": {"from", "to"}} for each run whose - number changed.""" + """Copy one identity's runs; dw/runs.py settles their versions under + the identity's run lock. Returns {"/": {"from", "to"}} + for each run whose number changed.""" source_identity = os.path.join( os.path.realpath(source.outputs), *identity.split("/") ) outputs = os.path.realpath(destination.outputs) identity_dir = validate_path(os.path.join(outputs, *identity.split("/")), outputs) - source_versions = run_versions(source_identity) - run_ids = sorted(run_ids, key=run_id_sort_key) - renumbered = {} - with FileLock(run_lock_path(identity_dir)): - os.makedirs(identity_dir, exist_ok=True) - held = run_versions(identity_dir) - wanted = [source_versions.get(run_id) for run_id in run_ids] - collides = any(v is not None and v in held.values() for v in wanted) - next_number = max(held.values(), default=0) + 1 - for run_id, version in zip(run_ids, wanted): - _place_run( - os.path.join(source_identity, run_id), identity_dir, run_id, totals - ) - if version is None: - continue - new = version - if collides: - new, next_number = next_number, next_number + 1 - # Pinned even when unchanged: a source run that recorded no - # number was ranked, and its rank in the destination is not - # the one it had - _set_version(os.path.join(identity_dir, run_id), new) - if new != version: - renumbered[f"{identity}/{run_id}"] = {"from": version, "to": new} - return renumbered + + def place(run_id): + _place_run(os.path.join(source_identity, run_id), identity_dir, run_id, totals) + + renumbered = copy_runs_in(source_identity, identity_dir, run_ids, place) + return {f"{identity}/{run_id}": change for run_id, change in renumbered.items()} def _absolute_paths(manifest, source_root): @@ -311,24 +272,8 @@ def _unresolved_inputs(manifest, outputs): return unresolved -def copy_runs(source, destination, runs, active_runs=()): - """Copy runs from one workspace into another. - - Args: - source, destination: Workspaces under one root, already resolved - and known to exist - runs: `/` or bare `` entries - active_runs: `/` of the source's runs whose job - is still queued or running - their manifest is not final - - Returns: - {copied: {runs, assets}, already_present, renumbered, warnings, - bytes_linked, bytes_copied} - - Raises: - InvalidInputError, PathTraversalError: A malformed or escaping entry - CopyRefused: Anything else that stops the copy; nothing was written - """ +def _refuse_unless_copyable(source, destination, runs): + """The checks on the request as a whole, before any entry is read.""" for side in (source, destination): if isinstance(side, ConfiguredWorkspace): raise CopyRefused( @@ -347,13 +292,16 @@ def copy_runs(source, destination, runs, active_runs=()): if not os.access(destination.outputs, os.W_OK): raise CopyRefused(f"Workspace '{destination.name}' is not writable") + +def _plan_copy(source, outputs, runs, active_runs): + """Every entry checked before anything is written: ({identity: [run + id]} to copy, [already present]), or CopyRefused naming the runs whose + job is still active.""" named = [] for entry in runs: for found in _source_runs(source, entry): if found not in named: named.append(found) - - outputs = os.path.realpath(destination.outputs) active = set(active_runs) conflicts = [] already_present = [] @@ -376,28 +324,16 @@ def copy_runs(source, destination, runs, active_runs=()): "are not final - wait for them, then copy: " + ", ".join(conflicts), conflicts=conflicts, ) + return to_copy, already_present - totals = {"bytes_linked": 0, "bytes_copied": 0} - renumbered = {} - copied = [] - for identity, run_ids in to_copy.items(): - renumbered.update( - _copy_identity(source, destination, identity, run_ids, totals) - ) - copied.extend(f"{identity}/{run_id}" for run_id in run_ids) - forget_workspace_usage() - logger.info( - f"Copied {len(copied)} run(s) from workspace {source.name} to " - f"{destination.name}" - ) +def _copy_warnings(copied, outputs, source_root): + """What the caller should know about the copied runs - never a refusal.""" warnings = {} - source_root = os.path.realpath(source.root) unresolved = {} absolute = {} for name in copied: - run_dir = os.path.join(outputs, *name.split("/")) - manifest = _read_manifest(run_dir) or {} + manifest = _read_manifest(os.path.join(outputs, *name.split("/"))) or {} refs = _unresolved_inputs(manifest, outputs) if refs: unresolved[name] = refs @@ -410,6 +346,44 @@ def copy_runs(source, destination, runs, active_runs=()): warnings["absolute_paths"] = absolute if copied: warnings["no_jobs"] = NO_JOBS_NOTE + return warnings + + +def copy_runs(source, destination, runs, active_runs=()): + """Copy runs from one workspace into another. + + Args: + source, destination: Workspaces under one root, already resolved + and known to exist + runs: `/` or bare `` entries + active_runs: `/` of the source's runs whose job + is still queued or running - their manifest is not final + + Returns: + {copied: {runs, assets}, already_present, renumbered, warnings, + bytes_linked, bytes_copied} + + Raises: + InvalidInputError, PathTraversalError: A malformed or escaping entry + CopyRefused: Anything else that stops the copy; nothing was written + """ + _refuse_unless_copyable(source, destination, runs) + outputs = os.path.realpath(destination.outputs) + to_copy, already_present = _plan_copy(source, outputs, runs, active_runs) + + totals = {"bytes_linked": 0, "bytes_copied": 0} + renumbered = {} + copied = [] + for identity, run_ids in to_copy.items(): + renumbered.update( + _copy_identity(source, destination, identity, run_ids, totals) + ) + copied.extend(f"{identity}/{run_id}" for run_id in run_ids) + forget_workspace_usage() + logger.info( + f"Copied {len(copied)} run(s) from workspace {source.name} to " + f"{destination.name}" + ) return { "source": source.name, @@ -417,6 +391,6 @@ def copy_runs(source, destination, runs, active_runs=()): "copied": {"runs": copied, "assets": []}, "already_present": already_present, "renumbered": renumbered, - "warnings": warnings, + "warnings": _copy_warnings(copied, outputs, os.path.realpath(source.root)), **totals, } diff --git a/tests/test_workspace_copy.py b/tests/test_workspace_copy.py index b84c79b9..f066f681 100644 --- a/tests/test_workspace_copy.py +++ b/tests/test_workspace_copy.py @@ -548,3 +548,47 @@ def test_an_explicit_source_passes_through(self): client, seen = self.recording("shots") copy_to_workspace(client, "final", ["wf/x"], source="other") assert json.loads(seen[0].content)["source"] == "other" + + +class TestTheOwners: + """The rules a copy relies on live with their owners (#859 arch review): + the entry shape in dw/security.py, the numbering in dw/runs.py.""" + + def test_an_entry_follows_the_output_reference_segment_rule(self): + from dw.security import validate_output_prefix, validate_output_reference + + assert validate_output_prefix("wf") == "wf" + assert validate_output_prefix(f"ltx2/Gyre/{R1}") == f"ltx2/Gyre/{R1}" + for name in ("wf@a", "w.f-1/x"): + validate_output_prefix(name) + validate_output_reference(f"{name}/latest/img.png") + for bad in ("@wf", "wf/.hidden", "a/b/c/d/e/f/g/h", "wf/x y"): + with pytest.raises(InvalidInputError): + validate_output_prefix(bad) + + def test_copy_runs_in_pins_the_destinations_unrecorded_runs_first(self, tmp_path): + from dw.runs import copy_runs_in + + source = tmp_path / "src" / "wf" + dest = tmp_path / "dst" / "wf" + for base, run_id, version in ((source, R2, 1), (dest, R1, None)): + run_dir = base / run_id + run_dir.mkdir(parents=True) + manifest = {"run_id": run_id} + if version is not None: + manifest["version"] = version + (run_dir / "manifest.json").write_text(json.dumps(manifest)) + + def place(run_id): + os.makedirs(dest / run_id) + (dest / run_id / "manifest.json").write_text( + (source / run_id / "manifest.json").read_text() + ) + + renumbered = copy_runs_in(str(source), str(dest), [R2], place) + + assert renumbered == {R2: {"from": 1, "to": 2}} + recorded = json.loads((dest / R1 / "manifest.json").read_text()) + assert recorded["version"] == 1 + assert run_versions(str(dest)) == {R1: 1, R2: 2} + assert json.loads((source / R2 / "manifest.json").read_text())["version"] == 1 From a5ee1af81a055e2a6546b60f999b47493e56c239 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 17:45:05 -0500 Subject: [PATCH 034/204] fix(roles): report only recorded roles; a run from before roles has none The server inferred roles for older runs from the realized workflow, falling back to subfolders and single-step runs, which labelled nearly every pre-roles file final. Drop the inference: an entry without a recorded role is unknown, not guessed, and the UI shows no tag. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/roles.py | 19 +-------- dw/server/api_models.py | 4 +- dw/server/outputs.py | 45 ++++---------------- tests/test_gallery_roles.py | 68 ++++++++---------------------- tests/test_roles.py | 12 ------ ui/src/lib/generated/api-schema.ts | 2 +- ui/src/lib/generated/openapi.json | 2 +- 8 files changed, 32 insertions(+), 122 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ecd3fb50..859760e2 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -65,7 +65,7 @@ one to open. | Concept | Owner | Rule | Enforced by | | --- | --- | --- | --- | | Run directories | `dw/runs.py`: `open_run`, `workflow_identity`, `new_run_id` | Each execution writes into `///` with a `manifest.json`, unless the flat layout is chosen (the test pins the flat-layout exception). | `tests/test_runs.py::TestRunDirectories::test_the_flat_layout_writes_where_it_always_did` | -| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server infers it identically for older runs. | `tests/test_roles.py` | +| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server reports only recorded roles, so an older run has none. | `tests/test_roles.py` | | Notes on runs, files, assets, workspaces | dw/notes.py | notes.json beside what it describes (run dir or asset dir; a file's own note else the run's), workspace.json at a workspace root; only the server writes them, never the engine; ≤4000 chars, plain text. | tests/test_notes.py | | Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions`, `copy_runs_in` | A run takes its number once, when it opens (or, copied in from another workspace, when `copy_runs_in` places it), so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | diff --git a/dw/roles.py b/dw/roles.py index b21d7fb7..ec307493 100644 --- a/dw/roles.py +++ b/dw/roles.py @@ -17,9 +17,8 @@ A composed child's rolled-up manifest entries carry `parent_step` and take that step's role. The engine records roles when a run ends, from the -`role_basis` it took as the run opened; the server infers them with the -same function for a run that predates this, and falls back to -`fallback_roles` when the run's steps cannot be read. Inputs +`role_basis` it took as the run opened; the server reads only what was +recorded, so a run that predates roles has none - unknown, not guessed. Inputs are the asset: and output: references the expanded steps carry - a run's 'made from' list. This module names no model. """ @@ -145,20 +144,6 @@ def annotate_roles(steps, manifest_steps, basis=None): return annotated -def fallback_roles(manifest_steps): - """Copies of `manifest_steps` for a run whose steps cannot be read (no - realized workflow, or one that no longer expands): rule 2 against each - entry's recorded subfolder, and a run of one step is final. Any other - entry is left without a role - unknown, not guessed.""" - entries = [e for e in manifest_steps or [] if isinstance(e, dict)] - single = len({e.get("parent_step") or e.get("step") for e in entries}) == 1 - annotated = [] - for entry in entries: - role = _subfolder_role(entry.get("subfolder")) or (FINAL if single else None) - annotated.append({**entry, ROLE_KEY: role} if role else dict(entry)) - return annotated - - def entry_role(entry): """A manifest entry's recorded role, or None when it predates roles.""" role = entry.get(ROLE_KEY) if isinstance(entry, dict) else None diff --git a/dw/server/api_models.py b/dw/server/api_models.py index b79ffe31..dbc25379 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -397,8 +397,8 @@ class ManifestEntry(ApiModel): ) parent_step: str = sometimes("The composing step a rolled-up entry came from.") role: Literal["final", "intermediate"] = sometimes( - "The step's role in its run, from the run's manifest.json or inferred " - "for an older run. Absent while the run is going, or when unknown." + "The step's role in its run, as the run's manifest.json records it. " + "Absent while the run is going, or for a run from before roles." ) diff --git a/dw/server/outputs.py b/dw/server/outputs.py index 792e1869..fcc42aa6 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -6,7 +6,6 @@ app's `state` explicitly, so two apps in one process resolve independently. """ -import json import logging import os import shutil @@ -34,10 +33,9 @@ library_path, ) from ..notes import read_notes -from ..roles import annotate_roles, entry_role, fallback_roles +from ..roles import entry_role from ..runs import ( MANIFEST_FILE_NAME, - REALIZED_FILE_NAME, _read_manifest, is_output_reference, is_run_id, @@ -57,7 +55,6 @@ validate_asset_reference, validate_path, ) -from ..workflow import Workflow from ..workspace import forget_workspace_usage logger = logging.getLogger("dw") @@ -534,32 +531,11 @@ def run_directory(name, root): _INPUT_KINDS = ("asset", "output") -def _realized_steps(run_dir): - """The realized workflow's expanded steps, or None when unreadable.""" - path = os.path.join(run_dir, REALIZED_FILE_NAME) - try: - with open(path) as handle: - definition = json.load(handle) - return Workflow(definition, None, path).expanded_definition().get("steps") or [] - except Exception as e: # an unreadable or unexpandable record - logger.debug(f"No expanded steps for the roles of {run_dir}: {e}") - return None - - -def _inferred_roles(run_dir, steps): - """Manifest entries annotated from the realized workflow, or by the - subfolder and single-step fallback when it cannot be read.""" - expanded = _realized_steps(run_dir) - if expanded is None: - return fallback_roles(steps) - return annotate_roles(expanded, steps) - - def run_roles(run_dir): """Each file of a run, by path relative to the run directory ('/' - separated), mapped to its role. Recorded roles win; an entry without one - (a run that predates them) is inferred from the realized workflow, or - from its subfolder. {} when the manifest is unreadable.""" + separated), mapped to the role its manifest entry records. An entry + without one (a run that predates roles) is left out - unknown, not + guessed. {} when the manifest is unreadable.""" try: key = ( run_dir, @@ -572,13 +548,9 @@ def run_roles(run_dir): _ROLE_CACHE.move_to_end(key) return _ROLE_CACHE[key] manifest = _read_manifest(run_dir) or {} - steps = [e for e in manifest.get("steps") or [] if isinstance(e, dict)] - inferred = steps - if any(entry_role(e) is None for e in steps): - inferred = _inferred_roles(run_dir, steps) roles = {} - for entry, guess in zip(steps, inferred): - role = entry_role(entry) or entry_role(guess) + for entry in manifest.get("steps") or []: + role = entry_role(entry) if role: for name in entry.get("files") or []: roles[str(name).replace(os.sep, "/")] = role @@ -631,9 +603,8 @@ def with_run_note(summary, path): def with_run_state(detail, root, run_dir, roles=True): """A job's detail with what its run directory says now: the run note, and each manifest entry's role. The worker reports the manifest it held - in memory, which carries no roles, and a job recorded before roles has - none - so each entry takes the role `run_roles` gives its files, a - recorded one first. `roles` false leaves the manifest alone.""" + in memory, which carries no roles - so each entry takes the role + `run_roles` gives its files; a run from before roles stays without. `roles` false leaves the manifest alone.""" path = job_run_directory(root, run_dir) if path is None: return detail diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 571c9c83..0431fe2e 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -24,9 +24,8 @@ def test_recorded_roles_reach_the_listing(gallery_client, output_root, write_run assert files["b.png"]["role"] == "final" -def test_older_run_roles_inferred_from_realized_workflow( - gallery_client, output_root, write_run -): +def test_older_run_has_no_role(gallery_client, output_root, write_run): + # A manifest from before roles: b reads a, but nothing is guessed realized = { "id": "wf", "steps": [ @@ -44,13 +43,16 @@ def test_older_run_roles_inferred_from_realized_workflow( output_root, "wf", "20261012-000001-bbbbbbbb", - [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}], + [ + {"step": "a", "files": ["intermediate/a.png"], "subfolder": "intermediate"}, + {"step": "b", "files": ["b.png"]}, + ], realized=realized, - files=["a.png", "b.png"], + files=["intermediate/a.png", "b.png"], ) files = {f["label"]: f for f in gallery_client.get("/api/gallery").json()["files"]} - assert files["a.png"]["role"] == "intermediate" - assert files["b.png"]["role"] == "final" + assert files["a.png"]["role"] is None + assert files["b.png"]["role"] is None def test_role_filter(gallery_client, output_root, write_run): @@ -134,7 +136,7 @@ def test_metadata_for_a_subfolder_file_of_a_nested_identity_run( def _role_less_run_script(output_dir): """A worker that writes a two-step run - b reads a's result - whose - manifest.json predates roles, and reports the manifest it held in + manifest.json records roles, and reports the manifest it held in memory, which never carries them.""" from dw.runs import new_run_id @@ -156,7 +158,10 @@ def script(command): }, ], } - steps = [{"step": "a", "files": ["a.png"]}, {"step": "b", "files": ["b.png"]}] + steps = [ + {"step": "a", "files": ["a.png"], "role": "intermediate"}, + {"step": "b", "files": ["b.png"], "role": "final"}, + ] for name in ("a.png", "b.png"): with open(os.path.join(run_dir, name), "wb") as handle: handle.write(b"\x89PNG\r\n\x1a\n") @@ -223,18 +228,7 @@ def test_job_detail_entries_carry_the_run_roles(tmp_path): } -def test_recorded_roles_win_in_a_mixed_manifest(tmp_path, write_run): - # a says final although b reads it; inference would call it intermediate - realized = { - "id": "wf", - "steps": [ - {"name": "a", "task": {"command": "noop", "arguments": {}}}, - { - "name": "b", - "task": {"command": "noop", "arguments": {"x": "previous_result:a"}}, - }, - ], - } +def test_a_mixed_manifest_reports_only_recorded_roles(tmp_path, write_run): run_dir = write_run( str(tmp_path), "wf", @@ -242,38 +236,10 @@ def test_recorded_roles_win_in_a_mixed_manifest(tmp_path, write_run): [ {"step": "a", "files": ["a.png"], "role": "final"}, {"step": "b", "files": ["b.png"]}, + {"step": "c", "files": ["c.png"], "role": "bogus"}, ], - realized=realized, - ) - assert run_roles(run_dir) == {"a.png": "final", "b.png": "final"} - - -def test_a_run_without_its_workflow_falls_back_to_subfolders(tmp_path, write_run): - run_dir = write_run( - str(tmp_path), - "wf", - "20261012-000011-aaaaaaaa", - [ - {"step": "a", "files": ["intermediate/a.png"], "subfolder": "intermediate"}, - {"step": "b", "files": ["final/x/b.png"], "subfolder": "final/x"}, - {"step": "c", "files": ["c.png"], "subfolder": ""}, - ], - ) - assert run_roles(run_dir) == { - "intermediate/a.png": "intermediate", - "final/x/b.png": "final", - } - - -def test_a_single_step_run_without_its_workflow_is_final(tmp_path, write_run): - run_dir = write_run( - str(tmp_path), - "wf", - "20261012-000012-aaaaaaaa", - [{"step": "a", "files": ["a.png", "a2.png"]}], - realized={"not": "a workflow"}, ) - assert run_roles(run_dir) == {"a.png": "final", "a2.png": "final"} + assert run_roles(run_dir) == {"a.png": "final"} def test_run_inputs_keeps_only_well_formed_entries(tmp_path, write_run): diff --git a/tests/test_roles.py b/tests/test_roles.py index caf0650c..a80a0c2d 100644 --- a/tests/test_roles.py +++ b/tests/test_roles.py @@ -186,15 +186,3 @@ def test_role_basis_survives_realization_of_the_steps(): steps[1]["arguments"]["image"] = object() # what a run does in place manifest = [_entry("image", "a.png"), _entry("video", "b.mp4")] assert step_roles(None, manifest, basis) == {"image": INTERMEDIATE, "video": FINAL} - - -def test_fallback_roles(): - from dw.roles import fallback_roles - - entries = [ - {"step": "a", "files": ["a.png"], "subfolder": "intermediate/x"}, - {"step": "b", "files": ["b.png"], "subfolder": ""}, - ] - assert [e.get("role") for e in fallback_roles(entries)] == [INTERMEDIATE, None] - assert fallback_roles([{"step": "a", "files": ["a.png"]}])[0]["role"] == FINAL - assert fallback_roles([]) == [] diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index a01bc09b..6fdd4c18 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -2850,7 +2850,7 @@ export interface components { reused?: boolean; /** * Role - * @description The step's role in its run, from the run's manifest.json or inferred for an older run. Absent while the run is going, or when unknown. + * @description The step's role in its run, as the run's manifest.json records it. Absent while the run is going, or for a run from before roles. * @enum {string} */ role?: "final" | "intermediate"; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 61d5ab6a..fb233b42 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -2579,7 +2579,7 @@ "type": "boolean" }, "role": { - "description": "The step's role in its run, from the run's manifest.json or inferred for an older run. Absent while the run is going, or when unknown.", + "description": "The step's role in its run, as the run's manifest.json records it. Absent while the run is going, or for a run from before roles.", "enum": [ "final", "intermediate" From 64d5f8e1753bb4d2a62a994bbdb17bce86f0f26d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 17:55:06 -0500 Subject: [PATCH 035/204] feat(workspace): #859 - tester bounce: set_note registered, empty run segments and bad destination names refused - set_note was documented in docs/MCP.md and named in the server instructions but never registered as a tool; it is now (WRITES), with EXPECTED_TOOLS, TOOL_WIRING, the tool count 63 -> 64 and SURFACE_BUDGET raised by its measured 150 tokens. - A runs entry with an empty segment ("wf/", "wf//") is refused rather than normalized. - The copy route takes {destination:path}, and api_path encodes a segment that is only dots, so "../x", "/tmp/x", ".", ".." and "" reach the route and are refused as "No such workspace: ", as the same names are for source. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/MCP.md | 2 +- dw/server/guides.py | 2 +- dw/server/routes/library.py | 2 +- dw/workspace_copy.py | 1 - dw_mcp/client.py | 5 ++++- dw_mcp/server.py | 1 + tests/test_mcp_client.py | 12 ++++++++++++ tests/test_mcp_server.py | 9 ++++++++- tests/test_workspace_copy.py | 30 +++++++++++++++++++++++++++++- 10 files changed, 58 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e7526730..6882e05f 100644 --- a/README.md +++ b/README.md @@ -83,7 +83,7 @@ once; without it the run fails partway through with a 401/403 from the Hub. ## Drive it from an agent -Then just ask. The agent has 63 tools covering the whole surface — the +Then just ask. The agent has 64 tools covering the whole surface — the workflow catalog, the real diffusers pipeline signatures, the job queue, the gallery, the model cache: diff --git a/docs/MCP.md b/docs/MCP.md index df53d504..c8983b03 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -187,7 +187,7 @@ Nothing in this sequence costs GPU time. ## Tool reference -63 tools in six groups. Names and arguments below are transcribed from +64 tools in six groups. Names and arguments below are transcribed from `dw_mcp/tools_*.py` — nothing here is renamed or reshaped for the docs. ### Catalog (read-only) diff --git a/dw/server/guides.py b/dw/server/guides.py index 2ce08aa1..75547f72 100644 --- a/dw/server/guides.py +++ b/dw/server/guides.py @@ -240,7 +240,7 @@ def get_guide(name, section=None): Without a `section` the answer is the *index*: the guide's preamble, its first section, and the headings of the rest - not the whole file. A full WORKFLOW_GUIDE.md is ~19.6k tokens and TASKS.md ~16k, which is - more in one call than the entire 63-tool MCP surface costs to connect, + more in one call than the entire 64-tool MCP surface costs to connect, and an agent can make that call twice before noticing (#101). Every withheld section is named in `sections` and fetched by name, so nothing is unreachable - only unspent by accident. diff --git a/dw/server/routes/library.py b/dw/server/routes/library.py index 16c7e327..ee015d7a 100644 --- a/dw/server/routes/library.py +++ b/dw/server/routes/library.py @@ -183,7 +183,7 @@ def _copy_side(state, root, name): @router.post( - "/api/workspaces/{destination}/copy", + "/api/workspaces/{destination:path}/copy", response_model=WorkspaceCopied, response_model_exclude_unset=True, ) diff --git a/dw/workspace_copy.py b/dw/workspace_copy.py index 83cca460..061b5744 100644 --- a/dw/workspace_copy.py +++ b/dw/workspace_copy.py @@ -93,7 +93,6 @@ def _run_entry(entry): name = ref_name(OUTPUT, name) or name if name.startswith(("/", "\\")) or os.path.isabs(name): raise PathTraversalError(f"'{entry}' is an absolute path - name a run") - name = name.strip("/") parts = name.split("/") if any(part in ("..", ".") for part in parts): raise PathTraversalError(f"'{entry}' leaves the outputs folder") diff --git a/dw_mcp/client.py b/dw_mcp/client.py index 307109d8..3463533f 100644 --- a/dw_mcp/client.py +++ b/dw_mcp/client.py @@ -113,8 +113,11 @@ def path_segment(name): and the server's own path-traversal check never runs on it. Quoting keeps the literal bytes intact on the wire, so it is the server's own validation - not this client - that decides what a name is allowed to - contain. + contain. A segment that is only dots ('.', '..') is not touched by + `quote`, so its dots are encoded too: unencoded, httpx would drop it. """ + if name and name.strip(".") == "": + return "%2E" * len(name) return quote(name, safe="") diff --git a/dw_mcp/server.py b/dw_mcp/server.py index 0042d681..a40ee3e4 100644 --- a/dw_mcp/server.py +++ b/dw_mcp/server.py @@ -166,6 +166,7 @@ def tool(fn, annotations): tool(asset.list_assets, READ_ONLY) tool(asset.upload_asset, WRITES) tool(asset.keep_output, WRITES) + tool(asset.set_note, WRITES) tool(asset.delete_asset, DELETES) tool(wks.list_workspaces, READ_ONLY) diff --git a/tests/test_mcp_client.py b/tests/test_mcp_client.py index 10a1bcd8..a8264e32 100644 --- a/tests/test_mcp_client.py +++ b/tests/test_mcp_client.py @@ -263,6 +263,18 @@ def test_api_path_quotes_dot_segments_so_they_cannot_traverse(): assert api_path("api", "workflows", "../escape") == "/api/workflows/..%2Fescape" +@pytest.mark.parametrize("name,encoded", [(".", "%2E"), ("..", "%2E%2E")]) +def test_api_path_encodes_a_segment_that_is_only_dots(name, encoded): + # quote() leaves '.' alone, and httpx drops a bare '.' or '..' segment + assert api_path("api", "workspaces", name, "copy") == ( + f"/api/workspaces/{encoded}/copy" + ) + + +def test_api_path_keeps_dots_inside_a_name(): + assert api_path("api", "workflows", "v1.2") == "/api/workflows/v1.2" + + # Every dw_mcp handler module that talks to the API must build request paths # through `api_path`, not by hand - `path_segment` alone is easy to forget on # one call site among many, and a bare f-string skips quoting entirely. This diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 9fcd5fc3..a68238ac 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -73,6 +73,7 @@ "list_assets", "upload_asset", "keep_output", + "set_note", "delete_asset", "list_workspaces", "use_workspace", @@ -102,6 +103,7 @@ "delete_output", "upload_asset", "keep_output", + "set_note", "delete_asset", "use_workspace", "copy_to_workspace", @@ -702,6 +704,7 @@ def _a_file_to_upload(): "POST", "/api/assets/keep", ), + ("set_note", {"target": "wf/run", "note": "keeper"}, "PUT", "/api/notes"), ("delete_asset", {"name": "hero.png"}, "DELETE", "/api/assets/hero.png"), ("list_workspaces", {}, "GET", "/api/workspaces"), ("use_workspace", {"name": "default"}, "GET", "/api/workspaces"), @@ -1580,7 +1583,11 @@ def test_the_stated_tool_count_is_the_registered_one(): # 2026-10-13 (#859, #853 stage A): copy_to_workspace (121.25 description + # 84.75 schema) - measured at 14_470.25 (9_276.25 / 4_683.75 / 510.25); the # ceiling moves to 14_471 with 0.75 of headroom. -SURFACE_BUDGET = 14_471 +# 2026-10-13 (#859 bounce): set_note, documented since the notes feature but +# never registered, is registered (66.75 description + 83.25 schema) - +# measured at 14_620.25 (9_343 / 4_767 / 510.25); the ceiling moves to +# 14_621 with 0.75 of headroom. +SURFACE_BUDGET = 14_621 @pytest.mark.asyncio diff --git a/tests/test_workspace_copy.py b/tests/test_workspace_copy.py index f066f681..505558c1 100644 --- a/tests/test_workspace_copy.py +++ b/tests/test_workspace_copy.py @@ -19,7 +19,7 @@ create_workspace, ) from dw.workspace_copy import CopyRefused, copy_runs -from dw_mcp.client import DwClient +from dw_mcp.client import DwClient, api_path from dw_mcp.workspaces import copy_to_workspace from .test_server import ScriptedWorkerManager, success_script @@ -233,6 +233,13 @@ def test_traversal(self, src, dst, entry): copy_runs(src, dst, [entry]) assert nothing_written(dst) + @pytest.mark.parametrize("entry", ["wf/", f"wf/{R1}/", f"wf//{R1}", "output:wf/"]) + def test_an_empty_segment(self, src, dst, entry): + make_run(src, "wf", R1) + with pytest.raises(InvalidInputError): + copy_runs(src, dst, [entry]) + assert nothing_written(dst) + def test_a_symlinked_run_folder(self, src, dst, tmp_path): make_run(src, "elsewhere", R1) os.makedirs(os.path.join(src.outputs, "wf")) @@ -464,6 +471,27 @@ def test_traversal_is_400(self, server, src, dst): ) assert response.status_code == 400 + @pytest.mark.parametrize( + "destination", + ["../dst", "/tmp/dst", "dst/../x", ".", "..", "", "dst/"], + ) + def test_a_bad_destination_name_is_refused_by_name( + self, server, root, src, dst, destination + ): + # Built as the MCP client builds it: each name reaches the route + # rather than being normalized into another URL on the way + make_run(src, "wf", R1) + before = sorted(os.listdir(root.root)) + with server() as client: + response = client.post( + api_path("api", "workspaces", destination, "copy"), + json={"source": "src", "runs": [f"wf/{R1}"]}, + ) + assert response.status_code == 404, response.text + assert response.json()["detail"] == f"No such workspace: {destination}" + assert sorted(os.listdir(root.root)) == before + assert nothing_written(dst) + def test_an_active_job_conflicts(self, server, src, dst): make_run(src, "wf", R1) with server() as client: From e12baf080bb55977bcbc8c00175054e9562beca8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 18:14:15 -0500 Subject: [PATCH 036/204] feat(workspace): #860 - copy assets between workspaces copy_to_workspace takes assets: files or folders, keeping their library path, hard-linked, with their notes and kept shots merged into fresh destination sidecars. An identical file is already present; a different one refuses the whole request. Uploads replace a file rather than writing into it, so a linked copy's bytes stay put. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/api_models.py | 8 +- dw/server/routes/assets.py | 18 +- dw/server/routes/library.py | 29 +-- dw/workspace_copy.py | 232 +++++++++++++++++++--- dw_mcp/tools_authoring.py | 24 ++- dw_mcp/workspaces.py | 13 +- tests/test_workspace_copy.py | 359 +++++++++++++++++++++++++++++++---- 8 files changed, 586 insertions(+), 99 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 859760e2..d59530b6 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -57,7 +57,7 @@ one to open. | Packaged builtins | `dw/library.py`: `builtin_root`, `resolve_sub_workflow_reference` | `builtin:` names the packaged `dw/workflows/`, not the top-level `workflows/` examples. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_builtin_resolves_in_the_packaged_root` | | Sub-workflow paths | `dw/library.py`: `resolve_sub_workflow_reference`, `resolve_sub_workflow` | A sub-workflow path is confined to the root it resolves in. The search order is in the docstring of `resolve_sub_workflow`. | `tests/test_sub_workflow_resolver.py::TestTheResolver::test_a_path_escaping_its_root_is_refused` | | Workspace | `dw/workspace.py`: `resolve_workspace`, `set_workspace` | `--workspace` beats `DW_WORKSPACE`, which beats the setting, which beats a working directory that looks like a workspace, which beats `~/diffusers-workspace`. | `tests/test_workspace.py::TestResolution::test_a_flag_wins_over_everything`, `tests/test_workspace.py::TestResolution::test_a_bare_working_directory_falls_back_to_the_home_workspace` | -| Workspace copy | `dw/workspace_copy.py`: `copy_runs` | Copies runs between workspaces under one root: media hard-linked, sidecars always copied, never a symlink; all or nothing; `vN` renumbered by `runs.copy_runs_in` under the run lock; entries checked by `security.validate_output_prefix`. | `tests/test_workspace_copy.py` | +| Workspace copy | `dw/workspace_copy.py`: `copy_between` | Copies runs and asset-library entries between workspaces under one root: media hard-linked, sidecars always copied, never a symlink; an asset keeps its library-relative path and its note and kept-shots entry are merged into the destination folder's sidecars; all or nothing, a differing asset refusing runs too; `vN` renumbered by `runs.copy_runs_in` under the run lock; entries checked by `security.validate_output_prefix`. | `tests/test_workspace_copy.py` | | Realized workflow | `dw/realize.py`: `realize_workflow`, `dw/runs.py`: `write_realized_workflow` | Each run directory holds `workflow.json`, with the run's arguments, seed and stored prompt text pinned. | `tests/test_realize.py::TestVariablesAndSeed::test_arguments_become_the_variable_defaults`, `tests/test_runs.py::TestRealizedWorkflow::test_the_run_directory_holds_a_realized_copy` | ## Runs and outputs diff --git a/dw/server/api_models.py b/dw/server/api_models.py index dbc25379..120e0cb6 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -666,7 +666,9 @@ class WorkspaceDeleted(ApiModel): class CopiedItems(ApiModel): runs: list[str] = Field(description="`/` of each run copied.") - assets: list[str] = Field(description="Always empty until assets can be copied.") + assets: list[str] = Field( + description="Library-relative path of each asset file copied." + ) class WorkspaceCopied(ApiModel): @@ -676,7 +678,9 @@ class WorkspaceCopied(ApiModel): destination: str copied: CopiedItems already_present: list[str] = Field( - description="Runs the destination already held, by run id - skipped." + description="Runs the destination already held (`/`) " + "and assets it held with identical bytes (library-relative path) - " + "skipped." ) renumbered: dict[str, dict[str, int]] = Field( description="Copied runs whose `vN` changed, keyed `/`: " diff --git a/dw/server/routes/assets.py b/dw/server/routes/assets.py index 7fce782d..d09778b0 100644 --- a/dw/server/routes/assets.py +++ b/dw/server/routes/assets.py @@ -72,8 +72,22 @@ def _write_bytes(path, data): - with open(path, "wb") as f: - f.write(data) + """Write a file whole, then rename it into place. An upload over an + existing name replaces that file rather than writing into it: the old + one may be a hard link - of a run's file (keep_output) or of another + workspace's asset (copy_to_workspace) - and an in-place write would + change those bytes too.""" + partial = f"{path}.{uuid.uuid4().hex}.part" + try: + with open(partial, "wb") as f: + f.write(data) + os.replace(partial, path) + except BaseException: + try: + os.remove(partial) + except OSError: + pass + raise def _note_or_400(note): diff --git a/dw/server/routes/library.py b/dw/server/routes/library.py index ee015d7a..28bb1248 100644 --- a/dw/server/routes/library.py +++ b/dw/server/routes/library.py @@ -37,7 +37,7 @@ from ... import references, validation from ...security import InvalidInputError, SecurityError, validate_prompt_reference from ...workflow import Workflow -from ...workspace_copy import CopyRefused, copy_runs +from ...workspace_copy import CopyRefused, copy_between from ...notes import read_workspace_note from ...library import ( builtin_root, @@ -160,7 +160,9 @@ class WorkspaceCopyRequest(BaseModel): "all its runs", ) assets: list[str] = Field( - default_factory=list, description="Not supported yet - leave empty" + default_factory=list, + description="Asset-library files or folders, as written after `asset:` " + "(`proj/clip.mp4`, `proj/`); each keeps its library-relative path", ) @@ -190,24 +192,23 @@ def _copy_side(state, root, name): def copy_into_workspace( http_request: Request, destination: str, request: WorkspaceCopyRequest ): - """Copy runs from another workspace into this one. + """Copy runs and asset-library entries from another workspace into this one. Media is hard-linked where the filesystem allows, so a copy costs - little disk; sidecars are copied. All or nothing: a run already here is - skipped as `already_present`, a run whose job is still running refuses - the request. A move is a copy, then `DELETE` of the source runs. + little disk; sidecars are copied, and an asset's note and kept shots are + merged into the destination folder's. All or nothing: a run already + here, or an asset with identical bytes, is skipped as `already_present`; + a run whose job is still running, or a different file at an asset's + path, refuses the whole request (409, naming each) with nothing written. + A move is a copy, then `DELETE` of the source runs and assets. """ state = http_request.app.state manager = state.job_manager root = workspace_root(state) - if request.assets: + if not request.runs and not request.assets: raise HTTPException( - status_code=400, - detail="Copying assets is not supported yet - copy runs, and " - "re-upload the assets they need", + status_code=400, detail="Name at least one run or asset to copy" ) - if not request.runs: - raise HTTPException(status_code=400, detail="Name at least one run to copy") source_name = request.source or DEFAULT_WORKSPACE_NAME if source_name == destination: raise HTTPException( @@ -226,7 +227,9 @@ def copy_into_workspace( and getattr(job, "run_dir", None) ] try: - return copy_runs(source, target, request.runs, active_runs=active) + return copy_between( + source, target, request.runs, request.assets, active_runs=active + ) except CopyRefused as e: detail = ( {"message": str(e), "conflicts": e.conflicts} if e.conflicts else str(e) diff --git a/dw/workspace_copy.py b/dw/workspace_copy.py index 061b5744..a45351e5 100644 --- a/dw/workspace_copy.py +++ b/dw/workspace_copy.py @@ -1,4 +1,5 @@ -"""Copying runs from one workspace to another under the same root (#853). +"""Copying runs and asset-library entries from one workspace to another +under the same root (#853). A run is self-contained on disk - `latest`, `vN`, `output:` resolution and the gallery all read the run folder and its manifests, never the jobs @@ -33,24 +34,39 @@ path), and that a copied run has no job row (it appears in the gallery, not in the jobs list). -Copying assets is stage B of #853. +Assets are named as after `asset:` - a file, or a folder for everything +under it - and keep their library-relative path, so `uploads/x` stays +`uploads/x`. A path the destination already holds with identical bytes is +`already_present`; with different bytes it is a conflict, and a conflict +refuses the whole request - runs included - before anything is written. +An asset's sidecars are not copied as files: its `notes.json` entry and its +kept-shots entry (`record_kept_shots`) are merged into the destination +folder's own, written fresh, so the shot map and the note travel while +`provenance.workspace` keeps naming the workspace the file was made in. +Media is linked here too: no asset path writes a file in place - an upload +over a name replaces it (`os.replace`), keep and delete unlink first. """ +import filecmp import logging import os import shutil -from .references import OUTPUT, ref_name +from .notes import NOTES_FILE_NAME, read_notes, set_file_note +from .references import ASSET, OUTPUT, ref_name from .runs import ( + MANIFEST_FILE_NAME, _read_manifest, copy_runs_in, identity_run_ids, is_run_id, + record_kept_shots, resolve_output_reference, ) from .security import ( InvalidInputError, PathTraversalError, + validate_asset_reference, validate_output_prefix, validate_path, ) @@ -61,6 +77,9 @@ # The sidecars a copy writes fresh rather than links: every JSON file SIDECAR_SUFFIX = ".json" +# An asset folder's own sidecars: merged per file, never copied as files +ASSET_SIDECARS = (NOTES_FILE_NAME, MANIFEST_FILE_NAME) + NO_JOBS_NOTE = ( "Copied runs appear in list_gallery and resolve through output: " "references, but not in list_jobs - a copy carries no job record." @@ -271,7 +290,140 @@ def _unresolved_inputs(manifest, outputs): return unresolved -def _refuse_unless_copyable(source, destination, runs): +def _asset_entry(entry): + """An `assets` entry as a library-relative name, and whether it names a + folder. An `asset:` prefix is accepted; a trailing `/` marks a folder, + but a folder is a folder without one too. + + Raises: + PathTraversalError: For an absolute path or a `..` + InvalidInputError: For anything else that is not an asset name + """ + if not isinstance(entry, str) or not entry.strip(): + raise InvalidInputError("An assets entry must be a non-empty string") + name = entry.strip() + name = ref_name(ASSET, name) or name + if name.startswith(("/", "\\")) or os.path.isabs(name): + raise PathTraversalError(f"'{entry}' is an absolute path - name an asset") + if any(part in ("..", ".") for part in name.split("/")): + raise PathTraversalError(f"'{entry}' leaves the asset library") + folder = name.endswith("/") + return validate_asset_reference(name[:-1] if folder else name), folder + + +def _source_assets(source, entry): + """[library-relative name] of each file one entry names in the source's + asset library - the file itself, or every file under a folder but the + folder sidecars, which travel per file instead. + + Raises: + PathTraversalError: When the entry escapes, or reaches a symlink + InvalidInputError: When it names a sidecar rather than an asset + CopyRefused (404): When there is no such asset + """ + name, _folder = _asset_entry(entry) + library = os.path.realpath(source.assets) + joined = os.path.join(library, *name.split("/")) + if os.path.isdir(library) and _symlink_on_the_way(library, joined): + raise PathTraversalError( + f"'{entry}' passes through a symlink - a copy never follows one" + ) + path = validate_path(joined, library) + if os.path.isfile(path): + if os.path.basename(path) in ASSET_SIDECARS: + raise InvalidInputError( + f"'{entry}' is a sidecar, not an asset - copy the file it " + f"describes and its entry comes with it" + ) + return [name] + if not os.path.isdir(path): + raise CopyRefused(f"No asset '{name}' in workspace '{source.name}'", status=404) + if _symlink_inside(path): + raise PathTraversalError( + f"Asset folder '{name}' holds a symlink - a copy never follows or " + f"creates one" + ) + found = [] + for current, dirs, files in os.walk(path): + dirs.sort() + for file_name in sorted(files): + if file_name in ASSET_SIDECARS: + continue + relative = os.path.relpath(os.path.join(current, file_name), library) + found.append(relative.replace(os.sep, "/")) + if not found: + raise CopyRefused( + f"Asset folder '{name}' in workspace '{source.name}' holds no files", + status=404, + ) + return found + + +def _same_bytes(a, b): + """Whether two files hold the same bytes - a hard link of one another + trivially does.""" + if os.path.samefile(a, b): + return True + return filecmp.cmp(a, b, shallow=False) + + +def _plan_assets(source, destination, assets): + """([name] to copy, [already present], [conflicting]) for the assets + named - every path checked against the destination, nothing written.""" + named = [] + for entry in assets: + for name in _source_assets(source, entry): + if name not in named: + named.append(name) + source_library = os.path.realpath(source.assets) + library = os.path.realpath(destination.assets) + to_copy, already_present, conflicts = [], [], [] + for name in named: + joined = os.path.join(library, *name.split("/")) + if os.path.isdir(library) and _symlink_on_the_way(library, joined): + raise PathTraversalError( + f"'{name}' passes through a symlink in workspace " + f"'{destination.name}' - a copy never writes through one" + ) + target = validate_path(joined, library) + if not os.path.lexists(target): + to_copy.append(name) + elif os.path.isfile(target) and _same_bytes( + os.path.join(source_library, *name.split("/")), target + ): + already_present.append(name) + else: + conflicts.append(name) + return to_copy, already_present, conflicts + + +def _kept_entry(directory, file_name): + """The kept-shots entry a folder's manifest holds for one file, or None.""" + for entry in (_read_manifest(directory) or {}).get("steps") or []: + if isinstance(entry, dict) and entry.get("files") == [file_name]: + return entry + return None + + +def _copy_asset(source, destination, name, totals): + """Link (or copy) one asset into place, then merge its note and its + kept-shots entry into the destination folder's sidecars - written + fresh, so an edit on either side stays on that side.""" + parts = name.split("/") + source_path = os.path.join(os.path.realpath(source.assets), *parts) + library = os.path.realpath(destination.assets) + target = validate_path(os.path.join(library, *parts), library) + os.makedirs(os.path.dirname(target), exist_ok=True) + _copy_file(source_path, target, totals) + + source_dir, target_dir = os.path.dirname(source_path), os.path.dirname(target) + file_name = parts[-1] + set_file_note(target_dir, file_name, read_notes(source_dir)["files"].get(file_name)) + kept = _kept_entry(source_dir, file_name) or {} + record_kept_shots(target_dir, file_name, kept.get("shots"), kept.get("provenance")) + + +def _refuse_unless_copyable(source, destination, runs, assets): """The checks on the request as a whole, before any entry is read.""" for side in (source, destination): if isinstance(side, ConfiguredWorkspace): @@ -285,17 +437,19 @@ def _refuse_unless_copyable(source, destination, runs): f"Source and destination are both '{source.name}' - name another " f"workspace to copy into" ) - if not runs: - raise InvalidInputError("Name at least one run to copy") - os.makedirs(destination.outputs, exist_ok=True) - if not os.access(destination.outputs, os.W_OK): - raise CopyRefused(f"Workspace '{destination.name}' is not writable") + if not runs and not assets: + raise InvalidInputError("Name at least one run or asset to copy") + for folder, named in ((destination.outputs, runs), (destination.assets, assets)): + if not named: + continue + os.makedirs(folder, exist_ok=True) + if not os.access(folder, os.W_OK): + raise CopyRefused(f"Workspace '{destination.name}' is not writable") -def _plan_copy(source, outputs, runs, active_runs): - """Every entry checked before anything is written: ({identity: [run - id]} to copy, [already present]), or CopyRefused naming the runs whose - job is still active.""" +def _plan_runs(source, outputs, runs, active_runs): + """({identity: [run id]} to copy, [already present], [active]) - every + entry checked before anything is written.""" named = [] for entry in runs: for found in _source_runs(source, entry): @@ -317,13 +471,29 @@ def _plan_copy(source, outputs, runs, active_runs): already_present.append(name) else: to_copy.setdefault(identity, []).append(run_id) - if conflicts: - raise CopyRefused( - "These runs' jobs are still queued or running, so their manifests " - "are not final - wait for them, then copy: " + ", ".join(conflicts), - conflicts=conflicts, + return to_copy, already_present, conflicts + + +def _refuse_conflicts(active, differing): + """One refusal naming every conflict the plan found, or nothing.""" + if not active and not differing: + return + reasons = [] + if active: + reasons.append( + "these runs' jobs are still queued or running, so their manifests " + "are not final - wait for them, then copy: " + ", ".join(active) + ) + if differing: + reasons.append( + "the destination already holds a different file at: " + + ", ".join(differing) + + " - a copy never replaces one; delete or rename it first" ) - return to_copy, already_present + raise CopyRefused( + "Nothing was copied: " + "; and ".join(reasons), + conflicts=active + differing, + ) def _copy_warnings(copied, outputs, source_root): @@ -348,13 +518,14 @@ def _copy_warnings(copied, outputs, source_root): return warnings -def copy_runs(source, destination, runs, active_runs=()): - """Copy runs from one workspace into another. +def copy_between(source, destination, runs=(), assets=(), active_runs=()): + """Copy runs and assets from one workspace into another. Args: source, destination: Workspaces under one root, already resolved and known to exist runs: `/` or bare `` entries + assets: Library-relative files or folders, as after `asset:` active_runs: `/` of the source's runs whose job is still queued or running - their manifest is not final @@ -366,9 +537,14 @@ def copy_runs(source, destination, runs, active_runs=()): InvalidInputError, PathTraversalError: A malformed or escaping entry CopyRefused: Anything else that stops the copy; nothing was written """ - _refuse_unless_copyable(source, destination, runs) + runs, assets = list(runs or []), list(assets or []) + _refuse_unless_copyable(source, destination, runs, assets) outputs = os.path.realpath(destination.outputs) - to_copy, already_present = _plan_copy(source, outputs, runs, active_runs) + to_copy, runs_present, active = _plan_runs(source, outputs, runs, active_runs) + assets_to_copy, assets_present, differing = ( + _plan_assets(source, destination, assets) if assets else ([], [], []) + ) + _refuse_conflicts(active, differing) totals = {"bytes_linked": 0, "bytes_copied": 0} renumbered = {} @@ -378,17 +554,19 @@ def copy_runs(source, destination, runs, active_runs=()): _copy_identity(source, destination, identity, run_ids, totals) ) copied.extend(f"{identity}/{run_id}" for run_id in run_ids) + for name in assets_to_copy: + _copy_asset(source, destination, name, totals) forget_workspace_usage() logger.info( - f"Copied {len(copied)} run(s) from workspace {source.name} to " - f"{destination.name}" + f"Copied {len(copied)} run(s) and {len(assets_to_copy)} asset(s) from " + f"workspace {source.name} to {destination.name}" ) return { "source": source.name, "destination": destination.name, - "copied": {"runs": copied, "assets": []}, - "already_present": already_present, + "copied": {"runs": copied, "assets": assets_to_copy}, + "already_present": runs_present + assets_present, "renumbered": renumbered, "warnings": _copy_warnings(copied, outputs, os.path.realpath(source.root)), **totals, diff --git a/dw_mcp/tools_authoring.py b/dw_mcp/tools_authoring.py index 139fdb5b..8150ed33 100644 --- a/dw_mcp/tools_authoring.py +++ b/dw_mcp/tools_authoring.py @@ -39,18 +39,24 @@ def create_workspace(self, name: str, use: bool = False) -> dict: return workspaces.create_workspace(self.client, name, use=use) def copy_to_workspace( - self, destination: str, runs: list[str], source: str | None = None + self, + destination: str, + runs: list[str] | None = None, + assets: list[str] | None = None, + source: str | None = None, ) -> dict: - """Copy runs from one workspace into another: each entry is + """Copy runs and assets from one workspace into another. runs: '/' as list_gallery names it, or a bare - '' for all its runs. source defaults to this session's - workspace. Media is hard-linked, so a copy costs little disk. Runs - the destination already has are skipped; a vN it already uses is - renumbered and reported. Copies get no job record: they show in - list_gallery and resolve as output: refs, not in list_jobs. To move, - copy then delete_output the source runs.""" + '' for all its runs. assets: paths as after asset:, a + folder ('proj/') for all under it; each keeps its path, note and + shots. source defaults to this session's workspace. Media is + hard-linked, so a copy costs little disk. What the destination + already has is skipped; a different file at an asset's path refuses + the whole request; a vN it already uses is renumbered and reported. + Copied runs get no job record: they show in list_gallery and resolve + as output: refs, not in list_jobs. To move, copy then delete.""" return workspaces.copy_to_workspace( - self.client, destination, runs, source=source + self.client, destination, runs, assets, source=source ) def delete_workspace(self, name: str, acknowledged_cost: bool = False) -> dict: diff --git a/dw_mcp/workspaces.py b/dw_mcp/workspaces.py index 754cf7a8..b327a2be 100644 --- a/dw_mcp/workspaces.py +++ b/dw_mcp/workspaces.py @@ -151,15 +151,16 @@ def create_workspace(client, name, use=False): return result -def copy_to_workspace(client, destination, runs, source=None): - """Copy runs into another workspace under the same root. `source` - defaults to this session's workspace. The server hard-links media and - copies sidecars, skips runs the destination already holds - (`already_present`), and reports renumbered `vN` and warnings.""" +def copy_to_workspace(client, destination, runs=None, assets=None, source=None): + """Copy runs and assets into another workspace under the same root. + `source` defaults to this session's workspace. The server hard-links + media and copies sidecars, skips what the destination already holds + (`already_present`), refuses a differing asset at the same path, and + reports renumbered `vN` and warnings.""" source = source or client.workspace return client.post_json( api_path("api", "workspaces", destination, "copy"), - {"source": source, "runs": list(runs or [])}, + {"source": source, "runs": list(runs or []), "assets": list(assets or [])}, ) diff --git a/tests/test_workspace_copy.py b/tests/test_workspace_copy.py index 505558c1..e64e7fd8 100644 --- a/tests/test_workspace_copy.py +++ b/tests/test_workspace_copy.py @@ -1,4 +1,4 @@ -"""Copying runs between workspaces (#853): the engine, the route and the MCP tool.""" +"""Copying runs and assets between workspaces (#853): the engine, the route and the MCP tool.""" import json import os @@ -9,7 +9,13 @@ import pytest from fastapi.testclient import TestClient -from dw.runs import RUN_ID_PATTERN, resolve_output_reference, run_versions +from dw.notes import read_notes, set_file_note +from dw.runs import ( + RUN_ID_PATTERN, + record_kept_shots, + resolve_output_reference, + run_versions, +) from dw.security import InvalidInputError, PathTraversalError, SecurityError from dw.server.app import create_app from dw.server.jobs import JobManager @@ -18,7 +24,7 @@ Workspace, create_workspace, ) -from dw.workspace_copy import CopyRefused, copy_runs +from dw.workspace_copy import CopyRefused, copy_between from dw_mcp.client import DwClient, api_path from dw_mcp.workspaces import copy_to_workspace @@ -93,7 +99,7 @@ def nothing_written(ws): class TestHappyPath: def test_one_run_is_copied(self, src, dst): make_run(src, "wf", R1) - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) assert set(reply) == { "source", "destination", @@ -115,29 +121,29 @@ def test_one_run_is_copied(self, src, dst): def test_a_bare_workflow_copies_all_its_runs(self, src, dst): make_run(src, "wf", R1, 1) make_run(src, "wf", R2, 2) - reply = copy_runs(src, dst, ["wf"]) + reply = copy_between(src, dst, ["wf"]) assert reply["copied"]["runs"] == [f"wf/{R1}", f"wf/{R2}"] def test_a_multi_segment_identity(self, src, dst): make_run(src, "ltx2/Gyre", R1) - reply = copy_runs(src, dst, [f"ltx2/Gyre/{R1}"]) + reply = copy_between(src, dst, [f"ltx2/Gyre/{R1}"]) assert reply["copied"]["runs"] == [f"ltx2/Gyre/{R1}"] assert os.path.isfile(os.path.join(dst.outputs, "ltx2", "Gyre", R1, "img.png")) def test_a_bare_multi_segment_identity(self, src, dst): make_run(src, "ltx2/Gyre", R1) - reply = copy_runs(src, dst, ["ltx2/Gyre"]) + reply = copy_between(src, dst, ["ltx2/Gyre"]) assert reply["copied"]["runs"] == [f"ltx2/Gyre/{R1}"] def test_an_output_prefix_is_accepted(self, src, dst): make_run(src, "wf", R1) - reply = copy_runs(src, dst, [f"output:wf/{R1}"]) + reply = copy_between(src, dst, [f"output:wf/{R1}"]) assert reply["copied"]["runs"] == [f"wf/{R1}"] def test_repeating_a_copy_is_idempotent(self, src, dst): make_run(src, "wf", R1) - copy_runs(src, dst, [f"wf/{R1}"]) - again = copy_runs(src, dst, [f"wf/{R1}"]) + copy_between(src, dst, [f"wf/{R1}"]) + again = copy_between(src, dst, [f"wf/{R1}"]) assert again["already_present"] == [f"wf/{R1}"] assert again["copied"]["runs"] == [] assert "no_jobs" not in again["warnings"] @@ -150,7 +156,7 @@ def test_a_collision_renumbers_in_run_id_order(self, src, dst): make_run(src, "wf", R1, 1) make_run(src, "wf", R3, 3) # v1 collides, so every copied run of the identity is renumbered - reply = copy_runs(src, dst, ["wf"]) + reply = copy_between(src, dst, ["wf"]) assert reply["renumbered"] == { f"wf/{R1}": {"from": 1, "to": 2}, f"wf/{R2}": {"from": 2, "to": 3}, @@ -165,7 +171,7 @@ def test_two_runs_renumber_after_one(self, src, dst): make_run(dst, "wf", "20251231-000000-dddddddd", 1) make_run(src, "wf", R1, 1) make_run(src, "wf", R2, 2) - reply = copy_runs(src, dst, ["wf"]) + reply = copy_between(src, dst, ["wf"]) assert reply["renumbered"] == { f"wf/{R1}": {"from": 1, "to": 2}, f"wf/{R2}": {"from": 2, "to": 3}, @@ -174,7 +180,7 @@ def test_two_runs_renumber_after_one(self, src, dst): def test_no_collision_keeps_numbers(self, src, dst): make_run(dst, "wf", "20251231-000000-dddddddd", 1) make_run(src, "wf", R2, 2) - reply = copy_runs(src, dst, [f"wf/{R2}"]) + reply = copy_between(src, dst, [f"wf/{R2}"]) assert reply["renumbered"] == {} copied = read_json(os.path.join(dst.outputs, "wf", R2, "manifest.json")) assert copied["version"] == 2 @@ -189,14 +195,14 @@ def test_unresolved_output_refs(self, src, dst): R2, inputs=["output:other/latest/img.png", "output:gone/latest/img.png"], ) - reply = copy_runs(src, dst, [f"wf/{R2}"]) + reply = copy_between(src, dst, [f"wf/{R2}"]) unresolved = reply["warnings"]["unresolved_output_refs"] assert unresolved == {f"wf/{R2}": ["output:gone/latest/img.png"]} def test_all_refs_resolving_gives_no_warning(self, src, dst): make_run(dst, "other", R1) make_run(src, "wf", R2, inputs=["output:other/latest/img.png"]) - reply = copy_runs(src, dst, [f"wf/{R2}"]) + reply = copy_between(src, dst, [f"wf/{R2}"]) assert "unresolved_output_refs" not in reply["warnings"] def test_absolute_paths_are_relative_to_the_source_root(self, src, dst, tmp_path): @@ -209,19 +215,19 @@ def test_absolute_paths_are_relative_to_the_source_root(self, src, dst, tmp_path workflow_file=workflow, steps=[{"step": "s", "files": [inside, "relative/img.png"]}], ) - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) paths = reply["warnings"]["absolute_paths"][f"wf/{R1}"] assert paths == sorted(["outputs/wf/earlier/img.png", "workflows/wf.json"]) assert str(tmp_path) not in json.dumps(reply) def test_a_path_outside_the_source_root_is_not_reported(self, src, dst): make_run(src, "wf", R1, workflow_file="/somewhere/else/wf.json") - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) assert "absolute_paths" not in reply["warnings"] def test_no_jobs_note_after_a_copy(self, src, dst): make_run(src, "wf", R1) - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) assert "no_jobs" in reply["warnings"] @@ -230,14 +236,14 @@ class TestRefusals: def test_traversal(self, src, dst, entry): make_run(src, "wf", R1) with pytest.raises(SecurityError): - copy_runs(src, dst, [entry]) + copy_between(src, dst, [entry]) assert nothing_written(dst) @pytest.mark.parametrize("entry", ["wf/", f"wf/{R1}/", f"wf//{R1}", "output:wf/"]) def test_an_empty_segment(self, src, dst, entry): make_run(src, "wf", R1) with pytest.raises(InvalidInputError): - copy_runs(src, dst, [entry]) + copy_between(src, dst, [entry]) assert nothing_written(dst) def test_a_symlinked_run_folder(self, src, dst, tmp_path): @@ -248,7 +254,7 @@ def test_a_symlinked_run_folder(self, src, dst, tmp_path): os.path.join(src.outputs, "wf", R1), ) with pytest.raises(PathTraversalError): - copy_runs(src, dst, [f"wf/{R1}"]) + copy_between(src, dst, [f"wf/{R1}"]) assert nothing_written(dst) def test_a_symlinked_identity_folder(self, src, dst): @@ -257,36 +263,36 @@ def test_a_symlinked_identity_folder(self, src, dst): os.path.join(src.outputs, "real"), os.path.join(src.outputs, "linked") ) with pytest.raises(PathTraversalError): - copy_runs(src, dst, [f"linked/{R1}"]) + copy_between(src, dst, [f"linked/{R1}"]) assert nothing_written(dst) def test_a_symlink_inside_a_run(self, src, dst, tmp_path): run_dir = make_run(src, "wf", R1) os.symlink(os.path.join(run_dir, "img.png"), os.path.join(run_dir, "link.png")) with pytest.raises(PathTraversalError): - copy_runs(src, dst, [f"wf/{R1}"]) + copy_between(src, dst, [f"wf/{R1}"]) assert nothing_written(dst) def test_a_missing_run(self, src, dst): make_run(src, "wf", R1) with pytest.raises(CopyRefused) as raised: - copy_runs(src, dst, [f"wf/{R2}"]) + copy_between(src, dst, [f"wf/{R2}"]) assert raised.value.status == 404 assert nothing_written(dst) def test_a_missing_workflow(self, src, dst): with pytest.raises(CopyRefused) as raised: - copy_runs(src, dst, ["nope"]) + copy_between(src, dst, ["nope"]) assert raised.value.status == 404 def test_same_workspace(self, src): make_run(src, "wf", R1) with pytest.raises(InvalidInputError): - copy_runs(src, src, [f"wf/{R1}"]) + copy_between(src, src, [f"wf/{R1}"]) def test_empty_runs(self, src, dst): with pytest.raises(InvalidInputError): - copy_runs(src, dst, []) + copy_between(src, dst, []) def configured(self, ws): return ConfiguredWorkspace(ws.workflows, ws.assets, ws.outputs, ws.prompts) @@ -294,14 +300,14 @@ def configured(self, ws): def test_configured_source(self, src, dst): make_run(src, "wf", R1) with pytest.raises(CopyRefused) as raised: - copy_runs(self.configured(src), dst, [f"wf/{R1}"]) + copy_between(self.configured(src), dst, [f"wf/{R1}"]) assert raised.value.status == 409 assert nothing_written(dst) def test_configured_destination(self, src, dst): make_run(src, "wf", R1) with pytest.raises(CopyRefused) as raised: - copy_runs(src, self.configured(dst), [f"wf/{R1}"]) + copy_between(src, self.configured(dst), [f"wf/{R1}"]) assert raised.value.status == 409 assert nothing_written(dst) @@ -309,7 +315,7 @@ def test_an_active_run_refuses_all_or_nothing(self, src, dst): make_run(src, "wf", R1, 1) make_run(src, "wf", R2, 2) with pytest.raises(CopyRefused) as raised: - copy_runs(src, dst, ["wf"], active_runs=[f"wf/{R2}"]) + copy_between(src, dst, ["wf"], active_runs=[f"wf/{R2}"]) assert raised.value.status == 409 assert raised.value.conflicts == [f"wf/{R2}"] assert nothing_written(dst) @@ -319,7 +325,7 @@ def test_an_active_run_refuses_all_or_nothing(self, src, dst): class TestResolution: def test_output_references_resolve_in_the_destination(self, src, dst): make_run(src, "wf", R1) - copy_runs(src, dst, ["wf"]) + copy_between(src, dst, ["wf"]) latest = resolve_output_reference("output:wf/latest/img.png", dst.outputs) pinned = resolve_output_reference(f"output:wf/{R1}/img.png", dst.outputs) expected = os.path.join(os.path.realpath(dst.outputs), "wf", R1, "img.png") @@ -331,7 +337,7 @@ def test_output_references_resolve_in_the_destination(self, src, dst): class TestIndependence: def test_notes_are_not_shared(self, src, dst): source_run = make_run(src, "wf", R1, notes={"note": "original"}) - copy_runs(src, dst, [f"wf/{R1}"]) + copy_between(src, dst, [f"wf/{R1}"]) copied = os.path.join(dst.outputs, "wf", R1, "notes.json") original = os.path.join(source_run, "notes.json") assert os.stat(copied).st_ino != os.stat(original).st_ino @@ -342,7 +348,7 @@ def test_notes_are_not_shared(self, src, dst): def test_a_renumbered_manifest_leaves_the_source_alone(self, src, dst): make_run(dst, "wf", "20251231-000000-dddddddd", 1) source_run = make_run(src, "wf", R1, 1) - copy_runs(src, dst, [f"wf/{R1}"]) + copy_between(src, dst, [f"wf/{R1}"]) copied = os.path.join(dst.outputs, "wf", R1, "manifest.json") original = os.path.join(source_run, "manifest.json") assert read_json(copied)["version"] == 2 @@ -353,7 +359,7 @@ def test_a_renumbered_manifest_leaves_the_source_alone(self, src, dst): class TestMediaLinking: def test_media_is_hard_linked(self, src, dst): source_run = make_run(src, "wf", R1, content=b"x" * 100) - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) a = os.stat(os.path.join(source_run, "img.png")) b = os.stat(os.path.join(dst.outputs, "wf", R1, "img.png")) assert a.st_ino == b.st_ino @@ -366,7 +372,7 @@ def refuse(*args, **kwargs): raise OSError("cross-device") monkeypatch.setattr("dw.workspace_copy.os.link", refuse) - reply = copy_runs(src, dst, [f"wf/{R1}"]) + reply = copy_between(src, dst, [f"wf/{R1}"]) target = os.path.join(dst.outputs, "wf", R1, "img.png") source_file = os.path.join(source_run, "img.png") assert os.stat(target).st_ino != os.stat(source_file).st_ino @@ -376,6 +382,211 @@ def refuse(*args, **kwargs): assert reply["bytes_copied"] >= 100 +def make_asset( + ws, name, content=b"asset-bytes", note=None, shots=None, provenance=None +): + path = os.path.join(ws.assets, *name.split("/")) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(content) + if note is not None: + set_file_note(os.path.dirname(path), os.path.basename(path), note) + if shots is not None or provenance is not None: + record_kept_shots( + os.path.dirname(path), os.path.basename(path), shots, provenance + ) + return path + + +def no_assets(ws): + return not os.path.exists(ws.assets) or not any(os.scandir(ws.assets)) + + +SHOTS = [{"start": 0.0, "end": 1.5}] +PROVENANCE = {"job_id": "j1", "workspace": "src", "run": f"wf/{R1}"} + + +class TestAssets: + def test_a_file_keeps_its_path(self, src, dst): + source_file = make_asset(src, "cast/iris.png") + reply = copy_between(src, dst, assets=["cast/iris.png"]) + target = os.path.join(dst.assets, "cast", "iris.png") + assert reply["copied"] == {"runs": [], "assets": ["cast/iris.png"]} + assert os.stat(target).st_ino == os.stat(source_file).st_ino + assert reply["bytes_linked"] == len(b"asset-bytes") + + def test_an_asset_prefix_is_accepted(self, src, dst): + make_asset(src, "iris.png") + reply = copy_between(src, dst, assets=["asset:iris.png"]) + assert reply["copied"]["assets"] == ["iris.png"] + + def test_a_folder_copies_every_file_under_it(self, src, dst): + make_asset(src, "proj/a.png", note="first") + make_asset(src, "proj/deep/b.wav") + make_asset(src, "other.png") + reply = copy_between(src, dst, assets=["proj/"]) + assert reply["copied"]["assets"] == ["proj/a.png", "proj/deep/b.wav"] + assert not os.path.exists(os.path.join(dst.assets, "other.png")) + # The folder's sidecar travels per file, not as a file of its own + assert read_json(os.path.join(dst.assets, "proj", "notes.json"))["files"] == { + "a.png": "first" + } + + def test_a_folder_without_a_slash_is_a_folder(self, src, dst): + make_asset(src, "proj/a.png") + reply = copy_between(src, dst, assets=["proj"]) + assert reply["copied"]["assets"] == ["proj/a.png"] + + def test_an_upload_stays_under_uploads(self, src, dst): + make_asset(src, "uploads/abc.png") + reply = copy_between(src, dst, assets=["uploads/abc.png"]) + assert reply["copied"]["assets"] == ["uploads/abc.png"] + assert os.path.isfile(os.path.join(dst.assets, "uploads", "abc.png")) + + def test_json_is_copied_not_linked(self, src, dst): + source_file = make_asset(src, "data/cues.json", content=b"{}") + copy_between(src, dst, assets=["data/cues.json"]) + target = os.path.join(dst.assets, "data", "cues.json") + assert os.stat(target).st_ino != os.stat(source_file).st_ino + + def test_an_identical_file_is_already_present(self, src, dst): + make_asset(src, "iris.png") + make_asset(dst, "iris.png") + reply = copy_between(src, dst, assets=["iris.png"]) + assert reply["copied"]["assets"] == [] + assert reply["already_present"] == ["iris.png"] + + def test_repeating_a_copy_is_idempotent(self, src, dst): + make_asset(src, "proj/a.png", note="n") + copy_between(src, dst, assets=["proj/"]) + reply = copy_between(src, dst, assets=["proj/"]) + assert reply["copied"]["assets"] == [] + assert reply["already_present"] == ["proj/a.png"] + + def test_a_different_file_refuses_everything(self, src, dst): + make_run(src, "wf", R1) + make_asset(src, "proj/a.png") + make_asset(src, "proj/b.png", content=b"mine") + make_asset(dst, "proj/b.png", content=b"theirs") + with pytest.raises(CopyRefused) as refused: + copy_between(src, dst, [f"wf/{R1}"], ["proj/"]) + assert refused.value.status == 409 + assert refused.value.conflicts == ["proj/b.png"] + assert "proj/b.png" in str(refused.value) + assert nothing_written(dst) + assert not os.path.exists(os.path.join(dst.assets, "proj", "a.png")) + with open(os.path.join(dst.assets, "proj", "b.png"), "rb") as f: + assert f.read() == b"theirs" + + def test_an_active_run_and_a_different_file_are_named_together(self, src, dst): + make_run(src, "wf", R1) + make_asset(src, "a.png", content=b"mine") + make_asset(dst, "a.png", content=b"theirs") + with pytest.raises(CopyRefused) as refused: + copy_between(src, dst, [f"wf/{R1}"], ["a.png"], active_runs=[f"wf/{R1}"]) + assert refused.value.conflicts == [f"wf/{R1}", "a.png"] + + def test_runs_and_assets_together(self, src, dst): + make_run(src, "wf", R1) + make_asset(src, "a.png") + reply = copy_between(src, dst, [f"wf/{R1}"], ["a.png"]) + assert reply["copied"] == {"runs": [f"wf/{R1}"], "assets": ["a.png"]} + + def test_notes_and_kept_shots_travel(self, src, dst): + make_asset( + src, "proj/a.mp4", note="the keeper", shots=SHOTS, provenance=PROVENANCE + ) + copy_between(src, dst, assets=["proj/a.mp4"]) + target_dir = os.path.join(dst.assets, "proj") + assert read_notes(target_dir)["files"] == {"a.mp4": "the keeper"} + manifest = read_json(os.path.join(target_dir, "manifest.json")) + [entry] = manifest["steps"] + assert entry["files"] == ["a.mp4"] + assert entry["shots"] == SHOTS + # Where it was kept from is history: it does not become the destination + assert entry["provenance"] == PROVENANCE + assert entry["provenance"]["workspace"] == "src" + + def test_a_destinations_own_entries_are_kept(self, src, dst): + make_asset(src, "proj/a.png", note="from src") + make_asset(dst, "proj/z.png", note="already here", shots=SHOTS) + copy_between(src, dst, assets=["proj/a.png"]) + target_dir = os.path.join(dst.assets, "proj") + assert read_notes(target_dir)["files"] == { + "a.png": "from src", + "z.png": "already here", + } + steps = read_json(os.path.join(target_dir, "manifest.json"))["steps"] + assert [s["files"] for s in steps] == [["z.png"]] + + def test_sidecars_are_not_shared(self, src, dst): + make_asset(src, "proj/a.mp4", note="original", shots=SHOTS) + source_dir = os.path.join(src.assets, "proj") + notes_before = read_json(os.path.join(source_dir, "notes.json")) + manifest_before = read_json(os.path.join(source_dir, "manifest.json")) + copy_between(src, dst, assets=["proj/"]) + target_dir = os.path.join(dst.assets, "proj") + for sidecar in ("notes.json", "manifest.json"): + assert ( + os.stat(os.path.join(target_dir, sidecar)).st_ino + != os.stat(os.path.join(source_dir, sidecar)).st_ino + ) + set_file_note(target_dir, "a.mp4", "edited") + record_kept_shots(target_dir, "a.mp4", [{"start": 0.0, "end": 9.0}]) + assert read_json(os.path.join(source_dir, "notes.json")) == notes_before + assert read_json(os.path.join(source_dir, "manifest.json")) == manifest_before + + @pytest.mark.parametrize("entry", ["../x.png", "/abs/x.png", "a/../b.png", "./a"]) + def test_traversal(self, src, dst, entry): + make_asset(src, "a/b.png") + with pytest.raises(PathTraversalError): + copy_between(src, dst, assets=[entry]) + assert no_assets(dst) + + def test_a_sidecar_named_directly(self, src, dst): + make_asset(src, "proj/a.png", note="n") + with pytest.raises(InvalidInputError): + copy_between(src, dst, assets=["proj/notes.json"]) + + def test_a_missing_asset(self, src, dst): + with pytest.raises(CopyRefused) as refused: + copy_between(src, dst, assets=["nope.png"]) + assert refused.value.status == 404 + + def test_a_symlink_on_the_way_in_the_source(self, src, dst, tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + (outside / "a.png").write_bytes(b"x") + os.makedirs(src.assets, exist_ok=True) + os.symlink(outside, os.path.join(src.assets, "linked")) + with pytest.raises(PathTraversalError): + copy_between(src, dst, assets=["linked/a.png"]) + assert no_assets(dst) + + def test_a_symlink_inside_a_folder(self, src, dst, tmp_path): + make_asset(src, "proj/a.png") + outside = tmp_path / "secret.png" + outside.write_bytes(b"x") + os.symlink(outside, os.path.join(src.assets, "proj", "b.png")) + with pytest.raises(PathTraversalError): + copy_between(src, dst, assets=["proj/"]) + assert no_assets(dst) + + def test_a_symlink_on_the_way_in_the_destination(self, src, dst, tmp_path): + make_asset(src, "proj/a.png") + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + os.makedirs(dst.assets, exist_ok=True) + os.symlink(elsewhere, os.path.join(dst.assets, "proj")) + with pytest.raises(PathTraversalError): + copy_between(src, dst, assets=["proj/a.png"]) + assert list(elsewhere.iterdir()) == [] + + def test_nothing_named(self, src, dst): + with pytest.raises(InvalidInputError): + copy_between(src, dst) + + @pytest.fixture def server(root, tmp_path): def make(with_root=True): @@ -412,17 +623,81 @@ def test_happy_path(self, server, root, src, dst): assert body["copied"]["runs"] == [f"wf/{R1}"] assert os.path.isfile(os.path.join(dst.outputs, "wf", R1, "img.png")) - def test_assets_are_not_supported(self, server, src, dst): + def test_assets(self, server, src, dst): + make_asset(src, "proj/a.png", note="n") + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "assets": ["proj/"]}, + ) + assert response.status_code == 200, response.text + assert response.json()["copied"] == {"runs": [], "assets": ["proj/a.png"]} + assert os.path.isfile(os.path.join(dst.assets, "proj", "a.png")) + + def test_runs_and_assets(self, server, src, dst): make_run(src, "wf", R1) + make_asset(src, "a.png") with server() as client: response = client.post( "/api/workspaces/dst/copy", json={"source": "src", "runs": [f"wf/{R1}"], "assets": ["a.png"]}, ) - assert response.status_code == 400 - assert "not supported" in response.text + assert response.status_code == 200, response.text + assert response.json()["copied"] == { + "runs": [f"wf/{R1}"], + "assets": ["a.png"], + } + + def test_a_different_asset_is_409_and_writes_nothing(self, server, src, dst): + make_run(src, "wf", R1) + make_asset(src, "a.png", content=b"mine") + make_asset(dst, "a.png", content=b"theirs") + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "runs": [f"wf/{R1}"], "assets": ["a.png"]}, + ) + assert response.status_code == 409 + assert response.json()["detail"]["conflicts"] == ["a.png"] assert nothing_written(dst) + def test_a_missing_asset_is_404(self, server, src, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "assets": ["nope.png"]}, + ) + assert response.status_code == 404 + + def test_asset_traversal_is_400(self, server, src, dst): + with server() as client: + response = client.post( + "/api/workspaces/dst/copy", + json={"source": "src", "assets": ["../x.png"]}, + ) + assert response.status_code == 400 + + def test_an_upload_over_a_copied_asset_leaves_the_source_alone( + self, server, src, dst + ): + source_file = make_asset(src, "uploads/cast.png", content=b"original") + copy_between(src, dst, assets=["uploads/cast.png"]) + with server() as client: + response = client.post( + "/api/uploads", + params={ + "filename": "cast.png", + "asset_name": "cast.png", + "workspace": "dst", + }, + content=b"replaced", + ) + assert response.status_code == 201, response.text + with open(os.path.join(dst.assets, "uploads", "cast.png"), "rb") as f: + assert f.read() == b"replaced" + with open(source_file, "rb") as f: + assert f.read() == b"original" + def test_empty_runs(self, server, src, dst): with server() as client: response = client.post( @@ -570,7 +845,13 @@ def test_source_defaults_to_the_session_workspace(self): assert seen[0].method == "POST" assert seen[0].url.path == "/api/workspaces/final/copy" body = json.loads(seen[0].content) - assert body == {"source": "shots", "runs": ["wf/x"]} + assert body == {"source": "shots", "runs": ["wf/x"], "assets": []} + + def test_assets_pass_through(self): + client, seen = self.recording("shots") + copy_to_workspace(client, "final", assets=["proj/"]) + body = json.loads(seen[0].content) + assert body == {"source": "shots", "runs": [], "assets": ["proj/"]} def test_an_explicit_source_passes_through(self): client, seen = self.recording("shots") From 500f53cc5b8c23e5b6444f0cd9836639ddc321cf Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 18:17:31 -0500 Subject: [PATCH 037/204] feat(workspace): #860 - docs, OpenAPI and surface budget for asset copying Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 2 +- docs/SERVER.md | 15 +++++++++------ docs/WORKSPACES.md | 21 +++++++++++++++++---- tests/test_mcp_server.py | 6 +++++- ui/src/lib/generated/api-schema.ts | 17 ++++++++++------- ui/src/lib/generated/openapi.json | 8 ++++---- 6 files changed, 46 insertions(+), 23 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index c8983b03..da1fbaac 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -278,7 +278,7 @@ tools) rather than relying on the session pin. | `list_workspaces()` | — | The server's workspaces and which one this session is using. Each has its own workflows, assets and outputs; the prompt library is shared by all of them | | `use_workspace(name)` | `name` | Work in that workspace for the rest of the session - every later call reads and writes there. This is how to keep your work out of another agent's namespace rather than sharing the default one. Checked against the server, so a typo fails here rather than scoping every later call to nothing. On a `dw.serve --mcp` endpoint the pin is shared by every connected client (#298, see the note above the table) - a `warning` field appears when this call overwrote a pin already pointed away from `default` | | `create_workspace(name, use=False)` | `name`, `use` | Create a workspace. Pass use=true to switch this session to it as well; otherwise the session stays where it was and the result says so. The result names the workspace (`name`, `default`, `current`, `next`), not its folders - `list_workspaces(detail=true)` is the opt-in for those. `use=true` carries the same mounted-server caveat and `warning` field as `use_workspace` | -| `copy_to_workspace(destination, runs, source=None)` | `destination`, `runs`, `source` | Copy runs from one workspace into another. Each `runs` entry is `/` as `list_gallery` names it, or a bare `` for all its runs; `source` defaults to this session's workspace. Free: no `acknowledged_cost`. Media is hard-linked (a copy across devices falls back to a real copy) and JSON sidecars are always copied, so an edit in one workspace never reaches the other. All or nothing; a run the destination already has is skipped as `already_present`, so a repeat is idempotent, and a `vN` it already uses is renumbered after its highest and reported in `renumbered`. Warnings: `unresolved_output_refs`, `absolute_paths`, `no_jobs`. A copied run has no job record - it shows in `list_gallery` and resolves as an `output:` reference, not in `list_jobs`. A run whose job is still queued or running is refused. To move, copy then `delete_output` the source runs. See [Workspaces](WORKSPACES.md#copying-between-workspaces) | +| `copy_to_workspace(destination, runs=None, assets=None, source=None)` | `destination`, `runs`, `assets`, `source` | Copy runs and assets from one workspace into another. Each `runs` entry is `/` as `list_gallery` names it, or a bare `` for all its runs; each `assets` entry is a path as after `asset:`, or a folder (`proj/`) for every file under it, keeping its path, note and kept shots. `source` defaults to this session's workspace. Free: no `acknowledged_cost`. Media is hard-linked (a copy across devices falls back to a real copy) and JSON sidecars are always copied, so an edit in one workspace never reaches the other. All or nothing; a run or identical asset the destination already has is skipped as `already_present`, a different file at an asset's path refuses the whole request, so a repeat is idempotent, and a `vN` it already uses is renumbered after its highest and reported in `renumbered`. Warnings: `unresolved_output_refs`, `absolute_paths`, `no_jobs`. A copied run has no job record - it shows in `list_gallery` and resolves as an `output:` reference, not in `list_jobs`. A run whose job is still queued or running is refused. To move, copy then `delete_output` the source runs and `delete_asset` the source assets. See [Workspaces](WORKSPACES.md#copying-between-workspaces) | | `delete_workspace(name, acknowledged_cost=False)` | `name`, `acknowledged_cost` | Permanently delete a workspace and everything in it. Refuses without the acknowledgement, reporting what it would remove | | `list_assets(limit=50, prefix)` | `limit`, `prefix` | The input media on the server, each with the `asset:` reference a workflow argument carries. Look here before asking for a file - what a workflow needs may already be there. `libraries` names the roots searched and which are writable; `shadowed` lists names a nearer library hides. Newest first; `prefix` keeps names starting with it (e.g. `cast/`), `limit` bounds the answer, `total` is the count matched before the cut | | `keep_output(name, asset_name=None, overwrite=False, shared=False, workspace=None, note=None)` | `name`, optional `asset_name`, `overwrite`, `shared`, `workspace`, `note` | Keep a generated file as an input asset under a stable `asset:` name, so a later workflow can rely on it. The copy happens on the server: nothing is downloaded or re-uploaded. `note` is kept with the asset; left out, the source file's note is copied. `asset_name` may name a folder and takes the kept file's extension when it has none; `shared=true` keeps it in the library every workspace shares, which is where a recurring cast belongs. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | diff --git a/docs/SERVER.md b/docs/SERVER.md index e647c64b..f470aabb 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -578,12 +578,15 @@ The editor's forms come from these; they are just as usable from scripts: exported job. It is a reserved name: no workspace can be called `exports`, and the folder is never listed as one. - `POST /api/workspaces/{destination}/copy` (`{"source": ..., "runs": [...], - "assets": []}`) — copy runs from one workspace into another. A `runs` entry - is `/`, or a bare `` for all its runs. A - non-empty `assets` is a 400: copying assets is not supported yet. An empty - `runs` or a source equal to the destination is a 400, an unknown workspace - or run a 404, and a server without a workspace root, a run whose job is still - queued or running, or a destination that is not writable a 409. All or + "assets": [...]}`) — copy runs and assets from one workspace into another. + A `runs` entry is `/`, or a bare `` for all its + runs; an `assets` entry is a library-relative path as after `asset:`, or a + folder (`proj/`) for every file under it. Naming neither, a source equal to + the destination, a path that escapes or reaches a symlink, or a sidecar + named directly is a 400, an unknown workspace, run or asset a 404, and a + server without a workspace root, a destination that is not writable, a run + whose job is still queued or running, or a different file already at an + asset's path a 409 (`detail.conflicts` names every one). All or nothing. The reply is `{source, destination, copied: {runs, assets}, already_present, renumbered, warnings: {unresolved_output_refs, absolute_paths, no_jobs}, bytes_linked, bytes_copied}`; `renumbered` maps diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 843a9aa3..8f780c4f 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -330,8 +330,13 @@ A run is self-contained on disk, so copying its folder into another workspace's `outputs/` is what makes it resolvable there. `copy_to_workspace` (MCP) and `POST /api/workspaces/{destination}/copy` ([SERVER.md](SERVER.md)) do that for runs, named `/` or, for all of a workflow's runs, -a bare ``. Only runs are copied for now; assets are not, and a -request naming some is refused. +a bare ``, and for assets, named as after `asset:` (`cast/iris.png`, +or a folder such as `proj/` for every file under it). An asset keeps its +library-relative path, so `uploads/` stays `uploads/` and an `asset:` reference +means the same file in both workspaces. Its note and its kept-shots entry (the +folder's `notes.json` and `manifest.json`, including where it was kept from) +are merged into the destination folder's own sidecars, which are written +fresh: naming a sidecar directly is a 400, since it travels with its files. Media is hard-linked, so a copy costs almost no disk, falling back to a real copy across devices. JSON sidecars (the manifest, `workflow.json`, notes, @@ -343,7 +348,15 @@ The copy is all or nothing: every entry is checked before anything is written. A run whose run id the destination already has is the same run, so it is skipped and listed as `already_present`, and repeating a copy changes nothing. A run whose job is still queued or running is refused, since its manifest is -not final. +not final. An asset the destination already holds with the same bytes is +skipped as `already_present` too; a different file at that path is a conflict, +never replaced. Every conflict, runs and assets alike, is named in one 409 +before anything is written. + +Because a copied asset is a hard link, nothing edits media in place: an upload +over an existing name writes a new file and renames it into place, and +`keep_output` and `delete_asset` unlink first, so an edit in one workspace +never reaches the other. A run keeps its `vN` unless the destination already uses that number for the workflow. Then this request's runs of that workflow are renumbered after the @@ -358,7 +371,7 @@ to the source workspace's root) and `no_jobs`. A copied run has no job record, s the gallery and resolves as an `output:` reference, but not in `list_jobs`. A move is a copy, then a delete of the source runs (`delete_output`, or -`DELETE` on the gallery runs); there is no move that could lose a run halfway. +`DELETE` on the gallery runs) and assets (`delete_asset`); there is no move that could lose a run halfway. ## Where this is going diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index a68238ac..29b3cb01 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1587,7 +1587,11 @@ def test_the_stated_tool_count_is_the_registered_one(): # never registered, is registered (66.75 description + 83.25 schema) - # measured at 14_620.25 (9_343 / 4_767 / 510.25); the ceiling moves to # 14_621 with 0.75 of headroom. -SURFACE_BUDGET = 14_621 +# 2026-10-13 (#860, #853 stage B): copy_to_workspace takes assets (its +# description names them; the schema gains the parameter, runs optional) - +# measured at 14_699.5 (9_381 / 4_808 / 510.5); the ceiling moves to 14_700 +# with 0.5 of headroom. +SURFACE_BUDGET = 14_700 @pytest.mark.asyncio diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 6fdd4c18..17012432 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -1704,12 +1704,15 @@ export interface paths { put?: never; /** * Copy Into Workspace - * @description Copy runs from another workspace into this one. + * @description Copy runs and asset-library entries from another workspace into this one. * * Media is hard-linked where the filesystem allows, so a copy costs - * little disk; sidecars are copied. All or nothing: a run already here is - * skipped as `already_present`, a run whose job is still running refuses - * the request. A move is a copy, then `DELETE` of the source runs. + * little disk; sidecars are copied, and an asset's note and kept shots are + * merged into the destination folder's. All or nothing: a run already + * here, or an asset with identical bytes, is skipped as `already_present`; + * a run whose job is still running, or a different file at an asset's + * path, refuses the whole request (409, naming each) with nothing written. + * A move is a copy, then `DELETE` of the source runs and assets. */ post: operations["copy_into_workspace_api_workspaces__destination__copy_post"]; delete?: never; @@ -2004,7 +2007,7 @@ export interface components { CopiedItems: { /** * Assets - * @description Always empty until assets can be copied. + * @description Library-relative path of each asset file copied. */ assets: string[]; /** @@ -3701,7 +3704,7 @@ export interface components { WorkspaceCopied: { /** * Already Present - * @description Runs the destination already held, by run id - skipped. + * @description Runs the destination already held (`/`) and assets it held with identical bytes (library-relative path) - skipped. */ already_present: string[]; /** Bytes Copied */ @@ -3734,7 +3737,7 @@ export interface components { WorkspaceCopyRequest: { /** * Assets - * @description Not supported yet - leave empty + * @description Asset-library files or folders, as written after `asset:` (`proj/clip.mp4`, `proj/`); each keeps its library-relative path */ assets?: string[]; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index fb233b42..c93d4914 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -426,7 +426,7 @@ "additionalProperties": false, "properties": { "assets": { - "description": "Always empty until assets can be copied.", + "description": "Library-relative path of each asset file copied.", "items": { "type": "string" }, @@ -4808,7 +4808,7 @@ "description": "What a copy between workspaces did (`POST /api/workspaces/{name}/copy`).", "properties": { "already_present": { - "description": "Runs the destination already held, by run id - skipped.", + "description": "Runs the destination already held (`/`) and assets it held with identical bytes (library-relative path) - skipped.", "items": { "type": "string" }, @@ -4868,7 +4868,7 @@ "WorkspaceCopyRequest": { "properties": { "assets": { - "description": "Not supported yet - leave empty", + "description": "Asset-library files or folders, as written after `asset:` (`proj/clip.mp4`, `proj/`); each keeps its library-relative path", "items": { "type": "string" }, @@ -9045,7 +9045,7 @@ }, "/api/workspaces/{destination}/copy": { "post": { - "description": "Copy runs from another workspace into this one.\n\nMedia is hard-linked where the filesystem allows, so a copy costs\nlittle disk; sidecars are copied. All or nothing: a run already here is\nskipped as `already_present`, a run whose job is still running refuses\nthe request. A move is a copy, then `DELETE` of the source runs.", + "description": "Copy runs and asset-library entries from another workspace into this one.\n\nMedia is hard-linked where the filesystem allows, so a copy costs\nlittle disk; sidecars are copied, and an asset's note and kept shots are\nmerged into the destination folder's. All or nothing: a run already\nhere, or an asset with identical bytes, is skipped as `already_present`;\na run whose job is still running, or a different file at an asset's\npath, refuses the whole request (409, naming each) with nothing written.\nA move is a copy, then `DELETE` of the source runs and assets.", "operationId": "copy_into_workspace_api_workspaces__destination__copy_post", "parameters": [ { From 0f5471bac0a520e127679fd03c8e24d2868bf7db Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 18:28:31 -0500 Subject: [PATCH 038/204] docs(proposals): #853 close-out - workspace copy design record Co-Authored-By: Claude Opus 5.5 --- .../complete/workspace-copy-complete.md | 160 ++++++++++++++++++ docs/proposals/todo.md | 9 + 2 files changed, 169 insertions(+) create mode 100644 docs/proposals/complete/workspace-copy-complete.md diff --git a/docs/proposals/complete/workspace-copy-complete.md b/docs/proposals/complete/workspace-copy-complete.md new file mode 100644 index 00000000..56884f1e --- /dev/null +++ b/docs/proposals/complete/workspace-copy-complete.md @@ -0,0 +1,160 @@ +# Copy runs and assets between workspaces: `copy_to_workspace` (#853) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-13). Plan v2 was approved by Don on 2026-10-13. Stage A, #859, and +stage B, #860, shipped the same day, at `develop` @ `b8a26e26` and +`d084ea6a`, and were verified on mini-ai (mps). + +## The idea + +A field report: an agent needed to move a project's work (the Soviet-anthem +assets and runs) from the `default` workspace into a workspace of its own. +No tool did it (`move_job` only reorders the queue), so the agent +re-uploaded every file and deleted the originals by hand. A re-upload loses +provenance: the file becomes an `uploads/` asset with no manifest and no +`output:` reference. + +## Verdict and scope (Don, 2026-10-13) + +**Build, in two stages.** Don's scope ruling came before the plan: + +1. **Copy only, no move.** A move is a copy followed by `delete_output` / + `delete_asset`, which already require `acknowledged_cost`. So there is no + new destructive path, and no jobs-DB `workspace`/`run_dir` rows are + rewritten. +2. **In scope:** asset-library entries and whole runs + (`//` with manifests). Saved workflows and prompts are + out of scope. Prompts are already shared by the whole root. +3. **`output:` references** are workspace-relative and are never + rewritten. A copied manifest that names a run not in the destination + gives a warning, not a refusal. +4. **Confinement:** both workspaces are under one root, checked with the + existing validators. Copying into the shared library stays with + `keep_output(shared=true)`. + +This also settled #854's deferred question about root placement for +uploads: a copied asset keeps its library-relative path exactly, so +`uploads/x` stays `uploads/x`. + +## Decisions (Don's review of plan v1, 2026-10-13) + +- **Q1. Hard-link or copy?** Hard-link **media only**, falling back to + `copy2`. JSON sidecars (`manifest.json`, `workflow.json`, `notes.json`, + `job.json`, any `.json`) are always copied, because `record_kept_shots` + and `write_realized_workflow` write in place, so a linked sidecar would + carry an edit across workspaces. +- **Q2.** No `acknowledged_cost` on the copy: it never overwrites. +- **Q3.** A colliding `vN` is renumbered after the destination's highest, + in run-id order, and the reply maps old to new. +- **Q4.** `SURFACE_BUDGET` was raised by the measured size, with a dated + note, once per stage. +- **Q5.** Copied runs get no jobs-DB rows. They appear in `list_gallery` + but not in `list_jobs`, and the reply and `WORKSPACES.md` say so. + +## What was built + +### Stage A, #859: copy runs + +- **Engine:** `dw/workspace_copy.py`. + - Both workspaces resolve through `named_workspace`. Every path gets + `validate_path` with that side's folder as the base. A symlink + anywhere on a source path or inside a run is refused, and a + `ConfiguredWorkspace` (no root) is refused with 409. + - All or nothing: every entry is checked before anything is written. A + run id the destination already holds goes to `already_present` and is + skipped, so a repeat copy is idempotent. A run whose job is queued or + running is refused by name (409). + - Media is hard-linked, falling back to a copy, and `.json` is always + copied. Warnings: `unresolved_output_refs`, `absolute_paths`, and the + no-jobs note. Afterwards it calls `forget_workspace_usage()`. +- **Run versions:** after the architecture review, numbering moved to + `dw/runs.py: copy_runs_in(source_identity_dir, identity_dir, run_ids, + place)`. It runs under `run_lock_path`, next to `record_run_versions`. +- **Run-entry shape:** `security.validate_output_prefix`, which shares one + `_OUTPUT_SEGMENT` with `OUTPUT_REFERENCE_PATTERN`. An entry with an empty + segment (`wf/`, `wf//`) is refused, not normalized. +- **Route:** `POST /api/workspaces/{destination:path}/copy` in + `dw/server/routes/library.py`, with body `{source, runs, assets}`. +- **MCP:** `copy_to_workspace(destination, runs, assets, source=None)`, a + thin client of the route, registered `WRITES`. +- **Outside the plan:** `set_note` was registered as an MCP tool. The + notes feature (`a6483383`) had written it, documented it and named it in + the server instructions, but never registered it, and the acceptance + case for note independence needed it. Tool count: 62 → 64. +- **Outside the plan:** `api_path` now encodes `.`/`..` segments + (`%2E`), so httpx no longer drops them. The fix is in the shared + `path_segment`, so it covers every MCP tool that builds a path. + +### Stage B, #860: copy assets + +- `copy_between` replaced `copy_runs` as the engine entry point. `runs` and + `assets` are both optional, but at least one is required (400 if + neither is given). +- An asset entry is a library-relative file or a folder (`proj/` or + `proj`), and an `asset:` prefix is accepted. `copied.assets` lists each + file copied, expanded from a folder. +- Each file's entries in its folder's `notes.json` and `manifest.json` + (kept shots) are merged into fresh destination sidecars. + `provenance.workspace` keeps naming the source. A sidecar can't be named + directly (400), and a folder copy doesn't copy sidecars as files. +- Identical bytes at the path go into `already_present`. Different bytes + are a conflict, and they refuse the whole request, runs included (409, + `detail.conflicts`). +- **Outside the plan:** the plan's risk said "copy, don't link, any path + that edits media in place". The build found one such writer: + `upload_media` wrote over an existing name with `open(path, "wb")`. It + now writes a temp file and `os.replace`s it, so linking stays safe. This + also fixes an existing bug, where an upload over a `keep_output` link + changed the linked original. + +### Tests and docs + +- `tests/test_workspace_copy.py`, on the real engine and route with no + mocks of the copy. It covers sidecar independence, the hard-link check + (same inode when `os.link` succeeds), traversal, symlinks, conflicts and + renumbering. +- `SURFACE_BUDGET` was raised once per stage, plus once for `set_note`. It + ended at 14_700. +- Docs: `docs/WORKSPACES.md` ("Copying between workspaces"), + `docs/MCP.md`, `docs/SERVER.md`, `docs/ARCHITECTURE.md` (the "Workspace + copy" row), and the regenerated OpenAPI documents. + +## Verified + +On mini-ai (mps), over MCP only. Stage A: C-F395 to C-F402 and SE-F047. +Stage B: C-F403 to C-F405 and SE-F048. Nothing here needs CUDA: it is all +filesystem work. + +## Bounces per stage + +- **Stage A, #859: 2.** + - **Architecture review (1):** three second-owner findings: the + reference prefix was spelled locally, run-version numbering lived + outside `dw/runs.py`, and the output name shape was restated. All three + were fixed by calling the owners. + - **Tester (1):** + - C-F398 couldn't run, because no MCP tool could change a note + (`set_note` was unregistered). + - SE-F047: `wf/` was normalized instead of refused, and bad + destination names missed the route (a bare 404, or a 405 when httpx + dropped `.`/`..`) instead of getting "No such workspace". + - The tester also filed three case-text problems as harnest#104; they + weren't counted. +- **Stage B, #860: 0.** The tester didn't count one SE-F048 arm: + `asset:proj/x.wav` was accepted, which the plan allows. It was filed as a + case amendment, harnest#106. + +Cost per stage isn't recorded: the stage comments name no `usage:` +figures. The plan estimated $8–12 for both. + +## Deferred + +- **A UI for copying.** It wasn't wanted for the field report. Build it if + copying from the web UI is ever asked for. +- **Jobs-DB rows for copied runs** (Q5). `get_job` and `job_provenance` + are empty for a copied run. Revisit if an agent needs job provenance on a + copy. +- **Saved workflows and prompts.** Out of scope by Don's ruling. +- **The `absolute_paths` warning on inline runs.** Every inline run warns + `workflows/__inline__.json`. It's accurate, but the tester called it + noise. Quiet it if it misleads an agent. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 31886077..0bb86dad 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -41,6 +41,15 @@ remaining deferred fix is recorded in ## Shipped since the ranking +- **Copy runs and assets between workspaces** (#853, stages #859-#860), + shipped 2026-10-13. `copy_to_workspace` and `POST + /api/workspaces/{destination}/copy` copy whole runs (with manifests) and + asset-library entries (with their notes and kept shots) between two + workspaces under one root. The copy is all or nothing, never overwrites + and is idempotent. Media is hard-linked, sidecars are copied, and + colliding versions are renumbered. A move is a copy followed by the + delete tools. Record, including what was deferred (a UI, job rows for + copied runs, workflows and prompts): `complete/workspace-copy-complete.md`. - **List and clear dw's own disk caches** (#828, stages #832-#834), shipped 2026-10-11. A registry (`dw/disk_caches.py`) of `inductor`, `triton` (Triton's own `~/.triton/cache`, left in place) and the `quantized` cache, with From 43cba931e398a3a37494fb9abf0ea1d3cdbbb950 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 18:30:49 -0500 Subject: [PATCH 039/204] docs(proposals): #853 record names no nonexistent workflow file Co-Authored-By: Claude Opus 5.5 --- docs/proposals/complete/workspace-copy-complete.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/proposals/complete/workspace-copy-complete.md b/docs/proposals/complete/workspace-copy-complete.md index 56884f1e..7fc3c65e 100644 --- a/docs/proposals/complete/workspace-copy-complete.md +++ b/docs/proposals/complete/workspace-copy-complete.md @@ -156,5 +156,5 @@ figures. The plan estimated $8–12 for both. copy. - **Saved workflows and prompts.** Out of scope by Don's ruling. - **The `absolute_paths` warning on inline runs.** Every inline run warns - `workflows/__inline__.json`. It's accurate, but the tester called it + the inline workflow's path. It's accurate, but the tester called it noise. Quiet it if it misleads an agent. From 9e59745404985d571184c6407ddee824ba453577 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:13:38 -0500 Subject: [PATCH 040/204] fix(mcp): #876 - warn when match_levels levels a multi-shot input as one unit Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/concat_videos.py | 49 ++++++++++++++++++++++++++++++++++--- tests/test_concat_videos.py | 34 +++++++++++++++++++++++++ 2 files changed, 80 insertions(+), 3 deletions(-) diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index 133627ef..ab21243e 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -12,7 +12,7 @@ import numpy -from ..dsp import as_channels_samples +from ..dsp import as_channels_samples, level_dbfs from ..events import emit_log, emit_warning from ..media_types import AudioVideo from ..shots import measured_num_samples, nested_shots, shot_record, trimmed_shots @@ -97,7 +97,10 @@ def concat_videos( land 10 dB apart routinely, and that jump is the one seam artifact none of the fade controls can hide, because it is not at the seam but either side of it. Left off, a spread wide - enough to hear is logged as a warning + enough to hear is logged as a warning. Each input is levelled + as one unit: an input that is itself a join (it carries + media.shots) keeps the level steps between its own shots, and + one wider than 3 dB is reported as an inner_level_step warning match_levels_dbfs: The level match_levels moves every shot to - defaults to -1 dBFS for "peak" and -20 dBFS for "rms". A shot that would clip at the target is held at -0.5 dBFS peak instead, @@ -223,8 +226,11 @@ def _prepare_inputs(videos, match_levels, match_levels_dbfs, sample_rate): names, videos = load_named_inputs(videos) clips = [frames_as_pil_list(v) for v in videos] check_same_frame_size(clips, "concat_videos") + native = _input_waveforms(videos) + if match_levels: + _warn_inner_level_steps(videos, names, native, match_levels) waveforms, sample_rate = reconcile_sample_rates( - "concat_videos", videos, names, _input_waveforms(videos), sample_rate + "concat_videos", videos, names, native, sample_rate ) waveforms = level_waveforms( "concat_videos", waveforms, match_levels, match_levels_dbfs, names @@ -232,6 +238,43 @@ def _prepare_inputs(videos, match_levels, match_levels_dbfs, sample_rate): return names, videos, clips, waveforms, sample_rate +INNER_LEVEL_STEP_WARN_DB = 3.0 + + +def _warn_inner_level_steps(videos, names, waveforms, measure): + """Say so when match_levels levels a multi-shot input as one unit and its + own shots differ by more than assess_output's seam_level_step threshold: + the step between them survives the join (#876).""" + for video, name, waveform in zip(videos, names, waveforms): + shots = getattr(video, "shots", None) + if waveform is None or not shots or len(shots) < 2: + continue + levels = [] + for shot in shots: + start, count = shot.get("start_sample"), shot.get("num_samples") + if start is None or not count: + continue + level = level_dbfs(waveform[:, start : start + count], measure) + if level is not None: + levels.append(level) + if len(levels) < 2: + continue + spread = max(levels) - min(levels) + if spread > INNER_LEVEL_STEP_WARN_DB: + emit_warning( + f"concat_videos: {name} is itself a join of {len(shots)} shots " + f"whose {measure} levels span {spread:.1f} dB - match_levels " + "levels an input as one unit, so that step survives the join. " + "Trim it into its shots (trim_video per media.shots entry) and " + "join those to level them", + kind="inner_level_step", + command="concat_videos", + video=name, + spread_db=round(spread, 1), + measure=measure, + ) + + def _fitted_audio(frames, audio, sample_rate, fps, videos, shots): """The joined track fitted to the frame grid, with the rate the file is written at: (audio, written_fps). Every shot is measured against it.""" diff --git a/tests/test_concat_videos.py b/tests/test_concat_videos.py index 29e72826..af777a77 100644 --- a/tests/test_concat_videos.py +++ b/tests/test_concat_videos.py @@ -582,6 +582,40 @@ def test_gaining_a_near_silent_shot_up_to_target_warns(self, caplog): assert "near-silent" in caplog.text +class TestInnerLevelStep: + """#876: match_levels levels a multi-shot input as one unit.""" + + def _events(self, call): + from dw.events import RunContext, activate_context, deactivate_context + + events = [] + token = activate_context(RunContext(on_event=events.append)) + try: + call() + finally: + deactivate_context(token) + return [e for e in events if "inner_level_step" in str(e)] + + def test_a_joined_input_with_a_step_between_its_shots_warns(self): + inner = concat_videos([audio_video(4, 0.5), audio_video(4, 0.1)]) + assert len(inner.shots) == 2 + + found = self._events( + lambda: concat_videos([inner, audio_video(4, 0.3)], match_levels="rms") + ) + + assert found + + def test_an_even_joined_input_draws_no_warning(self): + inner = concat_videos([audio_video(4, 0.3), audio_video(4, 0.3)]) + + found = self._events( + lambda: concat_videos([inner, audio_video(4, 0.3)], match_levels="rms") + ) + + assert not found + + class TestWarningsReachTheCaller: """A warning that only reaches the server's log does not exist from outside it - see issue #82, where match_levels verified but the spread From 68f9efca9d1a0a4fada3752196044d09fc877edc Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:22:15 -0500 Subject: [PATCH 041/204] fix(mcp): #876 - one owner for the inner-step threshold, place shots with assess.sample_span, honour audio_replaced_downstream Co-Authored-By: Claude Sonnet 5.5 --- dw/assessment_rules.py | 9 +++++---- dw/tasks/concat_videos.py | 15 +++++++++------ dw/tasks/joins.py | 4 ++++ tests/test_concat_videos.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 46 insertions(+), 10 deletions(-) diff --git a/dw/assessment_rules.py b/dw/assessment_rules.py index b0b55e08..71ced4ca 100644 --- a/dw/assessment_rules.py +++ b/dw/assessment_rules.py @@ -16,8 +16,9 @@ the threshold, the severity and what a crossing says. `magnitude` compares the value's absolute size, for a signed measurement whose direction is not the problem. `shot_level_spread` shares its threshold with the run-time -`LEVEL_SPREAD_WARN_DB`, so the join-time warning and the after-the-fact -probe cannot disagree about the same cut. +`LEVEL_SPREAD_WARN_DB`, and `seam_level_step` with `INNER_LEVEL_STEP_WARN_DB` +(concat_videos' warning for a multi-shot input), so the join-time warning +and the after-the-fact probe cannot disagree about the same cut. Three rules carry a guard the table alone cannot express, applied by the probe and named here in `unless` so it is written down beside the number: @@ -32,7 +33,7 @@ `tests/test_assessment_rules.py` pins those quotes to this table. """ -from .tasks.joins import LEVEL_SPREAD_WARN_DB +from .tasks.joins import INNER_LEVEL_STEP_WARN_DB, LEVEL_SPREAD_WARN_DB SEVERITIES = ("info", "warn") @@ -67,7 +68,7 @@ "probe": "analyze_seams", "field": "level_step_db", "comparator": ">", - "threshold": 3.0, + "threshold": INNER_LEVEL_STEP_WARN_DB, "severity": "warn", "says": "the shots either side of the seam sit this far apart in level", }, diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index ab21243e..920ca121 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -13,11 +13,13 @@ import numpy from ..dsp import as_channels_samples, level_dbfs -from ..events import emit_log, emit_warning +from ..events import emit_log, emit_warning, get_context from ..media_types import AudioVideo from ..shots import measured_num_samples, nested_shots, shot_record, trimmed_shots from ..task_domains import frames_to_samples +from .assess import sample_span from .joins import ( + INNER_LEVEL_STEP_WARN_DB, bleed_join, equal_power_crossfade_join, fit_audio_to_frames, @@ -238,20 +240,21 @@ def _prepare_inputs(videos, match_levels, match_levels_dbfs, sample_rate): return names, videos, clips, waveforms, sample_rate -INNER_LEVEL_STEP_WARN_DB = 3.0 - - def _warn_inner_level_steps(videos, names, waveforms, measure): """Say so when match_levels levels a multi-shot input as one unit and its - own shots differ by more than assess_output's seam_level_step threshold: + own shots differ by more than assess_output's seam_level_step threshold (shared with it), placing + shots with assess.sample_span: the step between them survives the join (#876).""" + # A later pair_audio replaces the track, so the step is never heard (#800) + if get_context().audio_replaced_downstream: + return for video, name, waveform in zip(videos, names, waveforms): shots = getattr(video, "shots", None) if waveform is None or not shots or len(shots) < 2: continue levels = [] for shot in shots: - start, count = shot.get("start_sample"), shot.get("num_samples") + start, count, _ = sample_span(shot, video) if start is None or not count: continue level = level_dbfs(waveform[:, start : start + count], measure) diff --git a/dw/tasks/joins.py b/dw/tasks/joins.py index 488962c4..041ae571 100644 --- a/dw/tasks/joins.py +++ b/dw/tasks/joins.py @@ -507,6 +507,10 @@ def crossfade_concat(waveforms, sample_rate, crossfade_ms, starts=None): LEVEL_SPREAD_WARN_DB = 6.0 +# The level step between a multi-shot input's own shots that match_levels cannot +# level (#876); assessment_rules' seam_level_step reads the same number +INNER_LEVEL_STEP_WARN_DB = 3.0 + # Mirrors audio_qc.py's NEAR_SILENT_WARN_DBFS: the same mean/rms level a job's # own near-silent check treats as having no real content. Gaining an input # already this quiet up to the target raises a noise floor rather than diff --git a/tests/test_concat_videos.py b/tests/test_concat_videos.py index af777a77..f76c0d1c 100644 --- a/tests/test_concat_videos.py +++ b/tests/test_concat_videos.py @@ -615,6 +615,34 @@ def test_an_even_joined_input_draws_no_warning(self): assert not found + def test_a_trimmed_joined_input_still_warns(self): + inner = concat_videos([audio_video(4, 0.5), audio_video(4, 0.1)]) + inner.fps = 4 + for shot in inner.shots: + shot.pop("start_sample", None) + shot.pop("num_samples", None) + + found = self._events( + lambda: concat_videos([inner, audio_video(4, 0.3)], match_levels="rms") + ) + + assert found + + def test_a_track_replaced_downstream_draws_no_warning(self): + from dw.events import RunContext, activate_context, deactivate_context + + inner = concat_videos([audio_video(4, 0.5), audio_video(4, 0.1)]) + events = [] + context = RunContext(on_event=events.append) + context.audio_replaced_downstream = True + token = activate_context(context) + try: + concat_videos([inner, audio_video(4, 0.3)], match_levels="rms") + finally: + deactivate_context(token) + + assert not [e for e in events if "inner_level_step" in str(e)] + class TestWarningsReachTheCaller: """A warning that only reaches the server's log does not exist from From 61838a9c7139d256ed08b710a17eb82f4c06ab7c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:25:09 -0500 Subject: [PATCH 042/204] fix(check-script): #873 - validate a lines[].shot name against the recorded shot map; accept the short @name Co-Authored-By: Claude Sonnet 5.5 --- dw/script_lines.py | 38 +++++++++++++++++-- dw/shot_span_preflight.py | 45 +++++++++++++++++++++- dw/tasks/script_check.py | 6 ++- dw/validation.py | 8 +++- tests/test_script_check.py | 18 +++++++++ tests/test_shot_span_preflight.py | 63 ++++++++++++++++++++++++++++++- tests/test_validation.py | 1 + 7 files changed, 171 insertions(+), 8 deletions(-) diff --git a/dw/script_lines.py b/dw/script_lines.py index 5542f842..8922518d 100644 --- a/dw/script_lines.py +++ b/dw/script_lines.py @@ -187,10 +187,21 @@ def parse_shots(shots): return list(shots) or None +def matching_shot_names(shot, names): + """The names in `names` a line's `shot` addresses: `shot` itself when + the map holds it, else every name carrying it as a `@` key + (`cold_open` for `shot@cold_open` or `...-shot@cold_open.0-0.0.mp4`).""" + if shot in names: + return [shot] + pattern = re.compile(r"@" + re.escape(shot) + r"(?![A-Za-z0-9_-])") + return list(dict.fromkeys(name for name in names if pattern.search(name or ""))) + + def shot_names_error(parsed_lines, records, source="argument"): """The refusal for a line naming a shot `records` does not hold once - - unknown, or a name `shots.duplicate_shot_names` finds repeated - or - None. The message lists the shots it does hold.""" + unknown, a short `@` name more than one shot carries, or a name + `shots.duplicate_shot_names` finds repeated - or None. The message lists + the shots it does hold.""" names = [record.get("name") for record in records] known = ", ".join(repr(name) for name in names) duplicated = { @@ -200,15 +211,34 @@ def shot_names_error(parsed_lines, records, source="argument"): shot = line["shot"] if shot is None: continue - if shot not in names: + matches = matching_shot_names(shot, names) + if not matches: return ( f"{COMMAND} 'lines'[{index}] names shot {shot!r}, which the take" f" does not have - its shots ({source}) are {known}" ) - if shot in duplicated: + if len(matches) > 1: + return ( + f"{COMMAND} 'lines'[{index}] names shot {shot!r}, which more" + f" than one of the take's shots ({source}) carries - name one" + f" in full: {', '.join(repr(name) for name in matches)}" + ) + if matches[0] in duplicated: return ( f"{COMMAND} 'lines'[{index}] names shot {shot!r}, which the" f" take's shot map ({source}) holds more than once - a line can" f" only name a shot the map holds once: {known}" ) return None + + +def with_full_shot_names(parsed_lines, records): + """`parsed_lines` with each line's short shot name replaced by the full + name `records` holds. Call after `shot_names_error` found nothing.""" + names = [record.get("name") for record in records] + resolved = [] + for line in parsed_lines: + shot = line["shot"] + matches = matching_shot_names(shot, names) if shot is not None else [] + resolved.append(dict(line, shot=matches[0]) if matches else line) + return resolved diff --git a/dw/shot_span_preflight.py b/dw/shot_span_preflight.py index 659d3acc..f8956fbe 100644 --- a/dw/shot_span_preflight.py +++ b/dw/shot_span_preflight.py @@ -24,6 +24,8 @@ from .media import probe_metadata from .probe_paths import resolve_probe_path from .references import iter_steps +from .script_lines import parse_lines, shot_names_error +from .runs import shots_beside PROBE_COMMANDS = ("analyze_shots", "analyze_seams", "analyze_sync_drift") @@ -103,4 +105,45 @@ def shot_span_warnings( return warnings -__all__ = ["shot_span_warnings"] +def script_shot_name_errors(workflow_definition, source_indices=None, base_dir=None): + """Every `check_script` step with a literal `lines` entry naming a shot + that the shot map recorded beside its statically-resolvable `audio` + (a manifest, via `shots_beside`) does not hold, as [{path, message}] + (#873). The run refuses the same line with the same message, but only + after queueing and loading Whisper. A `shots` argument, a + `previous_result:` audio or an unreadable manifest is left to the run.""" + errors = [] + for _, _, task, source, where in iter_steps( + workflow_definition.get("steps"), source_indices, "task" + ): + if task.get("command") != "check_script": + continue + task_args = task.get("arguments") + if not isinstance(task_args, dict) or task_args.get("shots"): + continue + try: + parsed_lines = parse_lines(task_args.get("lines")) + except ValueError: + continue # task_argument_errors reports a malformed `lines` + if not any(line["shot"] for line in parsed_lines): + continue + path = resolve_probe_path(task_args.get("audio"), base_dir, "an audio argument") + if path is None: + continue + records = shots_beside(path) + if not records: + continue + refusal = shot_names_error(parsed_lines, records, "manifest") + if refusal: + errors.append( + { + "path": render_path( + ("steps", source, "task", "arguments", "lines") + ), + "message": f"{refusal}{where}", + } + ) + return errors + + +__all__ = ["script_shot_name_errors", "shot_span_warnings"] diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 1832b78f..aecb5328 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -57,6 +57,7 @@ parse_lines, parse_shots, shot_names_error, + with_full_shot_names, ) from .assess import DEAD_AIR_WINDOW, resolve_shots, sample_span from .audio_transcription import WHISPER_DEFAULT_MODEL @@ -620,7 +621,9 @@ def check_script( audio: The take - a path, 'asset:'/'output:' reference, or an earlier step's audio or video (its soundtrack is taken). lines: The expected lines in order - a list of strings or - {text, shot}, 'shot' naming the take's shot the line belongs in. + {text, shot}, 'shot' naming the take's shot the line belongs in - + its full recorded name, or the short '@name' of exactly one shot + ('cold_open' for 'shot@cold_open'). H3 markup ([English] ..., , , [unclear], (S1)) is stripped, and a markup word heard spoken is 'tag_spoken'. [] means no speech is expected. @@ -655,6 +658,7 @@ def check_script( refusal = shot_names_error(parsed_lines, records, shots_source) if refusal: raise ValueError(refusal) + parsed_lines = with_full_shot_names(parsed_lines, records) waveform, rate = waveform_and_rate(audio, sample_rate, COMMAND) from .audio_transcription import transcribe_audio diff --git a/dw/validation.py b/dw/validation.py index 19f55ae6..2d309ceb 100644 --- a/dw/validation.py +++ b/dw/validation.py @@ -52,7 +52,7 @@ from .scalar_result_validation import scalar_result_errors from .schema import load_schema, validate_data_all from .security import InvalidInputError, SecurityError -from .shot_span_preflight import shot_span_warnings +from .shot_span_preflight import script_shot_name_errors, shot_span_warnings from .slice_preflight import slice_past_end_warnings from .subfolders import subfolder_errors from .step_value_checks import ( @@ -406,6 +406,12 @@ def _task_errors(context): c.expanded, c.source_indices, c.base_dir, probe=c.probe ), ), + # A check_script line naming a shot the take's recorded shot map lacks + # (dw/shot_span_preflight.py, #873) + Check( + "script_shot_names", + lambda c: script_shot_name_errors(c.expanded, c.source_indices, c.base_dir), + ), # A join_windows list not the length its statically-resolvable source # needs (dw/window_count_errors.py, #601) Check( diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 5728204c..b228a07c 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -18,6 +18,7 @@ parse_lines, parse_shots, shot_names_error, + with_full_shot_names, strip_markup, ) from dw.tasks import script_check @@ -841,6 +842,23 @@ def test_known_names_and_untagged_lines_pass(self): lines = parse_lines([{"text": "x", "shot": "B"}, "y"]) self.assertIsNone(shot_names_error(lines, self.RECORDS)) + def test_a_short_member_name_matches_the_one_shot_carrying_it(self): + records = [ + {"name": "x-shot@cold_open.0-0.0.mp4"}, + {"name": "x-shot@cold_open_two.0-0.0.mp4"}, + ] + lines = parse_lines([{"text": "x", "shot": "cold_open"}]) + self.assertIsNone(shot_names_error(lines, records)) + resolved = with_full_shot_names(lines, records) + self.assertEqual(resolved[0]["shot"], "x-shot@cold_open.0-0.0.mp4") + + def test_a_short_name_two_shots_carry_is_refused(self): + records = [{"name": "a-shot@open.mp4"}, {"name": "b-shot@open.mp4"}] + message = shot_names_error( + parse_lines([{"text": "x", "shot": "open"}]), records + ) + self.assertIn("more than one", message) + def test_parse_shots(self): self.assertIsNone(parse_shots(None)) self.assertIsNone(parse_shots([])) diff --git a/tests/test_shot_span_preflight.py b/tests/test_shot_span_preflight.py index 85b04d48..a49b3300 100644 --- a/tests/test_shot_span_preflight.py +++ b/tests/test_shot_span_preflight.py @@ -11,7 +11,7 @@ from dw import validation from dw.media import probe_metadata from dw.runs import activate_output_root, deactivate_output_root -from dw.shot_span_preflight import shot_span_warnings +from dw.shot_span_preflight import script_shot_name_errors, shot_span_warnings from dw.workflow import workflow_from_definition from tests.test_assess import write_mp4 @@ -174,3 +174,64 @@ def test_reachable_from_the_workflow_method(self, monkeypatch, tmp_path): assert len(warnings) == 1 assert "'b'" in warnings[0] + + +class TestScriptShotNames: + def _take(self, tmp_path): + import json + + from dw.runs import MANIFEST_FILE_NAME + from dw.shots import shot_record + + output_root = tmp_path / "out" + run_dir = output_root / "ep" / "20260101-000000-abc" + (run_dir / "final").mkdir(parents=True) + write_mp4(str(run_dir / "final" / "cut.mp4"), frames=48, fps=24) + records = [ + shot_record("clip-shot@cold_open.0-0.0.mp4", 0, 24), + shot_record("clip-shot@reveal.0-0.0.mp4", 24, 24), + ] + manifest = { + "steps": [{"step": "join", "files": ["final/cut.mp4"], "shots": records}] + } + (run_dir / MANIFEST_FILE_NAME).write_text(json.dumps(manifest)) + return str(output_root) + + def _definition(self, shot): + return { + "id": "check", + "steps": [ + { + "name": "check", + "task": { + "command": "check_script", + "arguments": { + "audio": "output:ep/20260101-000000-abc/final/cut.mp4", + "lines": [{"text": "hello", "shot": shot}], + }, + }, + "result": {"content_type": "application/json"}, + } + ], + } + + def _errors(self, tmp_path, shot): + token = activate_output_root(self._take(tmp_path)) + try: + return script_shot_name_errors(self._definition(shot)) + finally: + deactivate_output_root(token) + + def test_a_wrong_shot_name_is_an_error_at_validate_time(self, tmp_path): + errors = self._errors(tmp_path, "nope") + assert len(errors) == 1 + assert "does not have" in errors[0]["message"] + assert "cold_open" in errors[0]["message"] + + def test_the_short_name_and_the_full_name_pass(self, tmp_path): + token = activate_output_root(self._take(tmp_path)) + try: + for shot in ("cold_open", "clip-shot@reveal.0-0.0.mp4"): + assert script_shot_name_errors(self._definition(shot)) == [] + finally: + deactivate_output_root(token) diff --git a/tests/test_validation.py b/tests/test_validation.py index a19f7224..057fe61a 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -192,6 +192,7 @@ def test_validation_does_not_import_the_workflow_module(): "adapters", "task_argument_domains", "dissolve_frames", + "script_shot_names", "window_count", "video_sizes", "select", From 21a6f602035fbd4806e23c21193346ba2d133146 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:37:18 -0500 Subject: [PATCH 043/204] docs: #863 #864 - H3 base-model quality path, measured speed levers, series cinema path Co-Authored-By: Claude Sonnet 5.5 --- docs/ACCELERATION.md | 2 +- docs/RECIPES_24GB.md | 2 +- plugins/dw/skills/minimax-h3/SKILL.md | 5 +- .../references/checkpoints-and-loras.md | 11 ++- .../references/quality-and-speed.md | 96 +++++++++++++++++++ plugins/dw/skills/series-episodes/SKILL.md | 4 +- .../series-episodes/references/cinema-path.md | 23 +++++ 7 files changed, 135 insertions(+), 8 deletions(-) create mode 100644 plugins/dw/skills/minimax-h3/references/quality-and-speed.md create mode 100644 plugins/dw/skills/series-episodes/references/cinema-path.md diff --git a/docs/ACCELERATION.md b/docs/ACCELERATION.md index 1d64c59c..c05100e5 100644 --- a/docs/ACCELERATION.md +++ b/docs/ACCELERATION.md @@ -40,7 +40,7 @@ Simplest and broadest support. Compares first-block residuals to decide whether } ``` -Higher threshold = more speedup, more quality loss. Start with `0.05` and increase to taste. +Higher threshold = more speedup, more quality loss. Start with `0.05` and increase to taste. On MiniMax-H3 (RTX 3090, 1344x768, 4 references) `0.05` took 22% off a 30-step base-model job and nothing off a 12-step or turbo one, which it never skips; `torch.compile` there cost 289 s on the first pass for ~1% per pass, `text_kv` did nothing on Ref2VA, and `taylorseer` ran out of memory. **Example:** [step-caching.json](../workflows/templates/step-caching.json) diff --git a/docs/RECIPES_24GB.md b/docs/RECIPES_24GB.md index 5d1346f1..9f10f220 100644 --- a/docs/RECIPES_24GB.md +++ b/docs/RECIPES_24GB.md @@ -116,7 +116,7 @@ pipeline-level `offload`. The working 24GB configuration at 960x544: | `text_encoder` | `remove_modules: ["lm_head"]` - the encoder path never calls it | | `text_encoder.model` | SDNQ int4, `truncate_layers: {"language_model.layers": 51}`, `group_offload` `leaf_level` | | `vae`, `audio_vae` | SDNQ int8, `device: "cuda"`, `residency: "on_demand"` | -| pipeline | `cache: first_block` (`threshold: 0.1`) on the 20-step ref2va workflows only - measured on the 9-step turbo schedule it never skips (consecutive distilled steps differ too much for the threshold), so the turbo examples omit it rather than hold cache state for nothing | +| pipeline | `cache: first_block` (`threshold: 0.05`) only on a ~30-step base-model run (`lora_model_name: null`): -22% denoise there, identical staging and faces. It skips nothing at 12 steps or on a turbo schedule (consecutive distilled steps differ too much), and no template sets it today, so add it to the workflow. Measured on a 3090 with `sage_hub`; see the `minimax-h3` skill's `references/quality-and-speed.md` | The VAEs are the piece worth calling out. They hold roughly 3GiB, are used only to encode references and decode the result, and group offloading them is worse than useless diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index b824432e..e9b13691 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -70,7 +70,7 @@ arguments; prompt format is MiniMax's, from its text not here. `max(124, next 17n+5 ≥ lead + cut)`, the smallest `17n + 5` at or above `lead_frames + cut_frames`, at least 124; past 345, split the span. Before composing one, read `references/cuts.md`: the `shots` list (a music - video's six fields, plan then render), cost per entry, scoring and one voice across cuts. + video's six fields), cost per entry, scoring and one voice across cuts. - **Dialogue into a song**, not a concat: `slice_audio`, `join_into_song`, `pair_audio`, in that reference's last section. - **Unrelated shots, no cut**: `templates/minimax/shots-batch` - one H3 @@ -85,7 +85,7 @@ read the `workflows` guide's authoring section. ## Hard rules - `num_frames` is `17n + 5`, from 124 to 345, at a fixed 24 fps: 5.17 to 14.4 - seconds in one clip. Most default to 124 for fast iteration (storyboard 192); + seconds in one clip. Most default to 124 (storyboard 192); `num_frames=345` is the full length, and fits 24 GB at 544p only (#271). The 5-second floor is diffusers'; the model card says 4. - Canvas: 768-pixel short edge, at most 768x1344, in multiples of 32, aspect @@ -97,6 +97,7 @@ read the `workflows` guide's authoring section. Never put an FL2VA LoRA on a reference template: `ref2va` holds `transformer_ref` alone, so anything handed there degrades output. `validate_workflow` refuses it and warns on a `weight_name` naming neither. +- Smeary faces: int8 base model, `references/quality-and-speed.md`. - Fit a crowd's action to what it holds: candle-holders asked to clap rendered three-handed people. - A directed crowd move needs its landmark in frame: "marches up to the diff --git a/plugins/dw/skills/minimax-h3/references/checkpoints-and-loras.md b/plugins/dw/skills/minimax-h3/references/checkpoints-and-loras.md index dc510fe4..8e45ff66 100644 --- a/plugins/dw/skills/minimax-h3/references/checkpoints-and-loras.md +++ b/plugins/dw/skills/minimax-h3/references/checkpoints-and-loras.md @@ -6,7 +6,7 @@ A checkpoint comes with a canvas, a sigma shift and an alpha and they move together - change one, change all. `video_shift`, `audio_shift` and `lora_alpha` are variables everywhere, so a swap is arguments, not a file. -Four combinations are tested, plus one addition below, nothing else: +Four combinations are tested, plus the base model and one addition below, nothing else: **default** - 4-step FL2VA v1.2 (`minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors`), shift **6**/3, `num_inference_steps=5`, alpha unset, on every T2VA/FL2VA template at 960x544 and 1344x768 (since 2026-10-10; trained at 768p, @@ -19,11 +19,18 @@ Swap file, shift and steps together. The 544p 4-step v0.1 file renders noise. v1.2 records alpha 8; v1.0/v1.1 record 128 - never state an alpha the header doesn't. +**Base, no LoRA**: `lora_model_name: null`, int8, 12-30 steps (floor 12), shift +12/3 - the quality path for faces (4/5 against 2-3/5 on turbo or int4); on a +24 GB CUDA card 1344x768 takes ~13 min at 12 steps and ~23 min at 30 with +`first_block` 0.05. Numbers, caveats and prompting rules in +`references/quality-and-speed.md`. + **Realism People** (#585) as a second `loras` entry on a t2va step (`fal/MiniMax-H3-Realism-People-LoRA`, `h3-realism-people-t2v-i2v-r2v.safetensors`, scale 0.7, own `adapter_name`; a saved copy): validate warns the name fits neither partition, so never on -a `ref2va` template. +a `ref2va` template: its header says `base_model: minimax-h3-fl2va`, and +it is inert on `reference-to-video` whatever `r2v` the filename says. Acc PDD, HyperFlow and drozbay FastH3 need their own loaders, not `loras`: don't try them. Any other LoRA (a style, a motion fix): `list_loras` first - diff --git a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md new file mode 100644 index 00000000..7dac37bd --- /dev/null +++ b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md @@ -0,0 +1,96 @@ +# MiniMax H3: the base-model quality path and measured speed levers + +Part of the `minimax-h3` skill; its *Hard rules* say when to read this. + +Measured on a CUDA box (RTX 3090 24 GB, `sage_hub`), 1344x768, 4 references +(2 portraits, 2 voice clips), 124 frames, seed 42, #863. **These are CUDA +numbers**: MPS has no sage backend and runs several times slower, so take the +ordering, not the minutes. Face score is realism at native-resolution crops, +judged from frames by a model (5 = cinema-grade photographic). + +## Quality ladder + +Base model = `lora_model_name: null`, shift video 12 / audio 3. Turbo = the +shift the LoRA names. + +| Rung | Canvas | Weights | LoRA | Steps | Attn | Min | Face /5 | +|---|---|---|---|---|---|---|---| +| L0 | 960x544 | int4 | ref2v 4-step v0.1 | 5 | sage_hub | 3.5 | 2 | +| L1 | 1344x768 | int4 | ref2v 4-step v0.1 | 5 | sage_hub | 5.4 | 2.5 | +| L7 | 1344x768 | int8 | ref2v 4-step v0.1 | 5 | sage_hub | 7.2 | 3 | +| L3 | 1344x768 | int8 | ref2v 8-step v1.0 768p | 9 | sage_hub | 11.2 | 3 | +| L4 | 1344x768 | int8 | ref2v 8-step v1.0 768p | 9 | native | 14.5 | 3 | +| B4 | 960x544 | int4 | none | 12 | sage_hub | 6.6 | 3 | +| B1 | 960x544 | int4 | none | 30 | sage_hub | 15.1 | 3 | +| B5 | 1344x768 | int4 | none | 12 | sage_hub | 11.8 | 3.5 | +| B6 | 1344x768 | int4 | none | 8 | sage_hub | 8.5 | 1.5 (duplicated a character) | +| B3 | 1344x768 | int8 | none | 12 | sage_hub | 12.9 | 3.5 | +| L5 | 1344x768 | int8 | none | 30 | sage_hub | 29.5 | 4 | +| L6 | 1344x768 | int8 | none | 30 | native | 44.2 | 4 | +| P1 | 1344x768 | int8 | none | 30 | sage_hub + `first_block` 0.05 | 23.1 | 4 | +| C1 / C2 (close-up) | 1344x768 | int8 | none | 12 / 30 | sage_hub | 13.9 / 22.8 | 4 / 4 | + +- **Faces**: turbo LoRAs plus int4 cause the smeary faces, not resolution. The + base model at int8 fixes it: `lora_model_name: null`, 12-30 steps, shift + 12/3. A close-up is already 4/5 at 12 steps. +- **Composition** (who stands where): attention never moves it; quantization + doesn't on a turbo LoRA (L1 = L7) but does on the base model (int4 vs + int8); a different LoRA family re-stages; base step count doesn't. +- **Steps**: the base floor is 12. 8 steps duplicated a character (B6). +- **Two-character shots at 960x544 smear** (~60 px faces): render them at + 1344x768 from the start, not at 544 and upscaled. +- **Draft then final**: a 12-step draft (~13 min) of the wide, then the + 30-step final with `first_block` 0.05 (~23 min). Draft to final carries + blocking but not always props. +- **One int8 job at a time**: host RAM. +- **Frame ceiling** at 1344x768 with 4 references is 124; 141 was refused + (23.9 GB > 23.6). Write to 124 or drop a reference. + +## Prompting the base model + +- It invents an extra person unless every subject is placed explicitly + ("Howie at frame left, Pat at the right end of the desk"). A head-count + sentence ("exactly two people") alone failed three times. +- Name the set dressing (chair, desk items) in every shot of a sequence, or + each shot draws its own set: a chair changed between two close-ups. + +## What does not help + +- **Realism People LoRA**: its header says `base_model: minimax-h3-fl2va`, + `training: text_to_video`, rank 32. It is inert on `reference-to-video` + (Ref2VA) despite `r2v` in the filename. It works on FL2VA/I2VA (4/5) but + those lose the voice and identity references. +- **LTX-2.5 refine-in-place on an H3 shot** is not a face fix: cleaner edges, + waxy skin, identity drift (2.5/5, 2/5). + +## Speed levers + +Per phase, from the job event log (minutes unless noted): + +| Rung | Change | Denoise | s/pass | Total | vs baseline | +|---|---|---|---|---|---| +| B3 | 12 steps | 10.24 | 55.8 | 12.89 | | +| P2 | + `first_block` 0.05 | 10.31 | 56.3 | 12.79 | 0% | +| P4 | + `text_kv` | 10.24 | 55.9 | 12.69 | ~0% | +| P3 | + compile `repeated_blocks` | 14.02 | 288.6 first, 55.2 rest | 16.58 | +29% | +| L5 | 30 steps | 26.97 | 55.8 | 29.49 | | +| P1 | + `first_block` 0.05 | 20.61 | 42.2 avg | 23.06 | **-22%** | +| P5 | + `taylorseer` interval 3 order 1 | | | OOM at 2 min | | + +Denoise is 80-91% of a job, ~56 s/pass with int8 weights streamed by +block-level group offload: compute-bound, not PCIe-bound. + +- **`sage_hub`**: -23% at turbo 8-step, -33% at base 30, visually lossless. + Advise it first where the box has it (`attention_backend=sage_hub`). +- **`first_block` 0.05**: -22% at 30 base steps with identical staging and + faces (PSNR 27.5 dB against L5). It skips nothing at 12 steps or on a + turbo schedule: use it at 30 base steps only. +- **`torch.compile`**: +289 s on the first pass, ~1% per pass after. A net + loss for one job; it breaks even only across ~16 shots at one shape in one + warm worker. +- **`text_kv`**: no measurable effect on Ref2VA. +- **`taylorseer`**: OOM at this canvas with 4 references. + +Open: `first_block` 0.08-0.10 at 30 steps (speed against face cost); +`first_block` at 20 steps against plain 30; whether P1 holds on a close-up and +a second seed; MagCache needs a calibration run (no H3 preset). diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index f615f1f5..5d16ee20 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -138,8 +138,8 @@ composes into, not something to re-derive: - **pair**: the normalized track is muxed onto the cut - the episode's deliverable. -One `assemble-and-score` run per episode; each episode's `total_frames` and -`shots` list are its own. +For smeary two-shot faces, read `references/cinema-path.md` (1344x768 base +model); choose the path once, at episode 1. ## Run and judge diff --git a/plugins/dw/skills/series-episodes/references/cinema-path.md b/plugins/dw/skills/series-episodes/references/cinema-path.md new file mode 100644 index 00000000..a4649dd7 --- /dev/null +++ b/plugins/dw/skills/series-episodes/references/cinema-path.md @@ -0,0 +1,23 @@ +# Series episodes: the cinema path + +The default `dialogue-short` shots are the TV path (960x544, turbo), whose +two-character faces smear (~60 px). The cinema path renders every shot at +1344x768 on the base model at int8, no LoRA; the `minimax-h3` skill's +quality-and-speed reference owns the settings, timings (CUDA +measurements), the `first_block` lever and the base-model prompt rules - read +it, not restated here. What a multi-shot episode adds: + +- Wides as a 12-step draft, then a 30-step final; close-ups at 12 steps. + One int8 job at a time. A five-shot episode ran ~156 GPU-min with three + rejected drafts. +- Place every subject explicitly in every shot and name the set dressing, or + the base model adds a person and each shot draws its own chair. +- Four references at 1344x768 cap a shot at 124 frames: write tags to 124 or + drop a reference. +- Assemble at full size, no `fit`: `concat_videos` with `trim_frames: 0`, + `audio_bleed_ms: 0`, `match_levels: "rms"`, a looped room-tone bed, then + `normalize_audio` peak -1 (one episode: -16.7 LUFS). +- TV-path and cinema-path episodes look like different shows back to back: + choose the path once, at episode 1, and keep it unless the user says + otherwise. +- Choose the path once, at episode 1: TV-path and cinema-path episodes look like different shows back to back. From 510c47fd495f3f0e91c005656f7959835bd0c048 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:43:34 -0500 Subject: [PATCH 044/204] fix(tasks): #874 - no_loop_bed names the rule that removed the quiet windows, with the tonal closest miss Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/loop_bed.py | 58 ++++++++++++++++++++++++++++++++----- tests/test_find_loop_bed.py | 19 ++++++++++++ 2 files changed, 69 insertions(+), 8 deletions(-) diff --git a/dw/tasks/loop_bed.py b/dw/tasks/loop_bed.py index 117225b1..cd436d83 100644 --- a/dw/tasks/loop_bed.py +++ b/dw/tasks/loop_bed.py @@ -356,10 +356,16 @@ def _looped(segment, sample_rate, crossfade_ms, loop_seconds): } -def _empty_finding(rejected, criteria): +def _empty_finding(rejected, criteria, closest_tonal=None): """The one finding for a search that found nothing, naming the rule that threw the most out and the argument that relaxes it. + Rules tally in order, so `too_loud` counts every window above the level + limits, however many quiet windows then failed a later rule. Once any + window passed the level test, raising the level limits cannot help and + the dominant of the later rules is named instead. `closest_tonal` is the + nearest any tonal-rejected window came to the thresholds. + A window that crosses a shot boundary is counted under `shot_boundary` alone, so the other rules tally in-shot windows only. When any in-shot window existed the dominant of those is named: crossing windows @@ -369,7 +375,9 @@ def _empty_finding(rejected, criteria): rule = None else: in_shot = {n: c for n, c in rejected.items() if n != "shot_boundary"} - pool = in_shot if any(in_shot.values()) else rejected + quiet = {n: c for n, c in in_shot.items() if n != "too_loud"} + pool = quiet if any(quiet.values()) else in_shot + pool = pool if any(pool.values()) else rejected rule = max(pool, key=lambda name: pool[name]) message = { "too_loud": ( @@ -396,10 +404,25 @@ def _empty_finding(rejected, criteria): "tonal": ( "every quiet window is tonal or speech-like - faint " "programme that would be audible once it repeats; widen " - "the range" + "the range" + _closest_note(closest_tonal, criteria) ), }[rule] - return {"rule": "no_loop_bed", "rejected_by": rule, "message": message} + finding = {"rule": "no_loop_bed", "rejected_by": rule, "message": message} + if closest_tonal is not None: + finding["closest_tonal"] = closest_tonal + return finding + + +def _closest_note(closest, criteria): + """How near the tonal-rejected windows came to the thresholds, in words.""" + if closest is None: + return "" + return ( + f" (nearest misses: flatness {closest['flatness']:.3f} against the " + f"{criteria['tonal_flatness']:g} floor, harmonicity " + f"{closest['harmonicity']:.3f} against the {criteria['harmonicity']:g} " + "ceiling, each on a different window)" + ) def _coerce_arguments(shots, **raw): @@ -523,7 +546,10 @@ def _scan_windows( lengths, args, ): - """Judge every (start, length) window; returns (survivors, rejected). + """Judge every (start, length) window; returns (survivors, rejected, closest). + + `closest` holds the highest flatness and lowest harmonicity any + tonal-rejected window reached (empty with none). Vectorised per window length: every pair is judged at once and counted under the first rule it fails - loudness, silence, ticks, tonality - so @@ -539,6 +565,7 @@ def _scan_windows( if bin_shot is not None: rejected["shot_boundary"] = 0 survivors = [] # (source ripple, start bin, length in bins, readings) + closest = {} for length in range(shortest, longest + 1): windows = _sliding(power, length) # (starts, length) loudest = windows.max(axis=1) @@ -571,6 +598,17 @@ def _scan_windows( inside = _sliding(block_tonal, length - block_bins + 1) any_tonal |= inside[: windows.shape[0]].any(axis=1) tonal = ~crossing & ~loud & ~silent & ~ticked & any_tonal + if tonal.any(): + rows = numpy.flatnonzero(tonal) + for bins, flat, harm, _ in scales: + flat_in = _sliding(flat, length - bins + 1)[rows] + harm_in = _sliding(harm, length - bins + 1)[rows] + closest["flatness"] = max( + closest.get("flatness", 0.0), float(flat_in.min(axis=1).max()) + ) + closest["harmonicity"] = min( + closest.get("harmonicity", 1.0), float(harm_in.max(axis=1).min()) + ) rejected["too_loud"] += int(loud.sum()) rejected["silent"] += int(silent.sum()) rejected["spike"] += int(ticked.sum()) @@ -607,7 +645,7 @@ def _scan_windows( }, ) ) - return survivors, rejected + return survivors, rejected, closest def _rank_and_loop( @@ -683,6 +721,7 @@ def _answer( last, shots_source, spans, + closest=None, ): """The command's JSON answer, with the finding when nothing was found.""" criteria = { @@ -697,7 +736,9 @@ def _answer( "max_seconds": args.max_seconds, "target_bed_dbfs": args.target_bed_dbfs, } - findings = [] if candidates else [_empty_finding(rejected, criteria)] + findings = ( + [] if candidates else [_empty_finding(rejected, criteria, closest or None)] + ) logger.info( f"{COMMAND}: {survivor_count} windows passed level and tick tests, " f"{kept_count} looped, {len(candidates)} returned; rejected {rejected}" @@ -843,7 +884,7 @@ def find_loop_bed( searched, bin_length, sample_rate, bin_seconds, count, shortest, usable ) edges = _edge_ticks(power, peaks, bin_threshold) - survivors, rejected = _scan_windows( + survivors, rejected, closest = _scan_windows( power, peaks, bin_shot, scales, edges, (shortest, longest), args ) kept, candidates = _rank_and_loop( @@ -861,4 +902,5 @@ def find_loop_bed( last, shots_source, spans, + closest, ) diff --git a/tests/test_find_loop_bed.py b/tests/test_find_loop_bed.py index 88ca7508..ced15c45 100644 --- a/tests/test_find_loop_bed.py +++ b/tests/test_find_loop_bed.py @@ -216,6 +216,25 @@ def test_no_candidate_covers_the_tone(self): assert result["rejected"]["tonal"] >= 1 +class TestTonalFinding: + def test_names_tonal_when_loud_windows_outnumber_it(self): + rng = numpy.random.default_rng(3) + waveform = noise(rng, 20 * SR, -20.0) + t = numpy.arange(2 * SR) / SR + amplitude = rms_amplitude(-78.0) * math.sqrt(2) + waveform[8 * SR : 10 * SR] = amplitude * numpy.sin(2 * math.pi * 220.0 * t) + + result = find_loop_bed(track(waveform), min_seconds=0.5, max_seconds=1.0) + + assert result["candidates"] == [] + assert result["rejected"]["too_loud"] > result["rejected"]["tonal"] > 0 + finding = result["findings"][0] + assert finding["rejected_by"] == "tonal" + assert finding["closest_tonal"]["flatness"] < 0.3 + assert finding["closest_tonal"]["harmonicity"] >= 0.45 + assert "nearest misses" in finding["message"] + + class TestIntermittentVoice: """Speech under a bed comes and goes: 250 ms harmonic syllables every 400 ms, at the bed's own level. Measured over a whole 0.5-2 s window the From c1b0d2dac0f754c6cfd20896b8119b8799b9572d Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 19:46:25 -0500 Subject: [PATCH 045/204] docs: #880 - drop the repeated last bullet in series-episodes cinema-path.md Co-Authored-By: Claude Sonnet 5.5 --- plugins/dw/skills/series-episodes/references/cinema-path.md | 1 - 1 file changed, 1 deletion(-) diff --git a/plugins/dw/skills/series-episodes/references/cinema-path.md b/plugins/dw/skills/series-episodes/references/cinema-path.md index a4649dd7..2ae5ca20 100644 --- a/plugins/dw/skills/series-episodes/references/cinema-path.md +++ b/plugins/dw/skills/series-episodes/references/cinema-path.md @@ -20,4 +20,3 @@ it, not restated here. What a multi-shot episode adds: - TV-path and cinema-path episodes look like different shows back to back: choose the path once, at episode 1, and keep it unless the user says otherwise. -- Choose the path once, at episode 1: TV-path and cinema-path episodes look like different shows back to back. From f8fda594eab40670c7626ff635cb1cb3e2deb6e3 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 20:19:39 -0500 Subject: [PATCH 046/204] fix(check-script): #873 - member-key rule lives in references.member_keys; map names the shot-name preflight Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/references.py | 22 ++++++++++++++++++++++ dw/script_lines.py | 4 ++-- tests/test_script_check.py | 7 +++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d59530b6..4eb436de 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -129,7 +129,7 @@ one to open. | Opening media | `dw/media.py` | This is the only module that calls `av.open`. It imports no torch and no step types. | `tests/test_media_layering.py::test_av_open_appears_only_in_media`, `tests/test_media_layering.py::test_media_imports_no_step_types` (no torch: —) | | Signal processing | `dw/dsp.py` | It measures and transforms waveforms, decides nothing, and imports nothing from `dw`. | `tests/test_media_layering.py::test_dsp_imports_nothing_from_dw` | | Assessment probes | `dw/tasks/assess.py`, `dw/assessment_rules.py` | Exactly three registered commands are assessment probes. Each measures a finished file, and every rule names a real probe field. | `tests/test_assessment_rules.py::TestRuleProbesAreRealCommands::test_assessment_is_exactly_the_three_probes`, `tests/test_assessment_rules.py::TestRuleProbesAreRealCommands::test_every_rule_names_a_registered_json_command` | -| Script check | `dw/tasks/script_check.py`: `check`; `dw/script_lines.py`: `parse_lines`, `parse_shots`, `shot_names_error` | `check_script` aligns a take's heard words to its expected lines and guards out words heard over silence and repetition loops (Whisper's hallucination over music). It builds its findings with `assessment_rules.finding()` and reads the guard's floor and window (`DEAD_AIR_FLOOR_DBFS`, `DEAD_AIR_WINDOW`) rather than copying them; the loudest-window measure is `dsp.loudest_window_dbfs` (`script_check.word_level` wraps it), which `transcribe_audio`'s `suspect: "silent"` mark reads with the same floor and window; the repetition-loop detector (`script_lines.looped_texts`, `REPEAT_RUN_MIN`, `REPEAT_MAX_PERIOD`) lives in `dw/script_lines.py` and backs both `check_script`'s `repetition` discard and `transcribe_audio`'s `suspect: "repetition"`, and `script_lines.suspect_runs`/`SUSPECT_LEFT_OUT` are what `plan_cuts` groups and leaves out suspect chunks by - so neither task imports the other. It is not an assessment probe and adds nothing to `RULES`. Its literal `lines` is refused at validation through `task_problems` (`script_lines_errors`), which calls the same `parse_lines` the task runs, and a literal `shots` through the same `parse_shots` and `shot_names_error`; the parsers live in `dw/script_lines.py`, not the task, so that `task_problems` reaches them without importing `script_check` (the `import_cycles` ratchet in `scripts/arch_metrics.py`). Its shot map comes from `assess.resolve_shots`, the one the probes use, never a copy; it places each shot on the soundtrack with `assess.sample_span`, decides repeated shot names with `shots.duplicate_shot_names`, and asks `locations.is_http_url` whether the take is a URL. | `tests/test_script_check.py::TestRegistration::test_registered_as_json_command_without_assessment_flag`, `tests/test_script_check.py::TestEmptyLines::test_words_only_over_silence_are_discarded_not_found`, `tests/test_script_check.py::TestLinesErrors::test_literal_with_a_colon_is_not_a_reference` | +| Script check | `dw/tasks/script_check.py`: `check`; `dw/script_lines.py`: `parse_lines`, `parse_shots`, `shot_names_error` | `check_script` aligns a take's heard words to its expected lines and guards out words heard over silence and repetition loops (Whisper's hallucination over music). It builds its findings with `assessment_rules.finding()` and reads the guard's floor and window (`DEAD_AIR_FLOOR_DBFS`, `DEAD_AIR_WINDOW`) rather than copying them; the loudest-window measure is `dsp.loudest_window_dbfs` (`script_check.word_level` wraps it), which `transcribe_audio`'s `suspect: "silent"` mark reads with the same floor and window; the repetition-loop detector (`script_lines.looped_texts`, `REPEAT_RUN_MIN`, `REPEAT_MAX_PERIOD`) lives in `dw/script_lines.py` and backs both `check_script`'s `repetition` discard and `transcribe_audio`'s `suspect: "repetition"`, and `script_lines.suspect_runs`/`SUSPECT_LEFT_OUT` are what `plan_cuts` groups and leaves out suspect chunks by - so neither task imports the other. It is not an assessment probe and adds nothing to `RULES`. Its literal `lines` is refused at validation through `task_problems` (`script_lines_errors`), which calls the same `parse_lines` the task runs, and a literal `shots` through the same `parse_shots` and `shot_names_error`; the parsers live in `dw/script_lines.py`, not the task, so that `task_problems` reaches them without importing `script_check` (the `import_cycles` ratchet in `scripts/arch_metrics.py`). A line's `shot` may be the short `@name` of the one recorded shot that carries it (`script_lines.matching_shot_names`, `with_full_shot_names`, shared by the task and the literal-`shots` check); which member key a recorded name carries comes from `references.member_keys`, beside `MEMBER_SEPARATOR` and `for_each.member_name`, never a re-derived pattern. With `audio` an `output:`/`asset:`/path whose run manifest records shots, `dw/shot_span_preflight.py: script_shot_name_errors` (the `script_shot_names` error check) runs `shot_names_error` at validation, reading the map through `runs.shots_beside`. Its shot map comes from `assess.resolve_shots`, the one the probes use, never a copy; it places each shot on the soundtrack with `assess.sample_span`, decides repeated shot names with `shots.duplicate_shot_names`, and asks `locations.is_http_url` whether the take is a URL. | `tests/test_script_check.py::TestRegistration::test_registered_as_json_command_without_assessment_flag`, `tests/test_script_check.py::TestEmptyLines::test_words_only_over_silence_are_discarded_not_found`, `tests/test_script_check.py::TestLinesErrors::test_literal_with_a_colon_is_not_a_reference`, `tests/test_script_check.py::TestShotValidation::test_a_short_member_name_matches_the_one_shot_carrying_it`, `tests/test_shot_span_preflight.py::TestScriptShotNames::test_a_wrong_shot_name_is_an_error_at_validate_time` | | Audio sync against a source slice | `dw/tasks/measure_sync.py`: `measure_sync`; `dw/dsp.py`: `envelope_lag` | `measure_sync` cross-correlates the onset envelopes of a shot's audio and its source slice and answers the lag and a 0-1 confidence. A positive lag means the audio is LATE (its events fall after the reference's), negative early; `lag_seconds` is None for a silent or flat track. It builds its findings (`sync_unmeasurable`, `sync_low_confidence`, `audio_out_of_sync`) with `assessment_rules.finding()` over module-local rule dicts and adds nothing to `RULES`; it reads the silence floor from `beats.SILENT_DBFS`. `envelope_lag` imports nothing from `dw`. | `tests/test_measure_sync.py::TestLag::test_late_audio_is_a_positive_lag`, `tests/test_measure_sync.py::TestLag::test_early_audio_is_a_negative_lag`, `tests/test_measure_sync.py::TestUnmeasurable::test_a_silent_reference_has_no_lag` | ## Security and architecture guardrails diff --git a/dw/references.py b/dw/references.py index d0e37515..7d0a3561 100644 --- a/dw/references.py +++ b/dw/references.py @@ -103,6 +103,28 @@ def author_index(source_indices, index): # A for_each member is named `@` MEMBER_SEPARATOR = "@" +# A member's key is its entry's name (a variable name: letters, digits, '_', +# '-') or its index, so it ends at the first character outside this set +_MEMBER_KEY_CHARS = frozenset( + "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" +) + + +def member_keys(name): + """The for_each member keys a name carries: `cold_open` for + `shot@cold_open` and for a file named `...-shot@cold_open.0-0.0.mp4` + (the key ends where `member_name`'s key charset does). Empty for a name + with no separator.""" + if not isinstance(name, str): + return [] + keys = [] + for part in name.split(MEMBER_SEPARATOR)[1:]: + end = 0 + while end < len(part) and part[end] in _MEMBER_KEY_CHARS: + end += 1 + if end: + keys.append(part[:end]) + return keys def member_suffix(name): diff --git a/dw/script_lines.py b/dw/script_lines.py index 8922518d..8534b144 100644 --- a/dw/script_lines.py +++ b/dw/script_lines.py @@ -15,6 +15,7 @@ import re +from .references import member_keys from .shots import duplicate_shot_names COMMAND = "check_script" @@ -193,8 +194,7 @@ def matching_shot_names(shot, names): (`cold_open` for `shot@cold_open` or `...-shot@cold_open.0-0.0.mp4`).""" if shot in names: return [shot] - pattern = re.compile(r"@" + re.escape(shot) + r"(?![A-Za-z0-9_-])") - return list(dict.fromkeys(name for name in names if pattern.search(name or ""))) + return list(dict.fromkeys(name for name in names if shot in member_keys(name))) def shot_names_error(parsed_lines, records, source="argument"): diff --git a/tests/test_script_check.py b/tests/test_script_check.py index b228a07c..e863a85e 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -852,6 +852,13 @@ def test_a_short_member_name_matches_the_one_shot_carrying_it(self): resolved = with_full_shot_names(lines, records) self.assertEqual(resolved[0]["shot"], "x-shot@cold_open.0-0.0.mp4") + def test_member_keys_come_from_the_for_each_separator(self): + from dw.references import member_keys + + self.assertEqual(member_keys("shot@cold_open"), ["cold_open"]) + self.assertEqual(member_keys("x-shot@cold_open.0-0.0.mp4"), ["cold_open"]) + self.assertEqual(member_keys("plain.mp4"), []) + def test_a_short_name_two_shots_carry_is_refused(self): records = [{"name": "a-shot@open.mp4"}, {"name": "b-shot@open.mp4"}] message = shot_names_error( From 4515906bd035b9ae4e8b83b8fd214ef7ccd483af Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:45:38 -0500 Subject: [PATCH 047/204] feat(catalog): #883 - attention_backend variable on every H3 transformer template Co-Authored-By: Claude Opus 5.5 --- tests/test_catalog_structure.py | 97 ++++++++++++++++++- .../minimax/chain-matched-and-aligned.json | 4 +- .../minimax/chain-matched-to-audio.json | 4 +- .../minimax/chain-video-continuity.json | 4 +- .../templates/minimax/chained-segments.json | 4 +- .../minimax/composable-references.json | 4 +- .../templates/minimax/dialogue-short.json | 4 +- .../minimax/enhance-prompt-with-image.json | 4 +- .../templates/minimax/enhance-prompt.json | 4 +- .../minimax/first-and-last-frame.json | 4 +- .../minimax/generated-subject-reference.json | 4 +- .../templates/minimax/last-frame-only.json | 4 +- workflows/templates/minimax/music-video.json | 4 +- workflows/templates/minimax/shots-batch.json | 4 +- workflows/templates/minimax/storyboard.json | 4 +- .../minimax/voice-timbre-reference.json | 4 +- 16 files changed, 141 insertions(+), 16 deletions(-) diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index c9ff3ef9..4bf4773b 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -474,7 +474,11 @@ def test_no_stale_entry_in_the_allowlist(): # Then to 10_700 for the `attention_backend` variable on the four MiniMax-H3 # family baselines (2026-10-10), measured at 10_676 with six: the knob behind the # measured 28-33% per-step gain from sage_hub, null by default. -COMPACT_BUDGET = 10_700 +# Then to 10_800 for the same variable on every other MiniMax-H3 template (#865, +# stage #883, 2026-10-13), measured at 10_758: about 79 tokens across fifteen +# templates, so the cinema path's sage_hub is an argument on the shots, chains +# and episodes rather than a JSON edit. +COMPACT_BUDGET = 10_800 FILTERED_BUDGET = 1_500 @@ -573,6 +577,97 @@ def test_the_remote_code_check_sees_nested_keys(): ] +H3_MODEL = "MiniMaxAI/MiniMax-H3" + + +def _h3_steps(definition): + return [ + step + for step in definition.get("steps", []) + if (step.get("pipeline") or {}) + .get("from_pretrained_arguments", {}) + .get("model_name") + == H3_MODEL + ] + + +def _transformers(step): + components = step["pipeline"]["configuration"].get("components", {}) + # t2va/fl2va load `transformer`, ref2va loads `transformer_ref` + return {k: v for k, v in components.items() if k in ("transformer", "transformer_ref")} + + +H3_TEMPLATES = [path for path in TEMPLATES if _h3_steps(load(path))] + + +def test_there_are_h3_templates(): + assert len(H3_TEMPLATES) >= 19 + + +@pytest.mark.parametrize("path", H3_TEMPLATES) +def test_every_h3_transformer_step_wires_attention_backend(path): + """sage_hub is 23-33% faster per step and lossless on CUDA, and the only way + to reach it from `arguments` is a template variable wired to the + transformer (#865). Every H3 template exposes it, null by default so mps + and cpu keep the process default; one that doesn't makes the cinema path + a JSON edit again.""" + definition = load(path) + + assert definition["variables"].get("attention_backend", "missing") is None, ( + f"{path} declares no attention_backend variable defaulting to null" + ) + for step in _h3_steps(definition): + transformers = _transformers(step) + assert transformers, f"{path} step '{step['name']}' configures no transformer" + for name, component in transformers.items(): + assert component.get("attention_backend") == "variable:attention_backend", ( + f"{path} step '{step['name']}' does not wire {name}.attention_backend" + ) + + +def test_only_h3_steps_wire_attention_backend(): + """The Z-Image stills and the Music3 song beside an H3 shot keep the process + default: the variable reaches the H3 transformer and nothing else.""" + for path in H3_TEMPLATES: + definition = load(path) + h3 = {step["name"] for step in _h3_steps(definition)} + for step in definition["steps"]: + if step["name"] in h3: + continue + assert "attention_backend" not in json.dumps(step), ( + f"{path} step '{step['name']}' is not H3 but sets attention_backend" + ) + + +def _dialogue_short_after_substitution(**overrides): + from dw.variables import replace_variables + + definition = load( + os.path.join(REPO_ROOT, "workflows", "templates", "minimax", "dialogue-short.json") + ) + variables = {**definition["variables"], **overrides} + return { + step["name"]: replace_variables(step, variables) for step in definition["steps"] + } + + +def test_attention_backend_reaches_only_the_h3_shot(): + steps = _dialogue_short_after_substitution(attention_backend="sage_hub") + + shot = steps["shot"]["pipeline"]["configuration"]["components"]["transformer_ref"] + assert shot["attention_backend"] == "sage_hub" + for name, step in steps.items(): + if name != "shot": + assert "sage_hub" not in json.dumps(step), name + + +def test_a_default_attention_backend_drops_the_key(): + steps = _dialogue_short_after_substitution() + + shot = steps["shot"]["pipeline"]["configuration"]["components"]["transformer_ref"] + assert "attention_backend" not in shot + + def _step(definition, name): return next(step for step in definition["steps"] if step["name"] == name) diff --git a/workflows/templates/minimax/chain-matched-and-aligned.json b/workflows/templates/minimax/chain-matched-and-aligned.json index fcdd3bc4..0ecd6890 100644 --- a/workflows/templates/minimax/chain-matched-and-aligned.json +++ b/workflows/templates/minimax/chain-matched-and-aligned.json @@ -40,7 +40,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": 12345 + "seed": 12345, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -129,6 +130,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/chain-matched-to-audio.json b/workflows/templates/minimax/chain-matched-to-audio.json index a38fb395..01692b9a 100644 --- a/workflows/templates/minimax/chain-matched-to-audio.json +++ b/workflows/templates/minimax/chain-matched-to-audio.json @@ -36,7 +36,8 @@ "lora_adapter_name": "turbo", "weights_dtype": "{int4}", "hold_audio": null, - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -126,6 +127,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/chain-video-continuity.json b/workflows/templates/minimax/chain-video-continuity.json index 05f8cfd5..71df0d8e 100644 --- a/workflows/templates/minimax/chain-video-continuity.json +++ b/workflows/templates/minimax/chain-video-continuity.json @@ -40,7 +40,8 @@ "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", "weights_dtype": "{int4}", - "seed": 12345 + "seed": 12345, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -129,6 +130,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/chained-segments.json b/workflows/templates/minimax/chained-segments.json index 3bf9855e..1355e09f 100644 --- a/workflows/templates/minimax/chained-segments.json +++ b/workflows/templates/minimax/chained-segments.json @@ -49,7 +49,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -139,6 +140,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/composable-references.json b/workflows/templates/minimax/composable-references.json index 0a8c9b9b..54f528b8 100644 --- a/workflows/templates/minimax/composable-references.json +++ b/workflows/templates/minimax/composable-references.json @@ -36,7 +36,8 @@ "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", "weights_dtype": "{int4}", - "seed": 12345 + "seed": 12345, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -125,6 +126,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/dialogue-short.json b/workflows/templates/minimax/dialogue-short.json index 996b7581..e8a5080b 100644 --- a/workflows/templates/minimax/dialogue-short.json +++ b/workflows/templates/minimax/dialogue-short.json @@ -143,7 +143,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": 42 + "seed": 42, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -290,6 +291,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/enhance-prompt-with-image.json b/workflows/templates/minimax/enhance-prompt-with-image.json index 6e389c98..d535673c 100644 --- a/workflows/templates/minimax/enhance-prompt-with-image.json +++ b/workflows/templates/minimax/enhance-prompt-with-image.json @@ -27,7 +27,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -134,6 +135,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/enhance-prompt.json b/workflows/templates/minimax/enhance-prompt.json index e04b6c1a..208c7572 100644 --- a/workflows/templates/minimax/enhance-prompt.json +++ b/workflows/templates/minimax/enhance-prompt.json @@ -43,7 +43,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": 42 + "seed": 42, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -149,6 +150,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/first-and-last-frame.json b/workflows/templates/minimax/first-and-last-frame.json index 58dc4bc7..5ef4d4f6 100644 --- a/workflows/templates/minimax/first-and-last-frame.json +++ b/workflows/templates/minimax/first-and-last-frame.json @@ -28,7 +28,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -118,6 +119,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/generated-subject-reference.json b/workflows/templates/minimax/generated-subject-reference.json index 97e0226c..56fdb3fc 100644 --- a/workflows/templates/minimax/generated-subject-reference.json +++ b/workflows/templates/minimax/generated-subject-reference.json @@ -34,7 +34,8 @@ "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", "weights_dtype": "{int4}", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -151,6 +152,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/last-frame-only.json b/workflows/templates/minimax/last-frame-only.json index 66000755..8458da38 100644 --- a/workflows/templates/minimax/last-frame-only.json +++ b/workflows/templates/minimax/last-frame-only.json @@ -25,7 +25,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -115,6 +116,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/music-video.json b/workflows/templates/minimax/music-video.json index 1d966cb6..355c2c70 100644 --- a/workflows/templates/minimax/music-video.json +++ b/workflows/templates/minimax/music-video.json @@ -92,7 +92,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": 42 + "seed": 42, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -256,6 +257,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/shots-batch.json b/workflows/templates/minimax/shots-batch.json index 80b986c9..05b9ffdd 100644 --- a/workflows/templates/minimax/shots-batch.json +++ b/workflows/templates/minimax/shots-batch.json @@ -70,7 +70,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_fl2v_turbo_4step_v1.2_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -161,6 +162,7 @@ }, "components": { "transformer": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 2, diff --git a/workflows/templates/minimax/storyboard.json b/workflows/templates/minimax/storyboard.json index 7888e7bd..f14d1594 100644 --- a/workflows/templates/minimax/storyboard.json +++ b/workflows/templates/minimax/storyboard.json @@ -45,7 +45,8 @@ "lora_model_name": "lightx2v/Minimax-h3-Turbo", "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", - "seed": 42 + "seed": 42, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -214,6 +215,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, diff --git a/workflows/templates/minimax/voice-timbre-reference.json b/workflows/templates/minimax/voice-timbre-reference.json index 2e81b91b..f37adbad 100644 --- a/workflows/templates/minimax/voice-timbre-reference.json +++ b/workflows/templates/minimax/voice-timbre-reference.json @@ -36,7 +36,8 @@ "lora_weight_name": "minimax_h3_ref2v_turbo_8step_v1.0_768p_bf16.safetensors", "lora_adapter_name": "turbo", "weights_dtype": "{int4}", - "seed": null + "seed": null, + "attention_backend": null }, "variable_constraints": { "num_frames": { @@ -141,6 +142,7 @@ }, "components": { "transformer_ref": { + "attention_backend": "variable:attention_backend", "group_offload": { "offload_type": "block_level", "num_blocks_per_group": 1, From 0ec85cb6e2e21d00051460f0e00cc9d3702497d1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:45:43 -0500 Subject: [PATCH 048/204] style: #883 - ruff format Co-Authored-By: Claude Opus 5.5 --- tests/test_catalog_structure.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index 4bf4773b..f6b755a7 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -594,7 +594,9 @@ def _h3_steps(definition): def _transformers(step): components = step["pipeline"]["configuration"].get("components", {}) # t2va/fl2va load `transformer`, ref2va loads `transformer_ref` - return {k: v for k, v in components.items() if k in ("transformer", "transformer_ref")} + return { + k: v for k, v in components.items() if k in ("transformer", "transformer_ref") + } H3_TEMPLATES = [path for path in TEMPLATES if _h3_steps(load(path))] @@ -643,7 +645,9 @@ def _dialogue_short_after_substitution(**overrides): from dw.variables import replace_variables definition = load( - os.path.join(REPO_ROOT, "workflows", "templates", "minimax", "dialogue-short.json") + os.path.join( + REPO_ROOT, "workflows", "templates", "minimax", "dialogue-short.json" + ) ) variables = {**definition["variables"], **overrides} return { From 5e7b0cce049acc256ae8df5568b0e8830ff97886 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:51:06 -0500 Subject: [PATCH 049/204] docs: #883 - attention_backend on every H3 template in ACCELERATION, WORKFLOW_GUIDE, minimax README and the minimax-h3 skill Co-Authored-By: Claude Opus 5.5 --- docs/ACCELERATION.md | 19 ++++++++++--------- docs/WORKFLOW_GUIDE.md | 3 +++ plugins/dw/skills/minimax-h3/SKILL.md | 4 ++-- .../references/quality-and-speed.md | 5 ++++- workflows/templates/minimax/README.md | 6 +++++- 5 files changed, 24 insertions(+), 13 deletions(-) diff --git a/docs/ACCELERATION.md b/docs/ACCELERATION.md index c05100e5..743e0aa9 100644 --- a/docs/ACCELERATION.md +++ b/docs/ACCELERATION.md @@ -136,15 +136,16 @@ Select the attention implementation diffusers uses for the duration of each pipe Common values: `"flash"`, `"flash_hub"`, `"sage"`, `"sage_hub"`, `"native"`, `"flex"`. The full set is diffusers' `AttentionBackendName` enum - availability depends on what's installed (`flash-attn`, `sageattention`, etc.) and the platform. `_hub`-suffixed backends are fetched from the Hugging Face Hub kernel registry on first use, which needs the `kernels` package installed (`pip install kernels`) - it is not a dw dependency, and no bundled workflow sets a backend, so each runs on a plain install. diffusers also refuses a Hub kernel whose publisher it does not list as trusted - `sage_hub` among them, since SageAttention 2 comes from `SageAttention/sage-attention` - until `DIFFUSERS_TRUST_REMOTE_KERNELS=true`, which allows remote code from any such publisher. On an RTX 3090 `sage_hub` measured about 30% faster per MiniMax-H3 step (13.8 s against 19.2 s at 960x544x124, 54.7 s against 81.2 s at 345 frames); the local `sage` backend needs `sageattention>=2.1.1`, which PyPI does not carry. -The MiniMax-H3 baselines expose this as a variable: `attention_backend` on -[video-with-audio.json](../workflows/templates/minimax/video-with-audio.json), -[video-with-audio-768p.json](../workflows/templates/minimax/video-with-audio-768p.json), -[image-to-video.json](../workflows/templates/minimax/image-to-video.json) and -[reference-to-video.json](../workflows/templates/minimax/reference-to-video.json), pinned -on the transformer and `null` by default. Pass `attention_backend=sage_hub` to a box with -the `kernels` package and `DIFFUSERS_TRUST_REMOTE_KERNELS=true` in its `.env`; on MPS -leave it null. The other H3 templates take the same key under `components.transformer` -(`transformer_ref` for a ref2va workflow). LTX-2.5 cannot use it: its transformer +Every MiniMax-H3 template in [templates/minimax/](../workflows/templates/minimax/) +exposes this as a variable: `attention_backend`, pinned on the H3 step's transformer +(`components.transformer`, or `components.transformer_ref` for a ref2va workflow) and +`null` by default. That covers the shots, chains and episodes - `dialogue-short`, +`shots-batch`, `music-video` and `storyboard` among them - and only their H3 steps: a +Z-Image still or Music3 song beside an H3 shot keeps the process default. Pass +`attention_backend=sage_hub` to a box with the `kernels` package and +`DIFFUSERS_TRUST_REMOTE_KERNELS=true` in its `.env`; on MPS leave it null, since +validation does not check the backend against the device and a `_hub` kernel fails +there only at run time. LTX-2.5 cannot use it: its transformer passes an attention mask, and sage attention refuses one ("`attn_mask` is not supported for sage attention", measured 2026-10-10), so the LTX templates expose no backend. diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index b1d5c540..1275a82f 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -1225,6 +1225,9 @@ For components the pipeline loads itself — which is all of a modular pipeline' `diffusion_decoder`, for example). - `attention_backend` — a persistent `set_attention_backend` on one component, which a compiled component needs (the pipeline-level `attention_backend` applies per call). + Every MiniMax-H3 template wires an `attention_backend` variable (default `null`) here, + on its H3 step's `transformer` or `transformer_ref`; see + [ACCELERATION.md](ACCELERATION.md) for `sage_hub` and what it needs. - `attn_processor_type` — the attention processor the component runs, constructed with no arguments and handed to `set_attn_processor`. The `unet` and `transformer` blocks cover those two; this covers any other component that carries attention (LTX-2.5's diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index e9b13691..5f691439 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -51,8 +51,7 @@ arguments; prompt format is MiniMax's, from its text not here. - **As one take (a chain)**: `templates/minimax/chained-segments` (default `continuity: "last_frame"`; `continuity: "guide"`, `guide_frames` 22 or 39 - default 22 - holds the last frames and voice as a guide instead. - The guide prefix is trimmed off again, so the deliverable carries no - repeated frames. `carry_audio` (default true, a chain-block setting the + `carry_audio` (default true, a chain-block setting the template does not expose as a variable) is what brings the voice with the guide. Not on the ref2va chains, as guides stay off ref2va), `templates/minimax/chain-video-continuity` (previous segment's tail rides as a video reference - motion, camera, @@ -98,6 +97,7 @@ read the `workflows` guide's authoring section. `transformer_ref` alone, so anything handed there degrades output. `validate_workflow` refuses it and warns on a `weight_name` naming neither. - Smeary faces: int8 base model, `references/quality-and-speed.md`. +- CUDA: `attention_backend=sage_hub` (null elsewhere; mps fails at run). - Fit a crowd's action to what it holds: candle-holders asked to clap rendered three-handed people. - A directed crowd move needs its landmark in frame: "marches up to the diff --git a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md index 7dac37bd..3ae25a8a 100644 --- a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md +++ b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md @@ -81,7 +81,10 @@ Denoise is 80-91% of a job, ~56 s/pass with int8 weights streamed by block-level group offload: compute-bound, not PCIe-bound. - **`sage_hub`**: -23% at turbo 8-step, -33% at base 30, visually lossless. - Advise it first where the box has it (`attention_backend=sage_hub`). + Advise it first where the box has it: `attention_backend=sage_hub`, an + argument on every H3 template (default `null`, the H3 step only). It needs + CUDA, the `kernels` package and `DIFFUSERS_TRUST_REMOTE_KERNELS=true`; leave + it `null` on mps or cpu, where it passes validate and fails at run time. - **`first_block` 0.05**: -22% at 30 base steps with identical staging and faces (PSNR 27.5 dB against L5). It skips nothing at 12 steps or on a turbo schedule: use it at 30 base steps only. diff --git a/workflows/templates/minimax/README.md b/workflows/templates/minimax/README.md index fcae9da2..477e43f5 100644 --- a/workflows/templates/minimax/README.md +++ b/workflows/templates/minimax/README.md @@ -64,7 +64,11 @@ checkpoint comes with the canvas it was trained at, the sigma shift it was distilled against and the alpha it is meant to run at, and changing one means changing all of them. That is why `video_shift`, `audio_shift` and `lora_alpha` are variables on every template here - a checkpoint swap is arguments rather -than a new file. Four combinations are known good: +than a new file. So is `attention_backend`, `null` by default and wired to the +H3 step's transformer only (never a Z-Image or Music3 step beside it): +`attention_backend=sage_hub` is the measured 23-33% per-step win on a CUDA box +with the `kernels` package and `DIFFUSERS_TRUST_REMOTE_KERNELS=true`; leave it +null on mps, where it fails at run time ([ACCELERATION.md](../../../docs/ACCELERATION.md)). Four combinations are known good: | Checkpoint | Canvas | Shift (video/audio) | Alpha | Steps | Where | | --- | --- | --- | --- | --- | --- | From c467a543b8c3558cb63d33f9b85ef5e1d71c6dc7 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:53:13 -0500 Subject: [PATCH 050/204] fix(references): #873 - one owner of the variable-name characters; member_keys and validate_variable_name both read it Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/references.py | 16 ++++++++++------ dw/security.py | 4 +++- tests/test_script_check.py | 20 ++++++++++++++++++++ 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4eb436de..b8ac066e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -129,7 +129,7 @@ one to open. | Opening media | `dw/media.py` | This is the only module that calls `av.open`. It imports no torch and no step types. | `tests/test_media_layering.py::test_av_open_appears_only_in_media`, `tests/test_media_layering.py::test_media_imports_no_step_types` (no torch: —) | | Signal processing | `dw/dsp.py` | It measures and transforms waveforms, decides nothing, and imports nothing from `dw`. | `tests/test_media_layering.py::test_dsp_imports_nothing_from_dw` | | Assessment probes | `dw/tasks/assess.py`, `dw/assessment_rules.py` | Exactly three registered commands are assessment probes. Each measures a finished file, and every rule names a real probe field. | `tests/test_assessment_rules.py::TestRuleProbesAreRealCommands::test_assessment_is_exactly_the_three_probes`, `tests/test_assessment_rules.py::TestRuleProbesAreRealCommands::test_every_rule_names_a_registered_json_command` | -| Script check | `dw/tasks/script_check.py`: `check`; `dw/script_lines.py`: `parse_lines`, `parse_shots`, `shot_names_error` | `check_script` aligns a take's heard words to its expected lines and guards out words heard over silence and repetition loops (Whisper's hallucination over music). It builds its findings with `assessment_rules.finding()` and reads the guard's floor and window (`DEAD_AIR_FLOOR_DBFS`, `DEAD_AIR_WINDOW`) rather than copying them; the loudest-window measure is `dsp.loudest_window_dbfs` (`script_check.word_level` wraps it), which `transcribe_audio`'s `suspect: "silent"` mark reads with the same floor and window; the repetition-loop detector (`script_lines.looped_texts`, `REPEAT_RUN_MIN`, `REPEAT_MAX_PERIOD`) lives in `dw/script_lines.py` and backs both `check_script`'s `repetition` discard and `transcribe_audio`'s `suspect: "repetition"`, and `script_lines.suspect_runs`/`SUSPECT_LEFT_OUT` are what `plan_cuts` groups and leaves out suspect chunks by - so neither task imports the other. It is not an assessment probe and adds nothing to `RULES`. Its literal `lines` is refused at validation through `task_problems` (`script_lines_errors`), which calls the same `parse_lines` the task runs, and a literal `shots` through the same `parse_shots` and `shot_names_error`; the parsers live in `dw/script_lines.py`, not the task, so that `task_problems` reaches them without importing `script_check` (the `import_cycles` ratchet in `scripts/arch_metrics.py`). A line's `shot` may be the short `@name` of the one recorded shot that carries it (`script_lines.matching_shot_names`, `with_full_shot_names`, shared by the task and the literal-`shots` check); which member key a recorded name carries comes from `references.member_keys`, beside `MEMBER_SEPARATOR` and `for_each.member_name`, never a re-derived pattern. With `audio` an `output:`/`asset:`/path whose run manifest records shots, `dw/shot_span_preflight.py: script_shot_name_errors` (the `script_shot_names` error check) runs `shot_names_error` at validation, reading the map through `runs.shots_beside`. Its shot map comes from `assess.resolve_shots`, the one the probes use, never a copy; it places each shot on the soundtrack with `assess.sample_span`, decides repeated shot names with `shots.duplicate_shot_names`, and asks `locations.is_http_url` whether the take is a URL. | `tests/test_script_check.py::TestRegistration::test_registered_as_json_command_without_assessment_flag`, `tests/test_script_check.py::TestEmptyLines::test_words_only_over_silence_are_discarded_not_found`, `tests/test_script_check.py::TestLinesErrors::test_literal_with_a_colon_is_not_a_reference`, `tests/test_script_check.py::TestShotValidation::test_a_short_member_name_matches_the_one_shot_carrying_it`, `tests/test_shot_span_preflight.py::TestScriptShotNames::test_a_wrong_shot_name_is_an_error_at_validate_time` | +| Script check | `dw/tasks/script_check.py`: `check`; `dw/script_lines.py`: `parse_lines`, `parse_shots`, `shot_names_error` | `check_script` aligns a take's heard words to its expected lines and guards out words heard over silence and repetition loops (Whisper's hallucination over music). It builds its findings with `assessment_rules.finding()` and reads the guard's floor and window (`DEAD_AIR_FLOOR_DBFS`, `DEAD_AIR_WINDOW`) rather than copying them; the loudest-window measure is `dsp.loudest_window_dbfs` (`script_check.word_level` wraps it), which `transcribe_audio`'s `suspect: "silent"` mark reads with the same floor and window; the repetition-loop detector (`script_lines.looped_texts`, `REPEAT_RUN_MIN`, `REPEAT_MAX_PERIOD`) lives in `dw/script_lines.py` and backs both `check_script`'s `repetition` discard and `transcribe_audio`'s `suspect: "repetition"`, and `script_lines.suspect_runs`/`SUSPECT_LEFT_OUT` are what `plan_cuts` groups and leaves out suspect chunks by - so neither task imports the other. It is not an assessment probe and adds nothing to `RULES`. Its literal `lines` is refused at validation through `task_problems` (`script_lines_errors`), which calls the same `parse_lines` the task runs, and a literal `shots` through the same `parse_shots` and `shot_names_error`; the parsers live in `dw/script_lines.py`, not the task, so that `task_problems` reaches them without importing `script_check` (the `import_cycles` ratchet in `scripts/arch_metrics.py`). A line's `shot` may be the short `@name` of the one recorded shot that carries it (`script_lines.matching_shot_names`, `with_full_shot_names`, shared by the task and the literal-`shots` check); which member key a recorded name carries comes from `references.member_keys`, beside `MEMBER_SEPARATOR` and `for_each.member_name`, never a re-derived pattern: a key ends at the first character outside `references.VARIABLE_NAME_CHARACTERS`, the set `security.validate_variable_name` (which checks a for_each entry's name) builds its pattern from. With `audio` an `output:`/`asset:`/path whose run manifest records shots, `dw/shot_span_preflight.py: script_shot_name_errors` (the `script_shot_names` error check) runs `shot_names_error` at validation, reading the map through `runs.shots_beside`. Its shot map comes from `assess.resolve_shots`, the one the probes use, never a copy; it places each shot on the soundtrack with `assess.sample_span`, decides repeated shot names with `shots.duplicate_shot_names`, and asks `locations.is_http_url` whether the take is a URL. | `tests/test_script_check.py::TestRegistration::test_registered_as_json_command_without_assessment_flag`, `tests/test_script_check.py::TestEmptyLines::test_words_only_over_silence_are_discarded_not_found`, `tests/test_script_check.py::TestLinesErrors::test_literal_with_a_colon_is_not_a_reference`, `tests/test_script_check.py::TestShotValidation::test_a_short_member_name_matches_the_one_shot_carrying_it`, `tests/test_script_check.py::TestShotValidation::test_a_valid_variable_name_comes_back_whole_as_a_member_key`, `tests/test_shot_span_preflight.py::TestScriptShotNames::test_a_wrong_shot_name_is_an_error_at_validate_time` | | Audio sync against a source slice | `dw/tasks/measure_sync.py`: `measure_sync`; `dw/dsp.py`: `envelope_lag` | `measure_sync` cross-correlates the onset envelopes of a shot's audio and its source slice and answers the lag and a 0-1 confidence. A positive lag means the audio is LATE (its events fall after the reference's), negative early; `lag_seconds` is None for a silent or flat track. It builds its findings (`sync_unmeasurable`, `sync_low_confidence`, `audio_out_of_sync`) with `assessment_rules.finding()` over module-local rule dicts and adds nothing to `RULES`; it reads the silence floor from `beats.SILENT_DBFS`. `envelope_lag` imports nothing from `dw`. | `tests/test_measure_sync.py::TestLag::test_late_audio_is_a_positive_lag`, `tests/test_measure_sync.py::TestLag::test_early_audio_is_a_negative_lag`, `tests/test_measure_sync.py::TestUnmeasurable::test_a_silent_reference_has_no_lag` | ## Security and architecture guardrails diff --git a/dw/references.py b/dw/references.py index 7d0a3561..7e647c94 100644 --- a/dw/references.py +++ b/dw/references.py @@ -103,24 +103,28 @@ def author_index(source_indices, index): # A for_each member is named `@` MEMBER_SEPARATOR = "@" -# A member's key is its entry's name (a variable name: letters, digits, '_', -# '-') or its index, so it ends at the first character outside this set -_MEMBER_KEY_CHARS = frozenset( + +# The characters a variable name may hold after its first. +# `security.validate_variable_name` builds its pattern from this, and a +# for_each entry's name is checked by that validator, so `member_keys` reads +# the same set to find where a member key ends - one owner for both +VARIABLE_NAME_CHARACTERS = ( "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_-" ) def member_keys(name): """The for_each member keys a name carries: `cold_open` for - `shot@cold_open` and for a file named `...-shot@cold_open.0-0.0.mp4` - (the key ends where `member_name`'s key charset does). Empty for a name + `shot@cold_open` and for a file named `...-shot@cold_open.0-0.0.mp4`. + A key is an entry's name (a variable name) or its index, so it ends at + the first character outside `VARIABLE_NAME_CHARACTERS`. Empty for a name with no separator.""" if not isinstance(name, str): return [] keys = [] for part in name.split(MEMBER_SEPARATOR)[1:]: end = 0 - while end < len(part) and part[end] in _MEMBER_KEY_CHARS: + while end < len(part) and part[end] in VARIABLE_NAME_CHARACTERS: end += 1 if end: keys.append(part[:end]) diff --git a/dw/security.py b/dw/security.py index 4fa506de..5bbb2523 100644 --- a/dw/security.py +++ b/dw/security.py @@ -9,6 +9,8 @@ from urllib.parse import urlparse from typing import Union, List, Optional +from .references import VARIABLE_NAME_CHARACTERS + logger = logging.getLogger("dw") # Security constants @@ -307,7 +309,7 @@ def validate_variable_name(name: str) -> str: raise InvalidInputError("Variable name cannot be empty") # Allow only alphanumeric characters, underscores, and hyphens - if not re.match(r"^[a-zA-Z_][a-zA-Z0-9_-]*\Z", name): + if not re.match(rf"^[a-zA-Z_][{re.escape(VARIABLE_NAME_CHARACTERS)}]*\Z", name): raise InvalidInputError(f"Invalid variable name: {name}") if len(name) > MAX_VARIABLE_NAME_LENGTH: diff --git a/tests/test_script_check.py b/tests/test_script_check.py index e863a85e..18b3ea2c 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -859,6 +859,26 @@ def test_member_keys_come_from_the_for_each_separator(self): self.assertEqual(member_keys("x-shot@cold_open.0-0.0.mp4"), ["cold_open"]) self.assertEqual(member_keys("plain.mp4"), []) + def test_a_valid_variable_name_comes_back_whole_as_a_member_key(self): + # A for_each entry's name is checked by validate_variable_name; the + # key member_keys reads back must end exactly where that rule does, + # for every character either could meet + import string + + from dw.for_each import member_name + from dw.references import member_keys + from dw.security import InvalidInputError, validate_variable_name + + for char in string.printable: + key = f"a{char}b" + filename = member_name("shot", key) + ".0-0.0.mp4" + try: + validate_variable_name(key) + except InvalidInputError: + self.assertNotEqual(member_keys(filename)[:1], [key], repr(char)) + else: + self.assertEqual(member_keys(filename), [key], repr(char)) + def test_a_short_name_two_shots_carry_is_refused(self): records = [{"name": "a-shot@open.mp4"}, {"name": "b-shot@open.mp4"}] message = shot_names_error( From 60b1982899267efdbff2d074a655a459ee16457c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:55:55 -0500 Subject: [PATCH 051/204] fix(server): #881 - rank OOM score per dispatch: dispatching worker 1000, others 0 Co-Authored-By: Claude Sonnet 5.5 --- docs/SERVER.md | 7 ++++--- dw/server/jobs.py | 22 ++++++++++----------- dw/server/pool.py | 2 -- dw/worker_manager.py | 9 +++++---- tests/test_worker_pool.py | 40 ++++++++++++++++++++++++++++----------- 5 files changed, 49 insertions(+), 31 deletions(-) diff --git a/docs/SERVER.md b/docs/SERVER.md index f470aabb..c3cee1f9 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -228,9 +228,10 @@ on one card); else any card. A card's size is its GiB rounded up (a 3090 counts as 24), which a `cost` figure is compared to; a declared estimate is held to the card's GiB to one decimal (a 3090's 23.6), the same ceiling admission checks it against. Cancel reaches only the worker running that job, and a worker -crash fails only its own job. The worker started later gets a higher Linux -`oom_score_adj` (+100 over the highest other live worker), so a host-RAM -squeeze kills the later job; nothing is written with one worker. There is no +crash fails only its own job. Each dispatch sets its worker's Linux +`oom_score_adj` to 1000 and every other live worker's to 0, so a host-RAM +squeeze kills the newest job whatever the workers' sizes; nothing is written +with one worker. There is no host-RAM gate at dispatch: two big jobs can still exhaust host RAM together. Among the free cards a job fits, dispatch prefers the one it has an affinity for: a rerun goes to the card that ran the original, and any job to the card diff --git a/dw/server/jobs.py b/dw/server/jobs.py index 2593f186..36aa322b 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -861,23 +861,23 @@ def _run_on(self, slot, job): self._finish(job, status, error=error, traceback_text=traceback_text) def _rank_for_oom(self, slot): - """A worker this call just started is scored for the kernel's OOM - killer above every other live worker, so a host-RAM squeeze kills - the later-started worker - one job - rather than whichever is - largest. Nothing is written while it is the only worker, so one card - runs exactly as before.""" + """Score the dispatching worker 1000 for the kernel's OOM killer and + every other live worker 0, on every dispatch, so a host-RAM squeeze + kills the newest job whatever the workers' sizes (#881). Nothing is + written while it is the only worker, so one card runs exactly as + before. Lowering back to 0 needs no privilege.""" manager = slot.manager - pid = manager.pid() if hasattr(manager, "pid") else None - if pid is None or pid == slot.ranked_pid: + if not hasattr(manager, "set_oom_score_adj"): return - slot.ranked_pid = pid others = [ other.manager for other in self.slots if other is not slot and other.alive() ] - if not others or not hasattr(manager, "set_oom_score_adj"): + if not others: return - highest = max((other.oom_score_adj or 0) for other in others) - manager.set_oom_score_adj(min(1000, highest + 100)) + manager.set_oom_score_adj(1000) + for other in others: + if hasattr(other, "set_oom_score_adj"): + other.set_oom_score_adj(0) def is_busy(self): """True while a job is running on any card or queued - the window in diff --git a/dw/server/pool.py b/dw/server/pool.py index 063fe6aa..4f8e09ed 100644 --- a/dw/server/pool.py +++ b/dw/server/pool.py @@ -32,8 +32,6 @@ def __init__(self, manager): # This card's last memory reading and when it was taken self.last_memory = None self.last_memory_at = None - # The worker process _rank_for_oom last looked at - self.ranked_pid = None @property def device(self): diff --git a/dw/worker_manager.py b/dw/worker_manager.py index b0ef73b9..c18b9613 100644 --- a/dw/worker_manager.py +++ b/dw/worker_manager.py @@ -274,10 +274,11 @@ def ceiling_gb(self): def set_oom_score_adj(self, value): """Ask the kernel's OOM killer to pick this worker ahead of anything - scored lower (Linux only; best effort). In a pool the worker started - later is scored higher, so a host-RAM squeeze kills the newer job - rather than whichever worker happens to be larger (#462). Raising a - process's own score needs no privilege. Returns whether it took.""" + scored lower (Linux only; best effort). In a pool the worker of the + latest dispatch is scored 1000 and the others 0, so a host-RAM + squeeze kills the newest job rather than whichever worker happens to + be larger (#462, #881). Neither raising nor lowering to 0 needs + privilege. Returns whether it took.""" pid = self.pid() if pid is None or not sys.platform.startswith("linux"): return False diff --git a/tests/test_worker_pool.py b/tests/test_worker_pool.py index 2a0c37e5..fccbcd0c 100644 --- a/tests/test_worker_pool.py +++ b/tests/test_worker_pool.py @@ -409,30 +409,48 @@ def is_alive(self): class TestOomRanking: - def test_the_later_started_worker_is_ranked_first(self, pool): + def _pair(self, pool): a = card(success_script, "cuda:0", 24) b = card(success_script, "cuda:1", 24) written = [] for worker, pid in ((a, 101), (b, 102)): worker.pid = lambda pid=pid: pid + worker.worker_active = True + worker.worker_process = FakeProcess(pid) worker.set_oom_score_adj = lambda value, worker=worker: written.append( (worker.device, value) ) - manager = pool(a, b) + return pool(a, b), written + + def test_each_dispatch_scores_its_worker_highest(self, pool): + manager, written = self._pair(pool) first, second = manager.slots - # The only worker alive: nothing is written - a.worker_active = True - a.worker_process = FakeProcess(101) manager._rank_for_oom(first) - assert written == [] + assert written == [("cuda:0", 1000), ("cuda:1", 0)] - # A second starts beside it, with the first still at no adjustment - b.worker_active = True - b.worker_process = FakeProcess(102) - assert a.oom_score_adj is None + written.clear() + manager._rank_for_oom(second) + assert written == [("cuda:1", 1000), ("cuda:0", 0)] + + def test_a_dispatch_back_onto_an_unchanged_worker_flips_the_scores(self, pool): + manager, written = self._pair(pool) + first, second = manager.slots + manager._rank_for_oom(first) manager._rank_for_oom(second) - assert written == [("cuda:1", 100)] + + written.clear() + manager._rank_for_oom(first) # same pid as the first dispatch + assert written == [("cuda:0", 1000), ("cuda:1", 0)] + + def test_the_only_live_worker_is_left_alone(self, pool): + manager, written = self._pair(pool) + first, second = manager.slots + second.manager.worker_active = False + second.manager.worker_process = None + + manager._rank_for_oom(first) + assert written == [] class TestHealth: From 39d59692b4fac218d906af93fd6dfcf107c69db0 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 21:58:20 -0500 Subject: [PATCH 052/204] docs: #881 - architecture map names the renamed OOM ranking test Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b8ac066e..291a0aac 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -153,7 +153,7 @@ one to open. | App factory | `dw/server/app.py`: `create_app` | `create_app` builds the JobManager and `app.state`, installs the middleware and registers the routes, and all state lives in the JobManager. | — | | Routers | `dw/server/routes/*.py`, `dw/server/routes/__init__.py`: `ROUTERS`, `include_routers`, `include_file_routes` | Routers register in `ROUTERS` order, because a greedy `{name:path}` route must come after its specific siblings. The files router comes later, after `/mcp`. | `tests/test_server_downloads.py::test_download_workflow_sets_content_disposition_attachment` | | Job queue | `dw/server/jobs.py`: `JobManager`; `dw/server/job_results.py`: `consume_results` | A dispatcher thread hands queued jobs, in queue order, to the pool's workers (`Worker pool dispatch`); each worker runs one job at a time, and `consume_results` folds that worker's replies into the job, naming files relative to the job's own `output_dir`. A job carries its own `output_dir`, `asset_dir` and `workflow_dir`, so it stays in its workspace. | `tests/test_server_workspaces.py::TestRunning::test_rerun_stays_in_the_workspace_it_ran_in` (FIFO: —) | -| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_the_later_started_worker_is_ranked_first`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted` | +| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted` | | Card affinity | `dw/server/pool.py`: `choose_slot`; `dw/server/jobs.py`: `JobManager.route`; `dw/worker_protocol.py`: `workflow_identity` | Among the free cards a job fits it goes to the rerun's original card, else the card whose worker last ran its workflow identity (`workflow_identity`, the one rule the worker caches by too), else one with no worker running, else the first in `--devices` order; a cache probe and `plan.estimate` (`priced_for`) ask about the card `route` names, and memory is read and cleared per card. | `tests/test_worker_pool.py::TestIdentityAffinity::test_a_second_run_lands_on_the_card_that_ran_the_workflow_first`, `tests/test_worker_pool.py::TestRerunAffinity::test_a_rerun_lands_on_the_original_card_when_it_is_free`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle` | | Per-card memory and cache | `dw/server/worker_memory.py`: `memory_status`, `clear_memory`, `probe_cache` | Each card keeps its own last memory reading; an idle card's worker is asked live, a busy one answers with the run's last report and says why it is not live, and a clear leaves a card running a job alone. Every request takes the card's lock with a bounded wait, and a card's lock is never taken while the JobManager's lock is held. | `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_server.py::test_memory_says_why_a_reading_is_not_the_worker_s` | | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | From a27aaab4dd63d99f267e2a87a66495434bdd1dc8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 22:01:49 -0500 Subject: [PATCH 053/204] fix(cache): #891 - mag calibrate ratios reach job warnings and log events, both CFG contexts labeled Co-Authored-By: Claude Sonnet 5.5 --- docs/ACCELERATION.md | 2 +- dw/pipeline_processors/components.py | 82 +++++++++++++++++++++++++++- dw/pipeline_processors/pipeline.py | 2 + dw/workflow_schema.json | 2 +- tests/test_cache_blocks.py | 82 ++++++++++++++++++++++++++++ 5 files changed, 167 insertions(+), 3 deletions(-) diff --git a/docs/ACCELERATION.md b/docs/ACCELERATION.md index 743e0aa9..bec96dcb 100644 --- a/docs/ACCELERATION.md +++ b/docs/ACCELERATION.md @@ -75,7 +75,7 @@ MagCache needs to know how each denoising step's output magnitude typically beha - **A preset name** — `"mag_ratios": "flux"` resolves to the ratios diffusers ships for Flux. Any preset a later diffusers release adds is usable by name without a change here. - **An explicit array** — `"mag_ratios": [1.0, 0.98, 0.96, ...]`. The array is interpolated automatically when its length differs from `num_inference_steps`, so ratios measured at one step count can be reused at another. -For a model with no preset, run once with `"calibrate": true`. Calibration skips nothing and logs the measured ratios at the end of the run; paste that array into `mag_ratios` and drop the `calibrate` flag for subsequent runs. +For a model with no preset, run once with `"calibrate": true`. Calibration skips nothing. At the end of the run the measured ratios appear as a job warning and a `log` event (`get_job`'s `warnings`, `get_job_events`, `export_job`): `MagCache calibration (N steps): [ ... ] — paste into mag_ratios`. Paste that array into `mag_ratios` and drop the `calibrate` flag for subsequent runs. A pipeline that uses classifier-free guidance (an SD-style template, not a guidance-distilled one such as Flux) keeps a separate state for its `cond` and `uncond` passes, so it emits **two** lists, each labeled `[cond]` or `[uncond]`; use the `cond` one, since `mag_ratios` takes one array. ```json "cache": { "type": "mag", "calibrate": true, "num_inference_steps": 28 } diff --git a/dw/pipeline_processors/components.py b/dw/pipeline_processors/components.py index cc056cd5..3e350c9b 100644 --- a/dw/pipeline_processors/components.py +++ b/dw/pipeline_processors/components.py @@ -1,4 +1,5 @@ import torch +import ast import contextlib import logging from .config_objects import ( @@ -11,7 +12,7 @@ require_trusted_from_pretrained_arguments, ) from .. import get_device_type, resolve_device -from ..events import WorkflowCancelled, emit_phase +from ..events import WorkflowCancelled, emit_log, emit_phase, emit_warning from .. import download_watch from .. import quantized_cache from huggingface_hub.errors import HfHubHTTPError @@ -830,6 +831,85 @@ def get_cache_transformer(pipeline): return None +_MAG_LOGGER = "diffusers.hooks.mag_cache" +_MAG_RESULT_PREFIX = "MagCache Calibration Results:" + + +def _active_cache_context_name(transformer): + """The name of the cache context a stateful hook under the transformer is + running in, or None. Under CFG that is "cond" or "uncond".""" + for module in transformer.modules(): + registry = getattr(module, "_diffusers_hook", None) + for hook in getattr(registry, "hooks", {}).values(): + for attr in vars(hook).values(): + context = getattr(attr, "_context", None) + if context is not None and hasattr(attr, "get_state"): + return context.name + return None + + +class _MagCalibrationHandler(logging.Handler): + """Turns diffusers' end-of-run calibration log record into job events. + + diffusers prints the ratios, logs them at INFO and clears the list in the + same call (`MagCacheHook._advance_step`), so the record is the only moment + they can be read, and the worker's stdout reaches no consumer. The record + is emitted synchronously from inside the forward, so the state manager's + context is still the one the ratios belong to. + """ + + def __init__(self, transformer): + super().__init__(level=logging.INFO) + self.transformer = transformer + + def emit(self, record): + message = record.getMessage() + if not message.startswith(_MAG_RESULT_PREFIX): + return + try: + ratios = ast.literal_eval(message[len(_MAG_RESULT_PREFIX) :].strip()) + context = _active_cache_context_name(self.transformer) + except (ValueError, SyntaxError): + return + label = f" [{context}]" if context in ("cond", "uncond") else "" + text = ( + f"MagCache calibration ({len(ratios)} steps){label}: " + f"{ratios} \u2014 paste into mag_ratios" + ) + if context == "cond": + text += " (classifier-free guidance runs two lists; the cond one is the usual choice)" + emit_log(text) + emit_warning(text) + + +@contextlib.contextmanager +def captured_mag_calibration(pipeline): + """Report a mag cache's calibration ratios as job events (#891). + + A no-op unless the transformer's cache is a MagCacheConfig with calibrate + on. Under classifier-free guidance dw's pipelines run "cond" and "uncond" + contexts, each with its own state, so two labeled lists are emitted. + """ + transformer = get_cache_transformer(pipeline) + config = getattr(transformer, "_cache_config", None) + if not getattr(config, "calibrate", False): + yield + return + + mag_logger = logging.getLogger(_MAG_LOGGER) + handler = _MagCalibrationHandler(transformer) + old_level, old_propagate = mag_logger.level, mag_logger.propagate + mag_logger.addHandler(handler) + mag_logger.setLevel(logging.INFO) + mag_logger.propagate = False + try: + yield + finally: + mag_logger.removeHandler(handler) + mag_logger.setLevel(old_level) + mag_logger.propagate = old_propagate + + @contextlib.contextmanager def stateful_cache_context(pipeline): """Provide the context a stateful cache hook reads its state through. diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index 223083d8..a3d7d15e 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -23,6 +23,7 @@ get_component, load_and_configure_scheduler, load_component, + captured_mag_calibration, stateful_cache_context, ) from .placement import ( @@ -638,6 +639,7 @@ def _call_pipeline(self, arguments, attn_backend): logger.info(f"Using attention backend: {attn_backend}") stack.enter_context(attention_backend(attn_backend)) + stack.enter_context(captured_mag_calibration(self.pipeline)) stack.enter_context(stateful_cache_context(self.pipeline)) if not self._takes_step_callback(): # A modular pipeline takes no step callback at all, so this diff --git a/dw/workflow_schema.json b/dw/workflow_schema.json index abff7b82..f163a48d 100644 --- a/dw/workflow_schema.json +++ b/dw/workflow_schema.json @@ -805,7 +805,7 @@ ] }, "calibrate": { - "description": "Run mag cache in calibration mode: skip nothing, and log the magnitude ratios for this model at the end of the run so they can be pasted into mag_ratios. Default: false.", + "description": "Run mag cache in calibration mode: skip nothing, and report the magnitude ratios for this model at the end of the run as a job warning and log event, so they can be pasted into mag_ratios. Under classifier-free guidance two lists are reported, labeled [cond] and [uncond]; use the cond one. Default: false.", "type": "boolean" }, "cache_interval": { diff --git a/tests/test_cache_blocks.py b/tests/test_cache_blocks.py index b5026d8f..a2426109 100644 --- a/tests/test_cache_blocks.py +++ b/tests/test_cache_blocks.py @@ -438,3 +438,85 @@ def test_without_the_remap_the_text_stream_leaks_into_the_audio_slot(monkeypatch # The text conditioning's sequence length, carried into the audio stream assert audio_hidden_states.shape == (1, 3, 4) + + +def _mag_model(num_inference_steps): + from diffusers import MagCacheConfig + from diffusers.models.transformers.transformer_minimax_h3 import ( + MiniMaxH3Transformer3DModel, + ) + + register_cache_blocks() + model = MiniMaxH3Transformer3DModel( + num_attention_heads=2, + attention_head_dim=8, + hidden_size=16, + num_layers=2, + num_refiner_layers=1, + ffn_dim=32, + text_dim=16, + time_embed_hidden_dim=16, + time_embed_dim=8, + freq_dim=16, + rope_freq_dim=4, + ) + model.enable_cache( + MagCacheConfig(calibrate=True, num_inference_steps=num_inference_steps) + ) + return model + + +def _run_calibration_steps(model, context, steps): + """Drive the real mag hooks' tail path the way a forward does.""" + from diffusers.hooks import HookRegistry + from diffusers.hooks.mag_cache import MagCacheBlockHook + + tail = next( + hook + for block in model.transformer_blocks + for hook in HookRegistry.check_if_exists_or_initialize(block).hooks.values() + if isinstance(hook, MagCacheBlockHook) and hook.is_tail + ) + with model.cache_context(context): + state = tail.state_manager.get_state() + for i in range(steps): + tail._perform_calibration_step(state, torch.full((1, 3, 4), 1.0 + i)) + state.previous_residual = torch.full((1, 3, 4), 1.0 + i) + tail._advance_step(state) + + +def test_mag_calibration_ratios_reach_job_events_for_both_cfg_contexts(monkeypatch): + """#891: diffusers logs the ratios and clears them; both CFG lists must be emitted.""" + from dw.pipeline_processors import components + + logs, warnings = [], [] + monkeypatch.setattr(components, "emit_log", logs.append) + monkeypatch.setattr(components, "emit_warning", warnings.append) + + model = _mag_model(4) + with components.captured_mag_calibration(_FakePipeline(model)): + _run_calibration_steps(model, "cond", 4) + _run_calibration_steps(model, "uncond", 4) + + assert logs == warnings + assert len(warnings) == 2 + assert "[cond]" in warnings[0] and "[uncond]" in warnings[1] + for message in warnings: + assert "MagCache calibration (4 steps)" in message + assert "paste into mag_ratios" in message + + +def test_mag_calibration_silent_without_calibrate(monkeypatch): + from diffusers import MagCacheConfig + + from dw.pipeline_processors import components + + warnings = [] + monkeypatch.setattr(components, "emit_warning", warnings.append) + model = _mag_model(3) + model.disable_cache() + model.enable_cache(MagCacheConfig(mag_ratios=[1.0, 0.9, 0.8], num_inference_steps=3)) + + with components.captured_mag_calibration(_FakePipeline(model)): + pass + assert warnings == [] From 95813b3deb4bb57bbc67a37f76c0865862e54544 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 22:02:32 -0500 Subject: [PATCH 054/204] style: #891 - ruff format Co-Authored-By: Claude Sonnet 5.5 --- tests/test_cache_blocks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_cache_blocks.py b/tests/test_cache_blocks.py index a2426109..7a4fbba3 100644 --- a/tests/test_cache_blocks.py +++ b/tests/test_cache_blocks.py @@ -515,7 +515,9 @@ def test_mag_calibration_silent_without_calibrate(monkeypatch): monkeypatch.setattr(components, "emit_warning", warnings.append) model = _mag_model(3) model.disable_cache() - model.enable_cache(MagCacheConfig(mag_ratios=[1.0, 0.9, 0.8], num_inference_steps=3)) + model.enable_cache( + MagCacheConfig(mag_ratios=[1.0, 0.9, 0.8], num_inference_steps=3) + ) with components.captured_mag_calibration(_FakePipeline(model)): pass From 6f8e3a55b886e6e2530225573072b7236f722b51 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Tue, 13 Oct 2026 22:09:00 -0500 Subject: [PATCH 055/204] feat(plugin): #878 - minimax-h3 skill: locked frame lip-sync via Ref2VA keyframe completion + audio reuse Records #877's verdict (holds, first frame only). Trims existing text to stay under the 12 KiB cap; pins the task types and hold_audio in tests/test_plugin_skills.py. Co-Authored-By: Claude Opus 5.5 --- plugins/dw/skills/minimax-h3/SKILL.md | 38 +++++++++++++-------------- tests/test_plugin_skills.py | 29 ++++++++++++++++++++ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index 5f691439..a5426d42 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -28,6 +28,10 @@ arguments; prompt format is MiniMax's, from its text not here. first and last `templates/minimax/first-and-last-frame`; last only `templates/minimax/last-frame-only`; a one-line idea plus a picture `templates/minimax/enhance-prompt-with-image`. +- **A locked frame that lip-syncs to a song** (FL2VA takes no audio): Ref2VA + `[keyframe completion + audio reuse]`, `` the first frame, + `
    {/if} {#if parameter?.description} -
    {parameter.description}
    +
    + +
    {/if}
{/each} diff --git a/ui/src/lib/pages/WorkflowPage.svelte b/ui/src/lib/pages/WorkflowPage.svelte index 97dbe4fc..f4c64ca8 100644 --- a/ui/src/lib/pages/WorkflowPage.svelte +++ b/ui/src/lib/pages/WorkflowPage.svelte @@ -12,6 +12,7 @@ import { notify } from '../toast' import { confirmDialog } from '../confirm.svelte' import type { GalleryFile, WorkflowDefinition } from '../types' + import ClampedText from '../ui/ClampedText.svelte' let { name }: { name: string } = $props() @@ -175,7 +176,9 @@ {#if workflow} {#if workflow.description} -

{workflow.description}

+
+ +
{/if} diff --git a/ui/src/lib/schema/SchemaNode.svelte b/ui/src/lib/schema/SchemaNode.svelte index 6c09d997..a7c6658f 100644 --- a/ui/src/lib/schema/SchemaNode.svelte +++ b/ui/src/lib/schema/SchemaNode.svelte @@ -28,6 +28,7 @@ + +
+
+ {text} +
+ {#if overflows || expanded} + + {/if} +
+ + diff --git a/ui/src/lib/ui/ClampedText.test.ts b/ui/src/lib/ui/ClampedText.test.ts new file mode 100644 index 00000000..989051c6 --- /dev/null +++ b/ui/src/lib/ui/ClampedText.test.ts @@ -0,0 +1,36 @@ +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte' +import { afterEach, describe, expect, it, vi } from 'vitest' +import ClampedText from './ClampedText.svelte' + +// jsdom lays nothing out, so overflow is faked on the element's heights +function fakeHeights(scroll: number, client: number) { + vi.spyOn(HTMLElement.prototype, 'scrollHeight', 'get').mockReturnValue(scroll) + vi.spyOn(HTMLElement.prototype, 'clientHeight', 'get').mockReturnValue(client) +} + +describe('ClampedText', () => { + afterEach(() => { + cleanup() + vi.restoreAllMocks() + }) + + it('offers no toggle when the text fits', () => { + fakeHeights(40, 40) + render(ClampedText, { props: { text: 'short' } }) + expect(screen.getByText('short')).toBeTruthy() + expect(screen.queryByRole('button')).toBeNull() + }) + + it('folds long text and expands and collapses it', async () => { + fakeHeights(400, 80) + render(ClampedText, { props: { text: 'long text', lines: 3 } }) + const text = screen.getByText('long text') + expect(text.classList.contains('clamped')).toBe(true) + const toggle = screen.getByRole('button', { name: 'Show more' }) + expect(toggle.getAttribute('aria-expanded')).toBe('false') + await fireEvent.click(toggle) + expect(text.classList.contains('clamped')).toBe(false) + await fireEvent.click(screen.getByRole('button', { name: 'Show less' })) + expect(text.classList.contains('clamped')).toBe(true) + }) +}) diff --git a/ui/src/lib/ui/NoteEditor.svelte b/ui/src/lib/ui/NoteEditor.svelte index 5a46636f..35fd3901 100644 --- a/ui/src/lib/ui/NoteEditor.svelte +++ b/ui/src/lib/ui/NoteEditor.svelte @@ -1,6 +1,7 @@ + +
+ {#if tiles} +
+ {#each { length: tiles } as _, i (i)}{/each} +
+ {:else} + {#each { length: lines } as _, i (i)} + + {/each} + {/if} +
+ + +``` + +`OverviewPage.svelte`: import `Skeleton from '../ui/Skeleton.svelte'` and +replace each `loading…`: +- Recent outputs: `` +- Jobs: `` +- Workflows: `` + +If `OverviewPage.test.ts` asserts `getByText('loading…')`, change it to +`getByRole('status', { name: /loading/ })`. + +- [ ] **Step 4: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/ui/Skeleton.test.ts src/lib/pages/OverviewPage.test.ts && npm run check && npm run lint && npm test` +Expected: PASS (the design-rules test included) + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/lib/ui/Skeleton.svelte ui/src/lib/ui/Skeleton.test.ts ui/src/lib/pages/OverviewPage.svelte ui/src/lib/pages/OverviewPage.test.ts +git commit -m "feat(ui): placeholder blocks instead of 'loading…' on the overview" +``` + +--- + +### Task 6: Time in queue on the Jobs list + +**Files:** +- Create: `ui/src/lib/jobTimes.ts`, `ui/src/lib/jobTimes.test.ts` +- Modify: `ui/src/lib/pages/JobsPage.svelte` (the `.started` span around line 193) + +**Interfaces:** +- Produces: `queueWait(job: Pick, now: number): number | null` + +- [ ] **Step 1: Write the failing test** + +`ui/src/lib/jobTimes.test.ts`: +```ts +import { describe, expect, it } from 'vitest' +import { queueWait } from './jobTimes' + +describe('queueWait', () => { + it('is the time from submit to start for a job that started', () => { + expect( + queueWait({ created_at: 100, started_at: 160, status: 'succeeded' }, 999), + ).toBe(60) + }) + it('counts up to now for a job still queued', () => { + expect( + queueWait({ created_at: 100, started_at: null, status: 'queued' }, 400), + ).toBe(300) + }) + it('is null for a job cancelled before it started, or with no clock', () => { + expect( + queueWait({ created_at: 100, started_at: null, status: 'cancelled' }, 400), + ).toBeNull() + expect( + queueWait({ created_at: null, started_at: null, status: 'queued' }, 400), + ).toBeNull() + }) +}) +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `cd ui && npx vitest run src/lib/jobTimes.test.ts` +Expected: FAIL + +- [ ] **Step 3: Implement** + +`ui/src/lib/jobTimes.ts`: +```ts +import type { JobSummary } from './types' + +/** Seconds a job waited for a card: submit to start, or submit to now for + * one still waiting. Null when there is no wait to speak of - no clock, or + * cancelled before it ever started. */ +export function queueWait( + job: Pick, + now: number, +): number | null { + if (!job.created_at) return null + if (job.started_at) return Math.max(0, job.started_at - job.created_at) + return job.status === 'queued' ? Math.max(0, now - job.created_at) : null +} +``` + +`JobsPage.svelte`: import `queueWait`. Replace +`{when(job.started_at)}` with: +```svelte + {@const wait = queueWait(job, Date.now() / 1000)} + {#if job.status === 'queued'} + waiting {wait === null ? '' : formatDuration(wait)} + {:else} + = 1 + ? `waited ${formatDuration(wait)} in the queue` + : ''}>{when(job.started_at)} + {/if} +``` +`{@const}` must be the first child of a block. If Svelte rejects it here, +compute the value in a small function `waitOf(job)` in the script and call +that instead. + +- [ ] **Step 4: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/jobTimes.test.ts src/lib/pages/JobsPage.test.ts && npm run check && npm run lint` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/lib/jobTimes.ts ui/src/lib/jobTimes.test.ts ui/src/lib/pages/JobsPage.svelte +git commit -m "feat(ui): jobs list shows how long a job waited for a card" +``` + +--- + +## Phase B - UI, medium + +### Task 7: Notes rendered as Markdown (a safe subset) + +The renderer is our own small parser, and it outputs **Svelte nodes, never +HTML**, so no sanitizer or new dependency is needed and the +"never markup" rule holds. The subset: +- `#`/`##`/`###` headings; +- `-`/`*` and `1.` lists; +- fenced code blocks; +- paragraphs (single newlines kept as line breaks); +- inline `**bold**`, `*italic*`, `` `code` ``, and `[text](https://…)`. + +Underscore emphasis is left out on purpose: notes are full of `snake_case` +names (`cold_minutes`, `first_block`). + +`-webkit-line-clamp` does not reliably clamp across block children (`

`, +`

    `). So `ClampedText` switches to a max-height clamp (`lines × line-height`) +with a fade, which works for both plain and Markdown text. + +**Files:** +- Create: `ui/src/lib/markdown.ts`, `ui/src/lib/markdown.test.ts` +- Create: `ui/src/lib/ui/Markdown.svelte`, `ui/src/lib/ui/Markdown.test.ts` +- Modify: `ui/src/lib/ui/ClampedText.svelte` (prop `markdown`, max-height clamp) +- Modify: `ui/src/lib/ui/NoteEditor.svelte` (render notes as Markdown; hint under the textarea) + +**Interfaces:** +- Produces: types `Inline`, `Block`; `parseMarkdown(src: string): Block[]`; + `parseInline(src: string): Inline[]`; `safeHref(url: string): string | null` +- Produces: `Markdown` props `{ source: string }`; `ClampedText` prop `markdown?: boolean` + +- [ ] **Step 1: Write the failing tests** + +`ui/src/lib/markdown.test.ts`: +```ts +import { describe, expect, it } from 'vitest' +import { parseInline, parseMarkdown, safeHref } from './markdown' + +describe('safeHref', () => { + it('allows http, https and mailto only', () => { + expect(safeHref('https://x.dev/a')).toBe('https://x.dev/a') + expect(safeHref('mailto:a@b.c')).toBe('mailto:a@b.c') + expect(safeHref('javascript:alert(1)')).toBeNull() + expect(safeHref('JaVaScRiPt:alert(1)')).toBeNull() + expect(safeHref('data:text/html,x')).toBeNull() + expect(safeHref('/relative')).toBeNull() + }) +}) + +describe('parseInline', () => { + it('reads bold, italic, code and links', () => { + expect(parseInline('a **b** *c* `d` [e](https://x.dev)')).toEqual([ + { kind: 'text', text: 'a ' }, + { kind: 'strong', children: [{ kind: 'text', text: 'b' }] }, + { kind: 'text', text: ' ' }, + { kind: 'em', children: [{ kind: 'text', text: 'c' }] }, + { kind: 'text', text: ' ' }, + { kind: 'code', text: 'd' }, + { kind: 'text', text: ' ' }, + { + kind: 'link', + href: 'https://x.dev/', + children: [{ kind: 'text', text: 'e' }], + }, + ]) + }) + it('leaves an unsafe link as its literal text', () => { + expect(parseInline('[x](javascript:alert(1))')).toEqual([ + { kind: 'text', text: '[x](javascript:alert(1))' }, + ]) + }) + it('leaves snake_case alone', () => { + expect(parseInline('cold_minutes and warm_minutes')).toEqual([ + { kind: 'text', text: 'cold_minutes and warm_minutes' }, + ]) + }) + it('keeps markup-looking text as text', () => { + expect(parseInline('x')).toEqual([ + { kind: 'text', text: 'x' }, + ]) + }) +}) + +describe('parseMarkdown', () => { + it('reads headings, lists, code and paragraphs', () => { + const src = [ + '## RULE', + 'Two paths, pick by target.', + 'Second line.', + '', + '- TV: L0 -> L7', + '- Cinema: B3 -> P1', + '', + '1. one', + '2. two', + '', + '```', + 'dw validate x', + '```', + ].join('\n') + expect(parseMarkdown(src)).toEqual([ + { kind: 'heading', level: 2, children: [{ kind: 'text', text: 'RULE' }] }, + { + kind: 'paragraph', + children: [ + { kind: 'text', text: 'Two paths, pick by target.' }, + { kind: 'br' }, + { kind: 'text', text: 'Second line.' }, + ], + }, + { + kind: 'list', + ordered: false, + items: [ + [{ kind: 'text', text: 'TV: L0 -> L7' }], + [{ kind: 'text', text: 'Cinema: B3 -> P1' }], + ], + }, + { + kind: 'list', + ordered: true, + items: [[{ kind: 'text', text: 'one' }], [{ kind: 'text', text: 'two' }]], + }, + { kind: 'code', text: 'dw validate x' }, + ]) + }) + it('a plain note is one paragraph', () => { + expect(parseMarkdown('just text')).toEqual([ + { kind: 'paragraph', children: [{ kind: 'text', text: 'just text' }] }, + ]) + }) + it('an unclosed fence runs to the end', () => { + expect(parseMarkdown('```\nx')).toEqual([{ kind: 'code', text: 'x' }]) + }) +}) +``` + +`ui/src/lib/ui/Markdown.test.ts`: +```ts +import { cleanup, render, screen } from '@testing-library/svelte' +import { afterEach, describe, expect, it } from 'vitest' +import Markdown from './Markdown.svelte' + +describe('Markdown', () => { + afterEach(cleanup) + it('renders the subset as elements', () => { + render(Markdown, { + props: { source: '## Rule\n\n- **one**\n\n[docs](https://x.dev)' }, + }) + expect(screen.getByRole('heading', { name: 'Rule' })).toBeTruthy() + expect(screen.getByText('one').tagName).toBe('STRONG') + const link = screen.getByRole('link', { name: 'docs' }) + expect(link.getAttribute('href')).toBe('https://x.dev/') + expect(link.getAttribute('rel')).toBe('noopener noreferrer') + }) + it('never renders markup in the source', () => { + render(Markdown, { props: { source: '' } }) + expect(document.querySelector('img')).toBeNull() + expect(screen.getByText('')).toBeTruthy() + }) + it('never links a javascript: url', () => { + render(Markdown, { props: { source: '[x](javascript:alert(1))' } }) + expect(screen.queryByRole('link')).toBeNull() + }) +}) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `cd ui && npx vitest run src/lib/markdown.test.ts src/lib/ui/Markdown.test.ts` +Expected: FAIL + +- [ ] **Step 3: Implement the parser** + +`ui/src/lib/markdown.ts`: +```ts +/** A small, safe Markdown subset for notes, parsed to a tree that + * Markdown.svelte renders as Svelte nodes. Nothing here produces HTML, so + * text that looks like markup stays text, and a link is only ever an + * http(s)/mailto URL the browser's own URL parser accepted. + * + * Underscore emphasis is left out on purpose: notes quote snake_case + * names (`cold_minutes`, `first_block`) constantly. */ + +export type Inline = + | { kind: 'text'; text: string } + | { kind: 'br' } + | { kind: 'code'; text: string } + | { kind: 'strong' | 'em'; children: Inline[] } + | { kind: 'link'; href: string; children: Inline[] } + +export type Block = + | { kind: 'heading'; level: 1 | 2 | 3; children: Inline[] } + | { kind: 'paragraph'; children: Inline[] } + | { kind: 'list'; ordered: boolean; items: Inline[][] } + | { kind: 'code'; text: string } + +const SAFE_PROTOCOLS = new Set(['http:', 'https:', 'mailto:']) + +export function safeHref(url: string): string | null { + try { + const parsed = new URL(url) + return SAFE_PROTOCOLS.has(parsed.protocol) ? parsed.href : null + } catch { + return null + } +} + +// One alternation, tried left to right at each position: code first, so a +// `**` inside backticks stays literal +const INLINE = + /`([^`]+)`|\*\*(.+?)\*\*|\*([^*\s][^*]*?)\*|\[([^\]]+)\]\(([^)\s]+(?:\([^)\s]*\))?[^)\s]*)\)/g + +function pushText(out: Inline[], text: string) { + if (!text) return + const last = out[out.length - 1] + if (last?.kind === 'text') last.text += text + else out.push({ kind: 'text', text }) +} + +export function parseInline(src: string): Inline[] { + const out: Inline[] = [] + let at = 0 + for (const m of src.matchAll(INLINE)) { + pushText(out, src.slice(at, m.index)) + at = m.index + m[0].length + if (m[1] !== undefined) out.push({ kind: 'code', text: m[1] }) + else if (m[2] !== undefined) + out.push({ kind: 'strong', children: parseInline(m[2]) }) + else if (m[3] !== undefined) + out.push({ kind: 'em', children: parseInline(m[3]) }) + else { + const href = safeHref(m[5]) + if (href) out.push({ kind: 'link', href, children: parseInline(m[4]) }) + else pushText(out, m[0]) + } + } + pushText(out, src.slice(at)) + return out +} + +const HEADING = /^(#{1,3})\s+(.*)$/ +const BULLET = /^\s*[-*]\s+(.*)$/ +const NUMBERED = /^\s*\d+[.)]\s+(.*)$/ +const FENCE = /^\s*```/ + +const startsBlock = (line: string) => + FENCE.test(line) || + HEADING.test(line) || + BULLET.test(line) || + NUMBERED.test(line) + +export function parseMarkdown(src: string): Block[] { + const lines = src.replace(/\r\n?/g, '\n').split('\n') + const blocks: Block[] = [] + let i = 0 + while (i < lines.length) { + const line = lines[i] + if (!line.trim()) { + i++ + continue + } + if (FENCE.test(line)) { + const body: string[] = [] + i++ + while (i < lines.length && !FENCE.test(lines[i])) body.push(lines[i++]) + i++ // the closing fence, or past the end when it never closed + blocks.push({ kind: 'code', text: body.join('\n') }) + continue + } + const heading = HEADING.exec(line) + if (heading) { + blocks.push({ + kind: 'heading', + level: heading[1].length as 1 | 2 | 3, + children: parseInline(heading[2]), + }) + i++ + continue + } + const list = BULLET.test(line) ? BULLET : NUMBERED.test(line) ? NUMBERED : null + if (list) { + const items: Inline[][] = [] + while (i < lines.length && list.test(lines[i])) + items.push(parseInline(list.exec(lines[i++])![1])) + blocks.push({ kind: 'list', ordered: list === NUMBERED, items }) + continue + } + // A paragraph runs until a blank line or the start of another block; + // its single newlines are kept as line breaks - notes are written as + // lines, not reflowed prose + const children: Inline[] = [] + while (i < lines.length && lines[i].trim() && !startsBlock(lines[i])) { + if (children.length) children.push({ kind: 'br' }) + children.push(...parseInline(lines[i++])) + } + blocks.push({ kind: 'paragraph', children }) + } + return blocks +} +``` + +- [ ] **Step 4: Implement the component, and switch ClampedText and NoteEditor** + +`ui/src/lib/ui/Markdown.svelte`: +```svelte + + +{#snippet inline(nodes: Inline[])} + {#each nodes as node, i (i)} + {#if node.kind === 'text'}{node.text}{:else if node.kind === 'br'}
    {:else if node.kind === 'code'}{node.text}{:else if node.kind === 'strong'}{@render inline(node.children)}{:else if node.kind === 'em'}{@render inline(node.children)}{:else}    {@render inline(node.children)}{/if} + {/each} +{/snippet} + +
    + {#each blocks as block, i (i)} + {#if block.kind === 'heading'} +

    + {@render inline(block.children)} +

    + {:else if block.kind === 'paragraph'} +

    {@render inline(block.children)}

    + {:else if block.kind === 'list' && block.ordered} +
      + {#each block.items as item, j (j)}
    1. {@render inline(item)}
      2. {/each} +
    + {:else if block.kind === 'list'} +
      + {#each block.items as item, j (j)}
    • {@render inline(item)}
      • {/each} +
    + {:else} + 
    {block.text}
    + {/if} + {/each} +
    + + +``` + +`ClampedText.svelte`: +- Add the prop `markdown = false` (type `markdown?: boolean`) and + `import Markdown from './Markdown.svelte'`. +- The text element body becomes + `{#if markdown}{:else}{text}{/if}` and gets + `class:plain={!markdown}`. +- Replace the CSS for `.text` / `.clamped`: +```css + .text { + line-height: 1.5; + overflow-wrap: anywhere; + } + .plain { + white-space: pre-wrap; + } + /* A height clamp, not -webkit-line-clamp: that one does not reliably + clamp across block children, and a Markdown note is paragraphs and + lists. The fade says there is more */ + .clamped { + max-height: calc(var(--lines) * 1.5em); + overflow: hidden; + mask-image: linear-gradient(to bottom, #000 70%, transparent); + } +``` +- Update the comment at the top of the file to describe the height clamp. + +`NoteEditor.svelte`: +- Pass `markdown` to ``. +- Under the textarea, before `.actions`, add + `

    Markdown: # heading · - list · **bold** · *italic* · \`code\` · [link](https://…)

    ` + with `.hint { font-size: var(--t-sm); margin: 0.25rem 0 0; }`. + +- [ ] **Step 5: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/markdown.test.ts src/lib/ui && npm run check && npm run lint && npm test` +Expected: PASS. `NoteEditor.test.ts` "never markup" passes unchanged. + +- [ ] **Step 6: Commit** + +```bash +git add ui/src/lib/markdown.ts ui/src/lib/markdown.test.ts ui/src/lib/ui/Markdown.svelte ui/src/lib/ui/Markdown.test.ts ui/src/lib/ui/ClampedText.svelte ui/src/lib/ui/NoteEditor.svelte +git commit -m "feat(ui): notes render a safe Markdown subset" +``` + +--- + +### Task 8: Drag-and-drop uploads + +**Files:** +- Create: `ui/src/lib/dropFiles.ts`, `ui/src/lib/dropFiles.test.ts` +- Modify: `ui/src/app.css` (one `.dropping` rule) +- Modify: `ui/src/lib/pages/AssetsPage.svelte` (`upload` around line 235; wrap the page content from `.pagehead` (line 297) through the ``) +- Modify: `ui/src/lib/editor/MediaArgumentInput.svelte` (`pick` around line 74; the root `
    `) + +**Interfaces:** +- Produces: `dropFiles: Action` with + `DropOptions = { ondrop: (files: File[]) => void; accept?: (file: File) => boolean; disabled?: boolean }` + +- [ ] **Step 1: Write the failing test** + +`ui/src/lib/dropFiles.test.ts`: +```ts +import { describe, expect, it, vi } from 'vitest' +import { dropFiles } from './dropFiles' + +function dragEvent(type: string, files: File[]) { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { + value: { types: ['Files'], files, dropEffect: 'none' }, + }) + return event +} + +describe('dropFiles', () => { + it('hands over the accepted files and marks the target while dragging', () => { + const node = document.createElement('div') + const ondrop = vi.fn() + const png = new File(['x'], 'a.png', { type: 'image/png' }) + const txt = new File(['x'], 'b.txt', { type: 'text/plain' }) + const action = dropFiles(node, { + ondrop, + accept: (f) => f.type.startsWith('image/'), + }) + node.dispatchEvent(dragEvent('dragenter', [png, txt])) + expect(node.classList.contains('dropping')).toBe(true) + const drop = dragEvent('drop', [png, txt]) + node.dispatchEvent(drop) + expect(drop.defaultPrevented).toBe(true) + expect(ondrop).toHaveBeenCalledWith([png]) + expect(node.classList.contains('dropping')).toBe(false) + action?.destroy?.() + }) + + it('ignores drops while disabled', () => { + const node = document.createElement('div') + const ondrop = vi.fn() + dropFiles(node, { ondrop, disabled: true }) + node.dispatchEvent(dragEvent('drop', [new File(['x'], 'a.png')])) + expect(ondrop).not.toHaveBeenCalled() + }) +}) +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `cd ui && npx vitest run src/lib/dropFiles.test.ts` +Expected: FAIL + +- [ ] **Step 3: Implement** + +`ui/src/lib/dropFiles.ts`: +```ts +/** Drop files from the desktop onto an element. A drag that carries no + * files (text, a link, a tile being dragged inside the page) is left alone + * so it keeps its native behaviour. `dropping` marks the element while + * files hover; enter/leave are counted because they fire for every child + * the pointer crosses. */ +import type { Action } from 'svelte/action' + +export interface DropOptions { + ondrop: (files: File[]) => void + accept?: (file: File) => boolean + disabled?: boolean +} + +export const dropFiles: Action = (node, initial) => { + let options = initial + let depth = 0 + const carriesFiles = (e: DragEvent) => + !options.disabled && + Array.from(e.dataTransfer?.types ?? []).includes('Files') + const clear = () => { + depth = 0 + node.classList.remove('dropping') + } + const enter = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + depth++ + node.classList.add('dropping') + } + const over = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy' + } + const leave = () => { + if (depth > 0 && --depth === 0) node.classList.remove('dropping') + } + const drop = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + clear() + const files = Array.from(e.dataTransfer?.files ?? []).filter( + (f) => options.accept?.(f) ?? true, + ) + if (files.length) options.ondrop(files) + } + node.addEventListener('dragenter', enter) + node.addEventListener('dragover', over) + node.addEventListener('dragleave', leave) + node.addEventListener('drop', drop) + return { + update(next) { + options = next + }, + destroy() { + node.removeEventListener('dragenter', enter) + node.removeEventListener('dragover', over) + node.removeEventListener('dragleave', leave) + node.removeEventListener('drop', drop) + }, + } +} +``` + +`ui/src/app.css`, near the other global utility rules: +```css +/* A drop target with files hovering over it (dropFiles.ts). Ink, not + --select: it is the pointer's place, not a selection */ +.dropping { + outline: 2px dashed var(--ink); + outline-offset: 4px; +} +``` + +`AssetsPage.svelte`: +- Split `upload(event)` into the input handler and `uploadFiles(files: File[])`: +```ts + async function upload(event: Event) { + const input = event.target as HTMLInputElement + const files = Array.from(input.files ?? []) + // Clear it either way, so picking the same file twice still fires + input.value = '' + await uploadFiles(files) + } + + // One file asks for its library name; a dropped batch keeps each file's + // own, since a prompt per file would be a dialog storm + async function uploadFiles(files: File[]) { + if (!files.length) return + let names: (string | undefined)[] = files.map(() => undefined) + if (files.length === 1) { + const assetName = window.prompt( + `Upload — name in the ${uploadTarget} asset library:`, + files[0].name, + ) + if (assetName === null) return + names = [assetName || undefined] + } + busy = true + const failed: string[] = [] + for (const [i, file] of files.entries()) { + try { + const result = await api.uploadMedia( + file, + names[i], + uploadTarget === 'shared', + ) + if (files.length === 1) + notify.success(`Uploaded as ${result.reference ?? result.url}`) + } catch (e) { + failed.push(`${file.name}: ${e instanceof Error ? e.message : String(e)}`) + } + } + if (files.length > 1 && failed.length < files.length) + notify.success(`Uploaded ${files.length - failed.length} of ${files.length} files`) + if (failed.length) notify.error(failed.join('\n')) + busy = false + await load() + } +``` +- Import `dropFiles`. Wrap the content from `
    ` through + the end of `…` in + `
    `. + Give it `.dropzone { min-height: 60vh; }` so an empty library is still a + target. Add `multiple` to the file `` (line 314), since the handler + now takes several. + +`MediaArgumentInput.svelte`: +- Split `pick` into the event handler and `uploadFile(file: File)`. The + handler reads `e.currentTarget.files?.[0]`, clears the input, and calls + `uploadFile(file)` when there is a file. The body after `if (!file) return` + moves into `uploadFile`. +- On the root `
    ` add + `use:dropFiles={{ ondrop: (files) => uploadFile(files[0]), accept: (f) => f.type.startsWith(`${kind}/`), disabled: uploading }}`. +- Add to its title text: "or drop a file here". + +- [ ] **Step 4: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/dropFiles.test.ts src/lib/pages/AssetsPage.test.ts src/lib/editor/MediaArgumentInput.test.ts && npm run check && npm run lint && npm test` +Expected: PASS. If an existing `AssetsPage.test.ts` upload test fires +`change` with one file, it still works: one file still prompts. + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/lib/dropFiles.ts ui/src/lib/dropFiles.test.ts ui/src/app.css ui/src/lib/pages/AssetsPage.svelte ui/src/lib/editor/MediaArgumentInput.svelte +git commit -m "feat(ui): drop files onto the assets page and media inputs to upload" +``` + +--- + +### Task 9: Sidebar - hide workspaces, group by prefix + +The hidden list is **per browser** (`localStorage`). Workspaces are shared +server state, and hiding one is a viewing preference, not a change to the +workspace. The open workspace is always shown even if hidden, so you can't +lose the page you're on. Workspace names cannot contain `/` +(`^[a-zA-Z_][a-zA-Z0-9_-]*$`), so groups key on the part before the first +`-`. A group forms only when at least 3 names share that prefix. + +**Files:** +- Create: `ui/src/lib/wsGroups.ts`, `ui/src/lib/wsGroups.test.ts` +- Create: `ui/src/lib/hiddenWorkspaces.svelte.ts` +- Modify: `ui/src/lib/Sidebar.svelte` (the `{#each workspace.names …}` block, lines 144-200) +- Modify: `ui/src/lib/Sidebar.test.ts` +- Modify: `ui/src/lib/pages/OverviewPage.svelte` (Manage panel) + +**Interfaces:** +- Produces: `type WsEntry = { kind: 'ws'; name: string } | { kind: 'group'; prefix: string; names: string[] }`; + `groupWorkspaces(names: readonly string[], opts: { hidden: readonly string[]; open: string; min?: number }): { entries: WsEntry[]; hidden: string[] }` +- Produces: `hiddenWorkspaces: { names: string[] }` ($state), + `setHidden(name: string, hidden: boolean): void` + +- [ ] **Step 1: Write the failing tests** + +`ui/src/lib/wsGroups.test.ts`: +```ts +import { describe, expect, it } from 'vitest' +import { groupWorkspaces } from './wsGroups' + +const names = [ + 'acorn-wars', + 'regression-co', + 'h3-quality', + 'regression-mod', + 'regression-smoke', + 'sci0fi', +] + +describe('groupWorkspaces', () => { + it('groups three or more names that share a prefix, where the first one stood', () => { + expect(groupWorkspaces(names, { hidden: [], open: 'h3-quality' })).toEqual({ + entries: [ + { kind: 'ws', name: 'acorn-wars' }, + { + kind: 'group', + prefix: 'regression', + names: ['regression-co', 'regression-mod', 'regression-smoke'], + }, + { kind: 'ws', name: 'h3-quality' }, + { kind: 'ws', name: 'sci0fi' }, + ], + hidden: [], + }) + }) + it('leaves hidden names out, except the open one', () => { + const r = groupWorkspaces(names, { + hidden: ['sci0fi', 'h3-quality'], + open: 'h3-quality', + }) + expect(r.hidden).toEqual(['sci0fi']) + expect(r.entries).toContainEqual({ kind: 'ws', name: 'h3-quality' }) + expect(r.entries).not.toContainEqual({ kind: 'ws', name: 'sci0fi' }) + }) + it('does not group a pair', () => { + const r = groupWorkspaces(['a-1', 'a-2', 'b'], { hidden: [], open: 'b' }) + expect(r.entries.every((e) => e.kind === 'ws')).toBe(true) + }) + it('hiding names can dissolve a group below the minimum', () => { + const r = groupWorkspaces(names, { + hidden: ['regression-co'], + open: 'h3-quality', + }) + expect(r.entries.some((e) => e.kind === 'group')).toBe(false) + }) +}) +``` + +Add to `Sidebar.test.ts`, following that file's existing setup for +`workspace.names`: +```ts +it('folds a run of same-prefix workspaces into one group row', async () => { + workspace.names = ['regression-a', 'regression-b', 'regression-c', 'main'] + workspace.current = 'main' + render(Sidebar, { props: { collapsed: false, onToggle: () => {} } }) + const group = screen.getByRole('button', { name: /regression/ }) + expect(group.getAttribute('aria-expanded')).toBe('false') + expect(screen.queryByText('regression-a')).toBeNull() + await fireEvent.click(group) + expect(screen.getByText('regression-a')).toBeTruthy() +}) +``` +Use whatever props and state setup `Sidebar.test.ts` already uses. The names +above are the only new part. + +- [ ] **Step 2: Run them to see them fail** + +Run: `cd ui && npx vitest run src/lib/wsGroups.test.ts src/lib/Sidebar.test.ts` +Expected: FAIL + +- [ ] **Step 3: Implement** + +`ui/src/lib/wsGroups.ts`: +```ts +/** The sidebar's workspace list, folded: names that share a prefix (the + * part before the first '-') become one group row once there are `min` of + * them, at the place the first one stood; names this browser hid are left + * out. The open workspace is never hidden - hiding the page you are on + * would strand you. */ +export type WsEntry = + | { kind: 'ws'; name: string } + | { kind: 'group'; prefix: string; names: string[] } + +const prefixOf = (name: string) => { + const at = name.indexOf('-') + return at > 0 ? name.slice(0, at) : null +} + +export function groupWorkspaces( + names: readonly string[], + { + hidden, + open, + min = 3, + }: { hidden: readonly string[]; open: string; min?: number }, +): { entries: WsEntry[]; hidden: string[] } { + const shown = names.filter((n) => n === open || !hidden.includes(n)) + const counts = new Map() + for (const n of shown) { + const p = prefixOf(n) + if (p) counts.set(p, (counts.get(p) ?? 0) + 1) + } + const entries: WsEntry[] = [] + const groups = new Map() + for (const n of shown) { + const p = prefixOf(n) + if (p && (counts.get(p) ?? 0) >= min) { + let group = groups.get(p) + if (!group) { + group = { kind: 'group', prefix: p, names: [] } + groups.set(p, group) + entries.push(group) + } + group.names.push(n) + } else entries.push({ kind: 'ws', name: n }) + } + return { + entries, + hidden: names.filter((n) => n !== open && hidden.includes(n)), + } +} +``` + +`ui/src/lib/hiddenWorkspaces.svelte.ts`: +```ts +/** Workspaces this browser keeps out of the sidebar - a viewing choice, so + * it is local, not a change to the workspace on the server. */ +import { storageGet, storageSet } from './storage' + +export const hiddenWorkspaces = $state({ + names: storageGet('hidden-workspaces', []), +}) + +export function setHidden(name: string, hidden: boolean): void { + const rest = hiddenWorkspaces.names.filter((n) => n !== name) + hiddenWorkspaces.names = hidden ? [...rest, name] : rest + storageSet('hidden-workspaces', hiddenWorkspaces.names) +} +``` + +`Sidebar.svelte`: +1. Pull the body of the existing `{#each … as name (name)}` (the + open-workspace `
    ` branch and the shut `` branch) + into a snippet `{#snippet wsRow(name: string)} … {/snippet}`, unchanged. +2. Add these imports: `groupWorkspaces` from `./wsGroups`, + `hiddenWorkspaces` from `./hiddenWorkspaces.svelte`, `storageGet` and + `storageSet` from `./storage`, and `Eye` from `@lucide/svelte`. +3. Add this state: +```ts + const folded = $derived( + groupWorkspaces(workspace.names ?? [workspace.current], { + hidden: hiddenWorkspaces.names, + open: workspace.current, + }), + ) + let openGroups = $state(storageGet('open-ws-groups', [])) + function toggleGroup(prefix: string) { + openGroups = openGroups.includes(prefix) + ? openGroups.filter((p) => p !== prefix) + : [...openGroups, prefix] + storageSet('open-ws-groups', openGroups) + } + let showHidden = $state(false) +``` +4. Replace the `{#each}` with: +```svelte + {#each folded.entries as entry (entry.kind === 'ws' ? entry.name : `group:${entry.prefix}`)} + {#if entry.kind === 'ws'} + {@render wsRow(entry.name)} + {:else} + {@const isOpen = + openGroups.includes(entry.prefix) || + entry.names.includes(workspace.current)} + {#if collapsed} + {#each entry.names as name (name)}{@render wsRow(name)}{/each} + {:else} + + {#if isOpen} +
    + {#each entry.names as name (name)}{@render wsRow(name)}{/each} +
    + {/if} + {/if} + {/if} + {/each} + {#if folded.hidden.length && !collapsed} + + {#if showHidden} +
    + {#each folded.hidden as name (name)}{@render wsRow(name)}{/each} +
    + {/if} + {/if} +``` +5. Styles: + `.groupkids { padding-left: var(--space-3); }`, + `.dim { opacity: 0.7; }` (check AA contrast against `--muted` in both + themes; if it fails, drop the opacity and rely on the label), + `.hiddentoggle { display: inline-flex; gap: 0.35rem; align-items: center; font-size: var(--t-sm); padding: var(--space-1) var(--space-2); }`, + and `.wsgroup { width: 100%; text-align: left; }`. + +`OverviewPage.svelte`, in the Manage panel, before the delete button: +```svelte + +``` +(import `Eye, EyeOff` and `hiddenWorkspaces, setHidden`). + +- [ ] **Step 4: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/wsGroups.test.ts src/lib/Sidebar.test.ts && npm run check && npm run lint && npm test` +Expected: PASS (design-rules included: the group row uses no `--select`) + +- [ ] **Step 5: Commit** + +```bash +git add ui/src/lib/wsGroups.ts ui/src/lib/wsGroups.test.ts ui/src/lib/hiddenWorkspaces.svelte.ts ui/src/lib/Sidebar.svelte ui/src/lib/Sidebar.test.ts ui/src/lib/pages/OverviewPage.svelte +git commit -m "feat(ui): group same-prefix workspaces in the sidebar; hide workspaces per browser" +``` + +--- + +## Phase C - server and UI + +### Task 10: Star outputs + +Stars are saved in the run's existing `notes.json` as `"starred": [names]`. +That file is already the server-only sidecar for that run's per-file data, +and it already travels with the run. `read_notes` keeps its current return +shape, because existing tests compare it exactly. Stars get their own reader, +and the shared write path keeps both kinds of data. + +**Files:** +- Modify: `dw/notes.py` +- Modify: `dw/server/routes/media.py` (a `StarRequest` model and a `put_star` route next to `put_note`) +- Modify: `dw/server/routes/gallery.py` (listing, lines ~124-131) +- Modify: `dw/server/api_models.py` (`GalleryFile.starred`, new `StarSet`) +- Create: `tests/test_stars.py` +- Modify: `docs/SERVER.md` (notes section, ~line 804) +- Regenerate: `ui/src/lib/generated/openapi.json`, `api-schema.ts` +- Modify: `ui/src/lib/types.ts` (`StarSet`), `ui/src/lib/api.ts` (`setStarred`) +- Modify: `ui/src/lib/gallery/GalleryDetail.svelte`, `ui/src/lib/pages/GalleryPage.svelte` (+ test), `ui/src/lib/KeyboardHelp.svelte` + +**Interfaces:** +- Produces (Python): `read_starred(directory) -> set[str]`, + `set_file_starred(directory, name, starred: bool) -> bool`. The + `drop_file_note(directory, name)` function now also clears the star. +- Produces (HTTP): `PUT /api/stars` with body `{name: str, starred: bool}`, + returning `{name, starred}`. A run directory or a `workspace:`/`asset:` + target gets 400; a flat-layout file gets 409. +- Produces (TS): `api.setStarred(name: string, starred: boolean): Promise`. + `GalleryFile.starred?: boolean`. + +- [ ] **Step 1: Write the failing server tests** + +`tests/test_stars.py`: +```python +"""Stars: the "keep" mark on an output file, in the run's notes.json beside +its notes (dw/notes.py), set through PUT /api/stars and carried on the +gallery listing.""" + +from dw import notes as N + +from .test_notes_api import ( # noqa: F401 - fixtures + client, + make_run, + output_root, + run_name, + studio, +) + + +def test_star_round_trip_keeps_notes(tmp_path): + N.set_file_note(str(tmp_path), "a.png", "keeper") + assert N.set_file_starred(str(tmp_path), "a.png", True) is True + assert N.read_starred(str(tmp_path)) == {"a.png"} + # A note written after the star leaves it, and the other way round + N.set_run_note(str(tmp_path), "run note") + assert N.read_starred(str(tmp_path)) == {"a.png"} + assert N.read_notes(str(tmp_path))["files"] == {"a.png": "keeper"} + + +def test_unstarring_the_last_file_with_no_notes_removes_the_sidecar(tmp_path): + N.set_file_starred(str(tmp_path), "a.png", True) + N.set_file_starred(str(tmp_path), "a.png", False) + assert not (tmp_path / N.NOTES_FILE_NAME).exists() + + +def test_dropping_a_file_drops_its_star(tmp_path): + N.set_file_starred(str(tmp_path), "a.png", True) + N.drop_file_note(str(tmp_path), "a.png") + assert N.read_starred(str(tmp_path)) == set() + + +def test_put_star_and_the_listing_carries_it(client, output_root, make_run): + run_dir = make_run("wf", ["a.png", "b.png"]) + run = run_name(run_dir, output_root) + r = client.put("/api/stars", json={"name": f"{run}/a.png", "starred": True}) + assert r.status_code == 200 + assert r.json() == {"name": f"{run}/a.png", "starred": True} + files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} + assert files["a.png"]["starred"] is True + assert "starred" not in files["b.png"] + client.put("/api/stars", json={"name": f"{run}/a.png", "starred": False}) + files = {f["label"]: f for f in client.get("/api/gallery").json()["files"]} + assert "starred" not in files["a.png"] + + +def test_only_an_output_file_can_be_starred(client, output_root, make_run): + run = run_name(make_run("wf", ["a.png"]), output_root) + for name in (run, "workspace:default", "asset:x.png"): + r = client.put("/api/stars", json={"name": name, "starred": True}) + assert r.status_code == 400, name + + +def test_put_star_refuses_traversal(client): + r = client.put("/api/stars", json={"name": "../../etc/passwd", "starred": True}) + assert r.status_code in (400, 404) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pytest tests/test_stars.py -v -p no:xdist` +Expected: FAIL (`set_file_starred` is not defined; the route returns 404/405) + +- [ ] **Step 3: Implement the store (`dw/notes.py`)** + +Update the module docstring's `notes.json` bullet to: +`{"run": ..., "files": {...}, "starred": [, ...]}` - starred is the +"keep" mark on an output file. + +Replace `_store` and `_update`, and update the callers: +```python +def read_starred(directory): + """The names starred in `directory`'s notes.json, as a set.""" + starred = _read_json(os.path.join(directory, NOTES_FILE_NAME)).get("starred") + if not isinstance(starred, list): + return set() + return {name for name in starred if isinstance(name, str)} + + +def _store(directory, notes, starred): + path = os.path.join(directory, NOTES_FILE_NAME) + if notes["run"] is None and not notes["files"] and not starred: + try: + os.remove(path) + except FileNotFoundError: + pass + return + data = {k: v for k, v in notes.items() if v} + if starred: + data["starred"] = sorted(starred) + write_json_atomic(path, data) + + +def _update(directory, change): + """Read, change and write notes.json under its lock. `change` gets the + notes and the starred set and edits them in place - both, so a write of + one never drops the other.""" + with _lock(directory): + notes = read_notes(directory) + starred = read_starred(directory) + change(notes, starred) + _store(directory, notes, starred) +``` +`set_run_note` becomes +`_update(directory, lambda notes, _starred: notes.__setitem__("run", note))`. +`set_file_note`'s inner `change` becomes `def change(notes, _starred):`. +Add: +```python +def set_file_starred(directory, name, starred): + starred = bool(starred) + + def change(_notes, names): + if starred: + names.add(name) + else: + names.discard(name) + + _update(directory, change) + return starred + + +def drop_file_note(directory, name): + """Forget a deleted file: its note and its star.""" + if os.path.exists(os.path.join(directory, NOTES_FILE_NAME)): + + def change(notes, names): + notes["files"].pop(name, None) + names.discard(name) + + _update(directory, change) +``` +(this replaces the old `drop_file_note`). + +- [ ] **Step 4: Implement the route, model and listing** + +`dw/server/api_models.py`: +- In `GalleryFile`, after `note`, add: + `starred: bool = sometimes("True when the file is starred; absent otherwise.")` +- Next to `NoteSet`, add: +```python +class StarSet(ApiModel): + name: str + starred: bool +``` + +`dw/server/routes/media.py`, after `put_note`: +```python +class StarRequest(BaseModel): + name: str = Field(description="A gallery file name") + starred: bool + + +@router.put("/api/stars", response_model=StarSet) +def put_star( + request: Request, body: StarRequest, ws: Workspace = Depends(selected_workspace) +): + """Star or unstar one output file - the "keep" mark a person culls a run + with. It lives in the run's notes.json beside the notes, so it is + resolved exactly as a file note is, and only a file in a run directory + can carry one.""" + if is_ref(WORKSPACE, body.name) or is_asset_reference(body.name): + raise HTTPException(status_code=400, detail="Only an output file can be starred") + directory, name, kind = resolve_note_target(request.app.state, ws, body.name) + if kind != "file": + raise HTTPException(status_code=400, detail="Only an output file can be starred") + try: + starred = set_file_starred(directory, name, body.starred) + except OSError: + raise internal_error(f"Could not write the star on {body.name!r}") + return {"name": body.name, "starred": starred} +``` +Import `set_file_starred` with the other `..notes` imports and `StarSet` with +the other `..api_models` imports. These have to be added to existing import +lists in that file, so check them. + +`dw/server/routes/gallery.py`: import `read_starred`. Next to `notes_by_run`, +add `starred_by_run = {}`. In the `if split:` block: +```python + if split[0] not in starred_by_run: + starred_by_run[split[0]] = read_starred(split[0]) + if split[1] in starred_by_run[split[0]]: + entry["starred"] = True +``` +Find where `notes_by_run = {}` is set and add `starred_by_run = {}` next to it. + +`docs/SERVER.md`, after the notes paragraph (~line 807): +```markdown +`PUT /api/stars` with `{"name": , "starred": true|false}` +stars or unstars one output file - the "keep" mark for culling a run. A star +is stored in the run's `notes.json` (`"starred": [...]`) beside its notes, and +the gallery listing carries `"starred": true` on a starred file. Only a file +in a run directory can be starred; a run, asset or workspace target is a 400. +``` + +- [ ] **Step 5: Run the server tests** + +Run: `pytest tests/test_stars.py tests/test_notes.py tests/test_notes_api.py tests/test_notes_bookkeeping.py -v -p no:xdist` +Expected: PASS + +- [ ] **Step 6: Regenerate the UI types** + +Run: `python scripts/dump_openapi.py && cd ui && npm run gen:api` +Expected: `openapi.json` and `api-schema.ts` now contain `StarSet` and `GalleryFile.starred`. + +- [ ] **Step 7: Write the failing UI test** + +In `GalleryPage.test.ts`, add `setStarred` to the mocked `api` object, using +the file's `vi.hoisted` pattern: +`const setStarred = vi.hoisted(() => vi.fn((name: string, starred: boolean) => Promise.resolve({ name, starred })))`. +Then add: +```ts +it('stars the open file and can filter to starred files', async () => { + render(GalleryPage) + await openFile('a.png') + await fireEvent.click(screen.getByRole('button', { name: 'star this file' })) + expect(setStarred).toHaveBeenCalledWith('a.png', true) + await fireEvent.click(screen.getByRole('button', { name: 'Starred' })) + expect(screen.queryByTitle('b.png')).toBeNull() +}) +``` + +Run: `cd ui && npx vitest run src/lib/pages/GalleryPage.test.ts` +Expected: FAIL + +- [ ] **Step 8: Implement the UI** + +`types.ts`: `export type StarSet = Schemas['StarSet']`. + +`api.ts`, after `deleteOutput`: +```ts + setStarred: (name: string, starred: boolean) => + request('/api/stars', { + method: 'PUT', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ name, starred }), + }), +``` +(import the `StarSet` type.) + +`GalleryDetail.svelte`: +- Add the props `starred: boolean` and `onstar: () => void`. +- Import `Star`. +- Before the delete button, add: +```svelte + +``` + +`GalleryPage.svelte`: +- Add state `let starredOnly = $state(false)`. +- `filterActive` adds `|| starredOnly`. +- `visible`'s filter adds `&& (!starredOnly || f.starred)`. +- Add: +```ts + async function toggleStar() { + if (!selected) return + const name = selected.name + const next = !selected.starred + try { + await api.setStarred(name, next) + files = files.map((f) => (f.name === name ? { ...f, starred: next } : f)) + if (selected?.name === name) selected = { ...selected, starred: next } + } catch (e) { + notify.error(e instanceof Error ? e.message : String(e)) + } + } +``` +- In the keydown handler, after the arrow branch, add + `else if (e.key === 's' && !e.metaKey && !e.ctrlKey) { e.preventDefault(); void toggleStar() }`. +- Pass `starred={!!selected.starred}` and `onstar={toggleStar}` to ``. +- After the role chips, add + ``. +- On the tile, next to the note marker (`aria-label="Has a note"`), add + `{#if file.starred}{/if}`, + copying the note marker's classes. + +`KeyboardHelp.svelte`: add +`
    S
    star / unstar the open file (gallery details)
    `. + +- [ ] **Step 9: Run all checks** + +Run: `cd ui && npm run check && npm run lint && npm test && cd .. && pytest tests/ -v` +Expected: PASS + +- [ ] **Step 10: Commit** + +```bash +git add dw/notes.py dw/server/routes/media.py dw/server/routes/gallery.py dw/server/api_models.py tests/test_stars.py docs/SERVER.md ui/src/lib/generated ui/src/lib/types.ts ui/src/lib/api.ts ui/src/lib/gallery/GalleryDetail.svelte ui/src/lib/pages/GalleryPage.svelte ui/src/lib/pages/GalleryPage.test.ts ui/src/lib/KeyboardHelp.svelte +git commit -m "feat: star output files to keep them; gallery filters to starred" +``` + +--- + +### Task 11: Rerun lineage + +**Files:** +- Modify: `dw/server/jobs.py` (`submit` param plus spec around line 320; `rerun` passes it, ~line 545) +- Modify: `dw/server/job_record.py` (`__init__` around line 94; `summary()`) +- Modify: `dw/server/job_history.py` (migration, `record`, `recent_summaries`, `get`, `_to_detail`) +- Modify: `dw/server/api_models.py` (`JobSummary.rerun_of`) +- Create: `tests/test_rerun_lineage.py` +- Regenerate: UI types +- Modify: `ui/src/lib/pages/JobsPage.svelte`, `ui/src/lib/job/JobHeader.svelte` + +**Interfaces:** +- Produces: `JobSummary.rerun_of?: string`, which `JobDetail` inherits. It is + present only on a rerun and is kept in history. + +- [ ] **Step 1: Write the failing tests** + +`tests/test_rerun_lineage.py`: +```python +"""A rerun names the job it reran (`rerun_of`), live and from history.""" + +import sqlite3 + +from fastapi.testclient import TestClient + +from dw.server.app import create_app +from dw.server.job_history import JobHistory +from dw.server.jobs import JobManager + +from .test_server import ScriptedWorkerManager, success_script, valid_workflow +from .test_server import wait_for_status + + +def make_client(tmp_path): + (tmp_path / "workflows").mkdir(exist_ok=True) + manager = JobManager( + str(tmp_path / "outputs"), + worker_manager=ScriptedWorkerManager(success_script), + history_path=str(tmp_path / "jobs.sqlite"), + ) + app = create_app( + workflow_dir=str(tmp_path / "workflows"), + output_dir=str(tmp_path / "outputs"), + job_manager=manager, + ) + return TestClient(app, base_url="http://localhost") + + +def run_and_rerun(client): + original = client.post( + "/api/jobs", json={"workflow": valid_workflow("lineage"), "arguments": {}} + ).json()["id"] + wait_for_status(client, original, ["succeeded"]) + rerun = client.post(f"/api/jobs/{original}/rerun").json()["id"] + wait_for_status(client, rerun, ["succeeded"]) + return original, rerun + + +def test_a_rerun_names_the_job_it_reran(tmp_path): + with make_client(tmp_path) as client: + original, rerun = run_and_rerun(client) + assert client.get(f"/api/jobs/{rerun}").json()["rerun_of"] == original + assert "rerun_of" not in client.get(f"/api/jobs/{original}").json() + listed = {j["id"]: j for j in client.get("/api/jobs").json()["jobs"]} + assert listed[rerun]["rerun_of"] == original + + +def test_the_lineage_survives_a_restart(tmp_path): + with make_client(tmp_path) as client: + original, rerun = run_and_rerun(client) + with make_client(tmp_path) as client: + assert client.get(f"/api/jobs/{rerun}").json()["rerun_of"] == original + listed = {j["id"]: j for j in client.get("/api/jobs").json()["jobs"]} + assert listed[rerun]["rerun_of"] == original + + +def test_an_older_history_database_gains_the_column(tmp_path): + path = tmp_path / "old.sqlite" + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE jobs (id TEXT PRIMARY KEY, workflow TEXT, status TEXT," + " created_at REAL, started_at REAL, finished_at REAL, arguments TEXT," + " spec TEXT, manifest TEXT, warnings TEXT, error TEXT)" + ) + JobHistory(path) + with sqlite3.connect(path) as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(jobs)")} + assert "rerun_of" in columns +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `pytest tests/test_rerun_lineage.py -v -p no:xdist` +Expected: FAIL (KeyError `rerun_of`, and the column is missing) + +- [ ] **Step 3: Implement** + +`dw/server/jobs.py`: +- `submit` gains a keyword parameter `rerun_of=None`. Document it in the + docstring: "the job this one reruns, recorded so the list can say so". +- After `if note: spec["note"] = note`, add + `if rerun_of: spec["rerun_of"] = rerun_of`. +- In `rerun`, pass `rerun_of=job_id` to `self.submit(...)`. + +`dw/server/job_record.py`: +- In `__init__`, after `self.note = spec.get("note")`, add + `self.rerun_of = spec.get("rerun_of")`. +- In `summary()`, after the `note` entry, add + `**({"rerun_of": self.rerun_of} if self.rerun_of else {}),`. + +`dw/server/job_history.py`: +- Migration, after the `note` column block: +```python + # The job a rerun reran. NULL for a fresh submit and for every + # row before the column + if "rerun_of" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN rerun_of TEXT") +``` +- `record`: add `rerun_of` to the end of the column list, add one more `?` + (25 in total), and add `getattr(job, "rerun_of", None),` as the last tuple + value. +- `recent_summaries`: append `, rerun_of` after the `CASE … END` expression + in the SELECT, so it is `row[15]`. In the dict, add + `**({"rerun_of": row[15]} if row[15] else {}),`. +- `get`: append `, rerun_of` after `note` in the SELECT (`row[20]`). In + `_to_detail`, add `**({"rerun_of": row[20]} if row[20] else {}),`. + +`dw/server/api_models.py`, in `JobSummary` after `note`: +`rerun_of: str = sometimes("The job this one reran; absent for a job submitted fresh.")` + +- [ ] **Step 4: Run the server tests** + +Run: `pytest tests/test_rerun_lineage.py tests/test_rerun_new_seed.py tests/test_server_jobs.py tests/test_job_history_device.py -v -p no:xdist` +Expected: PASS + +- [ ] **Step 5: Regenerate the UI types and show the lineage** + +Run: `python scripts/dump_openapi.py && cd ui && npm run gen:api` + +`JobsPage.svelte`, after the `run_version` chip: +```svelte + {#if job.rerun_of} + rerun + {/if} +``` +(the row is itself an `    `, so the chip is not a link.) + +`JobHeader.svelte`: next to the job id / `CopyButton` in the title row, add: +```svelte + {#if job?.rerun_of} + rerun of {job.rerun_of} + {/if} +``` + +Add to `JobsPage.test.ts`, following its existing listing mock: +```ts +it('marks a rerun', async () => { + // a job in the mocked listing with rerun_of: 'abc' + ...render as the other tests do... + expect(screen.getByTitle('a rerun of abc')).toBeTruthy() +}) +``` +Fill in the setup by copying the nearest existing test in that file. The only +new part is the extra `rerun_of: 'abc'` field on one listed job. + +- [ ] **Step 6: Run all checks** + +Run: `cd ui && npm run check && npm run lint && npm test && cd .. && pytest tests/ -v` +Expected: PASS + +- [ ] **Step 7: Commit** + +```bash +git add dw/server/jobs.py dw/server/job_record.py dw/server/job_history.py dw/server/api_models.py tests/test_rerun_lineage.py ui/src/lib/generated ui/src/lib/pages/JobsPage.svelte ui/src/lib/pages/JobsPage.test.ts ui/src/lib/job/JobHeader.svelte +git commit -m "feat: a rerun records and shows the job it reran" +``` + +--- + +### Task 12: Compare view + +Pick 2–4 files in the gallery, then click **Compare**. They show side by side, +each with its version, model, seed, run time and note. Below the grid, a +"What differs" table lists only the arguments whose values differ between the +files. Videos play, pause and scrub together from one control. + +**Files:** +- Create: `ui/src/lib/compare.ts`, `ui/src/lib/compare.test.ts` +- Create: `ui/src/lib/pages/ComparePage.svelte`, `ui/src/lib/pages/ComparePage.test.ts` +- Modify: `ui/src/lib/routes.ts` (`WS_SECTIONS` adds `'compare'`) +- Modify: `ui/src/App.svelte` (route branch) +- Modify: `ui/src/lib/BulkBar.svelte` (optional `oncompare`) +- Modify: `ui/src/lib/pages/GalleryPage.svelte` (pass `oncompare`) + +**Interfaces:** +- Consumes: `api.gallery()`, `api.galleryMetadata(name)` (which returns + `{ metadata, job, ... }`), `api.getJob(id)`, `describeOutputMeta`, + `formatDuration` +- Produces: `differingArguments(metas: (Record | null)[]): string[]`, + `argumentOf(meta: Record | null, key: string): unknown`, + `valueText(value: unknown): string`, `COMPARE_MAX = 4` + +- [ ] **Step 1: Write the failing tests** + +`ui/src/lib/compare.test.ts`: +```ts +import { describe, expect, it } from 'vitest' +import { argumentOf, differingArguments, valueText } from './compare' + +const meta = (args: Record, extra = {}) => ({ + arguments: args, + ...extra, +}) + +describe('differingArguments', () => { + it('lists only the keys whose values differ, model and seed first', () => { + expect( + differingArguments([ + meta({ prompt: 'a', steps: 12, width: 1344 }, { model_name: 'h3', seed: 1 }), + meta({ prompt: 'a', steps: 30, width: 1344 }, { model_name: 'h3', seed: 2 }), + ]), + ).toEqual(['seed', 'steps']) + }) + it('a key one side lacks is a difference', () => { + expect( + differingArguments([meta({ lora: 'x' }), meta({})]), + ).toEqual(['lora']) + }) + it('compares structured values by content', () => { + expect( + differingArguments([meta({ q: { a: 1 } }), meta({ q: { a: 1 } })]), + ).toEqual([]) + }) + it('missing metadata counts as unknown, not as different', () => { + expect(differingArguments([meta({ steps: 1 }), null])).toEqual([]) + }) +}) + +describe('argumentOf / valueText', () => { + it('reads model and seed from the top level, the rest from arguments', () => { + const m = meta({ steps: 12 }, { model_name: 'h3', seed: 9 }) + expect(argumentOf(m, 'model')).toBe('h3') + expect(argumentOf(m, 'seed')).toBe(9) + expect(argumentOf(m, 'steps')).toBe(12) + }) + it('prints strings bare, the rest as JSON, cut long', () => { + expect(valueText('a cat')).toBe('a cat') + expect(valueText({ a: 1 })).toBe('{"a":1}') + expect(valueText(undefined)).toBe('—') + expect(valueText('x'.repeat(300))).toHaveLength(201) + }) +}) +``` + +`ui/src/lib/pages/ComparePage.test.ts`: +```ts +import { cleanup, render, screen } from '@testing-library/svelte' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const file = (name: string, kind = 'image') => ({ + name, + label: name.split('/').pop(), + kind, + url: `/outputs/${name}`, + folder: 'wf', + subfolder: '', + run_id: 'r', + size: 1, + mtime: 1, +}) + +vi.mock('../api', () => ({ + api: { + gallery: () => + Promise.resolve({ files: [file('wf/r1/a.png'), file('wf/r2/b.png')] }), + galleryMetadata: (name: string) => + Promise.resolve({ + metadata: { + model_name: 'h3', + seed: name.includes('a') ? 1 : 2, + arguments: { steps: 12 }, + }, + job: null, + }), + galleryThumbnailUrl: (n: string) => `/thumb/${n}`, + getJob: () => Promise.reject(new Error('none')), + }, +})) + +import ComparePage from './ComparePage.svelte' + +describe('ComparePage', () => { + afterEach(cleanup) + it('shows each file and only the argument that differs', async () => { + render(ComparePage, { props: { names: ['wf/r1/a.png', 'wf/r2/b.png'] } }) + expect(await screen.findByAltText('wf/r1/a.png')).toBeTruthy() + expect(await screen.findByRole('rowheader', { name: 'seed' })).toBeTruthy() + expect(screen.queryByRole('rowheader', { name: 'steps' })).toBeNull() + }) + it('says when a file is gone', async () => { + render(ComparePage, { props: { names: ['wf/r1/a.png', 'wf/gone.png'] } }) + expect(await screen.findByText(/no longer in the gallery/)).toBeTruthy() + }) +}) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `cd ui && npx vitest run src/lib/compare.test.ts src/lib/pages/ComparePage.test.ts` +Expected: FAIL + +- [ ] **Step 3: Implement the logic** + +`ui/src/lib/compare.ts`: +```ts +/** What a side-by-side compare reads off each file's metadata: the + * arguments that actually differ between the files. The rest are shared and + * would only bury the one change the comparison is about. */ + +export const COMPARE_MAX = 4 + +type Meta = Record | null + +export function argumentOf(meta: Meta, key: string): unknown { + if (!meta) return undefined + if (key === 'model') return meta.model_name + if (key === 'seed') + return meta.seed ?? (meta.arguments as Record)?.seed + return (meta.arguments as Record | undefined)?.[key] +} + +const same = (a: unknown, b: unknown) => JSON.stringify(a) === JSON.stringify(b) + +/** Keys that differ across the files that have metadata - model and seed + * first, then the arguments alphabetically. A file whose metadata could + * not be read is unknown, not different. */ +export function differingArguments(metas: Meta[]): string[] { + const known = metas.filter((m): m is Record => !!m) + if (known.length < 2) return [] + const keys = new Set() + for (const m of known) + for (const k of Object.keys((m.arguments as object) ?? {})) keys.add(k) + keys.delete('seed') + const ordered = ['model', 'seed', ...[...keys].sort()] + return ordered.filter((key) => { + const first = argumentOf(known[0], key) + return known.some((m) => !same(argumentOf(m, key), first)) + }) +} + +export function valueText(value: unknown): string { + if (value === undefined || value === null) return '—' + const text = typeof value === 'string' ? value : JSON.stringify(value) + return text.length > 200 ? `${text.slice(0, 200)}…` : text +} +``` + +- [ ] **Step 4: Implement the page, the route and the action** + +`routes.ts`: add `'compare'` to `WS_SECTIONS`, after `'gallery'`. +`WS_ITEMS` in `Sidebar.svelte` is a separate list, so no nav item appears. + +`App.svelte`: import `ComparePage` and add, beside the gallery branch: +`{:else if view.section === 'compare'}` + +`BulkBar.svelte`: +- Add the optional prop `oncompare?: () => void`. +- Import `Columns2` and `COMPARE_MAX`. +- Before the download button, add: +```svelte + {#if oncompare} + + {/if} +``` + +`GalleryPage.svelte`: import `go` from `'../router.svelte'` and `workspace` +from `'../workspace.svelte'` (check which are already imported). Pass +`oncompare={() => go('ws', workspace.current, 'compare', ...picks.names)}` +to ``. + +`ui/src/lib/pages/ComparePage.svelte`: +```svelte + + +
    +

    Compare

    + back to the gallery +
    + +{#if !loaded} +

    loading…

    +{:else} + {#if cells.some((c) => c.file?.kind === 'video')} +
    + + seek(Number(e.currentTarget.value))} + aria-label="position in every video" + /> + {time.toFixed(2)} s +
    + {/if} + +
    + {#each cells as cell, i (cell.name)} +
    + {#if !cell.file} +

    {cell.name} is no longer in the gallery.

    + {:else if cell.file.kind === 'video'} + + + {:else if cell.file.kind === 'image'} + {cell.name} + {:else} +

    {cell.file.label}

    + {/if} + {#if cell.file} +
    + {cell.file.label} + {#if cell.file.version}v{cell.file.version}{/if} + + {valueText(argumentOf(cell.meta, 'model'))} · seed {valueText( + argumentOf(cell.meta, 'seed'), + )}{cell.took !== null ? ` · ${formatDuration(cell.took)}` : ''} + + {#if cell.file.note}{cell.file.note}{/if} +
    + {/if} +
    + {/each} +
    + + {#if differing.length} +
    +

    What differs

    + + + {#each differing as key (key)} + + + {#each cells as cell (cell.name)} + + {/each} + + {/each} + +
    {key}{valueText(argumentOf(cell.meta, key))}
    +
    + {:else if cells.filter((c) => c.meta).length > 1} +

    Same model, seed and arguments - the files differ only in what the run did.

    + {/if} +{/if} + + +``` +If Task 5 has landed, swap the `loading…` line for +``. + +- [ ] **Step 5: Run the tests and checks** + +Run: `cd ui && npx vitest run src/lib/compare.test.ts src/lib/pages/ComparePage.test.ts src/lib/pages/GalleryPage.test.ts && npm run check && npm run lint && npm test` +Expected: PASS. Then check by hand on `npm run dev`: tick two videos, click +Compare, and confirm play, pause and scrub move both together. + +- [ ] **Step 6: Commit** + +```bash +git add ui/src/lib/compare.ts ui/src/lib/compare.test.ts ui/src/lib/pages/ComparePage.svelte ui/src/lib/pages/ComparePage.test.ts ui/src/lib/routes.ts ui/src/App.svelte ui/src/lib/BulkBar.svelte ui/src/lib/pages/GalleryPage.svelte +git commit -m "feat(ui): compare 2-4 outputs side by side with synced video and the differing arguments" +``` + +--- + +## Finish + +- [ ] From `ui/`: `npm run check && npm run lint && npm test && npx playwright test`. + Playwright starts its own server, so stop `dw.serve` on the same port first. +- [ ] From the repo root: `pytest tests/ -v` +- [ ] Rebuild the UI (`cd ui && npm run build`) and restart `dw.serve` so Tasks + 10 and 11 take effect. +- [ ] Optional follow-up, not in this plan: rewrite the long h3-quality + workspace note as Markdown (RULE / TIERS / LESSONS / PERF headings with + lists) so Task 7 shows off on real content. The MCP `set_note` tool can do + this. From a4791b9adccbf4d5ab9de98fec5f7d27f617ea46 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 18:54:52 -0500 Subject: [PATCH 114/204] feat(ui): tab title tracks the running job; alert when a job finishes Co-Authored-By: Claude Opus 5.5 --- ui/src/App.svelte | 118 ++++++++++++++++++++++++++++++--- ui/src/lib/editable.test.ts | 14 ++++ ui/src/lib/editable.ts | 13 ++++ ui/src/lib/finishAlert.test.ts | 22 ++++++ ui/src/lib/finishAlert.ts | 63 ++++++++++++++++++ ui/src/lib/jobWatch.test.ts | 104 +++++++++++++++++++++++++++++ ui/src/lib/jobWatch.ts | 71 ++++++++++++++++++++ 7 files changed, 394 insertions(+), 11 deletions(-) create mode 100644 ui/src/lib/editable.test.ts create mode 100644 ui/src/lib/editable.ts create mode 100644 ui/src/lib/finishAlert.test.ts create mode 100644 ui/src/lib/finishAlert.ts create mode 100644 ui/src/lib/jobWatch.test.ts create mode 100644 ui/src/lib/jobWatch.ts diff --git a/ui/src/App.svelte b/ui/src/App.svelte index 04709092..14a1acfd 100644 --- a/ui/src/App.svelte +++ b/ui/src/App.svelte @@ -3,6 +3,8 @@ import { cardReadings, meterText, meterTitle } from './lib/cardMemory' import { BookOpen, + Bell, + BellOff, Braces, KeyRound, Menu, @@ -12,9 +14,25 @@ } from '@lucide/svelte' import { Toaster } from 'svelte-sonner' import { route } from './lib/router.svelte' - import { api } from './lib/api' + import { api, TERMINAL_STATUSES } from './lib/api' + import { isEditable } from './lib/editable' + import { notify } from './lib/toast' + import { wsHref } from './lib/routes' + import { + endedSince, + endedTitle, + finishedText, + runningIds, + tabTitle, + } from './lib/jobWatch' + import { + alertFinished, + alertMode, + enableAlerts, + unlockChime, + } from './lib/finishAlert' import { storageGet, storageSet } from './lib/storage' - import type { HealthInfo, MemoryInfo } from './lib/types' + import type { HealthInfo, JobDetail, MemoryInfo } from './lib/types' import Sidebar from './lib/Sidebar.svelte' import Breadcrumb from './lib/Breadcrumb.svelte' import KeyboardHelp from './lib/KeyboardHelp.svelte' @@ -45,17 +63,80 @@ let statusTrigger = $state(null) let tokenTrigger = $state(null) const currentJob = $derived(health?.current_job ?? null) + const view = $derived(route.view) + + // The tab title and the finish alert (jobWatch.ts, finishAlert.ts). The + // first poll only learns what is running - a job already going when the + // page opened has not "just finished" when the next poll misses it + let watched: string[] | null = null + let runningJob = $state(null) + let othersRunning = $state(0) + let endedMark = $state(null) + let alertsOn = $state(storageGet('alert-finished', false)) + + async function watchJobs(info: HealthInfo | null) { + // A failed poll says nothing about the jobs - treating it as "none + // running" would announce every running job as finished + if (!info) return + const now = runningIds(info) + const ended = watched === null ? [] : endedSince(watched, now) + watched = now + othersRunning = Math.max(0, now.length - 1) + runningJob = now[0] ? await api.getJob(now[0]).catch(() => null) : null + for (const id of ended) { + const job = await api.getJob(id).catch(() => null) + if (job && TERMINAL_STATUSES.includes(job.status)) announce(job) + } + } + + function announce(job: JobDetail) { + const text = finishedText(job) + if (document.hidden) endedMark = endedTitle(job) + // The job's own page already shows its outcome + const onItsPage = + view.kind === 'ws' && view.section === 'jobs' && view.rest[0] === job.id + if (!onItsPage) { + if (job.status === 'succeeded') notify.success(text) + else notify.error(text) + } + if (alertsOn && document.hidden) + alertFinished(text, wsHref(job.workspace, 'jobs', job.id)) + } - function isEditable(target: EventTarget | null): boolean { - if (!(target instanceof HTMLElement)) return false - return ( - target instanceof HTMLInputElement || - target instanceof HTMLTextAreaElement || - target instanceof HTMLSelectElement || - target.isContentEditable - ) + async function toggleAlerts() { + if (alertsOn) { + alertsOn = false + } else if (await enableAlerts()) { + alertsOn = true + } else { + notify.error( + 'The browser blocked notifications for this site - allow them in its site settings', + ) + return + } + storageSet('alert-finished', alertsOn) } + $effect(() => { + document.title = endedMark ?? tabTitle(runningJob, othersRunning) + }) + $effect(() => { + const back = () => { + if (!document.hidden) endedMark = null + } + // A chime may only play after a gesture; after a reload the first + // click anywhere is that gesture + const unlock = () => { + if (alertsOn && alertMode() === 'chime') unlockChime() + } + document.addEventListener('visibilitychange', back) + window.addEventListener('pointerdown', unlock, { once: true }) + return () => { + document.removeEventListener('visibilitychange', back) + window.removeEventListener('pointerdown', unlock) + } + }) + function onKeydown(event: KeyboardEvent) { if (event.key === '?' && !isEditable(event.target)) { event.preventDefault() @@ -76,6 +157,7 @@ ]) memory = memoryInfo.status === 'fulfilled' ? memoryInfo.value : null health = healthInfo.status === 'fulfilled' ? healthInfo.value : null + void watchJobs(health) } return poll(refreshStatus, 5000) }) @@ -133,7 +215,6 @@ drawerOpen = false window.scrollTo(0, 0) }) - const view = $derived(route.view) const wide = $derived( (view.kind === 'ws' && view.section === 'edit') || (view.kind === 'shared' && view.section === 'prompt-edit'), @@ -238,6 +319,21 @@ + + {#if observedText(observed)} + {observedText(observed)} + {/if}
    @@ -137,6 +142,10 @@ background: var(--panel-2); color: var(--ink); } + .hint { + font-size: var(--t-sm); + margin: 0.25rem 0 0; + } .actions { display: flex; gap: 0.5rem; From 06b887e84c9e31d573e591075b30321f2924c87e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:10:29 -0500 Subject: [PATCH 124/204] fix(ui): linear-time link parsing; fade only clamped text that overflows Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/markdown.test.ts | 18 ++++++++++++++++++ ui/src/lib/markdown.ts | 2 +- ui/src/lib/ui/ClampedText.svelte | 3 +++ ui/src/lib/ui/ClampedText.test.ts | 10 ++++++++++ ui/src/lib/ui/NoteEditor.svelte | 4 ++-- 5 files changed, 34 insertions(+), 3 deletions(-) diff --git a/ui/src/lib/markdown.test.ts b/ui/src/lib/markdown.test.ts index 9085e1d8..1ba74e1e 100644 --- a/ui/src/lib/markdown.test.ts +++ b/ui/src/lib/markdown.test.ts @@ -34,6 +34,24 @@ describe('parseInline', () => { { kind: 'text', text: '[x](javascript:alert(1))' }, ]) }) + it('does not backtrack on unclosed links', () => { + for (const src of ['[a]('.repeat(1000), '[a](b'.repeat(800)]) { + const t = performance.now() + expect(parseInline(src)).toEqual([{ kind: 'text', text: src }]) + expect(performance.now() - t).toBeLessThan(100) + } + }) + it('links a url with balanced parentheses', () => { + expect(parseInline('[w](https://en.wikipedia.org/wiki/Foo_(bar))')).toEqual( + [ + { + kind: 'link', + href: 'https://en.wikipedia.org/wiki/Foo_(bar)', + children: [{ kind: 'text', text: 'w' }], + }, + ], + ) + }) it('leaves snake_case alone', () => { expect(parseInline('cold_minutes and warm_minutes')).toEqual([ { kind: 'text', text: 'cold_minutes and warm_minutes' }, diff --git a/ui/src/lib/markdown.ts b/ui/src/lib/markdown.ts index e0b0aaaa..59831a9a 100644 --- a/ui/src/lib/markdown.ts +++ b/ui/src/lib/markdown.ts @@ -34,7 +34,7 @@ export function safeHref(url: string): string | null { // One alternation, tried left to right at each position: code first, so a // `**` inside backticks stays literal const INLINE = - /`([^`]+)`|\*\*(.+?)\*\*|\*([^*\s][^*]*?)\*|\[([^\]]+)\]\(([^)\s]+(?:\([^)\s]*\))?[^)\s]*)\)/g + /`([^`]+)`|\*\*(.+?)\*\*|\*([^*\s][^*]*?)\*|\[([^\]]+)\]\(([^()\s]+(?:\([^()\s]*\)[^()\s]*)?)\)/g function pushText(out: Inline[], text: string) { if (!text) return diff --git a/ui/src/lib/ui/ClampedText.svelte b/ui/src/lib/ui/ClampedText.svelte index 3b1faa84..e12b953f 100644 --- a/ui/src/lib/ui/ClampedText.svelte +++ b/ui/src/lib/ui/ClampedText.svelte @@ -59,6 +59,7 @@
    { render(ClampedText, { props: { text: '- **one**', markdown: true } }) expect(screen.getByText('one').tagName).toBe('STRONG') }) + + it('fades only text the clamp actually hides', () => { + fakeHeights(40, 40) + const fits = render(ClampedText, { props: { text: 'short' } }) + expect(screen.getByText('short').classList.contains('faded')).toBe(false) + fits.unmount() + fakeHeights(400, 80) + render(ClampedText, { props: { text: 'long text' } }) + expect(screen.getByText('long text').classList.contains('faded')).toBe(true) + }) }) diff --git a/ui/src/lib/ui/NoteEditor.svelte b/ui/src/lib/ui/NoteEditor.svelte index 8ee84a7a..dbe11e08 100644 --- a/ui/src/lib/ui/NoteEditor.svelte +++ b/ui/src/lib/ui/NoteEditor.svelte @@ -3,8 +3,8 @@ import { setNote } from '../api' import ClampedText from './ClampedText.svelte' - // A note on a workspace, asset, run or file: shown as a safe Markdown subset (never HTML), edited - // in place. The server owns the 4000-character limit and says so in its + // A note on a workspace, asset, run or file: shown as a safe Markdown + // subset (never HTML), edited in place. The server owns the 4000-character limit and says so in its // error, which stays on screen beside the box let { note, From 56c45bc865bc18f4f582efefcd430022c62000f4 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:11:59 -0500 Subject: [PATCH 125/204] feat(ui): drop files onto the assets page and media inputs to upload Co-Authored-By: Claude Opus 5.5 --- ui/src/app.css | 7 + ui/src/lib/dropFiles.test.ts | 39 +++ ui/src/lib/dropFiles.ts | 62 +++++ ui/src/lib/editor/MediaArgumentInput.svelte | 16 +- ui/src/lib/pages/AssetsPage.svelte | 285 +++++++++++--------- 5 files changed, 277 insertions(+), 132 deletions(-) create mode 100644 ui/src/lib/dropFiles.test.ts create mode 100644 ui/src/lib/dropFiles.ts diff --git a/ui/src/app.css b/ui/src/app.css index e78f5506..ba72ab7e 100644 --- a/ui/src/app.css +++ b/ui/src/app.css @@ -560,6 +560,13 @@ button.frame:hover { max-width: 440px; z-index: var(--layer-modal); } +/* A drop target with files hovering over it (dropFiles.ts). Ink, not + --select: it is the pointer's place, not a selection */ +.dropping { + outline: 2px dashed var(--ink); + outline-offset: 4px; +} + .ui-sheet-wide { min-width: min(420px, 92vw); max-width: 480px; diff --git a/ui/src/lib/dropFiles.test.ts b/ui/src/lib/dropFiles.test.ts new file mode 100644 index 00000000..bf7bb52b --- /dev/null +++ b/ui/src/lib/dropFiles.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, vi } from 'vitest' +import { dropFiles } from './dropFiles' + +function dragEvent(type: string, files: File[]) { + const event = new Event(type, { bubbles: true, cancelable: true }) + Object.defineProperty(event, 'dataTransfer', { + value: { types: ['Files'], files, dropEffect: 'none' }, + }) + return event +} + +describe('dropFiles', () => { + it('hands over the accepted files and marks the target while dragging', () => { + const node = document.createElement('div') + const ondrop = vi.fn() + const png = new File(['x'], 'a.png', { type: 'image/png' }) + const txt = new File(['x'], 'b.txt', { type: 'text/plain' }) + const action = dropFiles(node, { + ondrop, + accept: (f) => f.type.startsWith('image/'), + }) + node.dispatchEvent(dragEvent('dragenter', [png, txt])) + expect(node.classList.contains('dropping')).toBe(true) + const drop = dragEvent('drop', [png, txt]) + node.dispatchEvent(drop) + expect(drop.defaultPrevented).toBe(true) + expect(ondrop).toHaveBeenCalledWith([png]) + expect(node.classList.contains('dropping')).toBe(false) + action?.destroy?.() + }) + + it('ignores drops while disabled', () => { + const node = document.createElement('div') + const ondrop = vi.fn() + dropFiles(node, { ondrop, disabled: true }) + node.dispatchEvent(dragEvent('drop', [new File(['x'], 'a.png')])) + expect(ondrop).not.toHaveBeenCalled() + }) +}) diff --git a/ui/src/lib/dropFiles.ts b/ui/src/lib/dropFiles.ts new file mode 100644 index 00000000..f1d4e746 --- /dev/null +++ b/ui/src/lib/dropFiles.ts @@ -0,0 +1,62 @@ +/** Drop files from the desktop onto an element. A drag that carries no + * files (text, a link, a tile being dragged inside the page) is left alone + * so it keeps its native behaviour. `dropping` marks the element while + * files hover; enter/leave are counted because they fire for every child + * the pointer crosses. */ +import type { Action } from 'svelte/action' + +export interface DropOptions { + ondrop: (files: File[]) => void + accept?: (file: File) => boolean + disabled?: boolean +} + +export const dropFiles: Action = (node, initial) => { + let options = initial + let depth = 0 + const carriesFiles = (e: DragEvent) => + !options.disabled && + Array.from(e.dataTransfer?.types ?? []).includes('Files') + const clear = () => { + depth = 0 + node.classList.remove('dropping') + } + const enter = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + depth++ + node.classList.add('dropping') + } + const over = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + if (e.dataTransfer) e.dataTransfer.dropEffect = 'copy' + } + const leave = () => { + if (depth > 0 && --depth === 0) node.classList.remove('dropping') + } + const drop = (e: DragEvent) => { + if (!carriesFiles(e)) return + e.preventDefault() + clear() + const files = Array.from(e.dataTransfer?.files ?? []).filter( + (f) => options.accept?.(f) ?? true, + ) + if (files.length) options.ondrop(files) + } + node.addEventListener('dragenter', enter) + node.addEventListener('dragover', over) + node.addEventListener('dragleave', leave) + node.addEventListener('drop', drop) + return { + update(next) { + options = next + }, + destroy() { + node.removeEventListener('dragenter', enter) + node.removeEventListener('dragover', over) + node.removeEventListener('dragleave', leave) + node.removeEventListener('drop', drop) + }, + } +} diff --git a/ui/src/lib/editor/MediaArgumentInput.svelte b/ui/src/lib/editor/MediaArgumentInput.svelte index 8a27f7cb..c4e2dfd5 100644 --- a/ui/src/lib/editor/MediaArgumentInput.svelte +++ b/ui/src/lib/editor/MediaArgumentInput.svelte @@ -13,6 +13,7 @@ import { ASSET, OUTPUT, referenceName } from '../references' import { ensureOutputLibrary, outputNamed } from '../outputlib.svelte' import type { MediaKind } from '../editor' + import { dropFiles } from '../dropFiles' let { id, @@ -74,8 +75,10 @@ async function pick(e: Event & { currentTarget: HTMLInputElement }) { const file = e.currentTarget.files?.[0] e.currentTarget.value = '' - if (!file) return + if (file) await uploadFile(file) + } + async function uploadFile(file: File) { error = '' setLocalPreview(URL.createObjectURL(file)) uploading = true @@ -103,7 +106,14 @@ }} /> -
    +
    uploadFile(files[0]), + accept: (f) => f.type.startsWith(`${kind}/`), + disabled: uploading, + }} +>
    fileInput?.click()} diff --git a/ui/src/lib/pages/AssetsPage.svelte b/ui/src/lib/pages/AssetsPage.svelte index e5f3705c..0ab640fb 100644 --- a/ui/src/lib/pages/AssetsPage.svelte +++ b/ui/src/lib/pages/AssetsPage.svelte @@ -9,6 +9,7 @@ import { api } from '../api' import Empty from '../Empty.svelte' import FolderGroups from '../FolderGroups.svelte' + import { dropFiles } from '../dropFiles' import HintBar from '../HintBar.svelte' import { confirmDialog } from '../confirm.svelte' import BulkBar from '../BulkBar.svelte' @@ -234,30 +235,49 @@ async function upload(event: Event) { const input = event.target as HTMLInputElement - const file = input.files?.[0] + const files = Array.from(input.files ?? []) // Clear it either way, so picking the same file twice still fires input.value = '' - if (!file) return - const suggestion = file.name - const assetName = window.prompt( - `Upload — name in the ${uploadTarget} asset library:`, - suggestion, - ) - if (assetName === null) return - busy = true - try { - const result = await api.uploadMedia( - file, - assetName || undefined, - uploadTarget === 'shared', + await uploadFiles(files) + } + + // One file asks for its library name; a dropped batch keeps each file's + // own, since a prompt per file would be a dialog storm + async function uploadFiles(files: File[]) { + if (!files.length) return + let names: (string | undefined)[] = files.map(() => undefined) + if (files.length === 1) { + const assetName = window.prompt( + `Upload — name in the ${uploadTarget} asset library:`, + files[0].name, ) - notify.success(`Uploaded as ${result.reference ?? result.url}`) - await load() - } catch (e) { - notify.error(e instanceof Error ? e.message : String(e)) - } finally { - busy = false + if (assetName === null) return + names = [assetName || undefined] + } + busy = true + const failed: string[] = [] + for (const [i, file] of files.entries()) { + try { + const result = await api.uploadMedia( + file, + names[i], + uploadTarget === 'shared', + ) + if (files.length === 1) + notify.success(`Uploaded as ${result.reference ?? result.url}`) + } catch (e) { + failed.push( + `${file.name}: ${e instanceof Error ? e.message : String(e)}`, + ) + } } + if (files.length > 1 && failed.length < files.length) + notify.success( + `Uploaded ${files.length - failed.length} of ${files.length} files`, + ) + if (failed.length) notify.error(failed.join('\n')) + busy = false + await load() } async function remove(asset: AssetFile) { @@ -294,130 +314,134 @@ }} /> -
    -

    {shared ? 'Shared assets' : 'Assets'}

    - {shownCount} files - - {#if !shared && elsewhereCount} - shared {elsewhereCount} - {/if} - - - -
    + +
    -{#if error}

    Could not read the asset library: {error}

    {/if} + {#if error}

    Could not read the asset library: {error}

    {/if} - - An asset is input a workflow names by reference: an argument set to asset:name - loads this file at run time, whatever run produced it. - {#if shared} - The shared library follows you between workspaces; an examples library is - read-only. - {:else} - This workspace's own assets come first, and a name here hides the same name - in the shared library. Shared assets are listed here only when one of this - workspace's workflows names them; the rest are on the Shared assets page. - {/if} - - -{#if loaded && !error && shownCount === 0} - - {#snippet icon()}{/snippet} - Nothing in the asset library yet — upload a file, or keep one from the gallery. - {#if !shared && elsewhereCount} - The shared library has {elsewhereCount} - more. + + An asset is input a workflow names by reference: an argument set to + asset:name loads this file at run time, whatever run produced it. + {#if shared} + The shared library follows you between workspaces; an examples library is + read-only. + {:else} + This workspace's own assets come first, and a name here hides the same + name in the shared library. Shared assets are listed here only when one of + this workspace's workflows names them; the rest are on the Shared assets + page. {/if} - -{/if} + - + {#if loaded && !error && shownCount === 0} + + {#snippet icon()}{/snippet} + Nothing in the asset library yet — upload a file, or keep one from the gallery. + {#if !shared && elsewhereCount} + The shared library has {elsewhereCount} + more. + {/if} + + {/if} -{#each sections as section (section.origin)} - toggleLibrary(section.origin)} - onupload={startUpload} + noun="asset" + ondownload={downloadPicked} + ondelete={removePicked} /> - {#if isOpen(section.origin)} - a.name)} - groupOf={(name) => folderByName.get(name) ?? ''} - collapseKey="collapsed-asset-folders-{section.origin}" - {filterActive} - minColumn="150px" - > - {#snippet card(name)} - {@const asset = byName.get(name)!} -
    - - {#if section.writable} - picks.toggle(name, e.shiftKey)} - /> - {/if} - -
    - {/snippet} -     + {#if asset.kind === 'image'} + {asset.name} + {:else if asset.kind === 'video'} + + {:else} + {asset.kind === 'audio' ? '♪' : '¶'} + {leafName(asset.name)} + {/if} + {leafName(asset.name)} + +
    + {/snippet} + - {#if section.shadowed.length} - + {#if section.shadowed.length} + + {/if} {/if} - {/if} -{/each} + {/each} +
    {#if loaded && assets.length > 0 && ordered.length === 0}

    Nothing matches "{filter}".

    @@ -434,6 +458,9 @@ {/if} diff --git a/ui/src/lib/pages/ComparePage.test.ts b/ui/src/lib/pages/ComparePage.test.ts new file mode 100644 index 00000000..6fc64fee --- /dev/null +++ b/ui/src/lib/pages/ComparePage.test.ts @@ -0,0 +1,48 @@ +import { cleanup, render, screen } from '@testing-library/svelte' +import { afterEach, describe, expect, it, vi } from 'vitest' + +const file = (name: string, kind = 'image') => ({ + name, + label: name.split('/').pop(), + kind, + url: `/outputs/${name}`, + folder: 'wf', + subfolder: '', + run_id: 'r', + size: 1, + mtime: 1, +}) + +vi.mock('../api', () => ({ + api: { + gallery: () => + Promise.resolve({ files: [file('wf/r1/a.png'), file('wf/r2/b.png')] }), + galleryMetadata: (name: string) => + Promise.resolve({ + metadata: { + model_name: 'h3', + seed: name.includes('a') ? 1 : 2, + arguments: { steps: 12 }, + }, + job: null, + }), + galleryThumbnailUrl: (n: string) => `/thumb/${n}`, + getJob: () => Promise.reject(new Error('none')), + }, +})) + +import ComparePage from './ComparePage.svelte' + +describe('ComparePage', () => { + afterEach(cleanup) + it('shows each file and only the argument that differs', async () => { + render(ComparePage, { props: { names: ['wf/r1/a.png', 'wf/r2/b.png'] } }) + expect(await screen.findByAltText('wf/r1/a.png')).toBeTruthy() + expect(await screen.findByRole('rowheader', { name: 'seed' })).toBeTruthy() + expect(screen.queryByRole('rowheader', { name: 'steps' })).toBeNull() + }) + it('says when a file is gone', async () => { + render(ComparePage, { props: { names: ['wf/r1/a.png', 'wf/gone.png'] } }) + expect(await screen.findByText(/no longer in the gallery/)).toBeTruthy() + }) +}) diff --git a/ui/src/lib/pages/GalleryPage.svelte b/ui/src/lib/pages/GalleryPage.svelte index 5fba3e26..02f16c8c 100644 --- a/ui/src/lib/pages/GalleryPage.svelte +++ b/ui/src/lib/pages/GalleryPage.svelte @@ -13,6 +13,7 @@ import { confirmDialog } from '../confirm.svelte' import type { GalleryFile, GalleryInput, OutputRole } from '../types' import { workspace } from '../workspace.svelte' + import { go } from '../router.svelte' import GalleryDetail from '../gallery/GalleryDetail.svelte' import RoleTag from '../gallery/RoleTag.svelte' @@ -308,6 +309,7 @@ noun="file" ondownload={downloadPicked} ondelete={removePicked} + oncompare={() => go('ws', workspace.current, 'compare', ...picks.names)} /> Date: Wed, 14 Oct 2026 19:35:53 -0500 Subject: [PATCH 134/204] fix(ui): compare reads its clock from the first video; reset transport on reload Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/pages/ComparePage.svelte | 14 +++++++++++--- ui/src/lib/pages/ComparePage.test.ts | 27 +++++++++++++++++++++++++-- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/ui/src/lib/pages/ComparePage.svelte b/ui/src/lib/pages/ComparePage.svelte index 8ebd9140..0a261eed 100644 --- a/ui/src/lib/pages/ComparePage.svelte +++ b/ui/src/lib/pages/ComparePage.svelte @@ -48,6 +48,9 @@ } async function load(wanted: string[]) { + length = 0 + time = 0 + playing = false const { files } = await api.gallery() const byName = new Map(files.map((f) => [f.name, f])) cells = await Promise.all( @@ -65,15 +68,20 @@ const differing = $derived(differingArguments(cells.map((c) => c.meta))) - // One transport for every video: the first is the clock the slider reads + // One transport for every video let videos = $state([]) let playing = $state(false) let time = $state(0) let length = $state(0) + // The first video is the clock the slider reads, whatever cells precede it + const clock = $derived(cells.findIndex((c) => c.file?.kind === 'video')) const present = () => videos.filter(Boolean) function playPause() { playing = !playing - for (const v of present()) void (playing ? v.play() : v.pause()) + for (const v of present()) { + if (!playing) v.pause() + else void Promise.resolve(v.play()).catch(() => (playing = false)) + } } function seek(to: number) { time = to @@ -126,7 +134,7 @@ muted playsinline ontimeupdate={(e) => { - if (i === 0) time = e.currentTarget.currentTime + if (i === clock) time = e.currentTarget.currentTime }} onloadedmetadata={(e) => { length = Math.max(length, e.currentTarget.duration) diff --git a/ui/src/lib/pages/ComparePage.test.ts b/ui/src/lib/pages/ComparePage.test.ts index 6fc64fee..b74c8779 100644 --- a/ui/src/lib/pages/ComparePage.test.ts +++ b/ui/src/lib/pages/ComparePage.test.ts @@ -1,4 +1,4 @@ -import { cleanup, render, screen } from '@testing-library/svelte' +import { cleanup, fireEvent, render, screen } from '@testing-library/svelte' import { afterEach, describe, expect, it, vi } from 'vitest' const file = (name: string, kind = 'image') => ({ @@ -16,7 +16,14 @@ const file = (name: string, kind = 'image') => ({ vi.mock('../api', () => ({ api: { gallery: () => - Promise.resolve({ files: [file('wf/r1/a.png'), file('wf/r2/b.png')] }), + Promise.resolve({ + files: [ + file('wf/r1/a.png'), + file('wf/r2/b.png'), + file('wf/v1.mp4', 'video'), + file('wf/v2.mp4', 'video'), + ], + }), galleryMetadata: (name: string) => Promise.resolve({ metadata: { @@ -45,4 +52,20 @@ describe('ComparePage', () => { render(ComparePage, { props: { names: ['wf/r1/a.png', 'wf/gone.png'] } }) expect(await screen.findByText(/no longer in the gallery/)).toBeTruthy() }) + it('reads the clock from the first video, not the first cell', async () => { + const { container } = render(ComparePage, { + props: { names: ['wf/r1/a.png', 'wf/v1.mp4', 'wf/v2.mp4'] }, + }) + await screen.findByLabelText('position in every video') + const [first, second] = container.querySelectorAll('video') + const slider = screen.getByLabelText( + 'position in every video', + ) as HTMLInputElement + second.currentTime = 7 + await fireEvent(second, new Event('timeupdate')) + expect(slider.value).toBe('0') + first.currentTime = 3 + await fireEvent(first, new Event('timeupdate')) + expect(screen.getByText('3.00 s')).toBeTruthy() + }) }) From f65d5d8ce041fe75d5a5f1a716c24f1ef810b802 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:45:42 -0500 Subject: [PATCH 135/204] fix(server): #913 - delete_workspace drops the workspace's job history Co-Authored-By: Claude Sonnet 5.5 --- dw/server/job_history.py | 10 ++++++++ dw/server/jobs.py | 13 ++++++++++ dw/server/routes/library.py | 1 + tests/test_forget_workspace_history.py | 33 ++++++++++++++++++++++++++ 4 files changed, 57 insertions(+) create mode 100644 tests/test_forget_workspace_history.py diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 288c6123..6482bbe0 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -414,6 +414,16 @@ def orphan_workflow_history(self, workspace, workflow_name): (workspace, workflow_name), ) + def forget_workspace(self, workspace): + """Delete every row recorded in this workspace (#913). A deleted + workspace takes its history with it, so a new workspace reusing the + name starts with none. Returns the number of rows removed.""" + with self._lock, self._connect() as connection: + cursor = connection.execute( + "DELETE FROM jobs WHERE workspace = ?", (workspace,) + ) + return cursor.rowcount + def events_for(self, job_id): """A finished job's persisted event tail. [] for a job recorded before events were kept, None for a job history has never seen - diff --git a/dw/server/jobs.py b/dw/server/jobs.py index aaa11dc3..491cf49a 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -832,6 +832,19 @@ def _finish(self, job, status, error=None, traceback_text=None): logger.warning(f"Could not persist job {job.id}: {e}") self._trim_terminal_jobs() + def forget_workspace(self, workspace): + """Drop a deleted workspace's finished jobs, in memory and in history + (#913), so a workspace later created under the same name lists none + of them. Queued or running jobs are the caller's to refuse first.""" + with self._lock: + for job in [ + j + for j in self.jobs.values() + if j.status in TERMINAL_STATES and j.spec.get("workspace") == workspace + ]: + del self.jobs[job.id] + return self.history.forget_workspace(workspace) + def _trim_terminal_jobs(self): """Drop the oldest finished jobs from memory - history has them, and get()/list() fall through to it. Recent ones stay for event replay.""" diff --git a/dw/server/routes/library.py b/dw/server/routes/library.py index bb0e61cf..62777d31 100644 --- a/dw/server/routes/library.py +++ b/dw/server/routes/library.py @@ -302,6 +302,7 @@ def remove_workspace(request: Request, name: str, acknowledged: bool = False): except (ValueError, FileNotFoundError) as e: raise HTTPException(status_code=400, detail=str(e)) forget_workspace_usage() + manager.forget_workspace(name) logger.info(f"Deleted workspace {name}") return {"name": name, "deleted": True, "contents": contents} diff --git a/tests/test_forget_workspace_history.py b/tests/test_forget_workspace_history.py new file mode 100644 index 00000000..a9bfb940 --- /dev/null +++ b/tests/test_forget_workspace_history.py @@ -0,0 +1,33 @@ +"""A deleted workspace takes its job history with it (#913).""" + +from dw.server.job_history import JobHistory +from dw.server.job_record import Job + + +def _job(job_id, workspace): + job = Job( + { + "workflow_name": "w", + "catalog_name": "w", + "arguments": {}, + "workspace": workspace, + } + ) + job.id = job_id + job.status = "succeeded" + job.started_at = 10.0 + job.finished_at = 70.0 + return job + + +def test_forgetting_a_workspace_leaves_a_same_named_one_empty(tmp_path): + history = JobHistory(str(tmp_path / "jobs.sqlite")) + history.record(_job("a", "sweep")) + history.record(_job("b", "sweep")) + history.record(_job("c", "other")) + + assert history.forget_workspace("sweep") == 2 + + assert history.recent_summaries(workspace="sweep") == [] + assert [r["id"] for r in history.recent_summaries(workspace="other")] == ["c"] + assert history.get("a") is None From c60359b487b2c193b1c6f6bbdfb50604e59dacf9 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:48:54 -0500 Subject: [PATCH 136/204] refactor(ui): split observedTitle, parseMarkdown and uploadFiles under the complexity ratchet Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/markdown.ts | 97 +++++++++++++++++------------- ui/src/lib/observed.ts | 40 +++++++----- ui/src/lib/pages/AssetsPage.svelte | 42 +++++++++---- 3 files changed, 110 insertions(+), 69 deletions(-) diff --git a/ui/src/lib/markdown.ts b/ui/src/lib/markdown.ts index 59831a9a..5dcbcc57 100644 --- a/ui/src/lib/markdown.ts +++ b/ui/src/lib/markdown.ts @@ -75,55 +75,70 @@ const startsBlock = (line: string) => BULLET.test(line) || NUMBERED.test(line) +// Each reader takes the block starting at lines[i] and returns it with the +// index just past it, or null when the line does not start that kind + +type Read = { block: Block; next: number } | null + +function readFence(lines: string[], i: number): Read { + if (!FENCE.test(lines[i])) return null + const body: string[] = [] + i++ + while (i < lines.length && !FENCE.test(lines[i])) body.push(lines[i++]) + // Past the closing fence, or past the end when it never closed + return { block: { kind: 'code', text: body.join('\n') }, next: i + 1 } +} + +function readHeading(lines: string[], i: number): Read { + const heading = HEADING.exec(lines[i]) + if (!heading) return null + return { + block: { + kind: 'heading', + level: heading[1].length as 1 | 2 | 3, + children: parseInline(heading[2]), + }, + next: i + 1, + } +} + +function readList(lines: string[], i: number): Read { + const list = [BULLET, NUMBERED].find((re) => re.test(lines[i])) + if (!list) return null + const items: Inline[][] = [] + while (i < lines.length && list.test(lines[i])) + items.push(parseInline(list.exec(lines[i++])![1])) + return { block: { kind: 'list', ordered: list === NUMBERED, items }, next: i } +} + +// A paragraph runs until a blank line or the start of another block; its +// single newlines are kept as line breaks - notes are written as lines, not +// reflowed prose +function readParagraph(lines: string[], i: number): Read { + const children: Inline[] = [] + while (i < lines.length && lines[i].trim() && !startsBlock(lines[i])) { + if (children.length) children.push({ kind: 'br' }) + children.push(...parseInline(lines[i++])) + } + return { block: { kind: 'paragraph', children }, next: i } +} + export function parseMarkdown(src: string): Block[] { const lines = src.replace(/\r\n?/g, '\n').split('\n') const blocks: Block[] = [] let i = 0 while (i < lines.length) { - const line = lines[i] - if (!line.trim()) { - i++ - continue - } - if (FENCE.test(line)) { - const body: string[] = [] - i++ - while (i < lines.length && !FENCE.test(lines[i])) body.push(lines[i++]) - i++ // the closing fence, or past the end when it never closed - blocks.push({ kind: 'code', text: body.join('\n') }) - continue - } - const heading = HEADING.exec(line) - if (heading) { - blocks.push({ - kind: 'heading', - level: heading[1].length as 1 | 2 | 3, - children: parseInline(heading[2]), - }) + if (!lines[i].trim()) { i++ continue } - const list = BULLET.test(line) - ? BULLET - : NUMBERED.test(line) - ? NUMBERED - : null - if (list) { - const items: Inline[][] = [] - while (i < lines.length && list.test(lines[i])) - items.push(parseInline(list.exec(lines[i++])![1])) - blocks.push({ kind: 'list', ordered: list === NUMBERED, items }) - continue - } - // A paragraph runs until a blank line or the start of another block; - // its single newlines are kept as line breaks - notes are written as - // lines, not reflowed prose - const children: Inline[] = [] - while (i < lines.length && lines[i].trim() && !startsBlock(lines[i])) { - if (children.length) children.push({ kind: 'br' }) - children.push(...parseInline(lines[i++])) - } - blocks.push({ kind: 'paragraph', children }) + const read = + readFence(lines, i) ?? + readHeading(lines, i) ?? + readList(lines, i) ?? + readParagraph(lines, i)! + blocks.push(read.block) + i = read.next } return blocks } diff --git a/ui/src/lib/observed.ts b/ui/src/lib/observed.ts index 56b8d10c..7e81951b 100644 --- a/ui/src/lib/observed.ts +++ b/ui/src/lib/observed.ts @@ -13,24 +13,34 @@ export function observedText(observed: Observed): string | null { return minutes === null ? null : `${formatMinutes(minutes)} here` } -export function observedTitle(observed: Observed): string | null { - if (!observed) return null - const parts: string[] = [] - const warm = num(observed.warm_minutes) - const cold = num(observed.cold_minutes) - if (warm !== null) - parts.push( - `${formatMinutes(warm)} with the model loaded (${runs(num(observed.warm_runs) ?? 0)})`, - ) - if (cold !== null) - parts.push( - `${formatMinutes(cold)} from cold (${runs(num(observed.cold_runs) ?? 0)})`, - ) - if (!parts.length) return null +// One clause per measured state, or nothing when that state has no runs +function timing(minutes: number | null, label: string, n: unknown) { + return minutes === null + ? [] + : [`${formatMinutes(minutes)} ${label} (${runs(num(n) ?? 0)})`] +} + +// Where and since when: the card's name in parentheses, the date it began +function provenance(observed: Record) { const card = observed.name ?? observed.device const since = typeof observed.since === 'string' ? ` since ${observed.since.slice(0, 10)}` : '' - return `measured on this server${card ? ` (${card})` : ''}: ${parts.join(', ')}; ${runs(num(observed.runs) ?? 0)}${since}` + return { card: card ? ` (${card})` : '', since } +} + +export function observedTitle(observed: Observed): string | null { + if (!observed) return null + const parts = [ + ...timing( + num(observed.warm_minutes), + 'with the model loaded', + observed.warm_runs, + ), + ...timing(num(observed.cold_minutes), 'from cold', observed.cold_runs), + ] + if (!parts.length) return null + const { card, since } = provenance(observed) + return `measured on this server${card}: ${parts.join(', ')}; ${runs(num(observed.runs) ?? 0)}${since}` } diff --git a/ui/src/lib/pages/AssetsPage.svelte b/ui/src/lib/pages/AssetsPage.svelte index 6f4d0249..4f43ce93 100644 --- a/ui/src/lib/pages/AssetsPage.svelte +++ b/ui/src/lib/pages/AssetsPage.svelte @@ -242,19 +242,26 @@ } // One file asks for its library name; a dropped batch keeps each file's - // own, since a prompt per file would be a dialog storm - async function uploadFiles(files: File[], target: 'workspace' | 'shared') { - if (!files.length) return - let names: (string | undefined)[] = files.map(() => undefined) - if (files.length === 1) { - const assetName = window.prompt( - `Upload — name in the ${target} asset library:`, - files[0].name, - ) - if (assetName === null) return - names = [assetName || undefined] - } - busy = true + // own, since a prompt per file would be a dialog storm. Null is a cancel + function askNames( + files: File[], + target: 'workspace' | 'shared', + ): (string | undefined)[] | null { + if (files.length !== 1) return files.map(() => undefined) + const assetName = window.prompt( + `Upload — name in the ${target} asset library:`, + files[0].name, + ) + return assetName === null ? null : [assetName || undefined] + } + + // Each file in turn, so one failing does not stop the rest; what failed + // comes back as "name: reason" lines + async function sendAll( + files: File[], + names: (string | undefined)[], + target: 'workspace' | 'shared', + ): Promise { const failed: string[] = [] for (const [i, file] of files.entries()) { try { @@ -271,6 +278,15 @@ ) } } + return failed + } + + async function uploadFiles(files: File[], target: 'workspace' | 'shared') { + if (!files.length) return + const names = askNames(files, target) + if (!names) return + busy = true + const failed = await sendAll(files, names, target) if (files.length > 1 && failed.length < files.length) notify.success( `Uploaded ${files.length - failed.length} of ${files.length} files`, From b45af1ac92fa0177c748b55a2cab19ac4fdc8c85 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:48:55 -0500 Subject: [PATCH 137/204] fix(ui): gallery keys step on from a hidden file's place; S ignores Shift; split the key handler Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/neighbor.test.ts | 7 +++ ui/src/lib/neighbor.ts | 13 ++-- ui/src/lib/pages/GalleryPage.svelte | 91 +++++++++++++++++----------- ui/src/lib/pages/GalleryPage.test.ts | 24 ++++++++ 4 files changed, 95 insertions(+), 40 deletions(-) diff --git a/ui/src/lib/neighbor.test.ts b/ui/src/lib/neighbor.test.ts index f87c4ec6..a363a3d5 100644 --- a/ui/src/lib/neighbor.test.ts +++ b/ui/src/lib/neighbor.test.ts @@ -14,4 +14,11 @@ describe('neighbor', () => { it('is null for a name no longer listed', () => { expect(neighbor(items, 'z', 1)).toBeNull() }) + it('goes on from where an unlisted name last stood', () => { + // 'x' stood at 1, between 'a' and 'b'; 'b' has taken its place + expect(neighbor(items, 'x', 1, 1)).toBe('b') + expect(neighbor(items, 'x', -1, 1)).toBe('a') + expect(neighbor(items, 'x', -1, 0)).toBeNull() + expect(neighbor(items, 'x', 1, 3)).toBeNull() + }) }) diff --git a/ui/src/lib/neighbor.ts b/ui/src/lib/neighbor.ts index b39ddd4a..be9fd18e 100644 --- a/ui/src/lib/neighbor.ts +++ b/ui/src/lib/neighbor.ts @@ -1,12 +1,17 @@ /** The item `step` places from `current` in `items`, or null at either end - * (no wrap: arriving back at the first file reads as "nothing happened") or - * when `current` is no longer listed. */ + * (no wrap: arriving back at the first file reads as "nothing happened"). + * When `current` is no longer listed - a filter hid it, or it was unstarred + * with Starred on - `lastAt`, where it last stood, says where to go on + * from: forward is whatever now fills its place, back the one before it. + * Without one an unlisted `current` has no neighbor. */ export function neighbor( items: readonly string[], current: string, step: 1 | -1, + lastAt = -1, ): string | null { const at = items.indexOf(current) - if (at === -1) return null - return items[at + step] ?? null + if (at !== -1) return items[at + step] ?? null + if (lastAt < 0) return null + return items[step === 1 ? lastAt : lastAt - 1] ?? null } diff --git a/ui/src/lib/pages/GalleryPage.svelte b/ui/src/lib/pages/GalleryPage.svelte index 02f16c8c..59b864e4 100644 --- a/ui/src/lib/pages/GalleryPage.svelte +++ b/ui/src/lib/pages/GalleryPage.svelte @@ -147,8 +147,20 @@ busy = false } + // Where the open file stood in the grid when last seen there, so stepping + // goes on from that place once a filter or an unstar has hidden it + let selectedAt = -1 + // The grid's order, noting the open file's place while it is still listed + function gridOrder() { + const order = visible.map((f) => f.name) + const at = selected ? order.indexOf(selected.name) : -1 + if (at !== -1) selectedAt = at + return order + } + function select(file: GalleryFile) { selected = file + selectedAt = visible.findIndex((f) => f.name === file.name) metadata = null metadataLoading = true sourceJob = null @@ -209,8 +221,10 @@ ) return const name = selected.name - const order = visible.map((f) => f.name) - const after = neighbor(order, name, 1) ?? neighbor(order, name, -1) + const order = gridOrder() + const after = + neighbor(order, name, 1, selectedAt) ?? + neighbor(order, name, -1, selectedAt) try { await api.deleteOutput(name) files = files.filter((f) => f.name !== name) @@ -225,46 +239,51 @@ notify.error(msg) } } - - { - if (overlayOpen()) return - // Escape closes the thing on top, and a dialog answers it itself. The - // selection is the more recent, more surprising state to be stuck in, - // so it clears first and the detail panel on a second press - if (e.key === 'Escape') { - if (picks.size) picks.clear() - else selected = null - return - } - // Culling a run: step through what the grid shows, delete as you go. - // A key typed into the note box, or a focused video seeking, is theirs - if (!selected || isEditable(e.target)) return - if (e.key === 'ArrowRight' || e.key === 'ArrowLeft') { - const next = neighbor( - visible.map((f) => f.name), - selected.name, - e.key === 'ArrowRight' ? 1 : -1, - ) - if (next) { - e.preventDefault() - select(byName.get(next)!) - } - } else if (e.key === 'Delete' || (e.key === 'Backspace' && e.metaKey)) { + // Escape closes the thing on top, and a dialog answers it itself. The + // selection is the more recent, more surprising state to be stuck in, + // so it clears first and the detail panel on a second press + function escape() { + if (picks.size) picks.clear() + else selected = null + } + + function step(e: KeyboardEvent, by: 1 | -1) { + if (!selected) return + const next = neighbor(gridOrder(), selected.name, by, selectedAt) + const file = next ? byName.get(next) : undefined + if (!file) return + e.preventDefault() + select(file) + } + + // A bare S: with Shift, Alt or a command modifier it is someone else's + const isStarKey = (e: KeyboardEvent) => + e.key.toLowerCase() === 's' && + !(e.metaKey || e.ctrlKey || e.altKey || e.shiftKey) + + // Culling a run: step through what the grid shows, delete as you go + function cullKey(e: KeyboardEvent) { + if (e.key === 'ArrowRight') step(e, 1) + else if (e.key === 'ArrowLeft') step(e, -1) + else if (e.key === 'Delete' || (e.key === 'Backspace' && e.metaKey)) { e.preventDefault() void removeFile() - } else if ( - e.key.toLowerCase() === 's' && - !e.metaKey && - !e.ctrlKey && - !e.altKey - ) { + } else if (isStarKey(e)) { e.preventDefault() void toggleStar() } - }} -/> + } + + function onKeydown(e: KeyboardEvent) { + if (overlayOpen()) return + if (e.key === 'Escape') return escape() + // A key typed into the note box, or a focused video seeking, is theirs + if (selected && !isEditable(e.target)) cullKey(e) + } + + +

    Gallery

    diff --git a/ui/src/lib/pages/GalleryPage.test.ts b/ui/src/lib/pages/GalleryPage.test.ts index 36e79114..a0ebb630 100644 --- a/ui/src/lib/pages/GalleryPage.test.ts +++ b/ui/src/lib/pages/GalleryPage.test.ts @@ -581,3 +581,27 @@ it('the S key stars the open file', async () => { await fireEvent.keyDown(window, { key: 's' }) await waitFor(() => expect(setStarred).toHaveBeenCalledWith('a.png', true)) }) + +it('the S key leaves a bare s to the page: Shift-S does nothing', async () => { + await renderGallery() + await openFile('a.png') + await fireEvent.keyDown(window, { key: 'S', shiftKey: true }) + expect(setStarred).not.toHaveBeenCalled() +}) + +it('unstarring the open file under Starred steps on from its place', async () => { + listing.files = [ + { ...file('a.png'), starred: true }, + { ...file('b.png'), starred: true }, + file('c.png'), + ] + await renderGallery() + await fireEvent.click(screen.getByRole('button', { name: 'Starred' })) + await openFile('a.png') + await waitFor(() => expect(galleryMetadata).toHaveBeenLastCalledWith('a.png')) + await fireEvent.keyDown(window, { key: 's' }) + await waitFor(() => expect(setStarred).toHaveBeenCalledWith('a.png', false)) + await waitFor(() => expect(screen.queryByTitle('a.png')).toBeNull()) + await fireEvent.keyDown(window, { key: 'ArrowRight' }) + expect(galleryMetadata).toHaveBeenLastCalledWith('b.png') +}) From 5ca3e68a8f2598be782c2ec70131d955ebd57d30 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:48:55 -0500 Subject: [PATCH 138/204] refactor(ui): move the health-poll watch decision into nextWatch, with tests Co-Authored-By: Claude Opus 5.5 --- ui/src/App.svelte | 31 ++++++++++++------------------- ui/src/lib/jobWatch.test.ts | 28 ++++++++++++++++++++++++++++ ui/src/lib/jobWatch.ts | 22 ++++++++++++++++++++++ 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/ui/src/App.svelte b/ui/src/App.svelte index 53b77668..fdb6ed6c 100644 --- a/ui/src/App.svelte +++ b/ui/src/App.svelte @@ -18,13 +18,7 @@ import { isEditable } from './lib/editable' import { notify } from './lib/toast' import { wsHref } from './lib/routes' - import { - endedSince, - endedTitle, - finishedText, - runningIds, - tabTitle, - } from './lib/jobWatch' + import { endedTitle, finishedText, nextWatch, tabTitle } from './lib/jobWatch' import { alertFinished, alertMode, @@ -66,9 +60,8 @@ const currentJob = $derived(health?.current_job ?? null) const view = $derived(route.view) - // The tab title and the finish alert (jobWatch.ts, finishAlert.ts). The - // first poll only learns what is running - a job already going when the - // page opened has not "just finished" when the next poll misses it + // The tab title and the finish alert (jobWatch.ts, finishAlert.ts); what + // a poll means for the watched jobs is nextWatch's to decide let watched: string[] | null = null let runningJob = $state(null) let othersRunning = $state(0) @@ -76,15 +69,15 @@ let alertsOn = $state(storageGet('alert-finished', false)) async function watchJobs(info: HealthInfo | null) { - // A failed poll says nothing about the jobs - treating it as "none - // running" would announce every running job as finished - if (!info) return - const now = runningIds(info) - const ended = watched === null ? [] : endedSince(watched, now) - watched = now - othersRunning = Math.max(0, now.length - 1) - runningJob = now[0] ? await api.getJob(now[0]).catch(() => null) : null - for (const id of ended) { + const next = nextWatch(watched, info) + watched = next.watched + // A failed poll leaves the title showing what it last knew + if (info) { + const now = next.running + othersRunning = Math.max(0, now.length - 1) + runningJob = now[0] ? await api.getJob(now[0]).catch(() => null) : null + } + for (const id of next.ended) { const job = await api.getJob(id).catch(() => null) if (job && TERMINAL_STATUSES.includes(job.status)) announce(job) } diff --git a/ui/src/lib/jobWatch.test.ts b/ui/src/lib/jobWatch.test.ts index 7e52a91b..ac279a84 100644 --- a/ui/src/lib/jobWatch.test.ts +++ b/ui/src/lib/jobWatch.test.ts @@ -4,6 +4,7 @@ import { BASE_TITLE, endedSince, endedTitle, + nextWatch, percentDone, runningIds, tabTitle, @@ -38,6 +39,33 @@ describe('endedSince', () => { }) }) +describe('nextWatch', () => { + it('first poll learns what is running and ends nothing', () => { + expect(nextWatch(null, health(['a', 'b']))).toEqual({ + watched: ['a', 'b'], + ended: [], + running: ['a', 'b'], + }) + }) + it('a failed poll ends nothing and keeps what was watched', () => { + expect(nextWatch(['a', 'b'], null)).toEqual({ + watched: ['a', 'b'], + ended: [], + running: ['a', 'b'], + }) + expect(nextWatch(null, null).watched).toBeNull() + }) + it('a later real poll ends only the job that vanished', () => { + const next = nextWatch(['a', 'b'], health(['b', 'c'])) + expect(next.ended).toEqual(['a']) + expect(next.watched).toEqual(['b', 'c']) + }) + it('a real poll after a failed one still ends a vanished job', () => { + const kept = nextWatch(['a'], null).watched + expect(nextWatch(kept, health([])).ended).toEqual(['a']) + }) +}) + describe('percentDone', () => { it('counts finished steps plus the denoise fraction of the current one', () => { expect( diff --git a/ui/src/lib/jobWatch.ts b/ui/src/lib/jobWatch.ts index 7d8beb7e..6570c38b 100644 --- a/ui/src/lib/jobWatch.ts +++ b/ui/src/lib/jobWatch.ts @@ -30,6 +30,28 @@ export function endedSince( return before.filter((id) => !now.includes(id)) } +export type Watch = { + watched: string[] | null + ended: string[] + running: string[] +} + +/** One health poll's step: what is running now and which watched jobs ended. + * The first poll (`watched` null) only learns what is running - a job + * already going when the page opened has not "just finished" when the next + * poll misses it. A failed poll (`info` null) says nothing about the jobs: + * reading it as "none running" would announce every running job as + * finished, so it leaves the state as it was. */ +export function nextWatch( + watched: string[] | null, + info: HealthInfo | null, +): Watch { + if (!info) return { watched, ended: [], running: watched ?? [] } + const running = runningIds(info) + const ended = watched === null ? [] : endedSince(watched, running) + return { watched: running, ended, running } +} + /** 0-99: steps behind it plus how far the current step's denoise loop is. * Never 100 - a running job is not done. Null with nothing to go on. */ export function percentDone(progress: JobDetail['progress']): number | null { From 8dd0ff8f1d6ae1565f75e872bb3b1cb8f3670808 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:48:55 -0500 Subject: [PATCH 139/204] docs: rerun_of on job records, starred on gallery entries, stars after the notes list Co-Authored-By: Claude Opus 5.5 --- docs/MCP.md | 6 ++++-- docs/SERVER.md | 25 +++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/docs/MCP.md b/docs/MCP.md index 534eb968..30481bd9 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -203,7 +203,9 @@ a request nothing on disk covers. The instructions also say the engine runs **one job per GPU** - one worker per `dw.serve --devices` card, each running its own job from the one shared queue - and point at `get_health`'s `workers` for what each card is running. -A job record's `device` (`get_job`, `list_jobs`) names the card it ran on. +A job record's `device` (`get_job`, `list_jobs`) names the card it ran on, +and a job started by `rerun_job` carries `rerun_of`, the id of the job it +reran (absent on a fresh submit). A request usually names a *subject* ("a lego movie trailer set in the marvel universe") while the catalog is written in *shapes* - a single image, an image @@ -231,7 +233,7 @@ when no single workflow covers it. | `get_health()` | — | Check that the server is alive, and which machine answered: `version`, `device`, whether any model process is resident (`worker_alive`), and the queue depth. The engine runs one job per GPU: `workers` has one entry per card - `device`, `name`, `vram_gb`, `alive`, its own `current_job`, and the worker's `host_memory_rss_mb` while it runs - and the top-level `current_job` is the longest-running job. `worker_alive: false` is the normal idle state on a server that has not run a job since startup or the last memory clear - not a fault - the on-demand worker starts with the next job (#206) | | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server. `runtime` (#222) reports the Python version, torch version and the CUDA version torch was built against, the NVIDIA driver version (when `nvidia-smi` is reachable), and the installed versions of diffusers, transformers, accelerate, bitsandbytes, peft, safetensors and sentencepiece (`null` for one not installed) - for diagnosing an environment mismatch between boxes without shelling in. `caches` lists dw's disk caches (`inductor`, `triton`, `quantized`), each with `name`, `bytes`, `files`, `enabled`, `clearable` and `disk_free`, plus `entry_count` for a cache with entries, and `cache_entries=true` lists them (`id`, `model`, `subfolder`, `component`, `bytes`, `files`, `unreadable`, `created`, `last_hit`; the quantizer and dtype are not in `/api/caches`); it is omitted when the server has no `/api/caches`. There is no MCP tool to clear one - that is `DELETE /api/caches/{name}` or `.../entries/{entry}` ([SERVER.md](SERVER.md)) | | `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first**. Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | -| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | +| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_gallery_metadata(name, envelope=False, workspace=None, interval_seconds=None)` | `name`, `workspace` | Get the metadata embedded in a generated file — or, when `name` is an `asset:` reference, what an *input* asset holds (`source` says which; `job` is null for an asset). Reading an input's duration, frame count, fps and sample rate before a run is how a caller learns the `total_frames`, `fps` and `sample_rate` a workflow expects it to supply: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) and `findings` - each level problem the server measured against `dw/audio_qc.py`'s thresholds (`full_scale`, `near_silent`), with the fix. Only an image (PNG/JPEG/WebP) embeds `metadata` this way — it is always null for audio and video, and `next` then points at `get_job_workflow(job_id)` when `job` is known, or says a kept asset carries no provenance at all when it isn't. `envelope=true` adds `media.envelope` — `rms_dbfs` and `peak_dbfs` one entry per second, or per `interval_seconds` (default 1.0, 0.05 to 3600; 0.1 places a vocal onset; a finer-than-default width that would need over 2400 bins is refused with the interval that fits) — which is what locates something in a track rather than measuring the whole of it. `media.shots` is set on an output joined from shots: one `{name, start_frame, num_frames, start_sample, num_samples}` per shot, as the join measured them (see the run manifest in docs/WORKFLOW_GUIDE.md), else null. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | ### Media diff --git a/docs/SERVER.md b/docs/SERVER.md index 6c43ef13..a0984e00 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -201,8 +201,8 @@ running: | Route | What it does | | --- | --- | | `POST /api/jobs` | Queue a run: `{"workflow_path": ...}` or an inline `{"workflow": {...}, "base_dir": ...}`, plus `arguments` for variable overrides. `workflow_path` accepts a stored workflow name as listed by `/api/workflows` (with or without `.json`, nested names included), or a relative/absolute path that still resolves under `--workflow-dir` - confined the same way the `/api/workflows` CRUD routes are; a path that names a real file outside that directory is rejected with 400, not opened. Answers with argument warnings from signature checking. Takes an optional `acknowledged_cost`: `true` is recorded as `acknowledged: boolean`; the object `{fingerprint, minutes, downloads}` from a validate answer's `plan` is `bound` - the server re-plans the run for the arguments given and answers **409** when the fingerprint differs or a repo in `downloads_required` is not in `downloads` (a download that has since vanished is not a refusal); the body is `{"detail": {message, reason: "fingerprint" \| "downloads" \| "unplannable", acknowledged, plan, acknowledge}}` with the current plan, so the caller re-quotes from it, and `acknowledge` - the `{fingerprint, minutes, downloads}` to resend - whenever there is a plan. A `null` in `downloads` (a URL with no repo) is ignored. `minutes` is recorded, never compared. Nothing is required: the web UI and every caller that sends nothing are `acknowledged: none`, and every job answer and history row carries `acknowledged` (and `acknowledged_cost` when bound). `POST /api/jobs/{id}/rerun` takes the same field and checks against the stored spec; a fresh seed does not change a fingerprint. | -| `GET /api/jobs?workspace=&status=&limit=` | Queue + history summaries, oldest first, with `total` beside them. `status` narrows to one state or a comma-separated set (`queued`, `running`, `succeeded`, `failed`, `cancelled`; anything else is a 400); `limit` keeps the newest N, and `total` still reports how many matched, so a bounded answer cannot be mistaken for a complete one. Each summary carries `device`, the card the job ran on (`"cuda:1 NVIDIA GeForce RTX 3090"`; null while queued and for older jobs). No parameters means every job, which is what the web UI polls | -| `GET /api/jobs/{id}` | Full detail: spec, events, manifest, error. A manifest entry for a step served from the step cache carries `reused: true`. Every entry carries `subfolder` - the in-run subfolder the step's `result.subfolder` chose, `''` for none. A `for_each` step appears in the manifest as its members (`shot@wide_open`, `shot@closeup`), because the manifest records what ran; the run's `workflow.json` keeps the `for_each` form, because it records what was asked Carries `output_kinds`: each manifest file mapped to its media kind (`image`, `video`, `audio`, `text`, or null), and each file-carrying run event (`step_end`) carries the same map for its own files. The entry of a pipeline step that ran carries `timing` (see [Step timing](WORKSPACES.md#step-timing)). | +| `GET /api/jobs?workspace=&status=&limit=` | Queue + history summaries, oldest first, with `total` beside them. `status` narrows to one state or a comma-separated set (`queued`, `running`, `succeeded`, `failed`, `cancelled`; anything else is a 400); `limit` keeps the newest N, and `total` still reports how many matched, so a bounded answer cannot be mistaken for a complete one. Each summary carries `device`, the card the job ran on (`"cuda:1 NVIDIA GeForce RTX 3090"`; null while queued and for older jobs), and `rerun_of`, present on a job started by rerun: the id of the job it reran; absent for a fresh submit. No parameters means every job, which is what the web UI polls | +| `GET /api/jobs/{id}` | Full detail: spec, events, manifest, error. A manifest entry for a step served from the step cache carries `reused: true`. Every entry carries `subfolder` - the in-run subfolder the step's `result.subfolder` chose, `''` for none. A `for_each` step appears in the manifest as its members (`shot@wide_open`, `shot@closeup`), because the manifest records what ran; the run's `workflow.json` keeps the `for_each` form, because it records what was asked Carries `output_kinds`: each manifest file mapped to its media kind (`image`, `video`, `audio`, `text`, or null), and each file-carrying run event (`step_end`) carries the same map for its own files. The entry of a pipeline step that ran carries `timing` (see [Step timing](WORKSPACES.md#step-timing)). Carries `rerun_of` - present on a job started by rerun: the id of the job it reran; absent for a fresh submit. | | `DELETE /api/jobs/{id}/run` | Delete the run directory a finished job wrote, whole, from the output root the job ran against (no workspace selector applies): `{job_id, run_dir, deleted, run_swept}`. 404 for an unknown job or one with no run directory, 409 for one still queued or running | | `GET /api/jobs/{id}/workflow` | The workflow the job ran: `{id, definition, realized, seed_variable}`. `seed_variable` names the variable a `new_seed` rerun would draw into (null when the workflow has none), read from the workflow as written rather than the realized copy, whose seed is pinned. `realized: true` is the copy the run itself wrote (`workflow.json` in its run directory), with arguments, seed, prompts and `output:latest` pinned; `false` falls back to the submitted definition, which is what a job from before run tracking has. 404 means neither is readable - the job itself still is. The equivalent MCP tool is `get_job_workflow` (see [MCP.md](MCP.md#diagnose)) | | `POST /api/jobs/{id}/export?workspace=&overwrite=` | Gather one finished job into `/exports//`: `workflow.json`, `manifest.json`, `job.json`, `README.md`, `assets/`, `inputs/`, `outputs/`. 201 with the file list, total bytes, anything it could not find, a `zip_url`, and the three JSON files inline. 404 unknown job, 409 for a job still running or an existing export without `overwrite` | @@ -210,7 +210,7 @@ running: | `GET /api/jobs/{id}/events` | Server-sent events stream; `?after=N` / `Last-Event-ID` replay missed events, so reconnects are lossless. Every event carries `seq` and `at` - seconds since the job started (since it was queued, for the events before that) - so a step's cost is a subtraction: `step_start` to `generating` is what a reused pipeline still pays before it runs; `generating` is emitted when the pipeline call starts, so `generating` to the first `pipeline_step` is the prompt and reference encoding plus the first denoise pass (`timing.encode_s` in the manifest is the encoding alone; see [Step timing](WORKSPACES.md#step-timing)). The server keeps only the last 200 events, so on a long job the early ones are gone | | `GET /api/jobs/{id}/event-log?after=-1&limit=200` | The same events as the SSE stream, as one JSON page: `{id, status, events, last_seq, truncated, note}`. `after` is exclusive; page by passing back the previous `last_seq`. A job restored from history serves the bounded event tail persisted with it; a job that finished before events were retained returns an empty list and a `note` saying so. | | `POST /api/jobs/{id}/cancel` | Cooperative cancel (takes effect at the next step boundary or denoise step) | -| `POST /api/jobs/{id}/rerun` | Re-queue a finished job's spec. Body `{"new_seed": true}` draws a fresh seed into the workflow's seed variable instead of repeating the original arguments; 400 when the workflow pins its seed to a literal or names none. A plain rerun of a seeded workflow is served whole from the step cache — the earlier run's files, republished with `reused: true`, generating nothing | +| `POST /api/jobs/{id}/rerun` | Re-queue a finished job's spec. Body `{"new_seed": true}` draws a fresh seed into the workflow's seed variable instead of repeating the original arguments; 400 when the workflow pins its seed to a literal or names none. A plain rerun of a seeded workflow is served whole from the step cache — the earlier run's files, republished with `reused: true`, generating nothing. The new job's summary and detail carry `rerun_of`: the id of the job it reran (absent on a fresh submit) | | `POST /api/jobs/{id}/move` | Reorder a queued job: `{"direction": "up"\|"down"\|"front"\|"back"}`. Job listings carry each waiting job's `queue_position`. | Each worker runs one job at a time, so with one device (the default) one job @@ -826,12 +826,6 @@ one that would leave its root is a 404 (a name a workspace cannot take, or an invalid asset name, a 400), and a write that fails a 500. Notes are free metadata and need no `acknowledged_cost`. -`PUT /api/stars` with `{"name": , "starred": true|false}` -stars or unstars one output file - the "keep" mark for culling a run. A star -is stored in the run's `notes.json` (`"starred": [...]`) beside its notes, and -the gallery listing carries `"starred": true` on a starred file. Only a file -in a run directory can be starred; a run, asset or workspace target is a 400. - Notes are set at creation too: - `POST /api/jobs` takes `note`. It is kept on the job (`note` in every job @@ -846,9 +840,16 @@ Notes are set at creation too: They are read back on every listing: a gallery entry's `note` is the effective one; the metadata route carries `note`, `note_source` and the run's `run_note`; an asset entry and a workspace entry carry `note`. Each is -absent when unset. Deleting an output drops its note; deleting an asset -drops its note and its kept provenance and shots, so a later file under the -same name inherits none of them. +absent when unset. Deleting an output drops its note and its star; deleting +an asset drops its note and its kept provenance and shots, so a later file +under the same name inherits none of them. + +`PUT /api/stars` with `{"name": , "starred": true|false}` +stars or unstars one output file - the "keep" mark for culling a run. A star +is stored in the run's `notes.json` (`"starred": [...]`) beside its notes, and +the gallery listing carries `"starred": true` on a starred file. Only a file +in a run directory can be starred: a flat-layout file has no run directory +and answers 409, and a run, asset or workspace target is a 400. ## Security model From 33df797a32a87fb36977c03bb11213387b688563 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:48:55 -0500 Subject: [PATCH 140/204] chore: sort the test_stars ignore; reflow a NoteEditor comment Co-Authored-By: Claude Opus 5.5 --- pyproject.toml | 2 +- ui/src/lib/ui/NoteEditor.svelte | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6ecd2abe..62bb4ed0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -170,10 +170,10 @@ ignore = ["E501"] "tests/test_cli.py" = ["F811"] "tests/test_h3_hold_audio.py" = ["F811"] "tests/test_jobs_listing.py" = ["F811"] -"tests/test_stars.py" = ["F811"] "tests/test_server_downloads.py" = ["F811"] "tests/test_server_exports.py" = ["F811"] "tests/test_server_jobs.py" = ["F811"] +"tests/test_stars.py" = ["F811"] [tool.setuptools.packages.find] # tasks/, pipeline_processors/ and community_pipelines/ are namespace diff --git a/ui/src/lib/ui/NoteEditor.svelte b/ui/src/lib/ui/NoteEditor.svelte index dbe11e08..206f16eb 100644 --- a/ui/src/lib/ui/NoteEditor.svelte +++ b/ui/src/lib/ui/NoteEditor.svelte @@ -4,8 +4,8 @@ import ClampedText from './ClampedText.svelte' // A note on a workspace, asset, run or file: shown as a safe Markdown - // subset (never HTML), edited in place. The server owns the 4000-character limit and says so in its - // error, which stays on screen beside the box + // subset (never HTML), edited in place. The server owns the 4000-character + // limit and says so in its error, which stays on screen beside the box let { note, target, From 13703bc356282d8bc69e625e1dd3310194a9219f Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:50:06 -0500 Subject: [PATCH 141/204] test(plan): #905 - pin the H3 inline round trip, seed out of the fingerprint, seed and file name in the cache key Co-Authored-By: Claude Opus 5.5 --- tests/test_plan.py | 24 ++++++++++++++++++++++++ tests/test_workflow_step_cache.py | 13 +++++++++++++ 2 files changed, 37 insertions(+) diff --git a/tests/test_plan.py b/tests/test_plan.py index b807694f..b0f01ff2 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -436,6 +436,30 @@ def test_cost_drivers_leave_the_fingerprint(self, plan, template): del other["vram_estimate"] assert plan(template)["fingerprint"] == plan(other)["fingerprint"] + def test_a_client_round_trip_keeps_it(self, plan, template): + # The inline get_workflow JSON comes back with video_shift 12.0 as + # 12 (and its kin); the by-name form keeps the float (#905) + def respell(value): + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, dict): + return {key: respell(item) for key, item in value.items()} + if isinstance(value, list): + return [respell(item) for item in value] + return value + + inline = respell(template) + # The template does hold one, or this checks nothing + assert json.dumps(inline) != json.dumps(template) + assert plan(template)["fingerprint"] == plan(inline)["fingerprint"] + + def test_the_seed_stays_out(self, plan, template): + # The #85 contract: an acknowledgement buys the cost, not the seed + assert ( + plan(template, arguments={"seed": 42})["fingerprint"] + == plan(template, arguments={"seed": 43})["fingerprint"] + ) + @pytest.mark.parametrize( "arguments", [{"attention_backend": "native"}, {"weights_dtype": "{int8}"}], diff --git a/tests/test_workflow_step_cache.py b/tests/test_workflow_step_cache.py index 7abd9e19..aeff9466 100644 --- a/tests/test_workflow_step_cache.py +++ b/tests/test_workflow_step_cache.py @@ -1368,3 +1368,16 @@ def test_a_subfolder_is_a_miss(self, tmp_path): other = copy.deepcopy(spec) other["steps"][0]["result"]["subfolder"] = "elsewhere" assert _runs_of(tmp_path, spec, other) == 2 + + def test_a_file_base_name_is_a_miss(self, tmp_path): + spec = _with_output_settings() + other = copy.deepcopy(spec) + other["steps"][0]["result"]["file_base_name"] = "renamed" + assert _runs_of(tmp_path, spec, other) == 2 + + def test_a_seed_is_a_miss(self, tmp_path): + # The fingerprint leaves the seed out (#85); the cache never does + spec = _with_output_settings() + other = copy.deepcopy(spec) + other["seed"] = 43 + assert _runs_of(tmp_path, spec, other) == 2 From 9dfb9e6c285822367ab4d67c400af57a975e85be Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 19:50:20 -0500 Subject: [PATCH 142/204] docs(architecture): #905 - fingerprint row states the number and seed rules Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 80a27873..0dbdb151 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -94,7 +94,7 @@ one to open. | Step-value checks | `dw/step_value_checks.py`: `fps_errors`, `cache_errors`, `null_media_errors`, `select_errors`, `chain_prompts_errors` | The only caller of these checks is the error registry. | `dw/validation.py`: `ERROR_CHECKS` | | Admission | `dw/server/admission.py`: `admit` | Validate, submit, rerun and enhance load and check a request once, and `JobManager.submit` does not check it again. | `tests/test_admission.py::test_a_submit_expands_once` | | Variable constraints | `dw/variable_constraints.py`: `apply_constraints`; the grid arithmetic `aligned` / `aligned_down` | A model's rule about a value is declared in the workflow's `variable_constraints`, and it is checked at validation and again before anything loads. `plan_cuts` (and its static check) rounds render lengths through `aligned` / `aligned_down`, never its own copy, so a planned `num_frames` is one the template's constraint accepts; `crop_face_track` pads its crops to the step's `modulus`/`remainder` grid through `aligned` too (`padding_to_grid`). | `tests/test_variable_constraints.py::TestTheGrid::test_without_snap_an_off_grid_value_is_refused_not_rounded`, `tests/test_variable_constraints.py::TestTheGrid::test_plan_cuts_rounds_through_the_constraint_owner`, `tests/test_face_track.py::TestPadding::test_padding_to_grid_rounds_through_the_constraint_owner`, `tests/test_variable_constraints.py::TestTheRunTimePass::test_an_illegal_value_is_refused_before_anything_loads` | -| Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | +| Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | | The estimate | `dw/plan.py`: `estimate`, `_tempered` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure` | From cf33dd181184ba4272beb7872733e36362f7613c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:04:43 -0500 Subject: [PATCH 143/204] docs(workflows): #914 - step cache is per card and per workflow identity Co-Authored-By: Claude Sonnet 5.5 --- docs/WORKFLOW_GUIDE.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 085f1b57..d125082c 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -2643,6 +2643,26 @@ is pinned to, not for every workspace that happens to hold a matching run. Deleting a workspace takes its cache entries with it, the same as deleting its `outputs/` directory would. +The step cache is also per card and per workflow identity. Identity is the stored +workflow's name, or an inline workflow's `id`; entries are keyed on the definition `id` +and step name (`StepCache.get` in `dw/step_cache.py`), and an inline job's identity is +`("inline", id)` (`workflow_identity` in `dw/worker_protocol.py`). Switching identity on a +card empties that card's cache (the worker's `_cleanup_all` calls `step_cache.clear()`), +so: + +- running any other workflow on the card in between starts the next run cold: A, then B, + then A runs the second A cold. This is the common case. On a multi-card server + warm-identity routing (`choose_slot` in `dw/server/pool.py`) mostly avoids it unless + that card is busy; on a single card it happens every time. +- a renamed copy, a new inline `id`, or switching between a stored workflow and its + inline copy also starts cold. + +To reuse cached upstream steps, rerun the same workflow (same name, or the same inline +`id`), for example with `rerun_job` or changed arguments. Only arguments that reach +*later* steps leave the earlier steps cached: changing an argument the expensive first +step reads re-runs that step. To check before running, `validate_workflow`'s +`plan.cached_steps` says how many steps would be reused. + ## Type System Dynamic type conversion applies to certain values: From ddd09708508ae5fcfc1b404893fef1c0cf4509db Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:18:29 -0500 Subject: [PATCH 144/204] feat(plan): #906 - scale the curated estimate along steps, canvas and frames A cost entry is measured at the workflow's variable defaults; a run that moves num_inference_steps, width/height or num_frames (per entry in a list, summed) is quoted scaled in proportion, basis derived, with `scaled` naming each moved axis. It multiplies with the list-length re-price and is what a thin observed figure tempers toward. dialogue-short's 42 min was measured at 1344x768 and is re-priced to its 960x544 defaults: 21.2. Co-Authored-By: Claude Opus 5.5 --- dw/plan.py | 252 +++++++++++---- tests/test_catalog_structure.py | 8 +- tests/test_plan.py | 294 ++++++++++++++++-- .../templates/minimax/dialogue-short.json | 4 +- 4 files changed, 476 insertions(+), 82 deletions(-) diff --git a/dw/plan.py b/dw/plan.py index 05d224d8..d47f6f79 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -470,44 +470,184 @@ def _list_entry_field_shifted(default_entries, effective_entries): return False -def _per_entry_variable(cost, device): - """The list variable the cost entry priced for `device` (else the first, - as `_price` chooses) re-prices by a measured per-entry rate, or None.""" - entries = [entry for entry in (cost or []) if isinstance(entry, dict)] - chosen = next((e for e in entries if e.get("device") == device), None) - if chosen is None and entries: - chosen = entries[0] - per = chosen.get("per_entry") if chosen else None - return per.get("variable") if isinstance(per, dict) else None +# The generic diffusers argument names a curated figure scales along when a +# run moves one off the default it was measured at (#866). Any other declared +# driver has no rule to scale by, so moving it leaves no figure (#267). +STEPS_DRIVER = "num_inference_steps" +FRAMES_DRIVER = "num_frames" +SCALABLE_DRIVERS = (STEPS_DRIVER, "width", "height", FRAMES_DRIVER) + + +def _positive(value): + """A driver value as a positive finite number, or None.""" + number = _driver_comparable(value) + if isinstance(number, float) and math.isfinite(number) and number > 0: + return number + return None + + +def _shown(number): + return f"{round(number, 1):g}" + + +def scale_curated(minutes, defaults, effective): + """`minutes` measured at `defaults`, scaled to a run at `effective`, as + (minutes, axes), or None when a moved value is not a number (#866). + + Both are {driver: value} over `SCALABLE_DRIVERS`. Plain proportional + scaling - steps/steps0 x (w.h)/(w0.h0) x frames/frames0 - with no fixed + term for the model load and decode, so it over-quotes a little when a + value rises and under-quotes when one falls. `axes` names each axis that + moved, for a caller to read ("steps 30 vs 9"); empty when none did. + """ + moved = { + name + for name in SCALABLE_DRIVERS + if name in effective + and _driver_comparable(effective[name]) + != _driver_comparable(defaults.get(name)) + } + if not moved: + return minutes, [] + values = {} + for name in SCALABLE_DRIVERS: + if name not in effective: + continue + pair = (_positive(defaults.get(name)), _positive(effective[name])) + if None in pair: + if name in moved: + return None + continue + values[name] = pair + factor = 1.0 + axes = [] + if STEPS_DRIVER in moved: + was, now = values[STEPS_DRIVER] + factor *= now / was + axes.append(f"steps {_shown(now)} vs {_shown(was)}") + canvas = [name for name in ("width", "height") if name in moved] + for name in canvas: + was, now = values[name] + factor *= now / was + if canvas and "width" in values and "height" in values: + (w0, w), (h0, h) = values["width"], values["height"] + axes.append(f"canvas {_shown(w)}x{_shown(h)} vs {_shown(w0)}x{_shown(h0)}") + else: + axes.extend( + f"{name} {_shown(values[name][1])} vs {_shown(values[name][0])}" + for name in canvas + ) + if FRAMES_DRIVER in moved: + was, now = values[FRAMES_DRIVER] + factor *= now / was + axes.append(f"frames {_shown(now)} vs {_shown(was)}") + return minutes * factor, axes + + +def _entry_frames(entries): + """The per-entry `num_frames` of a list's entries as numbers, or None + when an entry carries none (or a non-numeric one).""" + if not isinstance(entries, list) or not entries: + return None + frames = [] + for entry in entries: + number = ( + _positive(entry.get(FRAMES_DRIVER)) if isinstance(entry, dict) else None + ) + if number is None: + return None + frames.append(number) + return frames -def _scalar_driver_shifted(definition, expanded, list_entries, per_entry=None): - """Whether a declared, non-list `cost_driver` was overridden away from - the default value the curated `cost` was measured against (#267). +def _list_frames_scaling(name, default_entries, effective_entries, list_entries): + """A list driver's per-entry frames as (factor, axis) - (1.0, None) when + they did not move - or None when they moved in a way that cannot be + scaled (#593, #866). - A list driver is `_repriced`'s to catch - it changes the plan's list - count, not a bare variable's value. This only asks about a scalar one, - which `_repriced` never looks at, so a `catalog` figure measured for one - `num_frames` silently priced a run at a different one. + The frames a list renders are the sum over its entries. When the list's + length is already priced (`_repriced`, a `per_entry` rate), only the + mean per entry is left to scale, so the two multiply to the summed ratio + rather than counting the length twice. + """ + measured = _entry_frames(default_entries) + running = _entry_frames(effective_entries) + if measured is None or running is None: + # Nothing to sum on one side: only a frame count outside the values + # the default entries carried leaves no figure, as before (#593) + if _list_entry_field_shifted(default_entries, effective_entries): + return None + return 1.0, None + was, now = sum(measured), sum(running) + if was == now: + return 1.0, None + factor = now / was + if name in list_entries: + factor = (now / len(running)) / (was / len(measured)) + if math.isclose(factor, 1.0): + return 1.0, None + return factor, f"frames {_shown(now)} vs {_shown(was)} over {name}" + + +def _driver_scaling(definition, expanded, list_entries): + """How far the declared `cost_drivers` moved the run away from the + defaults the curated `cost` was measured at, as (factor, axes), or None + when one moved in a way nothing can scale (#267, #866). + + A list driver's length is `_repriced`'s to price; here only its entries' + `num_frames` are. A scalar one in `SCALABLE_DRIVERS` scales in + proportion; any other scalar driver that moved leaves no figure. """ defaults = definition.get("variables") or {} effective = expanded.get("variables") or {} + factor = 1.0 + axes = [] + at_default = {} + running = {} for name in _declared_drivers(definition): default_value = defaults.get(name) if isinstance(default_value, list): - # A list the cost prices per entry is already extrapolated in - # either direction; its entries' index/name fields are labels, - # not cost fields (#772) - if name != per_entry and _list_entry_field_shifted( - default_value, effective.get(name) - ): - return True + frames = _list_frames_scaling( + name, default_value, effective.get(name), list_entries + ) + if frames is None: + return None + factor *= frames[0] + if frames[1]: + axes.append(frames[1]) continue if name in list_entries: continue - if _driver_comparable(effective.get(name)) != _driver_comparable(default_value): - return True - return False + if name in SCALABLE_DRIVERS: + at_default[name] = default_value + running[name] = effective.get(name) + elif _driver_comparable(effective.get(name)) != _driver_comparable( + default_value + ): + return None + scaled = scale_curated(factor, at_default, running) + if scaled is None: + return None + return scaled[0], scaled[1] + axes + + +def _scaled(price, scaling): + """`price` scaled by a `_driver_scaling` answer: `derived` once any axis + moved (another device's figure stays `other_device`, as `_price` keeps it + for a resized list - #589), with `scaled` saying from what.""" + if scaling is None: + return {"minutes": None, "basis": UNKNOWN, "measured_on": None} + factor, axes = scaling + if not axes or price["minutes"] is None: + return price + basis = OTHER_DEVICE if price["basis"] == OTHER_DEVICE else DERIVED + return { + **price, + "minutes": price["minutes"] * factor, + "basis": basis, + "scaled": f"scaled from the curated {_shown(price['minutes'])} min: " + + ", ".join(axes), + } def _entry_cost_field_shifted(definition, expanded): @@ -523,23 +663,15 @@ def _entry_cost_field_shifted(definition, expanded): def _own_price(definition, expanded, list_entries, device, measured_entries): - """The workflow's own price, reset to unknown when a scalar driver moved.""" + """The workflow's own price, scaled along the cost drivers that moved.""" own = _price(definition.get("cost"), device, list_entries, measured_entries or {}) # `derived` and `per_entry` are re-priced by entry count alone, so a - # per-entry num_frames outside the measured values needs the same check (#796) - if own["basis"] in (CATALOG, OTHER_DEVICE, DERIVED, PER_ENTRY) and ( - _scalar_driver_shifted( - definition, - expanded, - list_entries, - _per_entry_variable(definition.get("cost"), device), - ) - ): - # A scalar cost_driver (H3's num_frames, say) moved away from the - # value the curated cost was measured against, and _repriced only - # re-prices a for_each list's length - so the catalog figure would - # otherwise be quoted for a run it was never measured for (#267) - own = {"minutes": None, "basis": UNKNOWN, "measured_on": None} + # per-entry num_frames is scaled as well (#796) + if own["basis"] in (CATALOG, OTHER_DEVICE, DERIVED, PER_ENTRY): + # _repriced only re-prices a for_each list's length, so a moved scalar + # cost_driver (H3's num_inference_steps, say) is scaled here rather + # than quoted at the figure measured for the defaults (#267, #866) + own = _scaled(own, _driver_scaling(definition, expanded, list_entries)) return own @@ -601,21 +733,17 @@ def _child_catalog_price(child_definition, child_cost, step_arguments, device): child_list_entries = _list_entries(child_definition, child_expanded) child = _price(child_cost, device, child_list_entries, child_measured_entries) if ( - child["basis"] in (CATALOG, OTHER_DEVICE) + child["basis"] in (CATALOG, OTHER_DEVICE, DERIVED, PER_ENTRY) and child_definition is not None - and _scalar_driver_shifted( - child_definition, - child_expanded, - child_list_entries, - _per_entry_variable(child_cost, device), - ) ): - # A scalar cost_driver the composing step overrode (H3's - # num_frames at 345 against a default of 124, say) is the - # same #267 failure one level down - the child's own - # catalog figure was never measured for the value this - # step actually passes it (#341) - child = {"minutes": None, "basis": UNKNOWN, "measured_on": None} + # A cost_driver the composing step overrode (H3's num_frames at 345 + # against a default of 124, say) is the same #267 case one level + # down - the child's figure was measured at its defaults, so it is + # scaled to the value this step actually passes it (#341, #866) + child = _scaled( + child, + _driver_scaling(child_definition, child_expanded, child_list_entries), + ) return child @@ -632,6 +760,7 @@ def __init__(self, minutes, partial, unpriced): self.runs = [] self.measured_on = set() self.bases = [] + self.scaled = [] def add(self, path, child, observed): self.had_child = True @@ -646,6 +775,8 @@ def add(self, path, child, observed): self.unpriced_children.append(path) return self.bases.append((child["basis"], child.get("measured_on"))) + if child.get("scaled"): + self.scaled.append(f"{path}: {child['scaled']}") if self.minutes is not None: self.minutes += child["minutes"] else: @@ -704,6 +835,10 @@ def _rolled_up_estimate(own, totals, device, cached_steps, total_steps): devices = {on for _, on in totals.bases} if len(devices) == 1: top_measured_on = next(iter(devices)) + if top_basis == CATALOG and any(basis == DERIVED for basis, _ in totals.bases): + # A child scaled for the arguments its step passes is an estimate, + # so the sum is one too (#866) + top_basis = DERIVED rounded = round(minutes, 1) if minutes is not None else None result = { "minutes": rounded, @@ -715,6 +850,9 @@ def _rolled_up_estimate(own, totals, device, cached_steps, total_steps): "runs": top_runs, "cached_minutes": _cached_minutes(rounded, cached_steps, total_steps), } + scaled = ([own["scaled"]] if own.get("scaled") else []) + totals.scaled + if scaled and minutes is not None and top_basis != OBSERVED: + result["scaled"] = "; ".join(scaled) if top_basis == OBSERVED: # The rolled-up-from-children case (#268): no curated figure of the # parent's own exists to blend toward (that is this branch's own @@ -779,6 +917,12 @@ def estimate( measured = None if measured is not None: measured = _tempered(measured, own["minutes"]) + if measured.get("tempered") and own.get("scaled"): + # What it blended toward was scaled for this run's drivers, and + # `curated_minutes` alone does not say so (#866) + measured["scaled"] = "; ".join( + part for part in (measured.get("scaled"), own["scaled"]) if part + ) measured["cached_minutes"] = _cached_minutes( measured["minutes"], cached_steps, total_steps ) diff --git a/tests/test_catalog_structure.py b/tests/test_catalog_structure.py index c7f43b86..d040038a 100644 --- a/tests/test_catalog_structure.py +++ b/tests/test_catalog_structure.py @@ -966,7 +966,7 @@ def test_every_readme_link_resolves(path): COSTED = { "workflows/templates/minimax/music-video.json": 35, - "workflows/templates/minimax/dialogue-short.json": 42, + "workflows/templates/minimax/dialogue-short.json": 21.2, "workflows/templates/assemble-and-score.json": 0.2, "workflows/templates/dissolve-between-shots.json": 0.2, } @@ -982,7 +982,11 @@ def test_the_cut_templates_quote_a_measured_cost(path, minutes): LoRA on those templates at 9 - so their figures went with the schedule they described rather than being scaled, `cost` being measured and never derived. The three here kept their step count and their canvas; only - which adapter loads changed, at the same rank and file size.""" + which adapter loads changed, at the same rank and file size. + + dialogue-short's 42 was measured at 1344x768, not its 960x544 defaults; + a cost entry is measured at the variable defaults, so #866 re-priced it + to 42 x 960*544 / (1344*768) = 21.2.""" definition = json.load(open(os.path.join(REPO_ROOT, path), encoding="utf-8")) entry = definition["cost"][0] assert entry["name"] == "RTX 3090" diff --git a/tests/test_plan.py b/tests/test_plan.py index b0f01ff2..1c687bd1 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -12,6 +12,7 @@ PLACEMENT_RESULT_KEYS, build_plan, gate_warnings, + scale_curated, ) from dw.validation import unseeded_cache_warnings from dw.runs import new_run_id @@ -655,17 +656,18 @@ def test_minutes_is_rounded_to_one_decimal(self, plan): spec["cost"] = [cost("cuda", 10.04)] assert plan(spec)["estimate"]["minutes"] == 10.0 - def test_a_shifted_scalar_cost_driver_falls_back_to_unknown(self, plan): + def test_a_shifted_unscalable_cost_driver_falls_back_to_unknown(self, plan): """The catalog figure was measured at frames=25; a caller who overrides a declared cost_driver away from that value is not - describing the run the figure was measured for, and `_repriced` - only re-prices a for_each list's length, not a bare variable (#267).""" + describing the run the figure was measured for (#267). `frames` is + not one of the drivers a figure scales along (#866), so there is + nothing to re-price it by.""" spec = definition() spec["cost_drivers"] = ["frames"] answer = plan(spec, arguments={"frames": 9})["estimate"] assert (answer["minutes"], answer["basis"]) == (None, "unknown") - def test_a_shifted_scalar_driver_is_unknown_on_another_device_too(self, plan): + def test_a_shifted_unscalable_driver_is_unknown_on_another_device_too(self, plan): """#589: the other_device branch skipped the #267 check, so a 241-frame run was quoted the 121-frame clip's figure.""" spec = definition() @@ -692,9 +694,10 @@ def test_a_list_driver_still_reprices_instead_of_going_unknown(self, plan): answer = plan(spec, arguments={"shots": shots})["estimate"] assert (answer["minutes"], answer["basis"]) == (50.0, "derived") - def test_a_shifted_per_entry_field_in_a_list_driver_is_unknown(self, plan): + def test_a_shifted_per_entry_field_in_a_list_driver_is_scaled(self, plan): """#593: a per-shot num_frames outside the values the default entries - were measured with has no matching bucket, same as a scalar (#267).""" + were measured with has no matching bucket, same as a scalar (#267) - + so the figure is scaled by the frames summed over the list (#866).""" spec = definition() spec["cost"] = [cost("mps", 40, name="M2")] spec["cost_drivers"] = ["shots"] @@ -710,20 +713,26 @@ def test_a_shifted_per_entry_field_in_a_list_driver_is_unknown(self, plan): {"name": "x", "prompt": "p", "num_frames": 243}, {"name": "y", "prompt": "p", "num_frames": 243}, ] + # Another device's figure stays other_device, scaled (#589) kept = plan(spec, arguments={"shots": same})["estimate"] - assert (kept["minutes"], kept["basis"]) == (40.0, "other_device") + assert (kept["minutes"], kept["basis"]) == (37.4, "other_device") + assert kept["scaled"].endswith("frames 248 vs 265 over shots") moved = plan(spec, arguments={"shots": long})["estimate"] - assert (moved["minutes"], moved["basis"]) == (None, "unknown") + assert (moved["minutes"], moved["basis"]) == (73.4, "other_device") as_text = [dict(entry, num_frames=str(entry["num_frames"])) for entry in long] moved = plan(spec, arguments={"shots": as_text})["estimate"] - assert (moved["minutes"], moved["basis"]) == (None, "unknown") + assert (moved["minutes"], moved["basis"]) == (73.4, "other_device") same_text = [dict(entry, num_frames="124") for entry in same] kept = plan(spec, arguments={"shots": same_text})["estimate"] - assert (kept["minutes"], kept["basis"]) == (40.0, "other_device") + assert (kept["minutes"], kept["basis"]) == (37.4, "other_device") + default = plan(spec)["estimate"] + assert (default["minutes"], default["basis"]) == (40.0, "other_device") + assert "scaled" not in default - def test_a_lengthened_list_with_a_shifted_field_is_unknown_not_derived(self, plan): + def test_a_lengthened_list_with_a_shifted_field_scales_both(self, plan): """#796: the derived basis scaled by entry count and skipped the #593 - field check, so 6 shots with one at 345 frames was quoted confidently.""" + field check, so 6 shots with one at 345 frames was quoted confidently. + The count and the mean frames per entry now both scale (#866).""" spec = definition() spec["cost_drivers"] = ["shots"] spec["variables"]["shots"] = [ @@ -732,9 +741,14 @@ def test_a_lengthened_list_with_a_shifted_field_is_unknown_not_derived(self, pla same = [{"name": f"s{n}", "prompt": "p", "num_frames": 124} for n in range(10)] kept = plan(spec, arguments={"shots": same})["estimate"] assert (kept["minutes"], kept["basis"]) == (20.0, "derived") + assert "scaled" not in kept long = same[:9] + [{"name": "z", "prompt": "p", "num_frames": 345}] moved = plan(spec, arguments={"shots": long})["estimate"] - assert (moved["minutes"], moved["basis"]) == (None, "unknown") + # 4 min x 10/5 shots x (1461/10)/(620/5) frames per shot + assert (moved["minutes"], moved["basis"]) == (23.6, "derived") + assert moved["scaled"] == ( + "scaled from the curated 20 min: frames 1461 vs 620 over shots" + ) def test_position_fields_do_not_block_a_derived_figure(self, plan): """#796: start_frame/cut_frames grow with the count; only num_frames @@ -760,7 +774,8 @@ def test_a_shifted_field_withholds_an_observed_figure(self, plan): long = [dict(e) for e in spec["variables"]["shots"]] long[3]["num_frames"] = 345 answer = plan(spec, arguments={"shots": long}, observed=observed())["estimate"] - assert (answer["minutes"], answer["basis"]) == (None, "unknown") + # The observed 8.04 is withheld; the curated 10 is scaled instead + assert (answer["minutes"], answer["basis"]) == (14.5, "derived") def test_a_per_entry_list_driver_prices_past_its_measured_count(self, plan): """#772: a windows list whose entries carry an `index` label priced @@ -934,6 +949,233 @@ def observed_for_child(path, child_definition, arguments=None): assert answer["low_confidence"] is True +def scalable(minutes=10): + """The fixture with steps and a canvas as declared cost drivers, its + curated figure measured at their defaults (#866).""" + spec = definition() + spec["cost"] = [cost("cuda", minutes)] + spec["variables"].update({"num_inference_steps": 9, "width": 960, "height": 544}) + spec["cost_drivers"] = ["num_inference_steps", "width", "height", "shots"] + return spec + + +class TestDriverScaling: + """#866: a curated figure is measured at the workflow's variable + defaults; a run that moves steps, the canvas or the frames is quoted + scaled along them rather than as no figure at all.""" + + def test_the_defaults_quote_the_curated_figure(self, plan): + answer = plan(scalable())["estimate"] + assert (answer["minutes"], answer["basis"]) == (10.0, "catalog") + assert "scaled" not in answer + + def test_steps_scale_in_proportion(self, plan): + answer = plan(scalable(), arguments={"num_inference_steps": 30})["estimate"] + assert (answer["minutes"], answer["basis"]) == (33.3, "derived") + assert answer["scaled"] == "scaled from the curated 10 min: steps 30 vs 9" + + def test_the_canvas_scales_by_its_area(self, plan): + arguments = {"width": 1344, "height": 768} + answer = plan(scalable(), arguments=arguments)["estimate"] + # 10 x 1344.768 / 960.544 + assert (answer["minutes"], answer["basis"]) == (19.8, "derived") + assert answer["scaled"] == ( + "scaled from the curated 10 min: canvas 1344x768 vs 960x544" + ) + + def test_one_side_of_the_canvas_scales_too(self, plan): + answer = plan(scalable(), arguments={"width": 1920})["estimate"] + assert (answer["minutes"], answer["basis"]) == (20.0, "derived") + assert "canvas 1920x544 vs 960x544" in answer["scaled"] + + def test_per_entry_frames_scale_by_their_sum(self, plan): + spec = scalable() + spec["variables"]["shots"] = [ + {"name": "a", "prompt": "one", "num_frames": 124}, + {"name": "b", "prompt": "two", "num_frames": 141}, + ] + shots = [ + {"name": "a", "prompt": "one", "num_frames": 124}, + {"name": "b", "prompt": "two", "num_frames": 345}, + ] + answer = plan(spec, arguments={"shots": shots})["estimate"] + # 10 x 469/265: the same two shots, one of them longer + assert (answer["minutes"], answer["basis"]) == (17.7, "derived") + assert answer["scaled"].endswith("frames 469 vs 265 over shots") + + def test_a_list_not_priced_by_length_scales_by_the_sum_alone(self, plan): + """No for_each runs over `takes`, so its length prices nothing - the + summed frames carry the whole ratio, length and all.""" + spec = scalable() + spec["variables"]["takes"] = [{"num_frames": 100}] + spec["cost_drivers"] = spec["cost_drivers"] + ["takes"] + takes = [{"num_frames": 100}, {"num_frames": 150}] + answer = plan(spec, arguments={"takes": takes})["estimate"] + assert (answer["minutes"], answer["basis"]) == (25.0, "derived") + assert answer["scaled"].endswith("frames 250 vs 100 over takes") + + def test_steps_canvas_and_frames_multiply(self, plan): + spec = scalable() + spec["variables"]["frames"] = 124 + spec["variables"]["num_frames"] = 124 + spec["steps"][0]["pipeline"]["arguments"]["num_frames"] = "variable:num_frames" + spec["cost_drivers"] = spec["cost_drivers"] + ["num_frames"] + arguments = { + "num_inference_steps": 30, + "width": 1344, + "height": 768, + "num_frames": 248, + } + answer = plan(spec, arguments=arguments)["estimate"] + # 10 x 30/9 x 1344.768/960.544 x 248/124 + assert (answer["minutes"], answer["basis"]) == (131.8, "derived") + assert answer["scaled"] == ( + "scaled from the curated 10 min: steps 30 vs 9, " + "canvas 1344x768 vs 960x544, frames 248 vs 124" + ) + + def test_it_multiplies_with_the_list_length(self, plan): + shots = [{"name": f"s{n}", "prompt": "x"} for n in range(4)] + arguments = {"shots": shots, "num_inference_steps": 18} + answer = plan(scalable(), arguments=arguments)["estimate"] + # 10 x 4/2 shots x 18/9 steps + assert (answer["minutes"], answer["basis"]) == (40.0, "derived") + assert answer["scaled"] == "scaled from the curated 20 min: steps 18 vs 9" + + def test_a_non_numeric_value_stays_unknown(self, plan): + # A text default keeps a text argument as text; an int one would + # have cast "wide" away to the default before the plan saw it + spec = scalable() + spec["variables"]["width"] = "960" + answer = plan(spec, arguments={"width": "wide"})["estimate"] + assert (answer["minutes"], answer["basis"]) == (None, "unknown") + assert "scaled" not in answer + + def test_a_number_spelled_as_text_scales(self, plan): + answer = plan(scalable(), arguments={"num_inference_steps": "18"})["estimate"] + assert (answer["minutes"], answer["basis"]) == (20.0, "derived") + + def test_another_devices_figure_stays_other_device(self, plan): + spec = scalable() + spec["cost"] = [cost("mps", 40)] + answer = plan(spec, arguments={"num_inference_steps": 18})["estimate"] + assert (answer["minutes"], answer["basis"]) == (80.0, "other_device") + assert answer["scaled"] == "scaled from the curated 40 min: steps 18 vs 9" + + def test_a_thin_observed_figure_tempers_toward_the_scaled_one(self, plan): + arguments = {"num_inference_steps": 18} + thin = observed(minutes=24, runs=1) + answer = plan(scalable(), arguments=arguments, observed=thin)["estimate"] + assert answer["basis"] == "observed" + assert answer["tempered"] is True + assert answer["curated_minutes"] == 20.0 + assert 20.0 < answer["minutes"] < 24.0 + assert answer["scaled"] == "scaled from the curated 10 min: steps 18 vs 9" + + def test_a_settled_observed_figure_is_not_scaled(self, plan): + arguments = {"num_inference_steps": 18} + answer = plan(scalable(), arguments=arguments, observed=observed(minutes=24))[ + "estimate" + ] + assert (answer["minutes"], answer["basis"]) == (24.0, "observed") + assert "scaled" not in answer + + +class TestScalingAccuracy: + """#866's bar: each measured pair on the RTX 3090 (#851's B and L runs) + is predicted within 30% by scaling the other.""" + + @pytest.mark.parametrize( + "measured, defaults, effective, actual", + [ + # B4 -> B1: steps 12 -> 30 at 960x544 + ( + 6.6, + {"num_inference_steps": 12, "width": 960, "height": 544}, + {"num_inference_steps": 30, "width": 960, "height": 544}, + 15.1, + ), + # B4 -> B5: canvas 960x544 -> 1344x768 at 12 steps + ( + 6.6, + {"num_inference_steps": 12, "width": 960, "height": 544}, + {"num_inference_steps": 12, "width": 1344, "height": 768}, + 11.8, + ), + # B3 -> L5: steps 12 -> 30 at 1344x768 + ( + 12.9, + {"num_inference_steps": 12, "width": 1344, "height": 768}, + {"num_inference_steps": 30, "width": 1344, "height": 768}, + 29.5, + ), + ], + ) + def test_within_thirty_percent(self, measured, defaults, effective, actual): + minutes, axes = scale_curated(measured, defaults, effective) + assert axes + assert abs(minutes - actual) / actual < 0.30 + + def test_nothing_moved_is_the_figure_itself(self): + defaults = {"num_inference_steps": 9, "width": 960, "height": 544} + assert scale_curated(7.88, defaults, dict(defaults)) == (7.88, []) + + def test_a_moved_non_number_is_none(self): + defaults = {"num_inference_steps": 9} + assert scale_curated(7.88, defaults, {"num_inference_steps": None}) is None + + +CATALOG_DIALOGUE = "workflows/templates/minimax/dialogue-short.json" + + +class TestTheCatalogTemplatesScale: + """#866 on the real templates: H3's reference-to-video and the + dialogue-short sequence, measured at their defaults.""" + + @staticmethod + def load(path): + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent + return json.loads((root / path).read_text()) + + def test_reference_to_video_at_its_defaults(self, plan): + answer = plan(self.load(CATALOG_H3))["estimate"] + assert (answer["minutes"], answer["basis"]) == (7.9, "catalog") + assert "scaled" not in answer + + def test_reference_to_video_at_thirty_steps(self, plan): + template = self.load(CATALOG_H3) + answer = plan(template, arguments={"num_inference_steps": 30})["estimate"] + # 7.88 x 30/9 + assert (answer["minutes"], answer["basis"]) == (26.3, "derived") + assert "steps 30 vs 9" in answer["scaled"] + + def test_reference_to_video_at_the_larger_canvas(self, plan): + template = self.load(CATALOG_H3) + arguments = {"width": 1344, "height": 768} + answer = plan(template, arguments=arguments)["estimate"] + assert (answer["minutes"], answer["basis"]) == (15.6, "derived") + assert "canvas 1344x768 vs 960x544" in answer["scaled"] + + def test_dialogue_short_at_its_defaults(self, plan): + answer = plan(self.load(CATALOG_DIALOGUE))["estimate"] + assert answer["basis"] == "catalog" + assert answer["minutes"] == pytest.approx(21, abs=0.5) + + def test_dialogue_short_with_one_shot_longer(self, plan): + template = self.load(CATALOG_DIALOGUE) + shots = copy.deepcopy(template["variables"]["shots"]) + assert [shot["num_frames"] for shot in shots] == [124, 124, 124, 124, 141] + shots[0]["num_frames"] = 192 + answer = plan(template, arguments={"shots": shots})["estimate"] + # 21.2 x 705/637 frames + assert (answer["minutes"], answer["basis"]) == (23.5, "derived") + assert answer["scaled"] == ( + "scaled from the curated 21.2 min: frames 705 vs 637 over shots" + ) + + def composing(child_path): return { "id": "parent", @@ -1015,11 +1257,11 @@ def test_a_child_measured_on_another_device_is_still_added(self, plan, tmp_path) # the parent's own basis is what is reported assert answer["basis"] == "catalog" - def test_a_composed_childs_shifted_scalar_driver_is_unpriced(self, plan, tmp_path): + def test_a_composed_childs_shifted_scalar_driver_is_scaled(self, plan, tmp_path): """The composing step's own `arguments` are what the child actually runs with, not its declared defaults - a scalar cost_driver moved away from the value the child's catalog cost was measured against - is the same #267 failure one level down (#341).""" + is the same #267 case one level down (#341), scaled (#866).""" child = { "id": "child", "cost": [cost("cuda", 5)], @@ -1031,8 +1273,12 @@ def test_a_composed_childs_shifted_scalar_driver_is_unpriced(self, plan, tmp_pat parent = composing("child.json") parent["steps"][1]["workflow"]["arguments"] = {"num_frames": 345} answer = plan(parent)["estimate"] - assert (answer["minutes"], answer["partial"]) == (2.0, True) - assert answer["unpriced"] == ["child.json"] + # 2 of its own + 5 x 345/124 + assert (answer["minutes"], answer["partial"]) == (15.9, False) + assert (answer["basis"], answer["unpriced"]) == ("derived", []) + assert answer["scaled"] == ( + "child.json: scaled from the curated 5 min: frames 345 vs 124" + ) def test_a_composed_child_at_its_default_is_still_priced(self, plan, tmp_path): """The companion case: a composing step that passes the child's own @@ -1166,16 +1412,16 @@ def test_an_unpriced_childs_history_leaves_the_estimate_partial( assert answer["basis"] == "unknown" assert answer["partial"] is False - def test_a_composed_childs_shifted_scalar_driver_stays_unpriced_despite_default_bucket_history( + def test_a_composed_childs_shifted_scalar_driver_is_scaled_despite_default_bucket_history( self, plan, tmp_path ): """The observed-history rollup must not paper over a shifted scalar driver by quoting the *default* bucket's figure just because it exists - `observed_for_child` is called with the composing step's own arguments, and answering None for the shifted bucket falls - through to the catalog path's `_scalar_driver_shifted` unpriced - check rather than silently reusing the default bucket's history - (#341).""" + through to the catalog path's driver check rather than silently + reusing the default bucket's history (#341), which now scales the + child's figure (#866).""" child = { "id": "child", "cost": [cost("cuda", 5)], @@ -1207,8 +1453,8 @@ def observed_for_child(path, child_definition, arguments=None): answer = plan(parent, observed_for_child=observed_for_child)["estimate"] assert calls == [{"num_frames": 345}] - assert answer["basis"] == "unknown" - assert (answer["minutes"], answer["partial"]) == (None, False) + assert answer["basis"] == "derived" + assert (answer["minutes"], answer["partial"]) == (13.9, False) def test_a_composed_childs_observed_history_is_bucketed_by_the_composing_steps_arguments( self, plan, tmp_path diff --git a/workflows/templates/minimax/dialogue-short.json b/workflows/templates/minimax/dialogue-short.json index 93cf471b..a4593702 100644 --- a/workflows/templates/minimax/dialogue-short.json +++ b/workflows/templates/minimax/dialogue-short.json @@ -7,7 +7,7 @@ "device": "cuda", "name": "RTX 3090", "vram_gb": 24, - "minutes": 42 + "minutes": 21.2 } ], "cost_drivers": [ @@ -25,7 +25,7 @@ "height", "num_frames" ], - "reason": "Ref2VA holds every reference's encoding resident beside the video latents, so the ceiling is a fixed base, a per-voxel term for the VAE decode's conv3d activation (width * height * num_frames), and a per-reference term. Classified against the 24 GB entry from the #479 field report (lem, RTX 3090 24 GB, int4 + 768p turbo LoRA, 9 steps, 2026-09-25/26) at 1344x768: 175 frames x 2 refs, 209 x 2 and 175 x 3 ran (projected 22.83, 23.77 and 23.83 GB); 209 x 3 OOM'd in reference encode needing a further 5.1 GB and 260 x 1 OOM'd at denoise step 1 needing a further 4.8 GB (projected 24.77 and 24.18 GB). The refused allocations are lower bounds, not peaks, so this is a classification with about 0.2 GB of margin either side, not a fitted curve. bytes_per_voxel is unchanged from #324's calibration (job 282172105da1); base_gb drops 0.3 GB now the references are counted separately" + "reason": "Ref2VA holds every reference's encoding resident beside the video latents, so the ceiling is a fixed base, a per-voxel term for the VAE decode's conv3d activation (width * height * num_frames), and a per-reference term. Classified against the 24 GB entry from the #479 field report (lem, RTX 3090 24 GB, int4 + 768p turbo LoRA, 9 steps, 2026-09-25/26) at 1344x768: 175 frames x 2 refs, 209 x 2 and 175 x 3 ran (projected 22.83, 23.77 and 23.83 GB); 209 x 3 OOM'd in reference encode needing a further 5.1 GB and 260 x 1 OOM'd at denoise step 1 needing a further 4.8 GB (projected 24.77 and 24.18 GB). The refused allocations are lower bounds, not peaks, so this is a classification with about 0.2 GB of margin either side, not a fitted curve. bytes_per_voxel is unchanged from #324's calibration (job 282172105da1); base_gb drops 0.3 GB now the references are counted separately. The cost entry's 42 min was that field run at 1344x768; re-priced to these 960x544 defaults it is 42 x 960*544 / (1344*768) = 21.2 (#866)" }, "variables": { "character_a_portrait_prompt": "a 3/4-length studio portrait of a wiry man in his early 30s with short curly dark hair and expressive eyebrows, wearing a mustard-yellow sweater vest over a white button-down shirt, standing in a 1990s multi-camera sitcom apartment set with a sagging olive-green couch and a brick wall behind him, warm even studio lighting, sharp focus, photographic", From 7f70be95df456d9f388b5e4a8c8e27e6261235ed Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:21:07 -0500 Subject: [PATCH 145/204] feat(plan): #906 - say what a figure was scaled along in the UI and the docs The web UI's figure line reads `scaled` for a derived or other-device estimate. The schema's cost description, WORKFLOW_GUIDE, SERVER.md and the architecture map state the rule: a cost entry is measured at the workflow's variable defaults. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/SERVER.md | 7 ++++++- docs/WORKFLOW_GUIDE.md | 11 ++++++++-- dw/server/api_models.py | 5 ++++- dw/workflow_schema.json | 2 +- ui/src/lib/generated/api-schema.ts | 2 +- ui/src/lib/generated/openapi.json | 2 +- ui/src/lib/plan.test.ts | 33 ++++++++++++++++++++++++++++++ ui/src/lib/plan.ts | 17 +++++++++------ 9 files changed, 67 insertions(+), 14 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0dbdb151..f8f89e2c 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,7 +97,7 @@ one to open. | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | -| The estimate | `dw/plan.py`: `estimate`, `_tempered` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure` | +| The estimate | `dw/plan.py`: `estimate`, `_tempered` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | | Cost drivers | `dw/server/observed_cost.py`: `declared_drivers` | A cost driver that names no declared variable is dropped. | `tests/test_observed_cost.py::TestComparability::test_a_driver_naming_no_variable_is_dropped`, `tests/test_observed_cost.py::TestTheCatalogsDriversAreReal::test_every_declared_driver_is_a_variable_of_its_workflow` | | Raw workflow GET | `dw/server/routes/library.py`: `get_workflow` | The raw workflow GET is served verbatim, with no `observed`, because the editor saves what it reads. | — | | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | diff --git a/docs/SERVER.md b/docs/SERVER.md index 6c2b204c..5b7306ac 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -438,7 +438,12 @@ The editor's forms come from these; they are just as usable from scripts: ones it was measured with), `per_entry` (re-priced from a measured per-entry rate, when the entry carries `per_entry`), `derived` (the stored total extrapolated linearly over a list whose length the caller - changed - an estimate, not a measurement), `other_device` (no entry for + changed, or scaled in proportion along `num_inference_steps`, + `width` x `height` and `num_frames` - per entry in a list, summed - that + the caller moved off the variable defaults a `cost` entry is measured at, + with `scaled` naming each moved axis; an estimate, not a measurement; a + moved driver with no rule to scale by, or a non-numeric value, is + `unknown`), `other_device` (no entry for the serving backend; the first entry's figure, which is a warning rather than a quote) or `unknown` (no cost block, or more than one list changed so there is nothing honest to extrapolate along) or `inherited` (an diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index d125082c..3310978b 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -708,7 +708,9 @@ by the source, and `validate_workflow` names the entries to add or drop. a measured per-entry rate re-priced for your list; `catalog` is a measured total for a run whose lists are the ones it was measured with; `derived` is that total extrapolated over a list you changed the length - of - say it is an estimate; `inherited` is an inline workflow priced + of, or scaled along the steps, canvas or frames you moved off the + defaults it was measured at (`scaled` names each axis that moved) - say + it is an estimate; `inherited` is an inline workflow priced from this box's runs of the catalog template with the same pipeline (`inherited_from`) - say it is an approximation, and relay the warning if offload, quantization or frame count differ; `other_device` is a figure from another @@ -977,7 +979,12 @@ longer than that is truncated with an ellipsis in every listing. Declare `shape`, `traits` or `summary` at the top level only when derivation gets it wrong; a declaration that merely repeats the derivation is noise that rots when the rules change, and the repo's catalog tests refuse it. `cost` is -never derived — leave it absent until a run has been measured. That makes it +never derived — leave it absent until a run has been measured. A `cost` entry +is measured at the workflow's variable defaults: a run that moves +`num_inference_steps`, `width`/`height` or `num_frames` (per entry in a list, +summed) is quoted scaled in proportion to them, so a figure measured at some +other size or step count is re-priced to the defaults before it is written +(dialogue-short's 42 min at 1344x768 is 21.2 at its 960x544). That makes it the catalog's verified marker as well: an entry carrying `cost` has been run to completion on the device it names, and one without has only been authored — its description may still say what it has not been able to check (VRAM at diff --git a/dw/server/api_models.py b/dw/server/api_models.py index b0cfef6b..067da53f 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -627,7 +627,10 @@ class PlanEstimate(ApiModel): "when nothing does." ) scaled: str = sometimes( - "With basis `inherited`: why the template's minutes were re-priced " + "With basis `derived` (or `other_device`, or a tempered `observed`): " + "what the curated figure was scaled along - steps, canvas or frames " + "moved off the variable defaults it was measured at (#866). With basis " + "`inherited`: why the template's minutes were re-priced " "linearly (more pipeline steps or frames than the template run measured) " "- an approximation, with `low_confidence` true." ) diff --git a/dw/workflow_schema.json b/dw/workflow_schema.json index 62f17c1f..080de78c 100644 --- a/dw/workflow_schema.json +++ b/dw/workflow_schema.json @@ -35,7 +35,7 @@ }, "cost": { "type": "array", - "description": "Measured runs, one per device the maintainer measured on. Never derived; absent means unknown. 'minutes' is the whole run's wall clock on a worker that has to load the models - what a consumer actually waits through, not the denoise loop alone; on a video template the two differ by minutes.", + "description": "Measured runs, one per device the maintainer measured on, each measured at the workflow's variable defaults: a run that moves num_inference_steps, width/height or num_frames off them is quoted scaled in proportion (plan basis 'derived'), so a figure measured anywhere else must be re-priced to the defaults before it is written. Never invented without a run; absent means unknown. 'minutes' is the whole run's wall clock on a worker that has to load the models - what a consumer actually waits through, not the denoise loop alone; on a video template the two differ by minutes.", "items": { "type": "object", "required": ["device", "vram_gb", "minutes"], diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index ac58aa08..87204023 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3276,7 +3276,7 @@ export interface components { runs: number | null; /** * Scaled - * @description With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true. + * @description With basis `derived` (or `other_device`, or a tempered `observed`): what the curated figure was scaled along - steps, canvas or frames moved off the variable defaults it was measured at (#866). With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true. */ scaled?: string; /** Tempered */ diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 444f9188..955d5291 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3636,7 +3636,7 @@ "title": "Runs" }, "scaled": { - "description": "With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true.", + "description": "With basis `derived` (or `other_device`, or a tempered `observed`): what the curated figure was scaled along - steps, canvas or frames moved off the variable defaults it was measured at (#866). With basis `inherited`: why the template's minutes were re-priced linearly (more pipeline steps or frames than the template run measured) - an approximation, with `low_confidence` true.", "title": "Scaled", "type": "string" }, diff --git a/ui/src/lib/plan.test.ts b/ui/src/lib/plan.test.ts index fbf15824..1cbdab29 100644 --- a/ui/src/lib/plan.test.ts +++ b/ui/src/lib/plan.test.ts @@ -62,6 +62,39 @@ describe('describePlan', () => { expect(figure.tone).toBe('warn') }) + it('a figure scaled along moved drivers says what moved', () => { + const [, figure] = describePlan({ + ...base, + list_entries: {}, + estimate: { + ...base.estimate, + minutes: 26.3, + basis: 'derived', + scaled: 'scaled from the curated 7.9 min: steps 30 vs 9', + }, + }) + expect(figure.text).toBe( + '~26.3 min on cuda - estimated (scaled from the curated 7.9 min: steps 30 vs 9), from a figure measured on RTX 3090', + ) + expect(figure.tone).toBe('warn') + }) + + it('a scaled figure from another accelerator says what moved too', () => { + const [, figure] = describePlan({ + ...base, + estimate: { + ...base.estimate, + minutes: 80, + basis: 'other_device', + device: 'mps', + scaled: 'scaled from the curated 40 min: steps 18 vs 9', + }, + }) + expect(figure.text).toBe( + 'no figure for mps - 80 min was measured on RTX 3090 (scaled from the curated 40 min: steps 18 vs 9)', + ) + }) + it('a figure from another accelerator is a warning, not a quote', () => { const [, figure] = describePlan({ ...base, diff --git a/ui/src/lib/plan.ts b/ui/src/lib/plan.ts index b4e4b45d..f8567cdc 100644 --- a/ui/src/lib/plan.ts +++ b/ui/src/lib/plan.ts @@ -62,7 +62,7 @@ export function describePlan(plan: Plan): PlanLine[] { } function figure(plan: Plan): PlanLine { - const { minutes, basis, device, measured_on, partial, unpriced } = + const { minutes, basis, device, measured_on, partial, unpriced, scaled } = plan.estimate const card = measured_on ? `measured on ${measured_on}` : 'measured' const missing = partial @@ -71,9 +71,12 @@ function figure(plan: Plan): PlanLine { if (basis === 'unknown' || minutes === null) { return { text: 'no measured cost', tone: 'warn' } } + // What a figure was scaled along (#866): steps, canvas or frames moved + // off the defaults it was measured at + const along = scaled ? ` (${scaled})` : '' if (basis === 'other_device') { return { - text: `no figure for ${device} - ${minutes} min was ${card}${missing}`, + text: `no figure for ${device} - ${minutes} min was ${card}${along}${missing}`, tone: 'warn', } } @@ -81,16 +84,18 @@ function figure(plan: Plan): PlanLine { .map(([name, count]) => `${count} ${name}`) .join(', ') // `derived` is the stored total stretched over a list the caller - // resized - an estimate, so it is said to be one rather than quoted + // resized, or scaled along the drivers they moved - an estimate, so it is + // said to be one rather than quoted if (basis === 'derived') { + const target = forList ? ` for ${forList}` : '' return { - text: `~${minutes} min on ${device} - estimated for ${forList}, from a figure ${card}${missing}`, + text: `~${minutes} min on ${device} - estimated${target}${along}, from a figure ${card}${missing}`, tone: 'warn', } } - const scaled = basis === 'per_entry' ? `re-priced for ${forList}, ` : '' + const repriced = basis === 'per_entry' ? `re-priced for ${forList}, ` : '' return { - text: `~${minutes} min on ${device} - ${scaled}${card}${missing}`, + text: `~${minutes} min on ${device} - ${repriced}${card}${missing}`, tone: 'plain', } } From d3efefec6b486b86b9670b1bfc2c3ab6beed5a35 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:21:34 -0500 Subject: [PATCH 146/204] docs(plan): #906 - the estimate docstring names the scaled derived basis Co-Authored-By: Claude Opus 5.5 --- dw/plan.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/dw/plan.py b/dw/plan.py index d47f6f79..e0d19e7c 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -889,8 +889,10 @@ def estimate( ones it was measured with, 'per_entry' that total re-priced from a measured per-entry rate, and 'derived' re-priced by extrapolating the stored total linearly over a list whose length the caller changed - (#85) - an estimate, not a measurement, and the only alternative to - quoting a 5-shot figure for a 10-shot run. + (#85), or scaled along the steps, canvas or frames moved off the + defaults it was measured at (#866, `scale_curated`) - an estimate, not + a measurement, and the only alternative to quoting a 5-shot figure for + a 10-shot run. `measured_entries` is what the stored defaults carry, which is the list the catalog figure was measured against; without it a catalog From 46357fdbea5fd87b48f4138a152262ed45eba809 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:38:45 -0500 Subject: [PATCH 147/204] feat(plan): #907 - flag a shift in a driver nothing scales instead of withholding the figure H3 templates declare weights_dtype, lora_model_name, attention_backend and cache as cost drivers, so observed runs bucket by them (#866 Q4, Q5). Co-Authored-By: Claude Opus 5.5 --- dw/plan.py | 50 ++++- tests/test_observed_cost.py | 57 +++++- tests/test_plan.py | 192 ++++++++++++++++-- .../minimax/chain-matched-and-aligned.json | 6 +- .../minimax/chain-matched-to-audio.json | 6 +- .../minimax/chain-video-continuity.json | 6 +- .../templates/minimax/chained-segments.json | 6 +- .../minimax/composable-references.json | 6 +- .../templates/minimax/dialogue-short.json | 6 +- .../minimax/enhance-prompt-with-image.json | 6 +- .../templates/minimax/enhance-prompt.json | 6 +- .../minimax/first-and-last-frame.json | 6 +- .../minimax/generated-subject-reference.json | 6 +- .../templates/minimax/image-to-video.json | 6 +- .../templates/minimax/last-frame-only.json | 6 +- workflows/templates/minimax/music-video.json | 6 +- .../templates/minimax/reference-to-video.json | 6 +- workflows/templates/minimax/shots-batch.json | 6 +- workflows/templates/minimax/storyboard.json | 6 +- .../minimax/video-with-audio-768p.json | 6 +- .../templates/minimax/video-with-audio.json | 6 +- .../minimax/voice-timbre-reference.json | 6 +- 22 files changed, 371 insertions(+), 42 deletions(-) diff --git a/dw/plan.py b/dw/plan.py index e0d19e7c..174cd5ec 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -472,7 +472,8 @@ def _list_entry_field_shifted(default_entries, effective_entries): # The generic diffusers argument names a curated figure scales along when a # run moves one off the default it was measured at (#866). Any other declared -# driver has no rule to scale by, so moving it leaves no figure (#267). +# scalar driver (H3's weights_dtype, a music audio_duration) has no rule to +# scale by, so moving it keeps the figure and flags it (`_scaled`). STEPS_DRIVER = "num_inference_steps" FRAMES_DRIVER = "num_frames" SCALABLE_DRIVERS = (STEPS_DRIVER, "width", "height", FRAMES_DRIVER) @@ -591,12 +592,15 @@ def _list_frames_scaling(name, default_entries, effective_entries, list_entries) def _driver_scaling(definition, expanded, list_entries): """How far the declared `cost_drivers` moved the run away from the - defaults the curated `cost` was measured at, as (factor, axes), or None - when one moved in a way nothing can scale (#267, #866). + defaults the curated `cost` was measured at, as (factor, axes, unscaled), + or None when a scalable one moved to a value that is not a number, or a + list's frames moved in a way nothing can scale (#267, #866). A list driver's length is `_repriced`'s to price; here only its entries' `num_frames` are. A scalar one in `SCALABLE_DRIVERS` scales in - proportion; any other scalar driver that moved leaves no figure. + proportion. Any other scalar driver that moved has no rule to scale by, + for every template: its name goes in `unscaled`, and the figure is kept + but flagged (#866, Q4). """ defaults = definition.get("variables") or {} effective = expanded.get("variables") or {} @@ -604,6 +608,7 @@ def _driver_scaling(definition, expanded, list_entries): axes = [] at_default = {} running = {} + unscaled = [] for name in _declared_drivers(definition): default_value = defaults.get(name) if isinstance(default_value, list): @@ -624,21 +629,28 @@ def _driver_scaling(definition, expanded, list_entries): elif _driver_comparable(effective.get(name)) != _driver_comparable( default_value ): - return None + unscaled.append(name) scaled = scale_curated(factor, at_default, running) if scaled is None: return None - return scaled[0], scaled[1] + axes + return scaled[0], scaled[1] + axes, unscaled def _scaled(price, scaling): """`price` scaled by a `_driver_scaling` answer: `derived` once any axis moved (another device's figure stays `other_device`, as `_price` keeps it - for a resized list - #589), with `scaled` saying from what.""" + for a resized list - #589), with `scaled` saying from what. A driver with + no rule to scale by that moved leaves the basis alone, and is named in + `differs` with `low_confidence` set: the figure was measured at another + setting of it (#866).""" if scaling is None: return {"minutes": None, "basis": UNKNOWN, "measured_on": None} - factor, axes = scaling - if not axes or price["minutes"] is None: + factor, axes, unscaled = scaling + if price["minutes"] is None: + return price + if unscaled: + price = {**price, "differs": list(unscaled), "low_confidence": True} + if not axes: return price basis = OTHER_DEVICE if price["basis"] == OTHER_DEVICE else DERIVED return { @@ -761,6 +773,7 @@ def __init__(self, minutes, partial, unpriced): self.measured_on = set() self.bases = [] self.scaled = [] + self.differs = [] def add(self, path, child, observed): self.had_child = True @@ -777,12 +790,18 @@ def add(self, path, child, observed): self.bases.append((child["basis"], child.get("measured_on"))) if child.get("scaled"): self.scaled.append(f"{path}: {child['scaled']}") + if not observed: + self.differs.extend(child.get("differs") or []) if self.minutes is not None: self.minutes += child["minutes"] else: self.minutes = child["minutes"] +def _unique(names): + return list(dict.fromkeys(names)) + + def _rolled_up_estimate(own, totals, device, cached_steps, total_steps): """The final estimate from the parent's own price and its children's.""" minutes = totals.minutes @@ -853,6 +872,12 @@ def _rolled_up_estimate(own, totals, device, cached_steps, total_steps): scaled = ([own["scaled"]] if own.get("scaled") else []) + totals.scaled if scaled and minutes is not None and top_basis != OBSERVED: result["scaled"] = "; ".join(scaled) + differs = _unique((own.get("differs") or []) + totals.differs) + if differs and minutes is not None and top_basis != OBSERVED: + # A figure, own or a child's, measured at another setting of a driver + # nothing scales (#866) + result["differs"] = differs + result["low_confidence"] = True if top_basis == OBSERVED: # The rolled-up-from-children case (#268): no curated figure of the # parent's own exists to blend toward (that is this branch's own @@ -925,6 +950,13 @@ def estimate( measured["scaled"] = "; ".join( part for part in (measured.get("scaled"), own["scaled"]) if part ) + if measured.get("tempered") and own.get("differs"): + # It blended toward a figure measured at another setting of a + # driver nothing scales (#866) + measured["differs"] = _unique( + (measured.get("differs") or []) + own["differs"] + ) + measured["low_confidence"] = True measured["cached_minutes"] = _cached_minutes( measured["minutes"], cached_steps, total_steps ) diff --git a/tests/test_observed_cost.py b/tests/test_observed_cost.py index 6bf3ebad..83f88ee4 100644 --- a/tests/test_observed_cost.py +++ b/tests/test_observed_cost.py @@ -662,8 +662,13 @@ def test_a_driver_is_a_size_not_a_piece_of_text(self): """A prompt, a seed or a URL moves the output, not the wall clock, and bucketing on one gives every run its own bucket and so a permanent `runs: 1`. `num_images_per_prompt` is a count and is - fine - the rule is the value's *type*, not the name's spelling.""" + fine - the rule is the value's *type*, not the name's spelling. + + The exception is a configuration choice drawn from a short list: + H3's weights dtype and LoRA move the wall clock and are text, and + bucket by exact value (#866, Q5).""" never = {"seed", "prompt", "negative_prompt"} + choices = {"weights_dtype", "lora_model_name"} for path in sorted( glob.glob( os.path.join(REPO_ROOT, "workflows", "**", "*.json"), recursive=True @@ -675,7 +680,7 @@ def test_a_driver_is_a_size_not_a_piece_of_text(self): where = os.path.relpath(path, REPO_ROOT) for name in definition.get("cost_drivers") or []: assert name not in never, f"{where} buckets on '{name}'" - assert not isinstance(variables.get(name), str), ( + assert name in choices or not isinstance(variables.get(name), str), ( f"{where} buckets on '{name}', whose default is text" ) @@ -752,6 +757,54 @@ def test_the_minimax_templates_declare_drivers(self): assert definition.get("cost_drivers"), os.path.relpath(path, REPO_ROOT) +class TestH3KnobsSplitTheBucket: + """#866 stage B: a run that overrode a knob with no rule to scale by no + longer pools with the runs at the default.""" + + @staticmethod + def h3(): + return workflow( + ["num_frames", "weights_dtype", "lora_model_name", "attention_backend"], + num_frames=124, + weights_dtype="{int4}", + lora_model_name="lightx2v/Minimax-h3-Turbo", + attention_backend=None, + ) + + def test_an_int8_run_does_not_pool_with_int4_runs(self): + rows = [ + run(8 * MINUTE, {}), + run(8 * MINUTE, {"weights_dtype": "{int4}"}), + run(20 * MINUTE, {"weights_dtype": "{int8}"}), + ] + default = observed_for(self.h3(), rows) + assert (default["runs"], default["cold_minutes"]) == (2, 8.0) + int8 = observed_for(self.h3(), rows, arguments={"weights_dtype": "{int8}"}) + assert (int8["runs"], int8["cold_minutes"]) == (1, 20.0) + + def test_a_lora_buckets_by_its_exact_name(self): + rows = [ + run(8 * MINUTE, {}), + run(12 * MINUTE, {"lora_model_name": None}), + run(9 * MINUTE, {"lora_model_name": "someone/other-lora"}), + ] + assert observed_for(self.h3(), rows)["runs"] == 1 + none = observed_for(self.h3(), rows, arguments={"lora_model_name": None}) + assert none["cold_minutes"] == 12.0 + other = observed_for( + self.h3(), rows, arguments={"lora_model_name": "someone/other-lora"} + ) + assert other["cold_minutes"] == 9.0 + + def test_a_native_attention_run_leaves_the_default_bucket(self): + rows = [run(8 * MINUTE, {}), run(11 * MINUTE, {"attention_backend": "native"})] + assert observed_for(self.h3(), rows)["runs"] == 1 + native = observed_for( + self.h3(), rows, arguments={"attention_backend": "native"} + ) + assert native["cold_minutes"] == 11.0 + + @pytest.mark.parametrize("value", [None, "", "[]", "not json"]) def test_a_manifest_that_will_not_parse_is_not_a_cached_run(value): from dw.server.observed_cost import _every_step_was_reused diff --git a/tests/test_plan.py b/tests/test_plan.py index 1c687bd1..72b2adfd 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -412,6 +412,61 @@ def test_a_results_format(self, plan): CATALOG_H3 = "workflows/templates/minimax/reference-to-video.json" +CATALOG_MUSIC = "workflows/templates/minimax/music.json" +# The knobs stage B of #866 made cost drivers of every H3 template +H3_UNSCALABLE = ("weights_dtype", "lora_model_name", "attention_backend", "cache") + + +def _repo_json(path): + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent + return json.loads((root / path).read_text()) + + +def _minimax_with_h3_drivers(): + import glob + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent + found = [] + for path in sorted(glob.glob(str(root / "workflows/templates/minimax/*.json"))): + drivers = json.loads(pathlib.Path(path).read_text()).get("cost_drivers") or [] + if "weights_dtype" in drivers: + found.append(str(pathlib.Path(path).relative_to(root))) + return found + + +@pytest.mark.parametrize("path", _minimax_with_h3_drivers()) +def test_h3_unscalable_drivers_leave_every_fingerprint(plan, path): + """Stage 0's pin, held for stage B (#866): declaring H3's unscalable + knobs as cost drivers changes no template's plan fingerprint.""" + template = _repo_json(path) + before = copy.deepcopy(template) + before["cost_drivers"] = [ + name for name in template["cost_drivers"] if name not in H3_UNSCALABLE + ] + assert plan(template)["fingerprint"] == plan(before)["fingerprint"] + + +def test_every_h3_template_with_drivers_declares_its_unscalable_knobs(): + """#866 stage B: a minimax H3 template that buckets on its size also + buckets on the knobs that move its wall clock with no rule to scale by.""" + import glob + import pathlib + + root = pathlib.Path(__file__).resolve().parent.parent + missing = [] + for path in sorted(glob.glob(str(root / "workflows/templates/minimax/*.json"))): + definition = json.loads(pathlib.Path(path).read_text()) + variables = definition.get("variables") or {} + if not definition.get("cost_drivers") or "weights_dtype" not in variables: + continue + absent = [n for n in H3_UNSCALABLE if n not in definition["cost_drivers"]] + if absent: + missing.append((pathlib.Path(path).name, absent)) + assert _minimax_with_h3_drivers() + assert missing == [] class TestTheCatalogH3Template: @@ -428,11 +483,8 @@ def template(self): def test_cost_drivers_leave_the_fingerprint(self, plan, template): other = copy.deepcopy(template) - other["cost_drivers"] = other["cost_drivers"] + [ - "weights_dtype", - "attention_backend", - "lora_model_name", - "cache", + other["cost_drivers"] = [ + name for name in other["cost_drivers"] if name not in H3_UNSCALABLE ] del other["vram_estimate"] assert plan(template)["fingerprint"] == plan(other)["fingerprint"] @@ -656,25 +708,29 @@ def test_minutes_is_rounded_to_one_decimal(self, plan): spec["cost"] = [cost("cuda", 10.04)] assert plan(spec)["estimate"]["minutes"] == 10.0 - def test_a_shifted_unscalable_cost_driver_falls_back_to_unknown(self, plan): + def test_a_shifted_unscalable_cost_driver_is_flagged(self, plan): """The catalog figure was measured at frames=25; a caller who overrides a declared cost_driver away from that value is not describing the run the figure was measured for (#267). `frames` is - not one of the drivers a figure scales along (#866), so there is - nothing to re-price it by.""" + not one of the drivers a figure scales along, so the figure is kept + and flagged rather than withheld (#866, Q4).""" spec = definition() spec["cost_drivers"] = ["frames"] answer = plan(spec, arguments={"frames": 9})["estimate"] - assert (answer["minutes"], answer["basis"]) == (None, "unknown") + assert (answer["minutes"], answer["basis"]) == (10.0, "catalog") + assert answer["low_confidence"] is True + assert answer["differs"] == ["frames"] - def test_a_shifted_unscalable_driver_is_unknown_on_another_device_too(self, plan): + def test_a_shifted_unscalable_driver_is_flagged_on_another_device_too(self, plan): """#589: the other_device branch skipped the #267 check, so a 241-frame - run was quoted the 121-frame clip's figure.""" + run was quoted the 121-frame clip's figure with nothing to say so.""" spec = definition() spec["cost"] = [cost("mps", 40, name="M2")] spec["cost_drivers"] = ["frames"] answer = plan(spec, arguments={"frames": 9})["estimate"] - assert (answer["minutes"], answer["basis"]) == (None, "unknown") + assert (answer["minutes"], answer["basis"]) == (40.0, "other_device") + assert answer["low_confidence"] is True + assert answer["differs"] == ["frames"] def test_an_unshifted_scalar_cost_driver_still_quotes_the_catalog_figure( self, plan @@ -1176,6 +1232,118 @@ def test_dialogue_short_with_one_shot_longer(self, plan): ) +class TestUnscalableDrivers: + """#866 stage B: a declared scalar driver with no rule to scale by keeps + the (scaled) figure, sets `low_confidence` and names itself in + `differs`, for every template (Q4).""" + + def test_reference_to_video_in_int8(self, plan): + template = _repo_json(CATALOG_H3) + answer = plan(template, arguments={"weights_dtype": "{int8}"})["estimate"] + assert (answer["minutes"], answer["basis"]) == (7.9, "catalog") + assert answer["low_confidence"] is True + assert answer["differs"] == ["weights_dtype"] + + def test_reference_to_video_without_its_lora_at_thirty_steps(self, plan): + template = _repo_json(CATALOG_H3) + arguments = {"lora_model_name": None, "num_inference_steps": 30} + answer = plan(template, arguments=arguments)["estimate"] + # 7.88 x 30/9, scaled and flagged + assert (answer["minutes"], answer["basis"]) == (26.3, "derived") + assert "steps 30 vs 9" in answer["scaled"] + assert answer["low_confidence"] is True + assert answer["differs"] == ["lora_model_name"] + + def test_every_moved_knob_is_named(self, plan): + template = _repo_json(CATALOG_H3) + arguments = { + "weights_dtype": "{int8}", + "attention_backend": "native", + "cache": {"type": "first_block"}, + } + answer = plan(template, arguments=arguments)["estimate"] + assert answer["minutes"] == 7.9 + assert sorted(answer["differs"]) == [ + "attention_backend", + "cache", + "weights_dtype", + ] + + def test_reference_to_video_at_its_defaults_is_not_flagged(self, plan): + template = _repo_json(CATALOG_H3) + answer = plan(template)["estimate"] + assert (answer["minutes"], answer["basis"]) == (7.9, "catalog") + assert "low_confidence" not in answer + assert "differs" not in answer + + def test_the_defaults_passed_explicitly_are_not_flagged(self, plan): + template = _repo_json(CATALOG_H3) + variables = template["variables"] + arguments = {name: variables[name] for name in H3_UNSCALABLE} + answer = plan(template, arguments=arguments)["estimate"] + assert "low_confidence" not in answer + + def test_music_at_another_duration(self, plan): + """#440's case, now a flagged figure rather than `unknown`.""" + template = _repo_json(CATALOG_MUSIC) + answer = plan(template, arguments={"audio_duration": 240})["estimate"] + assert (answer["minutes"], answer["basis"]) == (3.6, "catalog") + assert answer["low_confidence"] is True + assert answer["differs"] == ["audio_duration"] + + def test_a_thin_bucket_blends_toward_a_flagged_figure(self, plan): + """One int8 run, blended toward the int4 catalog figure: the blend + leans on a figure measured at another setting, so it says so.""" + spec = definition() + spec["cost_drivers"] = ["frames"] + answer = plan( + spec, arguments={"frames": 9}, observed=observed(minutes=6, runs=1) + )["estimate"] + assert answer["basis"] == "observed" + assert answer["tempered"] is True + assert answer["low_confidence"] is True + assert answer["differs"] == ["frames"] + + def test_a_full_bucket_is_not_flagged(self, plan): + """Three runs at the moved setting are a measurement of it.""" + spec = definition() + spec["cost_drivers"] = ["frames"] + answer = plan( + spec, arguments={"frames": 9}, observed=observed(minutes=6, runs=3) + )["estimate"] + assert answer["minutes"] == 6.0 + assert "low_confidence" not in answer + assert "differs" not in answer + + def test_a_composed_child_at_another_setting_flags_the_total(self, plan, tmp_path): + child = { + "id": "child", + "cost": [cost("cuda", 5)], + "variables": {"weights_dtype": "{int4}"}, + "cost_drivers": ["weights_dtype"], + "steps": [], + } + (tmp_path / "child.json").write_text(json.dumps(child)) + parent = { + "id": "parent", + "cost": [cost("cuda", 2)], + "steps": [ + {"name": "own", "task": {"command": "x", "arguments": {}}}, + { + "name": "child", + "workflow": { + "path": "child.json", + "arguments": {"weights_dtype": "{int8}"}, + }, + }, + ], + } + answer = plan(parent)["estimate"] + assert answer["minutes"] == 7.0 + assert answer["low_confidence"] is True + assert answer["differs"] == ["weights_dtype"] + + def composing(child_path): return { "id": "parent", diff --git a/workflows/templates/minimax/chain-matched-and-aligned.json b/workflows/templates/minimax/chain-matched-and-aligned.json index 5da13613..1e209abf 100644 --- a/workflows/templates/minimax/chain-matched-and-aligned.json +++ b/workflows/templates/minimax/chain-matched-and-aligned.json @@ -7,7 +7,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/chain-matched-to-audio.json b/workflows/templates/minimax/chain-matched-to-audio.json index 41360ab5..990b4f13 100644 --- a/workflows/templates/minimax/chain-matched-to-audio.json +++ b/workflows/templates/minimax/chain-matched-to-audio.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/chain-video-continuity.json b/workflows/templates/minimax/chain-video-continuity.json index ffc34283..31cbf738 100644 --- a/workflows/templates/minimax/chain-video-continuity.json +++ b/workflows/templates/minimax/chain-video-continuity.json @@ -8,7 +8,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/chained-segments.json b/workflows/templates/minimax/chained-segments.json index c088e893..9f171f61 100644 --- a/workflows/templates/minimax/chained-segments.json +++ b/workflows/templates/minimax/chained-segments.json @@ -26,7 +26,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "variables": { "prompts": null, diff --git a/workflows/templates/minimax/composable-references.json b/workflows/templates/minimax/composable-references.json index 3b61b3cf..c39c1559 100644 --- a/workflows/templates/minimax/composable-references.json +++ b/workflows/templates/minimax/composable-references.json @@ -7,7 +7,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/dialogue-short.json b/workflows/templates/minimax/dialogue-short.json index a4593702..2038241c 100644 --- a/workflows/templates/minimax/dialogue-short.json +++ b/workflows/templates/minimax/dialogue-short.json @@ -14,7 +14,11 @@ "shots", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/enhance-prompt-with-image.json b/workflows/templates/minimax/enhance-prompt-with-image.json index 7425825f..3abfa59b 100644 --- a/workflows/templates/minimax/enhance-prompt-with-image.json +++ b/workflows/templates/minimax/enhance-prompt-with-image.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "variables": { "idea": "Task: I2VA. Duration: 5.17 seconds. Idea: a blown glass figure's tentacles writhe as it speaks in a deep, echoing voice, calling forth ancient ancestors.", diff --git a/workflows/templates/minimax/enhance-prompt.json b/workflows/templates/minimax/enhance-prompt.json index 414647e4..e645985f 100644 --- a/workflows/templates/minimax/enhance-prompt.json +++ b/workflows/templates/minimax/enhance-prompt.json @@ -14,7 +14,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.3, diff --git a/workflows/templates/minimax/first-and-last-frame.json b/workflows/templates/minimax/first-and-last-frame.json index dc253ec4..05afbc10 100644 --- a/workflows/templates/minimax/first-and-last-frame.json +++ b/workflows/templates/minimax/first-and-last-frame.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "variables": { "prompt": "How the reference pictures align with the target video - Picture 1 (from Shot 1) aligns with the 0.00-second mark of the target video; Picture 2 (from Shot 1) aligns with the 5.17-second mark of the target video.\n\nintegrated_multimodal_description: [Shot 1] Cinematic, live-action macro photography with a shallow depth of field, one continuous shot from the first frame to the last. The video opens on : a translucent blown glass figure centred against a dark, softly graded background, its tentacles curled in close to the body, cool rim light running along every curve and small caustics pooling on the surface beneath it. The tentacles unfurl outward and upward through the first half of the clip, reaching their widest span near the midpoint with the glass flexing as though molten while staying perfectly clear, and the caustics below spreading with them. The figure (S1), a deep cavernous male voice with a long tail of reverb, unhurried and very low in pitch, says [English] Rise now. The old ones are listening. Through the second half the tentacles draw steadily back in along the same paths, the span narrowing and the caustics contracting, until the figure settles into exactly the pose, position and lighting of as the video ends. The camera holds a static shot throughout so the opening and closing frames match.\n\noverall_soundscape: A dark, near-silent room tone under a faint low-frequency hum. The tentacle movement carries a fine, dry glass-on-glass squeak that rises as the limbs spread and falls as they close, with an occasional soft chime as two limbs touch.\n\nnon_diegetic_music: A sustained low drone of bowed double bass and detuned synthesiser, almost subsonic, swelling as the tentacles reach their widest span and returning to its opening level as they close, with no discernible melody or pulse.", diff --git a/workflows/templates/minimax/generated-subject-reference.json b/workflows/templates/minimax/generated-subject-reference.json index 60a92525..709f3baa 100644 --- a/workflows/templates/minimax/generated-subject-reference.json +++ b/workflows/templates/minimax/generated-subject-reference.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/image-to-video.json b/workflows/templates/minimax/image-to-video.json index 57a4bb0e..be5a19c2 100644 --- a/workflows/templates/minimax/image-to-video.json +++ b/workflows/templates/minimax/image-to-video.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "variables": { "prompt": "For the target video, at 0.00 seconds into the target video, (from [Shot 1]) is fully referenced.\n\nintegrated_multimodal_description: [Shot 1] Cinematic, live-action macro photography with a shallow depth of field. The shot opens exactly on : a translucent blown glass figure fills the frame against a dark, softly graded background, its surface catching cool rim light along every curve and throwing small caustics onto the surface beneath it. Its tentacles, until now still, begin to writhe, curling and uncurling in slow deliberate coils, the glass flexing as though molten while staying perfectly clear. The figure (S1), a deep cavernous male voice with a long tail of reverb, unhurried and very low in pitch, says [English] Rise now. The old ones are listening. As the words land its central mass swells and contracts once, and the caustics on the surface below ripple with it. The camera pushes in with small amplitude at slow speed toward the figure's core through the final seconds, holding the writhing tentacles at the edges of frame.\n\noverall_soundscape: A dark, near-silent room tone under a faint low-frequency hum. The tentacle movement carries a fine, dry glass-on-glass squeak and an occasional soft chime as two limbs touch. A distant, slow swell of moving air rises beneath the voice and falls away after it.\n\nnon_diegetic_music: A sustained low drone of bowed double bass and detuned synthesiser, almost subsonic, rising very slowly in intensity across the clip with no discernible melody or pulse.", diff --git a/workflows/templates/minimax/last-frame-only.json b/workflows/templates/minimax/last-frame-only.json index 1efdc2aa..602986d0 100644 --- a/workflows/templates/minimax/last-frame-only.json +++ b/workflows/templates/minimax/last-frame-only.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "variables": { "prompt": "How the reference pictures align with the target video - (from [Shot 1]) aligns with the 5.17-second mark of the target video.\n\nintegrated_multimodal_description: [Shot 1] Cinematic, live-action macro photography with a shallow depth of field, one continuous shot from the first frame to the last. The video opens on a translucent blown glass figure held far back in a dark, softly graded field, small in frame and turned away from the lens, its tentacles trailing loosely behind it and only a thin cool rim light picking out their edges. The figure drifts steadily toward the lens through the whole clip, growing in the frame as it comes, and turns as it travels until it faces camera straight on. The tentacles gather in from their trailing spread and curl close to the body along the way, the rim light strengthening along every curve as the figure nears, and caustics begin to pool on the surface beneath it. The camera pushes in slowly to meet it. The clip ends on exactly the framing, pose and lighting of , the figure centred and still.\n\noverall_soundscape: A dark, near-silent room tone under a faint low-frequency hum, the hum rising very slightly in level as the figure closes on the lens. The tentacle movement carries a fine, dry glass-on-glass squeak that grows more present as the figure nears, with an occasional soft chime as two limbs touch, and the last second settles into near silence.\n\nnon_diegetic_music: A sustained low drone of bowed double bass and detuned synthesiser, almost subsonic, swelling steadily as the figure approaches and cutting to nothing on the final frame, with no discernible melody or pulse.", diff --git a/workflows/templates/minimax/music-video.json b/workflows/templates/minimax/music-video.json index 3cb021f2..dd5123ca 100644 --- a/workflows/templates/minimax/music-video.json +++ b/workflows/templates/minimax/music-video.json @@ -14,7 +14,11 @@ "audio_duration", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/reference-to-video.json b/workflows/templates/minimax/reference-to-video.json index 9e1409e9..ed8760c1 100644 --- a/workflows/templates/minimax/reference-to-video.json +++ b/workflows/templates/minimax/reference-to-video.json @@ -14,7 +14,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/shots-batch.json b/workflows/templates/minimax/shots-batch.json index 48e03023..619d624a 100644 --- a/workflows/templates/minimax/shots-batch.json +++ b/workflows/templates/minimax/shots-batch.json @@ -18,7 +18,11 @@ "shots", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.3, diff --git a/workflows/templates/minimax/storyboard.json b/workflows/templates/minimax/storyboard.json index 9d118adb..7532d214 100644 --- a/workflows/templates/minimax/storyboard.json +++ b/workflows/templates/minimax/storyboard.json @@ -13,7 +13,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, diff --git a/workflows/templates/minimax/video-with-audio-768p.json b/workflows/templates/minimax/video-with-audio-768p.json index 987c674b..6fd9d36c 100644 --- a/workflows/templates/minimax/video-with-audio-768p.json +++ b/workflows/templates/minimax/video-with-audio-768p.json @@ -14,7 +14,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.3, diff --git a/workflows/templates/minimax/video-with-audio.json b/workflows/templates/minimax/video-with-audio.json index af803a5e..5887652c 100644 --- a/workflows/templates/minimax/video-with-audio.json +++ b/workflows/templates/minimax/video-with-audio.json @@ -14,7 +14,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.3, diff --git a/workflows/templates/minimax/voice-timbre-reference.json b/workflows/templates/minimax/voice-timbre-reference.json index cc5f4292..052a2655 100644 --- a/workflows/templates/minimax/voice-timbre-reference.json +++ b/workflows/templates/minimax/voice-timbre-reference.json @@ -6,7 +6,11 @@ "num_frames", "num_inference_steps", "width", - "height" + "height", + "weights_dtype", + "lora_model_name", + "attention_backend", + "cache" ], "vram_estimate": { "base_gb": 16.0, From c2fcb09739cfc29509d7a70f385f164636499d49 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 20:39:22 -0500 Subject: [PATCH 148/204] docs(plan): #907 - an unscalable driver's shift keeps a flagged figure Co-Authored-By: Claude Opus 5.5 --- docs/SERVER.md | 8 ++++++-- docs/WORKFLOW_GUIDE.md | 7 ++++++- dw/server/api_models.py | 4 +++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/SERVER.md b/docs/SERVER.md index 5b7306ac..cb5a648d 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -442,8 +442,12 @@ The editor's forms come from these; they are just as usable from scripts: `width` x `height` and `num_frames` - per entry in a list, summed - that the caller moved off the variable defaults a `cost` entry is measured at, with `scaled` naming each moved axis; an estimate, not a measurement; a - moved driver with no rule to scale by, or a non-numeric value, is - `unknown`), `other_device` (no entry for + non-numeric value of one of those is `unknown`). Any other declared + driver moved off its default - H3's `weights_dtype` or LoRA, a music + `audio_duration` - has no rule to scale by: the (scaled) figure is kept + under whatever basis it has, with `low_confidence` true and `differs` + naming the drivers that moved, because it was measured at another setting + of them; `other_device` (no entry for the serving backend; the first entry's figure, which is a warning rather than a quote) or `unknown` (no cost block, or more than one list changed so there is nothing honest to extrapolate along) or `inherited` (an diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 3310978b..3ca67629 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -999,7 +999,12 @@ reports the result as `observed` beside the curated `cost`, so a 345-frame run never informs a 124-frame figure and a list driver buckets on its length. Declaring none is not neutral: the figure then falls back to runs that overrode nothing at all, which most real runs do, so a measured workflow with -no drivers keeps answering "unknown". Each name must be a variable the +no drivers keeps answering "unknown". A driver the estimate cannot scale +along - anything but steps, width, height and frames, such as a weights +dtype, a LoRA name or an audio duration - still splits the buckets (a text +value buckets by its exact value). When a run moves one off its default, the +curated figure is kept but carries `low_confidence: true`, and `differs` +names the drivers, since it was measured at the default. Each name must be a variable the workflow declares — `tests/test_observed_cost.py` sweeps the catalog for one that is not, since a driver bucketing on nothing looks exactly like a driver that works. diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 067da53f..ed7d029c 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -624,7 +624,9 @@ class PlanEstimate(ApiModel): differs: list[str] = sometimes( "With basis `inherited`: what this workflow sets differently from " "`inherited_from` among offload, quantization and frame count; empty " - "when nothing does." + "when nothing does. Otherwise: the cost drivers with no rule to scale " + "by that moved off the default the figure was measured at, with " + "`low_confidence` true." ) scaled: str = sometimes( "With basis `derived` (or `other_device`, or a tempered `observed`): " From 2566e6cc378bfaa12af5549cbb3baac29d48fcaf Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 21:04:57 -0500 Subject: [PATCH 149/204] feat(plan): #908 - the inherited estimate scales by inference steps and canvas, and names attention, cache and LoRA Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/SERVER.md | 8 +- dw/plan.py | 4 + dw/server/deps.py | 83 ++++++++++++--- dw/vram_inheritance.py | 63 +++++++---- tests/test_inherited_cost.py | 200 ++++++++++++++++++++++++++++++++++- 6 files changed, 317 insertions(+), 43 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index f8f89e2c..8f77f8ab 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,7 @@ one to open. | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | -| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization and frame count set differently from the template. More pipeline steps or frames than the template run re-price its minutes linearly (steps x mean frames per step) with `scaled` naming why and `low_confidence` set (#851). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns` | +| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache and LoRA set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps, frames, inference steps or canvas pixels than the template run re-price its minutes linearly (pipeline steps x the mean frames, inference steps and pixels per step) with `scaled` naming why and `low_confidence` set (#851, #866) - the same proportions `scale_curated` applies to a catalog figure, so an inline copy and its template quote one request alike. It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | | Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It projects and never refuses (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | diff --git a/docs/SERVER.md b/docs/SERVER.md index 430a4b51..72e1a25a 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -455,8 +455,12 @@ The editor's forms come from these; they are just as usable from scripts: `observed` runs of the catalog template that loads the same pipeline - `component_type` + `model_name` + `workflow`, as the VRAM ceiling is inherited - with `inherited_from` naming it and `differs` listing what the - workflow sets differently among offload, quantization and frame count; - validate warns when it does, and never refuses); a composed child's + workflow sets differently among offload, quantization, frame count, + attention backend, cache and LoRA, with `low_confidence` true when it lists + any; more pipeline steps, frames, inference steps or canvas pixels than the + template's run re-price it in proportion, as the catalog figure is scaled, + with `scaled` naming each axis; validate warns when it does, and never + refuses); a composed child's cost is added to a curated figure and `partial` is true when a child has none - an `observed` figure already measured the whole run, children included, so nothing is added to it and `partial` is false; and diff --git a/dw/plan.py b/dw/plan.py index 174cd5ec..6f080ed9 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -1027,6 +1027,10 @@ def _observed(observed, device): estimate["basis"] = INHERITED estimate["inherited_from"] = inherited_from estimate["differs"] = list(observed.get("differs") or []) + if estimate["differs"]: + # Measured at another attention backend, cache, LoRA, offload or + # quantization than this run's: kept, but flagged (#866) + estimate["low_confidence"] = True if observed.get("scaled"): # Re-priced linearly from the template's size (#851): an # approximation, not a measurement diff --git a/dw/server/deps.py b/dw/server/deps.py index ce8f5ba3..9ba259df 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -21,6 +21,8 @@ build_index, inherited_differences, pipeline_frames, + pipeline_values, + sets_differently, template_identity, ) from .observed_cost import declared_drivers @@ -272,13 +274,37 @@ def _pipeline_steps(definition): ) +def _canvases(definition, arguments=None): + """Each pipeline step's `(width, height)`, as `pipeline_values` reads + them, None for a step that does not set both as numbers.""" + widths = pipeline_values(definition, "width", arguments) + heights = pipeline_values(definition, "height", arguments) + return [(w, h) if w and h else None for w, h in zip(widths, heights)] + + +def _size(value): + # A canvas counts by its pixels + return value[0] * value[1] if isinstance(value, tuple) else value + + +def _described(values, mean): + if not isinstance(values[0], tuple): + return _number(mean) + if len(set(values)) == 1: + width, height = values[0] + return f"{_number(width)}x{_number(height)}" + return f"{_number(mean)} pixels" + + def _scale_past_template(definition, template, drivers, arguments): """How the workflow's size compares to the template run the bucket was - measured on (#821, #851): (factor, reason). The factor is the ratio of - pipeline steps times the ratio of mean frames per step - linear, as - `derived` re-prices a list of another length - and the reason names - each axis that moved. (1, None) when the workflow is the template's - size; (None, reason) when its size cannot be compared.""" + measured on (#821, #851, #866): (factor, reason). The factor is the + ratio of pipeline steps times the ratios of the mean frames, inference + steps and canvas pixels per step - linear, as `scale_curated` re-prices + a catalog figure along the same axes, so the two quote one request + alike - and the reason names each axis that moved. (1, None) when the + workflow is the template's size; (None, reason) when its size cannot be + compared.""" steps, template_steps = _pipeline_steps(definition), _pipeline_steps(template) # The template as the bucket's drivers set it bucket = {**template, "variables": {**(template.get("variables") or {}), **drivers}} @@ -288,18 +314,41 @@ def _scale_past_template(definition, template, drivers, arguments): reasons.append( f"runs the pipeline in {steps} steps, template measured {template_steps}" ) - # `pipeline_frames` takes the template's variables as the bucket sets them - mine = [f for f in pipeline_frames(definition, arguments) if f] - theirs = [f for f in pipeline_frames(bucket) if f] - if mine and theirs: - mean_mine, mean_theirs = sum(mine) / len(mine), sum(theirs) / len(theirs) - if mean_mine != mean_theirs: - factor *= mean_mine / mean_theirs - reasons.append( - f"frame count {_number(mean_mine)} vs {_number(mean_theirs)} per step" - ) - elif "frame count" in inherited_differences(definition, bucket, arguments): - return None, "frame count differs and cannot be compared" + # Each axis read per pipeline step, the template's variables as the + # bucket sets them; a step that does not set it as a number is left out + for label, keys, mine, theirs in ( + ( + "frame count", + ("num_frames",), + pipeline_frames(definition, arguments), + pipeline_frames(bucket), + ), + ( + "inference steps", + ("num_inference_steps",), + pipeline_values(definition, "num_inference_steps", arguments), + pipeline_values(bucket, "num_inference_steps"), + ), + ( + "canvas", + ("width", "height"), + _canvases(definition, arguments), + _canvases(bucket), + ), + ): + mine = [value for value in mine if value] + theirs = [value for value in theirs if value] + if mine and theirs: + mean_mine = sum(map(_size, mine)) / len(mine) + mean_theirs = sum(map(_size, theirs)) / len(theirs) + if mean_mine != mean_theirs: + factor *= mean_mine / mean_theirs + reasons.append( + f"{label} {_described(mine, mean_mine)} vs " + f"{_described(theirs, mean_theirs)} per step" + ) + elif any(sets_differently(definition, bucket, key, arguments) for key in keys): + return None, f"{label} differs and cannot be compared" return factor, "; ".join(reasons) or None diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index a113f38f..6befa03f 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -67,50 +67,69 @@ def _found(node, key, variables): # What changes a pipeline's cost without changing its identity (#797): -# (label, key) pairs read wherever a step sets them +# (label, key) pairs read wherever a step sets them. The last three are the +# keys a pipeline step sets its attention backend, step cache and LoRAs +# under (#866) _COST_SETTINGS = ( ("offload", "offload"), ("quantization", "quantization_config"), ("frame count", "num_frames"), + ("attention backend", "attention_backend"), + ("cache", "cache"), + ("LoRA", "loras"), ) def inherited_differences(definition, template, arguments=None): """What `definition` sets differently from `template` among the things that move a pipeline's cost without changing which pipeline it is - - offload, quantization, frame count - as short phrases (#797). A figure - inherited from `template` is an approximation to the extent these - differ. `arguments` (the caller's) override `definition`'s variables.""" + offload, quantization, frame count, attention backend, cache, LoRA - as + short phrases (#797, #866). A figure inherited from `template` is an + approximation to the extent these differ. `arguments` (the caller's) + override `definition`'s variables.""" + return [ + label + for label, key in _COST_SETTINGS + if sets_differently(definition, template, key, arguments) + ] + + +def sets_differently(definition, template, key, arguments=None): + """Whether `definition`'s steps set `key` to other values than + `template`'s, `variable:` references resolved (`arguments` override + `definition`'s variables).""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - template_variables = template.get("variables") or {} - differs = [] - for label, key in _COST_SETTINGS: - mine = _found(definition.get("steps"), key, variables) - theirs = _found(template.get("steps"), key, template_variables) - if mine != theirs: - differs.append(label) - return differs + mine = _found(definition.get("steps"), key, variables) + theirs = _found(template.get("steps"), key, template.get("variables") or {}) + return mine != theirs -def pipeline_frames(definition, arguments=None): - """The frame count each pipeline step sets (`num_frames`, a `variable:` - reference resolved; `arguments` override the variables), None for a - step that sets none or a non-numeric one - for scaling an inherited - price by frame count (#851).""" +def pipeline_values(definition, key, arguments=None): + """The number each pipeline step sets `key` to (a `variable:` reference + resolved; `arguments` override the variables; the largest where a step + sets it twice), None for a step that sets none or a non-numeric one - + for scaling an inherited price along it (#851, #866).""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - frames = [] + found = [] for step in definition.get("steps") or []: if not isinstance(step, dict) or not isinstance(step.get("pipeline"), dict): continue - counts = [ + numbers = [ value - for value in map(json.loads, _found(step, "num_frames", variables)) + for value in map(json.loads, _found(step, key, variables)) if isinstance(value, (int, float)) and not isinstance(value, bool) ] - frames.append(max(counts) if counts else None) - return frames + found.append(max(numbers) if numbers else None) + return found + + +def pipeline_frames(definition, arguments=None): + """The frame count each pipeline step sets (`num_frames`), as + `pipeline_values` reads it - for scaling an inherited price by frame + count (#851).""" + return pipeline_values(definition, "num_frames", arguments) def declarations(catalog): diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 64b07a3f..6f9ff9a6 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -4,12 +4,15 @@ template had none. `inherited_observed` finds the catalog template with the same pipeline identity (the key `dw/vram_inheritance.py` matches on) and quotes its runs under `basis: "inherited"`, warning where the inline -workflow's offload, quantization or frame count differ. +workflow's offload, quantization, frame count, attention backend, cache or +LoRA differ, and scaling its minutes by its size (#851, #866). """ from types import SimpleNamespace from unittest.mock import patch +import pytest + import dw.plan from dw.plan import build_plan from dw.server.observed_cost import observed_for @@ -317,3 +320,198 @@ def test_the_plan_carries_the_withheld_reason(self, tmp_path): assert estimate["inherited_from"] == "templates/t" assert estimate["withheld"] == "not priced: why" assert "withheld" in _inherited_cost_warnings(estimate)[0] + + +def sized(name, steps=9, width=960, height=544, **extra): + """`definition` whose step also sets inference steps and a canvas.""" + workflow = definition(name, **extra) + workflow["variables"].update( + {"num_inference_steps": steps, "width": width, "height": height} + ) + workflow["cost_drivers"] += ["num_inference_steps", "width", "height"] + workflow["steps"][0]["run"].update( + { + "num_inference_steps": "variable:num_inference_steps", + "width": "variable:width", + "height": "variable:height", + } + ) + return workflow + + +class TestScaledByStepsAndCanvas: + """#866 stage C: the inherited figure scales by the inference-step and + canvas ratios too, as the catalog figure does (`scale_curated`).""" + + HISTORY = {"templates/t": [run(8 * MINUTE, {})]} + CATALOG = [("templates/t", sized("t"))] + + def test_thirty_steps_scale_by_the_step_ratio(self): + block = inherited(sized("inline", steps=30), self.CATALOG, self.HISTORY) + assert block["cold_minutes"] == 8.0 * 30 / 9 + assert "inference steps 30 vs 9" in block["scaled"] + + def test_a_larger_canvas_scales_by_the_pixel_ratio(self): + block = inherited( + sized("inline", width=1344, height=768), self.CATALOG, self.HISTORY + ) + assert block["cold_minutes"] == 8.0 * (1344 * 768) / (960 * 544) + assert "canvas 1344x768 vs 960x544" in block["scaled"] + + def test_the_callers_arguments_scale_it_and_the_axes_multiply(self): + block = inherited( + sized("inline"), + self.CATALOG, + self.HISTORY, + {"num_inference_steps": 30, "width": 1344, "height": 768}, + ) + assert block["cold_minutes"] == pytest.approx( + 8.0 * 30 / 9 * (1344 * 768) / (960 * 544) + ) + assert "inference steps" in block["scaled"] and "canvas" in block["scaled"] + + def test_a_bucket_at_those_values_is_not_scaled_again(self): + history = { + "templates/t": [ + run(8 * MINUTE, {}), + run(20 * MINUTE, {"num_inference_steps": 30}), + ] + } + block = inherited(sized("inline", steps=30), self.CATALOG, history) + assert block["cold_minutes"] == 20.0 + assert "scaled" not in block + + def test_a_canvas_that_cannot_be_compared_is_withheld(self): + inline = sized("inline") + inline["steps"][0]["run"]["width"] = "variable:missing" + block = inherited(inline, self.CATALOG, self.HISTORY) + assert "canvas differs and cannot be compared" in block["withheld"] + + def test_the_plan_quotes_it_inherited_and_scaled(self, tmp_path): + inline = sized("inline", steps=30, width=1344, height=768) + block = inherited(inline, self.CATALOG, {"templates/t": [run(8 * MINUTE)] * 3}) + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + estimate = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: block, + )["estimate"] + assert estimate["basis"] == "inherited" + assert estimate["minutes"] == round(8.0 * 30 / 9 * 1.9765, 1) + assert "inference steps 30 vs 9" in estimate["scaled"] + assert "canvas 1344x768 vs 960x544" in estimate["scaled"] + + +class TestAttentionCacheAndLoraDiffer: + """#866 stage C: the attention backend, cache and LoRA a step sets are + named in `differs`, and a figure measured at others is low confidence.""" + + def with_settings(self, name, attention=None, cache=None, lora="acme/turbo"): + workflow = definition(name) + pipeline = { + **PIPELINE, + "configuration": { + **PIPELINE["configuration"], + "components": {"transformer": {"attention_backend": attention}}, + "cache": cache, + }, + "loras": [{"model_name": lora}] if lora else [], + } + workflow["steps"][0]["pipeline"] = pipeline + return workflow + + def test_each_is_named(self): + template = self.with_settings("t") + assert inherited_differences(self.with_settings("i"), template) == [] + assert inherited_differences( + self.with_settings("i", attention="native"), template + ) == ["attention backend"] + assert inherited_differences( + self.with_settings("i", cache={"type": "fbc"}), template + ) == ["cache"] + assert inherited_differences(self.with_settings("i", lora=None), template) == [ + "LoRA" + ] + + def test_the_plan_flags_it_low_confidence(self, tmp_path): + inline = self.with_settings("inline", attention="native") + block = inherited( + inline, + [("templates/t", self.with_settings("t"))], + {"templates/t": [run(8 * MINUTE, {"num_frames": 124})] * 3}, + ) + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + estimate = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: block, + )["estimate"] + assert estimate["basis"] == "inherited" + assert estimate["minutes"] == 8.0 + assert estimate["differs"] == ["attention backend"] + assert estimate["low_confidence"] is True + + +class TestTheTemplateAndAnInlineCopyAgree: + """#866 stage C: the catalog path (`scale_curated`) and the inherited path + quote the same request alike - an inline copy of the shipped H3 + reference-to-video and the template itself, at the same arguments.""" + + ARGUMENTS = {"num_inference_steps": 30, "width": 1344, "height": 768} + + def test_the_same_minutes_within_rounding(self, tmp_path): + import copy + import json + from pathlib import Path + + path = ( + Path(__file__).resolve().parents[1] + / "workflows/templates/minimax/reference-to-video.json" + ) + template = json.loads(path.read_text()) + curated = template["cost"][0]["minutes"] + inline = copy.deepcopy(template) + del inline["cost"], inline["cost_drivers"] + + def planned(definition, observed): + candidate = workflow_from_definition( + definition, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + return build_plan( + candidate, + self.ARGUMENTS, + device="cuda", + lookup_sizes=False, + observed=observed, + )["estimate"] + + # The template has no run at these arguments: its curated figure, + # scaled + catalog = planned(template, lambda arguments: None) + assert catalog["basis"] == "derived" + + # Its runs at the defaults took the curated minutes; the inline copy + # inherits them, scaled from the default bucket + block = inherited( + inline, + [("templates/minimax/reference-to-video", template)], + {"templates/minimax/reference-to-video": [run(curated * MINUTE)] * 3}, + self.ARGUMENTS, + ) + block["device"] = None + copied = planned(inline, lambda arguments: block) + assert copied["basis"] == "inherited" + assert "inference steps 30 vs 9" in copied["scaled"] + assert "canvas 1344x768 vs 960x544" in copied["scaled"] + assert abs(copied["minutes"] - catalog["minutes"]) <= 0.1 From e31b224d881a941d4f9d83ec3baa0500a54258fe Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 21:07:30 -0500 Subject: [PATCH 150/204] feat(api): enhance description for 'differs' field to clarify cost drivers with low confidence --- ui/src/lib/generated/api-schema.ts | 2 +- ui/src/lib/generated/openapi.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 19925299..cc20e505 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -3287,7 +3287,7 @@ export interface components { device: string; /** * Differs - * @description With basis `inherited`: what this workflow sets differently from `inherited_from` among offload, quantization and frame count; empty when nothing does. + * @description With basis `inherited`: what this workflow sets differently from `inherited_from` among offload, quantization and frame count; empty when nothing does. Otherwise: the cost drivers with no rule to scale by that moved off the default the figure was measured at, with `low_confidence` true. */ differs?: string[]; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 084e7dd5..1acca290 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -3578,7 +3578,7 @@ "type": "string" }, "differs": { - "description": "With basis `inherited`: what this workflow sets differently from `inherited_from` among offload, quantization and frame count; empty when nothing does.", + "description": "With basis `inherited`: what this workflow sets differently from `inherited_from` among offload, quantization and frame count; empty when nothing does. Otherwise: the cost drivers with no rule to scale by that moved off the default the figure was measured at, with `low_confidence` true.", "items": { "type": "string" }, From d90e2b64faaaf4d73d38a0f919b2dcb21e7ad3a4 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 21:15:03 -0500 Subject: [PATCH 151/204] refactor(plan): #908 - the inherited path scales through scale_curated, one rule for both estimates Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/deps.py | 60 +++++++++++++++++------------------- tests/test_inherited_cost.py | 12 ++++---- 3 files changed, 36 insertions(+), 38 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 8f77f8ab..cfb2bb92 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,7 @@ one to open. | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | -| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache and LoRA set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps, frames, inference steps or canvas pixels than the template run re-price its minutes linearly (pipeline steps x the mean frames, inference steps and pixels per step) with `scaled` naming why and `low_confidence` set (#851, #866) - the same proportions `scale_curated` applies to a catalog figure, so an inline copy and its template quote one request alike. It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | +| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache and LoRA set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | | Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It projects and never refuses (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | diff --git a/dw/server/deps.py b/dw/server/deps.py index 9ba259df..ec96cc7e 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -25,6 +25,7 @@ sets_differently, template_identity, ) +from ..plan import FRAMES_DRIVER, STEPS_DRIVER, scale_curated from .observed_cost import declared_drivers from ..library import ASSETS_KIND, LORAS_KIND, PROMPTS_KIND, library_path from ..workspace import ( @@ -282,29 +283,25 @@ def _canvases(definition, arguments=None): return [(w, h) if w and h else None for w, h in zip(widths, heights)] -def _size(value): - # A canvas counts by its pixels - return value[0] * value[1] if isinstance(value, tuple) else value - - -def _described(values, mean): - if not isinstance(values[0], tuple): - return _number(mean) - if len(set(values)) == 1: - width, height = values[0] - return f"{_number(width)}x{_number(height)}" - return f"{_number(mean)} pixels" +def _means(keys, values): + """{key: the mean over `values`}: one key per value, or a canvas's + `("width", "height")` over its `(width, height)` pairs.""" + if len(keys) == 1: + values = [(value,) for value in values] + return { + key: sum(value[index] for value in values) / len(values) + for index, key in enumerate(keys) + } def _scale_past_template(definition, template, drivers, arguments): """How the workflow's size compares to the template run the bucket was measured on (#821, #851, #866): (factor, reason). The factor is the - ratio of pipeline steps times the ratios of the mean frames, inference - steps and canvas pixels per step - linear, as `scale_curated` re-prices - a catalog figure along the same axes, so the two quote one request - alike - and the reason names each axis that moved. (1, None) when the - workflow is the template's size; (None, reason) when its size cannot be - compared.""" + ratio of pipeline steps times what `scale_curated` makes of the mean + frames, inference steps and canvas per pipeline step - one rule, so the + catalog figure and this one quote a request alike - and the reason names + each axis that moved. (1, None) when the workflow is the template's + size; (None, reason) when its size cannot be compared.""" steps, template_steps = _pipeline_steps(definition), _pipeline_steps(template) # The template as the bucket's drivers set it bucket = {**template, "variables": {**(template.get("variables") or {}), **drivers}} @@ -315,19 +312,21 @@ def _scale_past_template(definition, template, drivers, arguments): f"runs the pipeline in {steps} steps, template measured {template_steps}" ) # Each axis read per pipeline step, the template's variables as the - # bucket sets them; a step that does not set it as a number is left out + # bucket sets them, and averaged for `scale_curated`; a step that does + # not set it as a number is left out + measured, effective = {}, {} for label, keys, mine, theirs in ( ( "frame count", - ("num_frames",), + (FRAMES_DRIVER,), pipeline_frames(definition, arguments), pipeline_frames(bucket), ), ( "inference steps", - ("num_inference_steps",), - pipeline_values(definition, "num_inference_steps", arguments), - pipeline_values(bucket, "num_inference_steps"), + (STEPS_DRIVER,), + pipeline_values(definition, STEPS_DRIVER, arguments), + pipeline_values(bucket, STEPS_DRIVER), ), ( "canvas", @@ -339,16 +338,15 @@ def _scale_past_template(definition, template, drivers, arguments): mine = [value for value in mine if value] theirs = [value for value in theirs if value] if mine and theirs: - mean_mine = sum(map(_size, mine)) / len(mine) - mean_theirs = sum(map(_size, theirs)) / len(theirs) - if mean_mine != mean_theirs: - factor *= mean_mine / mean_theirs - reasons.append( - f"{label} {_described(mine, mean_mine)} vs " - f"{_described(theirs, mean_theirs)} per step" - ) + effective.update(_means(keys, mine)) + measured.update(_means(keys, theirs)) elif any(sets_differently(definition, bucket, key, arguments) for key in keys): return None, f"{label} differs and cannot be compared" + scaled = scale_curated(factor, measured, effective) + if scaled is None: + return None, "its size is not a positive number" + factor, axes = scaled + reasons.extend(axes) return factor, "; ".join(reasons) or None diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 6f9ff9a6..907eaadb 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -219,7 +219,7 @@ def test_a_longer_frame_count_scales_by_frames(self): inline["steps"][0]["run"] = {"num_frames": 248} block = inherited(inline, self.CATALOG, self.HISTORY) assert block["cold_minutes"] == 16.0 - assert "frame count 248 vs 124" in block["scaled"] + assert "frames 248 vs 124" in block["scaled"] assert "frame count" in block["differs"] def test_steps_and_frames_multiply(self): @@ -284,7 +284,7 @@ def test_a_variable_frame_count_scales_from_the_default_bucket(self): inline = definition("inline", num_frames=248) block = inherited(inline, self.CATALOG, self.HISTORY) assert block["cold_minutes"] == 16.0 - assert "frame count 248 vs 124" in block["scaled"] + assert "frames 248 vs 124" in block["scaled"] def test_variable_frames_and_steps_multiply(self): inline = definition("inline", num_frames=248) @@ -349,7 +349,7 @@ class TestScaledByStepsAndCanvas: def test_thirty_steps_scale_by_the_step_ratio(self): block = inherited(sized("inline", steps=30), self.CATALOG, self.HISTORY) assert block["cold_minutes"] == 8.0 * 30 / 9 - assert "inference steps 30 vs 9" in block["scaled"] + assert "steps 30 vs 9" in block["scaled"] def test_a_larger_canvas_scales_by_the_pixel_ratio(self): block = inherited( @@ -368,7 +368,7 @@ def test_the_callers_arguments_scale_it_and_the_axes_multiply(self): assert block["cold_minutes"] == pytest.approx( 8.0 * 30 / 9 * (1344 * 768) / (960 * 544) ) - assert "inference steps" in block["scaled"] and "canvas" in block["scaled"] + assert "steps" in block["scaled"] and "canvas" in block["scaled"] def test_a_bucket_at_those_values_is_not_scaled_again(self): history = { @@ -403,7 +403,7 @@ def test_the_plan_quotes_it_inherited_and_scaled(self, tmp_path): )["estimate"] assert estimate["basis"] == "inherited" assert estimate["minutes"] == round(8.0 * 30 / 9 * 1.9765, 1) - assert "inference steps 30 vs 9" in estimate["scaled"] + assert "steps 30 vs 9" in estimate["scaled"] assert "canvas 1344x768 vs 960x544" in estimate["scaled"] @@ -512,6 +512,6 @@ def planned(definition, observed): block["device"] = None copied = planned(inline, lambda arguments: block) assert copied["basis"] == "inherited" - assert "inference steps 30 vs 9" in copied["scaled"] + assert "steps 30 vs 9" in copied["scaled"] assert "canvas 1344x768 vs 960x544" in copied["scaled"] assert abs(copied["minutes"] - catalog["minutes"]) <= 0.1 From 4a7573a15ee7ad24c05185302fb38f0d0b286141 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 21:26:39 -0500 Subject: [PATCH 152/204] fix(plan): #908 - the inherited comparison resolves variable references at any depth, so a LoRA change is named The H3 loras entry carries its repo as a variable: reference inside the list; _found resolved only a top-level reference, so dropping or swapping the turbo LoRA compared equal. The inherited warning also agrees in number. Co-Authored-By: Claude Opus 5.5 --- dw/server/routes/jobs.py | 3 ++- dw/vram_inheritance.py | 16 ++++++++++++--- tests/test_inherited_cost.py | 40 ++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 4 deletions(-) diff --git a/dw/server/routes/jobs.py b/dw/server/routes/jobs.py index 37e30358..cfb4c277 100644 --- a/dw/server/routes/jobs.py +++ b/dw/server/routes/jobs.py @@ -958,7 +958,8 @@ def _inherited_cost_warnings(estimate): "(same pipeline), not of this workflow" + (f"; {estimate['scaled']}" if estimate.get("scaled") else "") + ( - f"; its {', '.join(differs)} differ from that template, so the real cost may too" + f"; its {', '.join(differs)} {'differs' if len(differs) == 1 else 'differ'}" + " from that template, so the real cost may too" if differs else "" ) diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 6befa03f..8ee60ce8 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -51,14 +51,24 @@ def template_identity(definition): return identities.pop() +def _deep_resolved(value, variables): + """`value` with every `variable:` reference in it resolved, at any depth - + a `loras` list carries its repo as a reference inside each entry (#866).""" + if isinstance(value, dict): + return {name: _deep_resolved(item, variables) for name, item in value.items()} + if isinstance(value, list): + return [_deep_resolved(item, variables) for item in value] + return _resolved(value, variables) + + def _found(node, key, variables): - """Every value `key` carries anywhere under `node`, a `variable:` - reference resolved against `variables`, as sorted JSON text.""" + """Every value `key` carries anywhere under `node`, its `variable:` + references resolved against `variables`, as sorted JSON text.""" values = set() if isinstance(node, dict): for name, value in node.items(): if name == key: - values.add(json.dumps(_resolved(value, variables), sort_keys=True)) + values.add(json.dumps(_deep_resolved(value, variables), sort_keys=True)) values |= _found(value, key, variables) elif isinstance(node, list): for item in node: diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 907eaadb..33dbcd43 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -515,3 +515,43 @@ def planned(definition, observed): assert "steps 30 vs 9" in copied["scaled"] assert "canvas 1344x768 vs 960x544" in copied["scaled"] assert abs(copied["minutes"] - catalog["minutes"]) <= 0.1 + + +class TestTheShippedTemplatesLoraIsCompared: + """#866 stage C (bounce): H3's `loras` entry carries its repo as a + `variable:` reference inside the list, so the comparison resolves + references at any depth - dropping or swapping the turbo LoRA is named.""" + + def template(self): + import json + from pathlib import Path + + path = ( + Path(__file__).resolve().parents[1] + / "workflows/templates/minimax/reference-to-video.json" + ) + return json.loads(path.read_text()) + + def test_the_defaults_differ_in_nothing(self): + template = self.template() + assert inherited_differences(template, template) == [] + + def test_dropping_the_lora_is_named(self): + template = self.template() + arguments = {"lora_model_name": None, "lora_weight_name": None} + assert inherited_differences(template, template, arguments) == ["LoRA"] + + def test_swapping_the_lora_is_named(self): + template = self.template() + arguments = {"lora_model_name": "lightx2v/Some-Other-Lora"} + assert inherited_differences(template, template, arguments) == ["LoRA"] + + +def test_the_warning_agrees_in_number(): + def warned(differs): + return _inherited_cost_warnings( + {"basis": "inherited", "inherited_from": "templates/t", "differs": differs} + )[0] + + assert "its LoRA differs from that template" in warned(["LoRA"]) + assert "its cache, LoRA differ from that template" in warned(["cache", "LoRA"]) From 6fe4917d29a00c8fff4a6579aafd65bd84805649 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 21:38:26 -0500 Subject: [PATCH 153/204] refactor(plan): #908 - the inherited comparison substitutes through replace_variables, the run's own rule Co-Authored-By: Claude Opus 5.5 --- dw/vram_inheritance.py | 55 ++++++++++++++++++++++++++---------- tests/test_inherited_cost.py | 15 ++++++++++ 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 8ee60ce8..941b8d8c 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -30,7 +30,8 @@ import json from .vram_estimate import KEY as ESTIMATE_KEY -from .vram_estimate import _resolved, pipeline_identity, vram_estimate_errors +from .variables import VariableNotFoundError, replace_variables +from .vram_estimate import pipeline_identity, vram_estimate_errors KIND = "vram_projection_inherited" @@ -51,28 +52,52 @@ def template_identity(definition): return identities.pop() -def _deep_resolved(value, variables): - """`value` with every `variable:` reference in it resolved, at any depth - - a `loras` list carries its repo as a reference inside each entry (#866).""" - if isinstance(value, dict): - return {name: _deep_resolved(item, variables) for name, item in value.items()} - if isinstance(value, list): - return [_deep_resolved(item, variables) for item in value] - return _resolved(value, variables) +def _found(node, key, variables): + """Every value `key` carries anywhere under `node`, as sorted JSON text, + after `node`'s `variable:` references are substituted the way a run + substitutes them (`replace_variables`: at any depth, and a key whose + reference resolves to null dropped). A `loras` list carries its repo as + a reference inside each entry (#866).""" + return _carried(_substituted(node, variables), key) -def _found(node, key, variables): - """Every value `key` carries anywhere under `node`, its `variable:` - references resolved against `variables`, as sorted JSON text.""" +def _substituted(node, variables): + """`replace_variables(node, variables)`, except that a reference to a + variable `variables` doesn't declare stays as written instead of failing + the whole node, so the rest of it is still compared.""" + try: + return replace_variables(node, variables) + except VariableNotFoundError: + pass + if isinstance(node, dict): + result = {} + for name, value in node.items(): + try: + result.update(replace_variables({name: value}, variables)) + except VariableNotFoundError: + result[name] = _substituted(value, variables) + return result + if isinstance(node, list): + result = [] + for item in node: + try: + result.extend(replace_variables([item], variables)) + except VariableNotFoundError: + result.append(_substituted(item, variables)) + return result + return node + + +def _carried(node, key): values = set() if isinstance(node, dict): for name, value in node.items(): if name == key: - values.add(json.dumps(_deep_resolved(value, variables), sort_keys=True)) - values |= _found(value, key, variables) + values.add(json.dumps(value, sort_keys=True)) + values |= _carried(value, key) elif isinstance(node, list): for item in node: - values |= _found(item, key, variables) + values |= _carried(item, key) return values diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 33dbcd43..9e56a803 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -555,3 +555,18 @@ def warned(differs): assert "its LoRA differs from that template" in warned(["LoRA"]) assert "its cache, LoRA differ from that template" in warned(["cache", "LoRA"]) + + +def test_the_comparison_substitutes_as_the_run_does(): + """#866 stage C (arch review): `_found` reads a step through + `replace_variables`, so a `variable:` resolving to null drops its key as + the run does, rather than comparing a value the run never sees.""" + from dw.vram_inheritance import _found + + steps = [{"pipeline": {"loras": [{"model_name": "variable:lora"}]}}] + assert _found(steps, "loras", {"lora": None}) == {"[{}]"} + assert _found(steps, "loras", {"lora": "a/b"}) == {'[{"model_name": "a/b"}]'} + # An undeclared name stays as written; the rest of the step still resolves + mixed = [{"run": {"width": "variable:missing", "num_frames": "variable:n"}}] + assert _found(mixed, "num_frames", {"n": 124}) == {"124"} + assert _found(mixed, "width", {"n": 124}) == {'"variable:missing"'} From d1a93db73dce80f620503ebcb8e3628e7cd488db Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:16:19 -0500 Subject: [PATCH 154/204] fix(plan): #915 - an inherited estimate names a reference count other than the template's in differs Co-Authored-By: Claude Sonnet 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/vram_inheritance.py | 39 +++++++++++++++++++++++++++++++++++- tests/test_inherited_cost.py | 22 ++++++++++++++++++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index cfb2bb92..56ca1520 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,7 @@ one to open. | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | -| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache and LoRA set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | +| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | | Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It projects and never refuses (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 941b8d8c..9c855439 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -122,11 +122,48 @@ def inherited_differences(definition, template, arguments=None): short phrases (#797, #866). A figure inherited from `template` is an approximation to the extent these differ. `arguments` (the caller's) override `definition`'s variables.""" - return [ + differs = [ label for label, key in _COST_SETTINGS if sets_differently(definition, template, key, arguments) ] + # A reference is encoded and attended on every pass, so a count other + # than the template's moves the cost (#915); it is named, not scaled + if reference_counts(definition, arguments) != reference_counts(template): + differs.append("reference count") + return differs + + +def reference_counts(definition, arguments=None): + """The set of `references` list lengths the pipeline steps of + `definition` pass, `variable:` references resolved. A step whose + `references` is an `item:` reference counts each entry of its + `for_each` list variable that carries a `references` list (#915).""" + variables = dict(definition.get("variables") or {}) + variables.update(arguments or {}) + counts = set() + for step in definition.get("steps") or []: + if not isinstance(step, dict) or not isinstance(step.get("pipeline"), dict): + continue + for value in _carried_values(_substituted(step, variables), "references"): + if isinstance(value, list): + counts.add(len(value)) + elif isinstance(value, str) and value.startswith("item:"): + each = _substituted(step.get("for_each"), variables) + for entry in each if isinstance(each, list) else []: + listed = ( + entry.get(value[len("item:") :]) + if isinstance(entry, dict) + else None + ) + if isinstance(listed, list): + counts.add(len(listed)) + return counts + + +def _carried_values(node, key): + """Every value `key` carries anywhere under `node`.""" + return [json.loads(text) for text in _carried(node, key)] def sets_differently(definition, template, key, arguments=None): diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 9e56a803..152fe420 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -164,6 +164,28 @@ def test_a_frame_count_the_caller_changed(self): ) assert differs == ["frame count"] + def test_a_reference_count_other_than_the_templates_is_named(self): + def refs(n): + return [{"from_file": f"r{i}"} for i in range(n)] + + def with_refs(value, **top): + d = definition("inline") + step = d["steps"][0] + step["run"]["references"] = value + d.update(top) + return d + + template = with_refs(refs(2)) + assert inherited_differences(with_refs(refs(2)), template) == [] + assert inherited_differences(with_refs(refs(3)), template) == [ + "reference count" + ] + # per-entry references, read off the step's for_each list + entries = [{"references": refs(2)}, {"references": refs(3)}] + each = with_refs("item:references", variables={"shots": entries}) + each["steps"][0]["for_each"] = "variable:shots" + assert "reference count" in inherited_differences(each, template) + class TestThePlanReportsIt: def test_the_estimate_is_inherited_and_the_validate_answer_warns(self, tmp_path): From aeca218f13d54bb1b4ce45512a997cc31538a5c7 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:26:23 -0500 Subject: [PATCH 155/204] fix(plan): #915 - reference counts come from the for_each-expanded steps through reference_count Co-Authored-By: Claude Sonnet 5.5 --- dw/vram_inheritance.py | 43 ++++++++++++++---------------------- tests/test_inherited_cost.py | 19 +++++++++++++++- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 9c855439..045848cf 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -29,7 +29,9 @@ import json +from .for_each import ForEachError, expand_for_each from .vram_estimate import KEY as ESTIMATE_KEY +from .vram_estimate import reference_count from .variables import VariableNotFoundError, replace_variables from .vram_estimate import pipeline_identity, vram_estimate_errors @@ -135,35 +137,22 @@ def inherited_differences(definition, template, arguments=None): def reference_counts(definition, arguments=None): - """The set of `references` list lengths the pipeline steps of - `definition` pass, `variable:` references resolved. A step whose - `references` is an `item:` reference counts each entry of its - `for_each` list variable that carries a `references` list (#915).""" + """The set of reference counts the pipeline steps of `definition` pass, + `variable:` references resolved and `for_each` expanded, so each member + step counts its own entry's `references` (#915). A step's count is + `vram_estimate.reference_count`'s, the one the VRAM projection uses.""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - counts = set() - for step in definition.get("steps") or []: - if not isinstance(step, dict) or not isinstance(step.get("pipeline"), dict): - continue - for value in _carried_values(_substituted(step, variables), "references"): - if isinstance(value, list): - counts.add(len(value)) - elif isinstance(value, str) and value.startswith("item:"): - each = _substituted(step.get("for_each"), variables) - for entry in each if isinstance(each, list) else []: - listed = ( - entry.get(value[len("item:") :]) - if isinstance(entry, dict) - else None - ) - if isinstance(listed, list): - counts.add(len(listed)) - return counts - - -def _carried_values(node, key): - """Every value `key` carries anywhere under `node`.""" - return [json.loads(text) for text in _carried(node, key)] + steps = _substituted(definition.get("steps"), variables) + try: + expanded = expand_for_each({"steps": steps}).get("steps") + except ForEachError: + return set() + return { + reference_count(step["pipeline"].get("arguments")) + for step in expanded or [] + if isinstance(step, dict) and isinstance(step.get("pipeline"), dict) + } def sets_differently(definition, template, key, arguments=None): diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 152fe420..786785a2 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -171,7 +171,10 @@ def refs(n): def with_refs(value, **top): d = definition("inline") step = d["steps"][0] - step["run"]["references"] = value + step["pipeline"] = { + **step["pipeline"], + "arguments": {"references": value}, + } d.update(top) return d @@ -185,6 +188,20 @@ def with_refs(value, **top): each = with_refs("item:references", variables={"shots": entries}) each["steps"][0]["for_each"] = "variable:shots" assert "reference count" in inherited_differences(each, template) + # the optional form with a default, and a null source, count as the + # run counts them + optional = [ + {"references": refs(2) + [{"from_file": None}]}, + {"label": "no references of its own"}, + ] + each = with_refs( + {"item": "references", "default": refs(2)}, + variables={"shots": optional, "num_frames": 124}, + ) + each["steps"][0]["for_each"] = "variable:shots" + assert "reference count" not in inherited_differences(each, template) + each["variables"]["shots"][1]["references"] = refs(3) + assert "reference count" in inherited_differences(each, template) class TestThePlanReportsIt: From 8ff5774c919bdc3c50863df84a57ba45a4c6ddd9 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:32:58 -0500 Subject: [PATCH 156/204] fix(plan): #916 - an inherited comparison expands for_each, so item: references compare by each entry's value Co-Authored-By: Claude Sonnet 5.5 --- dw/vram_inheritance.py | 34 ++++++++++++++++++++++++++-------- tests/test_inherited_cost.py | 21 +++++++++++++++++++++ 2 files changed, 47 insertions(+), 8 deletions(-) diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 045848cf..457b7325 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -143,27 +143,45 @@ def reference_counts(definition, arguments=None): `vram_estimate.reference_count`'s, the one the VRAM projection uses.""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - steps = _substituted(definition.get("steps"), variables) - try: - expanded = expand_for_each({"steps": steps}).get("steps") - except ForEachError: + expanded = _expanded_steps(definition, variables) + if expanded is None: return set() return { reference_count(step["pipeline"].get("arguments")) - for step in expanded or [] + for step in expanded if isinstance(step, dict) and isinstance(step.get("pipeline"), dict) } +def _expanded_steps(definition, variables): + """`definition`'s steps with `variables` substituted and `for_each` + expanded as the run does, so `item:` references carry each entry's own + value; None when the expansion fails.""" + steps = _substituted(definition.get("steps"), variables) + try: + return expand_for_each({"steps": steps}).get("steps") or [] + except ForEachError: + return None + + def sets_differently(definition, template, key, arguments=None): """Whether `definition`'s steps set `key` to other values than `template`'s, `variable:` references resolved (`arguments` override `definition`'s variables).""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - mine = _found(definition.get("steps"), key, variables) - theirs = _found(template.get("steps"), key, template.get("variables") or {}) - return mine != theirs + return _found_expanded(definition, key, variables) != _found_expanded( + template, key, template.get("variables") or {} + ) + + +def _found_expanded(definition, key, variables): + # A for_each step's `item:` references resolve per entry, so the value an + # entry sets is compared, not the reference text (#916) + expanded = _expanded_steps(definition, variables) + if expanded is None: + return _found(definition.get("steps"), key, variables) + return _carried(expanded, key) def pipeline_values(definition, key, arguments=None): diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 786785a2..ef7ce0a6 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -203,6 +203,27 @@ def with_refs(value, **top): each["variables"]["shots"][1]["references"] = refs(3) assert "reference count" in inherited_differences(each, template) + def test_a_for_each_whose_entries_match_the_templates_frames_is_not_named(self): + def with_frames(value, **top): + d = definition("inline") + step = d["steps"][0] + step["pipeline"] = { + **step["pipeline"], + "arguments": {"num_frames": value}, + } + d.update(top) + return d + + template = with_frames(124) + shots = [{"num_frames": 124}, {"num_frames": 124}] + each = with_frames( + "item:num_frames", variables={"shots": shots, "num_frames": 124} + ) + each["steps"][0]["for_each"] = "variable:shots" + assert inherited_differences(each, template) == [] + each["variables"]["shots"][1]["num_frames"] = 209 + assert inherited_differences(each, template) == ["frame count"] + class TestThePlanReportsIt: def test_the_estimate_is_inherited_and_the_validate_answer_warns(self, tmp_path): From 3fbe4ba60a79ac59796a5a36b6702a1c4cf00d62 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:43:52 -0500 Subject: [PATCH 157/204] fix(server): #917 - a job a kept asset names survives its workspace's deletion Co-Authored-By: Claude Sonnet 5.5 --- dw/server/job_history.py | 27 ++++++++++++++++++++++---- dw/server/routes/assets.py | 7 +++++++ tests/test_forget_workspace_history.py | 13 +++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 682bb4b6..3ca80c22 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -157,6 +157,10 @@ def __init__(self, db_path): # row before the column if "rerun_of" not in columns: connection.execute("ALTER TABLE jobs ADD COLUMN rerun_of TEXT") + # 1 once a kept asset's provenance names the job (#917), so a + # workspace delete leaves the row for get_job_workflow to read + if "kept" not in columns: + connection.execute("ALTER TABLE jobs ADD COLUMN kept INTEGER DEFAULT 0") if "device_ordinal" not in columns or "device_card" not in columns: connection.execute( "UPDATE jobs SET" @@ -421,15 +425,30 @@ def orphan_workflow_history(self, workspace, workflow_name): (workspace, workflow_name), ) + def mark_kept(self, job_id): + """Record that a kept asset's provenance names this job (#917), so + `forget_workspace` leaves its row.""" + with self._lock, self._connect() as connection: + connection.execute("UPDATE jobs SET kept = 1 WHERE id = ?", (job_id,)) + def forget_workspace(self, workspace): - """Delete every row recorded in this workspace (#913). A deleted + """Delete the rows recorded in this workspace (#913). A deleted workspace takes its history with it, so a new workspace reusing the - name starts with none. Returns the number of rows removed.""" + name starts with none. A job a kept asset names (#917) survives, its + workspace renamed to one no workspace can be called, so + `get_job_workflow` still resolves the asset's provenance. Returns + the number of rows removed.""" with self._lock, self._connect() as connection: cursor = connection.execute( - "DELETE FROM jobs WHERE workspace = ?", (workspace,) + "DELETE FROM jobs WHERE workspace = ? AND COALESCE(kept, 0) = 0", + (workspace,), + ) + removed = cursor.rowcount + connection.execute( + "UPDATE jobs SET workspace = ? WHERE workspace = ?", + (f"{workspace} (deleted)", workspace), ) - return cursor.rowcount + return removed def events_for(self, job_id): """A finished job's persisted event tail. [] for a job recorded diff --git a/dw/server/routes/assets.py b/dw/server/routes/assets.py index 786ac1aa..3de01c7e 100644 --- a/dw/server/routes/assets.py +++ b/dw/server/routes/assets.py @@ -514,6 +514,13 @@ def keep_output_as_asset( if job else None ) + if job: + # The asset now names this job, so deleting the workspace must not + # take its row (#917) + try: + request.app.state.job_manager.history.mark_kept(job["id"]) + except Exception as e: + logger.warning(f"Could not mark job {job['id']} kept: {e}") record_kept_shots( os.path.dirname(destination), os.path.basename(destination), diff --git a/tests/test_forget_workspace_history.py b/tests/test_forget_workspace_history.py index a9bfb940..2708e1cc 100644 --- a/tests/test_forget_workspace_history.py +++ b/tests/test_forget_workspace_history.py @@ -31,3 +31,16 @@ def test_forgetting_a_workspace_leaves_a_same_named_one_empty(tmp_path): assert history.recent_summaries(workspace="sweep") == [] assert [r["id"] for r in history.recent_summaries(workspace="other")] == ["c"] assert history.get("a") is None + + +def test_a_job_a_kept_asset_names_survives_its_workspace(tmp_path): + history = JobHistory(str(tmp_path / "jobs.sqlite")) + history.record(_job("a", "sweep")) + history.record(_job("b", "sweep")) + history.mark_kept("a") + + assert history.forget_workspace("sweep") == 1 + + assert history.recent_summaries(workspace="sweep") == [] + assert history.get("b") is None + assert history.get("a")["status"] == "succeeded" From 9038f0b72ab349b593010b1337962adf681721cd Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:51:47 -0500 Subject: [PATCH 158/204] fix(server): #918 - a kept asset's job gone from history reports forgotten and the hint stops promising get_job_workflow Co-Authored-By: Claude Sonnet 5.5 --- dw/server/routes/media.py | 7 +++++++ dw_mcp/catalog.py | 11 ++++++++++- tests/test_mcp_catalog.py | 19 +++++++++++++++++++ tests/test_notes_api.py | 10 ++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index b495a6e7..15ceece8 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -165,6 +165,13 @@ def gallery_metadata( kept = kept_provenance(path) if kept: job = kept.get("job") + if job and job.get("id") and ( + request.app.state.job_manager.get(job["id"]) is None + ): + # Pruned from history (the cap, or a deleted workspace): + # the id is a record, not something get_job_workflow can + # open, so the stale status is not reported as current (#918) + job = {**job, "status": "forgotten"} run_id = kept.get("run_id") or "" version = kept.get("version") inputs = kept.get("inputs") or None diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index 3cde6485..cce7ae5a 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -338,7 +338,16 @@ def get_gallery_metadata( job = body.get("job") hints = [] if body.get("metadata") is None: - if job and job.get("id"): + if job and job.get("status") == "forgotten": + hints.append( + "metadata is null because this file has none embedded, and " + f'its job {job.get("id")} is no longer in the server\'s ' + "history (status 'forgotten'), so get_job_workflow would " + "answer 'Unknown job'. What remains is this file's own " + "`note` and `inputs` when present, or get_job_workflow of " + "an older job that is still listed." + ) + elif job and job.get("id"): hints.append( "metadata is null because this file has none embedded - a " "pipeline's image carries it, but video, audio and an image " diff --git a/tests/test_mcp_catalog.py b/tests/test_mcp_catalog.py index f5931c14..424c47b0 100644 --- a/tests/test_mcp_catalog.py +++ b/tests/test_mcp_catalog.py @@ -450,3 +450,22 @@ def handler(request): catalog.list_gallery(client, role=["final", "intermediate"]) assert seen["roles"] == ["final", "intermediate"] + + +def test_gallery_metadata_does_not_promise_a_forgotten_job(): + """#918: a kept asset's job pruned from history reports 'forgotten', and + the hint must not point at get_job_workflow, which would say Unknown job.""" + body = { + "name": "asset:qa-cast/ep182-episode.mp4", + "source": "asset", + "metadata": None, + "job": {"id": "ad1adcb65d6d", "status": "forgotten"}, + } + client, _ = scripted( + {("GET", "/api/gallery/asset:qa-cast/ep182-episode.mp4/metadata"): (200, body)} + ) + + result = catalog.get_gallery_metadata(client, "asset:qa-cast/ep182-episode.mp4") + + assert "no longer in the server" in result["next"] + assert 'get_job_workflow(job_id="ad1adcb65d6d")' not in result["next"] diff --git a/tests/test_notes_api.py b/tests/test_notes_api.py index 45ccd329..289aecd3 100644 --- a/tests/test_notes_api.py +++ b/tests/test_notes_api.py @@ -600,3 +600,13 @@ def test_an_upload_under_a_fresh_name_leaves_other_provenance( assert up.status_code == 201 with open(manifest) as handle: assert handle.read() == '{"steps": [' + + +def test_a_kept_asset_whose_job_left_history_reports_it_forgotten( + client, output_root, make_run +): + """#918: the id stays, but the stale status is not reported as current.""" + run = run_name(make_run("wf", ["x.png"]), output_root) + _keep_with_provenance(client, run, "uploads/x.png") + meta = client.get("/api/gallery/asset:uploads/x.png/metadata").json() + assert meta["job"] == {"id": "abc", "status": "forgotten"} From 0788c56ff16367eee1d74adc99d8164634898158 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Wed, 14 Oct 2026 22:52:16 -0500 Subject: [PATCH 159/204] style: #918 - ruff format Co-Authored-By: Claude Sonnet 5.5 --- dw/server/routes/media.py | 6 ++++-- dw_mcp/catalog.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index 15ceece8..7e551557 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -165,8 +165,10 @@ def gallery_metadata( kept = kept_provenance(path) if kept: job = kept.get("job") - if job and job.get("id") and ( - request.app.state.job_manager.get(job["id"]) is None + if ( + job + and job.get("id") + and (request.app.state.job_manager.get(job["id"]) is None) ): # Pruned from history (the cap, or a deleted workspace): # the id is a record, not something get_job_workflow can diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index cce7ae5a..f7f5c8ec 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -341,7 +341,7 @@ def get_gallery_metadata( if job and job.get("status") == "forgotten": hints.append( "metadata is null because this file has none embedded, and " - f'its job {job.get("id")} is no longer in the server\'s ' + f"its job {job.get('id')} is no longer in the server's " "history (status 'forgotten'), so get_job_workflow would " "answer 'Unknown job'. What remains is this file's own " "`note` and `inputs` when present, or get_job_workflow of " From 8cda9f0fb52298624aba2837a206c023e3e705fa Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 00:51:19 -0500 Subject: [PATCH 160/204] feat(check-script): #919 - a line's start_frame picks one occurrence of a repeated shot name Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 2 +- docs/WORKFLOW_GUIDE.md | 2 +- dw/script_lines.py | 130 ++++++++++++++++++++++++++++++------- dw/tasks/script_check.py | 8 ++- tests/test_script_check.py | 42 +++++++++++- 5 files changed, 157 insertions(+), 27 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 6bcf24af..18c2eea6 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3007,7 +3007,7 @@ The step's `result.content_type` must be `"application/json"`; read the answer w | Argument | Required | Description | | -------- | -------- | ----------- | | `audio` | Yes | The take - a path, `asset:`/`output:` reference, or an earlier step's audio or video (its soundtrack is taken) | -| `lines` | Yes | The expected lines in order: a list of strings or `{text, shot}` objects, `shot` naming the take's shot the line belongs in. Markup is stripped (below). `[]` means no speech is expected | +| `lines` | Yes | The expected lines in order: a list of strings or `{text, shot}` objects, `shot` naming the take's shot the line belongs in; a shot name the map holds more than once (a cut joined from episodes sharing names) also needs `start_frame` on the line, `{text, shot, start_frame}`, and findings call it `name#start_frame`. Markup is stripped (below). `[]` means no speech is expected | | `shots` | No | The take's shot map, `[{name, start_frame, num_frames, start_sample, num_samples}]`. Default: the take's own `.shots` (an earlier step's video), else the run manifest or `keep_output` sidecar beside an `asset:`/`output:` file - resolved as the assessment probes resolve them (`assess.resolve_shots`). The argument wins over both | | `similarity` | No | Least similarity, 0..1, of a line's heard words to its expected words before it is a `line_mismatch` (default `0.85`) | | `model_name` | No | HuggingFace ID of a Whisper-class ASR model (default `openai/whisper-base`) | diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index 3ca67629..e872b4ec 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -769,7 +769,7 @@ by the source, and `validate_workflow` names the entries to add or drop. arguments={"input_audio": "output:", "lines": [...]})` first (free; it takes an audio file or a video's muxed soundtrack directly, and `lines` is the script in order, H3 markup allowed, `{text, shot}` to name - a line's shot, `[]` for a take that should be silent), then + a line's shot (plus `start_frame` when the cut repeats that shot's name), `[]` for a take that should be silent), then `run_workflow(..., acknowledged_cost={"fingerprint": ..., "minutes": ..., "downloads": [...]})` bound to that plan with `wait_seconds=60`, then `get_output_text` on the result, and `delete_output(job_id=...)` the diff --git a/dw/script_lines.py b/dw/script_lines.py index 8534b144..9354b570 100644 --- a/dw/script_lines.py +++ b/dw/script_lines.py @@ -128,9 +128,11 @@ def parse_lines(lines): """The expected lines as [{text, tokens, shot}], or ValueError naming `lines`. - Each entry is a string or {text, shot}; markup is stripped + Each entry is a string or {text, shot, start_frame}; markup is stripped (strip_markup) and `shot`, the name of the shot the line is spoken in, - is None when not given. `[]` means no speech is expected. + is None when not given. `start_frame` picks one occurrence of a shot + name the take's map holds more than once (#919); None when not given. + `[]` means no speech is expected. """ if isinstance(lines, str) or not isinstance(lines, (list, tuple)): raise ValueError( @@ -139,16 +141,29 @@ def parse_lines(lines): ) parsed = [] for index, entry in enumerate(lines): - shot = None + shot = start_frame = None if isinstance(entry, dict): - extra = sorted(set(entry) - {"text", "shot"}) + extra = sorted(set(entry) - {"text", "shot", "start_frame"}) if extra: raise ValueError( f"{COMMAND} 'lines'[{index}] has unknown keys {extra};" - " a line object takes only 'text' and 'shot'" + " a line object takes only 'text', 'shot' and 'start_frame'" ) text = entry.get("text") shot = entry.get("shot") + start_frame = entry.get("start_frame") + if start_frame is not None and ( + isinstance(start_frame, bool) or not isinstance(start_frame, int) + ): + raise ValueError( + f"{COMMAND} 'lines'[{index}] 'start_frame' must be an" + f" integer frame number, got {start_frame!r}" + ) + if start_frame is not None and shot is None: + raise ValueError( + f"{COMMAND} 'lines'[{index}] gives 'start_frame' without" + " the 'shot' it picks an occurrence of" + ) if shot is not None and (not isinstance(shot, str) or not shot.strip()): raise ValueError( f"{COMMAND} 'lines'[{index}] 'shot' must be a shot's name" @@ -162,7 +177,14 @@ def parse_lines(lines): " {text, shot}" ) stripped, tokens = strip_markup(text) - parsed.append({"text": stripped, "tokens": tokens, "shot": shot}) + parsed.append( + { + "text": stripped, + "tokens": tokens, + "shot": shot, + "start_frame": start_frame, + } + ) return parsed @@ -197,16 +219,46 @@ def matching_shot_names(shot, names): return list(dict.fromkeys(name for name in names if shot in member_keys(name))) +def _repeated(records): + """The (file, name) pairs `records` holds more than once.""" + return { + (file, name) + for file, dups in (duplicate_shot_names(records) or {}).items() + for name in dups + } + + +def occurrence_name(record): + """A repeated shot's name made unique by where it sits: `shot@bus#1248`.""" + return f"{record['name']}#{record.get('start_frame')}" + + +def disambiguated(records): + """`records` with each name the map holds more than once (within a file) + renamed by `occurrence_name`, so a span or finding can tell the + occurrences apart (#919). The same list when nothing repeats.""" + repeated = _repeated(records) + if not repeated: + return records + return [ + dict(record, name=occurrence_name(record)) + if (record.get("file"), record.get("name")) in repeated + else record + for record in records + ] + + def shot_names_error(parsed_lines, records, source="argument"): - """The refusal for a line naming a shot `records` does not hold once - - unknown, a short `@` name more than one shot carries, or a name - `shots.duplicate_shot_names` finds repeated - or None. The message lists - the shots it does hold.""" + """The refusal for a line naming a shot `records` does not hold - unknown, + a short `@` name more than one shot carries, or a name the map + holds more than once and the line gives no `start_frame` for - or None. + The message lists the shots it does hold.""" names = [record.get("name") for record in records] - known = ", ".join(repr(name) for name in names) - duplicated = { - name for dups in (duplicate_shot_names(records) or {}).values() for name in dups - } + repeated = {name for _, name in _repeated(records)} + known = ", ".join(repr(name) for name in dict.fromkeys(names)) + nameable = ", ".join( + repr(name) for name in dict.fromkeys(names) if name not in repeated + ) for index, line in enumerate(parsed_lines): shot = line["shot"] if shot is None: @@ -223,22 +275,56 @@ def shot_names_error(parsed_lines, records, source="argument"): f" than one of the take's shots ({source}) carries - name one" f" in full: {', '.join(repr(name) for name in matches)}" ) - if matches[0] in duplicated: - return ( - f"{COMMAND} 'lines'[{index}] names shot {shot!r}, which the" - f" take's shot map ({source}) holds more than once - a line can" - f" only name a shot the map holds once: {known}" + if matches[0] in repeated: + frames = [ + record.get("start_frame") + for record in records + if record.get("name") == matches[0] + ] + start_frame = line.get("start_frame") + if start_frame is None: + return ( + f"{COMMAND} 'lines'[{index}] names shot {shot!r}, which the" + f" take's shot map ({source}) holds more than once - add" + f" 'start_frame' to say which occurrence: one of {frames}." + f" Shots named without it: {nameable or 'none'}" + ) + if start_frame not in frames: + return ( + f"{COMMAND} 'lines'[{index}] 'start_frame' {start_frame}" + f" matches no occurrence of shot {shot!r} ({source}) - its" + f" start_frames are {frames}" + ) + elif line.get("start_frame") is not None: + frame = next( + record.get("start_frame") + for record in records + if record.get("name") == matches[0] ) + if frame != line["start_frame"]: + return ( + f"{COMMAND} 'lines'[{index}] 'start_frame'" + f" {line['start_frame']} does not match shot {shot!r}" + f" ({source}), which starts at frame {frame}" + ) return None def with_full_shot_names(parsed_lines, records): - """`parsed_lines` with each line's short shot name replaced by the full - name `records` holds. Call after `shot_names_error` found nothing.""" + """`parsed_lines` with each line's short shot name replaced by the name + `disambiguated(records)` holds: the full name, or for a repeated shot + `occurrence_name`. Call after `shot_names_error` found nothing.""" names = [record.get("name") for record in records] + repeated = {name for _, name in _repeated(records)} resolved = [] for line in parsed_lines: shot = line["shot"] matches = matching_shot_names(shot, names) if shot is not None else [] - resolved.append(dict(line, shot=matches[0]) if matches else line) + if not matches: + resolved.append(line) + continue + full = matches[0] + if full in repeated: + full = f"{full}#{line.get('start_frame')}" + resolved.append(dict(line, shot=full)) return resolved diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index aecb5328..904ca986 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -52,6 +52,7 @@ COMMAND, REPEAT_MAX_PERIOD, REPEAT_RUN_MIN, + disambiguated, looped_texts, normalize_words, parse_lines, @@ -623,7 +624,11 @@ def check_script( lines: The expected lines in order - a list of strings or {text, shot}, 'shot' naming the take's shot the line belongs in - its full recorded name, or the short '@name' of exactly one shot - ('cold_open' for 'shot@cold_open'). + ('cold_open' for 'shot@cold_open'). A name the shot map holds + more than once (a cut joined from episodes sharing shot names) + needs 'start_frame' on the line to pick the occurrence: + {text, shot, start_frame}; findings and 'shots' then call it + 'name#start_frame'. H3 markup ([English] ..., , , [unclear], (S1)) is stripped, and a markup word heard spoken is 'tag_spoken'. [] means no speech is expected. @@ -659,6 +664,7 @@ def check_script( if refusal: raise ValueError(refusal) parsed_lines = with_full_shot_names(parsed_lines, records) + records = disambiguated(records) waveform, rate = waveform_and_rate(audio, sample_rate, COMMAND) from .audio_transcription import transcribe_audio diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 18b3ea2c..08902c29 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -15,6 +15,7 @@ from dw.task_problems import script_lines_errors from dw.script_lines import ( normalize_words, + disambiguated, parse_lines, parse_shots, shot_names_error, @@ -118,8 +119,13 @@ def test_accepts_strings_and_text_objects(self): self.assertEqual( parsed, [ - {"text": "Hello", "tokens": ["cutoff"], "shot": None}, - {"text": "World", "tokens": ["s1"], "shot": None}, + { + "text": "Hello", + "tokens": ["cutoff"], + "shot": None, + "start_frame": None, + }, + {"text": "World", "tokens": ["s1"], "shot": None, "start_frame": None}, ], ) @@ -831,6 +837,38 @@ def test_duplicate_names_a_line_names_are_refused(self): records = [{"name": "A"}, {"name": "A"}] message = shot_names_error(parse_lines([{"text": "x", "shot": "A"}]), records) self.assertIn("more than once", message) + self.assertIn("start_frame", message) + + def test_start_frame_picks_one_occurrence_of_a_repeated_shot(self): + records = [ + {"name": "A", "start_frame": 0}, + {"name": "B", "start_frame": 10}, + {"name": "A", "start_frame": 20}, + ] + lines = parse_lines( + [{"text": "x", "shot": "A", "start_frame": 20}, {"text": "y", "shot": "B"}] + ) + self.assertIsNone(shot_names_error(lines, records)) + resolved = with_full_shot_names(lines, records) + self.assertEqual([line["shot"] for line in resolved], ["A#20", "B"]) + self.assertEqual( + [record["name"] for record in disambiguated(records)], ["A#0", "B", "A#20"] + ) + + def test_a_start_frame_no_occurrence_has_is_refused(self): + records = [{"name": "A", "start_frame": 0}, {"name": "A", "start_frame": 20}] + lines = parse_lines([{"text": "x", "shot": "A", "start_frame": 5}]) + self.assertIn("matches no occurrence", shot_names_error(lines, records)) + + def test_the_refusal_lists_only_the_nameable_shots(self): + records = [ + {"name": "A", "start_frame": 0}, + {"name": "A", "start_frame": 20}, + {"name": "B", "start_frame": 40}, + ] + message = shot_names_error(parse_lines([{"text": "x", "shot": "A"}]), records) + self.assertIn("Shots named without it: 'B'", message) + self.assertNotIn("'A', 'B'", message) def test_a_name_repeated_only_across_files_is_not_a_duplicate(self): # shots.duplicate_shot_names groups by file (#644 arch review). From b0ab82944895e84dc0be135c71344acbe424e9e1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 02:13:39 -0500 Subject: [PATCH 161/204] feat(check-script): #920 - unscripted_speech flags unmatched words inside a shot that has a scripted line Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 3 ++- dw/tasks/script_check.py | 50 ++++++++++++++++++++++++++++++++++++-- tests/test_script_check.py | 22 ++++++++++++++++- 3 files changed, 71 insertions(+), 4 deletions(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 18c2eea6..3e9372f0 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3023,8 +3023,9 @@ Markup stripped from an expected line: `[Language] ...`, ``, | `line_clipped_at_end` | A line's last heard word ends inside the final `CLIP_TAIL_SECONDS` of its shot (when the line has one) or of the file, and that tail is still above `GUARD_FLOOR_DBFS` | The tail's level in dBFS / `GUARD_FLOOR_DBFS` | `{line, seconds, word, shot}` - `shot` when the line has one | | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | +| `unscripted_speech` | Shots are known and a line names the shot; words were heard inside it that align to no line (a line spoken twice, an ad-lib) | Number of unmatched words in the shot / 0 | `{line: null, seconds, end, word, shot, repeats_line}` - the first such word; `repeats_line` is true when those words repeat or contain one of the shot's lines | -Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` is skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). +Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` and `unscripted_speech` are skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). The thresholds are module constants in `dw/tasks/script_check.py`, deliberately not in `RULES` (`dw/assessment_rules.py`): every rule there must fire on a synthetic file that holds no script. The answer echoes them under `thresholds`. diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index 904ca986..db034044 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -97,7 +97,7 @@ LINE_RULES = ("line_mismatch", "tag_spoken", "line_clipped_at_end") SILENT_RULES = ("speech_where_silent",) -SHOT_RULES = ("speech_in_silent_shot",) +SHOT_RULES = ("speech_in_silent_shot", "unscripted_speech") def _rule(name, threshold, says): @@ -139,6 +139,12 @@ def mismatch_rule(similarity): "no line names this shot, and this many words were heard in it above the" " dead-air floor and outside a repetition loop", ) +UNSCRIPTED_SPEECH = _rule( + "unscripted_speech", + 0, + "this many words were heard inside a shot that has a scripted line, matched to" + " no line - a line spoken twice (see 'repeats_line'), or an ad-lib; listen", +) def shot_spans(records, sample_rate, fps, duration): @@ -475,6 +481,9 @@ def check( if no_shot is None: rules_applied.extend(SHOT_RULES) findings.extend(_silent_shot_findings(spans, parsed_lines, heard, claimed)) + findings.extend( + _unscripted_findings(spans, parsed_lines, expected, heard, unassigned) + ) else: rules_skipped.extend({"rule": rule, "reason": no_shot} for rule in SHOT_RULES) @@ -581,6 +590,42 @@ def _silent_shot_findings(spans, parsed_lines, heard, claimed): return findings +def _unscripted_findings(spans, parsed_lines, expected, heard, unassigned): + """unscripted_speech: heard words align left to no line that sit inside a + shot a line names. `repeats_line` is set when the shot's words, in order, + contain one of its lines' words again.""" + findings = [] + for span in spans: + lines = [ + expected[i] + for i, line in enumerate(parsed_lines) + if line["shot"] == span["name"] + ] + if not lines: + continue + inside = [ + heard[j] + for j, _ in unassigned + if _heard_shot(spans, heard[j]) == span["name"] + ] + if not inside: + continue + at = _at(None, inside[0], shot=span["name"]) + words = [word["word"] for word in inside] + at["end"] = _round(inside[-1]["end"]) + at["repeats_line"] = any( + line and _contains(words, line) for line in lines + ) or any(line and _contains(line, words) for line in lines) + findings.append(finding(UNSCRIPTED_SPEECH, len(inside), at)) + return findings + + +def _contains(words, part): + """Whether `part` occurs in `words` as a contiguous run.""" + size = len(part) + return any(words[i : i + size] == part for i in range(len(words) - size + 1)) + + def _heard_entry(word): return { "text": word["text"], @@ -645,7 +690,8 @@ def check_script( Returns: A JSON document: 'findings' (line_mismatch, tag_spoken, - line_clipped_at_end, speech_where_silent, speech_in_silent_shot), + line_clipped_at_end, speech_where_silent, speech_in_silent_shot, + unscripted_speech), 'lines' (expected, heard, similarity, start, end, shot), 'shots' (name, start, end in seconds, or null), 'shots_source', 'discarded' (guarded words with their level and reason), 'unmatched' (heard words aligned to no line), 'transcript', diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 08902c29..7127fbdb 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -379,7 +379,7 @@ def test_exact_lines_have_no_findings(self): self.assertEqual(answer["rules_applied"], list(script_check.LINE_RULES)) self.assertEqual( [s["rule"] for s in answer["rules_skipped"]], - ["speech_where_silent", "speech_in_silent_shot"], + ["speech_where_silent", "speech_in_silent_shot", "unscripted_speech"], ) self.assertTrue(all(s["reason"] for s in answer["rules_skipped"])) @@ -635,6 +635,26 @@ def test_dropped_line_in_a_known_shot_is_at_the_shots_start(self): self.assertEqual(found[0]["at"]["shot"], "B") +class TestUnscriptedSpeech(unittest.TestCase): + def test_a_line_spoken_twice_in_its_shot_is_a_finding(self): + chunks = words("alpha bravo charlie", 0.0) + words("alpha bravo charlie", 1.0) + answer = shot_run([{"text": "alpha bravo charlie", "shot": "A"}], chunks) + (found,) = rules(answer, "unscripted_speech") + self.assertEqual(found["value"], 3) + self.assertEqual(found["at"]["shot"], "A") + self.assertTrue(found["at"]["repeats_line"]) + + def test_an_ad_lib_in_a_scripted_shot_is_not_a_repeat(self): + chunks = words("alpha bravo", 0.0) + words("zulu yankee", 1.0) + answer = shot_run([{"text": "alpha bravo", "shot": "A"}], chunks) + (found,) = rules(answer, "unscripted_speech") + self.assertFalse(found["at"]["repeats_line"]) + + def test_clean_take_has_none(self): + answer = shot_run([{"text": "alpha bravo", "shot": "A"}], words("alpha bravo")) + self.assertEqual(rules(answer, "unscripted_speech"), []) + + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From 91732f4608bfd19c3f1496a6895ee3a53e8edb7e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 03:10:07 -0500 Subject: [PATCH 162/204] fix(server): #921 - a task-only workflow with no cost drivers prices from every run, whatever inputs it was given Co-Authored-By: Claude Sonnet 5.5 --- dw/server/observed_cost.py | 8 +++++++- tests/test_observed_cost.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dw/server/observed_cost.py b/dw/server/observed_cost.py index 11ff5f46..cd399206 100644 --- a/dw/server/observed_cost.py +++ b/dw/server/observed_cost.py @@ -78,7 +78,8 @@ def _bucket_key(definition, arguments): With declared drivers: the effective value of each, which is what the run passed or else the stored default. Without: only a run that overrode nothing that matters is comparable, and since nothing is declared to - matter, that means a run that overrode nothing at all. + matter, that means a run that overrode nothing at all - except a + task-only workflow, whose runs are all comparable (#921). """ variables = definition.get("variables") or {} drivers = declared_drivers(definition) @@ -87,6 +88,11 @@ def _bucket_key(definition, arguments): (name, _comparable(arguments.get(name, variables.get(name)))) for name in sorted(drivers) ) + if _is_task_only(definition): + # No pipeline loads, so a task's inputs (an audio file, a list of + # lines) are data rather than a size; with nothing declared to + # scale the cost, every run is comparable (#921) + return () if any(key in variables for key in arguments): return None return () diff --git a/tests/test_observed_cost.py b/tests/test_observed_cost.py index 83f88ee4..9aac5e31 100644 --- a/tests/test_observed_cost.py +++ b/tests/test_observed_cost.py @@ -239,6 +239,21 @@ def test_a_task_only_workflow_reports_cold_even_though_had_load_is_false(self): assert observed["cold_minutes"] == 12.0 and observed["cold_runs"] == 24 assert "warm_minutes" not in observed + def test_a_task_only_workflow_without_drivers_counts_runs_with_overrides(self): + """#921: templates/check-script is called with its own audio and + lines every time; none of that makes runs incomparable.""" + definition = workflow(None, input_audio="asset:take.wav", lines=None) + definition["steps"] = [{"name": "check", "task": "check_script"}] + args = {"input_audio": "asset:other.mp4", "lines": ["a", "b"]} + + observed = observed_for( + definition, + [run(5, args), run(6, {"lines": ["x"]}), run(5, {})], + arguments=args, + ) + + assert observed["runs"] == 3 and observed["cold_runs"] == 3 + def test_a_pipeline_step_keeps_the_ordinary_had_load_split(self): """A workflow that does load a model is unaffected - the split still comes from what the run actually logged.""" From 4a79cb350fbe48aa6f0d9ace2a3aadcf6065f249 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 03:59:07 -0500 Subject: [PATCH 163/204] feat(mcp): #922 - get_gallery_metadata measures an image (kind, width, height, mode) and include_workflow=false omits the embedded workflow Co-Authored-By: Claude Sonnet 5.5 --- dw/server/api_models.py | 3 ++- dw/server/routes/media.py | 48 +++++++++++++++++++++++++++++++++----- dw_mcp/catalog.py | 12 ++++++++-- dw_mcp/tools_catalog.py | 7 ++++-- tests/test_mcp_catalog.py | 15 ++++++++++++ tests/test_mcp_server.py | 5 +++- tests/test_server.py | 49 +++++++++++++++++++++++++++++++++++++-- 7 files changed, 125 insertions(+), 14 deletions(-) diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 1bc25e0a..d9b2baf9 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -993,7 +993,8 @@ class GalleryMetadata(ApiModel): run_id: str version: int | None media: dict[str, Any] | None = Field( - description="The probe of an audio or video file; null for an image." + description="The probe of an audio or video file, or an image's kind, width, height " + "and mode; null for another file." ) findings: list[LevelFinding] role: Literal["final", "intermediate"] | None = sometimes( diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index 7e551557..164ea5a0 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -111,6 +111,7 @@ def gallery_metadata( name: str, envelope: bool = False, interval_seconds: float = 1.0, + include_workflow: bool = True, ws: Workspace = Depends(selected_workspace), ): """Generation metadata embedded in a saved image ('workflow' inside @@ -133,6 +134,12 @@ def gallery_metadata( refused (422) with the interval that fits; the default width is never refused for length (#823). + An image's `media` is its measured `{kind, width, height, mode}`, + read from the file header (#922) - not the width/height arguments + the pipeline was asked for. `include_workflow=false` drops the + embedded `metadata.workflow` (the whole definition, often 20 KB) and + leaves the rest of `metadata` with `workflow_omitted: true`. + `name` may also be an 'asset:' reference, and then it is the input asset of that name that is described rather than an output (#127). The numbers here - duration, frame count, fps, sample rate - are @@ -193,16 +200,27 @@ def gallery_metadata( inputs = run_inputs(split[0]) notes = _note_fields(read_notes(split[0]), split[1]) metadata = read_embedded_metadata(path) + if metadata is not None and not include_workflow and "workflow" in metadata: + metadata = { + **{k: v for k, v in metadata.items() if k != "workflow"}, + "workflow_omitted": True, + } extension = os.path.splitext(path)[1].lower() + kind = MEDIA_KINDS.get(extension) try: - media = ( - probe_media(path, envelope=envelope, interval_seconds=interval_seconds) - if MEDIA_KINDS.get(extension) in ("audio", "video") - else None - ) + if kind in ("audio", "video"): + media = probe_media( + path, envelope=envelope, interval_seconds=interval_seconds + ) + elif kind == "image": + media = _image_media(path) + else: + media = None except ValueError as error: raise HTTPException(status_code=422, detail=str(error)) from error - if media is not None and source == "output": + if kind == "image": + pass # no shots in a still + elif media is not None and source == "output": # Where each shot of a joined video sits, as the run that wrote # it recorded (dw/shots.py) - null for a file not joined from shots media["shots"] = recorded_shots(ws.outputs, name) @@ -226,6 +244,24 @@ def gallery_metadata( } +def _image_media(path): + """An image's `media` block - kind, pixel size and mode - read from + the header alone, so no pixel is decoded; None for a file Pillow cannot + read (#922).""" + from PIL import Image + + try: + with Image.open(path) as image: + return { + "kind": "image", + "width": image.width, + "height": image.height, + "mode": image.mode, + } + except (OSError, ValueError, SyntaxError): + return None + + def _note_fields(notes, name): """The metadata route's note keys for file `name` of a run, each only when set.""" diff --git a/dw_mcp/catalog.py b/dw_mcp/catalog.py index f7f5c8ec..d2551fc5 100644 --- a/dw_mcp/catalog.py +++ b/dw_mcp/catalog.py @@ -296,7 +296,12 @@ def list_gallery( def get_gallery_metadata( - client, name, envelope=False, workspace=None, interval_seconds=None + client, + name, + envelope=False, + workspace=None, + interval_seconds=None, + include_workflow=True, ): """Metadata embedded in a saved file: the full workflow that made it - the exact workflow, arguments and seed, so a result can be reproduced @@ -326,9 +331,12 @@ def get_gallery_metadata( measure, and the default answer has to stay small. `interval_seconds` (0.05 to 3600, default 1.0) narrows the bins; a finer-than-default width that would need over 2400 of them is refused with the interval that fits.""" - params = {"envelope": "true"} if envelope else None + params = {"envelope": "true"} if envelope else {} if envelope and interval_seconds is not None: params["interval_seconds"] = interval_seconds + if not include_workflow: + params["include_workflow"] = "false" + params = params or None body = client.get_json( api_path("api", "gallery", name, "metadata"), params=params, diff --git a/dw_mcp/tools_catalog.py b/dw_mcp/tools_catalog.py index d42b4a6d..f323a6da 100644 --- a/dw_mcp/tools_catalog.py +++ b/dw_mcp/tools_catalog.py @@ -307,6 +307,7 @@ def get_gallery_metadata( envelope: bool = False, workspace: str | None = None, interval_seconds: float | None = None, + include_workflow: bool = True, ) -> dict: """Get the metadata embedded in a generated file: the exact workflow, arguments and seed that produced it, so a result can be @@ -314,8 +315,9 @@ def get_gallery_metadata( and video `metadata` is null and `next` names `get_job_workflow(job_id)` when known. `media` carries duration, sample rate, channels, fps, size and level - how an agent that - cannot listen checks a deliverable; `media.shots` places a joined - video's shots. + cannot listen checks a deliverable, or an image's measured + `width`/`height`; `media.shots` places a joined video's shots. + `include_workflow=false` omits the large embedded workflow. `envelope=true` adds that level second by second (`media.envelope.rms_dbfs` / `peak_dbfs`), which says *where* in a track something is: a shot's last frame, a seam's hole, where a @@ -339,6 +341,7 @@ def get_gallery_metadata( envelope=envelope, workspace=workspace, interval_seconds=interval_seconds, + include_workflow=include_workflow, ) def list_guides( diff --git a/tests/test_mcp_catalog.py b/tests/test_mcp_catalog.py index 424c47b0..1b70f77f 100644 --- a/tests/test_mcp_catalog.py +++ b/tests/test_mcp_catalog.py @@ -469,3 +469,18 @@ def test_gallery_metadata_does_not_promise_a_forgotten_job(): assert "no longer in the server" in result["next"] assert 'get_job_workflow(job_id="ad1adcb65d6d")' not in result["next"] + + +def test_gallery_metadata_asks_the_server_to_leave_out_the_workflow(): + seen = [] + + def handler(request): + seen.append(dict(request.url.params)) + return httpx.Response(200, json={"name": "a.png", "metadata": None}) + + client = DwClient(transport=httpx.MockTransport(handler)) + + catalog.get_gallery_metadata(client, "a.png", include_workflow=False) + catalog.get_gallery_metadata(client, "a.png") + + assert seen == [{"include_workflow": "false"}, {}] diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py index 4b0857bd..a609dd8e 100644 --- a/tests/test_mcp_server.py +++ b/tests/test_mcp_server.py @@ -1700,7 +1700,10 @@ def test_the_stated_tool_count_is_the_registered_one(): # 2026-10-14 (#886, #867 stage 2): get_job_events' description names the # `cache_stats` kind (+15 description) - measured at 15_004.25 (9_485 / # 5_009 / 510.25); the ceiling moves to 15_005 with 0.75 of headroom. -SURFACE_BUDGET = 15_005 +# 2026-10-15 (#922): get_gallery_metadata gains `include_workflow` and says an +# image's `media` is its measured size (+46) - measured at 15_051.25; the +# ceiling moves to 15_052 with 0.75 of headroom. +SURFACE_BUDGET = 15_052 @pytest.mark.asyncio diff --git a/tests/test_server.py b/tests/test_server.py index 0bd653ee..ba71e331 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -1616,6 +1616,41 @@ def test_gallery_lists_media_and_reads_metadata(server, tmp_path): assert read_embedded_metadata(str(outputs / "meta.jpg"))["step_name"] == "gen" +def test_gallery_metadata_measures_an_image_and_can_omit_the_workflow(server, tmp_path): + """#922: an image's `media` was null, so its size was only in the + arguments the pipeline was asked for; `include_workflow=false` drops + the embedded definition.""" + from PIL import Image + from dw.writers import embed_image_metadata + + with server(success_script) as client: + outputs = tmp_path / "outputs" + metadata = {"step_name": "gen", "workflow": valid_workflow("from_image")} + embed_image_metadata( + Image.new("RGB", (6, 4)), + str(outputs / "m.0-0.0.jpg"), + "image/jpeg", + metadata, + ) + full = client.get("/api/gallery/m.0-0.0.jpg/metadata").json() + assert full["media"] == { + "kind": "image", + "width": 6, + "height": 4, + "mode": "RGB", + } + assert full["findings"] == [] + assert full["metadata"]["workflow"]["id"] == "from_image" + + lean = client.get( + "/api/gallery/m.0-0.0.jpg/metadata?include_workflow=false" + ).json() + assert "workflow" not in lean["metadata"] + assert lean["metadata"]["workflow_omitted"] is True + assert lean["metadata"]["step_name"] == "gen" + assert lean["media"]["width"] == 6 + + def test_gallery_metadata_describes_audio_and_video(server, tmp_path): """A generated mp3 answered metadata: null and nothing else, so every duration and level check was ffprobe by hand. The route now says what @@ -1640,7 +1675,12 @@ def test_gallery_metadata_describes_audio_and_video(server, tmp_path): assert shot["media"]["frame_count"] == 12 still = client.get("/api/gallery/still-gen.0-0.0.png/metadata").json() - assert still["media"] is None + assert still["media"] == { + "kind": "image", + "width": 4, + "height": 4, + "mode": "RGB", + } def test_gallery_metadata_accepts_an_output_reference(server, tmp_path): @@ -6068,7 +6108,12 @@ def test_gallery_metadata_reads_an_asset_reference(asset_server, tmp_path): # an image asset carries no media block, the same as an image output still = client.get("/api/gallery/asset:still.png/metadata").json() assert still["source"] == "asset" - assert still["media"] is None + assert still["media"] == { + "kind": "image", + "width": 4, + "height": 4, + "mode": "RGB", + } # and an output is unchanged - it still says where it came from write_wav(tmp_path / "outputs" / "score-gen.0-0.0.wav", seconds=1.0) From cd65e203c44b965e054d3deccedf741a12c76733 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 04:18:44 -0500 Subject: [PATCH 164/204] feat(check-script): #923 - speech_not_transcribed flags audible sound in a gap inside a matched line or a stretched word, where the ASR dropped speech Co-Authored-By: Claude Sonnet 5.5 --- docs/TASKS.md | 2 ++ dw/tasks/script_check.py | 65 +++++++++++++++++++++++++++++++++++++- tests/test_script_check.py | 36 +++++++++++++++++++++ 3 files changed, 102 insertions(+), 1 deletion(-) diff --git a/docs/TASKS.md b/docs/TASKS.md index 3e9372f0..974c80d3 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -3024,6 +3024,7 @@ Markup stripped from an expected line: `[Language] ...`, ``, | `speech_where_silent` | `lines` is `[]` and words were heard above the floor and outside a repetition loop | Number of words heard / 0 | `{line: null, seconds, word}` - the first; `shot` when shots are known | | `speech_in_silent_shot` | Shots are known, at least one line names a shot, and words were heard above the floor and outside a repetition loop in a shot no line names. A word is in the shot its midpoint falls in; one a line in another shot matched (a word straddling the cut) is that line's, not counted | Number of words heard in the shot / 0 | `{line: null, seconds, word, shot}` - the shot's first counted word, `seconds` no earlier than the shot's start | | `unscripted_speech` | Shots are known and a line names the shot; words were heard inside it that align to no line (a line spoken twice, an ad-lib) | Number of unmatched words in the shot / 0 | `{line: null, seconds, end, word, shot, repeats_line}` - the first such word; `repeats_line` is true when those words repeat or contain one of the shot's lines | +| `speech_not_transcribed` | Between two words of one matched line there is a gap of `UNHEARD_MIN_SECONDS` or more, or one of its words lasts `UNHEARD_MIN_SECONDS` + 2 `UNHEARD_MARGIN_SECONDS` or more, and the interior (trimmed by `UNHEARD_MARGIN_SECONDS` each side) holds a window above `UNHEARD_FLOOR_DBFS`: audible speech the ASR left out of its text, which the line's timing absorbed (#923) | The gap's length in seconds / `UNHEARD_FLOOR_DBFS` | `{line, seconds, end, shot}` - `shot` when the line has one | Every finding is severity `warn`; `at.shot` is set whenever the shot is known. A rule that does not apply to the call is listed in `rules_skipped` with its reason: `speech_in_silent_shot` and `unscripted_speech` are skipped when no shots are known (`line_clipped_at_end` then looks at the file's end only), when the shots carry neither a sample span nor a frame span the take's frame rate can place, or when no line names a shot (no shot is known to be meant silent). @@ -3033,6 +3034,7 @@ The thresholds are module constants in `dw/tasks/script_check.py`, deliberately | -------- | ----- | --------------------- | | `DEFAULT_SIMILARITY` | 0.85 | A line below this similarity is a `line_mismatch` (the `similarity` argument overrides it) | | `GUARD_FLOOR_DBFS` | -65 dBFS (= `DEAD_AIR_FLOOR_DBFS`, `shot_dead_air`'s floor) | A heard word whose loudest window is at or below it is discarded as unheard; also the level a clipped tail must exceed | +| `UNHEARD_MIN_SECONDS` / `UNHEARD_MARGIN_SECONDS` / `UNHEARD_FLOOR_DBFS` | 1.0 s / 0.25 s / -40 dBFS | A gap in a line's words (or a stretched word) this long, with a window above the floor in its margin-trimmed interior, is a `speech_not_transcribed` | | `GUARD_WINDOW_SECONDS` | 0.05 s (= `DEAD_AIR_WINDOW`, `shot_dead_air`'s window) | The window a heard word is measured in - its loudest one, so a span overhanging a pause does not average a real word away | | `CLIP_TAIL_SECONDS` | 0.25 s | The final stretch of the file a line's last word must end in to be `line_clipped_at_end` | | `REPEAT_RUN_MIN` | 6 | Heard words repeating the same unit this many times over, back to back, are a repetition loop: every chunk holding one is discarded (`reason: "repetition"`), however loud | diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index db034044..a605abe5 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -83,6 +83,17 @@ # GUARD_FLOOR_DBFS - still voiced when the file stops, not a word that # finished into a quiet tail CLIP_TAIL_SECONDS = 0.25 +# Audible speech the transcript has no words for (#923): Whisper can drop a +# passage from its text on a longer input (the ad-lib heard in a short file +# is gone), and the word alignment hands that time to a neighbouring word, +# which stretches. A gap between a line's words, +# or a word, lasting UNHEARD_MIN_SECONDS or more, whose interior - trimmed by +# UNHEARD_MARGIN_SECONDS each side - holds a window above UNHEARD_FLOOR_DBFS, +# is speech the words do not account for. The floor is far above the +# dead-air floor so room tone and a pause's tail do not trip it +UNHEARD_MIN_SECONDS = 1.0 +UNHEARD_MARGIN_SECONDS = 0.25 +UNHEARD_FLOOR_DBFS = -40.0 # REPEAT_RUN_MIN and REPEAT_MAX_PERIOD, the repetition-loop guard, live in # dw/script_lines.py beside the detector, which transcribe_audio shares @@ -91,11 +102,19 @@ "guard_floor_dbfs": GUARD_FLOOR_DBFS, "guard_window_seconds": GUARD_WINDOW_SECONDS, "clip_tail_seconds": CLIP_TAIL_SECONDS, + "unheard_min_seconds": UNHEARD_MIN_SECONDS, + "unheard_margin_seconds": UNHEARD_MARGIN_SECONDS, + "unheard_floor_dbfs": UNHEARD_FLOOR_DBFS, "repeat_run_min": REPEAT_RUN_MIN, "repeat_max_period": REPEAT_MAX_PERIOD, } -LINE_RULES = ("line_mismatch", "tag_spoken", "line_clipped_at_end") +LINE_RULES = ( + "line_mismatch", + "tag_spoken", + "line_clipped_at_end", + "speech_not_transcribed", +) SILENT_RULES = ("speech_where_silent",) SHOT_RULES = ("speech_in_silent_shot", "unscripted_speech") @@ -120,6 +139,13 @@ def mismatch_rule(similarity): "a markup word from this line was heard spoken aloud - the model read the" " tag as dialogue", ) +SPEECH_NOT_TRANSCRIBED = _rule( + "speech_not_transcribed", + UNHEARD_FLOOR_DBFS, + "audible sound lasting this many seconds sits between the line's words, or" + " inside one stretched word, with no word for it - the transcript left speech" + " out and the line's timing absorbed it; listen", +) SPEECH_WHERE_SILENT = _rule( "speech_where_silent", 0, @@ -468,6 +494,9 @@ def check( findings.extend( _clipped_findings(mono, sample_rate, tails, index, own, shot) ) + findings.extend( + _untranscribed_findings(mono, sample_rate, index, own, shot) + ) previous_end = end findings.extend(_unassigned_tags(heard, unassigned, parsed_lines, expected, spans)) @@ -534,6 +563,40 @@ def _silent_take(heard, discarded, spans, shots_source): } +def _untranscribed_findings(mono, sample_rate, index, own, shot): + """speech_not_transcribed: audible sound in a gap the line's words leave + - between two of them - or inside one word stretched over + UNHEARD_MIN_SECONDS (#923). Not the gap before a line's first word: a pause + between shots may hold music or ambience above the floor.""" + intervals = [(a["end"], b["start"]) for a, b in zip(own, own[1:])] + findings = [] + for start, end in intervals: + if end - start >= UNHEARD_MIN_SECONDS: + findings.append(_untranscribed(mono, sample_rate, index, start, end, shot)) + for word in own: + # A word is sound itself: only its interior past the margins is a gap + if ( + word["end"] - word["start"] + >= UNHEARD_MIN_SECONDS + 2 * UNHEARD_MARGIN_SECONDS + ): + findings.append( + _untranscribed( + mono, sample_rate, index, word["start"], word["end"], shot + ) + ) + return [found for found in findings if found is not None] + + +def _untranscribed(mono, sample_rate, index, start, end, shot): + lo, hi = start + UNHEARD_MARGIN_SECONDS, end - UNHEARD_MARGIN_SECONDS + level = word_level(mono, sample_rate, lo, hi) + if level is None or level <= UNHEARD_FLOOR_DBFS: + return None + at = _at(index, seconds=start, shot=shot) + at["end"] = _round(end) + return finding(SPEECH_NOT_TRANSCRIBED, _round(end - start, 2), at) + + def _clipped_findings(mono, sample_rate, tails, index, own, shot): """line_clipped_at_end for one heard line, as a list of zero or one finding: its last word ends in a voiced tail of its shot or the file.""" diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 7127fbdb..bc650a15 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -655,6 +655,42 @@ def test_clean_take_has_none(self): self.assertEqual(rules(answer, "unscripted_speech"), []) +class TestSpeechNotTranscribed(unittest.TestCase): + LINES = [{"text": "alpha bravo charlie delta", "shot": "A"}] + + def test_sound_in_a_gap_between_a_lines_words_is_a_finding(self): + # The ASR dropped an ad-lib between "bravo" and "charlie" + chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) + answer = shot_run(self.LINES, chunks, mono=voiced(6.0)) + (found,) = rules(answer, "speech_not_transcribed") + self.assertEqual(found["at"]["line"], 0) + self.assertEqual(found["at"]["shot"], "A") + self.assertAlmostEqual(found["at"]["seconds"], 0.9, places=2) + self.assertAlmostEqual(found["at"]["end"], 4.0, places=2) + + def test_a_stretched_word_over_sound_is_a_finding(self): + chunks = [ + {"start": 0.0, "end": 0.4, "text": " alpha"}, + {"start": 0.5, "end": 3.0, "text": " bravo"}, + {"start": 3.0, "end": 3.4, "text": " charlie"}, + {"start": 3.5, "end": 3.9, "text": " delta"}, + ] + answer = shot_run(self.LINES, chunks, mono=voiced(6.0)) + (found,) = rules(answer, "speech_not_transcribed") + self.assertEqual(found["at"]["seconds"], 0.5) + + def test_a_quiet_gap_is_not(self): + chunks = words("alpha bravo", 0.0) + words("charlie delta", 4.0) + mono = numpy.concatenate([voiced(0.9), silent(3.1), voiced(2.0)]) + answer = shot_run(self.LINES, chunks, mono=mono) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + + def test_a_short_gap_is_not(self): + chunks = words("alpha bravo", 0.0) + words("charlie delta", 1.5) + answer = shot_run(self.LINES, chunks, mono=voiced(3.0)) + self.assertEqual(rules(answer, "speech_not_transcribed"), []) + + class TestSpeechInSilentShot(unittest.TestCase): LINES = [{"text": "alpha bravo", "shot": "A"}] From d18903171c7c5e99bbfb99d35dbf9230b0b5a79c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 04:19:57 -0500 Subject: [PATCH 165/204] docs(check-script): #923 - name speech_not_transcribed in the template and docstring Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 2 +- workflows/templates/check-script.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index a605abe5..c718edea 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -753,7 +753,7 @@ def check_script( Returns: A JSON document: 'findings' (line_mismatch, tag_spoken, - line_clipped_at_end, speech_where_silent, speech_in_silent_shot, + line_clipped_at_end, speech_not_transcribed, speech_where_silent, speech_in_silent_shot, unscripted_speech), 'lines' (expected, heard, similarity, start, end, shot), 'shots' (name, start, end in seconds, or null), 'shots_source', 'discarded' (guarded words with their diff --git a/workflows/templates/check-script.json b/workflows/templates/check-script.json index fe8aa891..e577b9fd 100644 --- a/workflows/templates/check-script.json +++ b/workflows/templates/check-script.json @@ -1,6 +1,6 @@ { "id": "CheckScript", - "description": "Checks that a take speaks its script, line by line, and saves the answer as JSON for get_output_text. It runs 'check_script' on 'input_audio' (an audio file, or a video's muxed soundtrack) against 'lines', the expected lines in order: strings, or {text, shot} objects naming the shot a line belongs in. H3 markup in a line is stripped, and a markup word heard spoken is 'tag_spoken'; [] means no speech is expected. 'shots' is optional: by default the take's own recorded shots are used, from its run manifest or keep_output sidecar. Findings are line_mismatch, tag_spoken, line_clipped_at_end, speech_where_silent and speech_in_silent_shot, each a place to listen before re-rolling. The defaults come from a two-model measurement (#609, job af5de241ca83): whisper-large-v3-turbo scored correct lines 0.71 to 1.00 and wrong or missing ones 0.00, so 'similarity' 0.6 separates them. Write numbers as words, as turbo hears them.", + "description": "Checks that a take speaks its script, line by line, and saves the answer as JSON for get_output_text. It runs 'check_script' on 'input_audio' (an audio file, or a video's muxed soundtrack) against 'lines', the expected lines in order: strings, or {text, shot} objects naming the shot a line belongs in. H3 markup in a line is stripped, and a markup word heard spoken is 'tag_spoken'; [] means no speech is expected. 'shots' is optional: by default the take's own recorded shots are used, from its run manifest or keep_output sidecar. Findings are line_mismatch, tag_spoken, line_clipped_at_end, speech_not_transcribed, speech_where_silent and speech_in_silent_shot, each a place to listen before re-rolling. The defaults come from a two-model measurement (#609, job af5de241ca83): whisper-large-v3-turbo scored correct lines 0.71 to 1.00 and wrong or missing ones 0.00, so 'similarity' 0.6 separates them. Write numbers as words, as turbo hears them.", "variables": { "input_audio": "asset:take.wav", "lines": null, From 40e0d8918e8769f7ac25a24e82e0e2167c07b254 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 05:06:02 -0500 Subject: [PATCH 166/204] feat(timing): #924 - warn when a loaded pipeline's first denoise pass dwarfs the steady one Co-Authored-By: Claude Sonnet 5.5 --- dw/pipeline_processors/timing.py | 37 +++++++++++++++++++++++++++ tests/test_step_timing.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/dw/pipeline_processors/timing.py b/dw/pipeline_processors/timing.py index a459e346..b3dcfcf1 100644 --- a/dw/pipeline_processors/timing.py +++ b/dw/pipeline_processors/timing.py @@ -38,6 +38,10 @@ _active_timer = contextvars.ContextVar("dw_step_timer", default=None) +# A first pass this much slower than a steady one, on a loaded pipeline, is +# worth a warning (#924) +_SLOW_FIRST_PASS_EXTRA_S = 30.0 + def _synced_clock(): """The monotonic clock, read after the device has finished its queued work. @@ -137,6 +141,39 @@ def pass_ended(self): segment = self._current_segment() if segment is not None: segment.pass_ends.append(self._clock()) + self._warn_on_slow_first_pass(segment) + + def _warn_on_slow_first_pass(self, segment): + """Say so once when a cached pipeline's first pass dwarfs the steady one. + + The pipeline is loaded, so the minutes are not a load. A later entry + that runs the cached pipeline at a larger shape (more frames) pays a + one-off warm-up on its first pass - re-allocation, offload buffers, + kernels meeting a new shape - and without a word the consumer sees only + a `phase_stall` (#924). Judged at the second pass interval, when a + steady pass is known.""" + passes = segment.pass_ends + if ( + len(passes) != 3 + or self._load_cached is not True + or segment.denoise_started is None + ): + return + first = passes[0] - segment.denoise_started + steady = passes[2] - passes[1] + if first - steady < _SLOW_FIRST_PASS_EXTRA_S or first < steady * 3: + return + emit_warning( + f"First denoise pass took {first:.0f}s against {steady:.0f}s for " + "the passes after it, with the pipeline already loaded: a one-off " + "warm-up (typically a cached pipeline meeting a larger shape than " + "it last ran, such as more frames). Later passes run at the " + "steady rate.", + kind="first_pass_slow", + first_pass_s=round(first, 1), + pass_s=round(steady, 1), + step=self.step, + ) def call_ended(self): segment = self._current_segment() diff --git a/tests/test_step_timing.py b/tests/test_step_timing.py index 4a9f99a5..59881636 100644 --- a/tests/test_step_timing.py +++ b/tests/test_step_timing.py @@ -534,3 +534,47 @@ def test_default_clock_does_not_sync_on_cpu(monkeypatch): monkeypatch.setattr(timing_module, "get_device_type", lambda: "cpu") monkeypatch.setattr(torch.mps, "synchronize", lambda: pytest.fail("synced on cpu")) StepTimer().load_started() + + +def _passes(clock, timer, cached, first, steady): + timer.load_cached(cached) + marked( + clock, + timer, + (0.0, "call_started"), + (1.0, "denoise_started"), + (1.0 + first, "pass_ended"), + (1.0 + first + steady, "pass_ended"), + (1.0 + first + 2 * steady, "pass_ended"), + (2.0 + first + 2 * steady, "call_ended"), + ) + + +def test_a_cached_pipeline_whose_first_pass_dwarfs_the_rest_warns_once(monkeypatch): + """#924: 202 s then 37 s/pass on a loaded pipeline is said, not silent.""" + warnings = [] + monkeypatch.setattr( + timing_module, "emit_warning", lambda message, **data: warnings.append(data) + ) + clock, timer = new_timer() + _passes(clock, timer, True, 202.0, 37.0) + assert len(warnings) == 1 + assert warnings[0]["kind"] == "first_pass_slow" + assert warnings[0]["first_pass_s"] == 202.0 + assert warnings[0]["pass_s"] == 37.0 + + +@pytest.mark.parametrize( + "cached, first, steady", + [(True, 29.0, 28.4), (True, 60.0, 37.0), (False, 202.0, 37.0)], +) +def test_no_warning_for_a_normal_first_pass_or_a_fresh_load( + monkeypatch, cached, first, steady +): + warnings = [] + monkeypatch.setattr( + timing_module, "emit_warning", lambda message, **data: warnings.append(data) + ) + clock, timer = new_timer() + _passes(clock, timer, cached, first, steady) + assert warnings == [] From 47d47d3e70d56daf456011d3f56b23311fd7056c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 06:19:49 -0500 Subject: [PATCH 167/204] fix(check-script): #925 - transcribe a level-normalized copy so the verdict does not depend on input loudness Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 20 +++++++++++++++++++- tests/test_script_check.py | 15 +++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index c718edea..ee5f67b4 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -173,6 +173,24 @@ def mismatch_rule(similarity): ) +# Whisper drops soft speech from a quiet input, so the same shot got different +# verdicts by level (#925). The transcriber hears the take scaled so its loud +# end sits here; the level checks keep reading the original waveform. +TRANSCRIBE_PEAK = 0.5 +TRANSCRIBE_MAX_GAIN = 100.0 # 40 dB: a near-silent take is not boosted to noise + + +def level_for_transcription(waveform): + """The waveform scaled so its 99.9th-percentile magnitude is TRANSCRIBE_PEAK.""" + import numpy + + loud = float(numpy.percentile(numpy.abs(waveform), 99.9)) if waveform.size else 0.0 + if loud <= 0.0: + return waveform + gain = min(TRANSCRIBE_PEAK / loud, TRANSCRIBE_MAX_GAIN) + return (waveform * gain).astype(numpy.float32) + + def shot_spans(records, sample_rate, fps, duration): """Each shot record as {name, start, end} in seconds on the take, or None when one cannot be placed. Placed by `assess.sample_span`, the rule the @@ -779,7 +797,7 @@ def check_script( from .audio_transcription import transcribe_audio transcript = transcribe_audio( - waveform, + level_for_transcription(waveform), device=device, sample_rate=rate, model_name=model_name, diff --git a/tests/test_script_check.py b/tests/test_script_check.py index bc650a15..22c43f97 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -1127,3 +1127,18 @@ def test_check_script_defaults_to_the_one_whisper_model(): default = inspect.signature(script_check.check_script).parameters["model_name"] assert default.default is audio_transcription.WHISPER_DEFAULT_MODEL + + +def test_transcription_input_is_level_independent(): + import numpy + + from dw.tasks.script_check import level_for_transcription + + rng = numpy.random.default_rng(0) + loud = rng.uniform(-0.5, 0.5, (1, 48000)).astype(numpy.float32) + quiet = loud * 0.1 + a, b = level_for_transcription(loud), level_for_transcription(quiet) + assert numpy.allclose(a, b, atol=1e-4) + assert float(numpy.abs(a).max()) <= 0.51 + silent = numpy.zeros((1, 100), dtype=numpy.float32) + assert level_for_transcription(silent) is silent From c2cb4d6b937a520031aafe8aaed116126aca0606 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 06:24:32 -0500 Subject: [PATCH 168/204] fix(inherited-cost): #908 - a reference the copy doesn't declare compares as null, so a copy without its null-default variables matches its template Co-Authored-By: Claude Opus 5.5 --- docs/SERVER.md | 9 +++++---- dw/vram_inheritance.py | 23 +++++++++++++++++++---- tests/test_inherited_cost.py | 16 ++++++++++++++++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/docs/SERVER.md b/docs/SERVER.md index 72e1a25a..c78d9435 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -456,10 +456,11 @@ The editor's forms come from these; they are just as usable from scripts: `component_type` + `model_name` + `workflow`, as the VRAM ceiling is inherited - with `inherited_from` naming it and `differs` listing what the workflow sets differently among offload, quantization, frame count, - attention backend, cache and LoRA, with `low_confidence` true when it lists - any; more pipeline steps, frames, inference steps or canvas pixels than the - template's run re-price it in proportion, as the catalog figure is scaled, - with `scaled` naming each axis; validate warns when it does, and never + attention backend, cache and LoRA; more pipeline steps, frames, inference + steps or canvas pixels than the template's run re-price it in proportion, + as the catalog figure is scaled, with `scaled` naming each axis; an + inherited figure has `low_confidence` true when `differs` lists anything + or the figure was scaled; validate warns when it does, and never refuses); a composed child's cost is added to a curated figure and `partial` is true when a child has none - an `observed` figure already measured the whole run, children diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 457b7325..7562581e 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -32,7 +32,11 @@ from .for_each import ForEachError, expand_for_each from .vram_estimate import KEY as ESTIMATE_KEY from .vram_estimate import reference_count -from .variables import VariableNotFoundError, replace_variables +from .variables import ( + VariableNotFoundError, + replace_variables, + undeclared_variable_references, +) from .vram_estimate import pipeline_identity, vram_estimate_errors KIND = "vram_projection_inherited" @@ -168,11 +172,22 @@ def sets_differently(definition, template, key, arguments=None): """Whether `definition`'s steps set `key` to other values than `template`'s, `variable:` references resolved (`arguments` override `definition`'s variables).""" + return _found_expanded( + definition, key, _compared_variables(definition, arguments) + ) != _found_expanded(template, key, _compared_variables(template)) + + +def _compared_variables(definition, arguments=None): + """`definition`'s variables, `arguments` over them, with every name its + references use but it doesn't declare set to null. A copy that leaves out + a variable whose default is null (`lora_alpha`) then drops that key as + the template it was copied from does, instead of comparing the reference + text against nothing (#866).""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) - return _found_expanded(definition, key, variables) != _found_expanded( - template, key, template.get("variables") or {} - ) + for _, name in undeclared_variable_references(definition): + variables.setdefault(name, None) + return variables def _found_expanded(definition, key, variables): diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index ef7ce0a6..e0a5dd6f 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -606,6 +606,22 @@ def test_swapping_the_lora_is_named(self): arguments = {"lora_model_name": "lightx2v/Some-Other-Lora"} assert inherited_differences(template, template, arguments) == ["LoRA"] + def test_a_copy_without_its_null_variables_differs_in_nothing(self): + """#866 stage C (C-F485 bounce): an inline copy that leaves out the + variables whose default is null (`lora_alpha`) compares as if they + resolved to null, the way the template's own declarations do.""" + template = self.template() + copy = dict(template) + copy["variables"] = { + name: value + for name, value in template["variables"].items() + if value is not None + } + assert "lora_alpha" not in copy["variables"] + assert inherited_differences(copy, template) == [] + arguments = {"lora_model_name": "lightx2v/Some-Other-Lora"} + assert inherited_differences(copy, template, arguments) == ["LoRA"] + def test_the_warning_agrees_in_number(): def warned(differs): From d0fb3f4596e091849afeb6e9df51ffe16653b5f1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 06:38:57 -0500 Subject: [PATCH 169/204] fix(check-script): #925 - level the transcription input to 0.9 (the 0.5 target gained only 1.7 dB and still lost the soft 'Uh,'), and log the applied gain Co-Authored-By: Claude Sonnet 5.5 --- dw/tasks/script_check.py | 23 +++++++++++++++++------ tests/test_script_check.py | 10 +++++++--- 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/dw/tasks/script_check.py b/dw/tasks/script_check.py index ee5f67b4..c3c12dfd 100644 --- a/dw/tasks/script_check.py +++ b/dw/tasks/script_check.py @@ -42,11 +42,13 @@ import difflib import logging +import math from types import SimpleNamespace from .. import dsp from ..assessment_rules import DEAD_AIR_FLOOR_DBFS, finding from ..task_domains import check_arguments +from ..events import emit_log from ..locations import is_http_url from ..script_lines import ( COMMAND, @@ -175,20 +177,24 @@ def mismatch_rule(similarity): # Whisper drops soft speech from a quiet input, so the same shot got different # verdicts by level (#925). The transcriber hears the take scaled so its loud -# end sits here; the level checks keep reading the original waveform. -TRANSCRIBE_PEAK = 0.5 +# end sits near full scale - a take only 2 dB under that still lost a soft +# "Uh," that +6 dB brought back; the level checks keep reading the original +# waveform. +TRANSCRIBE_PEAK = 0.9 TRANSCRIBE_MAX_GAIN = 100.0 # 40 dB: a near-silent take is not boosted to noise def level_for_transcription(waveform): - """The waveform scaled so its 99.9th-percentile magnitude is TRANSCRIBE_PEAK.""" + """(waveform, gain) with the 99.9th-percentile magnitude scaled to + TRANSCRIBE_PEAK (spikes past full scale clipped); gain is 1.0 for silence.""" import numpy loud = float(numpy.percentile(numpy.abs(waveform), 99.9)) if waveform.size else 0.0 if loud <= 0.0: - return waveform + return waveform, 1.0 gain = min(TRANSCRIBE_PEAK / loud, TRANSCRIBE_MAX_GAIN) - return (waveform * gain).astype(numpy.float32) + leveled = numpy.clip(waveform * gain, -1.0, 1.0).astype(numpy.float32) + return leveled, gain def shot_spans(records, sample_rate, fps, duration): @@ -796,8 +802,13 @@ def check_script( from .audio_transcription import transcribe_audio + leveled, gain = level_for_transcription(waveform) + emit_log( + f"check_script: transcribing at {20 * math.log10(gain):+.1f} dB gain", + transcription_gain_db=round(20 * math.log10(gain), 2), + ) transcript = transcribe_audio( - level_for_transcription(waveform), + leveled, device=device, sample_rate=rate, model_name=model_name, diff --git a/tests/test_script_check.py b/tests/test_script_check.py index 22c43f97..f653f0a6 100644 --- a/tests/test_script_check.py +++ b/tests/test_script_check.py @@ -1137,8 +1137,12 @@ def test_transcription_input_is_level_independent(): rng = numpy.random.default_rng(0) loud = rng.uniform(-0.5, 0.5, (1, 48000)).astype(numpy.float32) quiet = loud * 0.1 - a, b = level_for_transcription(loud), level_for_transcription(quiet) + (a, gain_a), (b, gain_b) = ( + level_for_transcription(loud), + level_for_transcription(quiet), + ) + assert gain_b > gain_a assert numpy.allclose(a, b, atol=1e-4) - assert float(numpy.abs(a).max()) <= 0.51 + assert float(numpy.abs(a).max()) <= 1.0 silent = numpy.zeros((1, 100), dtype=numpy.float32) - assert level_for_transcription(silent) is silent + assert level_for_transcription(silent) == (silent, 1.0) From e243ec7eb1a65fce24e1c61592423c0b34c64402 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 06:44:29 -0500 Subject: [PATCH 170/204] fix(inherited-cost): #908 - compare cost settings by value, not spelling, so a client's JSON round trip (lora_scale 1.0 -> 1) doesn't name the LoRA The integral-float canonicalization the fingerprint already used moves from dw/plan.py to dw/variables.py (same_number) and is shared by the inherited comparison. Co-Authored-By: Claude Opus 5.5 --- dw/plan.py | 17 ++------------- dw/variables.py | 14 ++++++++++++ dw/vram_inheritance.py | 5 ++++- tests/test_inherited_cost.py | 42 ++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 16 deletions(-) diff --git a/dw/plan.py b/dw/plan.py index 6f080ed9..5ef6b9d4 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -30,6 +30,7 @@ from .realize import read_sub_workflow, realize_workflow from .security import validate_url from .validation import _is_seeded +from .variables import same_number logger = logging.getLogger("dw") @@ -255,25 +256,11 @@ def output_only(doc): return doc -def _same_number(value): - """`value` with every integral float spelled as an int, so 12.0 and 12 - digest alike - the same value to the run (and to the step cache's - equality), but a JSON round trip through a client can respell one as - the other (#905). A bool is left alone: True is not 1.0 here.""" - if isinstance(value, float) and value.is_integer(): - return int(value) - if isinstance(value, dict): - return {key: _same_number(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [_same_number(item) for item in value] - return value - - def _canonical_digest(doc): # default=repr: a realized 'constant:' can be any Python value, and the # fingerprint only needs it to be stable, not round-trippable serialized = json.dumps( - _same_number(doc), + same_number(doc), sort_keys=True, separators=(",", ":"), ensure_ascii=False, diff --git a/dw/variables.py b/dw/variables.py index 573c92ba..4d2f9a67 100644 --- a/dw/variables.py +++ b/dw/variables.py @@ -39,6 +39,20 @@ class VariableCycleError(ValueError): or through others, inside `resolve_variable_values`.""" +def same_number(value): + """`value` with every integral float spelled as an int, so 12.0 and 12 + compare and digest alike - the same value to the run (and to the step + cache's equality), but a JSON round trip through a client can respell one + as the other (#905, #908). A bool is left alone: True is not 1.0 here.""" + if isinstance(value, float) and value.is_integer(): + return int(value) + if isinstance(value, dict): + return {key: same_number(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [same_number(item) for item in value] + return value + + def _resolve_variable_reference(value, variables): """ If value is a "variable:name" reference, look it up and return (True, resolved). diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 7562581e..349f5151 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -35,6 +35,7 @@ from .variables import ( VariableNotFoundError, replace_variables, + same_number, undeclared_variable_references, ) from .vram_estimate import pipeline_identity, vram_estimate_errors @@ -99,7 +100,9 @@ def _carried(node, key): if isinstance(node, dict): for name, value in node.items(): if name == key: - values.add(json.dumps(value, sort_keys=True)) + # By value, not spelling: a client's JSON round trip turns + # the template's `lora_scale` 1.0 into 1 (#908) + values.add(json.dumps(same_number(value), sort_keys=True)) values |= _carried(value, key) elif isinstance(node, list): for item in node: diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index e0a5dd6f..0973c830 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -622,6 +622,48 @@ def test_a_copy_without_its_null_variables_differs_in_nothing(self): arguments = {"lora_model_name": "lightx2v/Some-Other-Lora"} assert inherited_differences(copy, template, arguments) == ["LoRA"] + def test_a_client_round_trip_differs_in_nothing(self, tmp_path): + """#866 stage C (second C-F485 bounce): an MCP client's JSON round + trip respells the template's `lora_scale` 1.0 as 1. The copy, sent + the way the tester's was (seed set, prompt replaced, catalog keys + dropped), is quoted from the template's runs with nothing named and + no low confidence: a number compares by value, not spelling.""" + from dw.variables import same_number + + template = self.template() + copy = same_number(template) + assert copy["variables"]["lora_scale"] == 1 + assert isinstance(copy["variables"]["lora_scale"], int) + for key in ("cost", "cost_drivers", "summary", "vram_estimate"): + copy.pop(key, None) + copy.pop("variable_constraints", None) + copy["variables"]["seed"] = 42 + copy["variables"]["prompt"] = "a lighthouse at dusk" + + assert inherited_differences(copy, template) == [] + block = inherited( + copy, + [("templates/minimax/reference-to-video", template)], + {"templates/minimax/reference-to-video": [run(8 * MINUTE)] * 3}, + ) + assert block["differs"] == [] + candidate = workflow_from_definition( + copy, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + estimate = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: {**block, "device": None}, + )["estimate"] + assert estimate["basis"] == "inherited" + assert not estimate.get("differs") + assert not estimate.get("low_confidence") + # A scale that really differs is still named + assert inherited_differences(copy, template, {"lora_scale": 0.5}) == ["LoRA"] + def test_the_warning_agrees_in_number(): def warned(differs): From 76558c39f707ba74c0876749fbcc32f2ce9810bd Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 07:02:03 -0500 Subject: [PATCH 171/204] feat(server): #910 - hold a queued job for host RAM at dispatch, with a queue_wait event and waiting_for A job that fits a free card waits while its projected host-RAM peak exceeds MemAvailable less the growth the running jobs' peaks still have to make, plus the RSS of the idle worker it would reuse. Backfill applies; a job alone, without a figure, or on a host whose RAM can't be read is never held. A held job is re-checked every 10 s, emits one queue_wait event per hold, and carries waiting_for on its detail and summary (REST, MCP, web UI) only while queued and held. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 4 +- docs/MCP.md | 6 +- docs/SERVER.md | 24 ++- docs/WORKSPACES.md | 6 +- dw/host_memory_projection.py | 2 +- dw/server/api_models.py | 5 + dw/server/job_record.py | 8 + dw/server/jobs.py | 131 +++++++++++-- dw/server/pool.py | 37 +++- dw_mcp/diagnose.py | 2 + .../references/quality-and-speed.md | 3 +- tests/test_worker_pool.py | 184 ++++++++++++++++++ ui/src/lib/generated/api-schema.ts | 19 +- ui/src/lib/generated/openapi.json | 24 ++- ui/src/lib/job/JobHeader.svelte | 7 + 15 files changed, 429 insertions(+), 33 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 56ca1520..d8e4328e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -104,7 +104,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | -| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It projects and never refuses (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | +| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It never refuses; dispatch may delay a start on it (`JobManager._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | | H3 audio hold, refine and guide layout | Five modules, imported one way: `dw/pipeline_processors/h3_rules.py` holds the names and rules, torch- and diffusers-free (`refine_problems`, the per-argument refine rules asked before validation and before the call; `not_h3`, the step rule; `snap_guide_length`, `guide_frame_problem`, `guide_chain_problems`, `guide_end_problem`; `CHAIN_CONTINUITY_MODES`; the chunk constants), `dw/pipeline_processors/h3_hold.py` puts the hold and refine blocks in (`insert_audio_hold`, `core_denoise_sequences`, `refines`), and `dw/pipeline_processors/h3_guides.py` the guide layout (`LAYOUT_ANCHORS`, `insert_guides`), which imports `h3_hold` and never the reverse; the blocks themselves, and the arithmetic they run, are `dw/pipeline_processors/h3_hold_steps.py` (`encode_audio_span`, `refine_sigmas`) and `dw/pipeline_processors/h3_guide_steps.py` (`splice_guide_rows`), which import diffusers when they load, so the two owners import them only inside `blocks()` and `guide_blocks()` (#790), and `h3_guide_steps` imports `h3_hold_steps` and never the reverse (`_with_guides` in `dw/pipeline_processors/pipeline.py` adds its blocks for a `guides` call), `dw/guides.py`: the `guides` validation check and the chain `continuity: "guide"` check, `guide_chain_errors`, which asks the `guides` check's own step rule (`_takes_guides_problem`, built on `h3_rules.not_h3` and `REFERENCE_WORKFLOWS`) and reads the mode names from `h3_rules.CHAIN_CONTINUITY_MODES`, the names `chain.py`'s `CONTINUITY_MODES` is zipped against (both registered in `dw/validation.py`; a `previous_result:` guide is checked at run time), `dw/hold_audio.py`: `hold_audio_errors`, `refine_strength_errors` | The one place dw modifies a diffusers modular pipeline's block graph: three blocks go in, before `set_timesteps`, `denoise` and `after_denoise`, in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio` / `refine_strength`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. `dw/introspection.py` builds the same graph without weights, hold included, so `get_pipeline_signature` lists a modular pipeline's block inputs. The copies of diffusers behaviour are pinned to the installed diffusers: `shifted_sigma_grid` (the spacing `refine_sigmas` rests on) to stock `set_timesteps`, `GUIDE_FRAMES_PER_CHUNK`/`GUIDE_LATENTS_PER_CHUNK` to the H3 video VAE's `clip_length` and latents per clip, and every name in `LAYOUT_ANCHORS` (`_fill_audio_positions` among them) to `before_denoise`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused`, `tests/test_h3_hold_audio.py::TestSignature::test_the_h3_signature_shows_hold_audio`, `tests/test_h3_refine.py::TestAnchors::test_inserted_before_denoise_in_every_shape`, `tests/test_h3_refine.py::TestSigmaGridDrift::test_the_full_grid_is_stock_set_timesteps`, `tests/test_h3_guides.py::TestChunkDrift::test_frames_per_chunk_is_the_vae_clip_length`, `tests/test_h3_guides.py::TestChunkDrift::test_latents_per_chunk_is_the_vae_tokens_per_clip`, `tests/test_h3_guides.py::TestAnchors::test_installed_diffusers_has_every_anchor`, `tests/test_guide_chain_validation.py::TestRefused::test_the_step_rule_is_the_guides_rule`, `tests/test_guide_chain_validation.py::TestRefused::test_every_run_mode_is_known_to_validate`, `tests/test_h3_rules.py::test_h3_rules_imports_neither_torch_nor_diffusers`, `tests/test_h3_rules.py::test_the_block_owners_import_no_diffusers` | @@ -159,7 +159,7 @@ one to open. | App factory | `dw/server/app.py`: `create_app` | `create_app` builds the JobManager and `app.state`, installs the middleware and registers the routes, and all state lives in the JobManager. | — | | Routers | `dw/server/routes/*.py`, `dw/server/routes/__init__.py`: `ROUTERS`, `include_routers`, `include_file_routes` | Routers register in `ROUTERS` order, because a greedy `{name:path}` route must come after its specific siblings. The files router comes later, after `/mcp`. | `tests/test_server_downloads.py::test_download_workflow_sets_content_disposition_attachment` | | Job queue | `dw/server/jobs.py`: `JobManager`; `dw/server/job_results.py`: `consume_results` | A dispatcher thread hands queued jobs, in queue order, to the pool's workers (`Worker pool dispatch`); each worker runs one job at a time, and `consume_results` folds that worker's replies into the job, naming files relative to the job's own `output_dir`. A job carries its own `output_dir`, `asset_dir` and `workflow_dir`, so it stays in its workspace. | `tests/test_server_workspaces.py::TestRunning::test_rerun_stays_in_the_workspace_it_ran_in` (FIFO: —) | -| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted` | +| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, a job that fits a free card is also held while its projected host-RAM peak exceeds `MemAvailable` less what running jobs have yet to add plus the idle worker's own RSS (a job alone is never held, no figure or no RAM reading means no gate, a smaller job behind a held one starts, and the dispatcher re-checks every 10 s; #869), and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_the_host_cannot_hold_beside_the_running_one_is_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_alone_is_never_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_smaller_job_starts_past_a_held_one` | | Card affinity | `dw/server/pool.py`: `choose_slot`; `dw/server/jobs.py`: `JobManager.route`; `dw/worker_protocol.py`: `workflow_identity` | Among the free cards a job fits it goes to the rerun's original card, else the card whose worker last ran its workflow identity (`workflow_identity`, the one rule the worker caches by too), else one with no worker running, else the first in `--devices` order; a cache probe and `plan.estimate` (`priced_for`) ask about the card `route` names, and memory is read and cleared per card. | `tests/test_worker_pool.py::TestIdentityAffinity::test_a_second_run_lands_on_the_card_that_ran_the_workflow_first`, `tests/test_worker_pool.py::TestRerunAffinity::test_a_rerun_lands_on_the_original_card_when_it_is_free`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle` | | Per-card memory and cache | `dw/server/worker_memory.py`: `memory_status`, `clear_memory`, `probe_cache` | Each card keeps its own last memory reading; an idle card's worker is asked live, a busy one answers with the run's last report and says why it is not live, and a clear leaves a card running a job alone. Every request takes the card's lock with a bounded wait, and a card's lock is never taken while the JobManager's lock is held. | `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_server.py::test_memory_says_why_a_reading_is_not_the_worker_s` | | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | diff --git a/docs/MCP.md b/docs/MCP.md index 3a3f4e02..0e2eff50 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -232,7 +232,7 @@ when no single workflow covers it. | `get_memory()` | `device` | Get VRAM and RAM statistics: `workers` has one reading per card (the top level repeats the first card's), and `device` (`cuda:1`) asks for that card alone. `gpu_*` is the card; `host_memory_rss_mb` / `host_memory_peak_rss_mb` are the worker process's resident and high-water host memory, beside the machine's `host_memory_total_mb` / `host_memory_available_mb`; on Linux `host_memory_rss_anon_mb` / `host_memory_rss_file_mb` split the resident figure into memory the worker owns and mapped-file pages (memory-mapped checkpoint weights) the kernel can drop, so a loader that maps rather than copies raises the resident and peak figures without raising committed memory (#709) - read both, since an offloading workflow keeps its weights in host RAM and the card says little about what it holds (a host field is absent, not null, where the platform cannot measure it). `host_pinned_reserved_mb` / `host_pinned_allocated_mb`, when present, are torch's pinned-host cache - the staging buffers group offloading moves weights through, part of `host_memory_rss_mb` and invisible in every `gpu_*` figure, which is what a worker holding GB after releasing every model is usually holding (#98). `live: true` means `info` was measured now and is the worker's own memory; only live readings are comparable with each other. `live: false` with `info: null` (and `stale: false`) means nothing has been measured because nothing is resident; `live: false` with a populated `info` is a cached earlier reading, with `reason` (`job_running`, `worker_stopped`, `worker_busy`, `worker_unreachable`) and `age_seconds` - one cached while a job loads a model understates what is resident, so ask again when the card is idle rather than comparing it with a live figure | | `get_health()` | — | Check that the server is alive, and which machine answered: `version`, `device`, whether any model process is resident (`worker_alive`), and the queue depth. The engine runs one job per GPU: `workers` has one entry per card - `device`, `name`, `vram_gb`, `alive`, its own `current_job`, and the worker's `host_memory_rss_mb` while it runs - and the top-level `current_job` is the longest-running job. `worker_alive: false` is the normal idle state on a server that has not run a job since startup or the last memory clear - not a fault - the on-demand worker starts with the next job (#206) | | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server. `runtime` (#222) reports the Python version, torch version and the CUDA version torch was built against, the NVIDIA driver version (when `nvidia-smi` is reachable), and the installed versions of diffusers, transformers, accelerate, bitsandbytes, peft, safetensors and sentencepiece (`null` for one not installed) - for diagnosing an environment mismatch between boxes without shelling in. `caches` lists dw's disk caches (`inductor`, `triton`, `quantized`), each with `name`, `bytes`, `files`, `enabled`, `clearable` and `disk_free`, plus `entry_count` for a cache with entries, and `cache_entries=true` lists them (`id`, `model`, `subfolder`, `component`, `bytes`, `files`, `unreadable`, `created`, `last_hit`; the quantizer and dtype are not in `/api/caches`); it is omitted when the server has no `/api/caches`. There is no MCP tool to clear one - that is `DELETE /api/caches/{name}` or `.../entries/{entry}` ([SERVER.md](SERVER.md)) | -| `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first**. Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | +| `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first** (a queued job held for host RAM carries `waiting_for`). Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | | `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_gallery_metadata(name, envelope=False, workspace=None, interval_seconds=None)` | `name`, `workspace` | Get the metadata embedded in a generated file — or, when `name` is an `asset:` reference, what an *input* asset holds (`source` says which; `job` is null for an asset). Reading an input's duration, frame count, fps and sample rate before a run is how a caller learns the `total_frames`, `fps` and `sample_rate` a workflow expects it to supply: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) and `findings` - each level problem the server measured against `dw/audio_qc.py`'s thresholds (`full_scale`, `near_silent`), with the fix. Only an image (PNG/JPEG/WebP) embeds `metadata` this way — it is always null for audio and video, and `next` then points at `get_job_workflow(job_id)` when `job` is known, or says a kept asset carries no provenance at all when it isn't. `envelope=true` adds `media.envelope` — `rms_dbfs` and `peak_dbfs` one entry per second, or per `interval_seconds` (default 1.0, 0.05 to 3600; 0.1 places a vocal onset; a finer-than-default width that would need over 2400 bins is refused with the interval that fits) — which is what locates something in a track rather than measuring the whole of it. `media.shots` is set on an output joined from shots: one `{name, start_frame, num_frames, start_sample, num_samples}` per shot, as the join measured them (see the run manifest in docs/WORKFLOW_GUIDE.md), else null. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | @@ -316,11 +316,11 @@ references written in the same session. | Tool | Arguments | Purpose | | --- | --- | --- | | `run_workflow(workflow_path=None, inline_workflow=None, arguments=None, acknowledged_cost=False, workspace=None, wait_seconds=0, note=None)` | exactly one of `workflow_path` (a catalog name from `list_workflows`, with or without `.json`, or a path to a workflow file on the server) or `inline_workflow`, optional `arguments`, `acknowledged_cost`, `workspace`, `wait_seconds`, `note` | Queue a workflow for generation. Returns as soon as the job is queued - unless `wait_seconds` is above 0, in which case the call then waits on the queued job exactly as `wait_for_job(job_id, timeout_seconds=wait_seconds)` would (same per-call cap, clamped not honoured) and the result carries the queued-job fields plus that wait's (`status`, `still_running`, `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds`, `timeout_capped`, the slim `job`); when the cap covers the job's runtime one call is the run and the wait, and a `still_running: true` result is followed with `wait_for_job` as before. `workspace` names the workspace for this one call without switching the session to it - use it to pin a job whose `output:` or `asset:` references live in a workspace other than the session's - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate, and nothing is waited on. `note` says what the run is for (see `set_note`) | -| `get_job(job_id)` | `job_id` | Get a job's status, warnings, output manifest, error and traceback; each manifest entry's `subfolder` is the in-run subfolder the step declared - by convention `final` for the deliverable, `intermediate` for scratch, `''` for none. A running job also carries `progress` (below) | +| `get_job(job_id)` | `job_id` | Get a job's status, warnings, output manifest, error and traceback; each manifest entry's `subfolder` is the in-run subfolder the step declared - by convention `final` for the deliverable, `intermediate` for scratch, `''` for none. A running job also carries `progress` (below); a queued job held for host RAM carries `waiting_for` (#869) | | `get_job_workflow(job_id)` | `job_id` | The REST equivalent is `GET /api/jobs/{id}/workflow` (see [SERVER.md](SERVER.md#jobs-api)). The workflow the job actually ran. `realized: true` means every mutable input is pinned (arguments, seed, prompts, `output:latest`); `false` means the job predates run tracking and this is the definition as submitted. Pass it to `save_workflow` to keep it under a name | | `export_job(job_id, overwrite=False)` | `job_id`, `overwrite` | Gather one finished job into `/exports//` on the server: the realized workflow, the run's manifest, the job row, a README, and copies of the assets, earlier-run inputs and outputs. Returns the directory, a zip URL, the file list with sizes and the total. The three JSON files are in the zip, not repeated here - get_job_workflow and get_job serve them individually. **The directory is on the machine running the server**, like `download_output`'s destination. The zip needs no token (`/exports/*.zip` is ungated like `/outputs`, #592), so `auth_required` is false and the agent fetches `open_url` itself (prefixing a relative one with the server address) and unpacks it into `exports/` under the session's working directory (a deliverable, not a temp file) - the archive already unpacks into one folder named after the job id, so do not create that folder first; only an agent without HTTP gives the user `open_url`. `true` is kept as a forward guard for a gated zip, when `open_url` is handed to the person instead | | `get_job_events(job_id, after=-1, limit=200)` | `job_id`, `after`, `limit` | Get a page of a job's progress events | -| `wait_for_job(job_id, timeout_seconds=20)` | `job_id`, `timeout_seconds` | Block until a job reaches a terminal status, or `timeout_seconds` elapses. **One call blocks for at most the server's cap** — 55 seconds unless the deployment sets `DW_MCP_MAX_WAIT_SECONDS` higher, and the tool's description states the live value; a larger `timeout_seconds` is clamped, not honoured. Ask for the job's `plan.estimate` plus a margin: under the cap that is one call for the whole job, over it one call per cap. A deployment raises the cap only as far as its clients (and anything between them and the server) hold one silent HTTP request open - Claude Code's limit is `MCP_TOOL_TIMEOUT`. Every reply carries `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds` and `timeout_capped`, so a capped return is distinguishable from an elapsed one. Use instead of hand-polling `get_job`/`get_job_events` in a loop; if it returns `still_running: true`, call it again. Returns a slim job - status, warnings, error, `run_id` and `run_version` (the run's `v5`, as the gallery labels it), and the manifest once finished - without the arguments; `get_job` has those. A running job also carries `progress` (below) | +| `wait_for_job(job_id, timeout_seconds=20)` | `job_id`, `timeout_seconds` | Block until a job reaches a terminal status, or `timeout_seconds` elapses. **One call blocks for at most the server's cap** — 55 seconds unless the deployment sets `DW_MCP_MAX_WAIT_SECONDS` higher, and the tool's description states the live value; a larger `timeout_seconds` is clamped, not honoured. Ask for the job's `plan.estimate` plus a margin: under the cap that is one call for the whole job, over it one call per cap. A deployment raises the cap only as far as its clients (and anything between them and the server) hold one silent HTTP request open - Claude Code's limit is `MCP_TOOL_TIMEOUT`. Every reply carries `waited_seconds`, `timeout_requested_seconds`, `timeout_applied_seconds` and `timeout_capped`, so a capped return is distinguishable from an elapsed one. Use instead of hand-polling `get_job`/`get_job_events` in a loop; if it returns `still_running: true`, call it again. Returns a slim job - status, warnings, error, `run_id` and `run_version` (the run's `v5`, as the gallery labels it), and the manifest once finished - without the arguments; `get_job` has those. A running job also carries `progress` (below); a queued job held for host RAM carries `waiting_for` | | `cancel_job(job_id)` | `job_id` | Ask a queued or running job to stop | | `clear_memory()` | `device` | Drop every loaded pipeline and the step cache on each idle card - `device`'s alone when named - freeing VRAM/RAM immediately instead of waiting for the next job to evict one model for another. Also drops the step cache, so a seeded workflow that would otherwise reuse cached results regenerates on its next run. A card running a job is skipped (`workers` reports it `job_running`); refused with a 409 when no card asked about is idle, so wait and retry rather than expecting this call to block (#221, #677) | | `rerun_job(job_id, acknowledged_cost=False, new_seed=False)` | `job_id`, `acknowledged_cost`, `new_seed` | Queue a fresh job from a previous job's stored specification. Costs GPU time, so it passes the same gate as `run_workflow`. `new_seed=true` draws a fresh seed into the workflow's seed variable — without it a seeded workflow's rerun repeats its arguments exactly and the step cache serves the whole run from the earlier one's files (`reused: true`), generating nothing. `get_job_workflow`'s `seed_variable` says whether there is one - `acknowledged_cost` is `true` or the bound `{fingerprint, minutes, downloads}` from the validate plan; a 409 means the plan changed and the message carries the new estimate | diff --git a/docs/SERVER.md b/docs/SERVER.md index c78d9435..efb1f09f 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -211,7 +211,7 @@ running: | `GET /api/jobs/{id}/event-log?after=-1&limit=200` | The same events as the SSE stream, as one JSON page: `{id, status, events, last_seq, truncated, note}`. `after` is exclusive; page by passing back the previous `last_seq`. A job restored from history serves the bounded event tail persisted with it; a job that finished before events were retained returns an empty list and a `note` saying so. | | `POST /api/jobs/{id}/cancel` | Cooperative cancel (takes effect at the next step boundary or denoise step) | | `POST /api/jobs/{id}/rerun` | Re-queue a finished job's spec. Body `{"new_seed": true}` draws a fresh seed into the workflow's seed variable instead of repeating the original arguments; 400 when the workflow pins its seed to a literal or names none. A plain rerun of a seeded workflow is served whole from the step cache — the earlier run's files, republished with `reused: true`, generating nothing. The new job's summary and detail carry `rerun_of`: the id of the job it reran (absent on a fresh submit) | -| `POST /api/jobs/{id}/move` | Reorder a queued job: `{"direction": "up"\|"down"\|"front"\|"back"}`. Job listings carry each waiting job's `queue_position`. | +| `POST /api/jobs/{id}/move` | Reorder a queued job: `{"direction": "up"\|"down"\|"front"\|"back"}`. Job listings carry each waiting job's `queue_position`, and `waiting_for` while one is held for host RAM. | Each worker runs one job at a time, so with one device (the default) one job runs at a time; submissions queue in order, and the waiting portion of the @@ -236,8 +236,26 @@ newest job whatever the workers' sizes; nothing is written with one worker, and a write the kernel refuses is a warning on the job. Under systemd the unit needs `OOMPolicy=continue` (`scripts/dw-serve.service` sets it): the default `stop` answers one worker's OOM kill by stopping the whole server, every other job -with it. There is no -host-RAM gate at dispatch: two big jobs can still exhaust host RAM together. +with it. A +queued job may also carry a projected host-RAM peak (see +`host_ram_need`, #869). When it fits a free card, dispatch checks host RAM +too: available is `MemAvailable` less, for each running job, what that job's +projected peak still has to add beyond its worker's resident size now, and +the job is held while its peak exceeds available plus the resident size of +the idle worker it would reuse. A running job with no figure is charged +nothing extra, since `MemAvailable` already reflects what it holds. Backfill +applies: a smaller job behind a held one may start. With no other job +running a job is never held, so it waits only for running jobs, never +forever; a job with no figure is never held; and where the host's RAM cannot +be read (macOS without psutil) there is no gate. With one card only one job +runs, so the gate is never exercised. A held job is re-checked every 10 s, +since memory can free outside dw. The hold emits one `queue_wait` event +(`reason: "host_ram"`, message `waiting for host RAM: needs ~N GB, ~M GB +free`; not a `warning`, so nothing lands in the job's `warnings`), and job +detail and summaries carry `waiting_for` - that message - only while the job +is queued and held. The web UI shows it beside "#n in line". The gate never +refuses a submit (#243); it only delays a start, and the OOM ranking above +(#881) stays the backstop for a wrong projection. Among the free cards a job fits, dispatch prefers the one it has an affinity for: a rerun goes to the card that ran the original, and any job to the card whose worker last ran the same workflow (the same file, or an inline diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 53d01fd0..5e9b4427 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -456,9 +456,9 @@ dw already has. 4. **One `run_workflow` per variant, with a `note`.** The note's first line is the overrides, `steps=8` or `steps=20 attention=sage`, so a listing that clips notes still shows which variant is which. Call `wait_for_job` - until the variant finishes before submitting the next. Dispatch is not - yet aware of host RAM (#869), so two large variants on two cards can run - the host out of memory together. + until the variant finishes before submitting the next. The server holds a + second large variant until the host can hold it beside the first (see + [SERVER.md](SERVER.md)), so waiting is for ordering, not for memory. 5. **Only the latest run of a step is cached.** The step cache holds one entry per workflow step: its most recent run. A variant whose step and diff --git a/dw/host_memory_projection.py b/dw/host_memory_projection.py index 3832cb7f..9710ea40 100644 --- a/dw/host_memory_projection.py +++ b/dw/host_memory_projection.py @@ -10,7 +10,7 @@ curated (no author-declared memory figure - `host_memory_job_peak_rss_mb` is a worker-reported field, not a schema key); warn, not refuse (host RAM headroom is a property of *this machine*, not something a caller chose, so it never -blocks a run); no cross-machine normalization; and a cold start - no history +refuses a run; dispatch may delay a start on the figure, #869); no cross-machine normalization; and a cold start - no history for this workflow at all - means no check, the same rule `observed_cost.py` uses for GPU minutes. """ diff --git a/dw/server/api_models.py b/dw/server/api_models.py index d9b2baf9..a6f807ac 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -386,6 +386,11 @@ class JobSummary(ApiModel): ) historical: bool = sometimes("Read from job history rather than a live job.") queue_position: int = sometimes("Index in the waiting queue; only while queued.") + waiting_for: str = sometimes( + "Why dispatch is holding this queued job though a card is free - " + "'waiting for host RAM: needs ~N GB, ~M GB free'; only while queued and " + "held. It starts when a running job frees the RAM." + ) note: str = sometimes("The note it was submitted with; also its run's note.") rerun_of: str = sometimes( "The job this one reran; absent for a job submitted fresh." diff --git a/dw/server/job_record.py b/dw/server/job_record.py index 403a5cd5..e178f1f0 100644 --- a/dw/server/job_record.py +++ b/dw/server/job_record.py @@ -104,6 +104,9 @@ def __init__(self, spec): # The host RAM it was projected to hold at its peak when admitted, # (mb, basis, runs) or None (dw/server/host_ram_need.py, #869) self.host_ram_need = None + # Why dispatch is holding it while it is queued - "waiting for host + # RAM: ..." - or None; a live state, never a warning (#869) + self.waiting_for = None self.events = [] # The running summary a poll reads - see _note_progress. Kept as the # events arrive rather than derived from the log on request, because @@ -311,6 +314,11 @@ def summary(self): if self.cancel_requested and self.status == RUNNING else {} ), + **( + {"waiting_for": self.waiting_for} + if self.waiting_for and self.status == QUEUED + else {} + ), **({"note": self.note} if self.note else {}), **({"rerun_of": self.rerun_of} if self.rerun_of else {}), } diff --git a/dw/server/jobs.py b/dw/server/jobs.py index fa719d6a..a6dfbea9 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -43,7 +43,7 @@ from ..events import NON_INTERRUPTIBLE_PHASES from ..worker_protocol import Execute, workflow_identity from ..devices import device_ordinal -from ..host_memory import process_rss_mb, read_oom_score_adj +from ..host_memory import host_memory_stats, process_rss_mb, read_oom_score_adj from ..worker_manager import WorkerManager from ..workflow_run import SEED_BITS from ..security import ( @@ -86,6 +86,33 @@ CANCEL_GRACE_SECONDS = 30.0 +# How often a job held for host RAM is looked at again while nothing else +# wakes the dispatcher - memory can be freed outside dw (#869) +HOST_RAM_RECHECK_SECONDS = 10.0 + + +class _HostReading: + """The host's MemAvailable for one dispatch pass, read on first ask.""" + + def __init__(self): + self._read = False + self._available = None + + def available_mb(self): + if not self._read: + self._read = True + # None where the platform can't say (macOS without psutil): no gate + self._available = (host_memory_stats() or {}).get("available_mb") + return self._available + + +def _worker_rss_mb(slot): + """What `slot`'s worker holds in host RAM now, or None (not running, or + unreadable).""" + pid = slot.manager.pid() if hasattr(slot.manager, "pid") else None + return process_rss_mb(pid) if pid is not None else None + + class JobsActive(Exception): """A job is running or queued.""" @@ -135,6 +162,9 @@ def __init__( self._wake = threading.Condition(self._lock) # The VRAM each job needs to start, by job id (dispatch reads it) self._needs = {} + # The host RAM (MB) each job is projected to peak at, by job id - only + # jobs with a figure; dispatch holds one the host can't fit (#869) + self._host_needs = {} # The card a rerun prefers - the original's ordinal - by job id self._preferred = {} # Callers holding dispatch (hold_dispatch): while any does, no queued @@ -270,7 +300,8 @@ def submit( `host_ram_need` is the admission's (`Admission.host_ram_need`): the host RAM the run is projected to hold at its peak, kept on the job - as `job.host_ram_need` (#869). Dispatch does not read it yet. + as `job.host_ram_need` (#869). The job waits to start while the + host cannot hold it beside the running jobs (`_host_ram_wait`). `preferred_device` is the card ('cuda:1') the job goes to when that card is free and fits it - a rerun's original card (`rerun`). @@ -336,6 +367,8 @@ def submit( with self._lock: self.jobs[job.id] = job self._needs[job.id] = pool.dispatch_need(self.slots, vram_need) + if host_ram_need: + self._host_needs[job.id] = host_ram_need[0] if preferred_device: self._preferred[job.id] = preferred_device job.add_event({"event": "job_status", "status": QUEUED}) @@ -670,8 +703,7 @@ def cancel(self, job_id): if job.status == QUEUED: if job.id in self._pending: self._pending.remove(job.id) - self._needs.pop(job.id, None) - self._preferred.pop(job.id, None) + self._forget_dispatch(job) self._finish(job, CANCELLED) return job.status slot = self._slot_running(job.id) @@ -742,6 +774,9 @@ def move(self, job_id, direction): else: index = min(len(self._pending), index + 1) self._pending.insert(index, job_id) + # The order decides which job the next pass tries first, a held + # one included (#869) + self._wake.notify() return list(self._pending) def shutdown(self): @@ -772,7 +807,10 @@ def _run_loop(self): picked = self._next_dispatch() if picked is not None: break - self._wake.wait() + # Host RAM frees outside dw too, so a job held for it is + # looked at again on a timer, not only on a wake (#869) + held = any(self._held_for_host_ram()) + self._wake.wait(HOST_RAM_RECHECK_SECONDS if held else None) slot, job = picked job.status = RUNNING job.started_at = time.time() @@ -797,31 +835,92 @@ def _next_dispatch(self): a free card - the one it has an affinity for (`_choose_slot`) - or None. A job too big for every free card keeps its place, and the scan goes on past it: a smaller job behind it takes the card - meanwhile (backfill). Called holding _lock.""" + meanwhile (backfill). A job the host's RAM can't hold beside the + running ones keeps its place the same way (`_host_ram_wait`). + Called holding _lock.""" if self._dispatch_holds: return None free = [slot for slot in self.slots if slot.current_job_id is None] if not free: return None + host = _HostReading() for job_id in list(self._pending): job = self.jobs.get(job_id) if job is None or job.status != QUEUED: self._pending.remove(job_id) # cancelled while waiting - self._needs.pop(job_id, None) - self._preferred.pop(job_id, None) + self._forget_dispatch(job, job_id) continue need = self._needs.get(job_id) fitting = [slot for slot in free if slot.fits(need)] - if fitting: - slot = pool.choose_slot( - fitting, self._job_identity(job), self._preferred.get(job_id) - ) - self._pending.remove(job_id) - self._needs.pop(job_id, None) - self._preferred.pop(job_id, None) - return slot, job + if not fitting: + # Waiting for a card, not for host RAM + job.waiting_for = None + continue + slot = pool.choose_slot( + fitting, self._job_identity(job), self._preferred.get(job_id) + ) + reason = self._host_ram_wait(job_id, slot, host) + if reason is not None: + if job.waiting_for is None: + # Once per hold, not once per pass: a wait is a moment, + # so an event and a live field, never a warning (#869) + job.add_event( + {"event": "queue_wait", "reason": "host_ram", "message": reason} + ) + job.waiting_for = reason + continue + self._pending.remove(job_id) + self._forget_dispatch(job) + return slot, job return None + def _forget_dispatch(self, job, job_id=None): + """Drop what dispatch keeps for a job leaving the queue - started or + cancelled. `job` may be None for one already gone.""" + job_id = job.id if job is not None else job_id + self._needs.pop(job_id, None) + self._host_needs.pop(job_id, None) + self._preferred.pop(job_id, None) + if job is not None: + job.waiting_for = None + + def _held_for_host_ram(self): + """The queued jobs held for host RAM now. Called holding _lock.""" + for job_id in self._pending: + job = self.jobs.get(job_id) + if job is not None and job.waiting_for is not None: + yield job + + def _host_ram_wait(self, job_id, slot, host): + """Why the queued `job_id` must not start on `slot` yet for host RAM, + or None (pool.host_ram_wait, #869). `host` is this pass's reading, + taken once and only when a job with a figure asks.""" + need = self._host_needs.get(job_id) + if need is None: + return None + running = [ + (other, self.jobs.get(other.current_job_id)) + for other in self.slots + if other.current_job_id is not None + ] + if not running: + return None + available = host.available_mb() + if available is None: + return None + return pool.host_ram_wait( + need, + available, + [ + ( + job.host_ram_need[0] if job and job.host_ram_need else None, + _worker_rss_mb(other), + ) + for other, job in running + ], + _worker_rss_mb(slot), + ) + def _slot_running(self, job_id): """The slot whose worker is running `job_id`, or None.""" with self._lock: diff --git a/dw/server/pool.py b/dw/server/pool.py index 4f8e09ed..f859f6cb 100644 --- a/dw/server/pool.py +++ b/dw/server/pool.py @@ -3,7 +3,9 @@ A WorkerSlot is one `--devices` card: its worker, the job it runs and the lock that serializes talking to it. The functions here read only the slots they are given and take no lock - fit (`check_fits`, `dispatch_need`), the -largest card (`largest_ceiling_gb`) and affinity (`choose_slot`). The +largest card (`largest_ceiling_gb`), affinity (`choose_slot`) and the +host-RAM wait (`host_ram_wait`): the cards share one host's RAM, so a job +that fits a free card may still wait for a running one to finish. The JobManager (`dw/server/jobs.py`) calls them under its own lock where the answer must agree with dispatch (`JobManager.route`, `_next_dispatch`). """ @@ -226,3 +228,36 @@ def dispatch_need(slots, need): if hard or any(capacity >= gb for _, capacity in capacities(slots)): return (gb, hard) return None + + +# ---------------------------------------------------------------- host RAM + + +def host_ram_wait(need_mb, available_mb, running, idle_rss_mb=None): + """Why a job projected to peak at `need_mb` of host RAM must wait to + start now, or None when it may start (#869). + + `available_mb` is the host's MemAvailable; `running` holds one + (projected peak MB or None, worker RSS MB now or None) per running job; + `idle_rss_mb` is what the idle worker the job would reuse holds, which + the job takes over. Each running job is charged the growth it has still + to make, `max(0, peak - rss)`: one still loading has not reached its + peak, and MemAvailable already reflects what it holds. A running job + with no figure is charged nothing more, and an unread RSS is a worker + holding nothing yet. + + Never a wait with no other job running, so a job waits only for jobs + that will finish; never without a figure for the job or a reading of + the host.""" + if need_mb is None or available_mb is None or not running: + return None + growth = sum( + max(0.0, peak - (rss or 0.0)) for peak, rss in running if peak is not None + ) + free = max(0.0, available_mb - growth + (idle_rss_mb or 0.0)) + if need_mb <= free: + return None + return ( + f"waiting for host RAM: needs ~{need_mb / 1024:.0f} GB, " + f"~{free / 1024:.0f} GB free" + ) diff --git a/dw_mcp/diagnose.py b/dw_mcp/diagnose.py index b99ecc50..305d8dfb 100644 --- a/dw_mcp/diagnose.py +++ b/dw_mcp/diagnose.py @@ -231,6 +231,8 @@ def get_job_events(client, job_id, after=-1, limit=200, kinds=None): # can be told apart from a run on the slower card "device", "queue_position", + # Why a queued job is held though a card is free (#869) + "waiting_for", "warnings", "error", "event_count", diff --git a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md index 15b4342b..0a3ab528 100644 --- a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md +++ b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md @@ -42,7 +42,8 @@ shift the LoRA names. - **Draft then final**: a 12-step draft (~13 min) of the wide, then the 30-step final with `first_block` 0.05 (~23 min). Draft to final carries blocking but not always props. -- **One int8 job at a time**: host RAM. +- **Submit freely**: the server holds a second heavy job, queued with + `waiting_for`, until host RAM frees. - **Frame ceiling** at 1344x768 with 4 references is 124; 141 was refused (23.9 GB > 23.6). Write to 124 or drop a reference. diff --git a/tests/test_worker_pool.py b/tests/test_worker_pool.py index c41eebed..5f3ce74c 100644 --- a/tests/test_worker_pool.py +++ b/tests/test_worker_pool.py @@ -1246,3 +1246,187 @@ def mark_crashed(self): job.cancel_requested = True outcome = job_results.consume_results(job, Dead(), None, str(tmp_path), None) assert outcome == ("cancelled", None, None) + + +# ---------------------------------------------------------------- host RAM + + +GB = 1024 + + +class TestHostRamGate: + """#869: a job that fits a free card waits while the host's RAM can't hold + it beside the running jobs, with the readings faked.""" + + @pytest.fixture + def host(self, monkeypatch): + """The host's MemAvailable and each worker's RSS, as a test sets them.""" + readings = {"available_mb": 64 * GB, "rss": {}} + monkeypatch.setattr( + "dw.server.jobs.host_memory_stats", + lambda: ( + None + if readings["available_mb"] is None + else {"available_mb": readings["available_mb"]} + ), + ) + monkeypatch.setattr( + "dw.server.jobs._worker_rss_mb", + lambda slot: readings["rss"].get(slot.device), + ) + return readings + + @staticmethod + def submit(manager, name, host_gb=None, vram_need=None): + return manager.submit( + admitted=admitted_for(manager, valid_workflow(name)), + workflow=valid_workflow(name), + vram_need=vram_need, + host_ram_need=None if host_gb is None else (host_gb * GB, "observed", 1), + ) + + def _two_cards(self, pool): + first, second = Gate(), Gate() + manager = pool(card(first, "cuda:0", 24), card(second, "cuda:1", 24)) + return manager, first, second + + @staticmethod + def queue_waits(job): + return [e for e in job.events if e.get("event") == "queue_wait"] + + def test_a_job_the_host_cannot_hold_beside_the_running_one_is_held( + self, pool, host + ): + manager, first_gate, second_gate = self._two_cards(pool) + host["available_mb"] = 40 * GB + running = self.submit(manager, "running", host_gb=30) + first_gate.wait_started() + host["rss"]["cuda:0"] = 30 * GB + host["available_mb"] = 10 * GB + + held = self.submit(manager, "held", host_gb=30) + assert wait_until(lambda: held.waiting_for is not None) + assert held.status == "queued" + assert held.waiting_for == "waiting for host RAM: needs ~30 GB, ~10 GB free" + assert not second_gate.started.is_set() + assert manager.get(held.id).summary()["waiting_for"] == held.waiting_for + + # The running job finishing is what frees the RAM and wakes dispatch + host["available_mb"] = 40 * GB + first_gate.release.set() + wait_for(running, {"succeeded"}) + wait_for(held, {"running", "succeeded"}) + assert held.waiting_for is None + assert "waiting_for" not in held.summary() + second_gate.release.set() + wait_for(held, {"succeeded"}) + + def test_memory_freed_outside_dw_starts_the_held_job_on_the_recheck( + self, pool, host, monkeypatch + ): + monkeypatch.setattr("dw.server.jobs.HOST_RAM_RECHECK_SECONDS", 0.05) + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "running", host_gb=10) + first_gate.wait_started() + host["rss"]["cuda:0"] = 10 * GB + host["available_mb"] = 5 * GB + held = self.submit(manager, "held", host_gb=20) + assert wait_until(lambda: held.waiting_for is not None) + + # Nothing in dw wakes the dispatcher: only the timer sees this + host["available_mb"] = 40 * GB + second_gate.wait_started() + wait_for(held, {"running"}) + + def test_a_smaller_job_starts_past_a_held_one(self, pool, host): + gates = [Gate(), Gate(), Gate()] + manager = pool(*(card(g, f"cuda:{i}", 24) for i, g in enumerate(gates))) + self.submit(manager, "running", host_gb=10) + gates[0].wait_started() + host["rss"]["cuda:0"] = 10 * GB + host["available_mb"] = 12 * GB + + big = self.submit(manager, "big", host_gb=30) + small = self.submit(manager, "small", host_gb=4) + wait_for(small, {"running"}) + assert big.status == "queued" + assert big.waiting_for is not None + assert small.waiting_for is None + + def test_a_job_alone_is_never_held(self, pool, host): + host["available_mb"] = 1 * GB + manager = pool(card(success_script, "cuda:0", 24)) + job = self.submit(manager, "alone", host_gb=200) + wait_for(job, {"succeeded"}) + assert not self.queue_waits(job) + + def test_a_job_with_no_figure_is_never_held(self, pool, host): + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "running", host_gb=30) + first_gate.wait_started() + host["available_mb"] = 0 + unknown = self.submit(manager, "unknown") + second_gate.wait_started() + wait_for(unknown, {"running"}) + assert not self.queue_waits(unknown) + + def test_no_host_reading_means_no_gate(self, pool, host): + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "running", host_gb=30) + first_gate.wait_started() + host["available_mb"] = None + job = self.submit(manager, "unread", host_gb=30) + second_gate.wait_started() + wait_for(job, {"running"}) + assert not self.queue_waits(job) + + def test_the_growth_a_still_loading_job_has_left_is_charged(self, pool, host): + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "loading", host_gb=30) + first_gate.wait_started() + # 40 GB free now, but the loading job will take 25 GB more of it + host["rss"]["cuda:0"] = 5 * GB + host["available_mb"] = 40 * GB + held = self.submit(manager, "next", host_gb=20) + assert wait_until(lambda: held.waiting_for is not None) + assert held.waiting_for == "waiting for host RAM: needs ~20 GB, ~15 GB free" + assert not second_gate.started.is_set() + + def test_the_idle_worker_it_would_reuse_counts_as_free(self, pool, host): + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "running", host_gb=10) + first_gate.wait_started() + host["rss"] = {"cuda:0": 10 * GB, "cuda:1": 15 * GB} + host["available_mb"] = 10 * GB + job = self.submit(manager, "reuses", host_gb=20) + second_gate.wait_started() + wait_for(job, {"running"}) + + def test_the_event_fires_once_per_hold(self, pool, host, monkeypatch): + monkeypatch.setattr("dw.server.jobs.HOST_RAM_RECHECK_SECONDS", 0.02) + manager, first_gate, second_gate = self._two_cards(pool) + self.submit(manager, "running", host_gb=10) + first_gate.wait_started() + host["rss"]["cuda:0"] = 10 * GB + host["available_mb"] = 5 * GB + held = self.submit(manager, "held", host_gb=20) + assert wait_until(lambda: held.waiting_for is not None) + time.sleep(0.2) # several rechecks, each still held + waits = self.queue_waits(held) + assert len(waits) == 1 + assert waits[0]["reason"] == "host_ram" + assert waits[0]["message"] == held.waiting_for + assert not held.warnings + + def test_cancel_clears_waiting_for_and_the_bookkeeping(self, pool, host): + manager, first_gate, _ = self._two_cards(pool) + self.submit(manager, "running", host_gb=10) + first_gate.wait_started() + host["rss"]["cuda:0"] = 10 * GB + host["available_mb"] = 5 * GB + held = self.submit(manager, "held", host_gb=20) + assert wait_until(lambda: held.waiting_for is not None) + assert manager.cancel(held.id) == "cancelled" + assert held.waiting_for is None + assert "waiting_for" not in held.summary() + assert held.id not in manager._host_needs diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index cc20e505..9b14329a 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -615,6 +615,12 @@ export interface paths { * refused (422) with the interval that fits; the default width is * never refused for length (#823). * + * An image's `media` is its measured `{kind, width, height, mode}`, + * read from the file header (#922) - not the width/height arguments + * the pipeline was asked for. `include_workflow=false` drops the + * embedded `metadata.workflow` (the whole definition, often 20 KB) and + * leaves the rest of `metadata` with `workflow_omitted: true`. + * * `name` may also be an 'asset:' reference, and then it is the input * asset of that name that is described rather than an output (#127). * The numbers here - duration, frame count, fps, sample rate - are @@ -2427,7 +2433,7 @@ export interface components { job: components["schemas"]["JobRef"] | null; /** * Media - * @description The probe of an audio or video file; null for an image. + * @description The probe of an audio or video file, or an image's kind, width, height and mode; null for another file. */ media: { [key: string]: unknown; @@ -2604,6 +2610,11 @@ export interface components { status: "queued" | "running" | "succeeded" | "failed" | "cancelled"; /** Traceback */ traceback: string | null; + /** + * Waiting For + * @description Why dispatch is holding this queued job though a card is free - 'waiting for host RAM: needs ~N GB, ~M GB free'; only while queued and held. It starts when a running job frees the RAM. + */ + waiting_for?: string; /** Warnings */ warnings: string[]; /** @@ -2812,6 +2823,11 @@ export interface components { * @enum {string} */ status: "queued" | "running" | "succeeded" | "failed" | "cancelled"; + /** + * Waiting For + * @description Why dispatch is holding this queued job though a card is free - 'waiting for host RAM: needs ~N GB, ~M GB free'; only while queued and held. It starts when a running job frees the RAM. + */ + waiting_for?: string; /** * Workflow * @description Null only on a row recorded before the workflow was stored. @@ -4735,6 +4751,7 @@ export interface operations { query?: { envelope?: boolean; interval_seconds?: number; + include_workflow?: boolean; workspace?: string | null; }; header?: never; diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index 1acca290..d0b00dfd 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -1449,7 +1449,7 @@ "type": "null" } ], - "description": "The probe of an audio or video file; null for an image.", + "description": "The probe of an audio or video file, or an image's kind, width, height and mode; null for another file.", "title": "Media" }, "metadata": { @@ -1874,6 +1874,11 @@ ], "title": "Traceback" }, + "waiting_for": { + "description": "Why dispatch is holding this queued job though a card is free - 'waiting for host RAM: needs ~N GB, ~M GB free'; only while queued and held. It starts when a running job frees the RAM.", + "title": "Waiting For", + "type": "string" + }, "warnings": { "items": { "type": "string" @@ -2409,6 +2414,11 @@ "title": "Status", "type": "string" }, + "waiting_for": { + "description": "Why dispatch is holding this queued job though a card is free - 'waiting for host RAM: needs ~N GB, ~M GB free'; only while queued and held. It starts when a running job frees the RAM.", + "title": "Waiting For", + "type": "string" + }, "workflow": { "anyOf": [ { @@ -6796,7 +6806,7 @@ }, "/api/gallery/{name}/metadata": { "get": { - "description": "Generation metadata embedded in a saved image ('workflow' inside\nit is the full definition the editor can reopen), plus the job that\nproduced the file when history remembers one, plus - for audio and\nvideo - what the file itself holds: duration, format and level,\nwhich is how an agent that cannot listen checks a track. Only an\nimage embeds 'metadata' this way - it is always null for audio and\nvideo, since neither format has a slot this writer uses; recover\nthe recipe from 'job' (GET /api/jobs/{id}/workflow) when one is\nknown, or from nothing when it isn't (a kept asset has no job).\n\n`envelope=true` adds the soundtrack's level second by second, which\nis what says *where* in a track something is - whether a shot is\nstill voiced at its last frame, how deep the hole at a seam goes.\nOpt-in: a ten-minute track is 600 numbers, and the default call has\nto stay small. `interval_seconds` (default 1.0, 0.05 to 3600) is\nthe bin width: 0.1 places a cut on a vocal onset. A track that\nwould need more than 2400 bins at a finer-than-default width is\nrefused (422) with the interval that fits; the default width is\nnever refused for length (#823).\n\n`name` may also be an 'asset:' reference, and then it is the input\nasset of that name that is described rather than an output (#127).\nThe numbers here - duration, frame count, fps, sample rate - are\nwhat decide whether a call will work at all, and for a file the\ncaller is about to *consume* they were previously unobtainable:\nthe only way to read a wav's length was to run a job that copied it\ninto the output directory. `job` is null for an asset with no\nrecorded provenance, and `source` says which of the two roots\nanswered.\n\nA kept asset (`keep_output`) is not always provenance-blind: when\nthe file it was kept from still had a job in history, `keep_output`\nrecorded `{job, run_id, version}` beside it, and this route reads it\nback the same way it reads a kept file's shots (#556). `run_id`/\n`version` are that source run's own - `job.id` is still the one to\npass `get_job_workflow`, since a pruned run leaves no `workflow.json`\nof its own to reopen. `inputs` is the source run's too (#902).", + "description": "Generation metadata embedded in a saved image ('workflow' inside\nit is the full definition the editor can reopen), plus the job that\nproduced the file when history remembers one, plus - for audio and\nvideo - what the file itself holds: duration, format and level,\nwhich is how an agent that cannot listen checks a track. Only an\nimage embeds 'metadata' this way - it is always null for audio and\nvideo, since neither format has a slot this writer uses; recover\nthe recipe from 'job' (GET /api/jobs/{id}/workflow) when one is\nknown, or from nothing when it isn't (a kept asset has no job).\n\n`envelope=true` adds the soundtrack's level second by second, which\nis what says *where* in a track something is - whether a shot is\nstill voiced at its last frame, how deep the hole at a seam goes.\nOpt-in: a ten-minute track is 600 numbers, and the default call has\nto stay small. `interval_seconds` (default 1.0, 0.05 to 3600) is\nthe bin width: 0.1 places a cut on a vocal onset. A track that\nwould need more than 2400 bins at a finer-than-default width is\nrefused (422) with the interval that fits; the default width is\nnever refused for length (#823).\n\nAn image's `media` is its measured `{kind, width, height, mode}`,\nread from the file header (#922) - not the width/height arguments\nthe pipeline was asked for. `include_workflow=false` drops the\nembedded `metadata.workflow` (the whole definition, often 20 KB) and\nleaves the rest of `metadata` with `workflow_omitted: true`.\n\n`name` may also be an 'asset:' reference, and then it is the input\nasset of that name that is described rather than an output (#127).\nThe numbers here - duration, frame count, fps, sample rate - are\nwhat decide whether a call will work at all, and for a file the\ncaller is about to *consume* they were previously unobtainable:\nthe only way to read a wav's length was to run a job that copied it\ninto the output directory. `job` is null for an asset with no\nrecorded provenance, and `source` says which of the two roots\nanswered.\n\nA kept asset (`keep_output`) is not always provenance-blind: when\nthe file it was kept from still had a job in history, `keep_output`\nrecorded `{job, run_id, version}` beside it, and this route reads it\nback the same way it reads a kept file's shots (#556). `run_id`/\n`version` are that source run's own - `job.id` is still the one to\npass `get_job_workflow`, since a pruned run leaves no `workflow.json`\nof its own to reopen. `inputs` is the source run's too (#902).", "operationId": "gallery_metadata_api_gallery__name__metadata_get", "parameters": [ { @@ -6828,6 +6838,16 @@ "type": "number" } }, + { + "in": "query", + "name": "include_workflow", + "required": false, + "schema": { + "default": true, + "title": "Include Workflow", + "type": "boolean" + } + }, { "in": "query", "name": "workspace", diff --git a/ui/src/lib/job/JobHeader.svelte b/ui/src/lib/job/JobHeader.svelte index c926ce13..25cd1519 100644 --- a/ui/src/lib/job/JobHeader.svelte +++ b/ui/src/lib/job/JobHeader.svelte @@ -130,6 +130,13 @@ >#{job.queue_position + 1} in line {/if} + {#if job.waiting_for} + {job.waiting_for} + {/if} {#if seed !== undefined} Date: Thu, 15 Oct 2026 07:06:27 -0500 Subject: [PATCH 172/204] docs(proposals): #866 - design record for estimate scaling, todo entry Co-Authored-By: Claude Opus 5.5 --- .../complete/estimate-scaling-complete.md | 115 ++++++++++++++++++ docs/proposals/todo.md | 11 ++ 2 files changed, 126 insertions(+) create mode 100644 docs/proposals/complete/estimate-scaling-complete.md diff --git a/docs/proposals/complete/estimate-scaling-complete.md b/docs/proposals/complete/estimate-scaling-complete.md new file mode 100644 index 00000000..6196db5f --- /dev/null +++ b/docs/proposals/complete/estimate-scaling-complete.md @@ -0,0 +1,115 @@ +# Scale the curated estimate instead of dropping or repeating it (#866) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-15). Plan v4, approved by Don as "build smaller" on 2026-10-14 (v1 → v2: +Q1-Q5 and the fingerprint ruling. v2 → v3: the stage 0 re-plan. v3 → v4: two +fixes to stage A's acceptance). Four stages: #905 (0), #906 (A), #907 (B) and #908 (C), +all verified on lem. + +## The report + +`POST /api/validate` quoted ~42 min for the ep 6 dialogue-short no matter what +changed. The H3 quality matrix (lem, workspace `h3-quality`, 15 rungs of one +Ref2VA shot, 124 frames, 4 refs) measured the same shot at 3.5 to 44.2 min, +depending on canvas, weights, LoRA, steps, attention and cache. Agents quote +`plan.estimate` before every run, so the quote was off by up to 12x in either +direction. It was the tenth estimate bug in this class (#267, #255, #301, +#589, #590, #593, #796, #821, #851). + +The code check (plan v1, at a27ad5aa) found that the flat figure had three +sources, not one: +- **Catalog path.** A move of steps, canvas or frames off the defaults made + `_own_price` return `basis: unknown`. It was not scaled. +- **Default bucket.** Runs that differed only in weights, LoRA, attention or + cache landed in it, because those were not drivers. +- **Inherited path.** `_scale_past_template` scaled by pipeline-step count + and frames only, not by `num_inference_steps` or the canvas. + +## What was built + +- **Stage 0 (#905): the fingerprint and step cache are keyed only by what + changes the output.** + - `cost_drivers`, `vram_estimate` and the other documentation and cost + metadata no longer reach the plan fingerprint or the step cache keys. The + exclusion list is pinned by a test. + - Integral floats are canonicalized in the fingerprint, so `12.0` and `12` + digest alike. That was the inline vs. by-name mismatch. + - Seed stays out of the fingerprint (the #85 contract). The step cache keys + on it. + - The step cache keeps `subfolder` and `file_base_name`, because a hit + replays files to their recorded paths. +- **Stage A (#906): the catalog figure scales.** + - `scale_curated` gives + `minutes = curated × steps/steps₀ × (w·h)/(w₀·h₀) × frames/frames₀` on the + four generic driver names (`num_inference_steps`, `width`, `height`, + `num_frames`). `num_frames` is summed across a list's entries. + - The result is `basis: "derived"`, and `scaled` names each axis that moved. + - It multiplies with list-length repricing (`_repriced`). `_tempered` blends + toward the scaled figure. + - A non-numeric driver value still gives `unknown`. + - The rule "a `cost` entry is measured at the workflow's variable defaults" + is written in the schema and WORKFLOW_GUIDE. + - dialogue-short's `cost` was re-priced from 42 to 21.2 for its 960x544 + defaults. + - The 30% accuracy bar is `TestScalingAccuracy`: B4→B1 16.5 vs 15.1, B4→B5 + 13.0 vs 11.8, B3→L5 32.3 vs 29.5. + - The UI's `derived` wording shows `scaled`. +- **Stage B (#907): unscalable drivers.** + - Any other declared scalar driver that moves off its default keeps the + (scaled) figure, sets `low_confidence` and is listed in `differs`. This + holds for every template, with no engine allow-list. + - The H3 templates add `weights_dtype`, `lora_model_name`, + `attention_backend` and `cache` to `cost_drivers`, so an int8 or native + attention run no longer pools with the default bucket. LoRA buckets by its + exact name. + - Thanks to stage 0, the new drivers leave H3 fingerprints unchanged. +- **Stage C (#908): the inherited path scales the same way.** + - `_scale_past_template` reuses `scale_curated`'s steps and canvas ratios, + so there is one owner and inline and template quotes agree (C-F484: + within 2%). + - `inherited_differences` compares attention, cache and LoRA too, through + `replace_variables` (no second owner of substitution). + - A reference the definition doesn't declare compares as null. + - Numbers compare by value: `same_number` moved to `dw/variables.py` and is + shared with the fingerprint. + +## What was cut or deferred, and why + +- **Compile as a cost driver.** No H3 template exposes a `compile` variable, + and a driver must be a declared variable. It comes back if a template + exposes compile. +- **A learned fixed + passes × s/pass fit.** Deferred by Don. Plain + proportional scaling over-quotes by about 9-10% on the matrix pairs, + because it ignores load, encode and decode time. It is also slightly + under-superlinear in pixels (2.27x measured for 1.98x pixels). Both are + inside the 30% bar. A miss past 30% is the trigger to build the fit. +- **`passes = steps − 1`.** That is H3 scheduler knowledge, so it can't live + in the engine. The engine uses a plain step ratio (Q2). +- **Price precedence for a saved copy with its own `cost` and no runs** + (catalog vs. inherited) is unchanged (v3 review). +- **The step cache across identities.** A renamed or inline copy can't hit + the cache: it is per card and cleared on an identity switch. That is a + memory-policy change, filed as #912 (idea). +- **The validate probe ignores the card's warm identity**, so it can report + `cached_steps` that the run's identity switch then clears. Filed as #911 + (bug). + +## Bounces per stage + +| Stage | Verify bounces | Architecture bounces | Notes | +|---|---|---|---| +| #905 (0) | 1 | 0 | The bounce showed that v2's "a metadata-only copy hits the step cache" can't hold under the cache's per-identity scope. Re-planned as v3 (in-place edit only), plus the integral-float fix. | +| #906 (A) | 0 | 0 | | +| #907 (B) | 0 | 0 | | +| #908 (C) | 3 | 2 | Architecture: a second owner of the scaling rule, then a second owner of `variable:` substitution. Both were folded into the single owners. Verify: C-F485 `differs: ["LoRA"]` at the defaults, three times. Parked after the 4th hand-off; Don ruled that the plan stands. The causes were an undeclared null-default variable and then `lora_scale` 1.0 vs 1 after a client's JSON round trip. | + +Cost per stage: the stage comments record no `usage:` figures, so none is +given here. The plan estimated ~$2 / $5 / $3 / $4. + +## Rules this feature added + +- A `cost` entry is measured at the workflow's variable defaults. +- The plan fingerprint and the step cache are keyed only by what can change + the output (pinned in `docs/ARCHITECTURE.md` and its tests). +- A number is its value, not its spelling, in the fingerprint and in the + inherited-cost comparison. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index b4f207f7..c7a3b3e0 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -37,6 +37,17 @@ remaining deferred fix is recorded in ## Shipped since the ranking +- **`plan.estimate` scales the curated figure** (#866, stages #905-#908), + shipped 2026-10-15. A move of `num_inference_steps`, `width`/`height` or + `num_frames` off the defaults now scales the catalog or inherited figure + in proportion (`basis: "derived"`, `scaled` names the axes) instead of + quoting `unknown` or the flat figure. Any other declared driver (H3 + weights, LoRA, attention, cache) keeps the figure with `low_confidence` + and `differs`. The fingerprint and step cache are keyed only by what + changes the output. Record, including what was cut (compile as a driver, + the learned fixed + per-pass fit) and follow-ups #911 and #912: + `complete/estimate-scaling-complete.md`. + - **H3 adapter check reads the LoRA catalog first** (#870, stage #888), shipped 2026-10-14. On the server, validate and admission look up the LoRA in the shared catalog: a `proven` entry whose `workflow` puts it on From cd9c8f1b497e711593d1c587107ddf337c32b7a1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 07:18:49 -0500 Subject: [PATCH 173/204] docs(architecture): #910 - name host_ram_wait as an owner of the dispatch row Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d8e4328e..4649da13 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -159,7 +159,7 @@ one to open. | App factory | `dw/server/app.py`: `create_app` | `create_app` builds the JobManager and `app.state`, installs the middleware and registers the routes, and all state lives in the JobManager. | — | | Routers | `dw/server/routes/*.py`, `dw/server/routes/__init__.py`: `ROUTERS`, `include_routers`, `include_file_routes` | Routers register in `ROUTERS` order, because a greedy `{name:path}` route must come after its specific siblings. The files router comes later, after `/mcp`. | `tests/test_server_downloads.py::test_download_workflow_sets_content_disposition_attachment` | | Job queue | `dw/server/jobs.py`: `JobManager`; `dw/server/job_results.py`: `consume_results` | A dispatcher thread hands queued jobs, in queue order, to the pool's workers (`Worker pool dispatch`); each worker runs one job at a time, and `consume_results` folds that worker's replies into the job, naming files relative to the job's own `output_dir`. A job carries its own `output_dir`, `asset_dir` and `workflow_dir`, so it stays in its workspace. | `tests/test_server_workspaces.py::TestRunning::test_rerun_stays_in_the_workspace_it_ran_in` (FIFO: —) | -| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, a job that fits a free card is also held while its projected host-RAM peak exceeds `MemAvailable` less what running jobs have yet to add plus the idle worker's own RSS (a job alone is never held, no figure or no RAM reading means no gate, a smaller job behind a held one starts, and the dispatcher re-checks every 10 s; #869), and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_the_host_cannot_hold_beside_the_running_one_is_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_alone_is_never_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_smaller_job_starts_past_a_held_one` | +| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`, `JobManager._host_ram_wait`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`, `host_ram_wait`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, a job that fits a free card is also held while its projected host-RAM peak exceeds `MemAvailable` less what running jobs have yet to add plus the idle worker's own RSS (a job alone is never held, no figure or no RAM reading means no gate, a smaller job behind a held one starts, and the dispatcher re-checks every 10 s; #869), and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_the_host_cannot_hold_beside_the_running_one_is_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_alone_is_never_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_smaller_job_starts_past_a_held_one` | | Card affinity | `dw/server/pool.py`: `choose_slot`; `dw/server/jobs.py`: `JobManager.route`; `dw/worker_protocol.py`: `workflow_identity` | Among the free cards a job fits it goes to the rerun's original card, else the card whose worker last ran its workflow identity (`workflow_identity`, the one rule the worker caches by too), else one with no worker running, else the first in `--devices` order; a cache probe and `plan.estimate` (`priced_for`) ask about the card `route` names, and memory is read and cleared per card. | `tests/test_worker_pool.py::TestIdentityAffinity::test_a_second_run_lands_on_the_card_that_ran_the_workflow_first`, `tests/test_worker_pool.py::TestRerunAffinity::test_a_rerun_lands_on_the_original_card_when_it_is_free`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle` | | Per-card memory and cache | `dw/server/worker_memory.py`: `memory_status`, `clear_memory`, `probe_cache` | Each card keeps its own last memory reading; an idle card's worker is asked live, a busy one answers with the run's last report and says why it is not live, and a clear leaves a card running a job alone. Every request takes the card's lock with a bounded wait, and a card's lock is never taken while the JobManager's lock is held. | `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_server.py::test_memory_says_why_a_reading_is_not_the_worker_s` | | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | From f3aab7eb8ff798808e9b5c1383b40d884b5defaa Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 08:05:27 -0500 Subject: [PATCH 174/204] fix(validate): #926 - warn on an unbraced type name handed to a *_dtype key Co-Authored-By: Claude Sonnet 5.5 --- dw/argument_warnings.py | 47 +++++++++++++++++++++++++++++ tests/test_unbraced_type_warning.py | 39 ++++++++++++++++++++++++ 2 files changed, 86 insertions(+) create mode 100644 tests/test_unbraced_type_warning.py diff --git a/dw/argument_warnings.py b/dw/argument_warnings.py index 225f98ec..20110962 100644 --- a/dw/argument_warnings.py +++ b/dw/argument_warnings.py @@ -6,7 +6,11 @@ Best effort and advisory: nothing here refuses a workflow. """ +import diffusers + from . import references +from .arguments import is_escaped +from .type_helpers import NON_TYPE_KEYS from .introspection import CLASS_NAME_PATTERN, unknown_call_arguments from .variables import undeclared_variable_references @@ -148,6 +152,46 @@ def _inert_match_levels_dbfs_warnings(step, command, arguments): ] +def _unbraced_type_warnings(node, values, path=""): + """A `*_type`/`*_dtype` key loads its value as a diffusers type unless it + is written `{name}`, so a variable handed to one with no braces + (`weights_dtype: "int8"`) passed validation and failed after the model + loaded with `module diffusers has no attribute int8` (#926). Only a bare + name is checked: a dotted one is the trust gate's, and a name diffusers + does define is a real type.""" + found = [] + if isinstance(node, dict): + for key, value in node.items(): + here = f"{path}.{key}" if path else key + if ( + isinstance(key, str) + and (key == "dtype" or key.endswith(("_dtype", "_type"))) + and key not in NON_TYPE_KEYS + and references.is_ref(references.VARIABLE, value) + ): + name = references.ref_name(references.VARIABLE, value) + given = values.get(name) + if ( + isinstance(given, str) + and given + and "." not in given + and not is_escaped(given) + and not hasattr(diffusers, given) + ): + found.append( + f"{here}: variable '{name}' is '{given}', but a " + f"'{key}' value loads as a diffusers type and " + f"'{given}' is not one; a type or quantization name " + f'that is only a name needs braces: "{{{given}}}"' + ) + else: + found.extend(_unbraced_type_warnings(value, values, here)) + elif isinstance(node, list): + for index, item in enumerate(node): + found.extend(_unbraced_type_warnings(item, values, f"{path}[{index}]")) + return found + + def workflow_argument_warnings(workflow_definition, arguments=None): """Best-effort pre-load check of a workflow's arguments. @@ -176,6 +220,9 @@ def workflow_argument_warnings(workflow_definition, arguments=None): f"{path}: '{references.make_ref(references.VARIABLE, name)}' names no declared variable{hint}; " f"declared: {', '.join(declared) or ''}" ) + warnings.extend( + _unbraced_type_warnings(workflow_definition.get("steps", []), values, "steps") + ) for step in workflow_definition.get("steps", []): task = step.get("task") if task and isinstance(task.get("arguments"), dict): diff --git a/tests/test_unbraced_type_warning.py b/tests/test_unbraced_type_warning.py new file mode 100644 index 00000000..5d8cce66 --- /dev/null +++ b/tests/test_unbraced_type_warning.py @@ -0,0 +1,39 @@ +"""#926: an unbraced type name handed to a `*_dtype` key warns at validate.""" + +from dw.argument_warnings import workflow_argument_warnings + + +def _definition(default): + return { + "variables": {"weights_dtype": default}, + "steps": [ + { + "name": "s", + "pipeline": { + "configuration": {"component_type": "{x}"}, + "arguments": {"weights_dtype": "variable:weights_dtype"}, + }, + } + ], + } + + +def test_unbraced_name_warns_with_the_braced_spelling(): + warnings = workflow_argument_warnings( + _definition("{int4}"), {"weights_dtype": "int8"} + ) + assert any("{int8}" in w and "weights_dtype" in w for w in warnings) + + +def test_braced_name_is_quiet(): + assert ( + workflow_argument_warnings(_definition("{int4}"), {"weights_dtype": "{int8}"}) + == [] + ) + + +def test_a_real_diffusers_type_is_quiet(): + warnings = workflow_argument_warnings( + _definition("{int4}"), {"weights_dtype": "DiffusionPipeline"} + ) + assert not any("needs braces" in w for w in warnings) From 215688db4e0bd8c599a2a241b04c11f16e274414 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 08:10:16 -0500 Subject: [PATCH 175/204] docs(proposals): #869 - design record for the host-RAM dispatch gate, todo entry Also drop series-episodes' "One int8 job at a time" (the server now holds a second heavy job) and mark the multi-gpu record's deferred gate as shipped. Co-Authored-By: Claude Opus 5.5 --- .../host-ram-dispatch-gate-complete.md | 120 ++++++++++++++++++ .../complete/multi-gpu-workers-complete.md | 3 +- docs/proposals/todo.md | 7 + .../series-episodes/references/cinema-path.md | 5 +- 4 files changed, 132 insertions(+), 3 deletions(-) create mode 100644 docs/proposals/complete/host-ram-dispatch-gate-complete.md diff --git a/docs/proposals/complete/host-ram-dispatch-gate-complete.md b/docs/proposals/complete/host-ram-dispatch-gate-complete.md new file mode 100644 index 00000000..dde69f65 --- /dev/null +++ b/docs/proposals/complete/host-ram-dispatch-gate-complete.md @@ -0,0 +1,120 @@ +# Hold a queued job for host RAM at dispatch (#869) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-15). Plan v2, approved by Don as "build smaller" on 2026-10-14 (v1 → +v2: Don's answers folded in as decisions D1-D3, no scope change). Two stages: +#909 (A) and #910 (B), both verified on lem. Part 2 of the report (the wrong +job died) was split out and shipped as #881. + +## The report + +On 2026-10-13, lem (2x RTX 3090, 125 GB host RAM) dispatched two int8 H3 +Ref2VA jobs to its two cards at once. Each holds ~62-63 GB RSS after load, and +two don't fit. The earlier job was SIGKILLed by the OOM killer and the later +one died with SIGTERM. The dispatcher (#462) routed on VRAM need only, and +`host_memory_projection` produced validate warnings only. Three places told +agents to work around it by hand: the workspaces guide ("wait between +variants"), the `minimax-h3` quality reference ("One int8 job at a time") and +`tools/ladder.py` waiting for lem to be idle. + +The code sweep (plan v1) found: +- **History had no (pipeline, weights dtype) key.** The dtype is usually a + template variable feeding a `quantization_config`, and history grouped by + workflow name. So the plan added a *memory key* computed from the admitted + definition. +- **`host_memory_projection` projects list runs only.** On its own it would + never have figured an H3 shot. +- **"Free" can't be MemAvailable alone.** A job that is still loading hasn't + reached its peak, so each running job's remaining growth is charged. +- **A `warning` event sticks to `job.warnings` for good.** A queue wait is a + moment, not a fact about the result, so it gets its own event and field. +- **The dispatcher only woke on submit, finish, hold release or shutdown.** A + held job needs a timed re-check. + +## What was built + +- **Stage A (#909): memory key and projected peak, nothing gated.** + - A `memory_key TEXT` history column (migrated with `ALTER TABLE`; older + rows are NULL and count as no data), written by `record()`. The key is the + sorted `pipeline_identity` of each pipeline step plus every + `quantization_config`, `offload` and `group_offload` value, with the + caller's arguments substituted through the single owner of `variable:` + substitution. Renamed and inline copies of one template share a key; + `weights_dtype` int4 and int8 don't. + - `dw/server/host_ram_need.py: projected_peak` returns `observed` (the max + `host_memory_job_peak_rss_mb` over successful same-key runs, D2), else + `fit` (`host_memory_projection.projected_mb`, a list run), else None. + - `host_memory_projection` now shares one `_projection` between + `projected_mb` and `host_memory_warnings`. The warning text and its 0.9 × + total-RAM ceiling are unchanged. + - `Admission.host_ram_need`, threaded through submit, rerun and enhance. + - Validate's `plan.host_memory: {projected_mb, basis, runs}`, absent with + no figure. +- **Stage B (#910): the gate.** + - `_host_needs` beside `_needs` in `JobManager`. The rule lives in + `dw/server/pool.py: host_ram_wait`: `available = MemAvailable − Σ running + max(0, projected_peak − worker RSS now)`, and a job is held while its + peak exceeds `available` plus the RSS of the idle worker it would reuse. + - Backfill: a held job keeps its place and smaller jobs behind it may start. + - Escapes: never held with no other job running, never held without a + figure, and no gate when `host_memory_stats()` can't read MemAvailable. + - While any job is held, the dispatcher re-checks every 10 s + (`HOST_RAM_RECHECK_SECONDS`), so memory freed outside dw is seen. + - A `queue_wait` event (`reason: "host_ram"`, "waiting for host RAM: needs + ~N GB, ~M GB free"), once per hold, never a `warning` (D1). + - `waiting_for` on the job summary and detail while queued and held: REST, + regenerated `openapi.json` and `api-schema.ts`, MCP `get_job`, + `list_jobs` and `wait_for_job`, and the UI's `JobHeader.svelte`. + - Cancel clears it; `move` wakes the dispatcher. + - Docs: SERVER.md states the rule and its escapes; WORKSPACES.md and the + `minimax-h3` quality reference drop the wait-by-hand advice; MCP.md and + the ARCHITECTURE.md dispatch row name the owners. + +On lem the verify held a second int8 H3 job ("needs ~87 GB, ~57 GB free") +with cuda:1 idle and started it the instant the first finished, while two +int4 jobs (~47 GB observed each) started at once. + +## What was cut or deferred, and why + +- **Refusal at submit.** Never: host RAM is this machine's state, not the + caller's choice, and #243's warn-not-refuse rule stands. +- **A "would wait now" prediction in validate's plan.** It guesses at a + queue that changes before submit. *Back when:* a field report of agents + submitting blind and being surprised by `waiting_for`. +- **A new job status.** Don's answer: the job stays `queued`, and + `waiting_for` says why. +- **`num_frames`, resolution and reference count in the memory key** (D3). + *Back when:* under-holding shows up from a key collision across sizes. + #881's OOM ranking is the backstop. +- **Pipelines inside a composed child workflow** are not in the parent's + key (stage A deviation). +- **No plugin version bump.** The plugin version is pinned to the engine's + and the release script bumps both. + +## Bounces per stage + +| Stage | Verify bounces | Architecture bounces | Notes | +|---|---|---|---| +| #909 (A) | 1 | 1 | Architecture (two findings in one bounce): a second owner of `variable:` substitution in `host_ram_need`, folded into the single owner, and the new module had no ARCHITECTURE.md row. Verify: C-F488 expected a projection warning that #243's 0.9 ceiling never emits at ~61.6 GB on lem. No code change; Don approved the case amendment. | +| #910 (B) | 0 | 1 | Architecture: the ARCHITECTURE.md dispatch row didn't name `pool.py: host_ram_wait` as the owner of the hold rule. | + +C-F495 (single worker or MPS, dispatch unchanged) was not run on lem, which +has two workers, so it stays `pending: #910`. + +Cost per stage: the stage comments record no `usage:` figures, so none is +given here. The plan estimated $4-6 (A) and $5-6 (B). + +## Found on the way + +- `validate_workflow` accepted `weights_dtype: "int8"` unbraced, and the run + failed at load. Fixed as #926 (validate warns on an unbraced type name + under a `*_dtype` key). + +## Rules this feature added + +- Dispatch may delay a job for host RAM but never refuses one; a job with no + other job running is never held. +- A queue wait is a `queue_wait` event and a live `waiting_for`, never a + `warning`. +- The observed host-RAM figure is the max over same-memory-key successful + runs on this machine. diff --git a/docs/proposals/complete/multi-gpu-workers-complete.md b/docs/proposals/complete/multi-gpu-workers-complete.md index 37e4c4e1..5e44eb2e 100644 --- a/docs/proposals/complete/multi-gpu-workers-complete.md +++ b/docs/proposals/complete/multi-gpu-workers-complete.md @@ -106,7 +106,8 @@ Every shape change is additive: none is a `breaking-change`. ## Deferred, and what brings each back - **Host-RAM admission gate** (Q3). *Back when:* a field report of a - cross-job OOM kill. + cross-job OOM kill. That came as #869, and the gate shipped (dispatch + delays, never refuses): `host-ram-dispatch-gate-complete.md`. - **An `overlapped` flag on jobs, excluded from observed cost** (Q4). *Back when:* the contention skew in cold medians is measured. - **`list_workflows`' cost listing per card.** It still quotes the diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index c7a3b3e0..72bcfc09 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -37,6 +37,13 @@ remaining deferred fix is recorded in ## Shipped since the ranking +- **Dispatch holds a queued job for host RAM** (#869, stages #909-#910), + shipped 2026-10-15. A job waits `queued`, with a `queue_wait` event and a + live `waiting_for`, while its projected peak (the max observed for its + memory key, else the list fit) won't fit beside the running jobs. It is + never refused, and never held alone. Record, including what was deferred + (a validate "would wait" prediction, frame count in the key): + `complete/host-ram-dispatch-gate-complete.md`. - **`plan.estimate` scales the curated figure** (#866, stages #905-#908), shipped 2026-10-15. A move of `num_inference_steps`, `width`/`height` or `num_frames` off the defaults now scales the catalog or inherited figure diff --git a/plugins/dw/skills/series-episodes/references/cinema-path.md b/plugins/dw/skills/series-episodes/references/cinema-path.md index 2ae5ca20..b5126d55 100644 --- a/plugins/dw/skills/series-episodes/references/cinema-path.md +++ b/plugins/dw/skills/series-episodes/references/cinema-path.md @@ -8,8 +8,9 @@ measurements), the `first_block` lever and the base-model prompt rules - read it, not restated here. What a multi-shot episode adds: - Wides as a 12-step draft, then a 30-step final; close-ups at 12 steps. - One int8 job at a time. A five-shot episode ran ~156 GPU-min with three - rejected drafts. + Submit the shots freely: the server holds a second int8 job for host RAM + (`waiting_for`). A five-shot episode ran ~156 GPU-min with three rejected + drafts. - Place every subject explicitly in every shot and name the set dressing, or the base model adds a person and each shot draws its own chair. - Four references at 1344x768 cap a shot at 124 frames: write tags to 124 or From 56245057c3981fd1cddada284589d27ee4626b2e Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 08:48:32 -0500 Subject: [PATCH 176/204] fix(validate): #926 - move the unbraced-type warning into type_references, reusing its key test, walk and resolver Co-Authored-By: Claude Sonnet 5.5 --- dw/argument_warnings.py | 47 --------------------------- dw/type_references.py | 50 ++++++++++++++++++++++++----- dw/validation.py | 12 ++++++- tests/test_unbraced_type_warning.py | 38 +++++++++++++--------- tests/test_validation.py | 1 + 5 files changed, 76 insertions(+), 72 deletions(-) diff --git a/dw/argument_warnings.py b/dw/argument_warnings.py index 20110962..225f98ec 100644 --- a/dw/argument_warnings.py +++ b/dw/argument_warnings.py @@ -6,11 +6,7 @@ Best effort and advisory: nothing here refuses a workflow. """ -import diffusers - from . import references -from .arguments import is_escaped -from .type_helpers import NON_TYPE_KEYS from .introspection import CLASS_NAME_PATTERN, unknown_call_arguments from .variables import undeclared_variable_references @@ -152,46 +148,6 @@ def _inert_match_levels_dbfs_warnings(step, command, arguments): ] -def _unbraced_type_warnings(node, values, path=""): - """A `*_type`/`*_dtype` key loads its value as a diffusers type unless it - is written `{name}`, so a variable handed to one with no braces - (`weights_dtype: "int8"`) passed validation and failed after the model - loaded with `module diffusers has no attribute int8` (#926). Only a bare - name is checked: a dotted one is the trust gate's, and a name diffusers - does define is a real type.""" - found = [] - if isinstance(node, dict): - for key, value in node.items(): - here = f"{path}.{key}" if path else key - if ( - isinstance(key, str) - and (key == "dtype" or key.endswith(("_dtype", "_type"))) - and key not in NON_TYPE_KEYS - and references.is_ref(references.VARIABLE, value) - ): - name = references.ref_name(references.VARIABLE, value) - given = values.get(name) - if ( - isinstance(given, str) - and given - and "." not in given - and not is_escaped(given) - and not hasattr(diffusers, given) - ): - found.append( - f"{here}: variable '{name}' is '{given}', but a " - f"'{key}' value loads as a diffusers type and " - f"'{given}' is not one; a type or quantization name " - f'that is only a name needs braces: "{{{given}}}"' - ) - else: - found.extend(_unbraced_type_warnings(value, values, here)) - elif isinstance(node, list): - for index, item in enumerate(node): - found.extend(_unbraced_type_warnings(item, values, f"{path}[{index}]")) - return found - - def workflow_argument_warnings(workflow_definition, arguments=None): """Best-effort pre-load check of a workflow's arguments. @@ -220,9 +176,6 @@ def workflow_argument_warnings(workflow_definition, arguments=None): f"{path}: '{references.make_ref(references.VARIABLE, name)}' names no declared variable{hint}; " f"declared: {', '.join(declared) or ''}" ) - warnings.extend( - _unbraced_type_warnings(workflow_definition.get("steps", []), values, "steps") - ) for step in workflow_definition.get("steps", []): task = step.get("task") if task and isinstance(task.get("arguments"), dict): diff --git a/dw/type_references.py b/dw/type_references.py index 337f2e62..f5502574 100644 --- a/dw/type_references.py +++ b/dw/type_references.py @@ -83,12 +83,18 @@ def _is_type_key(key): ) -def _loose_type_reference_error(key, value, path): +def _loose_type_reference_error(key, value, path, warnings=None): """Any other '*_type' / '*_dtype' / 'dtype' value the run loads as a type (a pipeline's torch_dtype, say), put to the same untrusted gate. Only the - gate's refusal is reported: a name that merely fails to resolve is left + gate's refusal is an error: a name that merely fails to resolve is left to the run, since realize_args reads some of these keys as something - other than a type.""" + other than a type. + + That rule stops short of a bare, undotted name (#926): a type or + quantization name written without braces (`weights_dtype: "int8"`) loads + as a diffusers attribute and dies after the model has loaded, so one that + fails to resolve is reported to `warnings` (when given) with its braced + spelling - a warning, since the key may still be read as something else.""" if not isinstance(value, str) or not _DOTTED_NAME_PATTERN.match(value): return None @@ -97,7 +103,17 @@ def _loose_type_reference_error(key, value, path): except UntrustedWorkflowError as e: return {"path": path, "message": str(e)} except (ImportError, AttributeError, ValueError): - pass + if warnings is not None and "." not in value: + warnings.append( + { + "path": path, + "message": ( + f"{key} {value!r} is not a diffusers type, and a " + f"'{key}' value loads as one; a type or quantization " + f'name that is only a name needs braces: "{{{value}}}"' + ), + } + ) return None @@ -116,7 +132,7 @@ def _constant_reference_error(value, path): return None -def _walk_type_references(node, path, errors): +def _walk_type_references(node, path, errors, warnings=None): # A {media_type, location} dict is loaded as media, and its media_type # names a kind rather than a type - realize_args never reads it as one if isinstance(node, dict) and not is_media_reference(node): @@ -124,20 +140,20 @@ def _walk_type_references(node, path, errors): if k in _TYPE_REFERENCE_KEYS: error = _type_reference_error(k, v, path + (k,)) elif _is_type_key(k): - error = _loose_type_reference_error(k, v, path + (k,)) + error = _loose_type_reference_error(k, v, path + (k,), warnings) else: error = _constant_reference_error(v, path + (k,)) if error is not None: errors.append(error) else: - _walk_type_references(v, path + (k,), errors) + _walk_type_references(v, path + (k,), errors, warnings) elif isinstance(node, list): for i, item in enumerate(node): error = _constant_reference_error(item, path + (i,)) if error is not None: errors.append(error) else: - _walk_type_references(item, path + (i,), errors) + _walk_type_references(item, path + (i,), errors, warnings) def component_type_errors(workflow_definition, source_indices=None): @@ -180,6 +196,24 @@ def component_type_errors(workflow_definition, source_indices=None): return errors +def unbraced_type_warnings(workflow_definition, source_indices=None): + """Every bare name handed to a `*_type`/`*_dtype`/`dtype` key that is not + a diffusers type, as warning strings (#926) - found by the same walk and + resolver `component_type_errors` uses, on the substituted definition, so a + value a `variable:` supplied is covered.""" + found = [] + for _, step, _, source, where in references.iter_steps( + workflow_definition.get("steps"), source_indices + ): + entries = [] + _walk_type_references(step, (), [], entries) + found.extend( + f"{render_path(('steps', source) + e['path'])}: {e['message']}{where}" + for e in entries + ) + return found + + def component_name_errors(workflow_definition, source_indices=None): """Every name under a step's `pipeline.configuration.components` that the named component_type's constructor does not register, as diff --git a/dw/validation.py b/dw/validation.py index 1d6fbc4f..06d69260 100644 --- a/dw/validation.py +++ b/dw/validation.py @@ -64,7 +64,11 @@ select_errors, ) from .task_domains import task_argument_errors -from .type_references import component_name_errors, component_type_errors +from .type_references import ( + component_name_errors, + component_type_errors, + unbraced_type_warnings, +) from .variable_constraints import ( ConstraintReferenceError, constraint_errors, @@ -586,6 +590,12 @@ def _null_variable_argument_warnings(context): "workflow_argument_warnings", lambda c: workflow_argument_warnings(c.definition, c.arguments), ), + # A type or quantization name written without braces, which the run loads + # as a diffusers attribute and fails on (dw/type_references.py, #926) + Check( + "unbraced_type_warnings", + lambda c: unbraced_type_warnings(c.expanded, c.source_indices), + ), # A value a declared constraint will round up - the silent half of #96: # the run changed the caller's frame count and only the server's log # said so diff --git a/tests/test_unbraced_type_warning.py b/tests/test_unbraced_type_warning.py index 5d8cce66..005651bc 100644 --- a/tests/test_unbraced_type_warning.py +++ b/tests/test_unbraced_type_warning.py @@ -1,17 +1,17 @@ """#926: an unbraced type name handed to a `*_dtype` key warns at validate.""" -from dw.argument_warnings import workflow_argument_warnings +from dw.type_references import unbraced_type_warnings +from dw.validation import WARNING_CHECKS -def _definition(default): +def _definition(value, extra=None): return { - "variables": {"weights_dtype": default}, "steps": [ { "name": "s", "pipeline": { "configuration": {"component_type": "{x}"}, - "arguments": {"weights_dtype": "variable:weights_dtype"}, + "arguments": {"weights_dtype": value, **(extra or {})}, }, } ], @@ -19,21 +19,27 @@ def _definition(default): def test_unbraced_name_warns_with_the_braced_spelling(): - warnings = workflow_argument_warnings( - _definition("{int4}"), {"weights_dtype": "int8"} - ) - assert any("{int8}" in w and "weights_dtype" in w for w in warnings) + warnings = unbraced_type_warnings(_definition("int8")) + assert len(warnings) == 1 + assert "{int8}" in warnings[0] and "weights_dtype" in warnings[0] def test_braced_name_is_quiet(): - assert ( - workflow_argument_warnings(_definition("{int4}"), {"weights_dtype": "{int8}"}) - == [] - ) + assert unbraced_type_warnings(_definition("{int8}")) == [] def test_a_real_diffusers_type_is_quiet(): - warnings = workflow_argument_warnings( - _definition("{int4}"), {"weights_dtype": "DiffusionPipeline"} - ) - assert not any("needs braces" in w for w in warnings) + assert unbraced_type_warnings(_definition("DiffusionPipeline")) == [] + + +def test_dotted_name_is_left_to_the_trust_gate(): + assert unbraced_type_warnings(_definition("os.path")) == [] + + +def test_media_reference_media_type_is_quiet(): + media = {"media_type": "image", "location": "x.png"} + assert unbraced_type_warnings(_definition("{int8}", {"image": media})) == [] + + +def test_registered_as_a_warning_check(): + assert "unbraced_type_warnings" in [c.name for c in WARNING_CHECKS] diff --git a/tests/test_validation.py b/tests/test_validation.py index 02b2c36f..b55ddb58 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -300,6 +300,7 @@ def test_a_childs_pathless_finding_is_placed_at_the_sub_workflow_step( WARNING_ORDER = [ "workflow_argument_warnings", + "unbraced_type_warnings", "constraint_warnings", "entry_field_warnings", "unseeded_cache_warnings", From f08cc3c8a6caa9727ee3a9e7c00f7ce981b5149c Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 08:53:41 -0500 Subject: [PATCH 177/204] feat(ui): enhance job status display with running jobs count and dropdown --- ui/src/App.svelte | 39 ++++++++++++++++++++++++++++++++------- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/ui/src/App.svelte b/ui/src/App.svelte index fdb6ed6c..13bfe6c6 100644 --- a/ui/src/App.svelte +++ b/ui/src/App.svelte @@ -6,6 +6,7 @@ Bell, BellOff, Braces, + ChevronDown, KeyRound, Menu, Moon, @@ -18,7 +19,13 @@ import { isEditable } from './lib/editable' import { notify } from './lib/toast' import { wsHref } from './lib/routes' - import { endedTitle, finishedText, nextWatch, tabTitle } from './lib/jobWatch' + import { + endedTitle, + finishedText, + nextWatch, + runningIds, + tabTitle, + } from './lib/jobWatch' import { alertFinished, alertMode, @@ -58,6 +65,14 @@ let statusTrigger = $state(null) let tokenTrigger = $state(null) const currentJob = $derived(health?.current_job ?? null) + // One job per card: with more than one running, the chip cannot link to + // them all, so it joins the status trigger and the popover lists each + const running = $derived(health ? runningIds(health) : []) + const statusLabel = $derived( + running.length > 1 + ? `${running.length} jobs running - server & worker status` + : 'server & worker status', + ) const view = $derived(route.view) // The tab title and the finish alert (jobWatch.ts, finishAlert.ts); what @@ -246,29 +261,38 @@
    - {#if currentJob} + {#if running.length === 1} running {/if} + {#snippet busy()} + + + {running.length} running + + + {/snippet} {#if metered.length} {/if} Date: Thu, 15 Oct 2026 10:42:08 -0500 Subject: [PATCH 178/204] fix(cache): #927 - report mag calibration ratios a call leaves unlogged diffusers logs a calibration only when its step counter reaches the config's num_inference_steps. MiniMax-H3's scheduler turns N sigmas into N - 1 forwards, so a calibrating H3 call ended with its ratios still in the mag state and the call's reset dropped them: a clean job with no ratios. stateful_cache_context now reports what a completed call left in each context's state before the reset; a finished calibration has nothing left (diffusers clears the list when it logs), and an errored call reports nothing partial. Docs and schema say num_inference_steps is the forwards the call runs, and to set it to the count reported. Co-Authored-By: Claude Opus 5.5 --- docs/ACCELERATION.md | 4 +- dw/pipeline_processors/components.py | 48 +++++++++++++++++---- dw/workflow_schema.json | 2 +- tests/test_cache_blocks.py | 63 ++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 12 deletions(-) diff --git a/docs/ACCELERATION.md b/docs/ACCELERATION.md index 34ae6c90..e864eb7b 100644 --- a/docs/ACCELERATION.md +++ b/docs/ACCELERATION.md @@ -78,7 +78,7 @@ Magnitude-based caching with error accumulation. Requires `num_inference_steps` | -------- | ------- | ----------- | | `mag_ratios` | required | Preset name or explicit per-step ratio array — see below | | `threshold` | 0.06 | Accumulated error threshold for skipping | -| `num_inference_steps` | required | Must match pipeline arguments | +| `num_inference_steps` | required | The transformer forwards the call runs: the pipeline's `num_inference_steps`, except where the scheduler runs fewer (MiniMax-H3 runs one fewer) | | `max_skip_steps` | 3 | Max consecutive steps to skip | | `retention_ratio` | 0.2 | Fraction of initial steps where skipping is disabled | | `calibrate` | false | Measure ratios for a new model instead of skipping — see below | @@ -90,7 +90,7 @@ MagCache needs to know how each denoising step's output magnitude typically beha - **A preset name** — `"mag_ratios": "flux"` resolves to the ratios diffusers ships for Flux. Any preset a later diffusers release adds is usable by name without a change here. - **An explicit array** — `"mag_ratios": [1.0, 0.98, 0.96, ...]`. The array is interpolated automatically when its length differs from `num_inference_steps`, so ratios measured at one step count can be reused at another. -For a model with no preset, run once with `"calibrate": true`. Calibration skips nothing. At the end of the run the measured ratios appear as a job warning and a `log` event (`get_job`'s `warnings`, `get_job_events`, `export_job`): `MagCache calibration (N steps): [ ... ] — paste into mag_ratios`. While calibrating, the run's own `num_inference_steps` is the step count, so the cache's can be left at its default (a mismatch is reported as a warning). The cache attaches to a pipeline's `transformer`, and only one that supports `enable_cache()`: a UNet pipeline such as SD 1.5 has no transformer, and Z-Image's transformer has no cache support, so both get a warning that the cache was not applied and no calibration. Paste that array into `mag_ratios` and drop the `calibrate` flag for subsequent runs. A pipeline that runs classifier-free guidance as two transformer passes in their own `cond` and `uncond` cache contexts (Qwen-Image-2.1 with `true_cfg_scale` above 1 and a `negative_prompt`, or the other Qwen-Image pipelines) keeps a separate state for each, so it emits **two** lists, each labeled `[cond]` or `[uncond]`; use the `cond` one, since `mag_ratios` takes one array. A guidance-distilled pipeline such as Flux, or one that batches both passes into one forward emits one unlabeled list. +For a model with no preset, run once with `"calibrate": true`. Calibration skips nothing. At the end of the run the measured ratios appear as a job warning and a `log` event (`get_job`'s `warnings`, `get_job_events`, `export_job`): `MagCache calibration (N steps): [ ... ] — paste into mag_ratios`. While calibrating, the run's own `num_inference_steps` is the step count, so the cache's can be left at its default (a mismatch is reported as a warning). The cache attaches to a pipeline's `transformer`, and only one that supports `enable_cache()`: a UNet pipeline such as SD 1.5 has no transformer, and Z-Image's transformer has no cache support, so both get a warning that the cache was not applied and no calibration. Paste that array into `mag_ratios` and drop the `calibrate` flag for subsequent runs. `N` is the forwards the run actually made, which is not always the call's `num_inference_steps`: MiniMax-H3's scheduler turns N sigmas into N - 1 evaluations, so a 30-step H3 call reports 29 ratios. Set the cache's `num_inference_steps` to that `N` when you paste them, so each ratio lines up with its forward. A pipeline that runs classifier-free guidance as two transformer passes in their own `cond` and `uncond` cache contexts (Qwen-Image-2.1 with `true_cfg_scale` above 1 and a `negative_prompt`, or the other Qwen-Image pipelines) keeps a separate state for each, so it emits **two** lists, each labeled `[cond]` or `[uncond]`; use the `cond` one, since `mag_ratios` takes one array. A guidance-distilled pipeline such as Flux, or one that batches both passes into one forward emits one unlabeled list. ```json "cache": { "type": "mag", "calibrate": true, "num_inference_steps": 28 } diff --git a/dw/pipeline_processors/components.py b/dw/pipeline_processors/components.py index 6e065381..47a6aedc 100644 --- a/dw/pipeline_processors/components.py +++ b/dw/pipeline_processors/components.py @@ -876,15 +876,43 @@ def emit(self, record): context = _active_cache_context_name(self.transformer) except (ValueError, SyntaxError): return - label = f" [{context}]" if context in ("cond", "uncond") else "" - text = ( - f"MagCache calibration ({len(ratios)} steps){label}: " - f"{ratios} \u2014 paste into mag_ratios" - ) - if context == "cond": - text += " (classifier-free guidance runs two lists; the cond one is the usual choice)" - emit_log(text) - emit_warning(text) + _report_mag_ratios(ratios, context) + + +def _report_mag_ratios(ratios, context): + label = f" [{context}]" if context in ("cond", "uncond") else "" + text = ( + f"MagCache calibration ({len(ratios)} steps){label}: " + f"{ratios} \u2014 paste into mag_ratios" + ) + if context == "cond": + text += " (classifier-free guidance runs two lists; the cond one is the usual choice)" + emit_log(text) + emit_warning(text) + + +def _report_unfinished_mag_calibration(transformer): + """Report the ratios a calibrating call left in its mag state (#927). + + diffusers logs a calibration only when its step counter reaches the + config's num_inference_steps, and a scheduler can run fewer forwards than + that: MiniMax-H3's turns N sigmas into N - 1 evaluations. Such a call ends + with its ratios still in the state, which the call's reset then drops. A + call that did finish has nothing left here - diffusers clears the list + when it logs - so nothing is reported twice. + """ + if not getattr(getattr(transformer, "_cache_config", None), "calibrate", False): + return + from diffusers.hooks.mag_cache import MagCacheBlockHook + + for module in transformer.modules(): + registry = getattr(module, "_diffusers_hook", None) + for hook in getattr(registry, "hooks", {}).values(): + if isinstance(hook, MagCacheBlockHook) and hook.is_tail: + for name, state in hook.state_manager._state_cache.items(): + if state.calibration_ratios: + _report_mag_ratios(list(state.calibration_ratios), name) + return @contextlib.contextmanager @@ -964,6 +992,8 @@ def stateful_cache_context(pipeline, segment=None): try: with transformer.cache_context(_CACHE_CONTEXT_NAME): yield + # Only a call that completed has ratios worth pasting + _report_unfinished_mag_calibration(transformer) finally: try: report_cache_stats(counter, segment) diff --git a/dw/workflow_schema.json b/dw/workflow_schema.json index 080de78c..a1c843b3 100644 --- a/dw/workflow_schema.json +++ b/dw/workflow_schema.json @@ -783,7 +783,7 @@ ] }, "num_inference_steps": { - "description": "Required for mag cache type. Must match the pipeline's num_inference_steps.", + "description": "Required for mag cache type. The transformer forwards the call runs: the pipeline's num_inference_steps, except where the scheduler runs fewer (MiniMax-H3 runs one fewer, so a calibration reports one ratio fewer than the call's steps; set this to the count reported).", "type": "integer" }, "max_skip_steps": { diff --git a/tests/test_cache_blocks.py b/tests/test_cache_blocks.py index 8cc06cfb..4485a889 100644 --- a/tests/test_cache_blocks.py +++ b/tests/test_cache_blocks.py @@ -570,3 +570,66 @@ class _Pipe: components.enable_cache_on_transformer(_Pipe(), MagCacheConfig(calibrate=True)) assert len(warnings) == 1 assert "was not applied" in warnings[0] and "enable_cache()" in warnings[0] + + +def _calibrate_call(components, model, steps_run, contexts=("cond", "uncond")): + """Nest the two contexts the way Pipeline.__call__ does around a 4-step call.""" + with components.captured_mag_calibration( + _FakePipeline(model), {"num_inference_steps": 4} + ): + with components.stateful_cache_context(_FakePipeline(model)): + for context in contexts: + _run_calibration_steps(model, context, steps_run) + + +def test_mag_calibration_reports_a_call_that_ends_one_forward_short(monkeypatch): + """#927: H3's scheduler runs num_inference_steps - 1 forwards, so diffusers + never reaches its end-of-run log; the ratios left in the state are reported + before the call's reset drops them.""" + from dw.pipeline_processors import components + + warnings = [] + monkeypatch.setattr(components, "emit_log", lambda *a, **k: None) + monkeypatch.setattr(components, "emit_warning", warnings.append) + + _calibrate_call(components, _mag_model(4), steps_run=3) + + reported = [w for w in warnings if w.startswith("MagCache calibration (")] + assert len(reported) == 2 + assert "[cond]" in reported[0] and "[uncond]" in reported[1] + assert all("MagCache calibration (3 steps)" in w for w in reported) + + +def test_mag_calibration_a_finished_call_is_reported_once(monkeypatch): + """#927: diffusers clears the list when it logs, so the end-of-call read + finds nothing left and does not repeat the lists.""" + from dw.pipeline_processors import components + + warnings = [] + monkeypatch.setattr(components, "emit_log", lambda *a, **k: None) + monkeypatch.setattr(components, "emit_warning", warnings.append) + + _calibrate_call(components, _mag_model(4), steps_run=4) + + reported = [w for w in warnings if w.startswith("MagCache calibration (")] + assert len(reported) == 2 + assert all("MagCache calibration (4 steps)" in w for w in reported) + + +def test_mag_calibration_an_errored_call_reports_no_partial_lists(monkeypatch): + from dw.pipeline_processors import components + + warnings = [] + monkeypatch.setattr(components, "emit_log", lambda *a, **k: None) + monkeypatch.setattr(components, "emit_warning", warnings.append) + + model = _mag_model(4) + with pytest.raises(RuntimeError): + with components.captured_mag_calibration( + _FakePipeline(model), {"num_inference_steps": 4} + ): + with components.stateful_cache_context(_FakePipeline(model)): + _run_calibration_steps(model, "cond", 2) + raise RuntimeError("out of memory") + + assert not [w for w in warnings if w.startswith("MagCache calibration (")] From 25aa95dda1b4a3822e7122a00b6bab117c1f5ff3 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 18:14:00 -0500 Subject: [PATCH 179/204] feat(plugin): #882 - minimax-h3 Ramp section: draft -> final paths, carryover with #861 seed counts, judging, cost Expands the "Draft then final" bullet in quality-and-speed.md into a Ramp section (TV L1 -> L7, cinema B3 -> L5/P1 by arguments, close-up ships the draft), rewords the minimax-h3 "Smeary faces" line to cover tiers, and points script-to-video's test-render passage at it for H3 shots. Co-Authored-By: Claude Opus 5.5 --- plugins/dw/skills/minimax-h3/SKILL.md | 2 +- .../references/quality-and-speed.md | 90 ++++++++++++++++++- plugins/dw/skills/script-to-video/SKILL.md | 1 + 3 files changed, 89 insertions(+), 4 deletions(-) diff --git a/plugins/dw/skills/minimax-h3/SKILL.md b/plugins/dw/skills/minimax-h3/SKILL.md index a5426d42..58869d10 100644 --- a/plugins/dw/skills/minimax-h3/SKILL.md +++ b/plugins/dw/skills/minimax-h3/SKILL.md @@ -98,7 +98,7 @@ section). Never put an FL2VA LoRA on a reference template: `ref2va` holds `transformer_ref` alone, so anything handed there degrades output. `validate_workflow` refuses it and warns on a `weight_name` naming neither. -- Smeary faces: int8 base model, `references/quality-and-speed.md`. +- Smeary faces (int8 base) or draft -> final tiers: `references/quality-and-speed.md`. - CUDA: `attention_backend=sage_hub` (null elsewhere; mps fails at run). - Fit a crowd's action to what it holds: candle-holders asked to clap rendered three-handed people. diff --git a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md index 0a3ab528..a705bec3 100644 --- a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md +++ b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md @@ -39,14 +39,98 @@ shift the LoRA names. - **Steps**: the base floor is 12. 8 steps duplicated a character (B6). - **Two-character shots at 960x544 smear** (~60 px faces): render them at 1344x768 from the start, not at 544 and upscaled. -- **Draft then final**: a 12-step draft (~13 min) of the wide, then the - 30-step final with `first_block` 0.05 (~23 min). Draft to final carries - blocking but not always props. +- **Draft then final**: pick the path by target and render a cheap draft + before the final. See *Ramp* below. - **Submit freely**: the server holds a second heavy job, queued with `waiting_for`, until host RAM frees. - **Frame ceiling** at 1344x768 with 4 references is 124; 141 was refused (23.9 GB > 23.6). Write to 124 or drop a reference. +## Ramp + +Spend GPU time on quality in two tiers: a draft to approve the shot, then a +final of the same shot. Pick the path by the target **before the first +render**, because a draft only carries to a final on its own path. + +### 1. Pick the path + +Both tiers use the same template (`templates/minimax/reference-to-video`, or +`templates/minimax/dialogue-short` for a cut). `attention_backend=sage_hub` +is an argument on every H3 template, `dialogue-short` included. Set it at +both tiers on CUDA: it never moves composition. + +| Target | Draft | Final | Only these change | +|---|---|---|---| +| **TV** (sitcom, social, previs) | **L1**, ~5.4 min | **L7**, ~7.2 min | `weights_dtype` `{int4}` to `{int8}` | +| **Cinema** (realistic faces) | **B3**, ~12.9 min | **L5**, ~29.5 min, or **P1**, ~23.1 min | `num_inference_steps` 12 to 30; P1 adds `cache` | +| **Close-up** | **C1**, ~13.9 min | ship the draft | nothing | + +- **TV.** Both rungs are 1344x768 on the 4-step Ref2VA v0.1 LoRA: + `lora_model_name` `lightx2v/Minimax-h3-Turbo`, `lora_weight_name` + `minimax_h3_ref2v_turbo_4step_v0.1_bf16.safetensors`, + `num_inference_steps` 5, `video_shift` 12, `audio_shift` 3. L1 is + `weights_dtype` `{int4}`, L7 is `{int8}`. **L0** (the same at 960x544, + ~3.5 min) is a throwaway sketch only. Its staging did carry to L7 (below), + but it smeared a face at both new seeds, so a face can't be approved on it, + and two-character shots at 960x544 smear. Render those at 1344x768 from + the start. The L7 final (3/5) is **not confirmed by ear or in playback**. +- **Cinema.** The base model: `lora_model_name` `null`, `weights_dtype` + `{int8}`, 1344x768, `video_shift` 12, `audio_shift` 3. B3 is 12 steps, L5 is + 30. **P1** is L5 plus the template's `cache` argument + `{"type": "first_block", "threshold": 0.05}` (-22%, the same staging and + faces). Both tiers warn `lora_disabled`. That is expected: the warning asks + for the base schedule, and these values are it. Never draft at B4 (960x544 + int4 base) for a 1344x768 final: its staging held on only 2 of 4 seeds. +- **Close-ups**: C1 is B3's arguments on a close-up shot. It scores 4/5 at + 12 steps (C1 = C2), so the draft is the deliverable. + +### 2. What carries from draft to final + +The composition rules under *Quality ladder* hold: attention never moves +staging, and neither does base step count; a different LoRA family +re-stages; so does int4 against int8 on the base model. On top of those: + +- keep the seed, canvas, prompt and references identical between the tiers; +- blocking carries, but props and set dressing sometimes don't (a jacket on + a chair in C1 only). + +Measured (#861) on seeds 42, 7 and 1234, judged by Don: B3 -> P1, C1 -> C2 +and L0 -> L7 each held staging on **3 of 3** seeds. L1 = L7 (the TV pair +itself) is measured on **1 seed** (42). B4 -> B5 held on only 2 of 4 seeds +(42, 7, 1234, 99). At seed 1234, B5 (int4) staged like B3 (int8): on the +base model the canvas moved staging more than the weights did. That is one +seed against the int4/int8 rule, which stands until it's measured further. + +### 3. What to approve at each tier + +- **Draft:** blocking, line delivery, timing and head count. Re-roll the + seed or the prompt here, and only here. +- **Final:** faces, skin and hair. On these paths a final that re-stages + usually has an argument the table doesn't allow: diff its arguments + against the draft's before suspecting the seed. + +### 4. How to judge + +- Face crops at native resolution, at fixed frames + (`get_output_frames`), the same frames at both tiers. +- A contact sheet (`get_output_frames(count=12)`) for staging. +- `check-script` for the lines. +- A 1-5 face score, recorded on the run with `set_note`. + +Claude can't judge audio by ear (voice timbre, lip-sync feel, room sound) +or motion (smear, flicker) in playback. Say so in the note, and ask a +human for those. + +### 5. Cost + +Quote `validate_workflow`'s `plan.estimate` for each tier, draft plus +final. It scales by steps, canvas and frames (#866), but not by +`weights_dtype`, `attention_backend`, `cache` or a null LoRA. When one of +those moves it sets `low_confidence` and names the variable in `differs`. +With `sage_hub` set, every rung above moves one. Quote that rung's minutes +from the ladder instead (they were measured with `sage_hub`), and say which +figure you are quoting. + ## Prompting the base model - It invents an extra person unless every subject is placed explicitly diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index b62a08dc..44b6a463 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -78,6 +78,7 @@ live there, not here. Render two to four representative shots first (a dialogue shot, an action shot, the hardest cut) and judge them before the full batch. +For H3 shots, `dw:minimax-h3`'s quality-and-speed reference (*Ramp*) has the draft and final tiers and what to approve at each. `validate_workflow` against the chosen template with the assembled `shots` argument, read `plan.estimate` and `plan.downloads_required`, and state the From 6834d68797c2a7dc0ca88f354fe096bc4005c121 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 18:24:21 -0500 Subject: [PATCH 180/204] feat(plugin): #882 - Ramp: 124-frame shots on dialogue-short at 1344x768 The default tag shot (141 frames, 4 refs) is over the VRAM ceiling at 1344x768, so every rung on dialogue-short needs a shots argument. Co-Authored-By: Claude Opus 5.5 --- .../minimax-h3/references/quality-and-speed.md | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md index a705bec3..8399a87b 100644 --- a/plugins/dw/skills/minimax-h3/references/quality-and-speed.md +++ b/plugins/dw/skills/minimax-h3/references/quality-and-speed.md @@ -59,6 +59,16 @@ Both tiers use the same template (`templates/minimax/reference-to-video`, or is an argument on every H3 template, `dialogue-short` included. Set it at both tiers on CUDA: it never moves composition. +**`dialogue-short` at 1344x768** (every rung below is that canvas): a shot +with 4 references must be 124 frames (*Frame ceiling* above). The default +`shots` list's `tag` (`shots[4]`) is 141 frames with 4 references, so +`validate_workflow` refuses it at `variables.shots[4]`. Pass your own +`shots` argument at both tiers: `get_workflow` the template, copy +`variables.shots`, set `shots[4].num_frames` to 124 (or drop a reference +from it), and pass the list unchanged at draft and final. A cut you write +follows the same rule: every 4-reference shot at 124. Check the trimmed +shot's line still fits 5.17 s. + | Target | Draft | Final | Only these change | |---|---|---|---| | **TV** (sitcom, social, previs) | **L1**, ~5.4 min | **L7**, ~7.2 min | `weights_dtype` `{int4}` to `{int8}` | @@ -76,7 +86,7 @@ both tiers on CUDA: it never moves composition. the start. The L7 final (3/5) is **not confirmed by ear or in playback**. - **Cinema.** The base model: `lora_model_name` `null`, `weights_dtype` `{int8}`, 1344x768, `video_shift` 12, `audio_shift` 3. B3 is 12 steps, L5 is - 30. **P1** is L5 plus the template's `cache` argument + 30. On `dialogue-short`, add the 124-frame `shots` above at both tiers. **P1** is L5 plus the template's `cache` argument `{"type": "first_block", "threshold": 0.05}` (-22%, the same staging and faces). Both tiers warn `lora_disabled`. That is expected: the warning asks for the base schedule, and these values are it. Never draft at B4 (960x544 From 6deed463edb412fa3fdafabf846c942bfb1d87d8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 18:30:50 -0500 Subject: [PATCH 181/204] docs: #862 close-out - H3 quality ramp design record Co-Authored-By: Claude Opus 5.5 --- .../complete/h3-quality-ramp-complete.md | 127 ++++++++++++++++++ docs/proposals/todo.md | 8 ++ 2 files changed, 135 insertions(+) create mode 100644 docs/proposals/complete/h3-quality-ramp-complete.md diff --git a/docs/proposals/complete/h3-quality-ramp-complete.md b/docs/proposals/complete/h3-quality-ramp-complete.md new file mode 100644 index 00000000..bf826e4a --- /dev/null +++ b/docs/proposals/complete/h3-quality-ramp-complete.md @@ -0,0 +1,127 @@ +# H3 quality ramp: draft -> final tiers (#862) + +Written by model `claude-opus-5-5` via provider `anthropic` (close-out, +2026-10-15). Don approved plan v2 on 2026-10-14. Its one stage, #882, +shipped on 2026-10-15 and is at `develop` @ `426b764b`. It was verified on +lem (CUDA) against the `regression-complete` workspace. + +## The idea + +Nothing in the plugin told an agent how to spend GPU time on quality. Every +H3 render before Don's quality matrix used int4 + a turbo LoRA, because the +templates default to that, and Known Issues faces smeared (2-2.5/5). The +matrix on lem (2x RTX 3090) showed a large, predictable trade-off. The base +model (no LoRA) at int8 gave photographic faces (4/5) at 2-4x the time. +Within one staging family, a cheap draft carried its staging to the final +at the same seed. The issue proposed a cross-cutting `quality-ramp` skill: +which tier to draft at, what to approve there, and what never to change +between tiers. + +## Verdict and scope + +**Build smaller.** No new skill and no new file. The ramp is a `## Ramp` +section in the existing `plugins/dw/skills/minimax-h3/references/quality-and-speed.md`. +It expands that file's "Draft then final" bullet. #863 had already put the +ladder, the speed levers and the composition invariants there, so the +section adds only the process around them. + +Why not a top-level skill: only H3 has a measured ladder, and `minimax-h3` +already loads on every H3 request. A new skill would cost up to 12 KB per +trigger and rows in the README, CLAUDE.md and the skill test. + +Corrections the plan made to the proposal: +- **TV path L1 -> L7, not L0 -> L7.** L0 changes the canvas. L0 stays as a + throwaway sketch. Plan v2 first said L0's staging doesn't carry. #861 then + measured L0 -> L7 holding on 3 of 3 seeds, so Don corrected the build: + L0 is for blocking only because it smears faces, not because its staging + moves. +- **No pin test.** The numbers and the advice sit in one file. +- **The base rung doesn't clear `lora_disabled`.** The warning is + unconditional on a null LoRA (`dw/adapter_compatibility.py`). The section + calls it expected on both cinema tiers. + +## Decisions (Don's review of plan v1, 2026-10-14) + +- **Q1.** A section in the existing `quality-and-speed.md`, not a new file + or skill. +- **Q2.** Wait for #861's seed runs before stating carryover claims. +- **Q3.** Quote `plan.estimate` once #866 lands, and the ladder until then. +- **Q4.** Ship L7 as the TV final, marked "not confirmed by ear or in + playback". +- **Ruling on dkackman/harnest#116:** #865 landed first, so P1 is L5 plus + the template's `cache` argument, and `sage_hub` is an `attention_backend` + argument on every H3 template. The plan's saved-copy path was dropped. + +## What was built + +### Stage 1, #882: the `## Ramp` section + +- **`quality-and-speed.md` `## Ramp`:** + 1. *Pick the path*, a table by target. TV: L1 (~5.4 min) -> L7 (~7.2 min); + only `weights_dtype` changes. Cinema: B3 (~12.9 min) -> L5 (~29.5 min) + or P1 (~23.1 min, + `cache` `{"type": "first_block", "threshold": 0.05}`); + only `num_inference_steps` changes, plus `cache` for P1. Close-up: ship + C1. Each rung's arguments are spelled out. L0 is a sketch, B4 is never + a cinema draft (2 of 4 seeds). + 2. *What carries*: a pointer to the *Quality ladder* rules, plus + identical seed/canvas/prompt/references and "blocking carries, props + sometimes don't", each with its #861 seed count. + 3. *What to approve* at draft and at final; re-roll only at the draft. + 4. *How to judge*: face crops and a contact sheet via + `get_output_frames`, `check-script`, a 1-5 score with `set_note`, and + what Claude can't judge (audio by ear, motion in playback). + 5. *Cost*: `plan.estimate` (#866), but the rung's ladder minutes when the + estimate is `low_confidence`, which every `sage_hub` rung is. +- **`dialogue-short` at 1344x768** (the bounce fix): the default `shots[4]` + (`tag`) is 141 frames with 4 references, over the 124-frame ceiling. The + section says to copy `variables.shots`, set `shots[4].num_frames` to 124 + (or drop a reference), and pass the list unchanged at both tiers. +- **`minimax-h3/SKILL.md`:** the "Smeary faces" line reworded to cover + draft -> final tiers, with no new line (12,278 of 12,288 bytes). +- **`script-to-video/SKILL.md`:** a one-line pointer for H3 shots in the + test-render passage. +- **Deviations:** no plugin version bump, because `plugin.json`'s version + must equal `pyproject.toml`'s and `scripts/release.sh` moves both. The + README and CLAUDE.md are unchanged. + +## Verified + +Cases C-F414 to C-F419, in `regression-suite-complete.md`; all six now +pass, with no GPU spend (`validate_workflow` only). C-F416 to C-F418 +validate B3/L5/P1 on `dialogue-short` with the 124-frame `shots`, each with +one `lora_disabled` warning per shot. The tester also checked the "drop a +reference" option: a 3-reference `tag` at 141 frames validates. + +Whether a session *follows* the section isn't tested over MCP. Don judges +that on his next H3 episode. + +## Bounces per stage + +- **Stage 1, #882: 1.** C-F416, C-F417 and C-F418 failed: the cinema path + sent `dialogue-short` to 1344x768, where its default 141-frame `tag` shot + is refused by the VRAM ceiling. Fixed in `6834d687` (`426b764b`) with the + `dialogue-short` paragraph above. Docs only; the template's default was + left alone (960x544 fits it). + +Stage costs aren't recorded, because the stage comments give no `usage:` +figures. The plan estimated about $2. + +## Deferred + +- **MagCache as the cinema-final cache.** After #882 was built, #875 and + #927 measured M2 (MagCache 0.06 with calibrated H3 ratios, 18.1 min, + -39% against L5), and Don chose it over `first_block` 0.05; `first_block` + 0.08 (P6, 19.5 min) is the fallback. The section still names P1 as the + cache final, and the file's closing "Open:" line still lists + `first_block` at 20 steps, P1 on a close-up or a second seed, and + MagCache calibration as open. All are answered on #862's body. Updating + the section needs the 29 ratios as a `cache` argument, so it is a + follow-up edit with its own check, not a close-out doc fix. +- **A top-level `quality-ramp` skill.** What brings it back: a measured + ladder for a second family (LTX-2.5, Flux). The family-agnostic half + (tier rules, approval per tier, judging) would then move up into + `skills/quality-ramp/`, and each family keeps its own ladder. +- **The 12-step base floor** stays advice (B6 at 8 steps duplicated a + character; 10 was never tried). An engine rule is #868's territory. +- **L1 = L7 on more seeds.** The TV pair itself is measured on one seed; + the section says so. diff --git a/docs/proposals/todo.md b/docs/proposals/todo.md index 72bcfc09..a992807a 100644 --- a/docs/proposals/todo.md +++ b/docs/proposals/todo.md @@ -37,6 +37,14 @@ remaining deferred fix is recorded in ## Shipped since the ranking +- **H3 quality ramp: draft -> final tiers** (#862, stage #882), shipped + 2026-10-15. A `## Ramp` section in the `minimax-h3` skill's + `references/quality-and-speed.md`, not a new skill: TV L1 -> L7, cinema + B3 -> L5/P1, ship a close-up draft, with the arguments per rung, what + carries (with #861's seed counts), what to approve per tier, how to judge + and how to quote cost. Record, including what was deferred (MagCache M2 + as the cinema-final cache, a top-level skill once a second family has a + ladder): `complete/h3-quality-ramp-complete.md`. - **Dispatch holds a queued job for host RAM** (#869, stages #909-#910), shipped 2026-10-15. A job waits `queued`, with a `queue_wait` event and a live `waiting_for`, while its projected peak (the max observed for its From 96d1b85dcbb61eee896c63ef13264c6c42fed7af Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:17:24 -0500 Subject: [PATCH 182/204] refactor(ui): pull figure() qualifiers into a helper; show scaled on plain figures figure() in plan.ts was at complexity 11 (limit 10). The card, along and missing qualifiers move into qualifiers(). dw/plan.py names `scaled` for every basis but observed, so the plain (catalog/per_entry) branch now carries it as the warn branches do. Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/plan.test.ts | 17 +++++++++++++++++ ui/src/lib/plan.ts | 25 +++++++++++++++---------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/ui/src/lib/plan.test.ts b/ui/src/lib/plan.test.ts index 1cbdab29..7e2edaf8 100644 --- a/ui/src/lib/plan.test.ts +++ b/ui/src/lib/plan.test.ts @@ -95,6 +95,23 @@ describe('describePlan', () => { ) }) + it('a re-priced figure scaled along a child says what moved', () => { + // dw/plan.py names `scaled` for every basis but observed, so the + // plain branch must carry it as the warn branches do + const [, figure] = describePlan({ + ...base, + estimate: { + ...base.estimate, + basis: 'per_entry', + scaled: 'scaled from the curated 7.9 min: steps 30 vs 9', + }, + }) + expect(figure.text).toBe( + '~42 min on cuda - re-priced for 5 shots, measured on RTX 3090 (scaled from the curated 7.9 min: steps 30 vs 9)', + ) + expect(figure.tone).toBe('plain') + }) + it('a figure from another accelerator is a warning, not a quote', () => { const [, figure] = describePlan({ ...base, diff --git a/ui/src/lib/plan.ts b/ui/src/lib/plan.ts index f8567cdc..10529d3b 100644 --- a/ui/src/lib/plan.ts +++ b/ui/src/lib/plan.ts @@ -61,19 +61,24 @@ export function describePlan(plan: Plan): PlanLine[] { return lines } +/** The qualifiers every priced figure carries: the card it was measured + * on, what it was scaled along (#866: steps, canvas or frames moved off the + * defaults it was measured at), and the steps it leaves unpriced. */ +function qualifiers(estimate: Plan['estimate']) { + const { measured_on, partial, unpriced, scaled } = estimate + return { + card: measured_on ? `measured on ${measured_on}` : 'measured', + along: scaled ? ` (${scaled})` : '', + missing: partial ? `, plus unpriced: ${(unpriced ?? []).join(', ')}` : '', + } +} + function figure(plan: Plan): PlanLine { - const { minutes, basis, device, measured_on, partial, unpriced, scaled } = - plan.estimate - const card = measured_on ? `measured on ${measured_on}` : 'measured' - const missing = partial - ? `, plus unpriced: ${(unpriced ?? []).join(', ')}` - : '' + const { minutes, basis, device } = plan.estimate if (basis === 'unknown' || minutes === null) { return { text: 'no measured cost', tone: 'warn' } } - // What a figure was scaled along (#866): steps, canvas or frames moved - // off the defaults it was measured at - const along = scaled ? ` (${scaled})` : '' + const { card, along, missing } = qualifiers(plan.estimate) if (basis === 'other_device') { return { text: `no figure for ${device} - ${minutes} min was ${card}${along}${missing}`, @@ -95,7 +100,7 @@ function figure(plan: Plan): PlanLine { } const repriced = basis === 'per_entry' ? `re-priced for ${forList}, ` : '' return { - text: `~${minutes} min on ${device} - ${repriced}${card}${missing}`, + text: `~${minutes} min on ${device} - ${repriced}${card}${along}${missing}`, tone: 'plain', } } From 111042d92c60b8a205d77aadc974b26958b58765 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:18:11 -0500 Subject: [PATCH 183/204] fix(downloads): #899 - write refs/main after a per-file repo download hf_hub_download pinned to a commit hash writes no ref, so a repo pulled through POST /api/downloads or MCP download_model had blobs and a snapshot but no refs/main: repo_download_incomplete read it as unfinished, validate kept it in downloads_required, and offline loads could not resolve main. _download_repo now writes refs/main = info.sha atomically (temp file + os.replace, paths through validate_path with the cache as base) once every file is in and no cancel came; a cancel that lands after the last file is still reported cancelled and writes no ref. Co-Authored-By: Claude Opus 5.5 --- dw/hub_cache.py | 52 ++++++++++++++++++++++++-- tests/test_hub_cache.py | 82 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 130 insertions(+), 4 deletions(-) diff --git a/dw/hub_cache.py b/dw/hub_cache.py index 2bf20053..38e736e0 100644 --- a/dw/hub_cache.py +++ b/dw/hub_cache.py @@ -11,7 +11,9 @@ import json import logging import os +import re import shutil +import tempfile import threading import time import uuid @@ -419,19 +421,26 @@ def _prune(self): del self._downloads[entry["id"]] -def _download_repo(repo_id, tqdm_class, max_workers=4): +def _download_repo(repo_id, tqdm_class, max_workers=4, cache_dir=None): """Fetch every file of repo_id, stopping promptly on a cancel (#899). Stands in for `snapshot_download`, whose file pool runs every queued file to the end after a cancel (its worker threads swallow the tracker's exception, and aborting the xet session only ends the files in flight). Here each file checks the cancel event before it starts, so a cancel - leaves at most the aborted in-flight files behind.""" + leaves at most the aborted in-flight files behind. + + Every file is pinned to the commit `repo_info` resolved for `main`, and + hf_hub_download writes no ref for a commit-hash revision, so this writes + `refs/main` itself once every file is in and no cancel came - without it + `repo_download_incomplete` reads the pull as unfinished and an offline + load cannot resolve `main`. A cancelled pull writes no ref.""" from concurrent.futures import ThreadPoolExecutor from huggingface_hub import HfApi, hf_hub_download cancel_event = tqdm_class._cancel_event + # repo_info's default revision is `main` - the ref written below info = HfApi().repo_info(repo_id) files = [sibling.rfilename for sibling in info.siblings or []] @@ -439,7 +448,11 @@ def fetch(filename): if cancel_event.is_set(): raise DownloadCancelled() hf_hub_download( - repo_id, filename=filename, revision=info.sha, tqdm_class=tqdm_class + repo_id, + filename=filename, + revision=info.sha, + tqdm_class=tqdm_class, + cache_dir=cache_dir, ) pool = ThreadPoolExecutor(max_workers=max_workers) @@ -448,6 +461,39 @@ def fetch(filename): future.result() finally: pool.shutdown(wait=True, cancel_futures=True) + if cancel_event.is_set(): + raise DownloadCancelled() + _write_main_ref(repo_id, info.sha, cache_dir) + + +_COMMIT_HASH = re.compile(r"^[0-9a-f]{40}$") + + +def _write_main_ref(repo_id, commit, cache_dir=None): + """Point repo_id's `refs/main` at `commit`, atomically: a temp file in + the refs folder, then os.replace over the ref.""" + if not isinstance(commit, str) or not _COMMIT_HASH.match(commit): + # The ref names a snapshot folder; anything but a commit hash is + # not a value to write into the cache + raise ValueError(f"'{repo_id}' resolved to no commit hash: {commit!r}") + resolved = _resolved_cache_dir(cache_dir) + repo_dir = validate_path( + os.path.join(resolved, _repo_folder_name(repo_id)), resolved + ) + refs_dir = validate_path(os.path.join(repo_dir, "refs"), repo_dir) + ref_path = validate_path(os.path.join(refs_dir, "main"), refs_dir) + os.makedirs(refs_dir, exist_ok=True) + fd, tmp_path = tempfile.mkstemp(dir=refs_dir, prefix=".main.", suffix=".tmp") + try: + with os.fdopen(fd, "w") as f: + f.write(commit) + os.replace(tmp_path, ref_path) + except BaseException: + try: + os.unlink(tmp_path) + except OSError: + pass + raise def _abort_xet_session(): diff --git a/tests/test_hub_cache.py b/tests/test_hub_cache.py index 232de664..99949b57 100644 --- a/tests/test_hub_cache.py +++ b/tests/test_hub_cache.py @@ -441,7 +441,7 @@ def test_cancel_skips_files_not_yet_started(self): cancel = threading.Event() fetched = [] - def fake_hub_download(repo_id, filename, revision, tqdm_class): + def fake_hub_download(repo_id, filename, revision, tqdm_class, **kwargs): fetched.append(filename) cancel.set() @@ -462,3 +462,83 @@ def fake_hub_download(repo_id, filename, revision, tqdm_class): else: raise AssertionError("expected DownloadCancelled") assert len(fetched) == 1 + + +class TestDownloadRepoMainRef: + """#899 regression: per-file hf_hub_download pinned to a commit hash never + writes refs/main, so a pulled repo read as incomplete and offline loads + could not resolve 'main'. _download_repo writes it once every file is in.""" + + SHA = "0123456789abcdef0123456789abcdef01234567" + + def run(self, tmp_path, fake_hub_download, cancel=None): + import threading + from types import SimpleNamespace + from unittest import mock + + from dw.hub_cache import _download_repo + + cancel = cancel or threading.Event() + info = SimpleNamespace( + sha=self.SHA, + siblings=[SimpleNamespace(rfilename=name) for name in ("a.bin", "b.bin")], + ) + tracker = SimpleNamespace(_cancel_event=cancel) + with ( + mock.patch("huggingface_hub.HfApi") as api, + mock.patch("huggingface_hub.hf_hub_download", fake_hub_download), + ): + api.return_value.repo_info.return_value = info + _download_repo("acme/tiny", tracker, max_workers=1, cache_dir=tmp_path) + + def fake_writer(self, tmp_path, on_fetch=None): + def fake_hub_download(repo_id, filename, revision, tqdm_class, cache_dir): + assert revision == self.SHA + snapshot = ( + tmp_path / "models--acme--tiny" / "snapshots" / revision / filename + ) + snapshot.parent.mkdir(parents=True, exist_ok=True) + snapshot.write_bytes(b"x") + if on_fetch: + on_fetch(filename) + + return fake_hub_download + + def test_a_completed_download_writes_refs_main_at_the_sha(self, tmp_path): + self.run(tmp_path, self.fake_writer(tmp_path)) + ref = tmp_path / "models--acme--tiny" / "refs" / "main" + assert ref.read_text() == self.SHA + assert repo_download_incomplete("acme/tiny", cache_dir=tmp_path) is False + # The temp file the ref was written through is gone + assert os.listdir(ref.parent) == ["main"] + + def test_a_cancelled_download_writes_no_ref(self, tmp_path): + import threading + + from dw.hub_cache import DownloadCancelled + + cancel = threading.Event() + with pytest.raises(DownloadCancelled): + self.run( + tmp_path, + self.fake_writer(tmp_path, on_fetch=lambda _: cancel.set()), + cancel=cancel, + ) + assert not (tmp_path / "models--acme--tiny" / "refs" / "main").exists() + + def test_a_cancel_after_the_last_file_writes_no_ref(self, tmp_path): + import threading + + cancel = threading.Event() + names = [] + + def on_fetch(name): + names.append(name) + if len(names) == 2: + cancel.set() + + from dw.hub_cache import DownloadCancelled + + with pytest.raises(DownloadCancelled): + self.run(tmp_path, self.fake_writer(tmp_path, on_fetch), cancel=cancel) + assert not (tmp_path / "models--acme--tiny" / "refs" / "main").exists() From 053c5fbacb13e9a9db3ea3f413c16d6ed79edcd8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:18:47 -0500 Subject: [PATCH 184/204] test(notes): leave the job history's SQLite out of the traversal snapshot test_put_note_refuses_traversal_targets snapshots the studio's parent, which also holds the client fixture's jobs.sqlite. JobHistory's connections commit but are closed by GC, and a WAL checkpoint then rewrites the file's bytes mid-test, failing the equality on Linux CI. The snapshot now skips jobs.sqlite* and still covers the rest of the parent, so a traversal write beside the studio is still caught. Co-Authored-By: Claude Opus 5.5 --- tests/test_notes_api.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_notes_api.py b/tests/test_notes_api.py index 289aecd3..703903fe 100644 --- a/tests/test_notes_api.py +++ b/tests/test_notes_api.py @@ -98,10 +98,15 @@ def asset_entry(client, name): def snapshot(root): """Every file under root with its bytes - what a refused write must not - change.""" + change. The job history's SQLite file and its -wal/-shm sidecars are left + out: the client fixture keeps them beside the studio, and a WAL + checkpoint rewrites their bytes on the connection's schedule, not the + request's.""" found = {} for current, _dirs, names in os.walk(root): for name in names: + if name.startswith("jobs.sqlite"): + continue path = os.path.join(current, name) with open(path, "rb") as handle: found[path] = handle.read() From e6e5fe698a1f5cc0b86a1676aced88281059beee Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:19:04 -0500 Subject: [PATCH 185/204] docs(release): draft 0.14.0 notes Co-Authored-By: Claude Opus 5.5 --- docs/RELEASING.md | 106 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 4 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 9ce20675..705bd5b0 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -9,10 +9,108 @@ when a release ships. ### 0.14.0 -- Gallery files are tagged final or intermediate (inferred when a - workflow doesn't say) and the gallery opens on finals; runs, files, - assets and workspaces take an optional note (`note` on `run_workflow`, - MCP `set_note`). + + +A release about what a run is and what it will cost. Gallery files say +whether they are a final or an intermediate, anything can carry a note, and +runs copy between workspaces. The cost estimate scales with steps, canvas +and frames instead of quoting a flat figure that was up to 12x off, and a +queued job waits for host RAM rather than being OOM-killed. + +**Roles and notes** (#855-#858) + + + +- Gallery files are tagged final or intermediate (inferred when a workflow + doesn't say), and the gallery opens on finals. A run lists what it was + Made from. +- Runs, files, assets and workspaces take an optional note: `note` on + `run_workflow`, `upload_asset` and `keep_output`, and MCP `set_note`. + Every listing carries notes; the web UI edits them in place as Markdown. +- Starring a file keeps it. A rerun records the job it reran (`rerun_of`). +- `copy_to_workspace` copies whole runs, with manifests, and asset-library + entries into another workspace (#853, #859, #860). + +**Cost and host RAM** (#866, #869, #881) + +- The estimate scales as steps x canvas x frames against the template's + defaults (`basis: "derived"`, `scaled` names the axes). Any other driver + that moves keeps the figure and sets `low_confidence`; H3 adds weights, + LoRA, attention and cache as drivers. On the H3 matrix it lands within + 30% (B3->L5: 32.3 min quoted, 29.5 run), and an inline copy prices + within 2% of its template (#906-#908, #915, #916, #921). +- A queued job is held at dispatch while its projected host-RAM peak won't + fit, and stays `queued` with `waiting_for` and a `queue_wait` event. It + is never held with nothing else running. On a 125 GB, 2x 3090 host a + second int8 H3 job waited ("needs ~87 GB, ~57 GB free") while two int4 + jobs ran at once (#909, #910). +- An OOM kill now takes the newest dispatch, not the earlier job (#881). + +**Timing and caches** (#867, #868, #905) + +- Manifest entries carry per-step timing (`first_pass_s`, `pass_s`) and + step-cache skip counts (`cache_stats`). A `first_pass_slow` warning names + a warm pipeline's slow first pass (#924: 202 s against 37 s/pass). +- Validate warns on acceleration that can't pay off or won't run: a cache + on a short schedule or with no transformer to attach to, `fullgraph` + with a cache, a CUDA-only attention backend, and an unbraced type name + under a `*_dtype` key (#887, #904, #926). +- The plan fingerprint and step cache key only on what changes the output, + so metadata edits and `12.0` vs `12` no longer miss the cache (#905, + #911). +- Every H3 template exposes `attention_backend` and `cache` (#883, #884). + MagCache calibration now reports its ratios on H3 (#927); with them, + MagCache 0.06 ran the cinema final in 18.1 min, 39% under L5 (#875). + +**H3 quality ramp** (#861, #862, #882) + +- The `minimax-h3` skill has measured draft -> final rungs. TV: L1 (~5.4 + min) -> L7 (~7.2 min), changing only `weights_dtype`. Cinema: B3 (~12.9 + min) -> L5 (~29.5 min), changing only steps. Staging carries at the same + seed; the skill says what to approve at each tier. +- A proven LoRA catalog entry decides which H3 step an adapter fits, + ahead of its filename (#870, #888). + +**Compare, sweeps and check-script** (#871, #872, #919-#925) + +- `compare_outputs` puts 2-4 outputs side by side with PSNR/SSIM; the web + UI's Compare page plays them in sync and lists the differing arguments. + The workspaces guide has a Running a sweep section. +- check-script refuses a wrong shot name at validate, adds + `unscripted_speech` and `speech_not_transcribed`, and levels its input + so the verdict no longer depends on loudness. + +**Web UI** + +- A running-jobs chip, a tab title that tracks the running job, measured + run times on cards, drag-and-drop upload, and keyboard culling in the + gallery. + +**Fixes** + +- `cancel_download` stops queued files and shows `cancel_requested` + (#899); a model download records `refs/main`, so it reads complete. + `cancel_job` takes effect during a load (#892). +- A kept asset keeps its lineage and its job survives the workspace's + deletion (#902, #913, #917, #918). +- `get_gallery_metadata` measures an image, and `include_workflow=false` + omits the embedded workflow (#922). +- An inline copy of a list-driven template is no longer quoted the + template's default-length price unscaled. +- The host-RAM gate no longer under-projects a list-driven run whose peak + grows with the list. +- Whole-module `compile: {fullgraph: true}` no longer fails at the first + denoise step after the timing hook. +- FasterCache runs on FLUX.1-dev (#897); denoise time on mps is no longer + billed to decode (#896). +- The UI's ratchet check and gallery sort are faster. + +**Dependencies and deploy** + +- No dependency floors changed. +- `scripts/dw-serve.service` sets `OOMPolicy=continue`, so one worker's + OOM kill no longer stops the server. Reinstall the unit to pick it up. +- The jobs history gains a `memory_key` column, migrated on start. ### 0.13.0 From 6b1b575d0d10e0dc91baa2e17443b581014949bb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:20:09 -0500 Subject: [PATCH 186/204] fix(plan): #851 - scale an inherited price by for_each member count d0565f83 let inherited_observed fall back to the template's default driver bucket when the caller's (shots=6, say) has no runs, trusting _scale_past_template to scale. It could not see list length: a for_each step counted as one pipeline step, pipeline_values read the authored step (item:num_frames -> None), and sets_differently compares sets of values, so 6 x 124 frames equalled 3 x 124. An inline six-shot copy of a template with one three-shot 30-min run was quoted 30 min, unscaled, differs [], no low_confidence. The fix scales rather than withholds, on the rule the block already uses for authored steps: vram_inheritance.pipeline_steps expands for_each with variables resolved (the expansion reference_counts and sets_differently use), and both the step count and the per-step values are read from it, the template's against the bucket's own list. A six-shot request against the three-shot bucket is 2x, with `scaled` naming "6 steps, template measured 3", which plan._observed already turns into low_confidence - the same basis/scaled/low_confidence shape the catalog figure's list re-price and #866's axes use. Mean frames, steps and canvas are now per member, so shorter shots scale down too. A step count of zero on either side is withheld with a reason rather than divided by. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/deps.py | 17 ++++---- dw/vram_inheritance.py | 25 +++++++++-- tests/test_inherited_cost.py | 82 ++++++++++++++++++++++++++++++++++++ 4 files changed, 112 insertions(+), 14 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4649da13..459883e4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -103,7 +103,7 @@ one to open. | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | -| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding` | +| Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run (a `for_each` counted per member, against the list the bucket ran, so a fallback to the default bucket re-prices a longer list) re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding`, `tests/test_inherited_cost.py::TestAListLongerThanTheFallbackBucket::test_an_inline_six_shot_copy_scales_by_members` | | Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It never refuses; dispatch may delay a start on it (`JobManager._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | diff --git a/dw/server/deps.py b/dw/server/deps.py index ec96cc7e..0dc62b73 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -21,6 +21,7 @@ build_index, inherited_differences, pipeline_frames, + pipeline_steps, pipeline_values, sets_differently, template_identity, @@ -267,14 +268,6 @@ def asset_users(state, ws): return users -def _pipeline_steps(definition): - return sum( - 1 - for step in definition.get("steps") or [] - if isinstance(step, dict) and isinstance(step.get("pipeline"), dict) - ) - - def _canvases(definition, arguments=None): """Each pipeline step's `(width, height)`, as `pipeline_values` reads them, None for a step that does not set both as numbers.""" @@ -302,9 +295,15 @@ def _scale_past_template(definition, template, drivers, arguments): catalog figure and this one quote a request alike - and the reason names each axis that moved. (1, None) when the workflow is the template's size; (None, reason) when its size cannot be compared.""" - steps, template_steps = _pipeline_steps(definition), _pipeline_steps(template) # The template as the bucket's drivers set it bucket = {**template, "variables": {**(template.get("variables") or {}), **drivers}} + # Counted with for_each expanded, so a list longer than the one the + # bucket ran (a fallback to the template's default bucket) re-prices by + # its member count + steps = len(pipeline_steps(definition, arguments)) + template_steps = len(pipeline_steps(bucket)) + if not steps or not template_steps: + return None, "its pipeline step count cannot be compared" factor, reasons = 1.0, [] if steps != template_steps: factor *= steps / template_steps diff --git a/dw/vram_inheritance.py b/dw/vram_inheritance.py index 349f5151..c6d37482 100644 --- a/dw/vram_inheritance.py +++ b/dw/vram_inheritance.py @@ -202,17 +202,34 @@ def _found_expanded(definition, key, variables): return _carried(expanded, key) +def pipeline_steps(definition, arguments=None): + """The pipeline steps `definition` runs, `variable:` references resolved + (`arguments` override the variables) and `for_each` expanded, so a + six-entry list is six steps, each with its entry's `item:` values - the + count and values an inherited price is scaled by (#851). The authored + steps when the expansion fails.""" + variables = dict(definition.get("variables") or {}) + variables.update(arguments or {}) + steps = _expanded_steps(definition, variables) + if steps is None: + steps = definition.get("steps") or [] + return [ + step + for step in steps + if isinstance(step, dict) and isinstance(step.get("pipeline"), dict) + ] + + def pipeline_values(definition, key, arguments=None): """The number each pipeline step sets `key` to (a `variable:` reference resolved; `arguments` override the variables; the largest where a step sets it twice), None for a step that sets none or a non-numeric one - - for scaling an inherited price along it (#851, #866).""" + for scaling an inherited price along it (#851, #866). A `for_each` + step is read per member, as `pipeline_steps` expands it.""" variables = dict(definition.get("variables") or {}) variables.update(arguments or {}) found = [] - for step in definition.get("steps") or []: - if not isinstance(step, dict) or not isinstance(step.get("pipeline"), dict): - continue + for step in pipeline_steps(definition, arguments): numbers = [ value for value in map(json.loads, _found(step, key, variables)) diff --git a/tests/test_inherited_cost.py b/tests/test_inherited_cost.py index 0973c830..882cd0d2 100644 --- a/tests/test_inherited_cost.py +++ b/tests/test_inherited_cost.py @@ -332,6 +332,88 @@ def test_a_shared_driver_name_still_prices_its_bucket(self): assert block["cold_minutes"] == 8.0 +def shot_list(count, num_frames=124): + return [{"prompt": f"p{n}", "num_frames": num_frames} for n in range(count)] + + +def shots_definition(name, shots): + return { + "id": name, + "variables": {"shots": shots}, + "cost_drivers": ["shots"], + "steps": [ + { + "name": "shot", + "for_each": "variable:shots", + "pipeline": PIPELINE, + "run": {"num_frames": "item:num_frames", "prompt": "item:prompt"}, + } + ], + } + + +class TestAListLongerThanTheFallbackBucket: + """A for_each step is one pipeline step per entry: an inline copy with + more entries than the default bucket the template ran is priced from + that bucket only once it is scaled by the expanded member count (#851 + regression - 6 x 124 frames was quoted as 3 x 124's minutes).""" + + CATALOG = [("templates/t", shots_definition("t", shot_list(3)))] + HISTORY = {"templates/t": [run(30 * MINUTE, {"shots": shot_list(3)})]} + + def test_the_template_sized_list_is_not_scaled(self): + block = inherited( + shots_definition("inline", shot_list(3)), self.CATALOG, self.HISTORY + ) + assert block["cold_minutes"] == 30.0 + assert "scaled" not in block + + def test_an_inline_six_shot_copy_scales_by_members(self): + block = inherited( + shots_definition("inline", shot_list(6)), self.CATALOG, self.HISTORY + ) + assert block["cold_minutes"] == 60.0 + assert "6 steps, template measured 3" in block["scaled"] + + def test_a_six_shot_argument_scales_by_members(self): + block = inherited( + shots_definition("inline", shot_list(3)), + self.CATALOG, + self.HISTORY, + {"shots": shot_list(6)}, + ) + assert block["cold_minutes"] == 60.0 + assert "6 steps, template measured 3" in block["scaled"] + + def test_shorter_shots_scale_by_members_and_frames(self): + block = inherited( + shots_definition("inline", shot_list(6, num_frames=62)), + self.CATALOG, + self.HISTORY, + ) + assert block["cold_minutes"] == 30.0 + assert "6 steps" in block["scaled"] and "62 vs 124" in block["scaled"] + + def test_the_plan_quotes_it_low_confidence(self, tmp_path): + inline = shots_definition("inline", shot_list(6)) + block = inherited(inline, self.CATALOG, self.HISTORY) + candidate = workflow_from_definition( + inline, str(tmp_path), str(tmp_path), str(tmp_path) + ) + with patch.object(dw.plan, "scan_models", return_value={"repos": []}): + plan = build_plan( + candidate, + {}, + device="cpu", + lookup_sizes=False, + observed=lambda arguments: block, + ) + estimate = plan["estimate"] + assert estimate["basis"] == "inherited" + assert estimate["minutes"] == 60.0 + assert estimate["low_confidence"] is True + + class TestVariableFrameCountAndWithheldReason: """#851 bounce: a frame count written as `variable:num_frames` that the template never ran at still prices from the template's default bucket, and From bb546858dd854611d404f8fd727a8c44317e1c8b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:22:23 -0500 Subject: [PATCH 187/204] fix(server): #910 - a longer resident list takes the host-RAM fit projected_peak let the observed max over the memory key always beat the list-run fit, and the key leaves list length out by design. For a shape that holds its pipeline across the list (shots-batch, music-video, best-of-n-to-video, restore-long), an 8-shot request with history at 2 shots/20 GB and 4 shots/30 GB was gated at 30 GB by dispatch and plan.host_memory, while validate's host_memory_warnings, from the same rows' fixed+per-entry fit, warned ~50 GB. Now, when the definition does not release between iterations and the fit is larger than the observed max, the fit is the figure (basis "fit"), so dispatch, plan.host_memory and the validate warning agree. A list inside the observed range keeps the observed max (the fit is not larger there), and a released shape keeps it whatever the length: its peak is one entry's, not the list's (#901). Under-projecting is the failure this gate exists for, so the larger figure wins. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/host_ram_need.py | 18 +++++-- tests/test_host_ram_need.py | 99 ++++++++++++++++++++++++++++++++++++- 3 files changed, 114 insertions(+), 5 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 459883e4..9f3b6e63 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -104,7 +104,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run (a `for_each` counted per member, against the list the bucket ran, so a fallback to the default bucket re-prices a longer list) re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding`, `tests/test_inherited_cost.py::TestAListLongerThanTheFallbackBucket::test_an_inline_six_shot_copy_scales_by_members` | -| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit; with neither there is no figure. It never refuses; dispatch may delay a start on it (`JobManager._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | +| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit, unless the shape holds its pipeline across the list and the fit is the larger (the key leaves list length out, #910); with neither there is no figure. It never refuses; dispatch may delay a start on it (`JobManager._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestALongerListThanObserved::test_an_accumulating_list_takes_the_larger_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | | H3 audio hold, refine and guide layout | Five modules, imported one way: `dw/pipeline_processors/h3_rules.py` holds the names and rules, torch- and diffusers-free (`refine_problems`, the per-argument refine rules asked before validation and before the call; `not_h3`, the step rule; `snap_guide_length`, `guide_frame_problem`, `guide_chain_problems`, `guide_end_problem`; `CHAIN_CONTINUITY_MODES`; the chunk constants), `dw/pipeline_processors/h3_hold.py` puts the hold and refine blocks in (`insert_audio_hold`, `core_denoise_sequences`, `refines`), and `dw/pipeline_processors/h3_guides.py` the guide layout (`LAYOUT_ANCHORS`, `insert_guides`), which imports `h3_hold` and never the reverse; the blocks themselves, and the arithmetic they run, are `dw/pipeline_processors/h3_hold_steps.py` (`encode_audio_span`, `refine_sigmas`) and `dw/pipeline_processors/h3_guide_steps.py` (`splice_guide_rows`), which import diffusers when they load, so the two owners import them only inside `blocks()` and `guide_blocks()` (#790), and `h3_guide_steps` imports `h3_hold_steps` and never the reverse (`_with_guides` in `dw/pipeline_processors/pipeline.py` adds its blocks for a `guides` call), `dw/guides.py`: the `guides` validation check and the chain `continuity: "guide"` check, `guide_chain_errors`, which asks the `guides` check's own step rule (`_takes_guides_problem`, built on `h3_rules.not_h3` and `REFERENCE_WORKFLOWS`) and reads the mode names from `h3_rules.CHAIN_CONTINUITY_MODES`, the names `chain.py`'s `CONTINUITY_MODES` is zipped against (both registered in `dw/validation.py`; a `previous_result:` guide is checked at run time), `dw/hold_audio.py`: `hold_audio_errors`, `refine_strength_errors` | The one place dw modifies a diffusers modular pipeline's block graph: three blocks go in, before `set_timesteps`, `denoise` and `after_denoise`, in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio` / `refine_strength`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. `dw/introspection.py` builds the same graph without weights, hold included, so `get_pipeline_signature` lists a modular pipeline's block inputs. The copies of diffusers behaviour are pinned to the installed diffusers: `shifted_sigma_grid` (the spacing `refine_sigmas` rests on) to stock `set_timesteps`, `GUIDE_FRAMES_PER_CHUNK`/`GUIDE_LATENTS_PER_CHUNK` to the H3 video VAE's `clip_length` and latents per clip, and every name in `LAYOUT_ANCHORS` (`_fill_audio_positions` among them) to `before_denoise`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused`, `tests/test_h3_hold_audio.py::TestSignature::test_the_h3_signature_shows_hold_audio`, `tests/test_h3_refine.py::TestAnchors::test_inserted_before_denoise_in_every_shape`, `tests/test_h3_refine.py::TestSigmaGridDrift::test_the_full_grid_is_stock_set_timesteps`, `tests/test_h3_guides.py::TestChunkDrift::test_frames_per_chunk_is_the_vae_clip_length`, `tests/test_h3_guides.py::TestChunkDrift::test_latents_per_chunk_is_the_vae_tokens_per_clip`, `tests/test_h3_guides.py::TestAnchors::test_installed_diffusers_has_every_anchor`, `tests/test_guide_chain_validation.py::TestRefused::test_the_step_rule_is_the_guides_rule`, `tests/test_guide_chain_validation.py::TestRefused::test_every_run_mode_is_known_to_validate`, `tests/test_h3_rules.py::test_h3_rules_imports_neither_torch_nor_diffusers`, `tests/test_h3_rules.py::test_the_block_owners_import_no_diffusers` | diff --git a/dw/server/host_ram_need.py b/dw/server/host_ram_need.py index ce9c34f5..6aef2709 100644 --- a/dw/server/host_ram_need.py +++ b/dw/server/host_ram_need.py @@ -20,7 +20,10 @@ successful runs with the same key (the max, not a median: under- projecting is the failure this exists for); - `fit`: `host_memory_projection`'s projection for a list run, from this - workflow's own history; + workflow's own history - also ahead of `observed` when it is the larger + for a shape that holds its pipeline across the list, since the key + leaves list length out and a longer list outgrows the observed max + (#910); - otherwise None - no figure. """ @@ -28,7 +31,7 @@ import json import logging -from ..host_memory_projection import projected_mb +from ..host_memory_projection import projected_mb, releases_between_iterations from ..variables import VariableNotFoundError, replace_variables from ..vram_estimate import pipeline_identity from ..vram_inheritance import _found @@ -86,12 +89,21 @@ def projected_peak(definition, arguments, history, *, rows=None, list_entries=No them, and `list_entries` the request's list lengths - the fit's inputs, absent for an inline definition, which has no history by name. """ + fit = projected_mb(definition, list_entries, rows or []) key = memory_key(definition, arguments) if key is not None and history is not None: peaks = history.peaks_for_key(key) if peaks: + # The key leaves list length out, so a resident list longer than + # any observed one outgrows the observed max: the fit's growth is + # taken when it is the larger, as validate's warning quotes (#910) + if ( + fit is not None + and fit["projected_mb"] > max(peaks) + and not releases_between_iterations(definition) + ): + return fit["projected_mb"], "fit", fit["runs"] return max(peaks), "observed", len(peaks) - fit = projected_mb(definition, list_entries, rows or []) if fit is not None: return fit["projected_mb"], "fit", fit["runs"] return None diff --git a/tests/test_host_ram_need.py b/tests/test_host_ram_need.py index 01b1a627..fff27332 100644 --- a/tests/test_host_ram_need.py +++ b/tests/test_host_ram_need.py @@ -15,7 +15,7 @@ import pytest from fastapi.testclient import TestClient -from dw.host_memory_projection import projected_mb +from dw.host_memory_projection import host_memory_warnings, projected_mb from dw.server.app import create_app from dw.server.host_ram_need import host_memory_block, memory_key, projected_peak from dw.server.job_history import JobHistory @@ -189,6 +189,103 @@ def test_another_key_is_not_observed(self, tmp_path): assert other is None +def shots_definition(released=False): + """A for_each over a shot list, holding its pipeline resident across the + list (the accumulating shape) unless `released`.""" + step = { + "name": "shot", + "for_each": "variable:shots", + "pipeline": { + "configuration": {"component_type": "ModularPipeline"}, + "from_pretrained_arguments": {"model_name": "acme/video"}, + }, + "run": {"prompt": "item:prompt"}, + } + if released: + step["release_pipeline"] = True + return { + "id": "shots", + "variables": {"shots": [{"prompt": "a"}]}, + "steps": [step], + } + + +def shot_rows(*runs): + """Fit rows of (shot count, peak MB).""" + return [ + { + "host_memory_job_peak_rss_mb": peak, + "arguments": json.dumps({"shots": [{"prompt": "p"}] * count}), + } + for count, peak in runs + ] + + +class TestALongerListThanObserved: + """#910: the memory key leaves list length out, so for an accumulating + shape the observed max (from shorter lists) under-projects a longer one; + the fit's growth is taken when it is the larger, so dispatch, the plan's + host_memory and validate's warning agree.""" + + def _history(self, tmp_path, definition): + history = JobHistory(str(tmp_path / "jobs.sqlite")) + for job_id, count, peak in (("a", 2, 20000), ("b", 4, 30000)): + history.record( + recorded_job( + job_id, + definition, + peak, + arguments={"shots": [{"prompt": "p"}] * count}, + ) + ) + return history + + def test_an_accumulating_list_takes_the_larger_fit(self, tmp_path): + definition = shots_definition() + history = self._history(tmp_path, definition) + rows = shot_rows((2, 20000), (4, 30000)) + + need = projected_peak( + definition, + {"shots": [{"prompt": "p"}] * 8}, + history, + rows=rows, + list_entries={"shots": 8}, + ) + + assert need == (50000, "fit", 2) + warnings = host_memory_warnings(definition, {"shots": 8}, rows, 40000) + assert warnings and "~50000 MB" in warnings[0] + + def test_a_list_inside_the_observed_range_keeps_the_observed_max(self, tmp_path): + definition = shots_definition() + history = self._history(tmp_path, definition) + + need = projected_peak( + definition, + {"shots": [{"prompt": "p"}] * 3}, + history, + rows=shot_rows((2, 20000), (4, 30000)), + list_entries={"shots": 3}, + ) + + assert need == (30000, "observed", 2) + + def test_a_released_list_keeps_the_observed_max(self, tmp_path): + definition = shots_definition(released=True) + history = self._history(tmp_path, definition) + + need = projected_peak( + definition, + {"shots": [{"prompt": "p"}] * 8}, + history, + rows=shot_rows((2, 20000), (4, 30000)), + list_entries={"shots": 8}, + ) + + assert need == (30000, "observed", 2) + + def old_shape_database(path): """A jobs.sqlite from before the memory_key column, holding one successful run with a peak.""" From 2645e7a93bd1780eb53343cbaf0d93af95dc51f2 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:22:43 -0500 Subject: [PATCH 188/204] refactor(ui): extract AlertToggle from App.svelte App.svelte was 604 lines (ceiling 600). The bell button moves to lib/AlertToggle.svelte, and alertsOn with toggleAlerts to the shared lib/alerts.svelte.ts store, which announce() and the chime-unlock listener still read. Behaviour is unchanged; the toggle gets tests. Co-Authored-By: Claude Opus 5.5 --- ui/src/App.svelte | 46 ++++------------------------- ui/src/lib/AlertToggle.svelte | 21 ++++++++++++++ ui/src/lib/AlertToggle.test.ts | 53 ++++++++++++++++++++++++++++++++++ ui/src/lib/alerts.svelte.ts | 25 ++++++++++++++++ 4 files changed, 105 insertions(+), 40 deletions(-) create mode 100644 ui/src/lib/AlertToggle.svelte create mode 100644 ui/src/lib/AlertToggle.test.ts create mode 100644 ui/src/lib/alerts.svelte.ts diff --git a/ui/src/App.svelte b/ui/src/App.svelte index 13bfe6c6..adb04525 100644 --- a/ui/src/App.svelte +++ b/ui/src/App.svelte @@ -3,8 +3,6 @@ import { cardReadings, meterText, meterTitle } from './lib/cardMemory' import { BookOpen, - Bell, - BellOff, Braces, ChevronDown, KeyRound, @@ -26,12 +24,8 @@ runningIds, tabTitle, } from './lib/jobWatch' - import { - alertFinished, - alertMode, - enableAlerts, - unlockChime, - } from './lib/finishAlert' + import { alertFinished, alertMode, unlockChime } from './lib/finishAlert' + import { alerts } from './lib/alerts.svelte' import { storageGet, storageSet } from './lib/storage' import type { HealthInfo, JobDetail, MemoryInfo } from './lib/types' import Sidebar from './lib/Sidebar.svelte' @@ -39,6 +33,7 @@ import KeyboardHelp from './lib/KeyboardHelp.svelte' import StatusPopover from './lib/StatusPopover.svelte' import TokenPopover from './lib/TokenPopover.svelte' + import AlertToggle from './lib/AlertToggle.svelte' import ConfirmDialog from './lib/ui/ConfirmDialog.svelte' import WorkflowsPage from './lib/pages/WorkflowsPage.svelte' import WorkflowPage from './lib/pages/WorkflowPage.svelte' @@ -81,7 +76,6 @@ let runningJob = $state(null) let othersRunning = $state(0) let endedMark = $state(null) - let alertsOn = $state(storageGet('alert-finished', false)) async function watchJobs(info: HealthInfo | null) { const next = nextWatch(watched, info) @@ -108,24 +102,10 @@ if (job.status === 'succeeded') notify.success(text) else notify.error(text) } - if (alertsOn && document.hidden) + if (alerts.on && document.hidden) alertFinished(text, wsHref(job.workspace, 'jobs', job.id)) } - async function toggleAlerts() { - if (alertsOn) { - alertsOn = false - } else if (await enableAlerts()) { - alertsOn = true - } else { - notify.error( - 'The browser blocked notifications for this site - allow them in its site settings', - ) - return - } - storageSet('alert-finished', alertsOn) - } - $effect(() => { document.title = endedMark ?? tabTitle(runningJob, othersRunning) }) @@ -136,7 +116,7 @@ // A chime may only play after a gesture; after a reload the first // click anywhere is that gesture const unlock = () => { - if (alertsOn && alertMode() === 'chime') unlockChime() + if (alerts.on && alertMode() === 'chime') unlockChime() } document.addEventListener('visibilitychange', back) window.addEventListener('pointerdown', unlock, { once: true }) @@ -338,21 +318,7 @@ - + diff --git a/ui/src/lib/AlertToggle.test.ts b/ui/src/lib/AlertToggle.test.ts new file mode 100644 index 00000000..2fb317df --- /dev/null +++ b/ui/src/lib/AlertToggle.test.ts @@ -0,0 +1,53 @@ +import { afterEach, expect, it, vi } from 'vitest' +import { cleanup, fireEvent, render, waitFor } from '@testing-library/svelte' + +const enableAlerts = vi.fn<() => Promise>() +const error = vi.fn() +vi.mock('./finishAlert', () => ({ + alertMode: () => 'chime', + enableAlerts: () => enableAlerts(), +})) +vi.mock('./toast', () => ({ notify: { error: (m: string) => error(m) } })) + +import AlertToggle from './AlertToggle.svelte' +import { alerts } from './alerts.svelte' +import { storageGet } from './storage' + +afterEach(() => { + cleanup() + alerts.on = false + enableAlerts.mockReset() + error.mockReset() + localStorage.clear() +}) + +const button = () => document.querySelector('button')! + +it('turns alerts on once the browser allows them, and remembers it', async () => { + enableAlerts.mockResolvedValue(true) + render(AlertToggle) + expect(button().getAttribute('aria-pressed')).toBe('false') + await fireEvent.click(button()) + await waitFor(() => expect(alerts.on).toBe(true)) + expect(button().getAttribute('aria-pressed')).toBe('true') + expect(button().getAttribute('aria-label')).toBe('stop job-finished alerts') + expect(storageGet('alert-finished', null)).toBe(true) +}) + +it('stays off and says why when the browser refuses', async () => { + enableAlerts.mockResolvedValue(false) + render(AlertToggle) + await fireEvent.click(button()) + await waitFor(() => expect(error).toHaveBeenCalledOnce()) + expect(alerts.on).toBe(false) + expect(button().getAttribute('title')).toMatch(/^chime when a job finishes/) +}) + +it('turns off without asking', async () => { + alerts.on = true + render(AlertToggle) + await fireEvent.click(button()) + await waitFor(() => expect(alerts.on).toBe(false)) + expect(enableAlerts).not.toHaveBeenCalled() + expect(storageGet('alert-finished', null)).toBe(false) +}) diff --git a/ui/src/lib/alerts.svelte.ts b/ui/src/lib/alerts.svelte.ts new file mode 100644 index 00000000..584b42fd --- /dev/null +++ b/ui/src/lib/alerts.svelte.ts @@ -0,0 +1,25 @@ +/** Whether a job finishing in a background tab raises an alert + * (finishAlert.ts) - a per-browser choice, so it is kept locally. Shared + * state: the toggle in the header sets it, and App's announce and its + * chime-unlock listener read it. */ +import { enableAlerts } from './finishAlert' +import { notify } from './toast' +import { storageGet, storageSet } from './storage' + +export const alerts = $state({ + on: storageGet('alert-finished', false), +}) + +export async function toggleAlerts(): Promise { + if (alerts.on) { + alerts.on = false + } else if (await enableAlerts()) { + alerts.on = true + } else { + notify.error( + 'The browser blocked notifications for this site - allow them in its site settings', + ) + return + } + storageSet('alert-finished', alerts.on) +} From 030968b648d0360b360b073255c4c671ae7c86cc Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:22:43 -0500 Subject: [PATCH 189/204] perf(ui): order gallery runs with a Map, not indexOf The visible derived sorted with runs.indexOf in the comparator and built runs with includes - quadratic, ~208 ms a keystroke at 10k files. Each run's place is now its first file's index from a Map; the order is the same, held by the existing newest-first-between-runs test. Co-Authored-By: Claude Opus 5.5 --- ui/src/lib/pages/GalleryPage.svelte | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ui/src/lib/pages/GalleryPage.svelte b/ui/src/lib/pages/GalleryPage.svelte index 59b864e4..f0d0f965 100644 --- a/ui/src/lib/pages/GalleryPage.svelte +++ b/ui/src/lib/pages/GalleryPage.svelte @@ -78,13 +78,13 @@ roles.includes(roleOf(f)) && (!starredOnly || f.starred), ) - const runs: string[] = [] - for (const f of shown) if (!runs.includes(runKey(f))) runs.push(runKey(f)) + // Each run's place is the index of its first (newest) file. Built + // back to front so the first file's index is the one that sticks; a + // Map keeps this linear at thousands of files + const runs = new Map(shown.map((f, i) => [runKey(f), i] as const).reverse()) + const place = (f: GalleryFile) => runs.get(runKey(f))! const rank = (f: GalleryFile) => Number(roleOf(f) === 'intermediate') - return shown.sort( - (a, b) => - runs.indexOf(runKey(a)) - runs.indexOf(runKey(b)) || rank(a) - rank(b), - ) + return shown.sort((a, b) => place(a) - place(b) || rank(a) - rank(b)) }) async function toggleStar() { if (!selected) return From 46506471663cf3a4914b804545765e14eb1b7be5 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:23:56 -0500 Subject: [PATCH 190/204] fix(server): close JobHistory's SQLite connections at the end of each block _connect returned a bare connection, and `with connection:` only commits or rolls back - the connection was closed by GC, which checkpoints the WAL at an arbitrary later moment and rewrote jobs.sqlite under a test watching its bytes. _connect is now a context manager that commits on success, rolls back on an exception and always closes; every call site already used it as `with self._connect() as connection`, so none change. Co-Authored-By: Claude Opus 5.5 --- dw/server/job_history.py | 14 ++++++++--- tests/test_job_history_wal.py | 44 +++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 3ca80c22..448269be 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -12,6 +12,7 @@ `device` in SQL when it is first opened, so no reader parses the label. """ +import contextlib import json import logging import os @@ -172,6 +173,7 @@ def __init__(self, db_path): " WHERE device IS NOT NULL" ) + @contextlib.contextmanager def _connect(self): # WAL mode lets a reader (the web UI polling job status, an MCP # get_job call) proceed without blocking behind whatever write the @@ -180,9 +182,15 @@ def _connect(self): # duration of a write. journal_mode is a property of the database # file, not the connection, but PRAGMA is cheap and idempotent, so # it is set on every connect rather than assumed to have stuck. - connection = sqlite3.connect(self.db_path, timeout=5) - connection.execute("PRAGMA journal_mode=WAL") - return connection + # + # The block commits on success and rolls back on an exception (what + # sqlite3's own context manager does), then closes the connection: + # one left to GC checkpoints the WAL whenever it is collected, + # rewriting jobs.sqlite at an arbitrary later moment. + with contextlib.closing(sqlite3.connect(self.db_path, timeout=5)) as connection: + connection.execute("PRAGMA journal_mode=WAL") + with connection: + yield connection def record(self, job): # The spec's workflow_name/warnings are derived; keep what rerun needs diff --git a/tests/test_job_history_wal.py b/tests/test_job_history_wal.py index b6ccab67..e1c5354f 100644 --- a/tests/test_job_history_wal.py +++ b/tests/test_job_history_wal.py @@ -12,3 +12,47 @@ def test_the_jobs_database_uses_wal_mode(tmp_path): (mode,) = connection.execute("PRAGMA journal_mode").fetchone() assert mode.lower() == "wal" + + +def test_every_connection_is_closed_when_its_block_ends(tmp_path, monkeypatch): + """A connection left to GC checkpoints the WAL whenever it is collected, + rewriting jobs.sqlite under whatever is watching it; each one closes at + the end of its `with` block instead.""" + import sqlite3 + + import pytest + + import dw.server.job_history as job_history + + opened = [] + real_connect = sqlite3.connect + + def tracking_connect(*args, **kwargs): + connection = real_connect(*args, **kwargs) + opened.append(connection) + return connection + + monkeypatch.setattr(job_history.sqlite3, "connect", tracking_connect) + history = JobHistory(str(tmp_path / "jobs.sqlite")) + history.get("missing") + history.watermark() + + assert len(opened) == 3 + for connection in opened: + with pytest.raises(sqlite3.ProgrammingError): + connection.execute("SELECT 1") + + +def test_a_failed_block_rolls_back_and_still_closes(tmp_path): + import sqlite3 + + import pytest + + history = JobHistory(str(tmp_path / "jobs.sqlite")) + with pytest.raises(RuntimeError): + with history._connect() as connection: + connection.execute("INSERT INTO jobs (id, status) VALUES ('x', 'failed')") + raise RuntimeError("boom") + with pytest.raises(sqlite3.ProgrammingError): + connection.execute("SELECT 1") + assert history.get("x") is None From af7d61b3407d4c228af403e10fd0620b63823e17 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:24:02 -0500 Subject: [PATCH 191/204] fix(timing): take denoise start outside a fullgraph-compiled denoiser denoise_start_hook registered a forward pre-hook on every denoiser. A denoiser compiled whole (compile without repeated_blocks) runs its hooks inside the compiled _call_impl, where Dynamo cannot trace the clock read (time.monotonic / device sync): fullgraph: true failed every job at the first denoise step, and a non-fullgraph compile broke its graph there. A whole-compiled denoiser (_compiled_call_impl set) now has that callable wrapped for the call instead: the wrapper marks once, outside the compiled region, then hands over, and is restored on the way out. The mark is the same instant a pre-hook stamped, so first_pass_s and the slow-first-pass warning (#924) read the same. Eager and repeated_blocks denoisers keep the self-removing pre-hook on the eager top-level module. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/pipeline_processors/timing.py | 42 ++++++++++++++++++++--- tests/test_step_timing.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 96 insertions(+), 5 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 4649da13..386a90bc 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,7 +31,7 @@ one to open. | Quantized component cache | `dw/quantized_cache.py`: `load_components`, `from_pretrained`, `cache_key` | A key that can't pin the weights isn't cached, the cache never fails a job, and the save comes before pruning, so an entry is the whole component. | `tests/test_quantized_cache.py` | | LoRAs | `dw/pipeline_processors/adapters.py`: `active_loras`, `load_loras` | A `loras` entry whose `model_name` is null is switched off. | `tests/test_lora_disable.py::TestLoadLoras::test_an_all_null_entry_loads_nothing` | | Step progress | `dw/pipeline_processors/progress.py`: `reported_progress_bars` | A pipeline with no step callback reports its denoise steps through its progress bars. | `tests/test_modular_progress.py::test_every_advance_of_the_bar_is_reported` | -| Step timing | `dw/pipeline_processors/timing.py`: `StepTimer`, `step_timer`, `timed_step`, `mark` | Only a pipeline step that ran is timed, on the worker's monotonic clock: `step_timer` opens a `StepTimer` for a step with a `pipeline` or `pipeline_reference` and None for any other, which also hides an outer step's timer from a composed child. Load, call, denoise start, each pass and save reach the open timer through `mark`, and nowhere else. A span with a missing mark is null, never guessed. `record_step` puts the reading on the step's manifest entry and `step_end`, and a reused step gets no `timing`. A chain sums its pipeline calls into one `timing` with `segments: N`. `ManifestEntry.timing` declares the shape (`docs/WORKSPACES.md` *Step timing*). | `tests/test_step_timing.py::test_a_missing_denoise_start_nulls_encode_and_first_pass`, `tests/test_step_timing.py::test_a_reused_step_carries_no_timing_even_when_one_is_passed`, `tests/test_step_timing.py::test_a_chain_sums_its_segments_and_takes_one_median_over_both`, `tests/test_step_timing.py::test_a_task_step_run_through_the_step_loop_has_no_timing`, `tests/test_step_timing.py::test_the_denoise_hook_marks_the_first_forward_once_and_removes_itself` | +| Step timing | `dw/pipeline_processors/timing.py`: `StepTimer`, `step_timer`, `timed_step`, `mark` | Only a pipeline step that ran is timed, on the worker's monotonic clock: `step_timer` opens a `StepTimer` for a step with a `pipeline` or `pipeline_reference` and None for any other, which also hides an outer step's timer from a composed child. Load, call, denoise start, each pass and save reach the open timer through `mark`, and nowhere else. Denoise start is never taken inside a compiled region: a whole-compiled denoiser (`_compiled_call_impl` set) has that callable wrapped for the call instead of a forward pre-hook, which `fullgraph: true` cannot trace. A span with a missing mark is null, never guessed. `record_step` puts the reading on the step's manifest entry and `step_end`, and a reused step gets no `timing`. A chain sums its pipeline calls into one `timing` with `segments: N`. `ManifestEntry.timing` declares the shape (`docs/WORKSPACES.md` *Step timing*). | `tests/test_step_timing.py::test_a_missing_denoise_start_nulls_encode_and_first_pass`, `tests/test_step_timing.py::test_a_reused_step_carries_no_timing_even_when_one_is_passed`, `tests/test_step_timing.py::test_a_chain_sums_its_segments_and_takes_one_median_over_both`, `tests/test_step_timing.py::test_a_task_step_run_through_the_step_loop_has_no_timing`, `tests/test_step_timing.py::test_the_denoise_hook_marks_the_first_forward_once_and_removes_itself`, `tests/test_step_timing.py::test_a_fullgraph_compiled_denoiser_runs_and_still_gets_the_start_mark` | | Step-cache skip counts | `dw/pipeline_processors/timing.py`: `CacheCounter`, `counting_forwards`, `report_cache_stats`, `StepTimer.cache_counted`; `dw/pipeline_processors/components.py`: `stateful_cache_context`, `get_cache_transformer` | For `first_block` and `mag`, a forward hook on the cache transformer counts computed or skipped once per forward, reading the leader hook's `should_compute` inside the cond/uncond `cache_context` the pipeline opened, so classifier-free guidance counts both forwards. `stateful_cache_context` registers the hook for one pipeline call; its `finally` emits `cache_stats` before `_reset_stateful_cache()` and removes the hook after the event. Private diffusers state that has moved gives null counts and one `cache_stats_unavailable` warning, and never fails the run; `taylorseer`, `faster` and `text_kv` report type and config only, with no warning. The counts reach the step's `timing.cache` only through `StepTimer.cache_counted`; a chain sums its calls' counts, and a null in any call nulls the sum. A step with no `cache` has no `cache` key and no event (`docs/ACCELERATION.md` *Reading the skip counts*). | `tests/test_cache_stats.py::test_cfg_counts_two_forwards_per_step_and_they_add_up`, `tests/test_cache_stats.py::test_the_event_names_the_step_and_segment_and_goes_out_before_the_reset`, `tests/test_cache_stats.py::test_moved_hook_state_nulls_the_counts_warns_once_and_the_run_finishes`, `tests/test_cache_stats.py::test_the_private_diffusers_names_are_where_the_counter_reads_them`, `tests/test_cache_stats.py::test_the_timer_sums_counts_and_nulls_on_any_null`, `tests/test_cache_stats.py::test_types_without_a_decision_report_type_only_and_no_warning` | | Community pipelines | `dw/community_pipelines/` (e.g. `pipeline_ltx2_refine.py`: `LTX2RefinePipeline`), `dw/introspection.py`: `community_pipelines`, `load_allowed_class` | A pipeline dw ships is named by its dotted path (`dw.community_pipelines..`), loads untrusted because it is under `dw`, and is listed by `list_pipelines` and resolved by `get_pipeline_signature` - only the modules in that directory, never a caller-chosen import. `LTX2RefinePipeline` VAE-encodes `video` at `width` x `height` and passes it unnormalized as `latents`, because `LTX2Pipeline.prepare_latents` normalizes 5-D latents itself. | `tests/test_ltx2_refine_pipeline.py::TestIntrospection::test_only_the_shipped_modules_resolve`, `tests/test_ltx2_refine_pipeline.py::TestEncode::test_the_latents_are_left_unnormalized_for_the_parent` | | Adding a task | `dw/tasks/registry.py`: `register_command`, `RegistryTable` | A task is a function registered with `@register_command`, and the signature of its implementation is its argument schema. Its validate-time rules are declared on the same decorator - `domains`, `choices`, `static_check`, `whole_numbers` and `media_arguments` - and `TASK_ARGUMENT_DOMAINS`, `TASK_ARGUMENT_CHOICES`, `TASK_STATIC_CHECKS`, `TASK_WHOLE_NUMBER_ARGUMENTS` and `TASK_MEDIA_ARGUMENTS` are read-only views derived from them, so there is no second table to forget (#692). The registry lives in `dw/tasks/registry.py`; `dw/tasks/task.py` re-exports it, and a task module may register its own handler, as `dw/tasks/beats.py` does, provided `dw/tasks/task.py` imports it (`docs/TASKS.md` *Adding a task*). The built-in handlers are grouped by family the same way - `dw/tasks/audio_handlers.py`, `dw/tasks/video_handlers.py`, `dw/tasks/finish_handlers.py` and `dw/tasks/model_handlers.py` (#790) - and `dw/tasks/task.py` keeps the general-purpose commands, the processor dispatch and `Task`. | `tests/test_task_discovery.py::TestDescribeTask::test_signature_becomes_the_schema`, `tests/test_task_registry_rules.py::test_every_derived_table_is_the_registrys_view`, `tests/test_task_registry_rules.py::test_every_static_check_is_registered_to_exactly_one_command`, `tests/test_task_registry_rules.py::test_every_media_argument_is_a_parameter_of_its_command` | diff --git a/dw/pipeline_processors/timing.py b/dw/pipeline_processors/timing.py index b3dcfcf1..4993c81b 100644 --- a/dw/pipeline_processors/timing.py +++ b/dw/pipeline_processors/timing.py @@ -3,7 +3,8 @@ A `StepTimer` is opened around one step by the step loop (`dw/workflow_run.py`) and fed marks from wherever they happen: the load around `create_step_action`, the pipeline call in `Pipeline._call_pipeline`, -the first denoiser forward (a self-removing pre-forward hook) or the opening +the first denoiser call (a self-removing pre-forward hook, or a wrapper +around a whole-compiled denoiser's compiled call) or the opening of a modular pipeline's denoise bar (`progress.py`), each step callback or bar advance, and the result save. Its `timing()` is the dict a step's manifest entry carries as `timing`. @@ -287,12 +288,45 @@ def is_compiled(module): @contextlib.contextmanager def denoise_start_hook(module): - """Mark denoise start at the first forward of `module`, then remove the - hook: one clock read, so a compiled denoiser sees it once and never - again. Removed on the way out too, for a call that never reached it.""" + """Mark denoise start at the first call of `module`, then stop marking: + one clock read, so a compiled denoiser sees it once and never again. + Undone on the way out too, for a call that never reached it. + + Where the mark is taken depends on how the denoiser is compiled. Eager, or + compiled by its repeated blocks, it is a self-removing forward pre-hook on + the denoiser, which runs eagerly. Compiled whole (`compile` without + `repeated_blocks`, which `module.compile()` does by setting + `_compiled_call_impl`), a pre-hook would run inside the compiled + `_call_impl`: Dynamo cannot trace the clock read (`time.monotonic`, the + device sync), so `fullgraph: true` fails the first denoise step and a + non-fullgraph compile breaks its graph there. So a whole-compiled + denoiser has its `_compiled_call_impl` wrapped instead, for this call + only: the wrapper marks before handing over to the compiled callable, so + the mark is outside the compiled region and is the same instant a + pre-hook would have stamped, and `first_pass_s` and the slow-first-pass + warning (#924) read the same in both cases.""" if module is None or not hasattr(module, "register_forward_pre_hook"): yield return + + compiled_call = getattr(module, "_compiled_call_impl", None) + if compiled_call is not None: + marked = False + + def first_call(*args, **kwargs): + nonlocal marked + if not marked: + marked = True + mark("denoise_started") + return compiled_call(*args, **kwargs) + + module._compiled_call_impl = first_call + try: + yield + finally: + module._compiled_call_impl = compiled_call + return + handle = None def first_forward(_module, _inputs): diff --git a/tests/test_step_timing.py b/tests/test_step_timing.py index 59881636..3884ca3c 100644 --- a/tests/test_step_timing.py +++ b/tests/test_step_timing.py @@ -319,6 +319,63 @@ def test_the_denoise_hook_is_removed_when_forward_never_ran(): assert not module._forward_pre_hooks +def _compiled_linear(**options): + """A tiny Linear compiled in place the way `apply_compile` does it, or a + skip where torch.compile cannot run on this platform.""" + module = torch.nn.Linear(2, 2) + try: + module.compile(**options) + module(torch.zeros(1, 2)) + except Exception as error: # pragma: no cover - platform dependent + pytest.skip(f"torch.compile is unavailable here: {error}") + return module + + +def test_a_fullgraph_compiled_denoiser_runs_and_still_gets_the_start_mark(): + # A forward pre-hook runs inside the compiled `_call_impl`, where its + # clock read is untraceable and fullgraph=True fails the first step + module = _compiled_linear(fullgraph=True) + original = module._compiled_call_impl + timer = StepTimer() + + with timed_step(timer): + timer.call_started() + with denoise_start_hook(module): + module(torch.zeros(1, 2)) + first = timer._segments[0].denoise_started + module(torch.zeros(1, 2)) + + assert first is not None + assert timer._segments[0].denoise_started == first + assert module._compiled_call_impl is original + + +def test_a_compiled_denoiser_is_restored_when_forward_never_ran(): + module = torch.nn.Linear(2, 2) + original = lambda *args, **kwargs: None # noqa: E731 + module._compiled_call_impl = original + + with denoise_start_hook(module): + assert module._compiled_call_impl is not original + + assert module._compiled_call_impl is original + + +def test_repeated_blocks_compiled_keeps_the_pre_hook_on_the_eager_parent(): + child = _compiled_linear(fullgraph=True) + parent = torch.nn.Sequential(child) + timer = StepTimer() + + with timed_step(timer): + timer.call_started() + with denoise_start_hook(parent): + assert len(parent._forward_pre_hooks) == 1 + parent(torch.zeros(1, 2)) + assert not parent._forward_pre_hooks + + assert timer._segments[0].denoise_started is not None + + def test_a_plain_module_is_not_compiled(): assert is_compiled(torch.nn.Sequential(torch.nn.Linear(2, 2))) is False From ef5f1e937a194572a32fe02a819e3d807ecbf776 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:36:45 -0500 Subject: [PATCH 192/204] refactor(plan): move driver scaling into dw/cost_scaling.py (ratchet) plan.py was 1,315 lines, over the 1,100 module ceiling. The cost-driver scaling helpers and the basis labels move out unchanged; plan.py imports them back, and dw/server/deps.py imports scale_curated from its new home. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/cost_scaling.py | 300 +++++++++++++++++++++++++++++++++++++++++++ dw/plan.py | 293 ++---------------------------------------- dw/server/deps.py | 2 +- tests/test_plan.py | 2 +- 5 files changed, 315 insertions(+), 284 deletions(-) create mode 100644 dw/cost_scaling.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e445d96e..165af7fe 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -97,7 +97,7 @@ one to open. | Bound acknowledgement | `dw/plan.py`: `fingerprint`, `dw/server/admission.py`: `check_bound_acknowledgement` | A bound `acknowledged_cost` is refused with 409 when the plan's fingerprint or its required downloads changed. The fingerprint is keyed only by what can change the output: `DOCUMENTATION_KEYS` (`cost`, `cost_drivers`, `vram_estimate`, descriptions, catalog shape/traits), a constraint's `reason` and a result's role, subfolder and file name are left out, of the workflow and of each composed child; every schema key is sorted into left out or kept. A number digests by value, not spelling (`12.0` as `12`), and the seed stays out (#85). | `tests/test_plan.py::TestFingerprintKeys::test_the_exclusion_lists_are_pinned`, `tests/test_plan.py::TestTheCatalogH3Template::test_a_client_round_trip_keeps_it`, `tests/test_plan.py::TestFingerprintKeys::test_every_top_level_schema_key_is_sorted`, `tests/test_server.py::TestBoundRerun::test_a_rerun_bound_to_a_stale_plan_is_refused`, `tests/test_server.py::TestBoundAcknowledgement::test_a_matching_fingerprint_queues` | | Per-repo gating | `dw/plan.py`: `build_plan` | Each gated repo, including a `loras` entry's repo, is probed and reported on its own. | `tests/test_plan.py::TestDownloadsRequired::test_a_gated_repo_this_token_lacks_access_to_is_blocked` | | Observed cost | `dw/server/observed_cost.py`: `declared_drivers`, `ObservedCosts._on_this_card`, `ObservedCosts.card` | `observed` comes only from this box's finished jobs on the named card (the one `route` names, for `plan.estimate`), else the server's own card; `ObservedCosts.card` owns a card's (kind, name, default). Rows are matched by card name whatever index they ran on (a job from before jobs carried a `device` counts only while the server is on the default card), and never writes `cost`. Cold and warm runs are reported separately. | `tests/test_observed_cost.py::TestColdIsNotWarm::test_the_two_are_reported_separately_each_with_its_runs`, `tests/test_observed_cost.py::TestByCard::test_only_rows_from_this_card_count_whichever_index_they_had`, `tests/test_observed_cost.py::TestByCard::test_rows_from_before_jobs_carried_a_device_do_not_count_off_the_default_card` | -| The estimate | `dw/plan.py`: `estimate`, `_tempered` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | +| The estimate | `dw/plan.py`: `estimate`, `_tempered`; the driver scaling is `dw/cost_scaling.py`: `scale_curated`, `_driver_scaling` | `plan.estimate` quotes the observed figure ahead of the curated one, and blends it toward the curated figure below three runs. A `cost` entry is measured at the workflow's variable defaults; a run moving `num_inference_steps`, `width`/`height` or `num_frames` (per list entry, summed) scales it in proportion (`scale_curated`, which `dw/server/deps.py` reuses for an inherited figure), basis `derived` with `scaled` naming each axis, times the list-length re-price, and that scaled figure is what a thin observed one blends toward (#866). | `tests/test_plan.py::TestObservedEstimate::test_history_beats_a_curated_figure`, `tests/test_plan.py::TestLowConfidenceObservedEstimate::test_a_single_run_blends_toward_the_curated_figure`, `tests/test_plan.py::TestDriverScaling::test_steps_canvas_and_frames_multiply`, `tests/test_plan.py::TestScalingAccuracy::test_within_thirty_percent` | | Cost drivers | `dw/server/observed_cost.py`: `declared_drivers` | A cost driver that names no declared variable is dropped. | `tests/test_observed_cost.py::TestComparability::test_a_driver_naming_no_variable_is_dropped`, `tests/test_observed_cost.py::TestTheCatalogsDriversAreReal::test_every_declared_driver_is_a_variable_of_its_workflow` | | Raw workflow GET | `dw/server/routes/library.py`: `get_workflow` | The raw workflow GET is served verbatim, with no `observed`, because the editor saves what it reads. | — | | VRAM projection | `dw/vram_estimate.py`, `dw/guides.py`: `guide_lengths` | A template's `vram_estimate` is projected per step after `for_each` expansion, and only the largest step over the ceiling is reported. With `bytes_per_guide_voxel` declared, each guide the step lays in (its own non-null `guides` plus a guide chain's) adds its snapped frames times the canvas; an unprobeable clip is charged at `num_frames`, and validate, admission and run project the same number. | `tests/test_vram_estimate.py::test_only_the_largest_over_ceiling_step_is_reported_once_per_cost_entry`, `tests/test_vram_guides.py::test_validate_apply_and_required_agree_on_a_guided_definition` | diff --git a/dw/cost_scaling.py b/dw/cost_scaling.py new file mode 100644 index 00000000..75badf16 --- /dev/null +++ b/dw/cost_scaling.py @@ -0,0 +1,300 @@ +"""How far a run's arguments moved it from the defaults a curated `cost` +figure was measured at, and that figure scaled to match (#267, #593, #866). + +A workflow's `cost` entry is measured at its variable defaults. The +variables its `cost_drivers` names are the ones the author says move that +figure: a generic diffusers driver (`num_inference_steps`, `width` and +`height`, `num_frames`, per list entry summed) scales it in proportion +(`scale_curated`); any other declared driver that moved has no rule to +scale by, so the figure is kept and flagged. `dw/plan.py` prices the run +and applies the answer (`_scaled`); `dw/server/deps.py` scales an inherited +figure with the same `scale_curated`, so an inline copy and its template +quote one request alike. The basis labels an estimate carries live here +because the scaled figure is the first place one is chosen. + +Nothing here knows a model: the drivers are the workflow's own +declaration, and the defaults its own `variables`. +""" + +import json +import math + +# The basis an estimate quotes its minutes on +UNKNOWN = "unknown" +CATALOG = "catalog" +PER_ENTRY = "per_entry" +DERIVED = "derived" +OTHER_DEVICE = "other_device" +OBSERVED = "observed" +INHERITED = "inherited" + +COST_DRIVERS_KEY = "cost_drivers" + + +def _declared_drivers(definition): + """The variables the author says move this workflow's cost - the same + rule `dw/server/observed_cost.py`'s `declared_drivers` applies, kept + local so this module does not import the server package (#267).""" + drivers = definition.get(COST_DRIVERS_KEY) + if not isinstance(drivers, list): + return [] + variables = definition.get("variables") or {} + return [name for name in drivers if isinstance(name, str) and name in variables] + + +def _driver_comparable(value): + """A driver value as something hashable and stable across JSON round + trips, mirroring `dw/server/observed_cost.py`'s `_comparable`.""" + if isinstance(value, bool) or value is None: + return value + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return value + return json.dumps(value, sort_keys=True, default=str) + + +def _numeric_fields(entries): + """{field: {values}} over the numeric fields of a list's dict entries. A + numeric string ("243") counts as its number, as `_driver_comparable` does + for a scalar driver.""" + fields = {} + if not isinstance(entries, list): + return fields + for entry in entries: + if not isinstance(entry, dict): + continue + for key, value in entry.items(): + number = _driver_comparable(value) + if isinstance(number, float) and math.isfinite(number): + fields.setdefault(key, set()).add(number) + return fields + + +LIST_ENTRY_COST_FIELDS = ("num_frames",) + + +def _list_entry_field_shifted(default_entries, effective_entries): + """Whether a list driver's entries carry a numeric field (a per-shot + `num_frames`, say) with a value none of the default entries had. The + curated figure was measured over the default entries' values, so a value + outside them has no matching bucket - the #267 rule applied inside a + list driver (#593).""" + measured = _numeric_fields(default_entries) + for field, values in _numeric_fields(effective_entries).items(): + # Only a field that sets how long a shot renders moves its cost; a + # start_frame or cut_frames is a position that moves with the count + if ( + field in LIST_ENTRY_COST_FIELDS + and field in measured + and not values <= measured[field] + ): + return True + return False + + +# The generic diffusers argument names a curated figure scales along when a +# run moves one off the default it was measured at (#866). Any other declared +# scalar driver (H3's weights_dtype, a music audio_duration) has no rule to +# scale by, so moving it keeps the figure and flags it (`_scaled`). +STEPS_DRIVER = "num_inference_steps" +FRAMES_DRIVER = "num_frames" +SCALABLE_DRIVERS = (STEPS_DRIVER, "width", "height", FRAMES_DRIVER) + + +def _positive(value): + """A driver value as a positive finite number, or None.""" + number = _driver_comparable(value) + if isinstance(number, float) and math.isfinite(number) and number > 0: + return number + return None + + +def _shown(number): + return f"{round(number, 1):g}" + + +def scale_curated(minutes, defaults, effective): + """`minutes` measured at `defaults`, scaled to a run at `effective`, as + (minutes, axes), or None when a moved value is not a number (#866). + + Both are {driver: value} over `SCALABLE_DRIVERS`. Plain proportional + scaling - steps/steps0 x (w.h)/(w0.h0) x frames/frames0 - with no fixed + term for the model load and decode, so it over-quotes a little when a + value rises and under-quotes when one falls. `axes` names each axis that + moved, for a caller to read ("steps 30 vs 9"); empty when none did. + """ + moved = { + name + for name in SCALABLE_DRIVERS + if name in effective + and _driver_comparable(effective[name]) + != _driver_comparable(defaults.get(name)) + } + if not moved: + return minutes, [] + values = {} + for name in SCALABLE_DRIVERS: + if name not in effective: + continue + pair = (_positive(defaults.get(name)), _positive(effective[name])) + if None in pair: + if name in moved: + return None + continue + values[name] = pair + factor = 1.0 + axes = [] + if STEPS_DRIVER in moved: + was, now = values[STEPS_DRIVER] + factor *= now / was + axes.append(f"steps {_shown(now)} vs {_shown(was)}") + canvas = [name for name in ("width", "height") if name in moved] + for name in canvas: + was, now = values[name] + factor *= now / was + if canvas and "width" in values and "height" in values: + (w0, w), (h0, h) = values["width"], values["height"] + axes.append(f"canvas {_shown(w)}x{_shown(h)} vs {_shown(w0)}x{_shown(h0)}") + else: + axes.extend( + f"{name} {_shown(values[name][1])} vs {_shown(values[name][0])}" + for name in canvas + ) + if FRAMES_DRIVER in moved: + was, now = values[FRAMES_DRIVER] + factor *= now / was + axes.append(f"frames {_shown(now)} vs {_shown(was)}") + return minutes * factor, axes + + +def _entry_frames(entries): + """The per-entry `num_frames` of a list's entries as numbers, or None + when an entry carries none (or a non-numeric one).""" + if not isinstance(entries, list) or not entries: + return None + frames = [] + for entry in entries: + number = ( + _positive(entry.get(FRAMES_DRIVER)) if isinstance(entry, dict) else None + ) + if number is None: + return None + frames.append(number) + return frames + + +def _list_frames_scaling(name, default_entries, effective_entries, list_entries): + """A list driver's per-entry frames as (factor, axis) - (1.0, None) when + they did not move - or None when they moved in a way that cannot be + scaled (#593, #866). + + The frames a list renders are the sum over its entries. When the list's + length is already priced (`_repriced`, a `per_entry` rate), only the + mean per entry is left to scale, so the two multiply to the summed ratio + rather than counting the length twice. + """ + measured = _entry_frames(default_entries) + running = _entry_frames(effective_entries) + if measured is None or running is None: + # Nothing to sum on one side: only a frame count outside the values + # the default entries carried leaves no figure, as before (#593) + if _list_entry_field_shifted(default_entries, effective_entries): + return None + return 1.0, None + was, now = sum(measured), sum(running) + if was == now: + return 1.0, None + factor = now / was + if name in list_entries: + factor = (now / len(running)) / (was / len(measured)) + if math.isclose(factor, 1.0): + return 1.0, None + return factor, f"frames {_shown(now)} vs {_shown(was)} over {name}" + + +def _driver_scaling(definition, expanded, list_entries): + """How far the declared `cost_drivers` moved the run away from the + defaults the curated `cost` was measured at, as (factor, axes, unscaled), + or None when a scalable one moved to a value that is not a number, or a + list's frames moved in a way nothing can scale (#267, #866). + + A list driver's length is `_repriced`'s to price; here only its entries' + `num_frames` are. A scalar one in `SCALABLE_DRIVERS` scales in + proportion. Any other scalar driver that moved has no rule to scale by, + for every template: its name goes in `unscaled`, and the figure is kept + but flagged (#866, Q4). + """ + defaults = definition.get("variables") or {} + effective = expanded.get("variables") or {} + factor = 1.0 + axes = [] + at_default = {} + running = {} + unscaled = [] + for name in _declared_drivers(definition): + default_value = defaults.get(name) + if isinstance(default_value, list): + frames = _list_frames_scaling( + name, default_value, effective.get(name), list_entries + ) + if frames is None: + return None + factor *= frames[0] + if frames[1]: + axes.append(frames[1]) + continue + if name in list_entries: + continue + if name in SCALABLE_DRIVERS: + at_default[name] = default_value + running[name] = effective.get(name) + elif _driver_comparable(effective.get(name)) != _driver_comparable( + default_value + ): + unscaled.append(name) + scaled = scale_curated(factor, at_default, running) + if scaled is None: + return None + return scaled[0], scaled[1] + axes, unscaled + + +def _scaled(price, scaling): + """`price` scaled by a `_driver_scaling` answer: `derived` once any axis + moved (another device's figure stays `other_device`, as `_price` keeps it + for a resized list - #589), with `scaled` saying from what. A driver with + no rule to scale by that moved leaves the basis alone, and is named in + `differs` with `low_confidence` set: the figure was measured at another + setting of it (#866).""" + if scaling is None: + return {"minutes": None, "basis": UNKNOWN, "measured_on": None} + factor, axes, unscaled = scaling + if price["minutes"] is None: + return price + if unscaled: + price = {**price, "differs": list(unscaled), "low_confidence": True} + if not axes: + return price + basis = OTHER_DEVICE if price["basis"] == OTHER_DEVICE else DERIVED + return { + **price, + "minutes": price["minutes"] * factor, + "basis": basis, + "scaled": f"scaled from the curated {_shown(price['minutes'])} min: " + + ", ".join(axes), + } + + +def _entry_cost_field_shifted(definition, expanded): + """Whether any declared list driver carries a per-entry cost field outside + the values the default entries had.""" + defaults = definition.get("variables") or {} + effective = expanded.get("variables") or {} + return any( + isinstance(defaults.get(name), list) + and _list_entry_field_shifted(defaults[name], effective.get(name)) + for name in _declared_drivers(definition) + ) diff --git a/dw/plan.py b/dw/plan.py index 5ef6b9d4..c01eac7b 100644 --- a/dw/plan.py +++ b/dw/plan.py @@ -17,13 +17,24 @@ import hashlib import json import logging -import math import os from huggingface_hub import model_info from huggingface_hub import get_hf_file_metadata, hf_hub_url from huggingface_hub.utils import GatedRepoError, HFValidationError, validate_repo_id +from .cost_scaling import ( + CATALOG, + DERIVED, + INHERITED, + OBSERVED, + OTHER_DEVICE, + PER_ENTRY, + UNKNOWN, + _driver_scaling, + _entry_cost_field_shifted, + _scaled, +) from .elision import elide_definition from .hub_cache import repo_download_incomplete, scan_models from . import references @@ -319,14 +330,6 @@ def fingerprint(expanded, definition, annotations=None): return FINGERPRINT_PREFIX + _canonical_digest(doc) -UNKNOWN = "unknown" -CATALOG = "catalog" -PER_ENTRY = "per_entry" -DERIVED = "derived" -OTHER_DEVICE = "other_device" -OBSERVED = "observed" -INHERITED = "inherited" - # #301: a single run is not the same statistical basis as a dozen. Below # this many observed runs, an "observed" figure is tempered rather than # quoted at full authority - see `_tempered`. @@ -389,278 +392,6 @@ def _cached_minutes(minutes, cached_steps, total_steps): return round(minutes * remaining / total_steps, 1) -COST_DRIVERS_KEY = "cost_drivers" - - -def _declared_drivers(definition): - """The variables the author says move this workflow's cost - the same - rule `dw/server/observed_cost.py`'s `declared_drivers` applies, kept - local so this module does not import the server package (#267).""" - drivers = definition.get(COST_DRIVERS_KEY) - if not isinstance(drivers, list): - return [] - variables = definition.get("variables") or {} - return [name for name in drivers if isinstance(name, str) and name in variables] - - -def _driver_comparable(value): - """A driver value as something hashable and stable across JSON round - trips, mirroring `dw/server/observed_cost.py`'s `_comparable`.""" - if isinstance(value, bool) or value is None: - return value - if isinstance(value, (int, float)): - return float(value) - if isinstance(value, str): - try: - return float(value) - except ValueError: - return value - return json.dumps(value, sort_keys=True, default=str) - - -def _numeric_fields(entries): - """{field: {values}} over the numeric fields of a list's dict entries. A - numeric string ("243") counts as its number, as `_driver_comparable` does - for a scalar driver.""" - fields = {} - if not isinstance(entries, list): - return fields - for entry in entries: - if not isinstance(entry, dict): - continue - for key, value in entry.items(): - number = _driver_comparable(value) - if isinstance(number, float) and math.isfinite(number): - fields.setdefault(key, set()).add(number) - return fields - - -LIST_ENTRY_COST_FIELDS = ("num_frames",) - - -def _list_entry_field_shifted(default_entries, effective_entries): - """Whether a list driver's entries carry a numeric field (a per-shot - `num_frames`, say) with a value none of the default entries had. The - curated figure was measured over the default entries' values, so a value - outside them has no matching bucket - the #267 rule applied inside a - list driver (#593).""" - measured = _numeric_fields(default_entries) - for field, values in _numeric_fields(effective_entries).items(): - # Only a field that sets how long a shot renders moves its cost; a - # start_frame or cut_frames is a position that moves with the count - if ( - field in LIST_ENTRY_COST_FIELDS - and field in measured - and not values <= measured[field] - ): - return True - return False - - -# The generic diffusers argument names a curated figure scales along when a -# run moves one off the default it was measured at (#866). Any other declared -# scalar driver (H3's weights_dtype, a music audio_duration) has no rule to -# scale by, so moving it keeps the figure and flags it (`_scaled`). -STEPS_DRIVER = "num_inference_steps" -FRAMES_DRIVER = "num_frames" -SCALABLE_DRIVERS = (STEPS_DRIVER, "width", "height", FRAMES_DRIVER) - - -def _positive(value): - """A driver value as a positive finite number, or None.""" - number = _driver_comparable(value) - if isinstance(number, float) and math.isfinite(number) and number > 0: - return number - return None - - -def _shown(number): - return f"{round(number, 1):g}" - - -def scale_curated(minutes, defaults, effective): - """`minutes` measured at `defaults`, scaled to a run at `effective`, as - (minutes, axes), or None when a moved value is not a number (#866). - - Both are {driver: value} over `SCALABLE_DRIVERS`. Plain proportional - scaling - steps/steps0 x (w.h)/(w0.h0) x frames/frames0 - with no fixed - term for the model load and decode, so it over-quotes a little when a - value rises and under-quotes when one falls. `axes` names each axis that - moved, for a caller to read ("steps 30 vs 9"); empty when none did. - """ - moved = { - name - for name in SCALABLE_DRIVERS - if name in effective - and _driver_comparable(effective[name]) - != _driver_comparable(defaults.get(name)) - } - if not moved: - return minutes, [] - values = {} - for name in SCALABLE_DRIVERS: - if name not in effective: - continue - pair = (_positive(defaults.get(name)), _positive(effective[name])) - if None in pair: - if name in moved: - return None - continue - values[name] = pair - factor = 1.0 - axes = [] - if STEPS_DRIVER in moved: - was, now = values[STEPS_DRIVER] - factor *= now / was - axes.append(f"steps {_shown(now)} vs {_shown(was)}") - canvas = [name for name in ("width", "height") if name in moved] - for name in canvas: - was, now = values[name] - factor *= now / was - if canvas and "width" in values and "height" in values: - (w0, w), (h0, h) = values["width"], values["height"] - axes.append(f"canvas {_shown(w)}x{_shown(h)} vs {_shown(w0)}x{_shown(h0)}") - else: - axes.extend( - f"{name} {_shown(values[name][1])} vs {_shown(values[name][0])}" - for name in canvas - ) - if FRAMES_DRIVER in moved: - was, now = values[FRAMES_DRIVER] - factor *= now / was - axes.append(f"frames {_shown(now)} vs {_shown(was)}") - return minutes * factor, axes - - -def _entry_frames(entries): - """The per-entry `num_frames` of a list's entries as numbers, or None - when an entry carries none (or a non-numeric one).""" - if not isinstance(entries, list) or not entries: - return None - frames = [] - for entry in entries: - number = ( - _positive(entry.get(FRAMES_DRIVER)) if isinstance(entry, dict) else None - ) - if number is None: - return None - frames.append(number) - return frames - - -def _list_frames_scaling(name, default_entries, effective_entries, list_entries): - """A list driver's per-entry frames as (factor, axis) - (1.0, None) when - they did not move - or None when they moved in a way that cannot be - scaled (#593, #866). - - The frames a list renders are the sum over its entries. When the list's - length is already priced (`_repriced`, a `per_entry` rate), only the - mean per entry is left to scale, so the two multiply to the summed ratio - rather than counting the length twice. - """ - measured = _entry_frames(default_entries) - running = _entry_frames(effective_entries) - if measured is None or running is None: - # Nothing to sum on one side: only a frame count outside the values - # the default entries carried leaves no figure, as before (#593) - if _list_entry_field_shifted(default_entries, effective_entries): - return None - return 1.0, None - was, now = sum(measured), sum(running) - if was == now: - return 1.0, None - factor = now / was - if name in list_entries: - factor = (now / len(running)) / (was / len(measured)) - if math.isclose(factor, 1.0): - return 1.0, None - return factor, f"frames {_shown(now)} vs {_shown(was)} over {name}" - - -def _driver_scaling(definition, expanded, list_entries): - """How far the declared `cost_drivers` moved the run away from the - defaults the curated `cost` was measured at, as (factor, axes, unscaled), - or None when a scalable one moved to a value that is not a number, or a - list's frames moved in a way nothing can scale (#267, #866). - - A list driver's length is `_repriced`'s to price; here only its entries' - `num_frames` are. A scalar one in `SCALABLE_DRIVERS` scales in - proportion. Any other scalar driver that moved has no rule to scale by, - for every template: its name goes in `unscaled`, and the figure is kept - but flagged (#866, Q4). - """ - defaults = definition.get("variables") or {} - effective = expanded.get("variables") or {} - factor = 1.0 - axes = [] - at_default = {} - running = {} - unscaled = [] - for name in _declared_drivers(definition): - default_value = defaults.get(name) - if isinstance(default_value, list): - frames = _list_frames_scaling( - name, default_value, effective.get(name), list_entries - ) - if frames is None: - return None - factor *= frames[0] - if frames[1]: - axes.append(frames[1]) - continue - if name in list_entries: - continue - if name in SCALABLE_DRIVERS: - at_default[name] = default_value - running[name] = effective.get(name) - elif _driver_comparable(effective.get(name)) != _driver_comparable( - default_value - ): - unscaled.append(name) - scaled = scale_curated(factor, at_default, running) - if scaled is None: - return None - return scaled[0], scaled[1] + axes, unscaled - - -def _scaled(price, scaling): - """`price` scaled by a `_driver_scaling` answer: `derived` once any axis - moved (another device's figure stays `other_device`, as `_price` keeps it - for a resized list - #589), with `scaled` saying from what. A driver with - no rule to scale by that moved leaves the basis alone, and is named in - `differs` with `low_confidence` set: the figure was measured at another - setting of it (#866).""" - if scaling is None: - return {"minutes": None, "basis": UNKNOWN, "measured_on": None} - factor, axes, unscaled = scaling - if price["minutes"] is None: - return price - if unscaled: - price = {**price, "differs": list(unscaled), "low_confidence": True} - if not axes: - return price - basis = OTHER_DEVICE if price["basis"] == OTHER_DEVICE else DERIVED - return { - **price, - "minutes": price["minutes"] * factor, - "basis": basis, - "scaled": f"scaled from the curated {_shown(price['minutes'])} min: " - + ", ".join(axes), - } - - -def _entry_cost_field_shifted(definition, expanded): - """Whether any declared list driver carries a per-entry cost field outside - the values the default entries had.""" - defaults = definition.get("variables") or {} - effective = expanded.get("variables") or {} - return any( - isinstance(defaults.get(name), list) - and _list_entry_field_shifted(defaults[name], effective.get(name)) - for name in _declared_drivers(definition) - ) - - def _own_price(definition, expanded, list_entries, device, measured_entries): """The workflow's own price, scaled along the cost drivers that moved.""" own = _price(definition.get("cost"), device, list_entries, measured_entries or {}) diff --git a/dw/server/deps.py b/dw/server/deps.py index 0dc62b73..b25a2fbf 100644 --- a/dw/server/deps.py +++ b/dw/server/deps.py @@ -26,7 +26,7 @@ sets_differently, template_identity, ) -from ..plan import FRAMES_DRIVER, STEPS_DRIVER, scale_curated +from ..cost_scaling import FRAMES_DRIVER, STEPS_DRIVER, scale_curated from .observed_cost import declared_drivers from ..library import ASSETS_KIND, LORAS_KIND, PROMPTS_KIND, library_path from ..workspace import ( diff --git a/tests/test_plan.py b/tests/test_plan.py index 72b2adfd..9c1c6dc1 100644 --- a/tests/test_plan.py +++ b/tests/test_plan.py @@ -12,8 +12,8 @@ PLACEMENT_RESULT_KEYS, build_plan, gate_warnings, - scale_curated, ) +from dw.cost_scaling import scale_curated from dw.validation import unseeded_cache_warnings from dw.runs import new_run_id from dw.workflow import workflow_from_definition From 7f7694445fe5bf8e2f681e18db1cb784af0c4ae7 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:36:51 -0500 Subject: [PATCH 193/204] refactor(server): move the dispatch scan and host-RAM hold out of jobs.py JobManager's _next_dispatch, _forget_dispatch, _held_for_host_ram and _host_ram_wait, with _HostReading, _worker_rss_mb and HOST_RAM_RECHECK_SECONDS, move verbatim into dw/server/dispatch.py as the Dispatcher mixin JobManager inherits. No behaviour change; jobs.py is 1,061 lines, back under the 1,100 ceiling. The host-RAM gate tests' monkeypatch targets follow the names to dw.server.dispatch. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 4 +- dw/server/dispatch.py | 136 ++++++++++++++++++++++++++++++++++++++ dw/server/jobs.py | 129 ++---------------------------------- dw/server/pool.py | 2 +- tests/test_worker_pool.py | 8 +-- 5 files changed, 150 insertions(+), 129 deletions(-) create mode 100644 dw/server/dispatch.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e445d96e..21698277 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -104,7 +104,7 @@ one to open. | H3 Ref2VA VRAM numbers | `workflows/templates/minimax/` | Every Ref2VA template declares `base_gb` 16.0, `bytes_per_voxel` 28.71 and `gb_per_reference` 1.0. | `tests/test_h3_vram_ceiling.py::test_every_ref2va_minimax_template_declares_gb_per_reference` | | Inherited VRAM ceiling | `dw/vram_inheritance.py`, `dw/server/deps.py`: `ceiling_index` (the catalog parse is `_catalog_listing`, shared with the cost lookup below) | A workflow with no `vram_estimate` is matched against the catalog by pipeline identity, and is warned, never refused. | `tests/test_vram_inheritance.py::test_every_template_declaring_one_identity_declares_the_same_numbers` | | Inherited cost | `dw/server/deps.py`: `inherited_observed`, `dw/vram_inheritance.py`: `inherited_differences`, `dw/plan.py`: `_observed` (basis `inherited`), `dw/server/routes/jobs.py`: `_inherited_cost_warnings` | An inline workflow (no catalog name) is priced from the observed runs of the catalog template with the same `component_type` + `model_name` + `workflow` key as `vram_inheritance`; the template with the most cold runs wins, then name. `differs` lists offload, quantization, frame count, attention backend, cache, LoRA and reference count (the `references` list lengths, per-entry `item:` ones read off the `for_each` list; named, not scaled, #915) set differently from the template, and sets `low_confidence` when it lists any (#866). More pipeline steps than the template run (a `for_each` counted per member, against the list the bucket ran, so a fallback to the default bucket re-prices a longer list) re-price its minutes linearly, and the mean frames, inference steps, width and height per pipeline step are scaled by `scale_curated` itself - the one rule the catalog figure uses, so an inline copy and its template quote one request alike - with `scaled` naming why and `low_confidence` set (#851, #866). It is warned, never refused. | `tests/test_inherited_cost.py::TestPricedByPipelineIdentity::test_an_inline_copy_inherits_the_templates_runs`, `tests/test_inherited_cost.py::TestDifferences::test_offload_quantization_and_frames_are_named`, `tests/test_inherited_cost.py::TestDifferences::test_a_reference_count_other_than_the_templates_is_named`, `tests/test_inherited_cost.py::TestThePlanReportsIt::test_the_estimate_is_inherited_and_the_validate_answer_warns`, `tests/test_inherited_cost.py::TestTheTemplateAndAnInlineCopyAgree::test_the_same_minutes_within_rounding`, `tests/test_inherited_cost.py::TestAListLongerThanTheFallbackBucket::test_an_inline_six_shot_copy_scales_by_members` | -| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit, unless the shape holds its pipeline across the list and the fit is the larger (the key leaves list length out, #910); with neither there is no figure. It never refuses; dispatch may delay a start on it (`JobManager._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestALongerListThanObserved::test_an_accumulating_list_takes_the_larger_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | +| Projected host-RAM peak | `dw/server/host_ram_need.py`: `memory_key`, `projected_peak`, `host_memory_block`; the fit is `dw/host_memory_projection.py`: `projected_mb`; history by key is `dw/server/job_history.py`: `peaks_for_key` | A run's memory key is the sorted pipeline identity of its pipeline steps plus every `quantization_config` / `offload` / `group_offload` value, with arguments substituted through `replace_variables`; never the workflow's name, so a renamed or inline copy shares the template's history. `observed` is the max peak over this machine's successful same-key runs and comes ahead of the list-run fit, unless the shape holds its pipeline across the list and the fit is the larger (the key leaves list length out, #910); with neither there is no figure. It never refuses; dispatch may delay a start on it (`dw/server/dispatch.py`: `Dispatcher._next_dispatch`, #869) (`Admission.host_ram_need`, validate's `plan.host_memory`). | `tests/test_host_ram_need.py::TestMemoryKey::test_an_inline_copy_without_its_metadata_has_the_same_key`, `tests/test_host_ram_need.py::TestMemoryKey::test_a_different_weights_dtype_is_a_different_key`, `tests/test_host_ram_need.py::TestProjectedPeak::test_an_observed_peak_beats_the_fit`, `tests/test_host_ram_need.py::TestALongerListThanObserved::test_an_accumulating_list_takes_the_larger_fit`, `tests/test_host_ram_need.py::TestProjectedPeak::test_a_failed_run_is_not_observed`, `tests/test_host_ram_need.py::TestValidateRoute::test_an_observed_run_is_the_plans_host_memory` | | Acceleration warnings | `dw/acceleration_warnings.py`: `acceleration_warnings`, `CACHE_MIN_USEFUL_STEPS`, `CUDA_ONLY_ATTENTION_BACKENDS` (the last entry of `WARNING_CHECKS`) | An acceleration setting that can't pay off (a `first_block`/`mag` cache under `CACHE_MIN_USEFUL_STEPS`, any `taylorseer` cache, `compile.fullgraph` with a cache, a cache on a pipeline class with no transformer or whose transformer lacks `enable_cache()`, a CUDA-only `attention_backend` on a non-CUDA device) is a warning and never a refusal, and an unresolvable value (steps, backend, device) leaves it silent. | `tests/test_validation.py::TestAccelerationWarnings::test_warnings_never_refuse`, `tests/test_validation.py::TestAccelerationWarnings::test_steps_that_resolve_to_nothing_are_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unknown_device_is_silent`, `tests/test_validation.py::TestAccelerationWarnings::test_an_unresolvable_backend_is_silent_on_mps` | | H3 adapter partition | `dw/adapter_compatibility.py`; the server's index is `dw/server/deps.py` `lora_index` | The LoRA catalog decides first, on the server only: a `proven` entry's `workflow` refuses a step in the other partition, a `trial`/`rejected` entry's only warns. A file no entry places falls to its name: an FL2VA LoRA on a reference step is refused, and one whose name says neither `ref2v` nor `fl2v` is warned. | `tests/test_h3_adapters.py::TestWhatIsRefused::test_a_keyframe_adapter_on_the_reference_path`, `tests/test_h3_adapters.py::TestWhatIsRefused::test_an_unrecognised_name_is_a_warning_not_a_refusal`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_proven_entry_refuses_the_other_partition`, `tests/test_h3_adapters.py::TestTheCatalogDecidesFirst::test_a_weaker_entry_only_warns`, `tests/test_server_loras.py::TestTheCatalogPlacesAnAdapter::test_a_trial_entry_warns_and_promoting_it_refuses` | | H3 audio hold, refine and guide layout | Five modules, imported one way: `dw/pipeline_processors/h3_rules.py` holds the names and rules, torch- and diffusers-free (`refine_problems`, the per-argument refine rules asked before validation and before the call; `not_h3`, the step rule; `snap_guide_length`, `guide_frame_problem`, `guide_chain_problems`, `guide_end_problem`; `CHAIN_CONTINUITY_MODES`; the chunk constants), `dw/pipeline_processors/h3_hold.py` puts the hold and refine blocks in (`insert_audio_hold`, `core_denoise_sequences`, `refines`), and `dw/pipeline_processors/h3_guides.py` the guide layout (`LAYOUT_ANCHORS`, `insert_guides`), which imports `h3_hold` and never the reverse; the blocks themselves, and the arithmetic they run, are `dw/pipeline_processors/h3_hold_steps.py` (`encode_audio_span`, `refine_sigmas`) and `dw/pipeline_processors/h3_guide_steps.py` (`splice_guide_rows`), which import diffusers when they load, so the two owners import them only inside `blocks()` and `guide_blocks()` (#790), and `h3_guide_steps` imports `h3_hold_steps` and never the reverse (`_with_guides` in `dw/pipeline_processors/pipeline.py` adds its blocks for a `guides` call), `dw/guides.py`: the `guides` validation check and the chain `continuity: "guide"` check, `guide_chain_errors`, which asks the `guides` check's own step rule (`_takes_guides_problem`, built on `h3_rules.not_h3` and `REFERENCE_WORKFLOWS`) and reads the mode names from `h3_rules.CHAIN_CONTINUITY_MODES`, the names `chain.py`'s `CONTINUITY_MODES` is zipped against (both registered in `dw/validation.py`; a `previous_result:` guide is checked at run time), `dw/hold_audio.py`: `hold_audio_errors`, `refine_strength_errors` | The one place dw modifies a diffusers modular pipeline's block graph: three blocks go in, before `set_timesteps`, `denoise` and `after_denoise`, in every H3 core-denoise sequence (the whole graph and the `workflow=`-pruned flat graph), and they are no-ops without `hold_audio` / `refine_strength`. A `hold_audio` path is opened only after `locations.validate_media_path` against the step's workflow directory. `dw/introspection.py` builds the same graph without weights, hold included, so `get_pipeline_signature` lists a modular pipeline's block inputs. The copies of diffusers behaviour are pinned to the installed diffusers: `shifted_sigma_grid` (the spacing `refine_sigmas` rests on) to stock `set_timesteps`, `GUIDE_FRAMES_PER_CHUNK`/`GUIDE_LATENTS_PER_CHUNK` to the H3 video VAE's `clip_length` and latents per clip, and every name in `LAYOUT_ANCHORS` (`_fill_audio_positions` among them) to `before_denoise`. | `tests/test_h3_hold_audio.py::TestAnchors::test_inserted_next_to_the_anchors_in_every_shape`, `tests/test_h3_hold_audio.py::TestHoldAudioLocation::test_a_path_outside_the_roots_is_refused_at_the_call`, `tests/test_h3_hold_audio.py::TestRefusals::test_a_non_h3_step_is_refused`, `tests/test_h3_hold_audio.py::TestSignature::test_the_h3_signature_shows_hold_audio`, `tests/test_h3_refine.py::TestAnchors::test_inserted_before_denoise_in_every_shape`, `tests/test_h3_refine.py::TestSigmaGridDrift::test_the_full_grid_is_stock_set_timesteps`, `tests/test_h3_guides.py::TestChunkDrift::test_frames_per_chunk_is_the_vae_clip_length`, `tests/test_h3_guides.py::TestChunkDrift::test_latents_per_chunk_is_the_vae_tokens_per_clip`, `tests/test_h3_guides.py::TestAnchors::test_installed_diffusers_has_every_anchor`, `tests/test_guide_chain_validation.py::TestRefused::test_the_step_rule_is_the_guides_rule`, `tests/test_guide_chain_validation.py::TestRefused::test_every_run_mode_is_known_to_validate`, `tests/test_h3_rules.py::test_h3_rules_imports_neither_torch_nor_diffusers`, `tests/test_h3_rules.py::test_the_block_owners_import_no_diffusers` | @@ -159,7 +159,7 @@ one to open. | App factory | `dw/server/app.py`: `create_app` | `create_app` builds the JobManager and `app.state`, installs the middleware and registers the routes, and all state lives in the JobManager. | — | | Routers | `dw/server/routes/*.py`, `dw/server/routes/__init__.py`: `ROUTERS`, `include_routers`, `include_file_routes` | Routers register in `ROUTERS` order, because a greedy `{name:path}` route must come after its specific siblings. The files router comes later, after `/mcp`. | `tests/test_server_downloads.py::test_download_workflow_sets_content_disposition_attachment` | | Job queue | `dw/server/jobs.py`: `JobManager`; `dw/server/job_results.py`: `consume_results` | A dispatcher thread hands queued jobs, in queue order, to the pool's workers (`Worker pool dispatch`); each worker runs one job at a time, and `consume_results` folds that worker's replies into the job, naming files relative to the job's own `output_dir`. A job carries its own `output_dir`, `asset_dir` and `workflow_dir`, so it stays in its workspace. | `tests/test_server_workspaces.py::TestRunning::test_rerun_stays_in_the_workspace_it_ran_in` (FIFO: —) | -| Worker pool dispatch | `dw/server/jobs.py`: `JobManager._next_dispatch`, `JobManager._host_ram_wait`; `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`, `host_ram_wait`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, a job that fits a free card is also held while its projected host-RAM peak exceeds `MemAvailable` less what running jobs have yet to add plus the idle worker's own RSS (a job alone is never held, no figure or no RAM reading means no gate, a smaller job behind a held one starts, and the dispatcher re-checks every 10 s; #869), and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_the_host_cannot_hold_beside_the_running_one_is_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_alone_is_never_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_smaller_job_starts_past_a_held_one` | +| Worker pool dispatch | `dw/server/dispatch.py`: `Dispatcher._next_dispatch`, `Dispatcher._host_ram_wait` (mixed into `JobManager`); `dw/server/pool.py`: `WorkerSlot`, `check_fits`, `largest_ceiling_gb`, `host_ram_wait`; `dw/vram_estimate.py`: `required_vram_gb`; `dw/server/admission.py`: `pool_capacity` | `--devices` runs one worker per card; a queued job goes to a free card that fits what it needs (none starts while a cache clear holds dispatch, `dw disk caches`), a job that fits no free card keeps its place while a smaller one behind it takes the card, a job bigger than every card is refused at submit, a job that fits a free card is also held while its projected host-RAM peak exceeds `MemAvailable` less what running jobs have yet to add plus the idle worker's own RSS (a job alone is never held, no figure or no RAM reading means no gate, a smaller job behind a held one starts, and the dispatcher re-checks every 10 s; #869), and cancel or a crash touches only the worker running that job. On a backend no `cost` entry describes, a declared `vram_estimate` is held at admission to the pool's largest card, so a job the first card cannot hold is not refused when a larger card can hold it. | `tests/test_worker_pool.py::TestConcurrency::test_two_workers_run_two_jobs_at_once`, `tests/test_worker_pool.py::TestBackfill::test_a_job_too_big_for_the_free_card_waits_while_a_smaller_one_takes_it`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_too_big_for_every_card_is_refused_naming_the_largest`, `tests/test_worker_pool.py::TestCancel::test_cancel_reaches_the_worker_running_the_job`, `tests/test_worker_pool.py::TestCrashIsolation::test_a_crashed_worker_fails_only_its_own_job`, `tests/test_worker_pool.py::TestOomRanking::test_each_dispatch_scores_its_worker_highest`, `tests/test_worker_pool.py::TestFitAtSubmit::test_a_job_only_the_larger_card_holds_is_admitted`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_the_host_cannot_hold_beside_the_running_one_is_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_job_alone_is_never_held`, `tests/test_worker_pool.py::TestHostRamGate::test_a_smaller_job_starts_past_a_held_one` | | Card affinity | `dw/server/pool.py`: `choose_slot`; `dw/server/jobs.py`: `JobManager.route`; `dw/worker_protocol.py`: `workflow_identity` | Among the free cards a job fits it goes to the rerun's original card, else the card whose worker last ran its workflow identity (`workflow_identity`, the one rule the worker caches by too), else one with no worker running, else the first in `--devices` order; a cache probe and `plan.estimate` (`priced_for`) ask about the card `route` names, and memory is read and cleared per card. | `tests/test_worker_pool.py::TestIdentityAffinity::test_a_second_run_lands_on_the_card_that_ran_the_workflow_first`, `tests/test_worker_pool.py::TestRerunAffinity::test_a_rerun_lands_on_the_original_card_when_it_is_free`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle` | | Per-card memory and cache | `dw/server/worker_memory.py`: `memory_status`, `clear_memory`, `probe_cache` | Each card keeps its own last memory reading; an idle card's worker is asked live, a busy one answers with the run's last report and says why it is not live, and a clear leaves a card running a job alone. Every request takes the card's lock with a bounded wait, and a card's lock is never taken while the JobManager's lock is held. | `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_server.py::test_memory_says_why_a_reading_is_not_the_worker_s` | | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | diff --git a/dw/server/dispatch.py b/dw/server/dispatch.py new file mode 100644 index 00000000..17633f3a --- /dev/null +++ b/dw/server/dispatch.py @@ -0,0 +1,136 @@ +"""The dispatcher's pick: which queued job starts next, on which card. + +`Dispatcher` is a mixin of `JobManager` (`dw/server/jobs.py`) - its methods +read the manager's queue and dispatch tables (`_pending`, `_needs`, +`_host_needs`, `_preferred`) and are called holding the manager's `_lock`. +Queue order, except that a job too big for every free card keeps its place +while a smaller one behind it takes the card (backfill), and a job that fits +a free card is also held while the host's RAM can't hold it beside the +running jobs (#869; the figure is `dw/server/host_ram_need.py`'s, the +comparison `pool.host_ram_wait`). +""" + +from ..host_memory import host_memory_stats, process_rss_mb +from . import pool +from .job_record import QUEUED + +# How often a job held for host RAM is looked at again while nothing else +# wakes the dispatcher - memory can be freed outside dw (#869) +HOST_RAM_RECHECK_SECONDS = 10.0 + + +class _HostReading: + """The host's MemAvailable for one dispatch pass, read on first ask.""" + + def __init__(self): + self._read = False + self._available = None + + def available_mb(self): + if not self._read: + self._read = True + # None where the platform can't say (macOS without psutil): no gate + self._available = (host_memory_stats() or {}).get("available_mb") + return self._available + + +def _worker_rss_mb(slot): + """What `slot`'s worker holds in host RAM now, or None (not running, or + unreadable).""" + pid = slot.manager.pid() if hasattr(slot.manager, "pid") else None + return process_rss_mb(pid) if pid is not None else None + + +class Dispatcher: + """The queue scan `JobManager._run_loop` asks for its next job.""" + + def _next_dispatch(self): + """(slot, job) for the first queued job, in queue order, that fits + a free card - the one it has an affinity for (`_choose_slot`) - or + None. A job too big for every free card keeps its place, and the + scan goes on past it: a smaller job behind it takes the card + meanwhile (backfill). A job the host's RAM can't hold beside the + running ones keeps its place the same way (`_host_ram_wait`). + Called holding _lock.""" + if self._dispatch_holds: + return None + free = [slot for slot in self.slots if slot.current_job_id is None] + if not free: + return None + host = _HostReading() + for job_id in list(self._pending): + job = self.jobs.get(job_id) + if job is None or job.status != QUEUED: + self._pending.remove(job_id) # cancelled while waiting + self._forget_dispatch(job, job_id) + continue + need = self._needs.get(job_id) + fitting = [slot for slot in free if slot.fits(need)] + if not fitting: + # Waiting for a card, not for host RAM + job.waiting_for = None + continue + slot = pool.choose_slot( + fitting, self._job_identity(job), self._preferred.get(job_id) + ) + reason = self._host_ram_wait(job_id, slot, host) + if reason is not None: + if job.waiting_for is None: + # Once per hold, not once per pass: a wait is a moment, + # so an event and a live field, never a warning (#869) + job.add_event( + {"event": "queue_wait", "reason": "host_ram", "message": reason} + ) + job.waiting_for = reason + continue + self._pending.remove(job_id) + self._forget_dispatch(job) + return slot, job + return None + + def _forget_dispatch(self, job, job_id=None): + """Drop what dispatch keeps for a job leaving the queue - started or + cancelled. `job` may be None for one already gone.""" + job_id = job.id if job is not None else job_id + self._needs.pop(job_id, None) + self._host_needs.pop(job_id, None) + self._preferred.pop(job_id, None) + if job is not None: + job.waiting_for = None + + def _held_for_host_ram(self): + """The queued jobs held for host RAM now. Called holding _lock.""" + for job_id in self._pending: + job = self.jobs.get(job_id) + if job is not None and job.waiting_for is not None: + yield job + + def _host_ram_wait(self, job_id, slot, host): + """Why the queued `job_id` must not start on `slot` yet for host RAM, + or None (pool.host_ram_wait, #869). `host` is this pass's reading, + taken once and only when a job with a figure asks.""" + need = self._host_needs.get(job_id) + if need is None: + return None + running = [ + (other, self.jobs.get(other.current_job_id)) + for other in self.slots + if other.current_job_id is not None + ] + if not running: + return None + available = host.available_mb() + if available is None: + return None + return pool.host_ram_wait( + need, + available, + [ + ( + job.host_ram_need[0] if job and job.host_ram_need else None, + _worker_rss_mb(other), + ) + for other, job in running + ], + _worker_rss_mb(slot), + ) diff --git a/dw/server/jobs.py b/dw/server/jobs.py index a6dfbea9..25015174 100644 --- a/dw/server/jobs.py +++ b/dw/server/jobs.py @@ -20,6 +20,8 @@ rest lives beside it, and JobManager keeps a thin delegator for every name a route or test calls: - `dw/server/pool.py`: WorkerSlot and the lock-free fit and affinity policy; +- `dw/server/dispatch.py`: the queue scan picking the next job and card, + with the host-RAM hold (`Dispatcher`, a mixin of JobManager); - `dw/server/job_results.py`: folding a run's worker messages into its Job; - `dw/server/worker_memory.py`: a card's memory reading, cache probe and memory clear. @@ -43,7 +45,7 @@ from ..events import NON_INTERRUPTIBLE_PHASES from ..worker_protocol import Execute, workflow_identity from ..devices import device_ordinal -from ..host_memory import host_memory_stats, process_rss_mb, read_oom_score_adj +from ..host_memory import process_rss_mb, read_oom_score_adj from ..worker_manager import WorkerManager from ..workflow_run import SEED_BITS from ..security import ( @@ -57,7 +59,8 @@ from ..runs import REALIZED_FILE_NAME from ..settings import resolve_path from ..workspace import DEFAULT_WORKSPACE_NAME -from . import job_results, pool, worker_memory +from . import dispatch, job_results, pool, worker_memory +from .dispatch import Dispatcher from .job_history import JobHistory from .outputs import job_run_directory, with_run_note from .job_record import ( @@ -86,38 +89,11 @@ CANCEL_GRACE_SECONDS = 30.0 -# How often a job held for host RAM is looked at again while nothing else -# wakes the dispatcher - memory can be freed outside dw (#869) -HOST_RAM_RECHECK_SECONDS = 10.0 - - -class _HostReading: - """The host's MemAvailable for one dispatch pass, read on first ask.""" - - def __init__(self): - self._read = False - self._available = None - - def available_mb(self): - if not self._read: - self._read = True - # None where the platform can't say (macOS without psutil): no gate - self._available = (host_memory_stats() or {}).get("available_mb") - return self._available - - -def _worker_rss_mb(slot): - """What `slot`'s worker holds in host RAM now, or None (not running, or - unreadable).""" - pid = slot.manager.pid() if hasattr(slot.manager, "pid") else None - return process_rss_mb(pid) if pid is not None else None - - class JobsActive(Exception): """A job is running or queued.""" -class JobManager: +class JobManager(Dispatcher): """Queues jobs and dispatches them onto one worker process per card.""" def __init__( @@ -810,7 +786,7 @@ def _run_loop(self): # Host RAM frees outside dw too, so a job held for it is # looked at again on a timer, not only on a wake (#869) held = any(self._held_for_host_ram()) - self._wake.wait(HOST_RAM_RECHECK_SECONDS if held else None) + self._wake.wait(dispatch.HOST_RAM_RECHECK_SECONDS if held else None) slot, job = picked job.status = RUNNING job.started_at = time.time() @@ -830,97 +806,6 @@ def _run_loop(self): self._job_threads.add(thread) thread.start() - def _next_dispatch(self): - """(slot, job) for the first queued job, in queue order, that fits - a free card - the one it has an affinity for (`_choose_slot`) - or - None. A job too big for every free card keeps its place, and the - scan goes on past it: a smaller job behind it takes the card - meanwhile (backfill). A job the host's RAM can't hold beside the - running ones keeps its place the same way (`_host_ram_wait`). - Called holding _lock.""" - if self._dispatch_holds: - return None - free = [slot for slot in self.slots if slot.current_job_id is None] - if not free: - return None - host = _HostReading() - for job_id in list(self._pending): - job = self.jobs.get(job_id) - if job is None or job.status != QUEUED: - self._pending.remove(job_id) # cancelled while waiting - self._forget_dispatch(job, job_id) - continue - need = self._needs.get(job_id) - fitting = [slot for slot in free if slot.fits(need)] - if not fitting: - # Waiting for a card, not for host RAM - job.waiting_for = None - continue - slot = pool.choose_slot( - fitting, self._job_identity(job), self._preferred.get(job_id) - ) - reason = self._host_ram_wait(job_id, slot, host) - if reason is not None: - if job.waiting_for is None: - # Once per hold, not once per pass: a wait is a moment, - # so an event and a live field, never a warning (#869) - job.add_event( - {"event": "queue_wait", "reason": "host_ram", "message": reason} - ) - job.waiting_for = reason - continue - self._pending.remove(job_id) - self._forget_dispatch(job) - return slot, job - return None - - def _forget_dispatch(self, job, job_id=None): - """Drop what dispatch keeps for a job leaving the queue - started or - cancelled. `job` may be None for one already gone.""" - job_id = job.id if job is not None else job_id - self._needs.pop(job_id, None) - self._host_needs.pop(job_id, None) - self._preferred.pop(job_id, None) - if job is not None: - job.waiting_for = None - - def _held_for_host_ram(self): - """The queued jobs held for host RAM now. Called holding _lock.""" - for job_id in self._pending: - job = self.jobs.get(job_id) - if job is not None and job.waiting_for is not None: - yield job - - def _host_ram_wait(self, job_id, slot, host): - """Why the queued `job_id` must not start on `slot` yet for host RAM, - or None (pool.host_ram_wait, #869). `host` is this pass's reading, - taken once and only when a job with a figure asks.""" - need = self._host_needs.get(job_id) - if need is None: - return None - running = [ - (other, self.jobs.get(other.current_job_id)) - for other in self.slots - if other.current_job_id is not None - ] - if not running: - return None - available = host.available_mb() - if available is None: - return None - return pool.host_ram_wait( - need, - available, - [ - ( - job.host_ram_need[0] if job and job.host_ram_need else None, - _worker_rss_mb(other), - ) - for other, job in running - ], - _worker_rss_mb(slot), - ) - def _slot_running(self, job_id): """The slot whose worker is running `job_id`, or None.""" with self._lock: diff --git a/dw/server/pool.py b/dw/server/pool.py index f859f6cb..ff9a432c 100644 --- a/dw/server/pool.py +++ b/dw/server/pool.py @@ -7,7 +7,7 @@ host-RAM wait (`host_ram_wait`): the cards share one host's RAM, so a job that fits a free card may still wait for a running one to finish. The JobManager (`dw/server/jobs.py`) calls them under its own lock where the -answer must agree with dispatch (`JobManager.route`, `_next_dispatch`). +answer must agree with dispatch (`JobManager.route`, `dispatch.Dispatcher._next_dispatch`). """ import logging diff --git a/tests/test_worker_pool.py b/tests/test_worker_pool.py index 5f3ce74c..3e9b3df5 100644 --- a/tests/test_worker_pool.py +++ b/tests/test_worker_pool.py @@ -1263,7 +1263,7 @@ def host(self, monkeypatch): """The host's MemAvailable and each worker's RSS, as a test sets them.""" readings = {"available_mb": 64 * GB, "rss": {}} monkeypatch.setattr( - "dw.server.jobs.host_memory_stats", + "dw.server.dispatch.host_memory_stats", lambda: ( None if readings["available_mb"] is None @@ -1271,7 +1271,7 @@ def host(self, monkeypatch): ), ) monkeypatch.setattr( - "dw.server.jobs._worker_rss_mb", + "dw.server.dispatch._worker_rss_mb", lambda slot: readings["rss"].get(slot.device), ) return readings @@ -1324,7 +1324,7 @@ def test_a_job_the_host_cannot_hold_beside_the_running_one_is_held( def test_memory_freed_outside_dw_starts_the_held_job_on_the_recheck( self, pool, host, monkeypatch ): - monkeypatch.setattr("dw.server.jobs.HOST_RAM_RECHECK_SECONDS", 0.05) + monkeypatch.setattr("dw.server.dispatch.HOST_RAM_RECHECK_SECONDS", 0.05) manager, first_gate, second_gate = self._two_cards(pool) self.submit(manager, "running", host_gb=10) first_gate.wait_started() @@ -1403,7 +1403,7 @@ def test_the_idle_worker_it_would_reuse_counts_as_free(self, pool, host): wait_for(job, {"running"}) def test_the_event_fires_once_per_hold(self, pool, host, monkeypatch): - monkeypatch.setattr("dw.server.jobs.HOST_RAM_RECHECK_SECONDS", 0.02) + monkeypatch.setattr("dw.server.dispatch.HOST_RAM_RECHECK_SECONDS", 0.02) manager, first_gate, second_gate = self._two_cards(pool) self.submit(manager, "running", host_gb=10) first_gate.wait_started() From 42b72552a24c134b8fb1eb99620f4148a29b0776 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:40:13 -0500 Subject: [PATCH 194/204] refactor(server): split the library response models out of api_models.py The workspace, workflow, prompt and enhancer models (and the Origin, Shape and Trait literals) move verbatim into api_models_library.py, and ApiModel, STRICT, sometimes and send_rejected_responses into api_models_base.py so the library module can derive from them without an import cycle. api_models.py re-exports every name, so routes and tests import as before; the OpenAPI document and the generated UI types are unchanged. api_models.py is 947 lines, back under the 1,100 ceiling. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/api_models.py | 313 ++++++-------------------------- dw/server/api_models_base.py | 55 ++++++ dw/server/api_models_library.py | 214 ++++++++++++++++++++++ 4 files changed, 330 insertions(+), 254 deletions(-) create mode 100644 dw/server/api_models_base.py create mode 100644 dw/server/api_models_library.py diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 21698277..689cb442 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -199,6 +199,6 @@ one to open. | The editors' document | `ui/src/lib/editorShell.svelte.ts`: `DocumentEditor` | The workflow and prompt editors keep one rule each for the view (remembered per editor), the JSON draft (a failed parse pins it), dirty against the last load or save, the save path, and the tab-close guard. | `ui/src/lib/editorShell.test.ts`, `ui/src/lib/pages/PromptEditorPage.test.ts`, `ui/src/lib/pages/EditorPage.test.ts` | | UI polling | `ui/src/lib/poll.ts`: `poll`, `sleep` | A page that refreshes on a timer starts it through `poll` and stops it with the function `poll` returns. | `ui/src/lib/poll.test.ts` | | Workspace state in the UI | `ui/src/lib/workspaceState.svelte.ts`: `workspace` | The current workspace is one `$state`; `api.ts` reads it from here, so the request layer imports no page state. | `import_cycles` in `ui/scripts/arch-metrics.mjs` | -| The UI's response contract | `dw/server/api_models.py`: `ApiModel`, `send_rejected_responses`; `scripts/dump_openapi.py` | Every JSON route `api.ts` calls declares a response model, and `types.ts` re-exports the types generated from it (`ui/src/lib/generated/`). The payload does not change: an absent key stays absent (`response_model_exclude_unset`), a sometimes-sent key is `sometimes()`. Runtime is lenient - an undeclared key passes and a rejected response is logged and sent as built; tests, the e2e server and the dump run strict (`DW_STRICT_RESPONSES=1`). The document is generated under the FastAPI and Pydantic `constraints-openapi.txt` pins: CI installs them, and the dump refuses to write under others. | `tests/test_api_contract.py`, `tests/test_ui_twins.py::test_every_json_route_the_ui_calls_declares_its_response`, CI's "Response contract" step | +| The UI's response contract | `dw/server/api_models.py` (the models; it re-exports `dw/server/api_models_base.py`: `ApiModel`, `send_rejected_responses`, and the library routes' models in `dw/server/api_models_library.py`); `scripts/dump_openapi.py` | Every JSON route `api.ts` calls declares a response model, and `types.ts` re-exports the types generated from it (`ui/src/lib/generated/`). The payload does not change: an absent key stays absent (`response_model_exclude_unset`), a sometimes-sent key is `sometimes()`. Runtime is lenient - an undeclared key passes and a rejected response is logged and sent as built; tests, the e2e server and the dump run strict (`DW_STRICT_RESPONSES=1`). The document is generated under the FastAPI and Pydantic `constraints-openapi.txt` pins: CI installs them, and the dump refuses to write under others. | `tests/test_api_contract.py`, `tests/test_ui_twins.py::test_every_json_route_the_ui_calls_declares_its_response`, CI's "Response contract" step | | Where `--live` appears | `ui/src/app.css` (its header) | The state colour marks machine state only, in the files `ui/scripts/design-rules.test.ts` lists. | `ui/scripts/design-rules.test.ts` | | UI architecture ratchet | `ui/scripts/arch-metrics.mjs` | No metric rises above `docs/stabilization/ui/baseline.json`. | the `ui` job in `.github/workflows/ci.yml`; `npm run preflight` | diff --git a/dw/server/api_models.py b/dw/server/api_models.py index a6f807ac..3331ae0c 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -13,57 +13,69 @@ not emit stays absent rather than arriving as null. """ -import logging -import os - from typing import Any, Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import ConfigDict, Field from .admission import AcknowledgedCost -from .catalog_shape import SHAPES, TRAITS - -STRICT = os.environ.get("DW_STRICT_RESPONSES") == "1" - -logger = logging.getLogger(__name__) - - -class ApiModel(BaseModel): - model_config = ConfigDict(extra="forbid" if STRICT else "allow") - - -def send_rejected_responses(app) -> None: - """Make runtime leniency cover a declared field of the wrong type too, - not just an undeclared key: the response is logged and sent as the - handler built it, with the route's own status. Some routes have acted - by then - POST /api/jobs has queued the job - and a 500 would invite a - retry that does it twice. Installed by create_app outside strict mode.""" - from fastapi.encoders import jsonable_encoder - from fastapi.exceptions import ResponseValidationError - from fastapi.responses import JSONResponse - - async def send_as_built(request, exc: ResponseValidationError): - route = request.scope.get("route") - logger.error( - "Response for %s %s does not match its model: %s", - request.method, - request.url.path, - exc.errors(), - ) - return JSONResponse( - jsonable_encoder(exc.body), - status_code=getattr(route, "status_code", None) or 200, - ) - - app.add_exception_handler(ResponseValidationError, send_as_built) - - -def sometimes(description: str | None = None) -> Any: - """A key the handler emits only in some states, and never as null when - it does: typed as its value, defaulting to an unvalidated None that - `response_model_exclude_unset` keeps out of the payload. The generated - type is then `key?: T`, not `key?: T | null`.""" - return Field(default=None, description=description) +from .api_models_base import STRICT, ApiModel, send_rejected_responses, sometimes +from .api_models_library import ( + Origin, + Shape, + Trait, + LibraryRoot, + ShadowedEntry, + DiskUsage, + WorkspaceInfo, + WorkspaceList, + WorkspaceDeleted, + CopiedItems, + WorkspaceCopied, + WorkflowCost, + WorkflowCard, + BuiltinWorkflows, + WorkflowList, + WorkflowSaved, + WorkflowDeleted, + PromptCard, + PromptList, + PromptSaved, + Deleted, + EnhancerPreset, + EnhancerPresets, +) + +# The library's models live in api_models_library, and the base in +# api_models_base; every route and test imports them from here +__all__ = [ + "STRICT", + "ApiModel", + "send_rejected_responses", + "sometimes", + "Origin", + "Shape", + "Trait", + "LibraryRoot", + "ShadowedEntry", + "DiskUsage", + "WorkspaceInfo", + "WorkspaceList", + "WorkspaceDeleted", + "CopiedItems", + "WorkspaceCopied", + "WorkflowCost", + "WorkflowCard", + "BuiltinWorkflows", + "WorkflowList", + "WorkflowSaved", + "WorkflowDeleted", + "PromptCard", + "PromptList", + "PromptSaved", + "Deleted", + "EnhancerPreset", + "EnhancerPresets", +] # ----------------------------------------------------------------- system @@ -704,215 +716,10 @@ class ValidationResult(ApiModel): ) -# ---------------------------------------------------------------- library +# --------------------------------------------------------- gallery, assets -Origin = Literal["workspace", "common", "examples", "builtin"] # Assets have no packaged (builtin) library AssetOrigin = Literal["workspace", "common", "examples"] -Shape = Literal[SHAPES] -Trait = Literal[TRAITS] - - -class LibraryRoot(ApiModel): - root: str - origin: Origin - writable: bool - - -class ShadowedEntry(ApiModel): - name: str - origin: Origin - shadowed_by: Origin - - -class DiskUsage(ApiModel): - files: int - bytes: int - - -class WorkspaceInfo(ApiModel): - name: str - default: bool - root: str | None - workflows: str - assets: str | None - outputs: str - prompts: str | None - common_assets: str | None - usage: DiskUsage = sometimes("Roughly how much disk it holds; listings only.") - note: str = sometimes("The workspace's note, when it has one; listings only.") - - -class WorkspaceList(ApiModel): - workspace_root: str | None - default: str - workspaces: list[WorkspaceInfo] - - -class WorkspaceDeleted(ApiModel): - name: str - deleted: bool - contents: dict[str, DiskUsage] = Field(description="What it held, per folder.") - - -class CopiedItems(ApiModel): - runs: list[str] = Field(description="`/` of each run copied.") - assets: list[str] = Field( - description="Library-relative path of each asset file copied." - ) - - -class WorkspaceCopied(ApiModel): - """What a copy between workspaces did (`POST /api/workspaces/{name}/copy`).""" - - source: str - destination: str - copied: CopiedItems - already_present: list[str] = Field( - description="Runs the destination already held (`/`) " - "and assets it held with identical bytes (library-relative path) - " - "skipped." - ) - renumbered: dict[str, dict[str, int]] = Field( - description="Copied runs whose `vN` changed, keyed `/`: " - "`{from, to}`." - ) - warnings: dict[str, Any] = Field( - description="`unresolved_output_refs`, `absolute_paths` and `no_jobs`, " - "each only when it applies." - ) - bytes_linked: int - bytes_copied: int - - -class WorkflowCost(ApiModel): - """A measured run, as the workflow's own `cost` block declares it.""" - - model_config = ConfigDict(extra="allow") - - device: str - name: str = sometimes() - vram_gb: int | float - minutes: int | float - - -class WorkflowCard(ApiModel): - """One workflow's entry in the full listing (the agent's - `view=compact` answer is a different, smaller shape outside this - model).""" - - kinds: list[str] - steps: int - variables: int - variable_names: list[str] - description: str - configures: str = sometimes( - "For a model config: the template it is a tuned instance of; '' for a " - "template. Absent when the file could not be read." - ) - configures_missing: str = sometimes("A `configures` that names no workflow.") - prompt_refs: list[str] - shape: Shape = Field(description="What the workflow makes, derived by the server.") - traits: list[Trait] = Field( - description="Sorted, independent facts about how the output is made or what " - "it needs." - ) - summary: str = Field( - description="The description's first sentence, clipped - what a card shows." - ) - lists: dict[str, Any] - constraints: dict[str, Any] - cost_drivers: dict[str, Any] - cost: list[WorkflowCost] | None = Field( - description="Measured runs, one per device the maintainer measured on. Null " - "means unknown - never derived." - ) - observed: dict[str, Any] = sometimes("This box's own history for it.") - origin: Origin - writable: bool = Field( - description="False for a read-only source: offer save-a-copy, not delete." - ) - - -class BuiltinWorkflows(ApiModel): - workflows: list[str] = Field( - description="The packaged fragments a sub-workflow step names as " - "`builtin:.json`, each as what follows the prefix." - ) - - -class WorkflowList(ApiModel): - workspace: str - libraries: list[LibraryRoot] = Field( - description="The search path in order; the writable workspace root is where " - "a save lands, whatever library a workflow was read from." - ) - workflows: list[str] - details: dict[str, WorkflowCard] - shadowed: list[ShadowedEntry] - cost_basis: str - - -class WorkflowSaved(ApiModel): - name: str - workspace: str - origin: Origin - warnings: list[str] - shape: Shape - traits: list[Trait] - summary: str - - -class WorkflowDeleted(ApiModel): - name: str - workspace: str - origin: Origin - deleted: bool - - -class PromptCard(ApiModel): - description: str - intended_model: str - tags: list[str] - text: str = sometimes("Absent when the listing was asked for without text.") - text_chars: int = sometimes("The text's length, in place of `text`.") - origin: Origin = Field( - description="Which library the prompt came from, and whether a save can " - "reach it." - ) - writable: bool - - -class PromptList(ApiModel): - libraries: list[LibraryRoot] - prompts: list[str] - details: dict[str, PromptCard] - shadowed: list[ShadowedEntry] - - -class PromptSaved(ApiModel): - name: str - - -class Deleted(ApiModel): - name: str - deleted: bool - - -class EnhancerPreset(ApiModel): - key: str - label: str - default_model: str - models: list[str] - intended_models: list[str] - placeholder: str - - -class EnhancerPresets(ApiModel): - presets: list[EnhancerPreset] - - -# --------------------------------------------------------- gallery, assets class GalleryFile(ApiModel): diff --git a/dw/server/api_models_base.py b/dw/server/api_models_base.py new file mode 100644 index 00000000..97553bb6 --- /dev/null +++ b/dw/server/api_models_base.py @@ -0,0 +1,55 @@ +"""The base every response model derives from, and its strictness. + +`dw/server/api_models.py` holds the contract this enforces and re-exports +every name here; import from it, not from this module. +""" + +import logging +import os + +from typing import Any + +from pydantic import BaseModel, ConfigDict, Field + + +STRICT = os.environ.get("DW_STRICT_RESPONSES") == "1" + +logger = logging.getLogger("dw.server.api_models") + + +class ApiModel(BaseModel): + model_config = ConfigDict(extra="forbid" if STRICT else "allow") + + +def send_rejected_responses(app) -> None: + """Make runtime leniency cover a declared field of the wrong type too, + not just an undeclared key: the response is logged and sent as the + handler built it, with the route's own status. Some routes have acted + by then - POST /api/jobs has queued the job - and a 500 would invite a + retry that does it twice. Installed by create_app outside strict mode.""" + from fastapi.encoders import jsonable_encoder + from fastapi.exceptions import ResponseValidationError + from fastapi.responses import JSONResponse + + async def send_as_built(request, exc: ResponseValidationError): + route = request.scope.get("route") + logger.error( + "Response for %s %s does not match its model: %s", + request.method, + request.url.path, + exc.errors(), + ) + return JSONResponse( + jsonable_encoder(exc.body), + status_code=getattr(route, "status_code", None) or 200, + ) + + app.add_exception_handler(ResponseValidationError, send_as_built) + + +def sometimes(description: str | None = None) -> Any: + """A key the handler emits only in some states, and never as null when + it does: typed as its value, defaulting to an unvalidated None that + `response_model_exclude_unset` keeps out of the payload. The generated + type is then `key?: T`, not `key?: T | null`.""" + return Field(default=None, description=description) diff --git a/dw/server/api_models_library.py b/dw/server/api_models_library.py new file mode 100644 index 00000000..777c197e --- /dev/null +++ b/dw/server/api_models_library.py @@ -0,0 +1,214 @@ +"""Response models for the library routes - workspaces, workflows, prompts +and enhancer presets. Part of the UI's response contract that +`dw/server/api_models.py` describes; it re-exports every name here. +""" + +from typing import Any, Literal + +from pydantic import ConfigDict, Field + +from .api_models_base import ApiModel, sometimes +from .catalog_shape import SHAPES, TRAITS + +Origin = Literal["workspace", "common", "examples", "builtin"] +Shape = Literal[SHAPES] +Trait = Literal[TRAITS] + + +class LibraryRoot(ApiModel): + root: str + origin: Origin + writable: bool + + +class ShadowedEntry(ApiModel): + name: str + origin: Origin + shadowed_by: Origin + + +class DiskUsage(ApiModel): + files: int + bytes: int + + +class WorkspaceInfo(ApiModel): + name: str + default: bool + root: str | None + workflows: str + assets: str | None + outputs: str + prompts: str | None + common_assets: str | None + usage: DiskUsage = sometimes("Roughly how much disk it holds; listings only.") + note: str = sometimes("The workspace's note, when it has one; listings only.") + + +class WorkspaceList(ApiModel): + workspace_root: str | None + default: str + workspaces: list[WorkspaceInfo] + + +class WorkspaceDeleted(ApiModel): + name: str + deleted: bool + contents: dict[str, DiskUsage] = Field(description="What it held, per folder.") + + +class CopiedItems(ApiModel): + runs: list[str] = Field(description="`/` of each run copied.") + assets: list[str] = Field( + description="Library-relative path of each asset file copied." + ) + + +class WorkspaceCopied(ApiModel): + """What a copy between workspaces did (`POST /api/workspaces/{name}/copy`).""" + + source: str + destination: str + copied: CopiedItems + already_present: list[str] = Field( + description="Runs the destination already held (`/`) " + "and assets it held with identical bytes (library-relative path) - " + "skipped." + ) + renumbered: dict[str, dict[str, int]] = Field( + description="Copied runs whose `vN` changed, keyed `/`: " + "`{from, to}`." + ) + warnings: dict[str, Any] = Field( + description="`unresolved_output_refs`, `absolute_paths` and `no_jobs`, " + "each only when it applies." + ) + bytes_linked: int + bytes_copied: int + + +class WorkflowCost(ApiModel): + """A measured run, as the workflow's own `cost` block declares it.""" + + model_config = ConfigDict(extra="allow") + + device: str + name: str = sometimes() + vram_gb: int | float + minutes: int | float + + +class WorkflowCard(ApiModel): + """One workflow's entry in the full listing (the agent's + `view=compact` answer is a different, smaller shape outside this + model).""" + + kinds: list[str] + steps: int + variables: int + variable_names: list[str] + description: str + configures: str = sometimes( + "For a model config: the template it is a tuned instance of; '' for a " + "template. Absent when the file could not be read." + ) + configures_missing: str = sometimes("A `configures` that names no workflow.") + prompt_refs: list[str] + shape: Shape = Field(description="What the workflow makes, derived by the server.") + traits: list[Trait] = Field( + description="Sorted, independent facts about how the output is made or what " + "it needs." + ) + summary: str = Field( + description="The description's first sentence, clipped - what a card shows." + ) + lists: dict[str, Any] + constraints: dict[str, Any] + cost_drivers: dict[str, Any] + cost: list[WorkflowCost] | None = Field( + description="Measured runs, one per device the maintainer measured on. Null " + "means unknown - never derived." + ) + observed: dict[str, Any] = sometimes("This box's own history for it.") + origin: Origin + writable: bool = Field( + description="False for a read-only source: offer save-a-copy, not delete." + ) + + +class BuiltinWorkflows(ApiModel): + workflows: list[str] = Field( + description="The packaged fragments a sub-workflow step names as " + "`builtin:.json`, each as what follows the prefix." + ) + + +class WorkflowList(ApiModel): + workspace: str + libraries: list[LibraryRoot] = Field( + description="The search path in order; the writable workspace root is where " + "a save lands, whatever library a workflow was read from." + ) + workflows: list[str] + details: dict[str, WorkflowCard] + shadowed: list[ShadowedEntry] + cost_basis: str + + +class WorkflowSaved(ApiModel): + name: str + workspace: str + origin: Origin + warnings: list[str] + shape: Shape + traits: list[Trait] + summary: str + + +class WorkflowDeleted(ApiModel): + name: str + workspace: str + origin: Origin + deleted: bool + + +class PromptCard(ApiModel): + description: str + intended_model: str + tags: list[str] + text: str = sometimes("Absent when the listing was asked for without text.") + text_chars: int = sometimes("The text's length, in place of `text`.") + origin: Origin = Field( + description="Which library the prompt came from, and whether a save can " + "reach it." + ) + writable: bool + + +class PromptList(ApiModel): + libraries: list[LibraryRoot] + prompts: list[str] + details: dict[str, PromptCard] + shadowed: list[ShadowedEntry] + + +class PromptSaved(ApiModel): + name: str + + +class Deleted(ApiModel): + name: str + deleted: bool + + +class EnhancerPreset(ApiModel): + key: str + label: str + default_model: str + models: list[str] + intended_models: list[str] + placeholder: str + + +class EnhancerPresets(ApiModel): + presets: list[EnhancerPreset] From 67fb8c0e6b8cb682d81048c3d823746b545afe29 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:42:18 -0500 Subject: [PATCH 195/204] refactor(server): drive JobHistory's column migrations from a table The sixteen ALTERs JobHistory.__init__ ran one if-block each become _ADDED_COLUMNS, a (column, ddl) list applied by _add_missing_columns; the workspace and device-field backfills still run once, when their columns were just added. Same DDL, same order, same comments; __init__ drops from C901 complexity 18 to under the limit. A new test opens a v0.13.0-shaped jobs.sqlite (no note/memory_key/rerun_of/kept) and checks it gains the four. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- dw/server/job_history.py | 192 +++++++++++++++++++------------------- tests/test_server_jobs.py | 37 ++++++++ 3 files changed, 133 insertions(+), 98 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 689cb442..f0b7a319 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -164,7 +164,7 @@ one to open. | Per-card memory and cache | `dw/server/worker_memory.py`: `memory_status`, `clear_memory`, `probe_cache` | Each card keeps its own last memory reading; an idle card's worker is asked live, a busy one answers with the run's last report and says why it is not live, and a clear leaves a card running a job alone. Every request takes the card's lock with a bounded wait, and a card's lock is never taken while the JobManager's lock is held. | `tests/test_worker_pool.py::TestMemoryPerCard::test_a_named_card_running_a_job_is_refused_though_the_other_is_idle`, `tests/test_worker_pool.py::TestProbeRouting::test_the_probe_goes_to_the_worker_the_job_would_be_routed_to`, `tests/test_server.py::test_memory_says_why_a_reading_is_not_the_worker_s` | | dw disk caches | `dw/disk_caches.py`; `dw/server/jobs.py`: `JobManager.hold_dispatch`; `dw/server/routes/system.py`: `clear_cache`, `delete_cache_entry` | Members `inductor`, `triton`, `quantized`. Clear only a root carrying the `.dw-cache` marker, never while a job is running or queued (`quantized` also not during a model download); an entry delete takes only a 64-hex id, checked before any path is built; clear renames into `.clearing-*` then deletes. The busy check and the dispatch hold are one step in `JobManager.hold_dispatch`, kept for the whole clear or entry delete: a job submitted meanwhile is accepted but stays queued until it ends. | `tests/test_disk_caches.py`, `tests/test_disk_caches.py::test_job_submitted_during_a_clear_waits_for_it` | | Job-to-run link | `dw/server/job_record.py`, `dw/server/jobs.py`: `JobManager.realized` | A job records `run_id`, `run_dir` and `run_version`, and `JobManager.realized` reads that run's `workflow.json`, confined to the output root. | `tests/test_server_jobs.py::test_realized_reads_the_file_the_run_wrote`, `tests/test_server_jobs.py::test_realized_refuses_a_run_dir_that_escapes_the_output_root` | -| Job history | `dw/server/job_history.py`: `JobHistory` | Finished jobs persist in `jobs.sqlite`, in WAL mode, so the Jobs view survives a restart. | `tests/test_server.py::test_job_history_survives_restart_and_reruns`, `tests/test_job_history_wal.py::test_the_jobs_database_uses_wal_mode` | +| Job history | `dw/server/job_history.py`: `JobHistory`, `_ADDED_COLUMNS` | Finished jobs persist in `jobs.sqlite`, in WAL mode, so the Jobs view survives a restart. A database from an older release gains each column it lacks from the `_ADDED_COLUMNS` table when it is opened. | `tests/test_server.py::test_job_history_survives_restart_and_reruns`, `tests/test_job_history_wal.py::test_the_jobs_database_uses_wal_mode`, `tests/test_server_jobs.py::test_a_v0_13_0_database_gains_the_columns_added_since` | | HTTP security | `dw/server/http_security.py`: `install_middleware`, `query_token_ok` | Every API route needs the token, and only a route marked `query_token_ok` takes it as `?token=`. | `tests/test_security_auth.py::TestTheTokenGate::test_every_api_spelling_needs_the_token`, `tests/test_security_auth.py::TestTheTokenGate::test_the_query_token_is_refused_where_it_is_not_allowed` | | Prompt enhancement | `dw/server/enhancers.py`: `PRESETS`, `_T2I_SYSTEM_PROMPT` | `PRESETS` is the Enhance panel's menu (label, curated LLMs, preselect hints); a preset's family-specific knowledge, its spec, lives in a builtin under `dw/workflows/`, never in Python. `t2i`'s generic, family-agnostic system prompt stays in Python. A third preset moves the menu into builtin metadata (#696). | `tests/test_prompt_library.py::test_every_enhancer_preset_targets_a_known_family`, `tests/test_server.py::TestEnhance::test_presets_are_listed`, `tests/test_server.py::TestEnhance::test_enhance_workflows_are_schema_valid`, `tests/test_server.py::TestEnhance::test_an_unknown_preset_is_a_client_error` | diff --git a/dw/server/job_history.py b/dw/server/job_history.py index 448269be..6d9d9243 100644 --- a/dw/server/job_history.py +++ b/dw/server/job_history.py @@ -42,6 +42,97 @@ def _memory_key_of(job): return None +# The columns added since the table was first written, in the order they +# were added: a database missing one gets it by ALTER, and its rows keep the +# column's default (NULL unless the DDL says otherwise) +_ADDED_COLUMNS = ( + # Databases written before events were persisted + ("events", "TEXT"), + # The workspace the job ran in; rows before it are backfilled to + # 'default' (`JobHistory.__init__`) + ("workspace", "TEXT DEFAULT 'default'"), + # The catalog name the job was run from, beside `workflow` (the + # definition's id). Ids are not unique across a catalog forever; names + # are, and a later runtime-by-workflow join wants the exact one. Rows + # before this column stay NULL: old history is unjoinable, new history + # is exact + ("workflow_name", "TEXT"), + # Which run of the workflow this job was - the directory under the + # output root that holds its manifest and its realized workflow. NULL + # for every row predating run tracking, and the manager refuses to + # guess one from file paths + ("run_id", "TEXT"), + ("run_dir", "TEXT"), + # That run's ordinal among the workflow's runs - the 'v4' the gallery + # shows. NULL before the column, and for a job that never opened a run + ("run_version", "INTEGER"), + # Which form of cost acknowledgement queued the job. Rows before the + # column are 'none' - nothing recorded is nothing recorded + ("acknowledged", "TEXT DEFAULT 'none'"), + # The worker's own high-water mark for this run (#243) - NULL for a row + # predating the column and for any run that never reported one + # (cancelled/errored before the worker's final memory_info) + ("host_memory_peak_rss_mb", "REAL"), + # This job's own contribution to that process-lifetime figure - growth + # since the job's first phase-boundary reading, or its current rss when + # it caused no growth (#272). NULL for a row predating the column and + # for any run that never got a memory_info message at all + ("host_memory_job_peak_rss_mb", "REAL"), + # The card the job ran on - "cuda:1 NVIDIA GeForce RTX 3090" (#462). + # NULL for a row predating the column: which card that was is not + # known, and guessing would mislabel a box whose card has been swapped + ("device", "TEXT"), + # The same card as two fields - its ordinal ("cuda:1") and its name + # ("NVIDIA GeForce RTX 3090") - so rerun affinity and observed cost read + # them rather than parse `device` (#693). `device` stays the label + # every reader returns. Rows before the columns are backfilled from it + # once (`_BACKFILL_DEVICE_FIELDS`) + ("device_ordinal", "TEXT"), + ("device_card", "TEXT"), + # The note the job was submitted with. NULL for a row before the column + # and for a job submitted without one + ("note", "TEXT"), + # What decides the run's host-RAM footprint - its pipelines, + # quantization and offload, arguments resolved + # (`host_ram_need.memory_key`, #869). NULL for a row predating the + # column, which counts as no data rather than a guess, and for a run + # that loads no pipeline + ("memory_key", "TEXT"), + # The job a rerun reran. NULL for a fresh submit and for every row + # before the column + ("rerun_of", "TEXT"), + # 1 once a kept asset's provenance names the job (#917), so a workspace + # delete leaves the row for get_job_workflow to read + ("kept", "INTEGER DEFAULT 0"), +) + +# `device_ordinal`/`device_card` from `device`, in SQL: an ordinal never +# holds a space, so the label splits at its first one, and a label with none +# (`cpu`, `mps`) is all ordinal with no card +_BACKFILL_DEVICE_FIELDS = ( + "UPDATE jobs SET" + " device_ordinal = CASE WHEN INSTR(device, ' ') > 0" + " THEN SUBSTR(device, 1, INSTR(device, ' ') - 1)" + " ELSE device END," + " device_card = CASE WHEN INSTR(device, ' ') > 0" + " THEN SUBSTR(device, INSTR(device, ' ') + 1) END" + " WHERE device IS NOT NULL" +) + + +def _add_missing_columns(connection): + """ALTER in each `_ADDED_COLUMNS` column the jobs table lacks; the names + added. ALTER is the whole migration - the column and its DDL are this + module's constants, never input.""" + present = {row[1] for row in connection.execute("PRAGMA table_info(jobs)")} + added = set() + for column, ddl in _ADDED_COLUMNS: + if column not in present: + connection.execute(f"ALTER TABLE jobs ADD COLUMN {column} {ddl}") + added.add(column) + return added + + class JobHistory: """Finished jobs, persisted so the Jobs view survives server restarts. @@ -69,109 +160,16 @@ def __init__(self, db_path): error TEXT, events TEXT )""") - # Databases written before events were persisted are missing the - # column; ALTER is the whole migration, and rows keep NULL - columns = {row[1] for row in connection.execute("PRAGMA table_info(jobs)")} - if "events" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN events TEXT") + added = _add_missing_columns(connection) # Every row predating workspaces belongs to the default one - # history that cannot say which workspace a job ran in stops # making sense the moment there are two - if "workspace" not in columns: - connection.execute( - "ALTER TABLE jobs ADD COLUMN workspace TEXT DEFAULT 'default'" - ) + if "workspace" in added: connection.execute( "UPDATE jobs SET workspace = 'default' WHERE workspace IS NULL" ) - # The catalog name the job was run from, beside `workflow` (the - # definition's id). Ids are not unique across a catalog forever; - # names are, and a later runtime-by-workflow join wants the exact - # one. Rows before this column stay NULL: old history is - # unjoinable, new history is exact - if "workflow_name" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN workflow_name TEXT") - # Which run of the workflow this job was - the directory under the - # output root that holds its manifest and its realized workflow. - # NULL for every row predating run tracking, and the manager - # refuses to guess one from file paths - if "run_id" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN run_id TEXT") - if "run_dir" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN run_dir TEXT") - # That run's ordinal among the workflow's runs - the 'v4' the - # gallery shows. NULL before the column, and for a job that - # never opened a run - if "run_version" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN run_version INTEGER") - # Which form of cost acknowledgement queued the job. Rows before - # the column are 'none' - nothing recorded is nothing recorded - if "acknowledged" not in columns: - connection.execute( - "ALTER TABLE jobs ADD COLUMN acknowledged TEXT DEFAULT 'none'" - ) - # The worker's own high-water mark for this run (#243) - NULL for - # a row predating the column and for any run that never reported - # one (cancelled/errored before the worker's final memory_info) - if "host_memory_peak_rss_mb" not in columns: - connection.execute( - "ALTER TABLE jobs ADD COLUMN host_memory_peak_rss_mb REAL" - ) - # This job's own contribution to that process-lifetime figure - - # growth since the job's first phase-boundary reading, or its - # current rss when it caused no growth (#272). NULL for a row - # predating the column and for any run that never got a - # memory_info message at all - if "host_memory_job_peak_rss_mb" not in columns: - connection.execute( - "ALTER TABLE jobs ADD COLUMN host_memory_job_peak_rss_mb REAL" - ) - # The card the job ran on - "cuda:1 NVIDIA GeForce RTX 3090" - # (#462). NULL for a row predating the column: which card that - # was is not known, and guessing would mislabel a box whose - # card has been swapped - if "device" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN device TEXT") - # The same card as two fields - its ordinal ("cuda:1") and its - # name ("NVIDIA GeForce RTX 3090") - so rerun affinity and - # observed cost read them rather than parse `device` (#693). - # `device` stays the label every reader returns. Rows before the - # columns are backfilled from it once, in SQL: an ordinal never - # holds a space, so the label splits at its first one, and a - # label with none (`cpu`, `mps`) is all ordinal with no card - if "device_ordinal" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN device_ordinal TEXT") - if "device_card" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN device_card TEXT") - # The note the job was submitted with. NULL for a row before the - # column and for a job submitted without one - if "note" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN note TEXT") - # What decides the run's host-RAM footprint - its pipelines, - # quantization and offload, arguments resolved - # (`host_ram_need.memory_key`, #869). NULL for a row predating - # the column, which counts as no data rather than a guess, and - # for a run that loads no pipeline - if "memory_key" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN memory_key TEXT") - # The job a rerun reran. NULL for a fresh submit and for every - # row before the column - if "rerun_of" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN rerun_of TEXT") - # 1 once a kept asset's provenance names the job (#917), so a - # workspace delete leaves the row for get_job_workflow to read - if "kept" not in columns: - connection.execute("ALTER TABLE jobs ADD COLUMN kept INTEGER DEFAULT 0") - if "device_ordinal" not in columns or "device_card" not in columns: - connection.execute( - "UPDATE jobs SET" - " device_ordinal = CASE WHEN INSTR(device, ' ') > 0" - " THEN SUBSTR(device, 1, INSTR(device, ' ') - 1)" - " ELSE device END," - " device_card = CASE WHEN INSTR(device, ' ') > 0" - " THEN SUBSTR(device, INSTR(device, ' ') + 1) END" - " WHERE device IS NOT NULL" - ) + if "device_ordinal" in added or "device_card" in added: + connection.execute(_BACKFILL_DEVICE_FIELDS) @contextlib.contextmanager def _connect(self): diff --git a/tests/test_server_jobs.py b/tests/test_server_jobs.py index f26e60ed..1f8449aa 100644 --- a/tests/test_server_jobs.py +++ b/tests/test_server_jobs.py @@ -144,6 +144,43 @@ def test_a_database_without_the_columns_is_migrated(tmp_path): assert row["run_id"] is None and row["run_dir"] is None +def test_a_v0_13_0_database_gains_the_columns_added_since(tmp_path): + """A jobs.sqlite as v0.13.0 wrote it - every column through + device_card, none of note/memory_key/rerun_of/kept - opens, gains the + four, and keeps its card fields as they were.""" + import sqlite3 + + path = str(tmp_path / "v0.13.0.sqlite") + with sqlite3.connect(path) as connection: + connection.execute( + "CREATE TABLE jobs (id TEXT PRIMARY KEY, workflow TEXT, status TEXT," + " created_at REAL, started_at REAL, finished_at REAL, arguments TEXT," + " spec TEXT, manifest TEXT, warnings TEXT, error TEXT, events TEXT," + " workspace TEXT DEFAULT 'default', workflow_name TEXT, run_id TEXT," + " run_dir TEXT, run_version INTEGER, acknowledged TEXT DEFAULT 'none'," + " host_memory_peak_rss_mb REAL, host_memory_job_peak_rss_mb REAL," + " device TEXT, device_ordinal TEXT, device_card TEXT)" + ) + connection.execute( + "INSERT INTO jobs (id, status, workspace, device, device_ordinal," + " device_card) VALUES ('old-1', 'succeeded', 'w', 'cuda:1 Card'," + " 'cuda:1', 'Card')" + ) + + history = JobHistory(path) + with history._connect() as connection: + columns = {row[1] for row in connection.execute("PRAGMA table_info(jobs)")} + kept = connection.execute("SELECT kept FROM jobs").fetchone()[0] + assert {"note", "memory_key", "rerun_of", "kept"} <= columns + assert kept == 0 + row = history.get("old-1") + assert row["workspace"] == "w" and "note" not in row and "rerun_of" not in row + assert history.device_ordinal("old-1") == "cuda:1" + assert history.peaks_for_key("any") == [] + # Opening it again finds nothing left to add + JobHistory(path) + + def _for_each_workflow(**overrides): """Copied from tests/test_workflow.py's helper of the same name (Task 5) - not imported across test modules, per that task's convention.""" From 3362bd3940dd7b24b27de9677ab667072c660c78 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:39:03 -0500 Subject: [PATCH 196/204] refactor(pipeline): move guide preparation into h3_guides.py (ratchet) pipeline.py was 1,103 lines, over the 1,100 module ceiling. Reading a step's guides entries into frames and soundtrack sits beside guide_frames_array and guide_audio_waveform, which it already called; moved unchanged as prepared_guide. Co-Authored-By: Claude Opus 5.5 --- dw/pipeline_processors/h3_guides.py | 63 ++++++++++++++++++++++++++++- dw/pipeline_processors/pipeline.py | 61 +--------------------------- tests/test_h3_guides.py | 5 +-- 3 files changed, 66 insertions(+), 63 deletions(-) diff --git a/dw/pipeline_processors/h3_guides.py b/dw/pipeline_processors/h3_guides.py index 7f7042ab..86ac51f4 100644 --- a/dw/pipeline_processors/h3_guides.py +++ b/dw/pipeline_processors/h3_guides.py @@ -20,7 +20,8 @@ Nothing restores them before decode: `after_denoise` slices the condition prefix off, and the target rows at the same times are generated, as with a video guide. -This module puts the guide blocks in; the blocks themselves, and the row and +This module puts the guide blocks in, and reads each `guides` entry a step +passes into the arrays they take (`prepared_guide`); the blocks themselves, and the row and position arithmetic they run (`splice_guide_rows`, `guide_audio_span`), are dw/pipeline_processors/h3_guide_steps.py, imported in `guide_blocks()` only, since it imports diffusers when it loads (#790). The hold blocks are @@ -33,6 +34,8 @@ import torch +from ..events import emit_warning +from ..tasks.video_utils import load_audio_video from .h3_hold import core_denoise_sequences from .h3_rules import ( GUIDE_AUDIO_BLOCK, @@ -41,6 +44,8 @@ HOLD_BEFORE, HOLD_BLOCK, LAYOUT_STEP, + guide_frame_problem, + snap_guide_length, ) logger = logging.getLogger("dw") @@ -190,6 +195,62 @@ def guide_frames_array(value): return np.ascontiguousarray(value) +def prepared_guide(where, index, guide, base_dir): + """One `guides` entry checked and read: its clip as uint8 frames cut to + a whole-latent length, with its soundtrack when it asks for one.""" + if not isinstance(guide, dict): + raise ValueError( + f"{where}[{index}] must be {{video, frame}}, got {type(guide).__name__}" + ) + unknown = sorted(set(guide) - {"video", "frame", "audio"}) + if unknown: + raise ValueError( + f"{where}[{index}] has unknown key(s) {unknown} - a guide is " + f"{{video, frame}}, with an optional 'audio'" + ) + if "video" not in guide or "frame" not in guide: + raise ValueError(f"{where}[{index}] needs both 'video' and 'frame'") + problem = guide_frame_problem(guide["frame"]) + if problem: + raise ValueError(f"{where}[{index}]: {problem}") + with_audio = guide.get("audio", False) + if not isinstance(with_audio, bool): + raise ValueError( + f"{where}[{index}]: 'audio' must be true or false, got " + f"{type(with_audio).__name__}" + ) + video = guide["video"] + if isinstance(video, str) or (isinstance(video, dict) and with_audio): + # argument_media imports diffusers, which this module waits for (#790) + from ..argument_media import fetch_video + + # Read with its soundtrack only when the guide holds it + video = ( + load_audio_video(video, base_dir) + if with_audio + else fetch_video(video, base_dir) + ) + try: + frames = guide_frames_array(video) + audio, sample_rate = guide_audio_waveform(video) if with_audio else (None, None) + except ValueError as error: + raise ValueError(f"{where}[{index}]: {error}") from error + length = snap_guide_length(frames.shape[0]) + if length != frames.shape[0]: + emit_warning( + f"{where}[{index}]: a guide clip encodes to whole latents only at " + f"1, 5 or 17m + 5 frames, so its {frames.shape[0]} frames are cut " + f"to the first {length}" + ) + frames = frames[:length] + return { + "video": frames, + "frame": guide["frame"], + "audio": audio, + "sample_rate": sample_rate, + } + + def guide_blocks(): """(DwH3GuideLayoutStep, the t2va condition step, the t2va latents step, DwH3GuideAudioStep) - the classes in diff --git a/dw/pipeline_processors/pipeline.py b/dw/pipeline_processors/pipeline.py index 675a21df..9297290e 100644 --- a/dw/pipeline_processors/pipeline.py +++ b/dw/pipeline_processors/pipeline.py @@ -32,7 +32,7 @@ attention_slicing_requested, place_component, ) -from .h3_guides import guide_audio_waveform, guide_frames_array, guides_refusal +from .h3_guides import guides_refusal, prepared_guide from .h3_hold import hold_audio_reference, holds_audio, refines from .h3_rules import ( GUIDES_INPUT, @@ -41,9 +41,7 @@ HELD_AUDIO_RATE_OUTPUT, HOLD_AUDIO_INPUT, REFINE_STRENGTH_INPUT, - guide_frame_problem, refine_problems, - snap_guide_length, ) from .progress import reported_blocks, reported_progress_bars from .timing import denoise_start_hook, is_compiled, mark @@ -59,8 +57,6 @@ # dw.prompt_weighting (transformers) and diffusers.hooks (peft, bitsandbytes) are # imported where they are used - at module scope they add seconds to every startup -from ..argument_media import fetch_video -from ..tasks.video_utils import load_audio_video from ..events import WorkflowCancelled, emit_log, emit_phase, emit_warning, get_context from ..media_types import AudioVideo from ..step_cache import component_names, copy_containers @@ -758,7 +754,7 @@ def _with_guides(self, arguments): f"{where} takes at most {GUIDE_LIMIT} clips, got {len(guides)}" ) prepared = [ - _prepared_guide(where, index, guide, self.base_dir) + prepared_guide(where, index, guide, self.base_dir) for index, guide in enumerate(guides) ] arguments[GUIDES_INPUT] = prepared @@ -930,59 +926,6 @@ def _denoiser(pipeline): return denoiser if denoiser is not None else getattr(pipeline, "unet", None) -def _prepared_guide(where, index, guide, base_dir): - """One `guides` entry checked and read: its clip as uint8 frames cut to - a whole-latent length, with its soundtrack when it asks for one.""" - if not isinstance(guide, dict): - raise ValueError( - f"{where}[{index}] must be {{video, frame}}, got {type(guide).__name__}" - ) - unknown = sorted(set(guide) - {"video", "frame", "audio"}) - if unknown: - raise ValueError( - f"{where}[{index}] has unknown key(s) {unknown} - a guide is " - f"{{video, frame}}, with an optional 'audio'" - ) - if "video" not in guide or "frame" not in guide: - raise ValueError(f"{where}[{index}] needs both 'video' and 'frame'") - problem = guide_frame_problem(guide["frame"]) - if problem: - raise ValueError(f"{where}[{index}]: {problem}") - with_audio = guide.get("audio", False) - if not isinstance(with_audio, bool): - raise ValueError( - f"{where}[{index}]: 'audio' must be true or false, got " - f"{type(with_audio).__name__}" - ) - video = guide["video"] - if isinstance(video, str) or (isinstance(video, dict) and with_audio): - # Read with its soundtrack only when the guide holds it - video = ( - load_audio_video(video, base_dir) - if with_audio - else fetch_video(video, base_dir) - ) - try: - frames = guide_frames_array(video) - audio, sample_rate = guide_audio_waveform(video) if with_audio else (None, None) - except ValueError as error: - raise ValueError(f"{where}[{index}]: {error}") from error - length = snap_guide_length(frames.shape[0]) - if length != frames.shape[0]: - emit_warning( - f"{where}[{index}]: a guide clip encodes to whole latents only at " - f"1, 5 or 17m + 5 frames, so its {frames.shape[0]} frames are cut " - f"to the first {length}" - ) - frames = frames[:length] - return { - "video": frames, - "frame": guide["frame"], - "audio": audio, - "sample_rate": sample_rate, - } - - def warn_if_safety_checker_blanked(output): """Say so when a safety checker replaced a generated image with a black one. diff --git a/tests/test_h3_guides.py b/tests/test_h3_guides.py index cac60ce4..fc8fac52 100644 --- a/tests/test_h3_guides.py +++ b/tests/test_h3_guides.py @@ -11,7 +11,6 @@ from dw.media_types import AudioVideo from dw.pipeline_processors import h3_guides, h3_hold, h3_rules -from dw.pipeline_processors import pipeline as pipeline_module from dw.pipeline_processors.h3_guides import ( LAYOUT_ANCHORS, default_num_frames, @@ -848,7 +847,7 @@ def guided(): def warnings_seen(monkeypatch): seen = [] monkeypatch.setattr( - pipeline_module, "emit_warning", lambda message, **data: seen.append(message) + h3_guides, "emit_warning", lambda message, **data: seen.append(message) ) return seen @@ -978,7 +977,7 @@ def load(location, base_dir=None): read.append(location) return AudioVideo(clip(5), np.ones((2, 10), np.float32), 16000) - monkeypatch.setattr(pipeline_module, "load_audio_video", load) + monkeypatch.setattr(h3_guides, "load_audio_video", load) result = Pipeline._with_guides( ns(guided), {"guides": [{"video": "clip.mp4", "frame": 0, "audio": True}]} ) From d1319ec6b3e2bb8f8e1075f352895d7bb654b41a Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:40:16 -0500 Subject: [PATCH 197/204] refactor(concat_videos): extract the join loop into _joined_tracks (ratchet) concat_videos was 152 lines, over the 150-line function limit. The loop that lays each input's frames, soundtrack and shot record onto the joined tracks moves unchanged into its own helper. Co-Authored-By: Claude Opus 5.5 --- dw/tasks/concat_videos.py | 43 ++++++++++++++++++++++++++++++++++----- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/dw/tasks/concat_videos.py b/dw/tasks/concat_videos.py index 920ca121..64963827 100644 --- a/dw/tasks/concat_videos.py +++ b/dw/tasks/concat_videos.py @@ -127,6 +127,43 @@ def concat_videos( videos, match_levels, match_levels_dbfs, sample_rate ) + frames, audio, shots = _joined_tracks( + videos, + names, + clips, + waveforms, + sample_rate, + trim_frames, + fps, + crossfade_ms, + audio_bleed_ms, + audio_bleed_gain_db, + seam_fade_ms, + ) + + _warn_fps_override(videos, names, fps) + audio, written_fps = _fitted_audio(frames, audio, sample_rate, fps, videos, shots) + logger.debug(f"Concatenated {len(videos)} videos into {len(frames)} frames") + return AudioVideo(frames, audio, sample_rate, fps=written_fps, shots=shots) + + +def _joined_tracks( + videos, + names, + clips, + waveforms, + sample_rate, + trim_frames, + fps, + crossfade_ms, + audio_bleed_ms, + audio_bleed_gain_db, + seam_fade_ms, +): + """The prepared inputs joined end to end, as (frames, audio, shots): + each video's frames after its head trim, its soundtrack joined at the + seam, and a shot record for where it landed. `audio` is None when no + input carries any.""" frames, audio, audio_native_rate = [], None, None # Where each video landed, measured on the joined picture and track as # they grow - never derived from the frame count, so a track that runs @@ -181,11 +218,7 @@ def concat_videos( ) _record_seam_fade(shots[first_shot[index]], applied) audio_native_rate = getattr(video, "sample_rate", None) - - _warn_fps_override(videos, names, fps) - audio, written_fps = _fitted_audio(frames, audio, sample_rate, fps, videos, shots) - logger.debug(f"Concatenated {len(videos)} videos into {len(frames)} frames") - return AudioVideo(frames, audio, sample_rate, fps=written_fps, shots=shots) + return frames, audio, shots def _record_seam_fade(shot, applied): From 9ad9ea5ee2a596b4632e87d8cebb19481ebfd1f8 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:41:24 -0500 Subject: [PATCH 198/204] refactor(media_compare): read the asset prefix from references (ratchet) short_name spelled "asset:" as a literal, the one prefix_literals hit. It now asks references.is_ref with references.ASSET, as dw/assets.py does. Co-Authored-By: Claude Opus 5.5 --- dw/media_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dw/media_compare.py b/dw/media_compare.py index 5f00c605..c051fcb1 100644 --- a/dw/media_compare.py +++ b/dw/media_compare.py @@ -22,7 +22,7 @@ import numpy from PIL import Image, ImageDraw, ImageFont -from . import media +from . import media, references from .media_frames import compose_grid, evenly_spaced_indices, moment_index, parse_crop MAX_COMPARE_NAMES = 8 @@ -360,7 +360,7 @@ def _cell(frame, box, max_dimension): def short_name(name): """A row's label: an asset reference as given, an output as its run id and file (the workflow folder is the same across a comparison).""" - if name.startswith("asset:"): + if references.is_ref(references.ASSET, name): return name parts = name.replace(os.sep, "/").split("/") return "/".join(parts[-2:]) From 8ed291b886920d7e72c34823b0d6e2b67d17ef4b Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:47:23 -0500 Subject: [PATCH 199/204] fix(roles): #928 - record a declared role, and name the files a later run read annotate_roles marks a role result.role stated (role_declared) so a later reading never demotes it. consumed_outputs walks an output root's runs once and resolves each run's output: inputs (and asset: inputs kept from an output) to the file that run read; an unpinned 'latest' is the newest run started before the reader (resolve_output_reference gains `before`). listed_role applies the read-time rule; nothing is rewritten. Co-Authored-By: Claude Opus 5.5 --- dw/roles.py | 165 +++++++++++++++++++++++++++++++++++++--- dw/runs.py | 34 ++++++++- tests/test_roles.py | 180 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 363 insertions(+), 16 deletions(-) diff --git a/dw/roles.py b/dw/roles.py index ec307493..abf8d346 100644 --- a/dw/roles.py +++ b/dw/roles.py @@ -20,14 +20,50 @@ `role_basis` it took as the run opened; the server reads only what was recorded, so a run that predates roles has none - unknown, not guessed. Inputs are the asset: and output: references the expanded steps carry - a run's -'made from' list. This module names no model. +'made from' list. + +Those rules see one run. A project built from separate workflows - shots, +narration, a score, then an assembly that cuts them together - leaves every +run's deliverable recorded final, though for the project they are +ingredients. So a listing applies one more rule when it reads, without +rewriting a manifest (`consumed_outputs`, `listed_role`): + +4. a file a later run in the same output root read is intermediate - through + an output: reference (a run id, 'v' or 'latest', resolved to the file + that run read: its inputs are pinned as it opened, and an unpinned + 'latest' is the newest run that started before it), or through an asset: + reference to an asset keep_output kept from that file. A role rule 1 + declared is recorded as such (`role_declared`) and never demoted; an entry + from before that flag is demotable, and a file with no recorded role + stays unknown. + +This module names no model. """ -from .references import ASSET, OUTPUT, is_ref, iter_previous_result_references +import json +import os + +from .references import ( + ASSET, + OUTPUT, + is_ref, + iter_previous_result_references, + make_ref, + ref_name, +) +from .runs import ( + MANIFEST_FILE_NAME, + REALIZED_FILE_NAME, + _read_manifest, + iter_run_directories, + kept_provenance, + resolve_output_reference, +) from .security import ( ALLOWED_AUDIO_EXTENSIONS, ALLOWED_IMAGE_EXTENSIONS, ALLOWED_VIDEO_EXTENSIONS, + SecurityError, ) from .step_cache import reference_resolves_to @@ -36,6 +72,9 @@ INPUT = "input" ROLES = (FINAL, INTERMEDIATE) ROLE_KEY = "role" +# On a manifest entry whose role the step's `result.role` declared (rule 1), +# the one role a later run's reading it does not demote +ROLE_DECLARED_KEY = "role_declared" _MEDIA_EXTENSIONS = frozenset( ALLOWED_IMAGE_EXTENSIONS | ALLOWED_VIDEO_EXTENSIONS | ALLOWED_AUDIO_EXTENSIONS @@ -51,15 +90,19 @@ def _subfolder_role(subfolder): return None +def _stated_role(step): + """Rule 1: the role a step's `result.role` names, or None.""" + result = step.get("result") + role = result.get(ROLE_KEY) if isinstance(result, dict) else None + return role if role in ROLES else None + + def _declared_role(step): """Rules 1 and 2: the role a step's result block states, or None.""" result = step.get("result") if not isinstance(result, dict): return None - role = result.get(ROLE_KEY) - if role in ROLES: - return role - return _subfolder_role(result.get("subfolder")) + return _stated_role(step) or _subfolder_role(result.get("subfolder")) def _writes_only_text(files): @@ -84,8 +127,8 @@ def _files_by_step(manifest_steps): def role_basis(steps): """What the rules read from expanded steps: each step's name, its - declared role (rules 1 and 2) and the previous_result references it - carries. A run rewrites its steps' references in place as they resolve, + declared role (rules 1 and 2), whether rule 1 declared it, and the + previous_result references it carries. A run rewrites its steps' references in place as they resolve, so the engine takes this as the run opens, before any step runs.""" basis = [] for step in steps or []: @@ -102,6 +145,7 @@ def role_basis(steps): { "name": step["name"], "declared": _declared_role(step), + "stated": _stated_role(step) is not None, "consumes": consumes, } ) @@ -132,15 +176,21 @@ def step_roles(steps, manifest_steps, basis=None): def annotate_roles(steps, manifest_steps, basis=None): - """Copies of `manifest_steps`, each with its step's `role`.""" - roles = step_roles(steps, manifest_steps, basis) + """Copies of `manifest_steps`, each with its step's `role`, and + `role_declared` where rule 1 decided it.""" + basis = role_basis(steps) if basis is None else basis + roles = step_roles(None, manifest_steps, basis) + stated = {b["name"] for b in basis if b.get("stated")} annotated = [] for entry in manifest_steps or []: if not isinstance(entry, dict): annotated.append(entry) continue name = entry.get("parent_step") or entry.get("step") - annotated.append({**entry, ROLE_KEY: roles.get(name, FINAL)}) + entry = {**entry, ROLE_KEY: roles.get(name, FINAL)} + if name in stated: + entry[ROLE_DECLARED_KEY] = True + annotated.append(entry) return annotated @@ -175,3 +225,96 @@ def run_inputs(steps): ): seen.append({"reference": value, "kind": kind}) return seen + + +def listed_role(role, declared, consumed): + """Rule 4: the role a listing shows for a file whose run recorded + `role` - intermediate once a later run `consumed` it, unless rule 1 + `declared` the role. An unknown role stays unknown.""" + if role is None or declared or not consumed: + return role + return INTERMEDIATE + + +# What each run read, by manifest path, keyed on the manifest's stat: a +# listing asks of every run on every call, and a manifest rarely changes +_READS = {} +_KINDS = {"asset": ASSET, "output": OUTPUT} + + +def _recorded_reads(run_dir): + """The (kind, reference) pairs a run read: its manifest's pinned + `inputs`, or for a run from before them the references its realized + workflow carries.""" + path = os.path.join(run_dir, MANIFEST_FILE_NAME) + try: + stat = os.stat(path) + except OSError: + return () + signature = (stat.st_mtime_ns, stat.st_size, stat.st_ino) + cached = _READS.get(path) + if cached is not None and cached[0] == signature: + return cached[1] + inputs = (_read_manifest(run_dir) or {}).get("inputs") + if not isinstance(inputs, list): + try: + with open(os.path.join(run_dir, REALIZED_FILE_NAME)) as handle: + inputs = run_inputs([json.load(handle)]) + except (OSError, ValueError): + inputs = [] + reads = tuple( + (entry["kind"], entry["reference"]) + for entry in inputs + if isinstance(entry, dict) + and isinstance(entry.get("reference"), str) + and is_ref(_KINDS.get(entry.get("kind"), ""), entry["reference"]) + ) + _READS[path] = (signature, reads) + return reads + + +def _relative(path, root): + return os.path.relpath(os.path.realpath(path), root).replace(os.sep, "/") + + +def consumed_outputs(output_root, find_asset=None, workspace=None): + """Rule 4's evidence: the files under `output_root` (relative, '/' + separated) that a run under it read. `find_asset` maps an asset name to + its file, or None; an asset counts when keep_output recorded it was kept + from a file of `workspace`. One call answers a whole listing.""" + root = os.path.realpath(output_root) + consumed = set() + kept = {} + for run_dir in iter_run_directories(output_root): + run_id = os.path.basename(run_dir) + for kind, reference in _recorded_reads(run_dir): + try: + if kind == "output": + path = resolve_output_reference(reference, root, before=run_id) + elif find_asset is None: + continue + else: + if reference not in kept: + kept[reference] = _kept_source( + reference, find_asset, workspace, root + ) + path = kept[reference] + if path and not _relative(path, root).startswith(".."): + consumed.add(_relative(path, root)) + except (OSError, ValueError, SecurityError): + # a reference that no longer names a file consumes nothing + continue + return consumed + + +def _kept_source(reference, find_asset, workspace, root): + """The output file an asset was kept from, when it was kept from + `workspace`, else None.""" + path = find_asset(ref_name(ASSET, reference).strip()) + provenance = kept_provenance(path) if path else None + if not isinstance(provenance, dict): + return None + source = provenance.get("source") + if provenance.get("workspace") != workspace or not isinstance(source, str): + return None + return resolve_output_reference(make_ref(OUTPUT, source), root) diff --git a/dw/runs.py b/dw/runs.py index 72600fda..75a1ed61 100644 --- a/dw/runs.py +++ b/dw/runs.py @@ -136,7 +136,13 @@ def _runs_newest_first(directory): return [] -def _resolve_segments(directory, parts, reference, root): +def _started_before(run, before): + """Whether `run` started no later than the run id `before` - the second + a run id opens with - and is not that run itself.""" + return run != before and run[:_RUN_ID_STAMP_LENGTH] <= before[:_RUN_ID_STAMP_LENGTH] + + +def _resolve_segments(directory, parts, reference, root, before=None): """Build the path a name stands for, expanding 'latest' or 'v' where it names a run. @@ -156,6 +162,9 @@ def _resolve_segments(directory, parts, reference, root): 'latest' or 'v4' segment in a directory that holds no runs is a name like any other, so a workflow or a file called either stays reachable. + `before`, a run id, narrows 'latest' to the runs that started before + it - what 'latest' named when that run read it. + Returns the path, or None when runs were found and none of them holds the file. """ @@ -166,6 +175,8 @@ def _resolve_segments(directory, parts, reference, root): runs = _runs_newest_first(directory) if runs: for run in runs: + if before and not _started_before(run, before): + continue candidate = _resolve_segments( os.path.join(directory, run), rest, reference, root ) @@ -199,10 +210,12 @@ def _resolve_segments(directory, parts, reference, root): if candidate and os.path.isfile(candidate): return candidate return None - return _resolve_segments(os.path.join(directory, part), rest, reference, root) + return _resolve_segments( + os.path.join(directory, part), rest, reference, root, before + ) -def resolve_output_reference(reference, root=None): +def resolve_output_reference(reference, root=None, before=None): """Resolve an 'output:' reference to the file it names. The name is a path under the output directory - '/ Date: Thu, 15 Oct 2026 20:47:23 -0500 Subject: [PATCH 200/204] fix(gallery): #928 - a file a later run consumed reads as intermediate GET /api/gallery (and its role= filter, which the web UI and MCP list_gallery use) and the gallery metadata route report a file another run of the workspace read as intermediate, unless its step declared the role. The MCP list_gallery description says later step or run; the surface stays within SURFACE_BUDGET. Docs: ARCHITECTURE roles row, WORKFLOW_GUIDE, WORKSPACES, SERVER, MCP. Co-Authored-By: Claude Opus 5.5 --- docs/ARCHITECTURE.md | 2 +- docs/MCP.md | 2 +- docs/SERVER.md | 6 +- docs/WORKFLOW_GUIDE.md | 7 ++- docs/WORKSPACES.md | 10 ++++ dw/server/outputs.py | 48 +++++++++++++-- dw/server/routes/gallery.py | 19 ++++-- dw/server/routes/media.py | 11 +++- dw_mcp/tools_catalog.py | 6 +- tests/test_gallery_roles.py | 116 ++++++++++++++++++++++++++++++++++++ 10 files changed, 206 insertions(+), 21 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index e445d96e..c89271ce 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -67,7 +67,7 @@ one to open. | Concept | Owner | Rule | Enforced by | | --- | --- | --- | --- | | Run directories | `dw/runs.py`: `open_run`, `workflow_identity`, `new_run_id` | Each execution writes into `///` with a `manifest.json`, unless the flat layout is chosen (the test pins the flat-layout exception). | `tests/test_runs.py::TestRunDirectories::test_the_flat_layout_writes_where_it_always_did` | -| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end; the server reports only recorded roles, so an older run has none. | `tests/test_roles.py` | +| A file's role (final / intermediate / input) | `dw/roles.py` | Decided per step: result.role, then a final/intermediate subfolder, then data flow (a step a media-writing or unsaved consumer reads is intermediate). Engine records it at run end, with `role_declared` where result.role stated it; the server reports only recorded roles, so an older run has none. At read time the gallery (`GET /api/gallery`, its `role=` filter, the metadata route) also demotes a file a later run of the same workspace read - an `output:` input (an unpinned `latest` is the newest run started before the reader) or an `asset:` keep_output kept from it - unless its role was declared (`consumed_outputs`, `listed_role`, once per listing; no manifest is rewritten). | `tests/test_roles.py`, `tests/test_roles.py::test_a_file_a_later_run_read_is_consumed`, `tests/test_roles.py::test_listed_role`, `tests/test_gallery_roles.py::test_a_file_a_later_run_read_lists_as_intermediate`, `tests/test_gallery_roles.py::test_a_declared_final_stays_final` | | Notes on runs, files, assets, workspaces | dw/notes.py | notes.json beside what it describes (run dir or asset dir; a file's own note else the run's), workspace.json at a workspace root; only the server writes them, never the engine; ≤4000 chars, plain text. | tests/test_notes.py | | Run versions | `dw/runs.py`: `open_run`, `run_versions`, `record_run_versions`, `copy_runs_in` | A run takes its number once, when it opens (or, copied in from another workspace, when `copy_runs_in` places it), so deleting a middle run leaves a gap rather than renumbering. | `tests/test_runs.py::TestRunVersions::test_a_deleted_middle_run_leaves_a_gap_rather_than_renumbering`, `tests/test_runs.py::TestRunVersions::test_runs_started_in_the_same_second_still_number_upward` | | Run-version surfaces | `dw/runs.py`: `run_versions`, `dw/server/routes/gallery.py`, `dw/server/outputs.py`, `dw_mcp/tools_catalog.py`: `list_gallery`, `ui/src/lib/pages/GalleryPage.svelte` | Only `dw/runs.py` computes the number, and every other module reads it back. | `tests/test_runs.py::TestRunVersions::test_an_output_reference_can_name_a_run_by_its_version` | diff --git a/docs/MCP.md b/docs/MCP.md index 0e2eff50..436c0bbb 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -233,7 +233,7 @@ when no single workflow covers it. | `get_health()` | — | Check that the server is alive, and which machine answered: `version`, `device`, whether any model process is resident (`worker_alive`), and the queue depth. The engine runs one job per GPU: `workers` has one entry per card - `device`, `name`, `vram_gb`, `alive`, its own `current_job`, and the worker's `host_memory_rss_mb` while it runs - and the top-level `current_job` is the longest-running job. `worker_alive: false` is the normal idle state on a server that has not run a job since startup or the last memory clear - not a fault - the on-demand worker starts with the next job (#206) | | `get_server_info()` | — | What this installation can do and where it keeps things: `device` (the accelerator a run will use), `version`, the `workspace` this session is working in and the workflow/asset/output/prompt `directories` of *that* workspace, the bind address and port, whether a token is required, and whether MCP is mounted. Check the device before authoring - a CUDA-only choice (bitsandbytes, `torch.compile`, flash attention) is not available on an `mps` or `cpu` server. `runtime` (#222) reports the Python version, torch version and the CUDA version torch was built against, the NVIDIA driver version (when `nvidia-smi` is reachable), and the installed versions of diffusers, transformers, accelerate, bitsandbytes, peft, safetensors and sentencepiece (`null` for one not installed) - for diagnosing an environment mismatch between boxes without shelling in. `caches` lists dw's disk caches (`inductor`, `triton`, `quantized`), each with `name`, `bytes`, `files`, `enabled`, `clearable` and `disk_free`, plus `entry_count` for a cache with entries, and `cache_entries=true` lists them (`id`, `model`, `subfolder`, `component`, `bytes`, `files`, `unreadable`, `created`, `last_hit`; the quantizer and dtype are not in `/api/caches`); it is omitted when the server has no `/api/caches`. There is no MCP tool to clear one - that is `DELETE /api/caches/{name}` or `.../entries/{entry}` ([SERVER.md](SERVER.md)) | | `list_jobs(limit=20, status=None, workspace=None)` | optional `limit` (newest N), `status` (one state or a comma-separated set of `queued`, `running`, `succeeded`, `failed`, `cancelled`), `workspace` | List queued, running and recent jobs, **newest first** (a queued job held for host RAM carries `waiting_for`). Bounded by default: the unbounded listing was over a client's tool-result limit on a server with a few months of history, which made it a tool that could not be called at all. `total` says how many matched and `truncated`/`next` say so when the answer was cut - raise `limit` or narrow with `status`. Without `workspace`, a named workspace lists its own jobs and the default one lists every job the server holds | -| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | +| `list_gallery(limit=50, subfolder=None, only_orphans=False, workspace=None, folder=None, version=None, media=False, role=None)` | `limit`, `subfolder`, `only_orphans`, `workspace`, `folder`, `version`, `media`, `role` | List generated output files, newest first. A name is `//`, where `` may sit in the subfolder the step chose (`final/episode.mp4`); each entry carries `folder` (the workflow) and `subfolder` (by convention `final` or `intermediate`, `''` when the step chose none, any path the workflow wrote otherwise), and `subfolder="final"` lists only deliverables. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with no run; `intermediate` once a later step or a later run read the file, unless `result.role` declared it); `role=["final"]` lists only those, and the `role` filter defaults to every file. See `dw/roles.py` for how a role is decided. A file starred in the web UI's gallery (its keep mark) carries `starred: true`; an unstarred one carries no `starred`. Each entry carries `run_id` and `version` - that run's ordinal among the workflow's runs, which is how one of several runs that wrote the same basename is named to a person: the web UI labels the same file `v5`. The number is assigned when the run opens and never renumbered, so deleting a run leaves a gap rather than sliding the rest down (a failed run, or a rerun that reused every step, leaves one too - it took a number and may have nothing to list), and it is `null` under the flat output layout, which has no runs. `folder` with `version` lists that one run's files, and `output:/v5/` names one in a workflow; every other tool takes `name`. Each entry also carries a ready-made `url`, already scoped to the workspace that made it - a hand-built `/outputs/` URL 404s for anything but the default workspace. `only_orphans=True` inverts the call: instead of files, it returns run directories with no media anywhere under them (`runs`, each `{name, mtime}`) - a run whose output was deleted before `delete_output` could remove it by name, or one that failed before writing anything; `subfolder` does not apply in this mode, and `name` is exactly what `delete_output` accepts (#170). `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | | `get_gallery_metadata(name, envelope=False, workspace=None, interval_seconds=None)` | `name`, `workspace` | Get the metadata embedded in a generated file — or, when `name` is an `asset:` reference, what an *input* asset holds (`source` says which; `job` is null for an asset). Reading an input's duration, frame count, fps and sample rate before a run is how a caller learns the `total_frames`, `fps` and `sample_rate` a workflow expects it to supply: the exact workflow and arguments that produced it, and, for audio/video, a `media` block (duration, rate, channels, fps, size, peak/mean dBFS) and `findings` - each level problem the server measured against `dw/audio_qc.py`'s thresholds (`full_scale`, `near_silent`), with the fix. Only an image (PNG/JPEG/WebP) embeds `metadata` this way — it is always null for audio and video, and `next` then points at `get_job_workflow(job_id)` when `job` is known, or says a kept asset carries no provenance at all when it isn't. `envelope=true` adds `media.envelope` — `rms_dbfs` and `peak_dbfs` one entry per second, or per `interval_seconds` (default 1.0, 0.05 to 3600; 0.1 places a vocal onset; a finer-than-default width that would need over 2400 bins is refused with the interval that fits) — which is what locates something in a track rather than measuring the whole of it. `media.shots` is set on an output joined from shots: one `{name, start_frame, num_frames, start_sample, num_samples}` per shot, as the join measured them (see the run manifest in docs/WORKFLOW_GUIDE.md), else null. `workspace` names the workspace for this one call without switching the session to it - the same pin `run_workflow` takes, so a job run into another workspace stays reachable from the session that queued it | ### Media diff --git a/docs/SERVER.md b/docs/SERVER.md index efb1f09f..5dd42d32 100644 --- a/docs/SERVER.md +++ b/docs/SERVER.md @@ -595,8 +595,10 @@ The editor's forms come from these; they are just as usable from scripts: `subfolders` list every distinct value over the whole tree, `''` always a member of each so root-level files stay selectable. Each entry also carries `role`, `final` or `intermediate` (null for a flat-layout file with - no run), and `GET /api/gallery?role=final` lists only those (repeatable; a - null role counts as final). See `dw/roles.py` for how a role is decided; + no run; `intermediate` once a later run read the file, unless the step's + `result.role` declared it), and `GET /api/gallery?role=final` lists only + those (repeatable; a null role counts as final). See `dw/roles.py` for how + a role is decided; the metadata route reports the file's `role` and the run's `inputs` too - `GET /api/gallery/{name:path}/download` — download an output file - `GET /api/builtin-workflows` — the packaged fragments a sub-workflow step diff --git a/docs/WORKFLOW_GUIDE.md b/docs/WORKFLOW_GUIDE.md index e872b4ec..c69a7fe6 100644 --- a/docs/WORKFLOW_GUIDE.md +++ b/docs/WORKFLOW_GUIDE.md @@ -273,8 +273,11 @@ document, which is where a draft that validates still fails. When exploring variations, pass `note` to `run_workflow` saying what each run tests ("video_shift 5 against 3"), and call `set_note` to mark the keeper: the note is shown with every file the run made. Each file carries a `role` -(`final` or `intermediate`) inferred from how a step's output is used; set -`result.role` on a step to override it when the default is wrong. +(`final` or `intermediate`) inferred from how a step's output is used - a file +a later run reads (`output:`, or an `asset:` kept from it) lists as +intermediate too; set `result.role` on a step to override it when the default +is wrong, and set `"role": "intermediate"` on a workflow whose deliverable is an +ingredient a later assembly consumes. ### References diff --git a/docs/WORKSPACES.md b/docs/WORKSPACES.md index 5e9b4427..4bfdf31d 100644 --- a/docs/WORKSPACES.md +++ b/docs/WORKSPACES.md @@ -235,6 +235,16 @@ A later workflow names what an earlier run made with an `output:` reference — longer needs files copied back by hand. See [Output References](WORKFLOW_GUIDE.md#output-references). +Each step's entry in `manifest.json` records its `role`, `final` or +`intermediate` (and `role_declared` when the step's `result.role` stated it), +from that one run's data flow. The gallery reads across runs as well: a file a +later run in the same workspace read - by an `output:` reference, or by an +`asset:` that `keep_output` kept from it - lists as `intermediate`, so a +project's shots, narration and score stop reading as finals once the assembly +that cuts them has run. Nothing is rewritten; a role `result.role` declared +`final` stays final, and a run from before roles stays unknown. See +`dw/roles.py`. + To keep the previous layout — everything at the output root, with only a `workflows/`-mirroring subfolder — use `--output-layout flat`, `DW_OUTPUT_LAYOUT=flat`, or `"output_layout": "flat"` in settings. Scripts that glob the output directory diff --git a/dw/server/outputs.py b/dw/server/outputs.py index fcc42aa6..908879f7 100644 --- a/dw/server/outputs.py +++ b/dw/server/outputs.py @@ -33,7 +33,7 @@ library_path, ) from ..notes import read_notes -from ..roles import entry_role +from ..roles import ROLE_DECLARED_KEY, consumed_outputs, entry_role, listed_role from ..runs import ( MANIFEST_FILE_NAME, _read_manifest, @@ -536,30 +536,68 @@ def run_roles(run_dir): separated), mapped to the role its manifest entry records. An entry without one (a run that predates roles) is left out - unknown, not guessed. {} when the manifest is unreadable.""" + return _run_role_record(run_dir)[0] + + +def declared_files(run_dir): + """The files of a run whose role the step declared (`role_declared`, + dw/roles.py rule 1), by the same names `run_roles` uses.""" + return _run_role_record(run_dir)[1] + + +def listed_file_role(run_dir, name, consumed): + """The role a listing shows for file `name` of a run: its recorded role, + demoted when `consumed` (dw/roles.py rule 4).""" + return listed_role( + run_roles(run_dir).get(name), name in declared_files(run_dir), consumed + ) + + +def consumed_in(state, ws): + """`consumed_outputs` over a workspace's outputs, its assets found down + the search path a run resolves 'asset:' in. Once per listing.""" + library = resolution_library(state, ws) + + def find_asset(name): + try: + validate_asset_reference(name) + except SecurityError: + return None + found = library.find(name) + return found[0] if found else None + + return consumed_outputs(ws.outputs, find_asset, ws.name) + + +def _run_role_record(run_dir): + """(roles, declared names) a run's manifest records, cached on its stat.""" try: key = ( run_dir, os.stat(os.path.join(run_dir, MANIFEST_FILE_NAME)).st_mtime_ns, ) except OSError: - return {} + return {}, frozenset() with _ROLE_CACHE_LOCK: if key in _ROLE_CACHE: _ROLE_CACHE.move_to_end(key) return _ROLE_CACHE[key] manifest = _read_manifest(run_dir) or {} - roles = {} + roles, declared = {}, set() for entry in manifest.get("steps") or []: role = entry_role(entry) if role: for name in entry.get("files") or []: roles[str(name).replace(os.sep, "/")] = role + if entry.get(ROLE_DECLARED_KEY) is True: + declared.add(str(name).replace(os.sep, "/")) + record = (roles, frozenset(declared)) with _ROLE_CACHE_LOCK: - _ROLE_CACHE[key] = roles + _ROLE_CACHE[key] = record _ROLE_CACHE.move_to_end(key) while len(_ROLE_CACHE) > _ROLE_CACHE_LIMIT: _ROLE_CACHE.popitem(last=False) - return roles + return record def run_inputs(run_dir): diff --git a/dw/server/routes/gallery.py b/dw/server/routes/gallery.py index 6a7325ce..947e4366 100644 --- a/dw/server/routes/gallery.py +++ b/dw/server/routes/gallery.py @@ -48,7 +48,8 @@ iter_gallery_files, resolve_output_file, run_directory, - run_roles, + consumed_in, + listed_file_role, served_url, split_run_name, strip_output_prefix, @@ -67,7 +68,7 @@ ) -def _gallery_entries(root, ws): +def _gallery_entries(root, ws, consumed=frozenset()): entries = [] try: files = list(iter_gallery_files(root)) @@ -129,7 +130,11 @@ def _version(folder, run_id): } # None under the flat layout, and for a file no manifest lists split = split_run_name(root, relative_name) if run_id else None - entry["role"] = run_roles(split[0]).get(split[1]) if split else None + entry["role"] = ( + listed_file_role(split[0], split[1], relative_name in consumed) + if split + else None + ) if split: if split[0] not in notes_by_run: notes_by_run[split[0]] = read_notes(split[0]) @@ -193,6 +198,7 @@ def _orphan_entries(root): "/api/gallery", response_model=GalleryList, response_model_exclude_unset=True ) def gallery( + request: Request, limit: int = 200, offset: int = 0, folder: Optional[str] = None, @@ -238,7 +244,9 @@ def gallery( stays proportional to the page size rather than the library size. Each entry carries `role`, `final` or `intermediate` (null for a - flat-layout file or one no manifest lists). `role` is a repeatable + flat-layout file or one no manifest lists); a file a later run read + lists as intermediate unless its step declared it final + (dw/roles.py rule 4). `role` is a repeatable filter; a null role counts as final. Omitted, every file is listed.""" if role: unknown = sorted(set(role) - set(ROLES)) @@ -263,7 +271,8 @@ def gallery( "workspace": ws.name, } ) - entries = _gallery_entries(ws.outputs, ws) + # Read once for the listing: which files a later run read (#928) + entries = _gallery_entries(ws.outputs, ws, consumed_in(request.app.state, ws)) folders = sorted({e["folder"] for e in entries} | {""}) subfolders = sorted({e["subfolder"] for e in entries} | {""}) if folder is not None: diff --git a/dw/server/routes/media.py b/dw/server/routes/media.py index 164ea5a0..e6bd3f4b 100644 --- a/dw/server/routes/media.py +++ b/dw/server/routes/media.py @@ -88,8 +88,9 @@ job_provenance, resolve_output_file, run_directory, + consumed_in, + listed_file_role, run_inputs, - run_roles, strip_output_prefix, validated_run_split, workspace_asset_library, @@ -196,7 +197,13 @@ def gallery_metadata( # run directory the manifest is read from is a validator's output split = validated_run_split(ws.outputs, path) if split: - role = run_roles(split[0]).get(split[1]) + # The role the listing shows: demoted once a later run read + # the file (#928) + relative = os.path.relpath(path, os.path.realpath(ws.outputs)) + consumed = relative.replace(os.sep, "/") in consumed_in( + request.app.state, ws + ) + role = listed_file_role(split[0], split[1], consumed) inputs = run_inputs(split[0]) notes = _note_fields(read_notes(split[0]), split[1]) metadata = read_embedded_metadata(path) diff --git a/dw_mcp/tools_catalog.py b/dw_mcp/tools_catalog.py index f323a6da..09a01e7d 100644 --- a/dw_mcp/tools_catalog.py +++ b/dw_mcp/tools_catalog.py @@ -272,7 +272,7 @@ def list_gallery( carries `folder` (the workflow), `subfolder` (the step's chosen subfolder, '' for none; `subfolder=` filters on it) and `role`: `final` is a deliverable, `intermediate` a file a later step - consumed, null with no run. `role=` filters. Each entry's `url` is workspace-scoped; use it as given. + or run consumed, null with no run. `role=` filters. Each entry's `url` is workspace-scoped; use it as given. Entries carry `run_id` and `version`, the run's stable ordinal (the web UI's `v5`) - quote it to a person. `folder=` plus @@ -282,8 +282,8 @@ def list_gallery( only bookkeeping (manifest.json, workflow.json, job.json) as `runs`, each `{name, mtime}` - output deleted, or a run that failed before writing anything; `name` is what `delete_output` accepts. - A run that wrote any file is not listed. `subfolder` does not - apply. Whether an entry is junk is yours to decide. + A run that wrote a file is not listed. `subfolder` is + ignored. Whether an entry is junk is yours to decide. `workspace` scopes one call without switching the session. diff --git a/tests/test_gallery_roles.py b/tests/test_gallery_roles.py index 0431fe2e..70da0182 100644 --- a/tests/test_gallery_roles.py +++ b/tests/test_gallery_roles.py @@ -293,3 +293,119 @@ def test_role_cache_is_lru_and_safe_under_threads(tmp_path, write_run, monkeypat run_roles(runs[4]) kept = {key[0] for key in outputs._ROLE_CACHE} assert runs[0] in kept and runs[1] not in kept + + +# A file a later run consumed reads as intermediate (#928) + +SHOT_RUN = "20261015-000000-aaaaaaaa" +CUT_RUN = "20261015-010000-bbbbbbbb" + + +def _shot_and_cut(output_root, write_run, reference, **shot_entry): + write_run( + output_root, + "shots", + SHOT_RUN, + [{"step": "shot", "files": ["final/shot.png"], "role": "final", **shot_entry}], + files=["final/shot.png"], + ) + write_run( + output_root, + "assembly", + CUT_RUN, + [{"step": "cut", "files": ["final/cut.png"], "role": "final"}], + files=["final/cut.png"], + inputs=[{"reference": reference, "kind": reference.split(":", 1)[0]}], + ) + + +def _roles(client, **params): + listing = client.get("/api/gallery", params=params).json()["files"] + return {f["label"]: f.get("role") for f in listing} + + +def test_a_file_a_later_run_read_lists_as_intermediate( + gallery_client, output_root, write_run +): + _shot_and_cut(output_root, write_run, "output:shots/latest/final/shot.png") + assert _roles(gallery_client) == {"shot.png": "intermediate", "cut.png": "final"} + assert _roles(gallery_client, role="final") == {"cut.png": "final"} + assert _roles(gallery_client, role="intermediate") == {"shot.png": "intermediate"} + meta = gallery_client.get( + f"/api/gallery/shots/{SHOT_RUN}/final/shot.png/metadata" + ).json() + assert meta["role"] == "intermediate" + + +def test_a_declared_final_stays_final(gallery_client, output_root, write_run): + _shot_and_cut( + output_root, + write_run, + "output:shots/v1/final/shot.png", + role_declared=True, + ) + assert _roles(gallery_client, role="final") == { + "shot.png": "final", + "cut.png": "final", + } + + +def test_a_pre_roles_file_a_later_run_read_stays_unknown( + gallery_client, output_root, write_run +): + write_run( + output_root, + "shots", + SHOT_RUN, + [{"step": "shot", "files": ["shot.png"]}], + files=["shot.png"], + ) + write_run( + output_root, + "assembly", + CUT_RUN, + [{"step": "cut", "files": ["cut.png"], "role": "final"}], + files=["cut.png"], + inputs=[{"reference": f"output:shots/{SHOT_RUN}/shot.png", "kind": "output"}], + ) + assert _roles(gallery_client)["shot.png"] is None + + +def test_an_asset_kept_from_a_file_demotes_it(tmp_path, write_run): + from fastapi.testclient import TestClient + + from dw.runs import record_kept_shots + from dw.server.app import create_app + from dw.server.jobs import JobManager + from dw.workspace import Workspace + + from .test_server import ScriptedWorkerManager, success_script + + studio = Workspace(tmp_path / "studio", "studio").ensure() + manager = JobManager( + studio.outputs, + worker_manager=ScriptedWorkerManager(success_script), + history_path=str(tmp_path / "jobs.sqlite"), + workflow_dir=studio.workflows, + ) + app = create_app( + workflow_dir=studio.workflows, + output_dir=studio.outputs, + job_manager=manager, + prompt_dir=studio.prompts, + asset_dir=studio.assets, + workspace=studio.root, + ) + with TestClient(app, base_url="http://localhost") as client: + name = client.get("/api/gallery").json()["workspace"] + os.makedirs(studio.assets, exist_ok=True) + with open(os.path.join(studio.assets, "hero.png"), "wb") as handle: + handle.write(b"\x89PNG\r\n\x1a\n") + record_kept_shots( + studio.assets, + "hero.png", + None, + {"workspace": name, "source": f"shots/{SHOT_RUN}/final/shot.png"}, + ) + _shot_and_cut(studio.outputs, write_run, "asset:hero.png") + assert _roles(client)["shot.png"] == "intermediate" From 1609df67a742eb7a4b36fdc681cf2713c35235f1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:48:45 -0500 Subject: [PATCH 201/204] docs(plugin): #928 - ingredient workflows set role intermediate script-to-video and series-episodes tell the agent that a workflow made for a shot, narration, score or cue, still or reference sets "role": "intermediate" in its saving step's result, so only the assembly's or episode's cut reads as final. series-episodes makes room under the 12 KiB cap by tightening three restated passages (no rule dropped); it is 12,182 bytes, script-to-video 8,728. Co-Authored-By: Claude Opus 5.5 --- plugins/dw/skills/script-to-video/SKILL.md | 8 ++++++++ plugins/dw/skills/series-episodes/SKILL.md | 24 +++++++++++----------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/plugins/dw/skills/script-to-video/SKILL.md b/plugins/dw/skills/script-to-video/SKILL.md index 44b6a463..f5229911 100644 --- a/plugins/dw/skills/script-to-video/SKILL.md +++ b/plugins/dw/skills/script-to-video/SKILL.md @@ -116,6 +116,14 @@ Once every shot exists, hand off to `templates/assemble-and-score` (or the a series) for the recut/bed/match_levels/normalize/pair pass. That skill already owns this step - do not re-derive it here. +The shots, narration, score or cues, stills and references are ingredients +the assembly consumes: a workflow you write or copy for one sets +`"role": "intermediate"` in its saving step's `result`, so it reads as +intermediate in the gallery (which opens on finals) before the assembly +runs. Only the assembly's cut is final. Once a later run reads a file +through `output:` or an `asset:` kept from it, the gallery lists that file +as intermediate on its own. + A sung multi-shot piece is reviewed by its lip-sync target: run `get_guide("tasks", section="Checking the lip-sync target")` on the cut before delivering it. diff --git a/plugins/dw/skills/series-episodes/SKILL.md b/plugins/dw/skills/series-episodes/SKILL.md index 5d16ee20..ac97f197 100644 --- a/plugins/dw/skills/series-episodes/SKILL.md +++ b/plugins/dw/skills/series-episodes/SKILL.md @@ -60,9 +60,8 @@ guard, so it has to be a deliberate, first step rather than an implicit one. Save each portrait prompt with `save_prompt` under one folder for the series and reference it as `prompt:/` from every episode. -`list_prompts(tag="")` is then the cast list, and a character -described the same way in episode 6 as in episode 1 is a reference rather -than a paragraph retyped - which is the drift this skill exists to stop. +`list_prompts(tag="")` is then the cast list, and a character is +referenced, never retyped - retyping is the drift this skill exists to stop. The same goes for a long shot list: a `run_workflow` call carrying 26 prompts is ~50 KB. `save_prompt(name, {"text": ...})` each shot under a series-prefixed name (the library @@ -76,9 +75,13 @@ voice clip, and the shot list's cast entry (`script-to-video` step 2) points at ## 1-5. Per episode: generate, then recut, bed, match_levels, normalize, pair Generate the episode's shots with `templates/minimax/dialogue-short` (or -`music-video`), `shots` referencing the fixed cast from step 0. Each shot -is its own H3 generation, so cast drift *within* one episode does not -happen - only *across* episodes does, which is what step 0 closes. +`music-video`), `shots` referencing the fixed cast from step 0. Drift +happens across episodes, not within one; step 0 closes it. + +A workflow you write for an ingredient - a shot, narration, a score or cue, +a still, a reference - sets `"role": "intermediate"` in its saving step's +`result`, so it reads as intermediate before the recut runs; only the +episode's cut is final. (A file a later run reads drops to intermediate.) **Optional look, before the recut.** Not a beat and skippable: to give every episode one look, grade each shot with `apply_lut` and one shared @@ -125,8 +128,7 @@ composes into, not something to re-derive: episode lower still. Match episodes **downward** anyway: pick one series `target_lufs` that every episode can reach - in practice, about the quietest episode the ceiling holds back - and pass that same value on every episode's - `assemble-and-score` run. Don't pick the loudest episode's loudness and - ask the others to climb to it. A `target_lufs_capped` warning on an + `assemble-and-score` run. A `target_lufs_capped` warning on an episode means that episode's own peak sets the series' ceiling; lower the series `target_lufs` rather than accepting the mismatch. Only when the series should sit louder than its most dynamic episode allows (e.g. -16 @@ -177,10 +179,8 @@ as a whole; `integrated_lufs` (BS.1770, whole-track) is the field that answers that, and is what to compare across episodes. A peak-normalized episode with one loud outlier (a studio-audience laugh, a sting) can sit 1-2 LU quieter than its neighbors even at the same -3 dBFS peak ceiling - -that gap is `target_lufs`'s to close, matched downward to a series-wide -value every episode's ceiling allows (see the **normalize** bullet above); -`target_lufs` alone cannot raise a capped episode to meet a louder one - -`limit: true` can, for a series that must sit louder (same bullet). To confirm every +that gap is `target_lufs`'s to close, matched downward (the **normalize** +bullet; `limit: true` for a series that must sit louder). To confirm every line rendered rather than judging it by ear, run `templates/check-script` against the episode's script: `validate_workflow(name="templates/check-script", arguments={"input_audio": "output:", "lines": [...]})` first (free; it From cefe43fe539edad499b2f33955b6c8d3f818a9c1 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:50:55 -0500 Subject: [PATCH 202/204] docs(api): #928 - GalleryFile.role names a later run as a consumer too Regenerated openapi.json and api-schema.ts (description only). Co-Authored-By: Claude Opus 5.5 --- dw/server/api_models.py | 4 ++-- ui/src/lib/generated/api-schema.ts | 6 ++++-- ui/src/lib/generated/openapi.json | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dw/server/api_models.py b/dw/server/api_models.py index 3331ae0c..17572e26 100644 --- a/dw/server/api_models.py +++ b/dw/server/api_models.py @@ -747,8 +747,8 @@ class GalleryFile(ApiModel): "An audio or video file's length, with `media=true`." ) role: Literal["final", "intermediate"] | None = sometimes( - "`final` for a deliverable, `intermediate` for a file a later step " - "consumed; null under the flat layout or when no manifest lists the file." + "`final` for a deliverable, `intermediate` for a file a later step or " + "run consumed; null under the flat layout or when no manifest lists the file." ) note: str = sometimes("The effective note: the file's own, else its run's.") starred: bool = sometimes("True when the file is starred; absent otherwise.") diff --git a/ui/src/lib/generated/api-schema.ts b/ui/src/lib/generated/api-schema.ts index 9b14329a..bd0bd948 100644 --- a/ui/src/lib/generated/api-schema.ts +++ b/ui/src/lib/generated/api-schema.ts @@ -341,7 +341,9 @@ export interface paths { * stays proportional to the page size rather than the library size. * * Each entry carries `role`, `final` or `intermediate` (null for a - * flat-layout file or one no manifest lists). `role` is a repeatable + * flat-layout file or one no manifest lists); a file a later run read + * lists as intermediate unless its step declared it final + * (dw/roles.py rule 4). `role` is a repeatable * filter; a null role counts as final. Omitted, every file is listed. */ get: operations["gallery_api_gallery_get"]; @@ -2363,7 +2365,7 @@ export interface components { note?: string; /** * Role - * @description `final` for a deliverable, `intermediate` for a file a later step consumed; null under the flat layout or when no manifest lists the file. + * @description `final` for a deliverable, `intermediate` for a file a later step or run consumed; null under the flat layout or when no manifest lists the file. */ role?: ("final" | "intermediate") | null; /** diff --git a/ui/src/lib/generated/openapi.json b/ui/src/lib/generated/openapi.json index d0b00dfd..949dd149 100644 --- a/ui/src/lib/generated/openapi.json +++ b/ui/src/lib/generated/openapi.json @@ -1273,7 +1273,7 @@ "type": "null" } ], - "description": "`final` for a deliverable, `intermediate` for a file a later step consumed; null under the flat layout or when no manifest lists the file.", + "description": "`final` for a deliverable, `intermediate` for a file a later step or run consumed; null under the flat layout or when no manifest lists the file.", "title": "Role" }, "run_id": { @@ -5979,7 +5979,7 @@ }, "/api/gallery": { "get": { - "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists). `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", + "description": "A page of media files in the output directory, newest first.\nStateless by design - the gallery survives server restarts because\nit reads the directory tree, not job history. 'folders' lists every\ndistinct workflow folder present (over the whole directory, not just\nthis page), for the UI's folder filter - a run id is not a folder of\nits own, so a workflow's runs group together; '' stands for files\nsaved directly at the output root, and is itself always a member so\nthat folder-less outputs stay selectable once anything is nested.\n'subfolders' is the other axis, over the whole directory the same\nway: the in-run subfolders steps wrote into ('final',\n'intermediate'), '' for files at a run's root. `folder` and\n`subfolder` filter independently and intersect when both are given.\n`version` narrows to the runs holding that ordinal - with `folder`,\nthe one run \"v4\" names; without it, that run of every workflow.\n\n`only_orphans=true` inverts the whole call: instead of media files,\nit returns run directories holding nothing but their own\nbookkeeping (manifest.json, workflow.json, job.json) as `runs`,\neach `{name, mtime}` - a run that wrote any file at all, a\ntext-shape prompt or a utility's side output included, is not\nlisted. `folder`/`subfolder` and the `folders`/`subfolders` facets\ndo not apply in this mode, since an orphan run has no file to\ncarry either. `name` is exactly what `DELETE /api/gallery/{name}`\naccepts, so listing and deleting an orphan is a two-call round\ntrip (#170).\n\n`media=true` adds `duration_seconds` to each audio/video entry,\nprobed the same way `get_gallery_metadata` reports it - which two\ntakes of the same workflow otherwise have no way to be told apart\nby, since size and mtime are misleading proxies for length (#356).\nOff by default and bounded by `limit`: only the page actually\nreturned is probed, not the whole listing, so the cost of asking\nstays proportional to the page size rather than the library size.\n\nEach entry carries `role`, `final` or `intermediate` (null for a\nflat-layout file or one no manifest lists); a file a later run read\nlists as intermediate unless its step declared it final\n(dw/roles.py rule 4). `role` is a repeatable\nfilter; a null role counts as final. Omitted, every file is listed.", "operationId": "gallery_api_gallery_get", "parameters": [ { From a691aa7c4a2ff71fe49bdfeddcf3e334a6e330fb Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:52:59 -0500 Subject: [PATCH 203/204] docs(release): 0.14.0 notes name #928 cross-run roles Co-Authored-By: Claude Opus 5.5 --- docs/RELEASING.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/RELEASING.md b/docs/RELEASING.md index 705bd5b0..997e7f1a 100644 --- a/docs/RELEASING.md +++ b/docs/RELEASING.md @@ -19,11 +19,15 @@ queued job waits for host RAM rather than being OOM-killed. **Roles and notes** (#855-#858) - - - Gallery files are tagged final or intermediate (inferred when a workflow doesn't say), and the gallery opens on finals. A run lists what it was Made from. +- A file a later run in the workspace read, through `output:` or an asset + kept from it, lists as intermediate, so a project's shots, narration and + score step aside once the cut that uses them exists. A step that sets + `result.role: "final"` stays final. The `script-to-video` and + `series-episodes` skills have ingredient workflows declare + `"role": "intermediate"` (#928). - Runs, files, assets and workspaces take an optional note: `note` on `run_workflow`, `upload_asset` and `keep_output`, and MCP `set_note`. Every listing carries notes; the web UI edits them in place as Markdown. @@ -103,7 +107,7 @@ queued job waits for host RAM rather than being OOM-killed. denoise step after the timing hook. - FasterCache runs on FLUX.1-dev (#897); denoise time on mps is no longer billed to decode (#896). -- The UI's ratchet check and gallery sort are faster. +- The gallery's sort no longer stalls each keystroke on a large library. **Dependencies and deploy** From 318a74b7a5e0a19f7fb5d5695bf87529cae6f692 Mon Sep 17 00:00:00 2001 From: Don Kackman Date: Thu, 15 Oct 2026 20:57:40 -0500 Subject: [PATCH 204/204] chore(ratchet): raise modules 211 -> 219 and test_dw_patch_targets 326 -> 327 --- docs/stabilization/baseline.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/stabilization/baseline.json b/docs/stabilization/baseline.json index 9d02b7fe..a4e7bc90 100644 --- a/docs/stabilization/baseline.json +++ b/docs/stabilization/baseline.json @@ -1,10 +1,10 @@ { - "modules": 211, + "modules": 219, "modules_over_size_ceiling": 0, "functions_over_150_lines": 0, "prefix_literals": 0, "prefix_handling": 0, - "test_dw_patch_targets": 326, + "test_dw_patch_targets": 327, "claude_md_lines": 129, "duplicate_blocks": 0, "complex_functions": 2,