From 18217f0e0c7204fd2ff57956da3b42d1eb599124 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 00:29:22 +0000 Subject: [PATCH 01/14] python auto-docs + clean up guides for demo-api + contributing.md cleanup --- .github/workflows/reuse-lint.yml | 4 +- AGENTS.md | 2 +- CONTRIBUTING.md | 215 +-- README.md | 200 +-- apps/README.md | 10 +- docs/source/.nav.yml | 6 +- docs/source/community/.nav.yml | 2 + docs/source/community/index.md | 5 +- docs/source/documentation/cli.md | 19 +- .../demo_api/api_reference/application.md | 416 +---- .../guides/adding_a_new_configuration.md | 102 ++ .../demo_api/guides/application_slugs.md | 59 - .../demo_api/guides/configuration.md | 104 -- .../demo_api/guides/create_demo.md | 254 ++- .../demo_api/guides/integrate_model.md | 100 -- .../integrate_model_adapter_for_a_demo.md | 123 ++ .../demo_api/guides/interactive_serving.md | 70 +- docs/source/documentation/demo_api/index.md | 11 +- .../inferencing_api/api_reference/core.md | 367 +--- .../inferencing_api/api_reference/pipeline.md | 1513 +---------------- .../inferencing_api/api_reference/runtime.md | 546 +----- .../inferencing_api/guides/create_model.md | 72 +- .../guides/stream_inference_pipeline.md | 15 + docs/source/documentation/tools/index.md | 3 +- docs/source/documentation/tools/testing.md | 69 + docs/source/index.md | 2 + docs/source/models/index.md | 2 +- docs/source/quickstart/index.md | 13 + docs/zensical.toml | 14 + flashdreams/flashdreams/infra/__init__.py | 4 + .../flashdreams/infra/diffusion/__init__.py | 4 + integrations_v2/README.md | 17 +- pyproject.toml | 1 + tests/README.md | 6 +- tests/test_docs_links.py | 95 +- uv.lock | 210 ++- 36 files changed, 1123 insertions(+), 3532 deletions(-) create mode 100644 docs/source/community/.nav.yml create mode 100644 docs/source/documentation/demo_api/guides/adding_a_new_configuration.md delete mode 100644 docs/source/documentation/demo_api/guides/application_slugs.md delete mode 100644 docs/source/documentation/demo_api/guides/configuration.md delete mode 100644 docs/source/documentation/demo_api/guides/integrate_model.md create mode 100644 docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md create mode 100644 docs/source/documentation/tools/testing.md create mode 100644 flashdreams/flashdreams/infra/__init__.py create mode 100644 flashdreams/flashdreams/infra/diffusion/__init__.py diff --git a/.github/workflows/reuse-lint.yml b/.github/workflows/reuse-lint.yml index f242fe993..26d712b75 100644 --- a/.github/workflows/reuse-lint.yml +++ b/.github/workflows/reuse-lint.yml @@ -73,14 +73,14 @@ jobs: echo "OK: $f" done - - name: CONTRIBUTING.md references the DCO / sign-off + - name: Contribution guide retains DCO policy run: | set -euo pipefail if ! grep -qE "Developer.{1,40}Certificate.{1,10}of.{1,10}Origin|Signed-off-by|sign-off" CONTRIBUTING.md; then echo "::error::CONTRIBUTING.md does not reference DCO / sign-off" exit 1 fi - echo "OK: CONTRIBUTING.md references DCO / sign-off" + echo "OK: CONTRIBUTING.md retains DCO policy" # Inline SPDX-License-Identifier on first-party source files. # REUSE.toml provides aggregate coverage so 'reuse lint' is green even diff --git a/AGENTS.md b/AGENTS.md index 4c37e354e..7e8220186 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -87,7 +87,7 @@ Before performance work, collect an nsys report with `uv run flashdreams-profile ## Testing Guidance -Every pytest test must carry exactly one of `ci_cpu`, `ci_gpu`, or `manual`; `CONTRIBUTING.md` has the exact rules, including how pytest discovers files and functions (`test_*.py`, `test_*`). Use module-level `pytestmark = pytest.mark.ci_cpu` for pure Python/metadata tests. Keep GPU, `libGL`/`cv2`, large-checkpoint, credential, and download-heavy checks out of `ci_cpu`. +Every pytest test must carry exactly one of `ci_cpu`, `ci_gpu`, or `manual`; [Testing](https://nvidia.github.io/flashdreams/main/documentation/tools/testing/) has the exact discovery and marker rules. Use module-level `pytestmark = pytest.mark.ci_cpu` for pure Python/metadata tests. Keep GPU, `libGL`/`cv2`, large-checkpoint, credential, and download-heavy checks out of `ci_cpu`. **v2 test ownership** — put a new test next to the thing it validates: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 55b528fef..73479ff21 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,6 +11,12 @@ We have intentionally kept the process light. If something below is unclear or feels heavier than it should be, that's a bug; please file an issue and we'll fix it. +Setup and build instructions: + +- [install and run FlashDreams](https://nvidia.github.io/flashdreams/main/quickstart/); +- [work on one integration package](https://nvidia.github.io/flashdreams/main/quickstart/#install-one-integration); +- [build and preview the website](https://nvidia.github.io/flashdreams/main/documentation/tools/build_website/). + ## Table of contents 1. [Ways to contribute](#ways-to-contribute) @@ -19,13 +25,15 @@ issue and we'll fix it. 4. [Submitting a pull request](#submitting-a-pull-request) 5. [Code review and merge](#code-review-and-merge) 6. [File Tree Of FlashDreams](#file-tree-of-flashdreams) -7. [Coding conventions](#coding-conventions) -8. [Testing](#testing) -9. [Dependency version bounds](#dependency-version-bounds) -10. [Working with a single integration package](#working-with-a-single-integration-package) -11. [Licensing of contributions](#licensing-of-contributions) -12. [Reporting issues](#reporting-issues) -13. [Code of Conduct](#code-of-conduct) +7. [Adding documentation](#adding-documentation) +8. [Coding conventions](#coding-conventions) +9. [Testing](#testing) +10. [Dependency version bounds](#dependency-version-bounds) +11. [Working with a single integration package](#working-with-a-single-integration-package) +12. [Adding an Agent Skill](#adding-an-agent-skill) +13. [Licensing of contributions](#licensing-of-contributions) +14. [Reporting issues](#reporting-issues) +15. [Code of Conduct](#code-of-conduct) ## Ways to contribute @@ -35,9 +43,9 @@ overhead" to "high overhead": - **Try FlashDreams and tell us what broke.** A clear bug report — what you ran, what you expected, what you saw — is one of the most valuable contributions a project of this kind can receive. -- **Improve documentation.** README clarifications, integration walkthroughs, - performance notes, and FAQ entries all land easily and benefit every - future user. +- **Improve documentation.** Website articles, integration walkthroughs, + performance notes, and troubleshooting entries all land easily and benefit + every future user. - **Fix bugs.** Issues labelled `good first issue` are a friendly starting point. Larger fixes are welcome too — please leave a comment on the relevant issue first so we can avoid duplicate work. @@ -171,6 +179,10 @@ pseudonymous identities cannot be accepted. ## Submitting a pull request +Use the website's [Get Started](https://nvidia.github.io/flashdreams/main/quickstart/) +guide for installation and the relevant model or demo authoring guide for +component-specific setup. Then follow the repository process below. + The short version: 1. Fork the repo on GitHub and create a feature branch from `main`. @@ -226,11 +238,13 @@ short ping comment. ## File Tree Of FlashDreams -Where a new app or model goes. Each documents its own layout: +Where a new app or model goes: ```text -apps/ # reusable apps; layout in apps/README.md -integrations_v2/ # model packages; layout in integrations_v2/README.md +apps/ # reusable, model-independent applications +integrations_v2/ # model packages and application adapters +docs/source/demos/ # canonical demo articles +docs/source/models/ # canonical model articles ``` The framework package, and the test and doc trees: @@ -258,6 +272,82 @@ tests/ # repo-wide test-runner scripts + meta checks, n docs/source/ # Zensical Markdown sources ``` +## Adding documentation + +Write documentation for users and contributors as a website article under +`docs/source/`, not as a package `README.md`. Update an existing article when +it already owns the topic. Otherwise, add a Markdown file in the section that +owns it: + +| Article | Website source | Example | +| --- | --- | --- | +| Install or first run | `docs/source/quickstart/` | `docs/source/quickstart/index.md` | +| Model requirements, setup, commands, or limitations | `docs/source/models/` | `docs/source/models/my_model.md` | +| Demo controls, options, or behavior | `docs/source/demos/` | `docs/source/demos/my_demo.md` | +| API concept or development guide | `docs/source/documentation//guides/` | `docs/source/documentation/demo_api/guides/my_topic.md` | +| API reference | `docs/source/documentation//api_reference/` | `docs/source/documentation/inferencing_api/api_reference/my_api.md` | +| Project or community process | `docs/source/community/` | `docs/source/community/my_process.md` | +| Common user failure | `docs/source/troubleshooting.md` | Add a focused section to the existing page | + +For example, a new demo article starts at `docs/source/demos/my_demo.md`: + +```markdown +--- +title: 'My Demo' +--- + + + + +Explain what the demo does, how to launch it, its controls, and its options. +``` + +Register a new page in the `.nav.yml` beside it; for this example, add +`My Demo: my_demo.md` to `docs/source/demos/.nav.yml`. Use a lowercase +snake-case filename that matches the model or demo package when applicable. + +The focused authoring guides show the expected locations in context: + +- [Create a demo](https://nvidia.github.io/flashdreams/main/documentation/demo_api/guides/create_demo/#package-layout) + uses `docs/source/demos/.md`. +- [Create a model](https://nvidia.github.io/flashdreams/main/documentation/inferencing_api/guides/create_model/#2-scaffold-the-package) + uses `docs/source/models/.md`. +- [Integrate model adapter for a demo](https://nvidia.github.io/flashdreams/main/documentation/demo_api/guides/integrate_model_adapter_for_a_demo/#5-document-once) + updates the model article and links to the shared demo article. + +Files outside `docs/source/` should link to the published website article +instead of copying its instructions. Add a package README only when external +tooling requires one; keep it to a pointer to the canonical website article. + +Build or preview the site with the commands in +[Build Website](https://nvidia.github.io/flashdreams/main/documentation/tools/build_website/#build-the-website). + +## Adding an Agent Skill + +Project-authored [Agent Skills](https://agentskills.io/specification) live in +`skills//SKILL.md`. Each skill requires YAML +frontmatter with a matching `name` and a specific `description` explaining what +the skill does and when to use it. Keep contribution policy in this guide and link to its exact section from +the skill instead of copying it. + +Add new skills to the **Skill Map** in `AGENTS.md`. Put sibling references, +scripts, or assets beside a skill only when they materially improve its +workflow. Validate skill structure with: + +```bash +uv run --group test pytest -m ci_cpu tests/test_agent_skills.py +``` + +Repository skills are opt-in. From the repository root, link all skills into +the directory used by your agent: + +```bash +mkdir -p .agents && ln -s ../skills .agents/skills +``` + +Replace `.agents` with `.cursor` or `.claude` as needed. These directories are +gitignored. To enable one skill, link only that skill directory instead. + ## Coding conventions - Python 3.10+. Type-annotate new code; the project type-checks with @@ -287,65 +377,8 @@ docs/source/ # Zensical Markdown sources ## Testing -Pytest collection is configured in the root `pyproject.toml`. There is -no `testpaths` setting, so `pytest` from the repo root walks the tree: - -- Files named `test_*.py`. Pytest's default also collects `*_test.py`; - don't use that name here. -- Classes named `Test*` (pytest default). Some modules use classes, - some only module-level functions. -- Functions and methods named `test_*` (pytest default), including - `async def test_*`. - -Same test filename in different folders is fine (several -`test_application.py` files exist). Root `pyproject.toml` already sets -`--import-mode=importlib` so pytest keeps them separate. Do not remove -that flag. - -A `test_*.py` next to the code it validates is collected with no CI -change. The root `pyproject.toml` skips `parity_check`, -`parity_check_v2`, `baseline_fastvideo`, and `baseline_lightx2v` via -`norecursedirs`. - -Every test function must be marked with exactly one **CI tier marker**. -A pytest plugin (`flashdreams._pytest_plugins.marker_enforcement`) -enforces this at collection time -- tests without a marker are -rejected, and tests with both `ci_cpu` and `ci_gpu` are rejected. - -| Marker | When to use | CI runner | -|--------|-------------|-----------| -| `@pytest.mark.ci_cpu` | Pure CPU logic, no GPU or `libGL` needed | CPU runner | -| `@pytest.mark.ci_gpu` | Needs CUDA, `libGL`, or transitive `cv2` | GPU runner (RTX Pro 6000) | -| `@pytest.mark.manual` | Heavy (OOM risk), flaky, needs credentials or large downloads | Not run in CI | - -Use a module-level `pytestmark` when every test in a file shares the -same marker: - -```python -import pytest - -pytestmark = pytest.mark.ci_cpu -``` - -Use per-function markers when tests in the same file have different -tiers: - -```python -@pytest.mark.ci_cpu -def test_basic_math(): ... - -@pytest.mark.ci_gpu -def test_cudagraph_path(): ... -``` - -Running tests locally: - -```bash -uv run pytest -m ci_cpu # CPU-safe tests only -uv run pytest -m ci_gpu # GPU tests only (needs CUDA) -uv run pytest -m "not manual" # everything that runs in CI -uv run pytest # all tests including manual -``` +See [Testing](https://nvidia.github.io/flashdreams/main/documentation/tools/testing/) for test discovery, +CI tier markers, and commands. ## Dependency version bounds @@ -366,41 +399,9 @@ the declared minimums. This means: ## Working with a single integration package -The workspace contains many integration packages under `integrations_v2/`. -A full `uv sync` installs dependencies for *all* of them. If you only -need one (e.g. you're working on `omnidreams`), use the distribution -package name with `--package` to sync only that package's dependencies: - -```bash -# Only install omnidreams + its deps (skips unrelated heavy packages) -uv sync --package flashdreams-omnidreams --extra dev - -# Run a script/test from that integration only -uv run --package flashdreams-omnidreams pytest integrations_v2/omnidreams/tests/ -m ci_gpu -``` - -This avoids pulling in (and compiling) dependencies that other -integrations require but yours does not, further reducing setup time. - -Available integration packages: - -| Path | `uv --package` name | -|------|---------------------| -| `integrations_v2/causal_forcing` | `flashdreams-causal-forcing` | -| `integrations_v2/cosmos_predict2` | `flashdreams-cosmos-predict2` | -| `integrations_v2/fastvideo_causal_wan22` | `flashdreams-fastvideo-causal-wan22` | -| `integrations_v2/flashvsr` | `flashdreams-flashvsr` | -| `integrations_v2/hy_worldplay` | `flashdreams-hy-worldplay` | -| `integrations_v2/lingbot` | `flashdreams-lingbot` | -| `integrations_v2/omnidreams` | `flashdreams-omnidreams` | -| `integrations_v2/self_forcing` | `flashdreams-self-forcing` | -| `integrations_v2/swiftvr` | `flashdreams-swiftvr` | -| `integrations_v2/wan21` | `flashdreams-wan21` | -| `integrations_v2/wan22` | `flashdreams-wan22` | - -The nested `integrations_v2/omnidreams/impl/ludus-renderer` workspace package is -named `ludus-renderer` and is installed as part of Omnidreams workflows -that need it. +See [Install one integration](https://nvidia.github.io/flashdreams/main/quickstart/#install-one-integration) +for package-specific setup. Available integrations and their setup instructions +are listed on the [Models](https://nvidia.github.io/flashdreams/main/models/) page. ## Licensing of contributions diff --git a/README.md b/README.md index 66566d773..5fb398e85 100644 --- a/README.md +++ b/README.md @@ -1,199 +1 @@ - - -

- - - FlashDreams - -

- -

- License: Apache 2.0 - Documentation -

- -**FlashDreams** is a high-performance inference and serving library for -interactive autoregressive video and world models. It began as the optimized -runtime behind the [NVIDIA OmniDreams closed-loop demo for GTC 2026][omnidreams-blog] -and has grown into a general platform for real-time world-model applications -across gaming, autonomous vehicles, robotics, simulated or virtual -environments, and more. - -[omnidreams-blog]: https://research.nvidia.com/labs/sil/projects/omnidreams-blog/ - -https://github.com/user-attachments/assets/2b000ce9-effe-4cc9-a227-5b4619413e4d - -## System Requirements - -- NVIDIA GPU with **80 GB VRAM or more** (e.g. H100 80GB), see notes below. -- NVIDIA driver from the **R580 series or newer** (default compatible with CUDA 13.x) -- **CUDA 13.x by default** (PyTorch `2.11.0+cu130` and the `nvidia-*-cu13` libraries are - resolved by `uv sync`. A system CUDA toolkit is needed only for the - developer extras and is included in `nvidia/cuda:13.2.1-cudnn-devel-ubuntu24.04`) -- **Python >= 3.10** -- **PyTorch >= 2.11.0+cu130** (`>= 2.9` for bare PyPI library install) -- Linux x86-64 or arm64 -- **100 GB+ free storage space** recommended for environment and model checkpoints. -- Docker with the - [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) - (optional, only for the container workflow) - -> Development and testing were performed on GPUs with **80 GB of VRAM or more**. -> Inference can fail (out-of-memory) on consumer and even enthusiast GPUs. -> Per-model GPU and VRAM requirements are listed on each model page in -> [the model gallery](https://nvidia.github.io/flashdreams/main/models/index.html). - -## Quickstart - -The complete setup is in -[the installation guide](https://nvidia.github.io/flashdreams/main/quickstart/index.html#install). -Assuming `uv` is [installed](https://docs.astral.sh/uv/getting-started/installation), the shortest viable path is: - -```bash -git clone https://github.com/NVIDIA/flashdreams.git -cd flashdreams -uv sync --extra runners -export HF_TOKEN= -uv run flashdreams-run --help -``` - -Note for developers/maintainers you would want to run `uv sync --extra dev --extra runners` instead. - -### Select CUDA Version - -FlashDreams defaults to the CUDA 13 PyTorch stack (the `cuda13` dependency -group is activated automatically via `default-groups`): - -- **Linux:** `uv sync` pulls `torch>=2.9` from PyPI, whose Linux wheels are - the CUDA 13 build (the `+cu130` local-version tag is just stripped on - PyPI). -- **Windows:** `uv sync` pulls torch from NVIDIA's `cu130` index - (`https://download.pytorch.org/whl/cu130`) — PyPI's Windows torch wheel - is CPU-only. - -CUDA 12.8 is available as an opt-in side profile (Linux only) via the -`cuda12` dependency group: - -```bash -uv sync --group cuda12 --extra runners -``` - -The two groups are declared mutually exclusive in `pyproject.toml`, so uv -deactivates the default `cuda13` group automatically when `--group cuda12` -is passed. - -Then launch your first model by following [the Get Started guide](https://nvidia.github.io/flashdreams/main/quickstart/index.html#run-your-first-model). -For example, the Self-Forcing T2V v2 application is: - -```bash -uv run --project integrations_v2/self_forcing \ - flashdreams-run-v2 t2v-self-forcing-wan2.1-t2v-1.3b \ - --output-path artifacts/t2v-self-forcing-wan2.1-t2v-1.3b.mp4 --timeout unbound -- \ - --prompt "A cat surfing." --total-blocks 7 -``` - -You can also install FlashDreams as a library from PyPI: - -```bash -pip install flashdreams -``` - -### Try the interactive driving demo - -Drive a world model in real time with the unified OmniDreams `local-window` or -`webrtc` launch mode. See the -**[interactive demo guide](https://nvidia.github.io/flashdreams/main/models/omnidreams.html#launch-the-interactive-demo)**. - -## Supported models - -FlashDreams ships first-party integrations under -[`integrations_v2/`](integrations_v2/). Each model has a dedicated docs page with -runner slugs, multi-GPU commands, and (where available) profiling benchmarks. - -| Model | Family | -| --- | --- | -| [Self-Forcing](https://nvidia.github.io/flashdreams/main/models/self_forcing.html) | Streaming Wan2.1 T2V | -| [OmniDreams](https://nvidia.github.io/flashdreams/main/models/omnidreams.html) | HDMap-conditioned driving world model | -| [LingBot-World](https://nvidia.github.io/flashdreams/main/models/lingbot_world.html) | Camera-controllable I2V world model | -| [Waypoint 1.5](https://nvidia.github.io/flashdreams/main/models/waypoint.html) | Interactive image-established keyboard/mouse-controlled world model | -| [Wan2.1](https://nvidia.github.io/flashdreams/main/models/wan21.html) | Bidirectional T2V / I2V | -| [Causal-Forcing](https://nvidia.github.io/flashdreams/main/models/causal_forcing.html) | Streaming Wan2.1 T2V / I2V | -| [Causal Wan2.2](https://nvidia.github.io/flashdreams/main/models/causal_wan22.html) | FastVideo Causal Wan 2.2 14B MoE T2V | -| [FlashVSR](https://nvidia.github.io/flashdreams/main/models/flashvsr.html) | Streaming video super-resolution | -| [Cosmos-Predict2.5](https://nvidia.github.io/flashdreams/main/models/cosmos_predict2.html) | Bidirectional T2V / I2V | - -See [the model gallery](https://nvidia.github.io/flashdreams/main/models/index.html) and -[the new method guide](https://nvidia.github.io/flashdreams/main/developer_guides/new_integration.html) -to add your own. - -## Developer guides - -- [Architecture](ARCHITECTURE.md) -- [Inference pipeline overview](https://nvidia.github.io/flashdreams/main/developer_guides/inference_pipeline_overview.html) -- [Config system](https://nvidia.github.io/flashdreams/main/developer_guides/config_system.html) -- [Add a new method](https://nvidia.github.io/flashdreams/main/developer_guides/new_integration.html) - -For day-to-day development: - -```bash -uv sync --extra dev --extra runners -uv run --group lint pre-commit run -a -uv run pytest -m "not manual" -``` - -See [`DEV.md`](DEV.md) for repository-specific workflow notes. - -## Contributing - -For how to contribute, see [`CONTRIBUTING.md`](CONTRIBUTING.md). -New integrations, bug reports, feature requests, performance tuning, and -documentation edits are all welcome. - -Use [GitHub Issues](https://github.com/NVIDIA/flashdreams/issues) to report defects or request improvements. - -Join us on the [NVIDIA Omniverse Discord](https://discord.com/invite/nvidiaomniverse) -to share your results and take part in technical discussion! Channel: [`#flashdreams`](https://discord.gg/cMt2mHm4aN) - -## Security - -To report a potential security vulnerability, follow the coordinated -disclosure process in [`SECURITY.md`](SECURITY.md). - -## License - -FlashDreams is released under the [Apache License 2.0](LICENSE). Third-party -components and their licenses are listed in -[`THIRD-PARTY-NOTICES`](THIRD-PARTY-NOTICES) and [`NOTICE`](NOTICE). The -repository is REUSE-compliant; see [`REUSE.toml`](REUSE.toml) and -[`LICENSES/`](LICENSES/). The independently packaged -[Ludus renderer](integrations_v2/omnidreams/impl/ludus-renderer/LICENSE) retains its own Apache license file; -[upstream cudaraster terms](integrations_v2/omnidreams/impl/ludus-renderer/ludus_renderer/_cpp/cudaraster/LICENSE) -and per-file exceptions are described in the third-party notices. - -## Citation - -If FlashDreams is useful in your research or product, please cite the project: - -```bibtex -@misc{flashdreams2026, - title = {FlashDreams: High-performance inference and serving for - interactive autoregressive video and world models}, - author = {{FlashDreams Contributors}}, - year = {2026}, - howpublished = {\url{https://github.com/NVIDIA/flashdreams}}, -} - -@misc{nvidia2026omnidreams, - title = {{NVIDIA} {OmniDreams}: Real-Time Generative World Model for Closed-Loop Autonomous Vehicle Simulation}, - author = {Basant, Aarti and Kar, Amlan and Paschalidou, Despoina and Wei, Fangyin and Ferroni, Francesco and Garcia Cobo, Guillermo and Turki, Haithem and Ling, Huan and Seo, Jaewoo and Lucas, James and Wu, Jay Zhangjie and Wang, Jialiang and Lorraine, Jonathan and Gao, Jun and He, Kai and Tothova, Katarina and Xie, Kevin and Tyszkiewicz, Micha{\l} and Wu, Qi and de Lutio, Riccardo and Li, Ruilong and Fidler, Sanja and Kim, Seung Wook and Shen, Tianchang and Cao, Tianshi and Pfaff, Tobias and Lew, William and Wu, Xindi and Ren, Xuanchi and Lu, Yifan and Zhang, Yuxuan and Gojcic, Zan and Wang, Zian}, - year = {2026}, - eprint = {2606.03159}, - archivePrefix = {arXiv}, - primaryClass = {cs.CV}, - doi = {10.48550/arXiv.2606.03159}, - url = {https://arxiv.org/abs/2606.03159}, -} -``` +[FlashDreams](docs/source/index.md#flashdreams) diff --git a/apps/README.md b/apps/README.md index eb7abe002..1a9460b85 100644 --- a/apps/README.md +++ b/apps/README.md @@ -15,7 +15,8 @@ apps// / # the app implementation tests/ # validate the app against a stub, not a real model pyproject.toml - README.md # purpose, controls, example command line, options + +docs/source/demos/.md # canonical website article ``` Tests live in `apps//tests/`, beside the package rather than inside @@ -37,5 +38,8 @@ model or its adapter belongs in `integrations_v2//tests/`. `omnidreams_game_engine` is not the OmniDreams model. That is `integrations_v2/omnidreams`. The names are close; the packages are unrelated. -Each app's own README covers how to launch it and what the controls are. To -bind a model to one of these, read `integrations_v2/README.md`. +Put launch instructions, controls, and options in +`docs/source/demos/.md`. Follow the website's +[Create a demo](../docs/source/documentation/demo_api/guides/create_demo.md) +and [Integrate model adapter for a demo](../docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md) +guides. diff --git a/docs/source/.nav.yml b/docs/source/.nav.yml index d922344e7..9b5249824 100644 --- a/docs/source/.nav.yml +++ b/docs/source/.nav.yml @@ -10,9 +10,8 @@ nav: - Demo API Overview: documentation/demo_api/index.md - Guides: - Create a demo: documentation/demo_api/guides/create_demo.md - - Integrate a model with a demo: documentation/demo_api/guides/integrate_model.md - - Demo configuration: documentation/demo_api/guides/configuration.md - - Application slugs and dispatch: documentation/demo_api/guides/application_slugs.md + - Integrate model adapter for a demo: documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md + - Adding a new configuration: documentation/demo_api/guides/adding_a_new_configuration.md - Interactive serving: documentation/demo_api/guides/interactive_serving.md - Local benchmarks: documentation/demo_api/guides/local_benchmarks.md - Profiling a demo: documentation/demo_api/guides/profiling.md @@ -34,5 +33,6 @@ nav: - Tooling Overview: documentation/tools/index.md - Offline Program Packager: documentation/tools/offline_program_packager.md - Build Website: documentation/tools/build_website.md + - Testing: documentation/tools/testing.md - Troubleshooting: troubleshooting.md - Community: community diff --git a/docs/source/community/.nav.yml b/docs/source/community/.nav.yml new file mode 100644 index 000000000..d921fbc54 --- /dev/null +++ b/docs/source/community/.nav.yml @@ -0,0 +1,2 @@ +nav: + - Overview: index.md \ No newline at end of file diff --git a/docs/source/community/index.md b/docs/source/community/index.md index 57493e5b5..2f06f40ec 100644 --- a/docs/source/community/index.md +++ b/docs/source/community/index.md @@ -6,9 +6,6 @@ title: 'Community' -Use the issue tracker for work that needs a record. Use Discord for informal -questions and conversation. Do not report security problems in public. -
ContributeDevelopment workflow, testing requirements, and contribution policy. Bugs and feature requestsSearch existing reports or open an issue with a reproducible example. @@ -55,7 +52,7 @@ application, session, and loop contracts. Start with [Create a model](../documentation/inferencing_api/guides/create_model.md), then -[integrate it with a demo](../documentation/demo_api/guides/integrate_model.md). +[integrate it with a demo](../documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md). ### Can I use FlashDreams commercially? diff --git a/docs/source/documentation/cli.md b/docs/source/documentation/cli.md index 00515332c..602b02b56 100644 --- a/docs/source/documentation/cli.md +++ b/docs/source/documentation/cli.md @@ -53,20 +53,8 @@ These options override the default demo session values requested by a particular | `--pixel-height N` | Override generated-frame height. | | `--fps N` | Override the generated-frame playback rate. | | `--layout {tchw,btchw,bcthw,bvtchw}` | Override generated tensor layout. | -| `--backpressure-mode {block,drop_oldest}` | Choose how the model thread handles a full presentation queue. | -| `--presentation-mode {on_demand,continuous}` | Choose when the presentation loop renders. | - -#### Backpressure Mode - -- `block` waits for queue capacity so generated chunks are retained. - -- `drop_oldest` discards the oldest queued chunk to favor recent output. - -#### Presentation Mode - -- `on_demand` presents each selected model frame when it arrives. - -- `continuous` keeps the UI responsive between model frames by reusing the newest frame. +| `--backpressure-mode {block,drop_oldest}` | Choose full-queue behavior: `block` waits for capacity to retain generated chunks; `drop_oldest` discards the oldest queued chunk to favor recent output. | +| `--presentation-mode {on_demand,continuous}` | Choose rendering behavior: `on_demand` presents each selected model frame when it arrives; `continuous` keeps the UI responsive between model frames by reusing the newest frame. | ### Examples @@ -110,7 +98,6 @@ uv run flashdreams-run-v2 DEMO_MODEL_SLUG --mode native-window --window-title Fl ## See also - [Quickstart](../quickstart/index.md) -- [Demo configuration](demo_api/guides/configuration.md) -- [Application slugs](demo_api/guides/application_slugs.md) +- [Adding a new configuration](demo_api/guides/adding_a_new_configuration.md) - [Demo API](demo_api/index.md) - [Offline Program Packager](tools/offline_program_packager.md) diff --git a/docs/source/documentation/demo_api/api_reference/application.md b/docs/source/documentation/demo_api/api_reference/application.md index 3ba277494..6a48b153b 100644 --- a/docs/source/documentation/demo_api/api_reference/application.md +++ b/docs/source/documentation/demo_api/api_reference/application.md @@ -8,424 +8,20 @@ title: 'Demo Application API Reference' This page documents the public application protocols in `flashdreams.api_v2` and the runtime-owned value types those protocols exchange. For implementation workflows, start with [Create a demo](../guides/create_demo.md) or -[Integrate a model with a demo](../guides/integrate_model.md). +[Integrate model adapter for a demo](../guides/integrate_model_adapter_for_a_demo.md). ## Application and session -### `flashdreams.api_v2.application.IApplication` +::: flashdreams.api_v2.application.IApplication -One application, for as long as the process runs. - -Parses its own arguments and holds whatever its sessions share, such as a -checkpoint or a compiled pipeline. It outlives every session it creates, so -that shared state is loaded once here and released in `close`. - -An application module implements this and `ISession`. The runtime -creates everything else and passes it in. - -[View source](https://github.com/NVIDIA/flashdreams/blob/main/flashdreams/flashdreams/api_v2/application.py#L14) - -#### `init` - -```python -def init(self, commandline_args: Sequence[str]) -> None -``` - -Parse application arguments and validate startup state. - -#### `session_desc` - -```python -def session_desc(self) -> SessionDesc | None -``` - -Return the description of a session this application would generate. - -A caller has to describe a session before there is one to describe, and -only the application knows what its model was trained for. Asked before -`init`, so describing a session costs nothing. - -**Returns** - -The session to create when nobody asks for another, or `None`, -the default, from an application that generates whatever it is -asked for. Its caller describes the session instead. - -#### `create_session` - -```python -def create_session(self, session_desc: SessionDesc) -> ISession -``` - -Create one isolated, uninitialized session for `session_desc`. - -**Parameters** - -- `session_desc`: Session the runtime is asking for. - -**Returns** - -A session for `session_desc`, resolved to what this application can -actually produce. - -**Raises** - -- `ValueError`: The application cannot honour `session_desc`. - -#### `close` - -```python -def close(self) -> None -``` - -Release whatever the application holds. - -Not abstract, and does nothing by default, so an application with nothing -to release does not implement it. - -### `flashdreams.api_v2.session.ISession` - -One application run with a model loop and a UI loop. - -Register the model loop in `init`. If no UI loop is registered, -the runtime uses `BlitModelOutputToScreenLoop`. - -[View source](https://github.com/NVIDIA/flashdreams/blob/main/flashdreams/flashdreams/api_v2/session.py#L21) - -- `_registered_ui_loop: IUILoop[Any] | None` - -- `_registered_model_loop: IModelLoop[Any] | None` - -#### `parallel_context` - -```python -def parallel_context(self) -> ParallelContext | None -``` - -Return the model mesh before initialization, or `None` for a local session. - -The runtime synchronizes model steps and inputs across this world. -Only rank zero owns a client window and output sinks. - -#### `init` - -```python -def init(self) -> None -``` - -Initialize state and register this session's loops. - -A model loop is required. Registering a UI loop is optional; without one -the runtime uses `BlitModelOutputToScreenLoop`. - -**Raises** - -- `RuntimeError`: No model loop was registered by the time the runtime - asks for the loops. - -#### `session_desc` - -```python -def session_desc(self) -> SessionDesc -``` - -Return the description used to configure the runtime. - -#### `register_ui_loop` - -```python -def register_ui_loop(self, loop_type: type[IUILoop[Any]], *, state: Any=None, **kwargs: Any) -> IUILoop[Any] -``` - -Create and register the UI loop, and return it. - -Omit this call to use the default UI. The loop is paced at the session's -`frames_per_second_for_ui`. - -**Parameters** - -- `loop_type`: UI loop class to instantiate. -- `state`: State the loop owns. Unlike a model loop, a UI loop is - allowed to hold none. -- `**kwargs`: Passed to `loop_type`. - -**Raises** - -- `RuntimeError`: The runtime already took the loops, or a UI loop was - registered already. -- `TypeError`: `loop_type` does not derive from `IUILoop`. - -#### `register_model_loop` - -```python -def register_model_loop(self, loop_type: type[IModelLoop[Any]], *, state: Any, **kwargs: Any) -> IModelLoop[Any] -``` - -Create and register the model loop, and return it. - -The loop is paced at the session's `frames_per_second_for_step`. - -**Parameters** - -- `loop_type`: Model loop class to instantiate. -- `state`: State the loop owns. Required, since a model loop with no - state has nothing to generate from. -- `**kwargs`: Passed to `loop_type`. - -**Raises** - -- `RuntimeError`: The runtime already took the loops, or a model loop - was registered already. -- `TypeError`: `loop_type` does not derive from `IModelLoop`. - -#### `ui_loop` - -```python -def ui_loop(self) -> IUILoop[Any] -``` - -Return the registered UI loop. - -#### `model_loop` - -```python -def model_loop(self) -> IModelLoop[Any] -``` - -Return the registered model-generation loop. - -#### `close` - -```python -def close(self) -> None -``` - -Release resources owned by the session. +::: flashdreams.api_v2.session.ISession ## Model and UI loops -### `flashdreams.api_v2.loop.IModelLoop` - -Loop that generates model results on the model thread. - -`ILoop.step` must return `list[StepResult]` here, one entry per -channel, with every channel reporting the same `frame_count`. An empty -list means the step produced no presentable output. - -Every admitted rank executes the same model step and ordered collectives. -The integration owns TP/CP partitioning and must gather all CP shards needed -for publication before the presenting rank returns non-empty results. -Workers return `[]` only after all required collectives, including gathers. -Gather before decoding when it needs the complete latent/view sequence; -decoding may run only on the presenting rank when the model permits it. -TP reductions that already replicate a complete value need no extra gather. - -A run whose steps all present nothing still ends when the model loop does. -Returning a bare `StepResult` or `None` raises `TypeError`. - -[View source](https://github.com/NVIDIA/flashdreams/blob/main/flashdreams/flashdreams/api_v2/loop.py#L289) - -#### `step` +::: flashdreams.api_v2.loop.IModelLoop -```python -def step(self, step_index: int, events: UserInputEvents) -> list[StepResult] -``` - -We expect IModelLoop's to return a list of StepResults, one result per channel. - -#### `inference_state` - -```python -def inference_state(self) -> ModelInferenceState -``` - -Return whether this model loop has started, is running, or finished. - -### `flashdreams.api_v2.loop.IUILoop` - -Loop whose output is sent to the client window. - -`ILoop.step` must return `list[StepResult]` here: one frame to -present, or an empty list to present nothing this step. A list longer than -one raises `TypeError`. Model frames to draw come from -`presented_model_frame` and `presented_model_frames` rather -than from the model loop directly. -`has_pending_model_frames` and `presented_model_frame_count` -report whether more model frames are waiting and how many have been -selected. - -[View source](https://github.com/NVIDIA/flashdreams/blob/main/flashdreams/flashdreams/api_v2/loop.py#L424) - -#### `step` - -```python -def step(self, step_index: int, events: UserInputEvents) -> list[StepResult] -``` - -Return zero or one `StepResult` to present this tick. - -#### `register_session_ui_loop_objects` - -```python -def register_session_ui_loop_objects(self, *, session_desc: SessionDesc, presentation_manager: PresentationManager) -> None -``` - -Store UI objects supplied when this loop is registered with a session. - -**Parameters** - -- `session_desc`: Description of the session this UI presents. -- `presentation_manager`: Buffer containing model frames. - -#### `model_inference_state` - -```python -def model_inference_state(self) -> ModelInferenceState -``` - -Return whether model inference has started, is running, or finished. - -#### `request_new_session` - -```python -def request_new_session(self, session_desc: SessionDesc) -> None -``` - -Ask the runtime to replace this session after the current UI step. - -**Parameters** - -- `session_desc`: Fully resolved description for the replacement session. - -#### `request_hide_cursor` - -```python -def request_hide_cursor(self, hide_cursor: bool) -> None -``` - -Request that the client window show or hide its cursor. - -#### `request_lock_cursor_to_window` - -```python -def request_lock_cursor_to_window(self, lock_cursor_to_window: bool) -> None -``` - -Request that the client window release or capture pointer motion. - -#### `request_new_window_size` - -```python -def request_new_window_size(self, new_window_size: tuple[int, int]) -> None -``` - -Request a positive downstream client-window width and height. - -**Parameters** - -- `new_window_size`: Requested `(width, height)` in pixels. - -**Raises** - -- `TypeError`: The size is not a pair of integers. -- `ValueError`: Either dimension is not positive. - -#### `get_or_create_ui_loop_requests` - -```python -def get_or_create_ui_loop_requests(self) -> UILoopRequests -``` - -#### `flush_ui_loop_requests` - -```python -def flush_ui_loop_requests(self) -> UILoopRequests | None -``` - -return UI loop requests and clear the internal state of requests. - -#### `presented_model_frame` - -```python -def presented_model_frame(self, channel_index: int=0) -> Tensor | None -``` - -Return the current frame from one model-result channel. - -**Parameters** - -- `channel_index`: Channel to read, indexed as the model loop returned - them. - -**Returns** - -A `[C, H, W]` frame with one, three or four channels, or `None` -before the first model result has been presented. - -**Raises** - -- `IndexError`: The presented result has no such channel. -- `ValueError`: The presentation stream is on a different CUDA device - from the presented result. - -#### `presented_model_frames` - -```python -def presented_model_frames(self) -> tuple[Tensor, ...] -``` - -Return the current frame from every model-result channel. - -**Returns** - -One `[C, H, W]` frame per channel, bottom channel first, or an -empty tuple before the first model result has been presented. - -**Raises** - -- `ValueError`: The presentation stream is on a different CUDA device - from a presented result. - -#### `presented_model_frame_count` - -```python -def presented_model_frame_count(self) -> int -``` - -Return how many model frames have been selected in this generation. - -#### `has_pending_model_frames` - -```python -def has_pending_model_frames(self) -> bool -``` - -Return whether another model frame is ready to present. +::: flashdreams.api_v2.loop.IUILoop ## Session description -### `flashdreams.runtime_v2.session_desc.SessionDesc` - -Description of a session, passed to create one and to open a window on it. - -The runtime fills this in to ask an application for a session, and the -session reports back what it resolved to. The same description then -configures the client window through `OutputSink.open`. - -[View source](https://github.com/NVIDIA/flashdreams/blob/main/flashdreams/flashdreams/runtime_v2/session_desc.py#L35) - -- `output_layout: VideoTensorLayout` - -- `backpressure_mode: BackpressureMode` - -- `presentation_mode: PresentationMode` - -- `frames_per_second_for_ui: int` - -- `frames_per_second_for_step: int` - -- `video_width: int` - -- `video_height: int` - -- `metadata: dict[str, Any]` +::: flashdreams.runtime_v2.session_desc.SessionDesc diff --git a/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md b/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md new file mode 100644 index 000000000..65153fdd9 --- /dev/null +++ b/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md @@ -0,0 +1,102 @@ +--- +title: 'Adding a new configuration' +--- + + + + +After [integrating a model adapter for a demo](integrate_model_adapter_for_a_demo.md), you can +add another preset without writing another app. This example adds a faster +pipeline config to the T2V adapter. + +## Files to update + +```text +integrations_v2// + config.py # define the pipeline variant + apps/t2v/adapter.py # add its application factory + pyproject.toml # register its CLI name + +docs/source/models/.md # add the preset and run command +``` + +## Define the pipeline config + +In `config.py`, derive a variant from the existing config: + +```python +from flashdreams.infra.config import derive_config + +PIPELINE_MY_MODEL_FAST = derive_config( + PIPELINE_MY_MODEL, + name="my-model-fast", + diffusion_model={"scheduler": {"num_inference_steps": 4}}, +) +``` + +Choose a step count your model supports. `derive_config` copies the original +config and applies nested changes; an unknown field raises `KeyError`. +Give the variant its own name and add it to the package's config dictionary: + +```python +MY_MODEL_CONFIGS = { + config.name: config + for config in (PIPELINE_MY_MODEL, PIPELINE_MY_MODEL_FAST) +} +``` + +The dictionary makes configs available to Python callers. Register a separate +application name to make the variant available from the CLI. + +## Add a factory + +In `apps/t2v/adapter.py`, reuse the defaults from the previous guide: + +```python +from dataclasses import replace + +from my_model.config import PIPELINE_MY_MODEL_FAST + + +def create_app_fast(): + return T2VApplication( + defaults=replace( + MY_MODEL_T2V_DEFAULTS, + pipeline_config=PIPELINE_MY_MODEL_FAST, + ) + ) +``` + +Like `create_app`, this factory takes no arguments and returns an uninitialized +application. Model loading happens in the app's `init()`. + +## Register and run the preset + +Add the new name to the existing entry-point table in `pyproject.toml`: + +```toml +[project.entry-points."flashdreams.applications_v2"] +"t2v-my-model" = "my_model.apps.t2v.adapter:create_app" +"t2v-my-model-fast" = "my_model.apps.t2v.adapter:create_app_fast" +``` + +Each CLI name selects a factory, not a key in `MY_MODEL_CONFIGS`. +Reinstall the package to update its entry points: + +```bash +uv sync --package flashdreams- --inexact +uv run --no-sync flashdreams-run-v2 --help +uv run --no-sync flashdreams-run-v2 t2v-my-model-fast --mode native-window +``` + +If a name is missing from `--help`, check that the model package is installed +and the name is registered under `flashdreams.applications_v2`. +For local experiments, the CLI can also import a module with a `create_app()` +function by replacing hyphens in the supplied name with underscores. These +module names are not listed in `--help`; use entry points for published presets. + +## Document the preset + +Add the new name, its differences, and a run command to +`docs/source/models/.md`. Link to the demo page for shared controls and +arguments. Use a unique name for each preset and keep existing names unchanged. diff --git a/docs/source/documentation/demo_api/guides/application_slugs.md b/docs/source/documentation/demo_api/guides/application_slugs.md deleted file mode 100644 index 4c00df5b3..000000000 --- a/docs/source/documentation/demo_api/guides/application_slugs.md +++ /dev/null @@ -1,59 +0,0 @@ ---- -title: 'Application slugs and model adapters' ---- - - - - -An application slug identifies one installed, zero-argument -`flashdreams.api_v2.IApplication` factory. It selects both an interaction and a -model binding; it is not a pipeline-config registry key. - -## Register an application - -Register factories in the model integration's `pyproject.toml`: - -```toml -[project.entry-points."flashdreams.applications_v2"] -"t2v-self-forcing-wan2.1-t2v-1.3b" = "self_forcing.apps.t2v.adapter:create_app" -"t2v-self-forcing-wan2.1-t2v-1.3b-taehv" = "self_forcing.apps.t2v.adapter:create_app_taehv" -``` - -Each target must be callable without arguments and return an uninitialized -`IApplication`. Importing the module or calling the factory must not load a -checkpoint, initialize CUDA, or open a window; `IApplication.init` owns that -work. - -Use a stable, lowercase, hyphenated slug. Existing integrations generally use -`-` and append a suffix for a supported variant. The -entry-point table is the source of truth; slug parsing is not an API. - -## Discover and inspect slugs - -```bash -uv run --no-sync flashdreams-run-v2 --help -uv run --no-sync flashdreams-run-v2 \ - t2v-self-forcing-wan2.1-t2v-1.3b -- --help -``` - -The first command lists registered entry points. The second constructs the -application and asks its argument parser for help. Arguments before `--` belong -to `flashdreams-run-v2`; arguments after it belong to the application. - -For small development-only examples, the registry can also import a module -whose name is the slug with hyphens changed to underscores and call its -`create_app`. This fallback is not listed by `--help`, so real integrations -should register an entry point. - -## Keep ownership one-way - -| Concern | Owner | -| --- | --- | -| Interaction, session, controls, UI, presentation policy | `apps//` | -| Model implementation, checkpoint mapping, pipeline configs | `integrations_v2//` | -| Binding and application factory | `integrations_v2//apps//adapter.py` | -| File, browser, or native-window transport | `flashdreams.runtime_v2` | - -Do not copy a reusable app into the model package or add `runner.py`, -`launch.py`, or `model_session.py` merely to expose a v2 slug. Follow -[Integrate a model with a demo](integrate_model.md) for the adapter workflow. diff --git a/docs/source/documentation/demo_api/guides/configuration.md b/docs/source/documentation/demo_api/guides/configuration.md deleted file mode 100644 index 1b7c2b236..000000000 --- a/docs/source/documentation/demo_api/guides/configuration.md +++ /dev/null @@ -1,104 +0,0 @@ ---- -title: 'Demo configuration' ---- - - - - -FlashDreams keeps model configuration separate from application and runtime -configuration. This separation lets one model serve several demos and lets one -demo run several models without either layer importing the other. - -| Configuration | Owner | Examples | -| --- | --- | --- | -| Pipeline architecture | `integrations_v2//config.py` | checkpoint, scheduler, cache, encoder, decoder | -| Application defaults and arguments | app plus model adapter | prompt, block count, native size, device | -| Runtime and presentation | `flashdreams-run-v2` | output mode, host, port, output path, step limit | - -## Define stable pipeline configs - -Export one named, module-level config for each supported model variant. Derive -small variants from a baseline instead of mutating it: - -```python -from typing import cast - -from flashdreams.infra.config import derive_config -from flashdreams.recipes.wan import WanInferencePipelineConfig - -PIPELINE_MY_MODEL = WanInferencePipelineConfig( - name="my-model", - encoder=..., - diffusion_model=..., - decoder=..., -) - -PIPELINE_MY_MODEL_FAST = cast( - WanInferencePipelineConfig, - derive_config( - PIPELINE_MY_MODEL, - name="my-model-fast", - diffusion_model={ - "scheduler": {"num_inference_steps": 4}, - "transformer": {"compile_network": True}, - }, - ), -) - -MY_MODEL_CONFIGS = { - config.name: config - for config in (PIPELINE_MY_MODEL, PIPELINE_MY_MODEL_FAST) -} -``` - -`derive_config` deep-copies the baseline, applies nested patches, and raises -`KeyError` for an unknown field. Always give a derived public variant its own -`name`; never mutate a shared literal in place. - -## Put components in the correct slots - -A `StreamInferencePipelineConfig` composes three model-side stages: - -- `encoder`: optional `StreamingEncoder` for per-step control, such as a camera - trajectory, HD map, or input video chunk; -- `diffusion_model`: transformer plus scheduler; -- `decoder`: optional `StreamingDecoder` from clean latent to output. - -A one-shot text or image context encoder belongs on the transformer's -`context_encoder`, not in the pipeline's per-step `encoder` slot. See the -[stream inference pipeline guide](../../inferencing_api/guides/stream_inference_pipeline.md) -for the lifecycle and cache boundaries. - -## Apply application overrides to a copy - -An application exposes only overrides it can validate. Parse them in -`IApplication.init`, then derive a private pipeline config before calling -`setup()`: - -```python -pipeline_config = derive_config( - PIPELINE_MY_MODEL, - diffusion_model={"seed": args.seed}, -) -pipeline = pipeline_config.setup().to(args.device).eval() -``` - -Keep output paths, browser settings, and presentation modes out of the model -config. They are runtime concerns. Keep prompts and UI policy out of shared -recipes. They are application concerns. - -## Review and test the boundary - -Before loading weights, verify on CPU that: - -- public config names are unique and stable; -- derived variants leave the baseline unchanged; -- component types and model-defining fields are correct; -- unknown override paths fail; -- checkpoint key transforms handle representative real keys; -- each adapter factory returns an uninitialized application. - -Use `ci_gpu` or `manual` checks for checkpoint downloads, real generation, -parity, and performance. Continue with [Create a model](../../inferencing_api/guides/create_model.md) -for model-package responsibilities or [Integrate a model with a demo](integrate_model.md) -for the application binding. diff --git a/docs/source/documentation/demo_api/guides/create_demo.md b/docs/source/documentation/demo_api/guides/create_demo.md index da7634292..88b5fc384 100644 --- a/docs/source/documentation/demo_api/guides/create_demo.md +++ b/docs/source/documentation/demo_api/guides/create_demo.md @@ -5,94 +5,224 @@ title: 'Create a demo' -A demo is a reusable, model-independent `flashdreams.api_v2` application under -`apps//`. It owns interaction, session lifecycle, input handling, UI, and -presentation policy. It must not import a model integration or own a -checkpoint. +A demo is a model-agnostic application built using the `flashdreams.api_v2` +library. It handles user input, sessions, and display. -First check the [demo gallery](../../../demos/index.md). If T2V, Cam2V, -Action2V, V2V, Interactive Drive, or Crazy Robotaxi already owns the required -interaction, add a [model adapter](integrate_model.md) instead of another app. +Check the [demo gallery](../../../demos/index.md) first. If an existing app +supports the interaction you need, [add a model adapter](integrate_model_adapter_for_a_demo.md) +instead. + +## Overview + +This example builds an app that sends a text prompt from an ImGui UI to a +text-to-video model (`StreamInferencePipeline`). ## Package layout +For a demo named `minimal_t2v`, add the app files and update the website: + ```text -apps// - / - __init__.py - application.py - defaults.py # optional adapter-supplied defaults - session.py - ui.py # only for custom composition or controls - tests/ +apps/minimal_t2v/ pyproject.toml - README.md + minimal_t2v/ + __init__.py # export MinimalT2VApplication + application.py # MinimalT2VApplication(IApplication) + session.py # session, model loop, and model state + ui.py # ImGui text input and async message + +docs/source/ + demos/ + minimal_t2v.md # add demo page + index.md # add demo card + .nav.yml # add sidebar entry + models/ + .md # add quick start and link to demo page ``` -Depend on `flashdreams`, never on `integrations_v2`. Add the `serving` or -`local-window` extra only when the app directly imports that optional stack. -The root workspace already discovers `apps/*` packages. +### `pyproject.toml` -## Implement the lifecycle +```toml +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" -The runtime calls the API in this order: +[project] +name = "flashdreams-minimal-t2v" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["flashdreams[local-window]"] -1. `IApplication.session_desc()` describes native output without loading the - model. -2. `IApplication.init(args)` parses arguments after `--` and initializes state - shared by sessions. -3. `IApplication.create_session(desc)` validates the requested description and - returns an uninitialized `ISession`. -4. `ISession.init()` creates per-run state and registers exactly one - `IModelLoop`; a custom `IUILoop` is optional. -5. `IModelLoop.step(index, events)` returns `list[StepResult]`. -6. `is_finished`, `reset`, and `close` manage the run lifecycle. +[tool.uv.sources] +flashdreams = { workspace = true } -A minimal session registration is: +[tool.setuptools.packages.find] +include = ["minimal_t2v*"] +``` + +### `application.py` + +The adapter supplies a pipeline and its default output description. No custom +command-line arguments are needed. ```python -from flashdreams.api_v2.session import ISession +from flashdreams.api_v2.application import IApplication +from flashdreams.infra.pipeline import StreamInferencePipeline +from flashdreams.runtime_v2.session_desc import SessionDesc + +from .session import MinimalT2VSession + + +class MinimalT2VApplication(IApplication): + def __init__(self, pipeline: StreamInferencePipeline, desc: SessionDesc): + self.pipeline = pipeline + self.desc = desc + + def init(self, commandline_args): + parser = argparse.ArgumentParser( + prog=f"flashdreams-run-v2 {self.defaults.slug} --", + description="Generate video from an image and keyboard/mouse actions.", + ) + parser.add_argument("--default-prompt", type=str, required=True) + args = parser.parse_args(commandline_args) + self.prompt = args.prompt + + def session_desc(self): + return self.desc + + def create_session(self, session_desc): + return MinimalT2VSession(self.pipeline, session_desc, self.prompt) +``` + +### `__init__.py` -class DemoSession(ISession): - def init(self) -> None: - self.register_model_loop(DemoModelLoop, state=DemoModelState(...)) +```python +from .application import MinimalT2VApplication +``` + +### `session.py` + +Keep the pipeline, prompt, and cache in the model-loop state: + +```python +from dataclasses import dataclass + +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.infra.pipeline import ( + StreamInferencePipeline, + StreamInferencePipelineCache, +) +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult + +from .ui import MinimalT2VUILoop, MinimalT2VUIState + + +@dataclass +class TextToVideoState: + pipeline: StreamInferencePipeline + desc: SessionDesc + prompt: str = "" + cache: StreamInferencePipelineCache | None = None + index: int = 0 + + def set_prompt(self, prompt): + self.prompt = prompt + self.cache = None + self.index = 0 + + +class TextToVideoLoop(IModelLoop[TextToVideoState]): + def step(self, step_index, events): + state = self.state + if not state.prompt: + return [] + if state.cache is None: + state.cache = state.pipeline.initialize_cache(text=[state.prompt]) + frames = state.pipeline.generate(state.index, state.cache) + state.pipeline.finalize(state.index, state.cache) + state.index += 1 + return [StepResult( + step_index=step_index, + output=frames, + frame_count=frames.shape[0], + output_layout=state.desc.output_layout, + )] + + +class MinimalT2VSession(ISession): + def __init__(self, pipeline: StreamInferencePipeline, desc: SessionDesc, prompt: str): + self.pipeline = pipeline + self.desc = desc + self.prompt = prompt @property - def session_desc(self) -> SessionDesc: - return self._session_desc + def session_desc(self): + return self.desc + + def init(self): + model_loop = self.register_model_loop( + TextToVideoLoop, + state=TextToVideoState(self.pipeline, self.desc), + ) + self.register_ui_loop( + MinimalT2VUILoop, + state=MinimalT2VUIState(model_loop), + width=self.desc.video_width, + height=self.desc.video_height, + ) + model_loop.state.set_prompt(self.prompt) ``` -If the session does not register a UI loop, the runtime presents model output -with its default blit loop. Model and UI loops run on separate threads and own -their state; use `flashdreams.api_v2.loop.invoke_async` for cross-loop changes. +Use a `SessionDesc` matching the pipeline's output size, frame rate, and layout. +This example expects frames in `tchw` layout. -## Expose a narrow adapter seam +### `ui.py` + +```python +from dataclasses import dataclass + +from flashdreams.api_v2.loop import IModelLoop, invoke_async +from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop + + +@dataclass +class MinimalT2VUIState: + model_loop: IModelLoop + text: str = "" + + +class MinimalT2VUILoop(ImGuiUILoop[MinimalT2VUIState]): + def step_ui(self, imgui, step_index, events): + imgui.begin("Minimal T2V") + try: + _, self.state.text = imgui.input_text("Prompt", self.state.text) + if imgui.button("Send"): + invoke_async( + self.state.model_loop, + lambda state, text=self.state.text: state.set_prompt(text), + ) + finally: + imgui.end() + return self.presented_model_frame() +``` -Put model-independent facts in the app and accept model-owned facts through a -small immutable defaults object or constructor arguments. Typical adapter -inputs are: +`invoke_async` runs the callback on the model thread before its next step. +The UI keeps its own editable text; it never writes model state directly. -- pipeline config or model factory; -- native width, height, frame rate, and tensor layout; -- default rollout length and device; -- model-specific input resolver callbacks. +The model loop waits for a prompt. Each `set_prompt` sets our pipeline cache to `None` to force our generation of `StreamInferencePipeline` to effectively reset (fill the cahce with the new prompt). -Keep expensive shared objects, such as the loaded pipeline, on the application. -Keep caches, counters, and resettable rollout state on the session's model-loop -state. +The [API reference](../api_reference/application.md) here describes our interfaces in more detail. -## Test without a real model +## Connect a model -Tests live in `apps//tests/` and must not import `integrations_v2`. Use a -stand-in pipeline to cover argument validation, session-description checks, -loop registration, step output, reset, finish, close, and scripted input. Mark -pure Python tests `ci_cpu`; put real-model behavior in the integration's tests. +[Add a model adapter](integrate_model_adapter_for_a_demo.md) to supply the pipeline and register +an application slug. The adapter would live in +`integrations_v2//apps/minimal_t2v/adapter.py`. +Run its registered slug with the standard format of: ```bash -uv sync --package flashdreams- --extra dev --no-default-groups --inexact -uv run --no-sync pytest apps//tests -m ci_cpu +uv run flashdreams-run-v2 minimal-t2v- --mode native-window ``` -Add `docs/source/demos/.md`, update `demos/.nav.yml`, and keep the package -README focused on development. The demo becomes publicly runnable when a model -adapter registers an application slug. +Use the [T2V demo](../../../demos/t2v.md) as feature complete examples. diff --git a/docs/source/documentation/demo_api/guides/integrate_model.md b/docs/source/documentation/demo_api/guides/integrate_model.md deleted file mode 100644 index 1c4dc89d4..000000000 --- a/docs/source/documentation/demo_api/guides/integrate_model.md +++ /dev/null @@ -1,100 +0,0 @@ ---- -title: 'Integrate a model with a demo' ---- - - - - -A demo integration is a thin adapter between a model config under -`integrations_v2//` and a reusable application under `apps//`. -The adapter supplies model-specific defaults and registers a zero-argument -`IApplication` factory. It does not duplicate either side. - -## 1. Choose the existing interaction - -Pick the app whose inputs and presentation match the model: T2V, Cam2V, -Action2V, V2V, Interactive Drive, or Crazy Robotaxi. Create a new app only when -none of them owns the required interaction or UI. - -Add that app package to the integration's dependencies. The app remains -model-independent; only the adapter imports both the app and model config. - -## 2. Add the adapter - -```text -integrations_v2//apps// - __init__.py - adapter.py - README.md # installation and launch details only -``` - -A T2V adapter can be this small: - -```python -from flashdreams.api_v2.application import IApplication -from t2v import T2VApplication, T2VApplicationDefaults - -from my_model.config import PIPELINE_MY_MODEL - -MY_MODEL_T2V_DEFAULTS = T2VApplicationDefaults( - pipeline_config=PIPELINE_MY_MODEL, - total_blocks=8, - pixel_width=832, - pixel_height=480, - fps=16, -) - - -def create_app() -> IApplication: - return T2VApplication(defaults=MY_MODEL_T2V_DEFAULTS) -``` - -Subclass the reusable application only when it exposes a hook the model really -needs, such as a different cache initializer or compile override. The factory -must not load weights or initialize CUDA. - -## 3. Register a stable slug - -```toml -[project.entry-points."flashdreams.applications_v2"] -"t2v-my-model" = "my_model.apps.t2v.adapter:create_app" -``` - -Use a readable `-` slug. Register each supported variant -explicitly with its own factory. Do not add `flashdreams.runner_configs`, a -`runner.py`, or a model-specific copy of the app solely for v2 discovery. - -## 4. Test the seam on CPU - -Put adapter tests in `integrations_v2//tests/`, not under the reusable -app. Inject a stand-in config and verify: - -- factory construction performs no model load; -- defaults report the correct size, rate, layout, and rollout length; -- initialize, generate, and finalize run in order; -- argument validation, reset, finish, and close work; -- every entry-point slug resolves to `IApplication`. - -Use the app's shared testing helpers when available. For example, `t2v.testing` -provides a stand-in pipeline and end-to-end adapter checks. - -## 5. Install and inspect - -```bash -uv sync --package flashdreams- --extra dev --inexact -uv run --no-sync pytest integrations_v2//tests -m ci_cpu -uv run --no-sync flashdreams-run-v2 --help -uv run --no-sync flashdreams-run-v2 - -- --help -``` - -A real launch uses the same slug with `--mode mp4`, `webrtc`, or -`native-window`. Runtime options precede `--`; application options follow it. - -## 6. Document once - -Put installation, supported bindings, and canonical commands on the model page. -Link to the demo page for shared controls. Keep the adapter README limited to -package-specific installation and launch details. - -See [Application slugs](application_slugs.md) for discovery details and -[Demo configuration](configuration.md) for ownership of overrides. diff --git a/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md new file mode 100644 index 000000000..7c3b1440f --- /dev/null +++ b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md @@ -0,0 +1,123 @@ +--- +title: 'Integrate model adapter for a demo' +--- + + + + +An adapter connects a model to an existing demo. It supplies the model settings +and registers a name for `flashdreams-run-v2`. + +## Overview + +This example connects a model pipeline config to the T2V app. The app handles +the prompt, sessions, and UI; the adapter selects the model and output defaults. + + + +## Choose a demo + +Pick an app from the [demo gallery](../../../demos/index.md) and check its +inputs. T2V accepts a pipeline config; V2V accepts a processor config. +If you need a new interaction, [create a demo](create_demo.md) first. + +## Files to add or update + +```text +integrations_v2// + apps/t2v/ + __init__.py # add an empty file + adapter.py # add the application factory + pyproject.toml # add the app dependency and CLI name + +docs/source/models/.md # add a quick start and presets +docs/source/demos/t2v.md # link to the model's quick start +``` + +## Add the adapter + +In `apps/t2v/adapter.py`: + +```python +from t2v import T2VApplication, T2VApplicationDefaults + +from my_model.config import PIPELINE_MY_MODEL + +MY_MODEL_T2V_DEFAULTS = T2VApplicationDefaults( + pipeline_config=PIPELINE_MY_MODEL, + total_blocks=8, + pixel_width=832, + pixel_height=480, + fps=16, +) + + +def create_app(): + return T2VApplication(defaults=MY_MODEL_T2V_DEFAULTS) +``` + +Set the dimensions, frame rate, and block count for your model. +The factory returns an uninitialized app. Load weights and initialize CUDA in +`IApplication.init()`, so listing apps and showing help do not load a model. + +For V2V, supply `V2VApplicationDefaults` with a processor, first and subsequent +chunk sizes, and a model name. See the +[SwiftVR adapter](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/swiftvr/apps/v2v/adapter.py) +or [FlashVSR adapter](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/flashvsr/apps/v2v/adapter.py). + +## Register the CLI name + +Add `flashdreams-t2v` to the integration's existing dependencies and workspace +sources in `pyproject.toml`: + +```toml +# Add to [project].dependencies: +# "flashdreams-t2v" + +[tool.uv.sources] +flashdreams-t2v = { workspace = true } + +[project.entry-points."flashdreams.applications_v2"] +"t2v-my-model" = "my_model.apps.t2v.adapter:create_app" +``` + +If the package lists modules explicitly in `[tool.setuptools].packages`, +include `.apps` and `.apps.t2v`. + +The name `t2v-my-model` is the application slug. Use a lowercase name such as +`-`. Each entry points to a factory callable without arguments. +Use `flashdreams.applications_v2` for apps; `flashdreams.postprocess_presets` +registers processors separately. + +## Install and run + +```bash +uv sync --package flashdreams- --inexact +uv run --no-sync flashdreams-run-v2 --help +uv run --no-sync flashdreams-run-v2 t2v-my-model --mode native-window +``` + +`--help` lists installed application names. For this T2V app's arguments, run +`flashdreams-run-v2 t2v-my-model -- --help`. +Runtime options go before `--`; demo options go after it. + + + +## Document the adapter + +Add installation instructions, a run command, and the registered preset to +`docs/source/models/.md`. Link to the demo page for controls and +arguments. Add a link back to the model's quick start on the demo page. +For a new model, add it to `docs/source/models/index.md` and +`docs/source/models/.nav.yml` too. + + + +The app stays in `apps//`. Model code and pipeline configs stay in +`integrations_v2//`; the adapter imports both. Browser and native-window +support come from the runtime. No model-specific copy of the app or `runner.py` +is needed. + +Continue with [Adding a new configuration](adding_a_new_configuration.md) to add another +preset to this adapter, or [Interactive serving](interactive_serving.md) to +run it in a browser. diff --git a/docs/source/documentation/demo_api/guides/interactive_serving.md b/docs/source/documentation/demo_api/guides/interactive_serving.md index dc45006b0..5ca648ff7 100644 --- a/docs/source/documentation/demo_api/guides/interactive_serving.md +++ b/docs/source/documentation/demo_api/guides/interactive_serving.md @@ -5,31 +5,29 @@ title: 'Interactive serving' -`flashdreams-run-v2` can present a compatible application in a browser or a -local native window. Applications generate frames and consume input events; -`flashdreams.runtime_v2` owns transport, pacing, and window lifecycle. +Run a demo in a browser or a local window with `flashdreams-run-v2`. +Use a registered name from `--help` or the [model pages](../../../models/index.md). -## Launch a client - -Serve a browser on the local machine: +## Browser ```bash uv run --no-sync flashdreams-run-v2 APPLICATION_SLUG \ --mode webrtc --host 127.0.0.1 --port 0 ``` -Port `0` selects an available port and prints the URL. Use `0.0.0.0` only when -remote clients must connect; it exposes the server on available network -interfaces, so apply the host's normal firewall and access controls. +Open the URL printed by the server. Port `0` picks an available port. +To accept connections from another machine, use `--host 0.0.0.0` and a fixed +port, then connect to the server's address. -Open a GPU-backed local window instead: +## Local window ```bash -uv run --no-sync flashdreams-run-v2 APPLICATION_SLUG \ - --mode native-window --window-title FlashDreams +uv run --no-sync flashdreams-run-v2 APPLICATION_SLUG --mode native-window ``` -Runtime options precede `--`. Application-specific options follow it: +## Demo arguments + +Runtime options go before `--`; demo options go after it: ```bash uv run --no-sync flashdreams-run-v2 cam2v-lingbot \ @@ -37,43 +35,31 @@ uv run --no-sync flashdreams-run-v2 cam2v-lingbot \ --example-data --total-blocks 21 ``` -Inspect both sides before launching: +List runtime options with `flashdreams-run-v2 --help`. For this demo's options, +use `flashdreams-run-v2 cam2v-lingbot -- --help`. -```bash -uv run --no-sync flashdreams-run-v2 --help -uv run --no-sync flashdreams-run-v2 cam2v-lingbot -- --help -``` - -## Understand the boundary +## Input and display -- `IApplication` owns shared model state and application arguments. -- `ISession` owns one run and registers model and optional UI loops. -- The model loop receives `UserInputEvents` and publishes `StepResult` channels. -- The UI loop composites frames and controls; without one, the runtime uses its - default blit loop. -- The selected client-window mode owns browser, file, or native presentation. +The runtime sends user input to the app's loops and displays their output. +The app handles controls and model inputs. A custom UI loop can draw controls +over the generated frames; otherwise, the runtime displays the frames directly. -The model and UI loops run on separate threads. They must not mutate each -other's state directly; use `flashdreams.api_v2.loop.invoke_async`. Keep camera, -action, and conditioning semantics in the app or adapter rather than teaching -the WebRTC transport about a model. +Model and UI loops run on separate threads. Use `invoke_async` to send changes +between them. [Create a demo](create_demo.md) shows an ImGui text field sending +a prompt to the model loop. -## Distributed applications +## Multiple GPUs -Launch all model ranks through the same command. Rank zero owns the client -window and output sinks; the session's parallel context synchronizes admitted -steps and input across ranks. +For a model adapter that supports distributed execution, launch all ranks: ```bash uv run torchrun --nproc_per_node=2 --no-python flashdreams-run-v2 \ - cam2v-lingbot --mode webrtc -- \ - --example-data --total-blocks 21 + cam2v-lingbot --mode webrtc -- --example-data --total-blocks 21 ``` -The integration remains responsible for its tensor/context parallelism and for -gathering any output required by the presenting rank. +Rank zero handles the window and output. The session's parallel context +synchronizes steps and inputs; the model integration handles parallel inference +and gathers the output for rank zero. -Use [Create a demo](create_demo.md) when the interaction needs new session or UI -behavior. Use [Integrate a model with a demo](integrate_model.md) when the -interaction already exists. The [CLI reference](../../cli.md) documents common -runtime and presentation options. +See the [CLI reference](../../cli.md) for runtime options, or +[Integrate model adapter for a demo](integrate_model_adapter_for_a_demo.md) to connect another model. diff --git a/docs/source/documentation/demo_api/index.md b/docs/source/documentation/demo_api/index.md index 411e0f3d3..34fda4570 100644 --- a/docs/source/documentation/demo_api/index.md +++ b/docs/source/documentation/demo_api/index.md @@ -9,18 +9,15 @@ Use the Demo API to build user-facing applications, manage sessions, expose controls, route inputs and outputs, configure a model-backed demo, and launch it through `flashdreams-run-v2`. -The current application protocol is `flashdreams.api_v2`. The separate -`flashdreams.runtime.demo` API is experimental and provides higher-level -scenario, presentation, replay, warmup, and benchmark utilities. +The current application protocol is `flashdreams.api_v2`. ## Guides
Create a demoBuild a model-independent application against the Demo API. - Integrate a modelConnect a model implementation to an existing demo. - Demo configurationDefine arguments, defaults, and application configuration. - Application slugs and dispatchRegister application names and route model adapters. - Interactive servingServe responsive sessions through supported client windows. + Integrate model adapter for a demoConnect a model to an existing demo. + Adding a new configurationAdd and register another model preset. + Interactive servingRun a demo in a browser or local window. Local benchmarksMeasure application performance in a local environment. Profiling a demoCollect Nsight Systems traces and per-step performance metrics.
diff --git a/docs/source/documentation/inferencing_api/api_reference/core.md b/docs/source/documentation/inferencing_api/api_reference/core.md index d88643b93..6f4f5461d 100644 --- a/docs/source/documentation/inferencing_api/api_reference/core.md +++ b/docs/source/documentation/inferencing_api/api_reference/core.md @@ -3,7 +3,6 @@ title: 'Core API Reference' --- - The `flashdreams.core` package collects the low-level attention primitives, @@ -14,368 +13,18 @@ KV-cache implementations, and process-group utilities that integrations share. The attention package provides the kernels used by the transformer and the block-structured KV cache that backs streaming inference. - -### `NativeAttention` - -```text -class NativeAttention ( * args : Any , ** kwargs : Any ) -``` - -Bases: `Module` - -Native attention module with configurable QKV layout and SDPA backend. - - -#### `set_context_parallel_group` - -```text -set_context_parallel_group ( cp_group : ProcessGroup | None ) → None -``` - -Enable or disable context parallelism for ring attention. - -##### `` - -```text -Parameters : -``` - -**cp\_group** – Process group for context parallel; use None to disable. - -#### `is_context_parallel_enabled` - -```text -is_context_parallel_enabled ( ) → bool -``` - -Return True if context parallelism is active. - -#### `context_parallel_size` - -```text -context_parallel_size ( ) → int -``` - -Return the context parallel world size, or 1 if disabled. - -#### `forward` - -```text -forward ( query : torch.Tensor , key : torch.Tensor , value : torch.Tensor ) → torch.Tensor -``` - -Run context-parallel SDPA (or single-rank SDPA when CP is disabled). - -##### `` - -```text -Parameters : -``` - -* **query** – Query tensor in configured `qkv_format`. -* **key** – Key tensor in configured `qkv_format`. -* **value** – Value tensor in configured `qkv_format`. - - -### `ContextParallelAttention` - -```text -class ContextParallelAttention ( * args : Any , ** kwargs : Any ) -``` - -Bases: `NativeAttention` - -Context-parallel attention with selectable method and SDPA backend. - - -### `BlockKVCache` - -```text -class BlockKVCache ( k_shape : tuple [ int , ... ] , v_shape : tuple [ int , ... ] , seq_dim : int , chunk_size : int , window_size : int , sink_size : int = 0 , device : torch.device | str = torch.device , dtype : torch.dtype = torch.float16 , _prev_chunk_idx : int = -1 , _curr_chunk_idx : int | None = None , _n_cached : int = 0 ) -``` - -Bases: `object` - -KV cache for causal attention with a fixed-size local window, CUDA-graph compatible. - -Keys and values can have arbitrary shape `[..., total_size, ...]`; the sequence -(rolling) dimension is given by `seq_dim` (dimension index, can be negative). -Layout along that dimension: [sink tokens | local window tokens]. Sink tokens are -never evicted; the local window rolls left as new chunks are added if full. Chunks are -non-overlapping: each update adds one chunk of `chunk_size` tokens at the -next logical position in the full sequence. - -#### `` - -```text -Phases : -``` - -* **- Filling** – cache not yet full; tokens are written contiguously; - `cached_k()` / `cached_v()` return only the valid prefix. -* **- Steady-state** – if adding a chunk would exceed the fixed cache size, the - local window rolls left by the overflow amount and the new chunk - overwrites the rightmost positions; `cached_k()` / `cached_v()` - return the full buffer. -The argument `chunk_idx` (0, 1, 2, …) is the index of the new chunk in the full -sequence (not an index into the cache). If `chunk_idx` is greater than -the previous one, the chunk is appended (or, in steady-state, written after -the roll). If `chunk_idx` equals the previous one, the same cache positions -are overwritten. - -#### `` - -```text -Per-step usage: -``` - -1. before\_update(chunk\_idx) — prepare (roll local window if steady-state). -2. update(k, v) — write the new chunk’s keys/values into the cache. -3. cached\_k() / cached\_v() — get cached keys/values for attention. -4. after\_update(chunk\_idx) — update internal bookkeeping. - -#### `k_shape` - -```text -k_shape : tuple [ int , ... ] -``` - -Shape of the keys. Must be the same as the values shape except for the last dimension. - -#### `v_shape` - -```text -v_shape : tuple [ int , ... ] -``` - -Shape of the values. Must be the same as the keys shape except for the last dimension. - -#### `seq_dim` - -```text -seq_dim : int -``` - -Sequence dimension that will be rolled. Can be negative. - -#### `chunk_size` - -```text -chunk_size : int -``` - -Number of tokens processed each time. - -#### `window_size` - -```text -window_size : int -``` - -Size of the local attention window (excluding sink tokens). - -#### `sink_size` - -```text -sink_size : int = 0 -``` - -Number of sink tokens at the start of the cache that are never evicted. Defaults to 0. - -#### `device` - -```text -device : torch.device | str = 'cuda' -``` - -Device to store the cache on. - -#### `dtype` - -```text -dtype : torch.dtype -``` +::: flashdreams.core.attention.NativeAttention -Data type to store the cache in. - -#### `size` +::: flashdreams.core.attention.ContextParallelAttention -```text -property size : int -``` - -Number of valid cached tokens visible to attention. - -#### `write_end` - -```text -property write_end : int -``` - -Right edge of the current chunk in the physical cache layout. - -#### `from_tensor` - -```text -classmethod from_tensor ( k : torch.Tensor , v : torch.Tensor , seq_dim : int ) → Self -``` - -Build a single-chunk cache pre-filled with the given key and value tensors. - -#### `is_steady_state` - -```text -is_steady_state ( ) → bool -``` - -Return True if the cache is full (steady-state phase). - -#### `before_update` - -```text -before_update ( chunk_idx : int ) → None -``` - -Prepare the cache before writing new tokens. - -If `chunk_idx` equals the previous chunk index, this is a no-op. Otherwise, -we expect the `chunk_idx` to be +1 from the previous chunk index. In this case, -we will roll the local window left if the cache is in steady-state, or no op -if the cache is in filling phase. - -##### `` - -```text -Parameters : -``` - -**chunk\_idx** – Chunk index of the new chunk in the full sequence. - -#### `update` - -```text -update ( k : torch.Tensor , v : torch.Tensor ) → None -``` - -Write the new chunk’s keys and values into the cache. - -Must be called after `before_update()` and before `after_update()`. - -##### `` - -```text -Parameters : -``` - -* **k** – Keys; shape must match cached keys except at seq\_dim, where length must be chunk\_size. -* **v** – Values; shape must match cached values except at seq\_dim, where length must be chunk\_size. - -#### `after_update` - -```text -after_update ( chunk_idx : int ) → None -``` - -Finalize bookkeeping after writing new tokens. - -Updates `_prev_chunk_idx` and, in filling phase, `_n_cached`. - -##### `` - -```text -Parameters : -``` - -**chunk\_idx** – The index of the new chunk in the full sequence. - -#### `cached_k` - -```text -cached_k ( ) → torch.Tensor -``` - -Return cached keys for attention (valid prefix in filling phase, full buffer in steady-state). - -#### `cached_v` - -```text -cached_v ( ) → torch.Tensor -``` - -Return cached values for attention (valid prefix in filling phase, full buffer in steady-state). - -#### `reset` - -```text -reset ( ) → None -``` - -Reset bookkeeping while preserving the allocated tensor storage. - -#### `clone_kv` - -```text -clone_kv ( ) → tuple [ torch.Tensor , torch.Tensor ] -``` - -Return clones of the full physical K/V buffers. - -#### `overwrite_kv_` - -```text -overwrite_kv_ ( k : torch.Tensor , v : torch.Tensor ) → None -``` - -Overwrite the full K/V buffers without changing their addresses. - -##### `` - -```text -Parameters : -``` - -* **k** – Replacement keys with the exact cache shape. -* **v** – Replacement values with the exact cache shape. +::: flashdreams.core.attention.BlockKVCache ## Distributed -Helpers for multi-GPU and multi-node inference. `init` boots the NCCL -process group with NVML-derived CPU affinity, a configurable heartbeat timeout, -and a larger L2 fetch granularity. - - -### `init` - -```text -init ( ) → int | None -``` - -Initialize distributed training. - - -### `Device` - -```text -class Device ( device_idx : int ) -``` - -Bases: `object` - -Lightweight wrapper around an NVML device handle for CPU-affinity queries. - - -#### `get_name` - -```text -get_name ( ) → str -``` - -Return the marketing name reported by NVML for this device. - -#### `get_cpu_affinity` +Helpers for multi-GPU / multi-node inference. `init` boots the NCCL process +group with sensible defaults and replaces the boilerplate at the top of +example launchers. -```text -get_cpu_affinity ( ) → list [ int ] -``` +::: flashdreams.core.distributed.init -Return the indices of CPUs ideally affined to this GPU per NVML. +::: flashdreams.core.distributed.Device diff --git a/docs/source/documentation/inferencing_api/api_reference/pipeline.md b/docs/source/documentation/inferencing_api/api_reference/pipeline.md index fb173d69e..78ffdad83 100644 --- a/docs/source/documentation/inferencing_api/api_reference/pipeline.md +++ b/docs/source/documentation/inferencing_api/api_reference/pipeline.md @@ -3,7 +3,6 @@ title: 'Pipeline API Reference' --- - The `flashdreams.infra` package defines the swappable abstractions that @@ -19,59 +18,11 @@ built via `config.setup()`. `PrintableConfig` provides readable configuration dumps, while `derive_config` deep-copies an existing config before applying nested overrides. - -### `PrintableConfig` - -```text -class PrintableConfig -``` - -Bases: `object` - -Config base class providing a multi-line `__str__` for human-readable dumps. - - -### `InstantiateConfig` - -```text -class InstantiateConfig ( _target : type ) -``` - -Bases: `PrintableConfig` - -Config carrying a `_target` class plus its kwargs, instantiable via `setup`. - - -#### `setup` - -```text -setup ( ** kwargs : Any ) → Any -``` - -Instantiate the configured object. - - -### `derive_config` - -```text -derive_config ( base_config : T , ** changes : Any ) → T -``` +::: flashdreams.infra.config.PrintableConfig -Deep-copy a base config and apply nested keyword overrides. +::: flashdreams.infra.config.InstantiateConfig -Nested `dict` values walk into both dataclass attributes and nested -dicts; leaf values overwrite directly. Raises `KeyError` on unknown -paths. - -Example: - -``` -new_config = derive_config( - base_config, - tokenizer=WanVAEInterfaceConfig(checkpoint_path=...), - dit=dict(len_t=3, checkpoint_path=...), -) -``` +::: flashdreams.infra.config.derive_config ## Pipeline @@ -81,1464 +32,80 @@ the diffusion model, and an optional decoder, with per-session caches for the configured streaming components. Call `finalize()` after `generate()` to advance the diffusion model's autoregressive cache. - -### `StreamInferencePipelineConfig` - -```text -class StreamInferencePipelineConfig ( * , _target: type[StreamInferencePipeline] = , name: str , diffusion_model: ~flashdreams.infra.diffusion.model.base.DiffusionModelConfig , decoder: ~flashdreams.infra.decoder.base.DecoderConfig | None = None , encoder: ~flashdreams.infra.encoder.base.EncoderConfig | None = None ) -``` - -Bases: `InstantiateConfig` - -Config for the streaming inference pipeline. - -Set `encoder=None` when the pipeline has no per-AR-step control input -(pure T2V). Set `decoder=None` to return the clean latent directly -(training, latent-space evaluation, or pipelines that own decoding). - - -#### `name` - -```text -name : str -``` - -Stable slug for this pipeline variant; the primary key of -`_CONFIGS`. Runners mirror it as `runner_name` so -`flashdreams-run ` resolves to this pipeline. - -#### `diffusion_model` - -```text -diffusion_model : DiffusionModelConfig -``` - -Transformer + scheduler config. - -#### `decoder` - -```text -decoder : DecoderConfig | None = None -``` - -Optional output `StreamingDecoder` with a per-rollout cache, -called as `decoder(input, autoregressive_index, cache)`. Use -`None` to return the clean latent unchanged. - -#### `encoder` - -```text -encoder : EncoderConfig | None = None -``` - -Optional per-AR-step input encoder. Must be a -`StreamingEncoder`; one-shot encoders go on -`transformer.context_encoder` instead. - -#### `enable_sync_and_profile` - -```text -property enable_sync_and_profile : bool -``` - -Whether to record synchronized per-stage CUDA timings. - -Enabled process-wide with `FLASHDREAMS_SYNC_AND_PROFILE=1` - - -### `StreamInferencePipeline` - -```text -class StreamInferencePipeline ( config : StreamInferencePipelineConfig ) -``` - -Bases: `Module`, `Generic`[`StreamingEncoderCacheT`, `TransformerCacheT`, `StreamingDecoderCacheT`] - -End-to-end streaming inference pipeline. - -Generic over the encoder, transformer, and decoder cache types. The -encoder’s input/output types are forwarded as `Any` so the -transformer’s `predict_flow` / `postprocess_clean_latent` overrides -own the typing on the `input` argument they receive. - -Examples - -cache = pipeline.initialize\_cache(transformer\_context={…}) -output = pipeline.generate(0, cache, input=…) -pipeline.finalize(0, cache) -output = pipeline.generate(1, cache, input=…) -pipeline.finalize(1, cache) # optional for the last rollout - - -#### `initialize_cache` - -```text -initialize_cache ( transformer_context : dict [ str , Any ] | None = None , encoder_context : dict [ str , Any ] | None = None , decoder_context : dict [ str , Any ] | None = None ) → StreamInferencePipelineCache [ StreamingEncoderCacheT , TransformerCacheT , StreamingDecoderCacheT ] -``` - -Build a fresh per-rollout cache. - -Each `*_context` dict is forwarded as keyword arguments to the -corresponding component’s `initialize_autoregressive_cache`. - -##### `` - -```text -Parameters : -``` - -* **transformer\_context** – Per-rollout state for the transformer - (e.g. `{"text_embeddings": ..., "image_embeddings": ...}`). -* **encoder\_context** – Per-rollout state for the encoder. Ignored - when there is no encoder. -* **decoder\_context** – Per-rollout state for the decoder. Ignored - when there is no decoder. - -#### `generate` - -```text -generate ( autoregressive_index : int , cache : StreamInferencePipelineCache [ StreamingEncoderCacheT , TransformerCacheT , StreamingDecoderCacheT ] , input : Any = None ) → Tensor -``` - -Generate one chunk for this AR step. - -##### `` - -```text -Parameters : -``` +::: flashdreams.infra.pipeline.StreamInferencePipelineConfig -* **autoregressive\_index** – Must be `cache.autoregressive_index + 1`, - or `0` for the first call after `initialize_cache`. -* **cache** – Per-rollout cache from `initialize_cache`. -* **input** – Raw input fed to the encoder. Required when an encoder - is configured, must be `None` otherwise. Use - `NullEncoderConfig` to pass an already-encoded tensor - straight through. - -#### `finalize` +::: flashdreams.infra.pipeline.StreamInferencePipeline -```text -finalize ( autoregressive_index : int , cache : StreamInferencePipelineCache [ StreamingEncoderCacheT , TransformerCacheT , StreamingDecoderCacheT ] ) → dict [ str , float ] | None -``` - -Advance the diffusion AR cache for the next AR step. - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – Must match the index passed to the most - recent `generate` (asserted). -* **cache** – Same cache used by `generate`. Consumes - `cache.final_state`. - - -### `StreamInferencePipelineCache` - -```text -class StreamInferencePipelineCache ( * , transformer_cache : TransformerCacheT , encoder_cache : StreamingEncoderCacheT | None = None , decoder_cache : StreamingDecoderCacheT | None = None , final_state : FinalState [ TransformerCacheT ] | None = None , autoregressive_index : int | None = None , event_profiler : EventProfiler | None = None ) -``` - -Bases: `Generic`[`StreamingEncoderCacheT`, `TransformerCacheT`, `StreamingDecoderCacheT`] - -Per-rollout cache held by the pipeline. - - -#### `transformer_cache` - -```text -transformer_cache : TransformerCacheT -``` - -Long-lived transformer AR cache (always present). - -#### `encoder_cache` - -```text -encoder_cache : StreamingEncoderCacheT | None = None -``` - -Encoder AR cache; `None` iff the pipeline has no encoder. - -#### `decoder_cache` - -```text -decoder_cache : StreamingDecoderCacheT | None = None -``` - -Decoder AR cache; `None` iff the pipeline has no decoder. - -#### `final_state` - -```text -final_state : FinalState [ TransformerCacheT ] | None = None -``` - -Diffusion-model state from the most recent `generate`, consumed -by `finalize`. - -#### `autoregressive_index` - -```text -autoregressive_index : int | None = None -``` - -AR step index of the most recent `generate`. - -#### `event_profiler` - -```text -event_profiler : EventProfiler | None = None -``` - -Per-step profiler, populated only when profiling is on. +::: flashdreams.infra.pipeline.StreamInferencePipelineCache ## Diffusion model -Combines a transformer backbone with a denoising scheduler. The denoising -iteration is hidden inside -`~flashdreams.infra.diffusion.model.DiffusionModel.generate`, which -returns the clean latent and the final state needed by `finalize()` to -advance the autoregressive cache. - - -### `DiffusionModelConfig` - -```text -class DiffusionModelConfig ( * , _target: type[DiffusionModel] = , transformer: ~flashdreams.infra.diffusion.transformer.base.TransformerConfig , scheduler: ~flashdreams.infra.diffusion.scheduler.base.SchedulerConfig , seed: int | None = None , context_noise: int = 0 , noise_in_unpatchified_shape: bool = False ) -``` - -Bases: `InstantiateConfig` - -Config for the autoregressive diffusion model. - - -#### `transformer` - -```text -transformer : TransformerConfig -``` - -Flow-prediction network config. - -#### `scheduler` - -```text -scheduler : SchedulerConfig -``` - -Denoising-loop config. - -#### `seed` - -```text -seed : int | None = None -``` - -RNG seed for initial-noise draws and scheduler sampling. -`None` uses the global RNG. - -#### `context_noise` - -```text -context_noise : int = 0 -``` - -Timestep used by `finalize` for the AR cache-update forward. -`0` skips `add_noise`. - -#### `noise_in_unpatchified_shape` - -```text -noise_in_unpatchified_shape : bool = False -``` - -draw the initial noise in the unpatchified shape, then -patchify. Slower than the default patchified path; useful when matching -another implementation’s RNG sequence. - -##### `` - -```text -Type : -``` - -Debug-only - - -### `DiffusionModel` - -```text -class DiffusionModel ( config : DiffusionModelConfig ) -``` - -Bases: `Module`, `Generic`[`TransformerCacheT`] - -Autoregressive diffusion model (scheduler + transformer). - -Generic over the transformer’s AR cache type so user-facing typing on -`cache` is preserved end-to-end. - -Examples - -model = config.setup().to(“cuda”) -cache = model.transformer.initialize\_autoregressive\_cache(…) -clean, final\_state = model.generate(autoregressive\_index=0, cache=cache) -model.finalize(final\_state) - - -#### `FinalState` - -```text -class FinalState ( * , clean_latent : Tensor , autoregressive_index : int , cache : _FinalStateCacheT , input : Any | None = None ) -``` +The diffusion model wraps a transformer backbone with a denoising scheduler. +Callers see only noise to clean latent conversion; the per-step flow prediction +and iteration loop remain inside `DiffusionModel.generate`. -Bases: `Generic`[`_FinalStateCacheT`] +::: flashdreams.infra.diffusion.model.DiffusionModelConfig -State passed from `generate` to `finalize`. - - -##### `clean_latent` - -```text -clean_latent : Tensor -``` - -Patchified clean latent at the end of denoising. - -##### `autoregressive_index` - -```text -autoregressive_index : int -``` - -AR step this state was produced at. - -##### `cache` - -```text -cache : _FinalStateCacheT -``` - -Long-lived AR cache used during generation. - -##### `input` - -```text -input : Any = None -``` - -Patchified per-AR-step encoder output, or `None`. - -#### `rng` - -```text -property rng : Generator | None -``` - -Per-model generator, lazily built on the current device. - -Returns `None` when `config.seed` is `None`. Rebuilt the first -time the model’s device changes after a `.to(...)`. A device move -resets the RNG stream — fine for “construct on CPU, `.to(gpu)` -once” but mid-rollout device hops lose RNG state. - -#### `generate` - -```text -generate ( autoregressive_index : int , cache : TransformerCacheT , input : Any | None = None ) → tuple [ Tensor , DiffusionModel.FinalState [ TransformerCacheT ] ] -``` - -Run the denoising loop for one AR step. - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – AR step index. -* **cache** – Long-lived AR cache, mutated in place. -* **input** – Optional per-AR-step encoder output. Patchified here and - forwarded to `predict_flow` / `postprocess_clean_latent`, - then stashed on the returned `FinalState` for `finalize`. - -#### `finalize` - -```text -finalize ( final_state : FinalState [ TransformerCacheT ] ) → None -``` - -Advance the AR cache using the clean latent from `generate`. - -Re-noises the clean latent to `config.context_noise` and runs the -transformer’s `finalize_kv_cache` (one forward for vanilla -transformers, multiple for dual-network DiTs). - -`context_noise == 0` skips `add_noise` (sigma=0 is identity) and -feeds the clean latent directly. This also dodges the requirement -for schedulers to support a `t=0` lookup (UniPC’s inference -schedule has no `t=0` entry). +::: flashdreams.infra.diffusion.model.DiffusionModel ## Transformer - -### `TransformerConfig` - -```text -class TransformerConfig ( * , _target: type[Transformer] = ) -``` - -Bases: `InstantiateConfig` - -Category base for every flow-prediction transformer config. - - -### `Transformer` - -```text -class Transformer ( config : TransformerConfig ) -``` - -Bases: `Module`, `ABC`, `Generic`[`TransformerCacheT`] - -Flow-prediction transformer, generic over its AR cache subclass. - -Subclasses implement `predict_flow` and the patchify hooks. AR -transformers also subclass `TransformerAutoregressiveCache` and -override `initialize_autoregressive_cache`. - -Example: - -``` -class MyTransformer(Transformer[MyCache]): - def predict_flow(self, noisy_latent, timestep, cache, input=None): - ... - - def initialize_autoregressive_cache(self, **context) -> MyCache: - ... -``` - - -#### `latent_shape` - -```text -abstract property latent_shape : tuple [ int , ... ] -``` - -Shape of the input/output latent tensor for this rank. - -Includes batch dims. May depend on hierarchical context-parallel -group sizes (V/T/HW), so subclasses typically derive this from -`self.cp_groups` rather than from static config alone. - -#### `predict_flow` - -```text -abstract predict_flow ( noisy_latent : Tensor , timestep : Tensor , cache : TransformerCacheT , input : Any | None = None ) → Tensor -``` - -Predict the flow at `timestep`. - -##### `` - -```text -Parameters : -``` - -* **noisy\_latent** – Patchified noisy latent for this denoising step. -* **timestep** – Scalar timestep tensor. -* **cache** – Per-rollout AR cache. -* **input** – Patchified encoder output for this AR step, or `None` - when the pipeline has no encoder. Subclasses should narrow - the type to their encoder’s output type. - -#### `finalize_kv_cache` - -```text -finalize_kv_cache ( noisy_latent : Tensor , timestep : Tensor , cache : TransformerCacheT , input : Any | None = None ) → None -``` - -Advance the AR cache so it is ready for the next AR step. - -Called by `DiffusionModel.finalize` after the denoising loop; the -flow is discarded — only the cache side effect matters. Default -runs a single `predict_flow` forward. Override for transformers -with multiple parallel networks that must stay in lock-step. - -##### `` - -```text -Parameters : -``` - -* **noisy\_latent** – Patchified latent at the AR-step’s context noise - (or the clean latent when `context_noise == 0`). -* **timestep** – 0-d context-noise timestep tensor. -* **cache** – Per-rollout AR cache. -* **input** – Same patchified encoder output passed to `predict_flow`. - -#### `initialize_autoregressive_cache` - -```text -initialize_autoregressive_cache ( ** context : Any ) → TransformerCacheT -``` - -Build a fresh AR cache for a new rollout. +::: flashdreams.infra.diffusion.transformer.TransformerConfig -Default returns an empty cache, correct for non-AR transformers. -Subclasses with custom cache types must override this and may -declare typed per-rollout context (e.g. `text_embeddings`). +::: flashdreams.infra.diffusion.transformer.Transformer -##### `` - -```text -Parameters : -``` - -**context** – Per-rollout state forwarded as keyword arguments. - -#### `initial_noise` - -```text -initial_noise ( * , latent_shape : tuple [ int , ... ] , rng : Generator | None , cache : TransformerCacheT , input : Any | None = None ) → Tensor -``` - -Draw the starting latent for one denoising loop. - -Default is Gaussian noise in `latent_shape`. Model integrations can -override this to seed from encoder outputs, pin I2V frames, or attach -per-step state to the cache while still using `DiffusionModel`. - -##### `` - -```text -Parameters : -``` - -* **latent\_shape** – Shape requested by the diffusion model. -* **rng** – Per-model generator on `self.device`, or `None`. -* **cache** – Per-rollout AR cache. -* **input** – Same patchified encoder output passed to `predict_flow`. - -#### `postprocess_clean_latent` - -```text -postprocess_clean_latent ( clean_latent : Tensor , cache : TransformerCacheT , input : Any | None = None ) → Tensor -``` - -Optional postprocessing hook for the predicted clean latent. - -Default is identity. Override to clamp or re-inject regions whose -clean value is known a priori (e.g. I2V first-frame pinning). -Called at the end of `DiffusionModel.generate`. - -##### `` - -```text -Parameters : -``` - -* **clean\_latent** – Patchified `x0` from the denoising loop. -* **cache** – Per-rollout AR cache. -* **input** – Same patchified encoder output passed to `predict_flow`. - -#### `patchify_and_maybe_split_cp` - -```text -abstract patchify_and_maybe_split_cp ( x : Any ) → Any -``` - -Patchify and (optionally) CP-split a noisy latent or encoder payload. - -Tensors patchify and split. Structured payloads (e.g. an image-control -struct with `latent` + `mask`) patchify each tensor field and -return the same struct type. Implement as identity when neither -token packing nor CP sharding applies. Output preserves the input -Python type — only shapes change. - -#### `unpatchify_and_maybe_gather_cp` - -```text -abstract unpatchify_and_maybe_gather_cp ( x : Tensor ) → Tensor -``` - -Inverse of `patchify_and_maybe_split_cp` for the network output. - - -### `TransformerAutoregressiveCache` - -```text -class TransformerAutoregressiveCache -``` - -Bases: `object` - -Cache that persists across an AR rollout. - -Empty by default; safe to instantiate directly for non-AR transformers -(default `start` / `finalize` are no-ops). Subclass and add fields -plus AR bookkeeping for real per-rollout state. - -Example: - -``` -cache.start(autoregressive_index) -# one or more denoising steps... -cache.finalize(autoregressive_index) -``` - - -#### `start` - -```text -start ( autoregressive_index : int ) → None -``` - -Mark the start of an AR step. Default is a no-op. - -##### `` - -```text -Parameters : -``` - -**autoregressive\_index** – Index of the AR step being started. - -#### `finalize` - -```text -finalize ( autoregressive_index : int ) → None -``` - -Finalize bookkeeping after use at this AR step. Default is a no-op. - -##### `` - -```text -Parameters : -``` - -**autoregressive\_index** – Index of the AR step just finalized. +::: flashdreams.infra.diffusion.transformer.TransformerAutoregressiveCache ## Schedulers -A scheduler owns the entire denoising loop. It is shape-agnostic: every -internal op is a broadcast against per-step scalar sigmas, so the same -scheduler works for any latent layout. - - -### `Scheduler` - -```text -class Scheduler ( config : SchedulerConfig ) -``` - -Bases: `Module`, `ABC` - -Denoising scheduler. - -Owns the entire denoising loop. Callers see only `noise → clean`; -the loop shape (renoise / multistep / plain ODE) is private. - -Concrete configs inherit `SchedulerConfig` and declare their own -`num_inference_steps` / `shift` fields (the base holds no -shared dataclass fields). - -Examples - -scheduler = config.setup() -clean = scheduler.sample(initial\_noise=noise, predict\_flow=predictor) -noisy = scheduler.add\_noise(clean\_input=clean, timestep=t) - - -#### `sample` - -```text -abstract sample ( initial_noise : Tensor , predict_flow : FlowPredictor , rng : Generator | None = None ) → Tensor -``` - -Run the full denoising loop and return the clean latent. +A scheduler owns the denoising loop. Its internal operations broadcast +per-step scalar sigmas, so the same scheduler works for any latent layout. -Schedulers are shape-agnostic: every internal op broadcasts against -per-step scalar sigmas. In practice `initial_noise` is a video -latent `[B, C, T, H, W]`, conventionally treated as a sample at -`sigma=1`. +::: flashdreams.infra.diffusion.scheduler.Scheduler -##### `` +::: flashdreams.infra.diffusion.scheduler.SchedulerConfig -```text -Parameters : -``` +::: flashdreams.infra.diffusion.scheduler.FlowPredictor -* **initial\_noise** – Gaussian noise on the caller’s device/dtype. -* **predict\_flow** – Per-step closure invoked - `num_inference_steps` times. -* **rng** – Generator on the same device. Used by self-forcing renoise - loops; pure ODE solvers ignore it. - -#### `add_noise` +::: flashdreams.infra.diffusion.scheduler.FlowMatchSchedulerConfig -```text -abstract add_noise ( clean_input : Tensor , timestep : Tensor , rng : Generator | None = None ) → Tensor -``` +::: flashdreams.infra.diffusion.scheduler.FlowMatchScheduler -Apply the forward corruption `x_t = (1 - sigma(t)) * x_0 + sigma(t) * eps`. +::: flashdreams.infra.diffusion.scheduler.FlowMatchEulerDiscreteSchedulerConfig -Timestep value semantics are scheduler-specific: all schedulers snap -to the nearest entry of their inference schedule. +::: flashdreams.infra.diffusion.scheduler.FlowMatchUniPCSchedulerConfig - -### `SchedulerConfig` - -```text -class SchedulerConfig ( * , _target: type[Scheduler] = ) -``` - -Bases: `InstantiateConfig` - -Category base for every denoising-scheduler config. - - -### `FlowPredictor` - -```text -class FlowPredictor ( * args , ** kwargs ) -``` - -Bases: `Protocol` - -Closure `(noisy_latent, timestep) -> predicted_flow`. - -Built by `DiffusionModel.generate` by binding the per-AR-step `cache` -/ `input` to the transformer’s `predict_flow`. A scheduler invokes -it once per denoising iteration. The scheduler decides the timestep -dtype (UniPC uses int64, flow-match uses float). - - -### `FlowMatchSchedulerConfig` - -```text -class FlowMatchSchedulerConfig ( * , _target: type[FlowMatchScheduler] = , num_inference_steps: int = 4 , shift: float = 8.0 , denoising_timesteps: list[int] = , warp_denoising_step: bool = True , num_train_timesteps: int = 1000 , sigma_max: float = 1.0 , sigma_min: float = 0.0 , extra_one_step: bool = True , timestep_dtype: ~torch.dtype = torch.float32 , enable_tqdm: bool = False ) -``` - -Bases: `SchedulerConfig` - -Config for the flow-matching scheduler. - - -#### `num_inference_steps` - -```text -num_inference_steps : int = 4 -``` - -Must equal `len(denoising_timesteps)`. - -#### `shift` - -```text -shift : float = 8.0 -``` - -Schedule warp factor. - -#### `denoising_timesteps` - -```text -denoising_timesteps : list [ int ] -``` - -Per-step diffusion timesteps in `[0, num_train_timesteps]`. - -#### `warp_denoising_step` - -```text -warp_denoising_step : bool = True -``` - -Map `denoising_timesteps` through the warped sigma schedule. - -#### `num_train_timesteps` - -```text -num_train_timesteps : int = 1000 -``` - -Length of the training sigma table. - -#### `sigma_max` - -```text -sigma_max : float = 1.0 -``` - -Top of the linspace before warping; `1.0` matches DiffSynth, upstream -Wan / Lingbot ships `0.999`. - -#### `sigma_min` - -```text -sigma_min : float = 0.0 -``` - -Bottom of the linspace before warping. Reserved for upstream parity; -only `0.0` is exercised. - -#### `extra_one_step` - -```text -extra_one_step : bool = True -``` - -If `True`, build the schedule from -`linspace(sigma_max, sigma_min, N+1)[:-1]` (matches DiffSynth / -upstream Wan); `False` uses `N` points and is kept for non-Wan -recipes. - -#### `timestep_dtype` - -```text -timestep_dtype : dtype = torch.float32 -``` - -Dtype of `denoising_step_list`. Set to an integer dtype (e.g. -`torch.int64`) when the network’s time embedding is sensitive to the -fractional part of the warped timestep — upstream Wan stores -`scheduler.timesteps` as `int64` and lets the embedding upcast to -`float64` internally. - -#### `enable_tqdm` - -```text -enable_tqdm : bool = False -``` - -Whether to enable tqdm progress bar. - - -### `FlowMatchScheduler` - -```text -class FlowMatchScheduler ( config : FlowMatchSchedulerConfig ) -``` - -Bases: `Scheduler` - -Flow-matching scheduler with self-forcing renoise (DiffSynth-style). - -Each iteration converts the predicted flow to an `x0` estimate, then -re-noises at the same sigma to feed the next iteration. The final -`x0` is returned: - -``` -x_t = initial_noise -for t in denoising_step_list: - v = predict_flow(x_t, t) - x0 = x_t - sigma(t) * v - x_t = (1 - sigma(t)) * x0 + sigma(t) * eps -return x0 -``` - -Example: - -``` -scheduler = FlowMatchSchedulerConfig( - num_inference_steps=4, - shift=8.0, - denoising_timesteps=[1000, 750, 500, 250], -).setup().to("cuda") -clean = scheduler.sample(initial_noise=noise, predict_flow=fn) -``` - -Schedule buffers are pinned to fp32 even after `module.to(bf16)`; -integer timesteps like 1000 would otherwise round to 1024. - - -#### `sample` - -```text -sample ( initial_noise : Tensor , predict_flow : FlowPredictor , rng : Generator | None = None ) → Tensor -``` - -Run the self-forcing flow-match denoising loop. - -Iteration 0 trusts `initial_noise` as the `sigma=1` sample; -later iterations re-noise the previous `x0` estimate to the new -sigma before the network forward. Schedule arithmetic auto-promotes -to fp32; the result is cast back to `initial_noise.dtype`. - -#### `add_noise` - -```text -add_noise ( clean_input : Tensor , timestep : Tensor , rng : Generator | None = None ) → Tensor -``` - -Apply the forward corruption at an arbitrary timestep. - -Snaps `timestep` to the nearest entry of the warped training table -and uses it as sigma in the standard lerp. - - -### `FlowMatchEulerDiscreteSchedulerConfig` - -```text -class FlowMatchEulerDiscreteSchedulerConfig ( * , _target: type[FlowMatchEulerDiscreteScheduler] = , num_inference_steps: int = 4 , shift: float = 5.0 , num_train_timesteps: int = 1000 , fixed_timesteps: tuple[float , ...] | None = None , enable_tqdm: bool = False ) -``` - -Bases: `SchedulerConfig` - -Config for the flow-matching Euler-discrete scheduler. - -Defaults match diffusers’ `FlowMatchEulerDiscreteScheduler` -with `num_train_timesteps=1000` and the standard -`linspace + shift` warp. Set `fixed_timesteps` to a length -`num_inference_steps + 1` tuple (terminal value typically `0.0`) -to pin an externally-derived distilled schedule. - - -#### `num_inference_steps` - -```text -num_inference_steps : int = 4 -``` - -Number of Euler steps. Matches the distilled WAN-5B 4-step path -out of the box; bump to 30-50 for non-distilled checkpoints. - -#### `shift` - -```text -shift : float = 5.0 -``` - -Schedule warp factor applied to the linspaced sigma grid. Ignored -when `fixed_timesteps` is set. - -#### `num_train_timesteps` - -```text -num_train_timesteps : int = 1000 -``` - -Length of the training sigma table. Used to convert between -integer timesteps and `[0, 1]` sigmas (`sigma = timestep / N`). - -#### `fixed_timesteps` - -```text -fixed_timesteps : tuple [ float , ... ] | None = None -``` - -Override the derived schedule with a precomputed list of -timesteps. When set, must have length `num_inference_steps + 1`; -the trailing entry is typically `0.0` so the last Euler step -lands on the clean latent. `None` (default) derives the schedule -from `num_inference_steps` + `shift` via the standard -flow-matching linspace + warp. - -The HY-WorldPlay distilled WAN-5B path pins this to -`(1000.0, 960.0, 888.8889, 727.2728, 0.0)`. - -#### `enable_tqdm` - -```text -enable_tqdm : bool = False -``` - -Whether to enable the tqdm progress bar inside `sample()`. - - Euler-discrete flow-matching scheduler. - - -### `FlowMatchUniPCSchedulerConfig` - -```text -class FlowMatchUniPCSchedulerConfig ( * , _target: type[FlowMatchUniPCScheduler] = , num_inference_steps: int = 50 , shift: float = 5.0 , num_train_timesteps: int = 1000 , solver_order: int = 2 , use_kerras_sigma: bool = False , enable_tqdm: bool = False ) -``` - -Bases: `SchedulerConfig` - -Config for the flow-matching UniPC scheduler. - -Defaults match the official Wan 2.1 inference integration (UniPC, BH2, -order 2, shift 5.0). Override `shift` per checkpoint as recommended -upstream (e.g. 3.0 for Wan 2.1 14B I2V 480P). - - -#### `num_inference_steps` - -```text -num_inference_steps : int = 50 -``` - -Number of UniPC denoising steps. - -#### `shift` - -```text -shift : float = 5.0 -``` - -Schedule warp factor. - -#### `num_train_timesteps` - -```text -num_train_timesteps : int = 1000 -``` - -Length of the training sigma table. - -#### `solver_order` - -```text -solver_order : int = 2 -``` - -UniPC solver order; only 2 is supported. - -#### `use_kerras_sigma` - -```text -use_kerras_sigma : bool = False -``` - -Whether to use the exact sigma used in edm sampler. - -#### `enable_tqdm` - -```text -enable_tqdm : bool = False -``` - -Whether to enable tqdm progress bar. - - -### `FlowMatchUniPCScheduler` - -```text -class FlowMatchUniPCScheduler ( config : FlowMatchUniPCSchedulerConfig ) -``` - -Bases: `Scheduler` - -Order-2 UniPC predictor-corrector for flow-matching. - -Specialized + pre-baked variant of the upstream Wan 2.1 UniPC solver. -Schedule buffers (sigmas + per-step coefficients) stay fp32 regardless -of `module.to(dtype)`. - -Example: - -``` -scheduler = FlowMatchUniPCSchedulerConfig( - num_inference_steps=50, - shift=5.0, -).setup().to("cuda") -clean = scheduler.sample(initial_noise=noise, predict_flow=fn) -``` - - -#### `sample` - -```text -sample ( initial_noise : Tensor , predict_flow : FlowPredictor , rng : Generator | None = None ) → Tensor -``` - -Run the order-2 UniPC predictor-corrector denoising loop. - -Each iteration: network → flow → `x0` → corrector (skipped at -step 0) → predictor. All per-step coefficients are pre-baked at -construction; the loop is pure tensor ops. Internal arithmetic is -fp32; the result is cast back to `initial_noise.dtype`. `rng` -is unused (deterministic ODE) but accepted for interface conformance. - -#### `add_noise` - -```text -add_noise ( clean_input : Tensor , timestep : Tensor , rng : Generator | None = None ) → Tensor -``` - -Apply the forward corruption at an arbitrary timestep. - -Snaps `timestep` to the nearest entry of the inference schedule -on-device (no Python sync) and uses the matching sigma in the lerp. +::: flashdreams.infra.diffusion.scheduler.FlowMatchUniPCScheduler ## Encoder -Encoders turn raw conditioning (text prompts, reference images, per-AR-step -control inputs, …) into latent tensors. Two flavours: - -- `Encoder` is stateless and one-shot. `forward(self, input)`. - Used as `transformer.context_encoder` for text / CLIP-image / identity. -- `StreamingEncoder` is stateful and per-AR-step. ``forward(self, - input, autoregressive_index, cache)`` with an - `StreamingEncoderCache`. Used as `pipeline.encoder` for - per-step control (HDMap, camera trajectory, I2V first-frame VAE). -- `StreamingVideoEncoder` extends `StreamingEncoder` with - the contracts a streaming pixel-video encoder always needs: spatial / - temporal compression ratios plus AR-step-aware temporal size mappers - between pixel and latent space. - - -### `EncoderConfig` - -```text -class EncoderConfig ( * , _target: type[Encoder] = ) -``` - -Bases: `InstantiateConfig` - -Category base for every encoder config (stateless or streaming). - - -### `Encoder` - -```text -class Encoder ( config : EncoderConfig ) -``` - -Bases: `ABC`, `Module` - -Stateless encoder. - -`forward` is not pinned by the base. Encoders used as a -`context_encoder` (one-shot, called once inside -`Transformer.initialize_autoregressive_cache()`) must match the -slim call shape `forward(self, input)`. - -For per-AR-step encoders that need a per-rollout cache, inherit -from `StreamingEncoder` instead. - - -### `StreamingEncoder` - -```text -class StreamingEncoder ( config : EncoderConfig ) -``` - -Bases: `ABC`, `Module`, `Generic`[`StreamingEncoderCacheT`] - -Streaming encoder, generic over the per-rollout cache type. - -`forward` is not pinned by the base. Streaming encoders called by -`StreamInferencePipeline` must match its call shape: -`forward(self, input, autoregressive_index=0, cache=None)`. - - -#### `initialize_autoregressive_cache` - -```text -abstract initialize_autoregressive_cache ( ** context : Any ) → StreamingEncoderCacheT -``` - -Build a fresh per-rollout cache. - -Override to return the encoder’s concrete cache type. - - -### `StreamingVideoEncoder` - -```text -class StreamingVideoEncoder ( config : EncoderConfig ) -``` - -Bases: `StreamingEncoder`[`StreamingEncoderCacheT`] - -Streaming pixel-video encoder. - -Pins down the contracts that every streaming pixel→latent video -encoder satisfies in addition to `StreamingEncoder`: - -* Spatial and temporal compression ratios between the pixel and - latent grids (constants of the architecture). -* AR-step-aware temporal size mappers, so a pipeline can size its - inputs and outputs without knowing the encoder’s concrete - temporal cache topology (causal first-frame padding, sliding - windows, etc.). - -Spatial scaling is trivially `side // spatial_compression_ratio` -in either direction; the AR-step-asymmetric piece is the temporal -size, which gets its own mapper. Typically AR 0 takes -`1 + (T_lat - 1) * r` pixel frames to produce `T_lat` latent -frames because of causal first-frame padding, while AR ≥ 1 takes -`T_lat * r` pixel frames. - - -#### `spatial_compression_ratio` - -```text -abstract property spatial_compression_ratio : int -``` - -Pixel side ÷ latent side. Constant across AR steps. - -#### `temporal_compression_ratio` - -```text -abstract property temporal_compression_ratio : int -``` - -Pixel frames ÷ latent frames in steady state (AR ≥ 1). - -AR 0 typically takes one extra (un-grouped) pixel frame to -produce its first latent frame because of causal first-frame -padding; that asymmetry lives inside -`get_output_temporal_size()` / -`get_input_temporal_size()`. - -#### `get_output_temporal_size` - -```text -abstract get_output_temporal_size ( autoregressive_index : int , input_temporal_size : int ) → int -``` - -Latent frame count produced from `input_temporal_size` pixel frames. - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – AR step index (0-based). -* **input\_temporal\_size** – Number of pixel frames fed at this step. - -#### `get_input_temporal_size` - -```text -abstract get_input_temporal_size ( autoregressive_index : int , output_temporal_size : int ) → int -``` - -Pixel frame count needed to produce `output_temporal_size` latents. - -Inverse of `get_output_temporal_size()`. Implementations -should assert `output_temporal_size` is achievable at this AR -step (i.e. the corresponding pixel count comes out as a -positive integer). - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – AR step index (0-based). -* **output\_temporal\_size** – Desired number of latent frames. - - -### `StreamingEncoderCache` - -```text -class StreamingEncoderCache -``` - -Bases: `object` +Encoders turn raw conditioning into latent tensors. `Encoder` is stateless and +one-shot, while `StreamingEncoder` carries per-step state. The video variant +adds spatial and temporal size contracts. -Per-rollout cache for `StreamingEncoder`. +::: flashdreams.infra.encoder.EncoderConfig -Empty by default; subclass to add fields (e.g. last-frame latent, -cross-step accumulators). +::: flashdreams.infra.encoder.Encoder - -### `NullEncoderConfig` +::: flashdreams.infra.encoder.StreamingEncoder -```text -class NullEncoderConfig ( * , _target: type[NullEncoder] = ) -``` +::: flashdreams.infra.encoder.StreamingVideoEncoder -Bases: `EncoderConfig` +::: flashdreams.infra.encoder.StreamingEncoderCache -Config for the identity encoder. +::: flashdreams.infra.encoder.NullEncoderConfig - -### `NullEncoder` - -```text -class NullEncoder ( config : EncoderConfig ) -``` - -Bases: `Encoder` - -Identity encoder: returns its input unchanged. - -Wire as the transformer’s `context_encoder` slot to pass already- -encoded tensors straight to the diffusion model. - -Example: - -``` -config = TransformerConfig( - context_encoder=NullEncoderConfig(), - ..., -) -``` - - -#### `forward` - -```text -forward ( input : Any ) → Any -``` - -Return `input` unchanged. +::: flashdreams.infra.encoder.NullEncoder ## Decoder -Decoders turn the latents emitted by the diffusion model back into pixel -frames. Single base class with two specialisations: - -- `StreamingDecoder` is stateful. ``forward(self, input, - autoregressive_index, cache)`` with a `StreamingDecoderCache`. - Use for chunk-by-chunk streaming decoders (e.g. WAN VAE that maintains - a temporal cache across AR steps); stateless decoders just return an - empty `StreamingDecoderCache` from - `StreamingDecoder.initialize_autoregressive_cache` and ignore - `autoregressive_index` / `cache` in `forward`. -- `StreamingVideoDecoder` extends `StreamingDecoder` with - the contracts a streaming pixel-video decoder always needs: spatial / - temporal compression ratios plus AR-step-aware temporal size mappers - between latent and pixel space. - - -### `DecoderConfig` - -```text -class DecoderConfig ( * , _target: type[StreamingDecoder] = ) -``` - -Bases: `InstantiateConfig` - -Category base for every decoder config. - - -### `StreamingDecoder` - -```text -class StreamingDecoder ( config : DecoderConfig ) -``` - -Bases: `ABC`, `Module`, `Generic`[`StreamingDecoderCacheT`] - -Streaming decoder, generic over the per-rollout cache type. - -`forward` is not pinned by the base. Streaming decoders called by -`StreamInferencePipeline` must match its call shape: -`forward(self, input, autoregressive_index=0, cache=None)`. - - -#### `initialize_autoregressive_cache` - -```text -abstract initialize_autoregressive_cache ( ** context : Any ) → StreamingDecoderCacheT -``` - -Build a fresh per-rollout cache. - -Override to return the decoder’s concrete cache type. - - -### `StreamingVideoDecoder` - -```text -class StreamingVideoDecoder ( config : DecoderConfig ) -``` - -Bases: `StreamingDecoder`[`StreamingDecoderCacheT`] - -Streaming pixel-video decoder. - -Pins down the contracts that every streaming latent→pixel video -decoder satisfies in addition to `StreamingDecoder`: - -* Spatial and temporal compression ratios between the latent and - pixel grids (constants of the architecture). -* AR-step-aware temporal size mappers, so a pipeline can size its - inputs and outputs without knowing the decoder’s concrete - temporal cache topology (causal first-frame padding, sliding - windows, etc.). - -Spatial scaling is trivially `side * spatial_compression_ratio` -in either direction; the AR-step-asymmetric piece is the temporal -size, which gets its own mapper. Typically AR 0 produces fewer -pixel frames per latent frame than AR ≥ 1 because of causal -first-frame padding. - - -#### `spatial_compression_ratio` - -```text -abstract property spatial_compression_ratio : int -``` - -Pixel side ÷ latent side. Constant across AR steps. - -#### `temporal_compression_ratio` - -```text -abstract property temporal_compression_ratio : int -``` - -Pixel frames ÷ latent frames in steady state (AR ≥ 1). - -AR 0 typically yields fewer pixel frames per latent frame -because of causal first-frame padding; that asymmetry lives -inside `get_output_temporal_size()` / -`get_input_temporal_size()`. - -#### `get_output_temporal_size` - -```text -abstract get_output_temporal_size ( autoregressive_index : int , input_temporal_size : int ) → int -``` - -Pixel frame count produced by `input_temporal_size` latent frames. - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – AR step index (0-based). -* **input\_temporal\_size** – Number of latent frames fed at this step. - -#### `get_input_temporal_size` - -```text -abstract get_input_temporal_size ( autoregressive_index : int , output_temporal_size : int ) → int -``` - -Latent frame count needed to produce `output_temporal_size` pixels. - -Inverse of `get_output_temporal_size()`. Implementations -should assert `output_temporal_size` is achievable at this AR -step (i.e. divisible by the right ratio after subtracting any -causal padding). - -##### `` - -```text -Parameters : -``` - -* **autoregressive\_index** – AR step index (0-based). -* **output\_temporal\_size** – Desired number of pixel frames. - - -### `StreamingDecoderCache` +Decoders turn emitted latents into frames. `StreamingDecoder` carries +per-rollout state; the video variant adds spatial and temporal size contracts. -```text -class StreamingDecoderCache -``` +::: flashdreams.infra.decoder.DecoderConfig -Bases: `object` +::: flashdreams.infra.decoder.StreamingDecoder -Per-rollout cache for `StreamingDecoder`. +::: flashdreams.infra.decoder.StreamingVideoDecoder -Empty by default; subclass to add fields (e.g. temporal feature -buffers carried across AR steps). +::: flashdreams.infra.decoder.StreamingDecoderCache diff --git a/docs/source/documentation/inferencing_api/api_reference/runtime.md b/docs/source/documentation/inferencing_api/api_reference/runtime.md index b0a6d4466..60d60a74e 100644 --- a/docs/source/documentation/inferencing_api/api_reference/runtime.md +++ b/docs/source/documentation/inferencing_api/api_reference/runtime.md @@ -3,7 +3,6 @@ title: 'Experimental Inference Runtime API Reference' --- - `flashdreams.runtime` is the experimental inference API. Its lifecycle is @@ -18,12 +17,8 @@ and references. Integrations provide the adapter and its model-specific input schema. ```python +from flashdreams.runtime import InferenceConfig, InferenceInput, ModelAdapter -from flashdreams.runtime import ( - InferenceConfig, - InferenceInput, - ModelAdapter, -) def run_one_step(adapter: ModelAdapter, global_conditioning, step): config = InferenceConfig(model_id=adapter.model_id) @@ -39,546 +34,35 @@ def run_one_step(adapter: ModelAdapter, global_conditioning, step): session.close() finally: runtime.close() - ``` Close sessions and runtimes even after errors. Session calls are sequential; the caller owns concurrency unless an integration says otherwise. -## API Reference - -### Runtime lifecycle - - -### `InferenceConfig` - -```text -class InferenceConfig ( * , model_id: str , preset_id: str | None = None , checkpoint: str | ~pathlib.Path | None = None , backend: ~typing.Literal['local' , 'local-distributed' , 'external' , 'hosted'] = 'local' , device: str | None = None , precision: ~typing.Literal['auto' , 'fp32' , 'fp16' , 'bf16'] = 'auto' , compile: bool | None = None , cuda_graph: bool | None = None , attention_backend: str | None = None , cache_policy: str | None = None , seed: int | None = None , runtime_options: ~collections.abc.Mapping[str , ~typing.Any] = , resource_hints: ~collections.abc.Mapping[str , ~typing.Any] = ) -``` - -Bases: `object` - -Runtime settings that affect model execution. - -Prompts, user controls, browser settings, output paths, and benchmark -directories intentionally live outside this object. The typed optimization -fields cover common cross-backend knobs; open-ended adapter-specific choices -can use `runtime_options`. - - -#### `model_id` - -```text -model_id : str -``` - -Stable identity for the model adapter or runtime integration. - -#### `preset_id` - -```text -preset_id : str | None -``` - -Optional preset identity under `model_id`. - -#### `checkpoint` - -```text -checkpoint : str | Path | None -``` - -Optional checkpoint or model-asset selector understood by the adapter. - -#### `backend` - -```text -backend : Literal [ 'local' , 'local-distributed' , 'external' , 'hosted' ] -``` - -Execution placement and backend family for inference compute. - -#### `device` - -```text -device : str | None -``` - -Optional device selector such as `cuda` or `cuda:0`; `None` leaves placement to the adapter/backend. - -#### `precision` - -```text -precision : Literal [ 'auto' , 'fp32' , 'fp16' , 'bf16' ] -``` - -Preferred compute precision. - -#### `compile` - -```text -compile : bool | None -``` - -Optional - Whether model compilation is requested or disabled. None means left to the adapter to decide. - -#### `cuda_graph` - -```text -cuda_graph : bool | None -``` - -Optional - Whether CUDA graph capture is requested or disabled. None means left to the adapter to decide. - -#### `attention_backend` - -```text -attention_backend : str | None -``` - -Optional attention implementation selector; `None` leaves the choice to the adapter. - -#### `cache_policy` - -```text -cache_policy : str | None -``` - -Optional cache policy selector; `None` leaves the choice to the adapter. - -#### `seed` - -```text -seed : int | None -``` - -Optional seed used when resolving deterministic demo/runtime behavior. - -#### `runtime_options` - -```text -runtime_options : Mapping [ str , Any ] -``` - -Adapter/backend-specific runtime options. - -#### `resource_hints` - -```text -resource_hints : Mapping [ str , Any ] -``` - -Resource hints for launchers, schedulers, or hosted backends. - - -### `ModelAdapter` - -```text -class ModelAdapter ( * args , ** kwargs ) -``` - -Bases: `Protocol` - -Model-specific boundary that declares defaults and creates runtimes. - -Adapters declare model-facing input requirements, the canonical modalities -their default mapping consumes, and an optional default mapping between the -two. Runtime, application, or benchmark code may override that mapping while -preserving the same `CanonicalInputs` to `InferenceInput` boundary. - - -#### `model_id` - -```text -property model_id : str -``` - -Stable identity for the model adapter or runtime integration. - -#### `inference_input_schema` - -```text -property inference_input_schema : InferenceInputSchema -``` - -Model-facing global conditioning and per-step input requirements. - -#### `canonical_input_schema` - -```text -property canonical_input_schema : CanonicalInputSchema | None -``` - -Canonical modalities the adapter’s default mapping consumes. - -#### `default_input_mapping` - -```text -default_input_mapping ( ) → InputMapping | None -``` - -Return the model-provided default canonical-to-model mapping. - -#### `validate_config` - -```text -validate_config ( config : InferenceConfig ) → None -``` - -Fail early for unsupported runtime settings. - -#### `create_runtime` - -```text -create_runtime ( config : InferenceConfig ) → InferenceRuntime -``` - -Initialize and return the heavyweight runtime. - - -### `InferenceRuntime` - -```text -class InferenceRuntime ( * args , ** kwargs ) -``` - -Bases: `Protocol` - -Heavyweight reusable runtime created from `InferenceConfig`. - - -#### `start_session` - -```text -start_session ( inputs : InferenceInput ) → InferenceSession -``` - -Create an isolated session from global conditioning inputs. - -#### `close` - -```text -close ( ) → None -``` - -Release model/backend resources. - - -### `InferenceSession` - -```text -class InferenceSession ( * args , ** kwargs ) -``` - -Bases: `Protocol` - -One rollout or stream with isolated model/cache state. - - -#### `next_step_request` - -```text -next_step_request ( ) → StepRequest | None -``` - -Return the next step’s runtime request, or `None` when complete. - -#### `step` - -```text -step ( inputs : InferenceInput ) → StepResult -``` - -Run one sequential inference step. - -#### `reset` - -```text -reset ( inputs : InferenceInput | None = None ) → None -``` - -Reset this session’s rollout state when the backend supports it. - -#### `close` - -```text -close ( ) → None -``` - -Release per-session resources. - -### Inputs and results - - -### `InferenceInput` - -```text -class InferenceInput ( * , global_conditioning: ~collections.abc.Mapping[str , ~typing.Any] = , step: ~collections.abc.Mapping[str , ~typing.Any] = , metadata: ~collections.abc.Mapping[str , ~typing.Any] = ) -``` - -Bases: `object` - -Encoded inputs for one `InferenceSession` call. - -Two conditioning slots: - -* `global_conditioning`: values that condition the whole rollout, such as - the conditioning frame or prompt. Session start/reset establishes this - state; a step call may carry a non-empty payload to request an update when - the model supports it. -* `step`: values needed to generate the next chunk or frame. - - -#### `for_phase` - -```text -for_phase ( phase : Literal [ 'global_conditioning' , 'step' ] ) → Mapping [ str , Any ] -``` - -Return the payload mapping for `phase`. - - -### `InferenceInputSchema` - -```text -class InferenceInputSchema ( * , global_conditioning_fields : tuple [ InputField , ... ] = () , step_fields : tuple [ InputField , ... ] = () , description : str = '' ) -``` - -Bases: `object` - -Minimal metadata for global conditioning and per-step inputs. - - -#### `global_conditioning_fields` - -```text -global_conditioning_fields : tuple [ InputField , ... ] -``` - -Model inputs carried in the global conditioning slot. - -#### `step_fields` - -```text -step_fields : tuple [ InputField , ... ] -``` - -Model inputs required for one session step. - -#### `fields_for` - -```text -fields_for ( phase : Literal [ 'global_conditioning' , 'step' ] ) → tuple [ InputField , ... ] -``` - -Return every declared field for `phase`. - -#### `required_fields` - -```text -required_fields ( phase : Literal [ 'global_conditioning' , 'step' ] | None = None ) → tuple [ tuple [ Literal [ 'global_conditioning' , 'step' ] , InputField ] , ... ] -``` - -Return required fields as `(phase, field)`, optionally filtered. - -#### `optional_fields` - -```text -optional_fields ( phase : Literal [ 'global_conditioning' , 'step' ] | None = None ) → tuple [ tuple [ Literal [ 'global_conditioning' , 'step' ] , InputField ] , ... ] -``` - -Return optional fields as `(phase, field)`, optionally filtered. - -#### `field_for` - -```text -field_for ( * , name : str , phase : Literal [ 'global_conditioning' , 'step' ] ) → InputField | None -``` - -Return one declared field, if present. - -#### `missing_global_conditioning` - -```text -missing_global_conditioning ( inputs : InferenceInput ) → tuple [ str , ... ] -``` - -Return required global conditioning fields absent from `inputs`. - -#### `missing_step` - -```text -missing_step ( inputs : InferenceInput ) → tuple [ str , ... ] -``` - -Return required per-step fields absent from `inputs`. - -#### `require_global_conditioning` - -```text -require_global_conditioning ( inputs : InferenceInput ) → None -``` - -Raise if required global conditioning fields are absent. - -#### `require_step` - -```text -require_step ( inputs : InferenceInput ) → None -``` - -Raise if required per-step fields are absent. - - -### `StepRequest` - -```text -class StepRequest ( * , step_index: int , inference_input_schema: ~flashdreams.runtime.inputs.InferenceInputSchema | None = None , user_input_window: ~flashdreams.infra.time.TimeWindow | None = None , metadata: ~collections.abc.Mapping[str , ~typing.Any] = ) -``` - -Bases: `object` - -Per-step runtime request emitted by an inference session. - -This is not a schema declaration. `user_input_window` lets a runner drain -or slice timestamped user events for the current step before invoking the -selected `InputMapping`. - - -### `StepResult` - -```text -class StepResult ( * , step_index: int , output: ~typing.Any | None = None , frame_count: int = 0 , layout: ~typing.Literal['tchw' , 'btchw' , 'bcthw' , 'bvtchw'] | None = None , output_window: ~flashdreams.infra.time.TimeWindow | None = None , metadata: ~collections.abc.Mapping[str , ~typing.Any] = , metrics: ~collections.abc.Mapping[str , float | int] = ) -``` - -Bases: `object` - -Generated output and metadata returned by one inference step. - -Video results use `from_video_chunk()`, which records a required tensor -layout and derives the frame count once. Non-video results may use the -regular constructor without a layout. - - -#### `from_video_chunk` - -```text -classmethod from_video_chunk ( * , step_index : int , video_chunk : Tensor , layout : Literal [ 'tchw' , 'btchw' , 'bcthw' , 'bvtchw' ] , output_window : TimeWindow | None = None , metadata : Mapping [ str , Any ] | None = None , metrics : Mapping [ str , float | int ] | None = None ) → StepResult -``` - -Build one layout-aware generated-video result. - -#### `video_chunk` - -```text -property video_chunk : Tensor -``` - -Return the video tensor or fail if this is not a video result. - -#### `lazy_rgb_frames` - -```text -lazy_rgb_frames ( * , batch_index : int = 0 , view_index : int = 0 , record_cuda_event : bool = True ) → list [ LazyCudaFrame ] -``` - -Expose this video result as lazy per-frame RGB handles. - -#### `video_hwc_uint8` - -```text -video_hwc_uint8 ( * , batch_index : int = 0 , view_index : int = 0 ) → Tensor -``` - -Return this video result as uint8 `[T,H,W,C]` on its device. +## Runtime lifecycle -### Input mapping +::: flashdreams.runtime.InferenceConfig - -### `CanonicalInputs` +::: flashdreams.runtime.ModelAdapter -```text -class CanonicalInputs ( * , values: ~collections.abc.Mapping[str , ~typing.Any] = , metadata: ~collections.abc.Mapping[str , ~typing.Any] = ) -``` +::: flashdreams.runtime.InferenceRuntime -Bases: `object` +::: flashdreams.runtime.InferenceSession -Canonicalized user input for one step, keyed by modality name. +## Inputs and results -Values are level-triggered and normally present every step: a key held down -emits no events but still means full throttle. Global conditioning does not -appear here; it is application-owned and reaches `InferenceInput` -directly. +::: flashdreams.runtime.InferenceInput - -### `InputMapping` +::: flashdreams.runtime.InferenceInputSchema -```text -class InputMapping ( * args , ** kwargs ) -``` +::: flashdreams.runtime.StepRequest -Bases: `Protocol` +::: flashdreams.runtime.StepResult -Convert user-facing inputs into model-facing inputs. +## Input mapping -A mapping may be supplied by the model adapter as a default or by an -application/runtime override. Step mappings usually receive a timestamped -event window selected by the runner for the current model step or chunk. - - -#### `validate` - -```text -validate ( * , canonical_schema : CanonicalInputSchema | None = None , inference_input_schema : InferenceInputSchema | None = None ) → None -``` - -Fail early for obvious app, event-source, and model mismatches. - -#### `map_global_conditioning_inputs` - -```text -map_global_conditioning_inputs ( * , canonical_inputs : CanonicalInputs , inference_input : InferenceInput ) → InferenceInput -``` - -Build global conditioning inputs for session start or reset. - -#### `map_step_inputs` - -```text -map_step_inputs ( * , canonical_inputs : CanonicalInputs , inference_input : InferenceInput , request : StepRequest ) → InferenceInput -``` +::: flashdreams.runtime.CanonicalInputs -Build model inputs for one session step from the current input window. - - -### `InputMappingSchema` - -```text -class InputMappingSchema ( * , name: str = 'input-mapping' , consumes: tuple[~flashdreams.runtime.inputs.CanonicalModality , ...] = () , produces_global_conditioning: tuple[~flashdreams.runtime.inputs.InputField , ...] = () , produces_step: tuple[~flashdreams.runtime.inputs.InputField , ...] = () , metadata: ~collections.abc.Mapping[str , ~typing.Any] = ) -``` - -Bases: `object` - -Declarative compatibility surface for one mapping. - -`InputMapping.validate` fails a run late and opaquely: it raises, but it -cannot answer which optional model inputs a source would enable, or which -missing user capability is responsible for an unreachable model input. This -schema makes those questions answerable before runtime initialization. - - -#### `produces_for` - -```text -produces_for ( phase : Literal [ 'global_conditioning' , 'step' ] ) → tuple [ InputField , ... ] -``` - -Return the fields this mapping produces for `phase`. - -#### `can_produce` - -```text -can_produce ( phase : Literal [ 'global_conditioning' , 'step' ] , required : InputField ) → bool -``` +::: flashdreams.runtime.InputMapping -Return whether this mapping can produce `required` in `phase`. +::: flashdreams.runtime.InputMappingSchema diff --git a/docs/source/documentation/inferencing_api/guides/create_model.md b/docs/source/documentation/inferencing_api/guides/create_model.md index 59071c9d6..b0f3a7073 100644 --- a/docs/source/documentation/inferencing_api/guides/create_model.md +++ b/docs/source/documentation/inferencing_api/guides/create_model.md @@ -5,50 +5,53 @@ title: 'Create a model' -A model integration is a standalone workspace package under -`integrations_v2//`. It owns model code, checkpoint translation, -pipeline configs, and model-specific tests. It does not copy reusable demo, -UI, or transport behavior. +A model integration is a workspace package in `integrations_v2//`. +It contains model code, checkpoint conversion, pipeline configs, and tests. +Reuse existing apps for demos, UI, and serving. -## 1. Scope before implementing + -Record the facts that determine the integration: +## 1. Check the model requirements + +Check the following before writing code: - closest existing recipe or integration; - checkpoint source, format, dtype, license, and access requirements; - native spatial and temporal shape; - scheduler, denoising steps, guidance, and random-seed behavior; - streaming or one-shot execution, chunk length, window, and sink policy; -- model-specific conditioning, attention, memory, or decoder deltas; +- differences in conditioning, attention, memory use, or decoding; - upstream revision, inputs, and command used for parity. Reuse `flashdreams.core`, `flashdreams.infra`, and `flashdreams.recipes`. If the -model is a variant of an existing backbone, implement only the delta. Keep +model is a variant of an existing backbone, implement only the differences. Keep model-specific branches out of shared framework layers. -## 2. Scaffold the package + + +## 2. Create the package ```text integrations_v2// __init__.py - config.py # public pipeline literals + config.py # pipeline configs impl/ __init__.py - checkpoint.py # only when keys or envelopes need translation + checkpoint.py # checkpoint conversion, if needed ... # model-specific implementation - apps/ # added when binding a reusable demo + apps/ # model adapters for existing apps /adapter.py tests/ pyproject.toml - README.md + +docs/source/models/.md # model documentation ``` Use a distribution name such as `flashdreams-` and depend on -`flashdreams`. Add a reusable app dependency only when adding its adapter. Do -not make `flashdreams` import the integration; dependency direction is -framework to recipe to integration. +`flashdreams`. Add a reusable app dependency only when adding its adapter. The +integration can import `flashdreams`; `flashdreams` must not import the integration. -## 3. Compose a stable pipeline +## 3. Configure the pipeline Start from the closest recipe and replace only the components that differ: @@ -58,11 +61,10 @@ Start from the closest recipe and replace only the components that differ: - scheduler and exact denoising schedule; - `StreamingDecoder`; - checkpoint path and state-dict transform; -- a pipeline subclass only when cache initialization needs a genuinely - different public signature. +- a pipeline subclass only when cache initialization needs different arguments. -Export one module-level config literal per supported variant and a name-keyed -mapping: +Define a config for each supported variant and a dictionary keyed by config +name: ```python from flashdreams.infra.pipeline import StreamInferencePipelineConfig @@ -80,28 +82,28 @@ MY_MODEL_CONFIGS = { } ``` -Use `derive_config` for small variants. Keep model-defining values explicit: +Use `derive_config` for small variants. Set these values explicitly: checkpoint, scheduler, steps, chunk and cache sizes, guidance, precision, compile, CUDA graphs, and attention backend. -## 4. Prove checkpoint compatibility on CPU +## 4. Check checkpoint compatibility on CPU -A checkpoint transform should unwrap only known envelopes, remove known -prefixes, apply ordered key renames, and leave tensor values unchanged. Before +A checkpoint transform should unwrap only known checkpoint containers, remove +known prefixes, apply ordered key renames, and leave tensor values unchanged. Before loading a GPU model, compare the transformed checkpoint with `network.state_dict()` using CPU or meta tensors: - no missing model keys; - no unexpected checkpoint keys; - identical shape for every matching key; -- representative real-key spot checks for every rename family. +- checks using real checkpoint keys for each rename rule. Prefer checkpoint metadata or safetensors headers when full weights are too large. If switching checkpoint sources, compare the transformed tensors from both sources before claiming equivalence. A successful import is not enough; unexplained missing or extra keys are integration failures. -## 5. Test the model boundary +## 5. Test the integration Put model tests in `integrations_v2//tests/`. Every pytest test must have exactly one of `ci_cpu`, `ci_gpu`, or `manual`. @@ -110,27 +112,27 @@ Keep these checks CPU-safe when possible: - package imports and metadata; - config names, component types, and derived-variant independence; -- checkpoint key and shape bijection; +- matching checkpoint keys and tensor shapes; - small-tensor component shapes and cache/reset behavior; -- adapter-free pipeline flow with a stand-in; +- pipeline execution with a fake model; - application adapter flow with the reusable app's testing helpers. Use `ci_gpu` or `manual` for real checkpoints, generation, parity, quality, and performance. Validate at least one step beyond initial cache fill for streaming models, because rolling-window bugs often appear only after the boundary. -## 6. Bind and document after the model works +## 6. Add a demo adapter and documentation A model package can exist without a public application slug. Once its config -and model tests are stable, follow -[Integrate a model with a demo](../../demo_api/guides/integrate_model.md) to add +and model tests pass, follow +[Integrate model adapter for a demo](../../demo_api/guides/integrate_model_adapter_for_a_demo.md) to add `integrations_v2//apps//adapter.py` and a `flashdreams.applications_v2` entry point. Add or update `docs/source/models/.md` with requirements, installation, -supported application slugs, canonical commands, and validated limitations. -Keep implementation and parity details in the package README or tests rather -than duplicating the public model page. +supported application slugs, run commands, known limitations, and +implementation notes useful to contributors. Keep parity test data and results +beside the tests. Do not add a package README. ```bash uv sync --package flashdreams- --extra dev --inexact diff --git a/docs/source/documentation/inferencing_api/guides/stream_inference_pipeline.md b/docs/source/documentation/inferencing_api/guides/stream_inference_pipeline.md index d6c6da442..7ddf9b79a 100644 --- a/docs/source/documentation/inferencing_api/guides/stream_inference_pipeline.md +++ b/docs/source/documentation/inferencing_api/guides/stream_inference_pipeline.md @@ -80,6 +80,14 @@ rejects skipped, repeated, or mismatched indices. When synchronized profiling is enabled, `finalize` returns stage timings and GPU-memory metrics; otherwise it returns `None`. +An application does not always own this loop directly. The V2V application +accepts a `VideoPostProcessorConfig`; each model's processor session owns the +pipeline cache and calls `generate`/`finalize` after converting and buffering +input chunks. FlashVSR builds its pipeline after the input shape is known, +while SwiftVR keeps a resident pipeline and creates an isolated stream cache. +See [Integrate model adapter for a demo](../../demo_api/guides/integrate_model_adapter_for_a_demo.md) +for the paired adapters. + ## Shape and state rules Outside the transformer network, video tensors are normally pre-patchify @@ -128,3 +136,10 @@ for uncontrolled streaming T2V, for per-step camera control, and [Wan 2.1](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/wan21/config.py) for single-step bidirectional generation. + +Pipeline-backed V2V processors include +[FlashVSR](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/flashvsr/impl/postprocess.py) +and +[SwiftVR](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/swiftvr/impl/postprocess.py). +They keep application-level video handling independent from each model's tensor +layout, cache initialization, buffering, and flush behavior. diff --git a/docs/source/documentation/tools/index.md b/docs/source/documentation/tools/index.md index b07b5697a..55fa5cf8d 100644 --- a/docs/source/documentation/tools/index.md +++ b/docs/source/documentation/tools/index.md @@ -5,9 +5,10 @@ title: 'Tooling' -Choose a tool for packaging applications or maintaining the documentation website. +Tools for packaging applications, building documentation, and running tests.
Offline Program PackagerPackage an application for offline use without Python, uv, or network access. Build WebsiteValidate, build, and preview the FlashDreams documentation website. + TestingTest discovery, CI tier markers, and local commands.
diff --git a/docs/source/documentation/tools/testing.md b/docs/source/documentation/tools/testing.md new file mode 100644 index 000000000..4a4e36d6b --- /dev/null +++ b/docs/source/documentation/tools/testing.md @@ -0,0 +1,69 @@ +--- +title: 'Testing' +--- + + + + +Pytest collection is configured in the root `pyproject.toml`. There is +no `testpaths` setting, so `pytest` from the repo root walks the tree: + +- Files named `test_*.py`. Pytest's default also collects `*_test.py`; + don't use that name here. +- Classes named `Test*` (pytest default). Some modules use classes, + some only module-level functions. +- Functions and methods named `test_*` (pytest default), including + `async def test_*`. + +Same test filename in different folders is fine (several +`test_application.py` files exist). Root `pyproject.toml` already sets +`--import-mode=importlib` so pytest keeps them separate. Do not remove +that flag. + +A `test_*.py` next to the code it validates is collected with no CI +change. The root `pyproject.toml` skips `parity_check`, +`parity_check_v2`, `baseline_fastvideo`, and `baseline_lightx2v` via +`norecursedirs`. + +Every test function must be marked with exactly one **CI tier marker**. +A pytest plugin (`flashdreams._pytest_plugins.marker_enforcement`) +enforces this at collection time -- tests without a marker are +rejected, and tests with both `ci_cpu` and `ci_gpu` are rejected. + +| Marker | When to use | CI runner | +|--------|-------------|-----------| +| `@pytest.mark.ci_cpu` | Pure CPU logic, no GPU or `libGL` needed | CPU runner | +| `@pytest.mark.ci_gpu` | Needs CUDA, `libGL`, or transitive `cv2` | GPU runner (RTX Pro 6000) | +| `@pytest.mark.manual` | Heavy (OOM risk), flaky, needs credentials or large downloads | Not run in CI | + +Use a module-level `pytestmark` when every test in a file shares the +same marker: + +```python +import pytest + +pytestmark = pytest.mark.ci_cpu +``` + +Use per-function markers when tests in the same file have different +tiers: + +```python +@pytest.mark.ci_cpu +def test_basic_math(): ... + +@pytest.mark.ci_gpu +def test_cudagraph_path(): ... +``` + +Running tests locally: + +```bash +uv run pytest -m ci_cpu # CPU-safe tests only +uv run pytest -m ci_gpu # GPU tests only (needs CUDA) +uv run pytest -m "not manual" # everything that runs in CI +uv run pytest # all tests including manual +``` + +For local and Docker test-runner scripts, see +[tests/README.md](https://github.com/NVIDIA/flashdreams/blob/main/tests/README.md). diff --git a/docs/source/index.md b/docs/source/index.md index 4ce95aa0f..d895a8839 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -9,6 +9,8 @@ hide: + +

FlashDreams

diff --git a/docs/source/models/index.md b/docs/source/models/index.md index 606214963..55ba7f83d 100644 --- a/docs/source/models/index.md +++ b/docs/source/models/index.md @@ -68,7 +68,7 @@ These models enhance an existing video stream rather than generating a scene fro Follow [Create a model](../documentation/inferencing_api/guides/create_model.md) for the model package, then -[Integrate a model with a demo](../documentation/demo_api/guides/integrate_model.md) +[Integrate model adapter for a demo](../documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md) to register a runnable application. ## Related diff --git a/docs/source/quickstart/index.md b/docs/source/quickstart/index.md index 44e6a28f9..ac1475e67 100644 --- a/docs/source/quickstart/index.md +++ b/docs/source/quickstart/index.md @@ -63,6 +63,19 @@ See our [How to Run a Model](../models/index.md#running-a-model) for more inform - See our [Omnidreams Overview](../models/omnidreams.md) for a brief Omnidreams world-model overview. - See our [Interactive Drive Overview](../demos/interactive_drive.md) for more information on the demo. +## Install one integration + +Install only the model package you need with `--package`. Add `--extra dev` +when working on its code: + +```bash +uv sync --package flashdreams-omnidreams --extra dev --inexact +``` + +Use the distribution name from the integration's `pyproject.toml`. +See [Models](../models/index.md) for available integrations, requirements, and +setup commands. See [Testing](../documentation/tools/testing.md) for test commands. + ## Where to next - [Model Gallery](../models/index.md): Every FlashDreams default-shipped model & how to run models! diff --git a/docs/zensical.toml b/docs/zensical.toml index 8fcb3c864..ebb22a92b 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -49,5 +49,19 @@ pymdownx.arithmatex.generic = true [project.plugins.awesome-nav] +[project.plugins.mkdocstrings.handlers.python] +paths = ["../flashdreams"] + +[project.plugins.mkdocstrings.handlers.python.options] +docstring_style = "google" +members_order = "source" +merge_init_into_class = true +separate_signature = true +show_root_full_path = true +show_root_heading = true +show_signature_annotations = true +show_source = true +signature_crossrefs = true + [project.extra] generator = false diff --git a/flashdreams/flashdreams/infra/__init__.py b/flashdreams/flashdreams/infra/__init__.py new file mode 100644 index 000000000..9ef615b94 --- /dev/null +++ b/flashdreams/flashdreams/infra/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared infrastructure for model integrations.""" diff --git a/flashdreams/flashdreams/infra/diffusion/__init__.py b/flashdreams/flashdreams/infra/diffusion/__init__.py new file mode 100644 index 000000000..592a63868 --- /dev/null +++ b/flashdreams/flashdreams/infra/diffusion/__init__.py @@ -0,0 +1,4 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Diffusion model, transformer, and scheduler interfaces.""" diff --git a/integrations_v2/README.md b/integrations_v2/README.md index fc574ab29..19a7cfd89 100644 --- a/integrations_v2/README.md +++ b/integrations_v2/README.md @@ -6,10 +6,10 @@ SPDX-License-Identifier: Apache-2.0 Applications built on the v2 API. Each directory is a standalone package that depends on `flashdreams` and holds no framework code of its own. -This is the guide to writing one. If the model generates video from a prompt, -read [`apps/t2v`](../apps/t2v/README.md) -instead, that path is a subclass and a `pyproject.toml`, and most of what -follows is already done for you. +For new work, follow the website's +[Create a model](../docs/source/documentation/inferencing_api/guides/create_model.md) +and [Integrate model adapter for a demo](../docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md) +guides. They are the canonical authoring instructions. ## What is here @@ -44,7 +44,6 @@ follows is already done for you. ```text integrations_v2// pyproject.toml - README.md __init__.py config.py # collection of pipeline definitions for a particular `` impl/ # implementation details of a model @@ -53,7 +52,8 @@ integrations_v2// / __init__.py adapter.py # all entry point definitions for `` (ex: `create_app`) - README.md # launch instructions only + +docs/source/models/.md # canonical website article ``` The workspace glob in the root `pyproject.toml` picks up any directory here @@ -280,5 +280,6 @@ halves for text-to-video models. in detail. - [Runtime](../flashdreams/flashdreams/runtime_v2/README.md) - the buffering between those threads, and the command line that starts them. -- [`apps/t2v`](../apps/t2v/README.md) - adding a - text-to-video model. +- [Create a model](../docs/source/documentation/inferencing_api/guides/create_model.md) + and [Integrate model adapter for a demo](../docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md) + cover new model and text-to-video integrations. diff --git a/pyproject.toml b/pyproject.toml index 699b18631..84d92991c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -200,6 +200,7 @@ lint = [ # Documentation site generator. docs = [ + "mkdocstrings-python>=2.0.9", "zensical==0.0.67", ] dev = [ diff --git a/tests/README.md b/tests/README.md index d762841c4..bcc20d235 100644 --- a/tests/README.md +++ b/tests/README.md @@ -25,9 +25,9 @@ Both scripts resolve paths relative to their own location and can be invoked fro ## What gets run -When no `TEST_TARGET` is given, each script runs `pytest` from the repo -root. See `CONTRIBUTING.md` Testing for how files and functions are -named. +When no `TEST_TARGET` is given, each script runs `pytest` from the repo root. +See [Testing](https://nvidia.github.io/flashdreams/main/documentation/tools/testing/) for discovery +and CI tier markers. Pytest is invoked with `-m "not manual"` so any test marked `@pytest.mark.manual` is skipped. diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index 21c946487..af1184d9e 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Check local media and script paths in the documentation sources.""" +"""Check local links and media paths in the documentation sources.""" import re from pathlib import Path @@ -12,6 +12,12 @@ pytestmark = pytest.mark.ci_cpu _DOCS_SOURCE = Path(__file__).parents[1] / "docs" / "source" +_REPO_ROOT = _DOCS_SOURCE.parents[1] +_MARKDOWN_LINK_RE = re.compile(r"\[[^]]+\]\(([^)#]+\.md)#([a-z0-9-]+)\)") +_CUSTOM_ANCHOR_RE = re.compile(r']*\btype="video/' @@ -48,3 +54,90 @@ def test_documentation_motion_media_uses_avif() -> None: legacy.append(f"{page.relative_to(_DOCS_SOURCE)}:{line}: {match.group(0)}") assert not legacy, "Documentation motion media must use AVIF:\n" + "\n".join(legacy) + + +def _anchors(path: Path) -> set[str]: + source = path.read_text(encoding="utf-8") + anchors = set(_CUSTOM_ANCHOR_RE.findall(source)) + for heading in _HEADING_RE.findall(source): + slug = re.sub(r"[^\w -]", "", heading.lower()).strip().replace(" ", "-") + anchors.add(slug) + return anchors + + +def _assert_local_section_link(source: Path, target: str, fragment: str) -> None: + page = (source.parent / target).resolve() + assert page.is_relative_to(_DOCS_SOURCE), ( + f"{source.relative_to(_REPO_ROOT)}: canonical link leaves docs/source: " + f"{target}#{fragment}" + ) + assert page.is_file(), ( + f"{source.relative_to(_REPO_ROOT)}: missing canonical page: {target}" + ) + assert fragment in _anchors(page), ( + f"{source.relative_to(_REPO_ROOT)}: missing section #{fragment} in " + f"{page.relative_to(_REPO_ROOT)}" + ) + + +def test_contributing_stays_on_github() -> None: + source = (_REPO_ROOT / "CONTRIBUTING.md").read_text(encoding="utf-8") + headings = set(_HEADING_RE.findall(source)) + assert { + "Ways to contribute", + "Submitting a pull request", + "Adding documentation", + } <= headings + assert "Adding an Agent Skill" in headings + assert not (_DOCS_SOURCE / "community/contributing.md").exists() + assert _CONTRIBUTING_URL in (_DOCS_SOURCE / "community/index.md").read_text() + + +def test_repository_links_resolve_to_canonical_sections() -> None: + sources = [_REPO_ROOT / relative for relative in _CANONICAL_POINTERS] + for source in sources: + assert len(source.read_text(encoding="utf-8").splitlines()) == 1, ( + f"{source.relative_to(_REPO_ROOT)} must be one canonical link" + ) + sources.append(_REPO_ROOT / "tests/README.md") + sources.extend(sorted((_REPO_ROOT / "skills").glob("*/SKILL.md"))) + + found = 0 + for source in sources: + for target, fragment in _MARKDOWN_LINK_RE.findall( + source.read_text(encoding="utf-8") + ): + if target == _CONTRIBUTING_URL: + assert fragment in _anchors(_REPO_ROOT / "CONTRIBUTING.md") + found += 1 + continue + page = (source.parent / target).resolve() + if page.is_relative_to(_DOCS_SOURCE): + _assert_local_section_link(source, target, fragment) + found += 1 + + assert found, "expected canonical links from repository guidance to docs/source" + + +def test_demo_guide_navigation_matches_pages_and_cards() -> None: + navigation = (_DOCS_SOURCE / ".nav.yml").read_text(encoding="utf-8") + overview = (_DOCS_SOURCE / "documentation/demo_api/index.md").read_text( + encoding="utf-8" + ) + entries = re.findall( + r"^\s+- ([^:]+): (documentation/demo_api/guides/[^\s]+\.md)$", + navigation, + re.MULTILINE, + ) + assert entries, "expected Demo API guides in the sidebar" + for label, target in entries: + page = _DOCS_SOURCE / target + assert page.is_file(), f"Sidebar links to missing guide: {target}" + title = re.search(r"^title: ['\"](.+?)['\"]$", page.read_text(), re.MULTILINE) + assert title and title[1] == label, f"Sidebar title differs from {target}" + route = f"guides/{page.stem}/" + assert f'href="{route}"' in overview, f"Missing guide card: {route}" + + routes = re.findall(r'href="(guides/[^\"]+/)"', overview) + expected = {f"guides/{Path(target).stem}/" for _, target in entries} + assert set(routes) == expected, "Guide cards and sidebar link to different pages" diff --git a/uv.lock b/uv.lock index 6685cb421..256fdb192 100644 --- a/uv.lock +++ b/uv.lock @@ -61,7 +61,10 @@ build-constraints = [{ name = "setuptools", specifier = ">=83.0.0,<84" }] [manifest.dependency-groups] dev = [{ name = "python-dotenv", specifier = ">=1.2.2" }] -docs = [{ name = "zensical", specifier = "==0.0.67" }] +docs = [ + { name = "mkdocstrings-python", specifier = ">=2.0.9" }, + { name = "zensical", specifier = "==0.0.67" }, +] lint = [ { name = "imageio-ffmpeg", specifier = ">=0.5" }, { name = "pre-commit", specifier = ">=4.3.0" }, @@ -1801,6 +1804,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, ] +[[package]] +name = "ghp-import" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d9/29/d40217cbe2f6b1359e00c6c307bb3fc876ba74068cbab3dde77f03ca0dc4/ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343", size = 10943, upload-time = "2022-05-02T15:47:16.11Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, +] + [[package]] name = "google-crc32c" version = "1.8.0" @@ -1826,6 +1841,34 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9c/97/7d75fe37a7a6ed171a2cf17117177e7aab7e6e0d115858741b41e9dd4254/google_crc32c-1.8.0-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f639065ea2042d5c034bf258a9f085eaa7af0cd250667c0635a3118e8f92c69c", size = 28800, upload-time = "2025-12-16T00:40:30.322Z" }, ] +[[package]] +name = "griffelib" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11' and sys_platform == 'win32'", + "python_full_version < '3.11' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/27/af/018c10bc9edd42b6ef6db2e96b09542050d5253f9b195e74bc910b2d13ab/griffelib-2.3.0.tar.gz", hash = "sha256:7b0952caf5bca6afa4bb5ee8c6a2d183fe3f21b62efc5f6c7243cb2b26d2d115", size = 234534, upload-time = "2026-09-04T15:08:17.472Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/63/e876e789525063c840ccfa8857febdabd6523bcef9ce7eb979b9305ea895/griffelib-2.3.0-py3-none-any.whl", hash = "sha256:1b8f9cd525681c26b1d6d574faa1371651e8459ca51d209684f50b8096ae06e0", size = 169423, upload-time = "2026-09-04T15:08:12.956Z" }, +] + +[[package]] +name = "griffelib" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'win32'", +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/27/b55f1a5278918be765fb2fd8b20966bc72bbdd3f789f031937cceea7834a/griffelib-2.3.2.tar.gz", hash = "sha256:df00c7a0dee3d86268d76788997a1859272cb1fb7b865658e043d2c0c3d52e60", size = 234729, upload-time = "2026-10-06T09:54:37.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/e5/0ae74c83c1cab2c14daadf28bd0143eb14bb81503834af987b66badc7b61/griffelib-2.3.2-py3-none-any.whl", hash = "sha256:8e710afededd5607f95bf3c8ccc175ee306e7084459faf27b9f9f44ef2a7a274", size = 169420, upload-time = "2026-10-06T09:54:32.038Z" }, +] + [[package]] name = "grpcio" version = "1.81.1" @@ -2510,6 +2553,130 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/8c/52f0299f1675cdfa1ab39a6028a2e5adf9032ae1118c9895c84b08af162b/mediapy-1.2.6-py3-none-any.whl", hash = "sha256:0a0ea00eb0da83c3c54d588b49c49a41ba456174aa33e530ffe13e17269c9072", size = 27494, upload-time = "2026-02-03T10:29:30.245Z" }, ] +[[package]] +name = "mergedeep" +version = "1.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/41/580bb4006e3ed0361b8151a01d324fb03f420815446c7def45d02f74c270/mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8", size = 4661, upload-time = "2021-02-05T18:55:30.623Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/19/04f9b178c2d8a15b076c8b5140708fa6ffc5601fb6f1e975537072df5b2a/mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307", size = 6354, upload-time = "2021-02-05T18:55:29.583Z" }, +] + +[[package]] +name = "mkdocs" +version = "1.6.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "colorama", marker = "sys_platform == 'win32' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "ghp-import" }, + { name = "jinja2" }, + { name = "markdown", version = "3.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markdown", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markupsafe" }, + { name = "mergedeep" }, + { name = "mkdocs-get-deps" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "pyyaml" }, + { name = "pyyaml-env-tag" }, + { name = "watchdog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bc/c6/bbd4f061bd16b378247f12953ffcb04786a618ce5e904b8c5a01a0309061/mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2", size = 3889159, upload-time = "2024-08-30T12:24:06.899Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/5b/dbc6a8cddc9cfa9c4971d59fb12bb8d42e161b7e7f8cc89e49137c5b279c/mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e", size = 3864451, upload-time = "2024-08-30T12:24:05.054Z" }, +] + +[[package]] +name = "mkdocs-autorefs" +version = "1.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown", version = "3.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markdown", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markupsafe" }, + { name = "mkdocs" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/52/c0/f641843de3f612a6b48253f39244165acff36657a91cc903633d456ae1ac/mkdocs_autorefs-1.4.4.tar.gz", hash = "sha256:d54a284f27a7346b9c38f1f852177940c222da508e66edc816a0fa55fc6da197", size = 56588, upload-time = "2026-02-10T15:23:55.105Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/28/de/a3e710469772c6a89595fc52816da05c1e164b4c866a89e3cb82fb1b67c5/mkdocs_autorefs-1.4.4-py3-none-any.whl", hash = "sha256:834ef5408d827071ad1bc69e0f39704fa34c7fc05bc8e1c72b227dfdc5c76089", size = 25530, upload-time = "2026-02-10T15:23:53.817Z" }, +] + +[[package]] +name = "mkdocs-get-deps" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mergedeep" }, + { name = "platformdirs" }, + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/25/b3cccb187655b9393572bde9b09261d267c3bf2f2cdabe347673be5976a6/mkdocs_get_deps-0.2.2.tar.gz", hash = "sha256:8ee8d5f316cdbbb2834bc1df6e69c08fe769a83e040060de26d3c19fad3599a1", size = 11047, upload-time = "2026-03-10T02:46:33.632Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/29/744136411e785c4b0b744d5413e56555265939ab3a104c6a4b719dad33fd/mkdocs_get_deps-0.2.2-py3-none-any.whl", hash = "sha256:e7878cbeac04860b8b5e0ca31d3abad3df9411a75a32cde82f8e44b6c16ff650", size = 9555, upload-time = "2026-03-10T02:46:32.256Z" }, +] + +[[package]] +name = "mkdocstrings" +version = "1.0.6" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11' and sys_platform == 'win32'", + "python_full_version < '3.11' and sys_platform != 'win32'", +] +dependencies = [ + { name = "jinja2", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markdown", version = "3.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markupsafe", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocs", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocs-autorefs", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "pymdown-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/53/71/f85bdf13355073ae15a7375f09879375a830553552e58c1c4b7e0bbc5c8b/mkdocstrings-1.0.6.tar.gz", hash = "sha256:a0b8c2bdd29a6416c80d717aa369bbf7831946bd9f23c2a66db1b1dbe7693dbd", size = 100649, upload-time = "2026-07-11T19:38:05.732Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/5b/4c1902e8bdd5c4db63284e9d101dece4038d4025d6d88850ffe0a1578980/mkdocstrings-1.0.6-py3-none-any.whl", hash = "sha256:2703708697487d1b6d6d7b412e176fa436edf120c1bf81dc9e126b12d00893c7", size = 35787, upload-time = "2026-07-11T19:38:04.417Z" }, +] + +[[package]] +name = "mkdocstrings" +version = "1.0.7" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12' and sys_platform == 'win32'", + "python_full_version >= '3.12' and sys_platform != 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform != 'win32'", +] +dependencies = [ + { name = "jinja2", marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markdown", version = "3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "markupsafe", marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocs", marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocs-autorefs", marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "pymdown-extensions", marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f5/e8/445b19834bf9559ebbbaa76897e965665b83d2861461eed8d8de4d122057/mkdocstrings-1.0.7.tar.gz", hash = "sha256:0951b147f352118a45a3db85f0f3b660833d34cb5120a2751315b8d08dafd159", size = 103739, upload-time = "2026-10-10T13:50:59.957Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/c3/9062531735ff1fd27f3a79c353c4cf4d822eeffd062fd90b2c553a96f4bb/mkdocstrings-1.0.7-py3-none-any.whl", hash = "sha256:9563828667eeb35256f87a6b61fe857676ac628ec1abbce9aea9839a502c9cb1", size = 41597, upload-time = "2026-10-10T13:50:58.43Z" }, +] + +[[package]] +name = "mkdocstrings-python" +version = "2.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffelib", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "griffelib", version = "2.3.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocs-autorefs" }, + { name = "mkdocstrings", version = "1.0.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "mkdocstrings", version = "1.0.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/11/640067d1ad713ea99c233e53e57a4c300d9885f904a61371d7ef90dea549/mkdocstrings_python-2.0.9.tar.gz", hash = "sha256:ae945637dc0618c6beedbee169f10c0234b77f322c53db3fc9fb4c29b3013038", size = 203006, upload-time = "2026-09-22T13:43:27.712Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/0f/f7f926cfa06317bbdbd3cf0ff82f9e3a91767352eb01519c81499fddf575/mkdocstrings_python-2.0.9-py3-none-any.whl", hash = "sha256:c0233eff3f84d78110df50541918e7ec2bcff8e1614faddea34290a90321a416", size = 105698, upload-time = "2026-09-22T13:43:26.345Z" }, +] + [[package]] name = "ml-dtypes" version = "0.5.4" @@ -3991,6 +4158,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, ] +[[package]] +name = "pyyaml-env-tag" +version = "1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyyaml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/2e/79c822141bfd05a853236b504869ebc6b70159afc570e1d5a20641782eaa/pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff", size = 5737, upload-time = "2025-05-13T15:24:01.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/11/432f32f8097b03e3cd5fe57e88efb685d964e2e5178a48ed61e841f7fdce/pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04", size = 4722, upload-time = "2025-05-13T15:23:59.629Z" }, +] + [[package]] name = "regendoc" version = "0.9.0" @@ -5082,6 +5261,35 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2c/02/3623e6169bed617ed1e2d372f7c69f92ec28d54c4dfc997055c8578ec148/virtualenv-21.5.1-py3-none-any.whl", hash = "sha256:55aa670b67bbfb991b03fda39bd3276d92c419d702376e98c5df1c9989a26783", size = 4558820, upload-time = "2026-06-16T16:23:56.963Z" }, ] +[[package]] +name = "watchdog" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/7d/7f3d619e951c88ed75c6037b246ddcf2d322812ee8ea189be89511721d54/watchdog-6.0.0.tar.gz", hash = "sha256:9ddf7c82fda3ae8e24decda1338ede66e1c99883db93711d8fb941eaa2d8c282", size = 131220, upload-time = "2024-11-01T14:07:13.037Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/56/90994d789c61df619bfc5ce2ecdabd5eeff564e1eb47512bd01b5e019569/watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26", size = 96390, upload-time = "2024-11-01T14:06:24.793Z" }, + { url = "https://files.pythonhosted.org/packages/55/46/9a67ee697342ddf3c6daa97e3a587a56d6c4052f881ed926a849fcf7371c/watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112", size = 88389, upload-time = "2024-11-01T14:06:27.112Z" }, + { url = "https://files.pythonhosted.org/packages/44/65/91b0985747c52064d8701e1075eb96f8c40a79df889e59a399453adfb882/watchdog-6.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c897ac1b55c5a1461e16dae288d22bb2e412ba9807df8397a635d88f671d36c3", size = 89020, upload-time = "2024-11-01T14:06:29.876Z" }, + { url = "https://files.pythonhosted.org/packages/e0/24/d9be5cd6642a6aa68352ded4b4b10fb0d7889cb7f45814fb92cecd35f101/watchdog-6.0.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6eb11feb5a0d452ee41f824e271ca311a09e250441c262ca2fd7ebcf2461a06c", size = 96393, upload-time = "2024-11-01T14:06:31.756Z" }, + { url = "https://files.pythonhosted.org/packages/63/7a/6013b0d8dbc56adca7fdd4f0beed381c59f6752341b12fa0886fa7afc78b/watchdog-6.0.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ef810fbf7b781a5a593894e4f439773830bdecb885e6880d957d5b9382a960d2", size = 88392, upload-time = "2024-11-01T14:06:32.99Z" }, + { url = "https://files.pythonhosted.org/packages/d1/40/b75381494851556de56281e053700e46bff5b37bf4c7267e858640af5a7f/watchdog-6.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:afd0fe1b2270917c5e23c2a65ce50c2a4abb63daafb0d419fde368e272a76b7c", size = 89019, upload-time = "2024-11-01T14:06:34.963Z" }, + { url = "https://files.pythonhosted.org/packages/39/ea/3930d07dafc9e286ed356a679aa02d777c06e9bfd1164fa7c19c288a5483/watchdog-6.0.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:bdd4e6f14b8b18c334febb9c4425a878a2ac20efd1e0b231978e7b150f92a948", size = 96471, upload-time = "2024-11-01T14:06:37.745Z" }, + { url = "https://files.pythonhosted.org/packages/12/87/48361531f70b1f87928b045df868a9fd4e253d9ae087fa4cf3f7113be363/watchdog-6.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c7c15dda13c4eb00d6fb6fc508b3c0ed88b9d5d374056b239c4ad1611125c860", size = 88449, upload-time = "2024-11-01T14:06:39.748Z" }, + { url = "https://files.pythonhosted.org/packages/5b/7e/8f322f5e600812e6f9a31b75d242631068ca8f4ef0582dd3ae6e72daecc8/watchdog-6.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f10cb2d5902447c7d0da897e2c6768bca89174d0c6e1e30abec5421af97a5b0", size = 89054, upload-time = "2024-11-01T14:06:41.009Z" }, + { url = "https://files.pythonhosted.org/packages/30/ad/d17b5d42e28a8b91f8ed01cb949da092827afb9995d4559fd448d0472763/watchdog-6.0.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:c7ac31a19f4545dd92fc25d200694098f42c9a8e391bc00bdd362c5736dbf881", size = 87902, upload-time = "2024-11-01T14:06:53.119Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ca/c3649991d140ff6ab67bfc85ab42b165ead119c9e12211e08089d763ece5/watchdog-6.0.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:9513f27a1a582d9808cf21a07dae516f0fab1cf2d7683a742c498b93eedabb11", size = 88380, upload-time = "2024-11-01T14:06:55.19Z" }, + { url = "https://files.pythonhosted.org/packages/a9/c7/ca4bf3e518cb57a686b2feb4f55a1892fd9a3dd13f470fca14e00f80ea36/watchdog-6.0.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:7607498efa04a3542ae3e05e64da8202e58159aa1fa4acddf7678d34a35d4f13", size = 79079, upload-time = "2024-11-01T14:06:59.472Z" }, + { url = "https://files.pythonhosted.org/packages/5c/51/d46dc9332f9a647593c947b4b88e2381c8dfc0942d15b8edc0310fa4abb1/watchdog-6.0.0-py3-none-manylinux2014_armv7l.whl", hash = "sha256:9041567ee8953024c83343288ccc458fd0a2d811d6a0fd68c4c22609e3490379", size = 79078, upload-time = "2024-11-01T14:07:01.431Z" }, + { url = "https://files.pythonhosted.org/packages/d4/57/04edbf5e169cd318d5f07b4766fee38e825d64b6913ca157ca32d1a42267/watchdog-6.0.0-py3-none-manylinux2014_i686.whl", hash = "sha256:82dc3e3143c7e38ec49d61af98d6558288c415eac98486a5c581726e0737c00e", size = 79076, upload-time = "2024-11-01T14:07:02.568Z" }, + { url = "https://files.pythonhosted.org/packages/ab/cc/da8422b300e13cb187d2203f20b9253e91058aaf7db65b74142013478e66/watchdog-6.0.0-py3-none-manylinux2014_ppc64.whl", hash = "sha256:212ac9b8bf1161dc91bd09c048048a95ca3a4c4f5e5d4a7d1b1a7d5752a7f96f", size = 79077, upload-time = "2024-11-01T14:07:03.893Z" }, + { url = "https://files.pythonhosted.org/packages/2c/3b/b8964e04ae1a025c44ba8e4291f86e97fac443bca31de8bd98d3263d2fcf/watchdog-6.0.0-py3-none-manylinux2014_ppc64le.whl", hash = "sha256:e3df4cbb9a450c6d49318f6d14f4bbc80d763fa587ba46ec86f99f9e6876bb26", size = 79078, upload-time = "2024-11-01T14:07:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/62/ae/a696eb424bedff7407801c257d4b1afda455fe40821a2be430e173660e81/watchdog-6.0.0-py3-none-manylinux2014_s390x.whl", hash = "sha256:2cce7cfc2008eb51feb6aab51251fd79b85d9894e98ba847408f662b3395ca3c", size = 79077, upload-time = "2024-11-01T14:07:06.376Z" }, + { url = "https://files.pythonhosted.org/packages/b5/e8/dbf020b4d98251a9860752a094d09a65e1b436ad181faf929983f697048f/watchdog-6.0.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:20ffe5b202af80ab4266dcd3e91aae72bf2da48c0d33bdb15c66658e685e94e2", size = 79078, upload-time = "2024-11-01T14:07:07.547Z" }, + { url = "https://files.pythonhosted.org/packages/07/f6/d0e5b343768e8bcb4cda79f0f2f55051bf26177ecd5651f84c07567461cf/watchdog-6.0.0-py3-none-win32.whl", hash = "sha256:07df1fdd701c5d4c8e55ef6cf55b8f0120fe1aef7ef39a1c6fc6bc2e606d517a", size = 79065, upload-time = "2024-11-01T14:07:09.525Z" }, + { url = "https://files.pythonhosted.org/packages/db/d9/c495884c6e548fce18a8f40568ff120bc3a4b7b99813081c8ac0c936fa64/watchdog-6.0.0-py3-none-win_amd64.whl", hash = "sha256:cbafb470cf848d93b5d013e2ecb245d4aa1c8fd0504e863ccefa32445359d680", size = 79070, upload-time = "2024-11-01T14:07:10.686Z" }, + { url = "https://files.pythonhosted.org/packages/33/e8/e40370e6d74ddba47f002a32919d91310d6074130fe4e17dabcafc15cbf1/watchdog-6.0.0-py3-none-win_ia64.whl", hash = "sha256:a1914259fa9e1454315171103c6a30961236f508b9b623eae470268bbcc6a22f", size = 79067, upload-time = "2024-11-01T14:07:11.845Z" }, +] + [[package]] name = "wcwidth" version = "0.8.1" From 881050b33882dd38f750ad0ae95534f1a9f32bbc Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 03:36:07 +0000 Subject: [PATCH 02/14] remove garbage pages --- docs/source/.nav.yml | 3 - .../demo_api/guides/local_benchmarks.md | 79 ------------- docs/source/documentation/demo_api/index.md | 1 - .../inferencing_api/api_reference/runtime.md | 68 ------------ .../inferencing_api/guides/accelerated.md | 4 - .../inferencing_api/guides/latency_tuning.md | 105 ------------------ .../documentation/inferencing_api/index.md | 2 - flashdreams/tools/benchmarks/README.md | 3 - 8 files changed, 265 deletions(-) delete mode 100644 docs/source/documentation/demo_api/guides/local_benchmarks.md delete mode 100644 docs/source/documentation/inferencing_api/api_reference/runtime.md delete mode 100644 docs/source/documentation/inferencing_api/guides/latency_tuning.md diff --git a/docs/source/.nav.yml b/docs/source/.nav.yml index 9b5249824..d696b0a6c 100644 --- a/docs/source/.nav.yml +++ b/docs/source/.nav.yml @@ -13,7 +13,6 @@ nav: - Integrate model adapter for a demo: documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md - Adding a new configuration: documentation/demo_api/guides/adding_a_new_configuration.md - Interactive serving: documentation/demo_api/guides/interactive_serving.md - - Local benchmarks: documentation/demo_api/guides/local_benchmarks.md - Profiling a demo: documentation/demo_api/guides/profiling.md - API Reference: - Application API: documentation/demo_api/api_reference/application.md @@ -22,11 +21,9 @@ nav: - Guides: - Create a model: documentation/inferencing_api/guides/create_model.md - Stream inference pipeline: documentation/inferencing_api/guides/stream_inference_pipeline.md - - Latency tuning: documentation/inferencing_api/guides/latency_tuning.md - Profiling a pipeline: documentation/inferencing_api/guides/profiling.md - Accelerated building blocks: documentation/inferencing_api/guides/accelerated.md - API Reference: - - Inference Runtime API: documentation/inferencing_api/api_reference/runtime.md - Pipeline API: documentation/inferencing_api/api_reference/pipeline.md - Core API: documentation/inferencing_api/api_reference/core.md - Tooling: diff --git a/docs/source/documentation/demo_api/guides/local_benchmarks.md b/docs/source/documentation/demo_api/guides/local_benchmarks.md deleted file mode 100644 index 99df93c52..000000000 --- a/docs/source/documentation/demo_api/guides/local_benchmarks.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: 'Local benchmarks' ---- - - - - -`flashdreams-benchmark` runs registered commands, captures their environment, -logs, metrics, and media, and writes an HTML report. It is a manual local -workflow; real-model scenarios require their normal GPU, checkpoint, asset, -and FFmpeg dependencies. - -## Run the maintained v2 suite - -```bash -uv sync --package flashdreams --group cuda13 --extra runners --inexact -uv run --no-sync flashdreams-benchmark --list-scenarios \ - --scenario-file configs/v2_model_benchmarks.json -uv run --no-sync flashdreams-benchmark \ - --scenario-file configs/v2_model_benchmarks.json \ - --scenario t2v-self-forcing-quality-10s \ - --output-dir artifacts/benchmarks/self-forcing -``` - -Use `--dry-run` first when reviewing a new or expensive scenario. Each run -directory contains the rendered command, manifest, environment metadata, -normalized metrics, logs, media, and `report.html`. Keep the complete directory -when comparing a candidate with a baseline. - -## Define a scenario - -Commands are argument arrays, not shell strings. Use `{output_dir}` for files -that belong to the benchmark artifact: - -```json -{ - "schema_version": 1, - "scenarios": [ - { - "id": "my-app-smoke", - "name": "My application smoke", - "command": [ - "flashdreams-run-v2", - "t2v-self-forcing-wan2.1-t2v-1.3b", - "--mode", "mp4", - "--output-path", "{output_dir}/clip.mp4", - "--stats-path", "{output_dir}/stats.json", - "--timeout", "unbound", - "--", - "--no-ui", - "--prompt", "A cat surfing.", - "--total-blocks", "8" - ], - "warmup_steps": 1, - "requires_runtime_stats": true - } - ] -} -``` - -Load it with `--scenario-file` and select it with `--scenario`. Do not put -credentials in scenario files; pass them through the environment expected by -the model. - -## Make comparisons defensible - -- Keep input, seed, resolution, block count, checkpoint, precision, and runtime - flags fixed. -- Record the commit, GPU, driver, CUDA, PyTorch, and compiler-cache state. -- Separate startup/compile time from steady-state measurements. -- Exclude declared warmup steps and report a distribution, not one best sample. -- Compare output quality when a cache, decoder, attention backend, precision, - or compile change can affect numerics. - -Use `--quality-baseline-dir` for the harness's non-gating MP4 comparison. Treat -long moving autoregressive clips as smoke tests rather than strict parity: small -numerical changes can alter later content. The -[latency tuning guide](../../inferencing_api/guides/latency_tuning.md) explains -how to interpret pipeline-stage timings. diff --git a/docs/source/documentation/demo_api/index.md b/docs/source/documentation/demo_api/index.md index 34fda4570..6c7fa5bd2 100644 --- a/docs/source/documentation/demo_api/index.md +++ b/docs/source/documentation/demo_api/index.md @@ -18,7 +18,6 @@ The current application protocol is `flashdreams.api_v2`. Integrate model adapter for a demoConnect a model to an existing demo. Adding a new configurationAdd and register another model preset. Interactive servingRun a demo in a browser or local window. - Local benchmarksMeasure application performance in a local environment. Profiling a demoCollect Nsight Systems traces and per-step performance metrics.
diff --git a/docs/source/documentation/inferencing_api/api_reference/runtime.md b/docs/source/documentation/inferencing_api/api_reference/runtime.md deleted file mode 100644 index 60d60a74e..000000000 --- a/docs/source/documentation/inferencing_api/api_reference/runtime.md +++ /dev/null @@ -1,68 +0,0 @@ ---- -title: 'Experimental Inference Runtime API Reference' ---- - - - - -`flashdreams.runtime` is the experimental inference API. Its lifecycle is -`ModelAdapter` → reusable `InferenceRuntime` → isolated -`InferenceSession`. See the [Demo API overview](../../demo_api/index.md) for -the application layer above it, or the -[Inferencing API overview](../index.md) for the surrounding model-side guides -and references. - -## Minimal direct use - -Integrations provide the adapter and its model-specific input schema. - -```python -from flashdreams.runtime import InferenceConfig, InferenceInput, ModelAdapter - - -def run_one_step(adapter: ModelAdapter, global_conditioning, step): - config = InferenceConfig(model_id=adapter.model_id) - adapter.validate_config(config) - runtime = adapter.create_runtime(config) - try: - initial_input = InferenceInput(global_conditioning=global_conditioning) - session = runtime.start_session(initial_input) - try: - request = session.next_step_request() - return None if request is None else session.step(InferenceInput(step=step)) - finally: - session.close() - finally: - runtime.close() -``` - -Close sessions and runtimes even after errors. Session calls are sequential; -the caller owns concurrency unless an integration says otherwise. - -## Runtime lifecycle - -::: flashdreams.runtime.InferenceConfig - -::: flashdreams.runtime.ModelAdapter - -::: flashdreams.runtime.InferenceRuntime - -::: flashdreams.runtime.InferenceSession - -## Inputs and results - -::: flashdreams.runtime.InferenceInput - -::: flashdreams.runtime.InferenceInputSchema - -::: flashdreams.runtime.StepRequest - -::: flashdreams.runtime.StepResult - -## Input mapping - -::: flashdreams.runtime.CanonicalInputs - -::: flashdreams.runtime.InputMapping - -::: flashdreams.runtime.InputMappingSchema diff --git a/docs/source/documentation/inferencing_api/guides/accelerated.md b/docs/source/documentation/inferencing_api/guides/accelerated.md index af2e501f1..b0db96de7 100644 --- a/docs/source/documentation/inferencing_api/guides/accelerated.md +++ b/docs/source/documentation/inferencing_api/guides/accelerated.md @@ -10,10 +10,6 @@ integrations. It currently covers tensor/linear quantization and streaming multi-head attention. These APIs are implementation tools, not a switch that makes every pipeline faster. -Start with the [latency tuning guide](latency_tuning.md). Use an accelerated -component only after profiling identifies its stage as important, and keep the -existing precision/backend path as the correctness reference. - ## Quantization ### Tensor quantization diff --git a/docs/source/documentation/inferencing_api/guides/latency_tuning.md b/docs/source/documentation/inferencing_api/guides/latency_tuning.md deleted file mode 100644 index 97415c50f..000000000 --- a/docs/source/documentation/inferencing_api/guides/latency_tuning.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -title: 'Latency tuning' ---- - - - - -Tune latency from measurements, not from model size or code shape. Interactive -latency includes model work, decode, transfer, queueing, and presentation; a -faster kernel cannot fix a backed-up presenter, and transport changes cannot -make a slow denoising loop faster. - -## 1. Establish a reproducible baseline - -Fix the model slug, checkpoint, input, seed, resolution, block count, precision, -compile flags, cache policy, and presentation mode. Record the commit, GPU, -driver, CUDA, PyTorch, cuDNN, and compiler-cache state. - -Use `--stats-path` before the application separator. It enables synchronized -pipeline profiling and writes model-step metrics: - -```bash -uv run --package flashdreams-self-forcing flashdreams-run-v2 \ - t2v-self-forcing-wan2.1-t2v-1.3b \ - --mode mp4 \ - --output-path artifacts/latency/baseline.mp4 \ - --stats-path artifacts/latency/baseline.json \ - --timeout unbound -- \ - --no-ui \ - --prompt "A fixed benchmark prompt." \ - --total-blocks 14 \ - --seed 1 -``` - -Profiling synchronizes CUDA and changes throughput. Use it to attribute stages, -not as the headline uninstrumented result. - -## 2. Separate cold and steady state - -The first steps may include checkpoint loading, text/context encoding, -`torch.compile`, kernel autotuning, graph capture, and cache fill. Report at -least: - -- application initialization or preload time; -- first visible chunk latency; -- warmup steps excluded; -- steady-state median and p90 chunk time; -- frames per chunk and effective generated FPS; -- peak and reserved GPU memory. - -Run long enough to reach a full rolling cache. Use fresh processes when -comparing compile, autotune, or persistent-cache behavior. - -## 3. Read the pipeline stages - -The base pipeline reports `encode_ms`, `diffuse_ms`, `decode_ms`, -`finalize_ms`, `total_ms_wo_finalize`, and `total_ms` when those stages are -present. It also reports allocated, reserved, and peak GPU memory. - -| Dominant observation | Investigate first | -| --- | --- | -| `encode_ms` | control preprocessing, VAE encoder, input layout/copies | -| `diffuse_ms` | denoising steps, attention/GEMM backend, compile, CUDA graphs, model cache | -| `decode_ms` | decoder choice, streaming cache, layout conversion, memory format | -| `finalize_ms` | K/V maintenance, synchronization, overlap opportunity | -| Pipeline fast but display late | device-to-host transfer, encoding, queue depth, pacing, network | -| Latency grows over time | unbounded cache/history, allocator growth, presenter backlog | - -Measure transfer and presentation separately when they are not represented by -pipeline metrics. For interactive generation, also report estimated -input-to-visible latency rather than only model FPS. - -## 4. Change one bottleneck at a time - -Prefer the smallest option that addresses the measured hot stage: - -- reduce resolution, denoising steps, or context only when the quality tradeoff - is acceptable; -- bound and preallocate K/V caches before compiler or graph work; -- compile the smallest fixed-shape compute region; -- capture CUDA graphs only with stable shapes, pointers, and stream ordering; -- compare attention backends on the target GPU instead of assuming portability; -- keep the quality decoder as the reference when evaluating a faster decoder; -- tune ordered pacing and bounded queues after generation gets faster. - -Keep optimized paths opt-in until their startup, reset, scene-switch, and -shape-change behavior is validated. Hardware-specific results belong on the -model page, not as universal defaults in this generic guide. - -## 5. Validate performance and quality together - -Use the same latent for decoder comparisons, compare cache changes before and -after the rolling-window boundary, and use controlled short schedules for -attention or compile changes. Preserve the fallback path. - -A defensible result includes the exact command and environment, warmup policy, -median and p90 totals, stage timings, memory, output artifacts, quality method, -and whether the candidate is recommended, useful only as an opt-in, rejected, -or still unvalidated. - -For repeatable command execution and reports, use the -[local benchmark guide](../../demo_api/guides/local_benchmarks.md). For low-level -attention and quantization options, continue with -[Accelerated building blocks](accelerated.md) only after profiling identifies -that stage as the bottleneck. diff --git a/docs/source/documentation/inferencing_api/index.md b/docs/source/documentation/inferencing_api/index.md index 95d065cc3..ea6402fd1 100644 --- a/docs/source/documentation/inferencing_api/index.md +++ b/docs/source/documentation/inferencing_api/index.md @@ -18,7 +18,6 @@ the lower-level pipeline and model building blocks.
Create a modelImplement and register a model integration with FlashDreams. Stream inference pipelineBuild stateful generation with streaming pipeline contracts. - Latency tuningProfile and reduce model, decode, transfer, and presentation latency. Profiling a pipelineMeasure pipeline stages with CUDA event or NVTX profilers. Accelerated building blocksUse optimized attention, quantization, caching, and execution components.
@@ -26,7 +25,6 @@ the lower-level pipeline and model building blocks. ## API Reference
- Inference Runtime APIReference for model loops, sessions, and runtime contracts. Pipeline APIReference for inference pipelines and reusable infrastructure. Core APIReference for model-agnostic core components and utilities.
diff --git a/flashdreams/tools/benchmarks/README.md b/flashdreams/tools/benchmarks/README.md index 983880594..185509360 100644 --- a/flashdreams/tools/benchmarks/README.md +++ b/flashdreams/tools/benchmarks/README.md @@ -13,9 +13,6 @@ so their clips can be put side by side; a camera-to-video model is conditioned on a first frame that its prompt belongs to, so it is compared against runs of itself rather than against the text-to-video clips. -The v1 demo suites that run through `flashdreams-run` are a separate workflow, -in [the local benchmarks guide](../../../docs/source/documentation/demo_api/guides/local_benchmarks.md). - ## Set up ```bash From f315172d5ddbc1d4d255652503e515c8efe4ec4f Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 03:38:59 +0000 Subject: [PATCH 03/14] add benchmark tool to tooling --- docs/source/.nav.yml | 1 + .../source/documentation/tools/benchmarking.md | 6 +++++- docs/source/documentation/tools/index.md | 3 ++- flashdreams/flashdreams/runtime_v2/README.md | 2 +- 4 files changed, 9 insertions(+), 3 deletions(-) rename flashdreams/tools/benchmarks/README.md => docs/source/documentation/tools/benchmarking.md (97%) diff --git a/docs/source/.nav.yml b/docs/source/.nav.yml index d696b0a6c..2ca03e555 100644 --- a/docs/source/.nav.yml +++ b/docs/source/.nav.yml @@ -31,5 +31,6 @@ nav: - Offline Program Packager: documentation/tools/offline_program_packager.md - Build Website: documentation/tools/build_website.md - Testing: documentation/tools/testing.md + - Benchmarking: documentation/tools/benchmarking.md - Troubleshooting: troubleshooting.md - Community: community diff --git a/flashdreams/tools/benchmarks/README.md b/docs/source/documentation/tools/benchmarking.md similarity index 97% rename from flashdreams/tools/benchmarks/README.md rename to docs/source/documentation/tools/benchmarking.md index 185509360..14d4b8422 100644 --- a/flashdreams/tools/benchmarks/README.md +++ b/docs/source/documentation/tools/benchmarking.md @@ -1,3 +1,7 @@ +--- +title: 'Benchmarking' +--- + -Tools for packaging applications, building documentation, and running tests. +Tools for packaging applications, building documentation, running tests, and benchmarking models.
Offline Program PackagerPackage an application for offline use without Python, uv, or network access. Build WebsiteValidate, build, and preview the FlashDreams documentation website. TestingTest discovery, CI tier markers, and local commands. + BenchmarkingGenerate model benchmarks, score clips, and compare runs against a baseline.
diff --git a/flashdreams/flashdreams/runtime_v2/README.md b/flashdreams/flashdreams/runtime_v2/README.md index 064974c7c..6f2cd1398 100644 --- a/flashdreams/flashdreams/runtime_v2/README.md +++ b/flashdreams/flashdreams/runtime_v2/README.md @@ -218,7 +218,7 @@ then the new session receives zero remaining time and stops at its first boundar results as they are published, not the UI loop's output, so a benchmark measures what the model generated while the window still sees one composited frame per tick. See [`configs/v2_model_benchmarks.json`](../../../configs/v2_model_benchmarks.json) -and the [benchmark README](../../tools/benchmarks/README.md) for the suite. +and the [benchmarking guide](https://nvidia.github.io/flashdreams/main/documentation/tools/benchmarking/) for the suite. The runtime augments those model records with step wall time and presentation-queue depth/publish-wait measurements under the reserved `runtime_` metric prefix. UI and window timings are not folded into a later From 38202e7f89dc2002c08c9f64058908e536a3ad00 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 16:57:04 +0000 Subject: [PATCH 04/14] accelerated api clean up --- docs/source/.nav.yml | 1 + .../api_reference/accelerated.md | 81 +++++++++++++++++++ .../inferencing_api/guides/accelerated.md | 3 + .../documentation/inferencing_api/index.md | 9 +-- .../flashdreams/accelerated/__init__.py | 1 + .../accelerated/quantization/__init__.py | 1 + 6 files changed, 90 insertions(+), 6 deletions(-) create mode 100644 docs/source/documentation/inferencing_api/api_reference/accelerated.md create mode 100644 flashdreams/flashdreams/accelerated/__init__.py create mode 100644 flashdreams/flashdreams/accelerated/quantization/__init__.py diff --git a/docs/source/.nav.yml b/docs/source/.nav.yml index 2ca03e555..d917ebe6b 100644 --- a/docs/source/.nav.yml +++ b/docs/source/.nav.yml @@ -26,6 +26,7 @@ nav: - API Reference: - Pipeline API: documentation/inferencing_api/api_reference/pipeline.md - Core API: documentation/inferencing_api/api_reference/core.md + - Accelerated API: documentation/inferencing_api/api_reference/accelerated.md - Tooling: - Tooling Overview: documentation/tools/index.md - Offline Program Packager: documentation/tools/offline_program_packager.md diff --git a/docs/source/documentation/inferencing_api/api_reference/accelerated.md b/docs/source/documentation/inferencing_api/api_reference/accelerated.md new file mode 100644 index 000000000..952797752 --- /dev/null +++ b/docs/source/documentation/inferencing_api/api_reference/accelerated.md @@ -0,0 +1,81 @@ +--- +title: 'Accelerated API Reference' +--- + +The `flashdreams.accelerated` package provides optimized attention, quantization, +and shared inference components. See [Accelerated building blocks](../guides/accelerated.md) +for usage and integration guidance. + +## Common components + +::: flashdreams.accelerated.common.NonPersistentLinear + +::: flashdreams.accelerated.common.rms_norm + +## Quantization + +::: flashdreams.accelerated.quantization.quantizer.Granularity + +::: flashdreams.accelerated.quantization.quantizer.quantize + +::: flashdreams.accelerated.quantization.quantizer.dequantize + +::: flashdreams.accelerated.quantization.linear.WeightGranularity + +::: flashdreams.accelerated.quantization.linear.QuantizedNonPersistentLinear + +## Attention contracts + +::: flashdreams.accelerated.multi_head_attention.AttentionType + +::: flashdreams.accelerated.multi_head_attention.AttentionConfig + +::: flashdreams.accelerated.multi_head_attention.AttentionMask + +::: flashdreams.accelerated.multi_head_attention.QKNormScope + +::: flashdreams.accelerated.multi_head_attention.RoPEStyle + +::: flashdreams.accelerated.multi_head_attention.RoPEScope + +::: flashdreams.accelerated.multi_head_attention.RoPEConfig + +::: flashdreams.accelerated.multi_head_attention.MultiHeadAttention + +## Attention implementations + +::: flashdreams.accelerated.multi_head_attention.torch.TorchMultiHeadAttention + +::: flashdreams.accelerated.multi_head_attention.optimized.SDPABackend + +::: flashdreams.accelerated.multi_head_attention.optimized.QKVFusionOption + +::: flashdreams.accelerated.multi_head_attention.optimized.QuantizationOption + +::: flashdreams.accelerated.multi_head_attention.optimized.OptimizedImplConfig + +::: flashdreams.accelerated.multi_head_attention.optimized.OptimizedMultiHeadAttention + +::: flashdreams.accelerated.multi_head_attention.flex.FlexAttentionOptions + +::: flashdreams.accelerated.multi_head_attention.flex.compiled_flex_attention + +::: flashdreams.accelerated.multi_head_attention.MoTRoute + +::: flashdreams.accelerated.multi_head_attention.mot.MoTMultiHeadAttention + +## Masked attention functions + +::: flashdreams.accelerated.multi_head_attention.functional.masked_attention + +::: flashdreams.accelerated.multi_head_attention.reference.reference_masked_attention + +## KV arena + +::: flashdreams.accelerated.multi_head_attention.kv_arena.KVSegmentLayout + +::: flashdreams.accelerated.multi_head_attention.kv_arena.allocate_kv_arena + +::: flashdreams.accelerated.multi_head_attention.kv_arena.validate_kv_arena_segments + +::: flashdreams.accelerated.multi_head_attention.kv_arena.stage_current_kv_ diff --git a/docs/source/documentation/inferencing_api/guides/accelerated.md b/docs/source/documentation/inferencing_api/guides/accelerated.md index b0db96de7..4eae7036d 100644 --- a/docs/source/documentation/inferencing_api/guides/accelerated.md +++ b/docs/source/documentation/inferencing_api/guides/accelerated.md @@ -10,6 +10,9 @@ integrations. It currently covers tensor/linear quantization and streaming multi-head attention. These APIs are implementation tools, not a switch that makes every pipeline faster. +See the [Accelerated API reference](../api_reference/accelerated.md) for generated +signatures and docstrings. + ## Quantization ### Tensor quantization diff --git a/docs/source/documentation/inferencing_api/index.md b/docs/source/documentation/inferencing_api/index.md index ea6402fd1..6ffce5b3b 100644 --- a/docs/source/documentation/inferencing_api/index.md +++ b/docs/source/documentation/inferencing_api/index.md @@ -5,13 +5,9 @@ title: 'Inferencing API' -Use the Inferencing API to implement model components, pipeline configuration, -streaming generation, caching, and optimized execution behind a Demo API model -loop. +Use the Inferencing API to implement model components, and pipelines to run inference on World Models. -`flashdreams.runtime` provides the experimental model and session contracts. -`flashdreams.infra`, `flashdreams.recipes`, and `flashdreams.core` provide -the lower-level pipeline and model building blocks. +`flashdreams.infra` and `flashdreams.core` provide the lower-level pipeline and model building blocks. ## Guides @@ -27,4 +23,5 @@ the lower-level pipeline and model building blocks.
Pipeline APIReference for inference pipelines and reusable infrastructure. Core APIReference for model-agnostic core components and utilities. + Accelerated APIReference for optimized attention, quantization, and shared inference components.
diff --git a/flashdreams/flashdreams/accelerated/__init__.py b/flashdreams/flashdreams/accelerated/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/flashdreams/flashdreams/accelerated/__init__.py @@ -0,0 +1 @@ + diff --git a/flashdreams/flashdreams/accelerated/quantization/__init__.py b/flashdreams/flashdreams/accelerated/quantization/__init__.py new file mode 100644 index 000000000..8b1378917 --- /dev/null +++ b/flashdreams/flashdreams/accelerated/quantization/__init__.py @@ -0,0 +1 @@ + From 3ab4464f1160abaeb61075a3b24080810e51595e Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 16:57:42 +0000 Subject: [PATCH 05/14] remove useless sentence --- docs/source/documentation/inferencing_api/index.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/source/documentation/inferencing_api/index.md b/docs/source/documentation/inferencing_api/index.md index 6ffce5b3b..fc469f925 100644 --- a/docs/source/documentation/inferencing_api/index.md +++ b/docs/source/documentation/inferencing_api/index.md @@ -7,8 +7,6 @@ title: 'Inferencing API' Use the Inferencing API to implement model components, and pipelines to run inference on World Models. -`flashdreams.infra` and `flashdreams.core` provide the lower-level pipeline and model building blocks. - ## Guides
From a577e96d45bc82636cbcaccbc0a40fa1979c9e06 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 17:48:34 +0000 Subject: [PATCH 06/14] clean up community page --- docs/source/community/index.md | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/docs/source/community/index.md b/docs/source/community/index.md index 2f06f40ec..fe20cfeab 100644 --- a/docs/source/community/index.md +++ b/docs/source/community/index.md @@ -44,17 +44,10 @@ uv sync --package flashdreams-wan21 --extra dev ### Which Python API should I use? -Use `flashdreams.api_v2`; the -[Demo API reference](../documentation/demo_api/index.md) maps its -application, session, and loop contracts. +Make demos using world-models with our [Demo API](../documentation/demo_api/index.md). +Implement world-models for inferencing with our [Inferencing API](../documentation/inferencing_api/index.md). -### How do I add a model? +### How do I implement a model with flashdreams? -Start with -[Create a model](../documentation/inferencing_api/guides/create_model.md), then -[integrate it with a demo](../documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md). - -### Can I use FlashDreams commercially? - -The Apache 2.0 license permits commercial use. Model weights and datasets may -have separate licenses. +1. [Implement a model in FlashDreams](../documentation/inferencing_api/guides/create_model.md), then +2. [Integrate your new model with a demo](../documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md). From 8aadf27a0e26c2dfb435b1c5bedd78bd44e938bb Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 18:14:46 +0000 Subject: [PATCH 07/14] contyributing update + readme simplification --- CONTRIBUTING.md | 53 ++--------------- README.md | 102 ++++++++++++++++++++++++++++++++- docs/source/community/index.md | 2 +- 3 files changed, 107 insertions(+), 50 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 73479ff21..0be27d651 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,10 +30,9 @@ Setup and build instructions: 9. [Testing](#testing) 10. [Dependency version bounds](#dependency-version-bounds) 11. [Working with a single integration package](#working-with-a-single-integration-package) -12. [Adding an Agent Skill](#adding-an-agent-skill) -13. [Licensing of contributions](#licensing-of-contributions) -14. [Reporting issues](#reporting-issues) -15. [Code of Conduct](#code-of-conduct) +12. [Licensing of contributions](#licensing-of-contributions) +13. [Reporting issues](#reporting-issues) +14. [Code of Conduct](#code-of-conduct) ## Ways to contribute @@ -179,10 +178,6 @@ pseudonymous identities cannot be accepted. ## Submitting a pull request -Use the website's [Get Started](https://nvidia.github.io/flashdreams/main/quickstart/) -guide for installation and the relevant model or demo authoring guide for -component-specific setup. Then follow the repository process below. - The short version: 1. Fork the repo on GitHub and create a feature branch from `main`. @@ -306,48 +301,12 @@ Register a new page in the `.nav.yml` beside it; for this example, add `My Demo: my_demo.md` to `docs/source/demos/.nav.yml`. Use a lowercase snake-case filename that matches the model or demo package when applicable. -The focused authoring guides show the expected locations in context: - -- [Create a demo](https://nvidia.github.io/flashdreams/main/documentation/demo_api/guides/create_demo/#package-layout) - uses `docs/source/demos/.md`. -- [Create a model](https://nvidia.github.io/flashdreams/main/documentation/inferencing_api/guides/create_model/#2-scaffold-the-package) - uses `docs/source/models/.md`. -- [Integrate model adapter for a demo](https://nvidia.github.io/flashdreams/main/documentation/demo_api/guides/integrate_model_adapter_for_a_demo/#5-document-once) - updates the model article and links to the shared demo article. - Files outside `docs/source/` should link to the published website article -instead of copying its instructions. Add a package README only when external -tooling requires one; keep it to a pointer to the canonical website article. +instead of copying its instructions. `README.md` outside website, should try to point to the canonical website as much as possible. Build or preview the site with the commands in [Build Website](https://nvidia.github.io/flashdreams/main/documentation/tools/build_website/#build-the-website). -## Adding an Agent Skill - -Project-authored [Agent Skills](https://agentskills.io/specification) live in -`skills//SKILL.md`. Each skill requires YAML -frontmatter with a matching `name` and a specific `description` explaining what -the skill does and when to use it. Keep contribution policy in this guide and link to its exact section from -the skill instead of copying it. - -Add new skills to the **Skill Map** in `AGENTS.md`. Put sibling references, -scripts, or assets beside a skill only when they materially improve its -workflow. Validate skill structure with: - -```bash -uv run --group test pytest -m ci_cpu tests/test_agent_skills.py -``` - -Repository skills are opt-in. From the repository root, link all skills into -the directory used by your agent: - -```bash -mkdir -p .agents && ln -s ../skills .agents/skills -``` - -Replace `.agents` with `.cursor` or `.claude` as needed. These directories are -gitignored. To enable one skill, link only that skill directory instead. - ## Coding conventions - Python 3.10+. Type-annotate new code; the project type-checks with @@ -399,9 +358,7 @@ the declared minimums. This means: ## Working with a single integration package -See [Install one integration](https://nvidia.github.io/flashdreams/main/quickstart/#install-one-integration) -for package-specific setup. Available integrations and their setup instructions -are listed on the [Models](https://nvidia.github.io/flashdreams/main/models/) page. +See our [model gallery](https://nvidia.github.io/flashdreams/main/models/) page. ## Licensing of contributions diff --git a/README.md b/README.md index 5fb398e85..d0b35c90c 100644 --- a/README.md +++ b/README.md @@ -1 +1,101 @@ -[FlashDreams](docs/source/index.md#flashdreams) + + +

+ + + FlashDreams + +

+ +

+ License: Apache 2.0 + Documentation +

+ +**FlashDreams** is a high-performance inference and serving library for +interactive autoregressive video and world models. It began as the optimized +runtime behind the [NVIDIA OmniDreams closed-loop demo for GTC 2026][omnidreams-blog] +and has grown into a general platform for real-time world-model applications +across gaming, autonomous vehicles, robotics, simulated or virtual +environments, and more. + + +## Quickstart + +The complete setup is in our +[quick-start guide](https://nvidia.github.io/flashdreams/main/quickstart/index.html#install). + +## Checkout Our Demos! 🎉 + +Quick-start to our interactive driving demo is [here](https://nvidia.github.io/flashdreams/main/quickstart/index.html#run-your-first-model). + +![](docs/source/_static/demo_clips/interactive-drive.avif) + +Checkout our game-like crazy-robotaxi demo [here](https://nvidia.github.io/flashdreams/main/demos/crazy_robotaxi.md). + +![](docs/source/_static/demo_clips/crazy-robotaxi.avif) + +## Developer guides + +### For Demo Developers + +Make demos using world-models with our [Demo API](../documentation/demo_api/index.md). + +### For Model Developers + +Implement world-models with our [Inferencing API](../documentation/inferencing_api/index.md). + +## Contributing + +For how to contribute, see [`CONTRIBUTING.md`](CONTRIBUTING.md). +New integrations, bug reports, feature requests, performance tuning, and +documentation edits are all welcome. + +Use [GitHub Issues](https://github.com/NVIDIA/flashdreams/issues) to report defects or request improvements. + +Join us on the [NVIDIA Omniverse Discord](https://discord.com/invite/nvidiaomniverse) +to share your results and take part in technical discussion! Channel: [`#flashdreams`](https://discord.gg/cMt2mHm4aN) + +## Security + +To report a potential security vulnerability, follow the coordinated +disclosure process in [`SECURITY.md`](SECURITY.md). + +## License + +FlashDreams is released under the [Apache License 2.0](LICENSE). Third-party +components and their licenses are listed in +[`THIRD-PARTY-NOTICES`](THIRD-PARTY-NOTICES) and [`NOTICE`](NOTICE). The +repository is REUSE-compliant; see [`REUSE.toml`](REUSE.toml) and +[`LICENSES/`](LICENSES/). The independently packaged +[Ludus renderer](integrations_v2/omnidreams/impl/ludus-renderer/LICENSE) retains its own Apache license file; +[upstream cudaraster terms](integrations_v2/omnidreams/impl/ludus-renderer/ludus_renderer/_cpp/cudaraster/LICENSE) +and per-file exceptions are described in the third-party notices. + +## Citation + +If FlashDreams is useful in your research or product, please cite the project: + +```bibtex +@misc{flashdreams2026, + title = {FlashDreams: High-performance inference and serving for + interactive autoregressive video and world models}, + author = {{FlashDreams Contributors}}, + year = {2026}, + howpublished = {\url{https://github.com/NVIDIA/flashdreams}}, +} + +@misc{nvidia2026omnidreams, + title = {{NVIDIA} {OmniDreams}: Real-Time Generative World Model for Closed-Loop Autonomous Vehicle Simulation}, + author = {Basant, Aarti and Kar, Amlan and Paschalidou, Despoina and Wei, Fangyin and Ferroni, Francesco and Garcia Cobo, Guillermo and Turki, Haithem and Ling, Huan and Seo, Jaewoo and Lucas, James and Wu, Jay Zhangjie and Wang, Jialiang and Lorraine, Jonathan and Gao, Jun and He, Kai and Tothova, Katarina and Xie, Kevin and Tyszkiewicz, Micha{\l} and Wu, Qi and de Lutio, Riccardo and Li, Ruilong and Fidler, Sanja and Kim, Seung Wook and Shen, Tianchang and Cao, Tianshi and Pfaff, Tobias and Lew, William and Wu, Xindi and Ren, Xuanchi and Lu, Yifan and Zhang, Yuxuan and Gojcic, Zan and Wang, Zian}, + year = {2026}, + eprint = {2606.03159}, + archivePrefix = {arXiv}, + primaryClass = {cs.CV}, + doi = {10.48550/arXiv.2606.03159}, + url = {https://arxiv.org/abs/2606.03159}, +} +``` diff --git a/docs/source/community/index.md b/docs/source/community/index.md index fe20cfeab..ccfada5b6 100644 --- a/docs/source/community/index.md +++ b/docs/source/community/index.md @@ -45,7 +45,7 @@ uv sync --package flashdreams-wan21 --extra dev ### Which Python API should I use? Make demos using world-models with our [Demo API](../documentation/demo_api/index.md). -Implement world-models for inferencing with our [Inferencing API](../documentation/inferencing_api/index.md). +Implement world-models with our [Inferencing API](../documentation/inferencing_api/index.md). ### How do I implement a model with flashdreams? From 6000e6707fac0b29a981e69babdc83394998267a Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 19:24:16 +0000 Subject: [PATCH 08/14] format --- docs/source/community/.nav.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/community/.nav.yml b/docs/source/community/.nav.yml index d921fbc54..fbe5d798b 100644 --- a/docs/source/community/.nav.yml +++ b/docs/source/community/.nav.yml @@ -1,2 +1,2 @@ nav: - - Overview: index.md \ No newline at end of file + - Overview: index.md From 4e257c88a949fc927af0fd70ef050b2fae392b90 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Fri, 16 Oct 2026 19:31:53 +0000 Subject: [PATCH 09/14] add liscence --- flashdreams/flashdreams/accelerated/__init__.py | 3 ++- flashdreams/flashdreams/accelerated/quantization/__init__.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/flashdreams/flashdreams/accelerated/__init__.py b/flashdreams/flashdreams/accelerated/__init__.py index 8b1378917..b7040b748 100644 --- a/flashdreams/flashdreams/accelerated/__init__.py +++ b/flashdreams/flashdreams/accelerated/__init__.py @@ -1 +1,2 @@ - + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 diff --git a/flashdreams/flashdreams/accelerated/quantization/__init__.py b/flashdreams/flashdreams/accelerated/quantization/__init__.py index 8b1378917..b7040b748 100644 --- a/flashdreams/flashdreams/accelerated/quantization/__init__.py +++ b/flashdreams/flashdreams/accelerated/quantization/__init__.py @@ -1 +1,2 @@ - + # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + # SPDX-License-Identifier: Apache-2.0 From 37ddc77ae06523bfda23e26ae17afd03576e967c Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Sat, 17 Oct 2026 01:07:17 +0000 Subject: [PATCH 10/14] human touch-ups to guide for demo-api --- README.md | 8 +- apps/README.md | 1 + apps/minimal_t2v/minimal_t2v/__init__.py | 8 + apps/minimal_t2v/minimal_t2v/application.py | 74 ++++++ apps/minimal_t2v/minimal_t2v/model_loop.py | 77 ++++++ apps/minimal_t2v/minimal_t2v/session.py | 45 ++++ apps/minimal_t2v/minimal_t2v/ui_loop.py | 46 ++++ apps/minimal_t2v/pyproject.toml | 19 ++ apps/minimal_t2v/tests/test_application.py | 136 ++++++++++ docs/source/.nav.yml | 1 - docs/source/demos/.nav.yml | 1 + docs/source/demos/index.md | 1 + docs/source/demos/minimal_t2v.md | 25 ++ docs/source/documentation/cli.md | 1 - .../demo_api/api_reference/application.md | 6 + .../guides/adding_a_new_configuration.md | 102 ------- .../demo_api/guides/create_demo.md | 249 +++++------------- .../integrate_model_adapter_for_a_demo.md | 132 ++++------ docs/source/documentation/demo_api/index.md | 3 - docs/source/models/wan21.md | 22 +- docs/zensical.toml | 1 + .../wan21/apps/minimal_t2v/__init__.py | 2 + .../wan21/apps/minimal_t2v/adapter.py | 22 ++ integrations_v2/wan21/pyproject.toml | 5 +- .../wan21/tests/test_minimal_t2v.py | 57 ++++ pyproject.toml | 2 + tests/test_docs_links.py | 46 +++- uv.lock | 55 +--- 28 files changed, 711 insertions(+), 436 deletions(-) create mode 100644 apps/minimal_t2v/minimal_t2v/__init__.py create mode 100644 apps/minimal_t2v/minimal_t2v/application.py create mode 100644 apps/minimal_t2v/minimal_t2v/model_loop.py create mode 100644 apps/minimal_t2v/minimal_t2v/session.py create mode 100644 apps/minimal_t2v/minimal_t2v/ui_loop.py create mode 100644 apps/minimal_t2v/pyproject.toml create mode 100644 apps/minimal_t2v/tests/test_application.py create mode 100644 docs/source/demos/minimal_t2v.md delete mode 100644 docs/source/documentation/demo_api/guides/adding_a_new_configuration.md create mode 100644 integrations_v2/wan21/apps/minimal_t2v/__init__.py create mode 100644 integrations_v2/wan21/apps/minimal_t2v/adapter.py create mode 100644 integrations_v2/wan21/tests/test_minimal_t2v.py diff --git a/README.md b/README.md index d0b35c90c..529a53cdb 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Quick-start to our interactive driving demo is [here](https://nvidia.github.io/f ![](docs/source/_static/demo_clips/interactive-drive.avif) -Checkout our game-like crazy-robotaxi demo [here](https://nvidia.github.io/flashdreams/main/demos/crazy_robotaxi.md). +Checkout our game-like crazy-robotaxi demo [here](https://nvidia.github.io/flashdreams/main/demos/crazy_robotaxi/). ![](docs/source/_static/demo_clips/crazy-robotaxi.avif) @@ -42,11 +42,11 @@ Checkout our game-like crazy-robotaxi demo [here](https://nvidia.github.io/flash ### For Demo Developers -Make demos using world-models with our [Demo API](../documentation/demo_api/index.md). +Make demos using world-models with our [Demo API](https://nvidia.github.io/flashdreams/main/documentation/demo_api/). ### For Model Developers -Implement world-models with our [Inferencing API](../documentation/inferencing_api/index.md). +Implement world-models with our [Inferencing API](https://nvidia.github.io/flashdreams/main/documentation/inferencing_api/). ## Contributing @@ -99,3 +99,5 @@ If FlashDreams is useful in your research or product, please cite the project: url = {https://arxiv.org/abs/2606.03159}, } ``` + +[omnidreams-blog]: https://research.nvidia.com/labs/sil/projects/omnidreams-blog/ diff --git a/apps/README.md b/apps/README.md index 1a9460b85..8c72baa92 100644 --- a/apps/README.md +++ b/apps/README.md @@ -25,6 +25,7 @@ model or its adapter belongs in `integrations_v2//tests/`. ## What is here +- `minimal_t2v` — Minimal T2V app from the Create a demo tutorial. - `t2v` — text-to-video: prompt in, frames out. The reference app to copy. - `cam2v` — interactive camera-to-video. - `action2v` — world models driven by direct action input. diff --git a/apps/minimal_t2v/minimal_t2v/__init__.py b/apps/minimal_t2v/minimal_t2v/__init__.py new file mode 100644 index 000000000..4e7852900 --- /dev/null +++ b/apps/minimal_t2v/minimal_t2v/__init__.py @@ -0,0 +1,8 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reference application for the Create a demo tutorial.""" + +from .application import MinimalT2VApplication + +__all__ = ["MinimalT2VApplication"] diff --git a/apps/minimal_t2v/minimal_t2v/application.py b/apps/minimal_t2v/minimal_t2v/application.py new file mode 100644 index 000000000..2235eb68b --- /dev/null +++ b/apps/minimal_t2v/minimal_t2v/application.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal text-to-video application with lazy model initialization.""" + +import argparse +from collections.abc import Sequence +from typing import Any + +from flashdreams.api_v2.application import IApplication +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + +from .session import MinimalT2VSession + + +class MinimalT2VApplication(IApplication): + """Tutorial app configured by a model adapter. + + The pipeline must accept text and latent dimensions in ``initialize_cache`` + and return ``tchw`` video from ``generate``, as Wan T2V pipelines do. + """ + + def __init__( + self, pipeline_config: Any, desc: SessionDesc, total_blocks: int = 8 + ) -> None: + """Keep model configuration without loading weights.""" + if desc.output_layout is not VideoTensorLayout.tchw: + raise ValueError("Minimal T2V requires tchw output.") + if total_blocks <= 0: + raise ValueError("total_blocks must be positive.") + self.pipeline_config = pipeline_config + self.desc = desc + self.total_blocks = total_blocks + self.pipeline: Any = None + self.prompt = "" + + def init(self, commandline_args: Sequence[str]) -> None: + """Parse the initial prompt and load the shared pipeline.""" + parser = argparse.ArgumentParser(description="Generate video from text.") + parser.add_argument("--default-prompt", default="") + parser.add_argument("--device", default="cuda") + args = parser.parse_args(commandline_args) + self.prompt = args.default_prompt.strip() + self.pipeline = self.pipeline_config.setup().to(args.device).eval() + + def session_desc(self) -> SessionDesc: + """Return the model adapter's output description.""" + return self.desc + + def create_session(self, session_desc: SessionDesc) -> MinimalT2VSession: + """Create a session at the model's supported size and layout.""" + if self.pipeline is None: + raise RuntimeError("Call init before create_session.") + if ( + session_desc.video_width != self.desc.video_width + or session_desc.video_height != self.desc.video_height + or session_desc.output_layout != self.desc.output_layout + or session_desc.frames_per_second_for_step + != self.desc.frames_per_second_for_step + ): + raise ValueError("The session must match the model's output description.") + ratio = self.pipeline.decoder.spatial_compression_ratio + if session_desc.video_width % ratio or session_desc.video_height % ratio: + raise ValueError( + "Video dimensions must be divisible by decoder compression." + ) + return MinimalT2VSession( + self.pipeline, session_desc, self.prompt, self.total_blocks + ) + + def close(self) -> None: + """Release the shared pipeline reference.""" + self.pipeline = None diff --git a/apps/minimal_t2v/minimal_t2v/model_loop.py b/apps/minimal_t2v/minimal_t2v/model_loop.py new file mode 100644 index 000000000..1d4dc0b00 --- /dev/null +++ b/apps/minimal_t2v/minimal_t2v/model_loop.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Prompt and autoregressive cache owned by the tutorial model loop.""" + +from dataclasses import dataclass +from typing import Any + +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.user_input_events import UserInputEvents + + +@dataclass +class TextToVideoState: + """Mutable rollout state, accessed on the model thread.""" + + pipeline: Any + """Text-conditioned pipeline shared across sessions.""" + desc: SessionDesc + """Pixel dimensions and frame layout requested by the runtime.""" + total_blocks: int + """Maximum blocks the selected model supports per prompt.""" + prompt: str = "" + """Current prompt; empty text pauses generation.""" + cache: Any = None + """Per-prompt cache, created lazily on the model thread.""" + index: int = 0 + """Next autoregressive block, independent of the runtime step index.""" + + def set_prompt(self, prompt: str) -> None: + """Start a fresh rollout or pause generation for empty text.""" + self.prompt = prompt.strip() + self.cache = None + self.index = 0 + + +class TextToVideoLoop(IModelLoop[TextToVideoState]): + """Generate bounded rollouts while waiting for further UI prompts.""" + + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + """Generate and finalize one block, or wait for another prompt.""" + state = self.state + if not state.prompt or state.index >= state.total_blocks: + return [] + if state.cache is None: + ratio = state.pipeline.decoder.spatial_compression_ratio + state.cache = state.pipeline.initialize_cache( + text=[state.prompt], + height=state.desc.video_height // ratio, + width=state.desc.video_width // ratio, + ) + frames = state.pipeline.generate( + autoregressive_index=state.index, cache=state.cache + ) + metrics = state.pipeline.finalize( + autoregressive_index=state.index, cache=state.cache + ) + state.index += 1 + return [ + StepResult( + step_index=step_index, + output=frames.detach(), + frame_count=int(frames.shape[0]), + output_layout=state.desc.output_layout, + metrics=metrics, + ) + ] + + def reset(self) -> None: + """Restart the current prompt after a runtime reset.""" + self.state.set_prompt(self.state.prompt) + + def close(self) -> None: + """Release the per-prompt cache.""" + self.state.cache = None diff --git a/apps/minimal_t2v/minimal_t2v/session.py b/apps/minimal_t2v/minimal_t2v/session.py new file mode 100644 index 000000000..1e5c83387 --- /dev/null +++ b/apps/minimal_t2v/minimal_t2v/session.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Session wiring for the Minimal T2V model and UI loops.""" + +from typing import Any + +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.session_desc import SessionDesc + +from .model_loop import TextToVideoLoop, TextToVideoState +from .ui_loop import MinimalT2VUILoop, MinimalT2VUIState + + +class MinimalT2VSession(ISession): + """One interactive tutorial session sharing the application's pipeline.""" + + def __init__( + self, pipeline: Any, desc: SessionDesc, prompt: str, total_blocks: int + ) -> None: + """Keep session inputs until the runtime initializes the loops.""" + self.pipeline = pipeline + self.desc = desc + self.prompt = prompt + self.total_blocks = total_blocks + + @property + def session_desc(self) -> SessionDesc: + """Return the description used to configure the window.""" + return self.desc + + def init(self) -> None: + """Register independent model and UI state.""" + model_loop = self.register_model_loop( + TextToVideoLoop, + state=TextToVideoState( + self.pipeline, self.desc, self.total_blocks, self.prompt + ), + ) + self.register_ui_loop( + MinimalT2VUILoop, + state=MinimalT2VUIState(model_loop, self.prompt), + width=self.desc.video_width, + height=self.desc.video_height, + ) diff --git a/apps/minimal_t2v/minimal_t2v/ui_loop.py b/apps/minimal_t2v/minimal_t2v/ui_loop.py new file mode 100644 index 000000000..704487da7 --- /dev/null +++ b/apps/minimal_t2v/minimal_t2v/ui_loop.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""ImGui prompt controls sending messages to the tutorial model loop.""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +from torch import Tensor + +from flashdreams.api_v2.loop import IModelLoop, invoke_async +from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +if TYPE_CHECKING: + from .model_loop import TextToVideoState + + +@dataclass +class MinimalT2VUIState: + """Editable prompt owned by the UI thread.""" + + model_loop: IModelLoop["TextToVideoState"] + """Destination of prompt update messages.""" + text: str = "" + """Text currently displayed in the prompt field.""" + + +class MinimalT2VUILoop(ImGuiUILoop[MinimalT2VUIState]): + """Send prompt updates without directly mutating model state.""" + + def step_ui( + self, imgui: Any, step_index: int, events: UserInputEvents + ) -> Tensor | None: + """Draw prompt controls over the latest generated frame.""" + imgui.begin("Minimal T2V") + try: + _, self.state.text = imgui.input_text("Prompt", self.state.text) + if imgui.button("Send"): + invoke_async( + self.state.model_loop, + lambda state, text=self.state.text: state.set_prompt(text), + ) + finally: + imgui.end() + return self.presented_model_frame() diff --git a/apps/minimal_t2v/pyproject.toml b/apps/minimal_t2v/pyproject.toml new file mode 100644 index 000000000..e4b94edbd --- /dev/null +++ b/apps/minimal_t2v/pyproject.toml @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "flashdreams-minimal-t2v" +version = "0.1.0" +description = "Reference implementation for the Create a demo tutorial" +requires-python = ">=3.10" +dependencies = ["flashdreams[local-window]"] + +[tool.uv.sources] +flashdreams = { workspace = true } + +[tool.setuptools.packages.find] +include = ["minimal_t2v*"] diff --git a/apps/minimal_t2v/tests/test_application.py b/apps/minimal_t2v/tests/test_application.py new file mode 100644 index 000000000..8146a29b4 --- /dev/null +++ b/apps/minimal_t2v/tests/test_application.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU checks for the tutorial's startup, prompt messages, and cache lifecycle.""" + +from dataclasses import replace +from unittest.mock import Mock + +import pytest +import torch +from minimal_t2v import MinimalT2VApplication +from minimal_t2v.ui_loop import MinimalT2VUILoop + +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + +pytestmark = pytest.mark.ci_cpu + + +def _application() -> tuple[MinimalT2VApplication, Mock, Mock]: + pipeline = Mock() + pipeline.to.return_value = pipeline + pipeline.eval.return_value = pipeline + pipeline.decoder.spatial_compression_ratio = 8 + pipeline.generate.return_value = torch.zeros(2, 3, 16, 32) + pipeline.finalize.return_value = {"total_ms": 1.0} + config = Mock() + config.setup.return_value = pipeline + app = MinimalT2VApplication( + config, SessionDesc(video_width=32, video_height=16), total_blocks=2 + ) + return app, pipeline, config + + +def test_help_and_description_do_not_load_the_model() -> None: + app, _, config = _application() + assert app.session_desc().video_width == 32 + with pytest.raises(SystemExit) as exit_info: + app.init(["--help"]) + assert exit_info.value.code == 0 + config.setup.assert_not_called() + with pytest.raises(RuntimeError, match="init"): + app.create_session(app.session_desc()) + + +def test_prompt_generation_reset_and_close() -> None: + app, pipeline, _ = _application() + app.init(["--default-prompt", " First prompt ", "--device", "cpu"]) + pipeline.to.assert_called_once_with("cpu") + session = app.create_session(app.session_desc()) + session.init() + loop = session.model_loop + events = UserInputEvents([]) + pipeline.initialize_cache.assert_not_called() + + first = loop.step(7, events)[0] + cache = pipeline.initialize_cache.return_value + pipeline.initialize_cache.assert_called_once_with( + text=["First prompt"], height=2, width=4 + ) + pipeline.generate.assert_called_once_with(autoregressive_index=0, cache=cache) + pipeline.finalize.assert_called_once_with(autoregressive_index=0, cache=cache) + assert first.step_index == 7 + assert first.frame_count == 2 + assert first.output_layout is VideoTensorLayout.tchw + assert first.read_output().shape == (2, 3, 16, 32) + assert first.metrics == {"total_ms": 1.0} + loop.step(8, events) + assert loop.step(9, events) == [] + assert pipeline.generate.call_count == 2 + assert not loop.is_finished() # The UI can still send another prompt. + + loop.reset() + loop.step(10, events) + assert pipeline.initialize_cache.call_count == 2 + pipeline.generate.assert_called_with(autoregressive_index=0, cache=cache) + loop.close() + assert loop.state.cache is None + app.close() + assert app.pipeline is None + + +def test_ui_captures_prompt_and_queues_model_thread_update() -> None: + app, pipeline, _ = _application() + app.init(["--device", "cpu"]) + session = app.create_session(app.session_desc()) + session.init() + loop, ui = session.model_loop, session.ui_loop + assert isinstance(ui, MinimalT2VUILoop) + events = UserInputEvents([]) + assert loop.step(0, events) == [] + imgui = Mock() + imgui.input_text.return_value = (True, "Second prompt") + imgui.button.return_value = True + + ui.step_ui(imgui, 0, events) + imgui.end.assert_called_once() + ui.state.text = "Edited after clicking Send" + assert loop.state.prompt == "" + loop._run_message_batch() + assert loop.state.prompt == "Second prompt" + loop.step(4, events) + pipeline.initialize_cache.assert_called_once_with( + text=["Second prompt"], height=2, width=4 + ) + + imgui.input_text.return_value = (True, " ") + ui.step_ui(imgui, 1, events) + loop._run_message_batch() + assert loop.step(5, events) == [] + assert loop.state.cache is None + + +@pytest.mark.parametrize( + "changes", + [ + {"video_width": 64}, + {"video_height": 32}, + {"output_layout": VideoTensorLayout.btchw}, + {"frames_per_second_for_step": 16}, + ], +) +def test_session_must_match_model_output(changes: dict) -> None: + app, _, _ = _application() + app.init(["--device", "cpu"]) + with pytest.raises(ValueError, match="output description"): + app.create_session(replace(app.session_desc(), **changes)) + + +def test_rejects_dimensions_that_truncate_latents() -> None: + app, _, _ = _application() + app.desc = replace(app.desc, video_width=31) + app.init(["--device", "cpu"]) + with pytest.raises(ValueError, match="divisible"): + app.create_session(app.session_desc()) diff --git a/docs/source/.nav.yml b/docs/source/.nav.yml index d917ebe6b..4ccf41a32 100644 --- a/docs/source/.nav.yml +++ b/docs/source/.nav.yml @@ -11,7 +11,6 @@ nav: - Guides: - Create a demo: documentation/demo_api/guides/create_demo.md - Integrate model adapter for a demo: documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md - - Adding a new configuration: documentation/demo_api/guides/adding_a_new_configuration.md - Interactive serving: documentation/demo_api/guides/interactive_serving.md - Profiling a demo: documentation/demo_api/guides/profiling.md - API Reference: diff --git a/docs/source/demos/.nav.yml b/docs/source/demos/.nav.yml index 7a1aa40a1..8664b90b8 100644 --- a/docs/source/demos/.nav.yml +++ b/docs/source/demos/.nav.yml @@ -6,5 +6,6 @@ nav: - Model Interfaces: - Action2V: action2v.md - Cam2V: cam2v.md + - Minimal T2V: minimal_t2v.md - T2V: t2v.md - V2V: v2v.md diff --git a/docs/source/demos/index.md b/docs/source/demos/index.md index 025f07577..f5b50600a 100644 --- a/docs/source/demos/index.md +++ b/docs/source/demos/index.md @@ -31,4 +31,5 @@ For how to run a particular **demo/model pair**, refer to the format listed here
Cam2VControl a camera through a generated world interactively.
T2VGenerate video from text and submit prompts without unloading the model.
V2VRestore or upscale a local or remote video source.
+
Minimal T2VTutorial application with an ImGui prompt field and a per-prompt model cache.
diff --git a/docs/source/demos/minimal_t2v.md b/docs/source/demos/minimal_t2v.md new file mode 100644 index 000000000..646569c8f --- /dev/null +++ b/docs/source/demos/minimal_t2v.md @@ -0,0 +1,25 @@ +--- +title: 'Minimal T2V' +--- + + + + +Minimal T2V is the tutorial application built in [Create a demo](../documentation/demo_api/guides/create_demo.md). +It owns an ImGui prompt field, a model loop, and a per-prompt generation cache. + +## Quick Start: + +
+ [Wan 2.1](../models/wan21.md#minimal-t2v){ .fd-button } +
+ +## Demo Arguments + +Edit the prompt and click **Send** to start a new clip. Empty text pauses +generation. The window remains open between clips; close it to stop. + +| App argument | Default | Purpose | +| --- | --- | --- | +| `--default-prompt` | Empty | Initial prompt; omit it to wait for input. | +| `--device` | `cuda` | Device on which to load the model. | diff --git a/docs/source/documentation/cli.md b/docs/source/documentation/cli.md index 602b02b56..af47d1066 100644 --- a/docs/source/documentation/cli.md +++ b/docs/source/documentation/cli.md @@ -98,6 +98,5 @@ uv run flashdreams-run-v2 DEMO_MODEL_SLUG --mode native-window --window-title Fl ## See also - [Quickstart](../quickstart/index.md) -- [Adding a new configuration](demo_api/guides/adding_a_new_configuration.md) - [Demo API](demo_api/index.md) - [Offline Program Packager](tools/offline_program_packager.md) diff --git a/docs/source/documentation/demo_api/api_reference/application.md b/docs/source/documentation/demo_api/api_reference/application.md index 6a48b153b..0c2e03ed0 100644 --- a/docs/source/documentation/demo_api/api_reference/application.md +++ b/docs/source/documentation/demo_api/api_reference/application.md @@ -22,6 +22,12 @@ workflows, start with [Create a demo](../guides/create_demo.md) or ::: flashdreams.api_v2.loop.IUILoop +::: flashdreams.runtime_v2.imgui_ui_loop.ImGuiUILoop + ## Session description ::: flashdreams.runtime_v2.session_desc.SessionDesc + +::: flashdreams.runtime_v2.session_desc.BackpressureMode + +::: flashdreams.runtime_v2.session_desc.PresentationMode \ No newline at end of file diff --git a/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md b/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md deleted file mode 100644 index 65153fdd9..000000000 --- a/docs/source/documentation/demo_api/guides/adding_a_new_configuration.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: 'Adding a new configuration' ---- - - - - -After [integrating a model adapter for a demo](integrate_model_adapter_for_a_demo.md), you can -add another preset without writing another app. This example adds a faster -pipeline config to the T2V adapter. - -## Files to update - -```text -integrations_v2// - config.py # define the pipeline variant - apps/t2v/adapter.py # add its application factory - pyproject.toml # register its CLI name - -docs/source/models/.md # add the preset and run command -``` - -## Define the pipeline config - -In `config.py`, derive a variant from the existing config: - -```python -from flashdreams.infra.config import derive_config - -PIPELINE_MY_MODEL_FAST = derive_config( - PIPELINE_MY_MODEL, - name="my-model-fast", - diffusion_model={"scheduler": {"num_inference_steps": 4}}, -) -``` - -Choose a step count your model supports. `derive_config` copies the original -config and applies nested changes; an unknown field raises `KeyError`. -Give the variant its own name and add it to the package's config dictionary: - -```python -MY_MODEL_CONFIGS = { - config.name: config - for config in (PIPELINE_MY_MODEL, PIPELINE_MY_MODEL_FAST) -} -``` - -The dictionary makes configs available to Python callers. Register a separate -application name to make the variant available from the CLI. - -## Add a factory - -In `apps/t2v/adapter.py`, reuse the defaults from the previous guide: - -```python -from dataclasses import replace - -from my_model.config import PIPELINE_MY_MODEL_FAST - - -def create_app_fast(): - return T2VApplication( - defaults=replace( - MY_MODEL_T2V_DEFAULTS, - pipeline_config=PIPELINE_MY_MODEL_FAST, - ) - ) -``` - -Like `create_app`, this factory takes no arguments and returns an uninitialized -application. Model loading happens in the app's `init()`. - -## Register and run the preset - -Add the new name to the existing entry-point table in `pyproject.toml`: - -```toml -[project.entry-points."flashdreams.applications_v2"] -"t2v-my-model" = "my_model.apps.t2v.adapter:create_app" -"t2v-my-model-fast" = "my_model.apps.t2v.adapter:create_app_fast" -``` - -Each CLI name selects a factory, not a key in `MY_MODEL_CONFIGS`. -Reinstall the package to update its entry points: - -```bash -uv sync --package flashdreams- --inexact -uv run --no-sync flashdreams-run-v2 --help -uv run --no-sync flashdreams-run-v2 t2v-my-model-fast --mode native-window -``` - -If a name is missing from `--help`, check that the model package is installed -and the name is registered under `flashdreams.applications_v2`. -For local experiments, the CLI can also import a module with a `create_app()` -function by replacing hyphens in the supplied name with underscores. These -module names are not listed in `--help`; use entry points for published presets. - -## Document the preset - -Add the new name, its differences, and a run command to -`docs/source/models/.md`. Link to the demo page for shared controls and -arguments. Use a unique name for each preset and keep existing names unchanged. diff --git a/docs/source/documentation/demo_api/guides/create_demo.md b/docs/source/documentation/demo_api/guides/create_demo.md index 88b5fc384..16393a984 100644 --- a/docs/source/documentation/demo_api/guides/create_demo.md +++ b/docs/source/documentation/demo_api/guides/create_demo.md @@ -5,224 +5,99 @@ title: 'Create a demo' -A demo is a model-agnostic application built using the `flashdreams.api_v2` -library. It handles user input, sessions, and display. +A demo implements `IApplication` and `ISession` from `flashdreams.api_v2`. +Reuse an app from the [demo gallery](../../../demos/index.md) when it already +supports your interaction; then [add a model adapter](integrate_model_adapter_for_a_demo.md). -Check the [demo gallery](../../../demos/index.md) first. If an existing app -supports the interaction you need, [add a model adapter](integrate_model_adapter_for_a_demo.md) -instead. +This tutorial builds the [Minimal T2V app](https://github.com/NVIDIA/flashdreams/tree/main/apps/minimal_t2v) +(`minimal_t2v`), which sends ImGui prompts to a model loop. All source below is included from the repository. -## Overview - -This example builds an app that sends a text prompt from an ImGui UI to a -text-to-video model (`StreamInferencePipeline`). - -## Package layout - -For a demo named `minimal_t2v`, add the app files and update the website: +## New application files ```text apps/minimal_t2v/ pyproject.toml minimal_t2v/ - __init__.py # export MinimalT2VApplication - application.py # MinimalT2VApplication(IApplication) - session.py # session, model loop, and model state - ui.py # ImGui text input and async message - -docs/source/ - demos/ - minimal_t2v.md # add demo page - index.md # add demo card - .nav.yml # add sidebar entry - models/ - .md # add quick start and link to demo page + __init__.py + application.py + session.py + model_loop.py + ui_loop.py + tests/test_application.py + +docs/source/demos/minimal_t2v.md # Add the app page with controls, arguments, and a link to its model's quick start. +docs/source/demos/.nav.yml # Add `Minimal T2V: minimal_t2v.md` to the sidebar. +docs/source/demos/index.md # Add a Minimal T2V gallery card linking to `minimal_t2v/`. ``` -### `pyproject.toml` - -```toml -[build-system] -requires = ["setuptools>=69", "wheel"] -build-backend = "setuptools.build_meta" - -[project] -name = "flashdreams-minimal-t2v" -version = "0.1.0" -requires-python = ">=3.10" -dependencies = ["flashdreams[local-window]"] +## Package -[tool.uv.sources] -flashdreams = { workspace = true } +Define a new demo package and export its application slug. -[tool.setuptools.packages.find] -include = ["minimal_t2v*"] +`apps/minimal_t2v/pyproject.toml`: +```toml +--8<-- "apps/minimal_t2v/pyproject.toml" ``` -### `application.py` - -The adapter supplies a pipeline and its default output description. No custom -command-line arguments are needed. - +`apps/minimal_t2v/minimal_t2v/__init__.py`: ```python -from flashdreams.api_v2.application import IApplication -from flashdreams.infra.pipeline import StreamInferencePipeline -from flashdreams.runtime_v2.session_desc import SessionDesc - -from .session import MinimalT2VSession - - -class MinimalT2VApplication(IApplication): - def __init__(self, pipeline: StreamInferencePipeline, desc: SessionDesc): - self.pipeline = pipeline - self.desc = desc +--8<-- "apps/minimal_t2v/minimal_t2v/__init__.py" +``` - def init(self, commandline_args): - parser = argparse.ArgumentParser( - prog=f"flashdreams-run-v2 {self.defaults.slug} --", - description="Generate video from an image and keyboard/mouse actions.", - ) - parser.add_argument("--default-prompt", type=str, required=True) - args = parser.parse_args(commandline_args) - self.prompt = args.prompt +## IApplication - def session_desc(self): - return self.desc +Implement an [`IApplication`](../api_reference/application.md#flashdreams.api_v2.application.IApplication) to be implemented by *every model* that desires to integrate with this demo: - def create_session(self, session_desc): - return MinimalT2VSession(self.pipeline, session_desc, self.prompt) +- `__init__` is your constructor/contract to any implementor of this demo. + - `pipeline_config` is the model's pipeline configuration. + - `desc` is our session description - more on this later. + - `total_blocks` is the model's total number of blocks to generate for the entire session. +- `init()` parses arguments & is where we load/download our model. +- `session_desc()` is how our [`ISession`](../api_reference/application.md#flashdreams.api_v2.session.ISession) collects a session description from our IApplication. + - [`SessionDesc`](../api_reference/application.md#flashdreams.runtime_v2.session_desc.SessionDesc) +- `create_session()` is called every time a session is made, passing `IApplication` data to `ISession` +- `close()` is called when the application is shutting down. +```python +--8<-- "apps/minimal_t2v/minimal_t2v/application.py" ``` -### `__init__.py` +## Session -```python -from .application import MinimalT2VApplication -``` +[`ISession`](../api_reference/application.md#flashdreams.api_v2.session.ISession) owns the two main loops driving our application *session* (running in separate threads): -### `session.py` +- [`IModelLoop`](../api_reference/application.md#flashdreams.api_v2.loop.IModelLoop): Drives model inference/logic. +- [`IUILoop`](../api_reference/application.md#flashdreams.api_v2.loop.IUILoop): Drives UI/user-interaction. -Keep the pipeline, prompt, and cache in the model-loop state: +These are set with `register_*_loop()` methods part of `ISession`. These must be called during `ISession.init()`. ```python -from dataclasses import dataclass - -from flashdreams.api_v2.loop import IModelLoop -from flashdreams.api_v2.session import ISession -from flashdreams.infra.pipeline import ( - StreamInferencePipeline, - StreamInferencePipelineCache, -) -from flashdreams.runtime_v2.session_desc import SessionDesc -from flashdreams.runtime_v2.step_result import StepResult - -from .ui import MinimalT2VUILoop, MinimalT2VUIState - - -@dataclass -class TextToVideoState: - pipeline: StreamInferencePipeline - desc: SessionDesc - prompt: str = "" - cache: StreamInferencePipelineCache | None = None - index: int = 0 - - def set_prompt(self, prompt): - self.prompt = prompt - self.cache = None - self.index = 0 - - -class TextToVideoLoop(IModelLoop[TextToVideoState]): - def step(self, step_index, events): - state = self.state - if not state.prompt: - return [] - if state.cache is None: - state.cache = state.pipeline.initialize_cache(text=[state.prompt]) - frames = state.pipeline.generate(state.index, state.cache) - state.pipeline.finalize(state.index, state.cache) - state.index += 1 - return [StepResult( - step_index=step_index, - output=frames, - frame_count=frames.shape[0], - output_layout=state.desc.output_layout, - )] - - -class MinimalT2VSession(ISession): - def __init__(self, pipeline: StreamInferencePipeline, desc: SessionDesc, prompt: str): - self.pipeline = pipeline - self.desc = desc - self.prompt = prompt - - @property - def session_desc(self): - return self.desc - - def init(self): - model_loop = self.register_model_loop( - TextToVideoLoop, - state=TextToVideoState(self.pipeline, self.desc), - ) - self.register_ui_loop( - MinimalT2VUILoop, - state=MinimalT2VUIState(model_loop), - width=self.desc.video_width, - height=self.desc.video_height, - ) - model_loop.state.set_prompt(self.prompt) +--8<-- "apps/minimal_t2v/minimal_t2v/session.py" ``` -Use a `SessionDesc` matching the pipeline's output size, frame rate, and layout. -This example expects frames in `tchw` layout. +## Model loop + +`IModelLoop` governs all model logic and will be where we run our model inference: -### `ui.py` +- `MinimalT2VModelState` is the type of `ILoop.state` for our `IModelLoop`. +- `step()` is called for each loop of our `IModelLoop`. + - `pipeline.initialize_cache()` is called to initialize our model's cache. + - `pipeline.generate()` is called to generate a chunk of frames given our current model's state (`pipeline`). + - `pipeline.finalize()` is called to finalize the generated chunk & produce metrics. +- `reset()` is called when the session is reset. +- `close()` is called when the session is closed. ```python -from dataclasses import dataclass - -from flashdreams.api_v2.loop import IModelLoop, invoke_async -from flashdreams.runtime_v2.imgui_ui_loop import ImGuiUILoop - - -@dataclass -class MinimalT2VUIState: - model_loop: IModelLoop - text: str = "" - - -class MinimalT2VUILoop(ImGuiUILoop[MinimalT2VUIState]): - def step_ui(self, imgui, step_index, events): - imgui.begin("Minimal T2V") - try: - _, self.state.text = imgui.input_text("Prompt", self.state.text) - if imgui.button("Send"): - invoke_async( - self.state.model_loop, - lambda state, text=self.state.text: state.set_prompt(text), - ) - finally: - imgui.end() - return self.presented_model_frame() +--8<-- "apps/minimal_t2v/minimal_t2v/model_loop.py" ``` -`invoke_async` runs the callback on the model thread before its next step. -The UI keeps its own editable text; it never writes model state directly. +## UI loop -The model loop waits for a prompt. Each `set_prompt` sets our pipeline cache to `None` to force our generation of `StreamInferencePipeline` to effectively reset (fill the cahce with the new prompt). +`IUILoop` governs all UI logic and will be where we handle user-interaction: -The [API reference](../api_reference/application.md) here describes our interfaces in more detail. +- `MinimalT2VUIState` is the type of `ILoop.state` for our `IUILoop`. +- `invoke_async(self.state.model_loop, lambda...)` is called when the user submits a new prompt (via imgui button) to submit a `lambda` to run on our `IModelLoop` thread before the next new `IModelLoop.step()`. +- `step_ui(...)` is called for each loop of our `IUILoop`. It contains `imgui` state when `IUILoop` inherits from [`ImGuiUILoop`](../api_reference/application.md#flashdreams.runtime_v2.imgui_ui_loop.ImGuiUILoop). -## Connect a model - -[Add a model adapter](integrate_model_adapter_for_a_demo.md) to supply the pipeline and register -an application slug. The adapter would live in -`integrations_v2//apps/minimal_t2v/adapter.py`. -Run its registered slug with the standard format of: - -```bash -uv run flashdreams-run-v2 minimal-t2v- --mode native-window -``` - -Use the [T2V demo](../../../demos/t2v.md) as feature complete examples. +```python +--8<-- "apps/minimal_t2v/minimal_t2v/ui_loop.py" +``` \ No newline at end of file diff --git a/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md index 7c3b1440f..8b59bbfc6 100644 --- a/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md +++ b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md @@ -5,119 +5,81 @@ title: 'Integrate model adapter for a demo' -An adapter connects a model to an existing demo. It supplies the model settings -and registers a name for `flashdreams-run-v2`. +An adapter is a package that contains the logic to connect a demo to a model. -## Overview - -This example connects a model pipeline config to the T2V app. The app handles -the prompt, sessions, and UI; the adapter selects the model and output defaults. +In this example we connect the [Minimal T2V application](create_demo.md) to Wan 2.1. -## Choose a demo - -Pick an app from the [demo gallery](../../../demos/index.md) and check its -inputs. T2V accepts a pipeline config; V2V accepts a processor config. -If you need a new interaction, [create a demo](create_demo.md) first. + ## Files to add or update +After creating `apps/minimal_t2v/`, add the adapter package and update the website: + ```text -integrations_v2// - apps/t2v/ - __init__.py # add an empty file +integrations_v2/wan21/ + config.py # add a new pipeline config if desired + apps/minimal_t2v/ + __init__.py # add the package initializer adapter.py # add the application factory - pyproject.toml # add the app dependency and CLI name - -docs/source/models/.md # add a quick start and presets -docs/source/demos/t2v.md # link to the model's quick start + pyproject.toml # update dependencies, packages, and entry points + +docs/source/ + models/ + wan21.md # update: add Minimal T2V quick start, preset, and app-page link + .nav.yml # update when adding a new model page + index.md # add a card when adding a new model page + demos/ + minimal_t2v.md # update: link to the model's quick start ``` -## Add the adapter - -In `apps/t2v/adapter.py`: +## Create the adapter entry point -```python -from t2v import T2VApplication, T2VApplicationDefaults +Our adapter entry point is a factory function that creates an instance of our application based on the `IApplication` we defined. -from my_model.config import PIPELINE_MY_MODEL +This factory has the contract of: -MY_MODEL_T2V_DEFAULTS = T2VApplicationDefaults( - pipeline_config=PIPELINE_MY_MODEL, - total_blocks=8, - pixel_width=832, - pixel_height=480, - fps=16, -) +```python +def some_create_app() -> IApplication: + ... +``` +In our case we then make in `integrations_v2/wan21/apps/minimal_t2v/adapter.py`: -def create_app(): - return T2VApplication(defaults=MY_MODEL_T2V_DEFAULTS) +```python +--8<-- "integrations_v2/wan21/apps/minimal_t2v/adapter.py" ``` -Set the dimensions, frame rate, and block count for your model. -The factory returns an uninitialized app. Load weights and initialize CUDA in -`IApplication.init()`, so listing apps and showing help do not load a model. - -For V2V, supply `V2VApplicationDefaults` with a processor, first and subsequent -chunk sizes, and a model name. See the -[SwiftVR adapter](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/swiftvr/apps/v2v/adapter.py) -or [FlashVSR adapter](https://github.com/NVIDIA/flashdreams/blob/main/integrations_v2/flashvsr/apps/v2v/adapter.py). +## Register the application -## Register the CLI name +In `integrations_v2/wan21/pyproject.toml`, add our application's package name to `tool.uv.sources` -Add `flashdreams-t2v` to the integration's existing dependencies and workspace -sources in `pyproject.toml`: +Then add our `slug` to run our new application entry point to `[project.entry-points."flashdreams.applications_v2"]` +This is seen in `integrations_v2/wan21/pyproject.toml`: ```toml -# Add to [project].dependencies: -# "flashdreams-t2v" - -[tool.uv.sources] -flashdreams-t2v = { workspace = true } - -[project.entry-points."flashdreams.applications_v2"] -"t2v-my-model" = "my_model.apps.t2v.adapter:create_app" +--8<-- "integrations_v2/wan21/pyproject.toml" ``` -If the package lists modules explicitly in `[tool.setuptools].packages`, -include `.apps` and `.apps.t2v`. - -The name `t2v-my-model` is the application slug. Use a lowercase name such as -`-`. Each entry points to a factory callable without arguments. -Use `flashdreams.applications_v2` for apps; `flashdreams.postprocess_presets` -registers processors separately. - -## Install and run +## Install and run entry point ```bash -uv sync --package flashdreams- --inexact -uv run --no-sync flashdreams-run-v2 --help -uv run --no-sync flashdreams-run-v2 t2v-my-model --mode native-window +uv sync --package flashdreams-wan21 --inexact +uv run --no-sync flashdreams-run-v2 minimal-t2v-wan21 --mode native-window -- --default-prompt "A cat surfing" ``` +## Adding a new pipeline configuration to pick from -`--help` lists installed application names. For this T2V app's arguments, run -`flashdreams-run-v2 t2v-my-model -- --help`. -Runtime options go before `--`; demo options go after it. - - +Modify `integrations_v2/wan21/config.py` to add a new pipeline config to `wan21`: -## Document the adapter - -Add installation instructions, a run command, and the registered preset to -`docs/source/models/.md`. Link to the demo page for controls and -arguments. Add a link back to the model's quick start on the demo page. -For a new model, add it to `docs/source/models/index.md` and -`docs/source/models/.nav.yml` too. - - +```python +PIPELINE_WAN21_T2V_SEED123 = derive_config( + PIPELINE_WAN21_T2V_1PT3B_480P, + name="wan21-t2v-1.3b-480p-seed123", + diffusion_model={"seed": 123}, +) +``` -The app stays in `apps//`. Model code and pipeline configs stay in -`integrations_v2//`; the adapter imports both. Browser and native-window -support come from the runtime. No model-specific copy of the app or `runner.py` -is needed. +This defines a new pipeline config that is a copy of `PIPELINE_WAN21_T2V_1PT3B_480P` with the addition of a `seed` parameter to the diffusion model, setting the pipeline to always run diffusion with the same seed of `123`. -Continue with [Adding a new configuration](adding_a_new_configuration.md) to add another -preset to this adapter, or [Interactive serving](interactive_serving.md) to -run it in a browser. +To learn more about demo serving, see [Interactive serving](interactive_serving.md). \ No newline at end of file diff --git a/docs/source/documentation/demo_api/index.md b/docs/source/documentation/demo_api/index.md index 6c7fa5bd2..fd4fea51e 100644 --- a/docs/source/documentation/demo_api/index.md +++ b/docs/source/documentation/demo_api/index.md @@ -9,14 +9,11 @@ Use the Demo API to build user-facing applications, manage sessions, expose controls, route inputs and outputs, configure a model-backed demo, and launch it through `flashdreams-run-v2`. -The current application protocol is `flashdreams.api_v2`. - ## Guides
Create a demoBuild a model-independent application against the Demo API. Integrate model adapter for a demoConnect a model to an existing demo. - Adding a new configurationAdd and register another model preset. Interactive servingRun a demo in a browser or local window. Profiling a demoCollect Nsight Systems traces and per-step performance metrics.
diff --git a/docs/source/models/wan21.md b/docs/source/models/wan21.md index 6e46f26ab..b9b57036b 100644 --- a/docs/source/models/wan21.md +++ b/docs/source/models/wan21.md @@ -35,10 +35,22 @@ uv run --no-sync flashdreams-run-v2 \ ```
- Demo Presets + T2V Presets Demo Arguments
+### Minimal T2V + +```bash +uv sync --package flashdreams-wan21 --inexact +uv run --no-sync flashdreams-run-v2 minimal-t2v-wan21 --mode native-window -- --default-prompt "A cat surfing" +``` + +
+ Minimal T2V Presets + Demo Arguments +
+
## Demo Presets @@ -51,6 +63,14 @@ uv run --no-sync flashdreams-run-v2 \ | --- | --- | | `t2v-wan21-t2v-1.3b-480p` | Wan 2.1 T2V 1.3B at 480p with a single bidirectional block. | + + +### Minimal T2V + +| Preset | Description | +| --- | --- | +| `minimal-t2v-wan21` | [Minimal T2V app](../demos/minimal_t2v.md) with Wan 2.1 T2V 1.3B at 480p, generating one block per prompt. | +
## Developer Details diff --git a/docs/zensical.toml b/docs/zensical.toml index ebb22a92b..ba55481b6 100644 --- a/docs/zensical.toml +++ b/docs/zensical.toml @@ -45,6 +45,7 @@ md_in_html = {} tables = {} pymdownx.details = {} pymdownx.superfences = {} +pymdownx.snippets = { base_path = ["."], check_paths = true } pymdownx.arithmatex.generic = true [project.plugins.awesome-nav] diff --git a/integrations_v2/wan21/apps/minimal_t2v/__init__.py b/integrations_v2/wan21/apps/minimal_t2v/__init__.py new file mode 100644 index 000000000..52a7a9daf --- /dev/null +++ b/integrations_v2/wan21/apps/minimal_t2v/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/integrations_v2/wan21/apps/minimal_t2v/adapter.py b/integrations_v2/wan21/apps/minimal_t2v/adapter.py new file mode 100644 index 000000000..236307d0d --- /dev/null +++ b/integrations_v2/wan21/apps/minimal_t2v/adapter.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Wan 2.1 adapter for the Minimal T2V application.""" + +from minimal_t2v import MinimalT2VApplication + +from flashdreams.runtime_v2.session_desc import SessionDesc +from wan21.config import PIPELINE_WAN21_T2V_1PT3B_480P + + +def create_app() -> MinimalT2VApplication: + """Return Minimal T2V with Wan 2.1's single-block pipeline.""" + return MinimalT2VApplication( + pipeline_config=PIPELINE_WAN21_T2V_1PT3B_480P, + desc=SessionDesc( + video_width=832, + video_height=480, + frames_per_second_for_step=16, + ), + total_blocks=1, + ) diff --git a/integrations_v2/wan21/pyproject.toml b/integrations_v2/wan21/pyproject.toml index 2881c915f..d5977188c 100644 --- a/integrations_v2/wan21/pyproject.toml +++ b/integrations_v2/wan21/pyproject.toml @@ -16,10 +16,12 @@ dependencies = [ "mediapy>=1.1", "opencv-python-headless>=4.5", "flashdreams-t2v", + "flashdreams-minimal-t2v", ] [tool.uv.sources] flashdreams-t2v = { workspace = true } +flashdreams-minimal-t2v = { workspace = true } flashdreams = { workspace = true } [project.optional-dependencies] @@ -29,9 +31,10 @@ dev = [ [project.entry-points."flashdreams.applications_v2"] "t2v-wan21-t2v-1.3b-480p" = "wan21.apps.t2v.adapter:create_app" +"minimal-t2v-wan21" = "wan21.apps.minimal_t2v.adapter:create_app" [tool.setuptools] -packages = ["wan21", "wan21.apps", "wan21.apps.t2v", "wan21.impl"] +packages = ["wan21", "wan21.apps", "wan21.apps.t2v", "wan21.apps.minimal_t2v", "wan21.impl"] [tool.setuptools.package-dir] wan21 = "." diff --git a/integrations_v2/wan21/tests/test_minimal_t2v.py b/integrations_v2/wan21/tests/test_minimal_t2v.py new file mode 100644 index 000000000..6a79ca819 --- /dev/null +++ b/integrations_v2/wan21/tests/test_minimal_t2v.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU check for the separately packaged Minimal T2V adapter.""" + +from importlib.metadata import EntryPoint +from pathlib import Path +from unittest.mock import Mock + +import pytest +import tomli +import torch +from minimal_t2v import MinimalT2VApplication +from wan21.config import PIPELINE_WAN21_T2V_1PT3B_480P + +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +pytestmark = pytest.mark.ci_cpu + + +def test_registered_minimal_t2v_adapter() -> None: + metadata = tomli.loads((Path(__file__).parents[1] / "pyproject.toml").read_text()) + assert "wan21.apps.minimal_t2v" in metadata["tool"]["setuptools"]["packages"] + target = metadata["project"]["entry-points"]["flashdreams.applications_v2"][ + "minimal-t2v-wan21" + ] + assert target == "wan21.apps.minimal_t2v.adapter:create_app" + app = EntryPoint( + name="minimal-t2v-wan21", value=target, group="flashdreams.applications_v2" + ).load()() + assert isinstance(app, MinimalT2VApplication) + assert app.pipeline is None + assert app.pipeline_config is PIPELINE_WAN21_T2V_1PT3B_480P + assert app.total_blocks == 1 + desc = app.session_desc() + assert (desc.video_width, desc.video_height) == (832, 480) + assert desc.frames_per_second_for_step == 16 + + pipeline = Mock() + pipeline.to.return_value = pipeline + pipeline.eval.return_value = pipeline + pipeline.decoder.spatial_compression_ratio = 8 + pipeline.generate.return_value = torch.zeros(1, 3, 480, 832) + pipeline.finalize.return_value = {} + app.pipeline_config = Mock() + app.pipeline_config.setup.return_value = pipeline + app.init(["--default-prompt", "A cat surfing", "--device", "cpu"]) + session = app.create_session(desc) + session.init() + result = session.model_loop.step(0, UserInputEvents([]))[0] + pipeline.initialize_cache.assert_called_once_with( + text=["A cat surfing"], height=60, width=104 + ) + assert result.read_output().shape == (1, 3, 480, 832) + assert session.model_loop.step(1, UserInputEvents([])) == [] + session.model_loop.close() + app.close() diff --git a/pyproject.toml b/pyproject.toml index 93dcfe516..c6802a8ce 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,7 @@ extraPaths = [ "apps", "apps/cam2v", "apps/t2v", + "apps/minimal_t2v", "apps/crazy_robotaxi", "apps/interactive_drive", "apps/omnidreams_game_engine", @@ -98,6 +99,7 @@ extra-paths = [ "apps", "apps/cam2v", "apps/t2v", + "apps/minimal_t2v", "apps/action2v", "apps/crazy_robotaxi", "apps/interactive_drive", diff --git a/tests/test_docs_links.py b/tests/test_docs_links.py index af1184d9e..2a155215f 100644 --- a/tests/test_docs_links.py +++ b/tests/test_docs_links.py @@ -13,10 +13,9 @@ _DOCS_SOURCE = Path(__file__).parents[1] / "docs" / "source" _REPO_ROOT = _DOCS_SOURCE.parents[1] -_MARKDOWN_LINK_RE = re.compile(r"\[[^]]+\]\(([^)#]+\.md)#([a-z0-9-]+)\)") +_MARKDOWN_LINK_RE = re.compile(r"\[[^]]*\]\(([^)#]+)(?:#([a-z0-9-]+))?\)") _CUSTOM_ANCHOR_RE = re.compile(r' None assert page.is_file(), ( f"{source.relative_to(_REPO_ROOT)}: missing canonical page: {target}" ) - assert fragment in _anchors(page), ( + assert not fragment or fragment in _anchors(page), ( f"{source.relative_to(_REPO_ROOT)}: missing section #{fragment} in " f"{page.relative_to(_REPO_ROOT)}" ) @@ -88,18 +87,12 @@ def test_contributing_stays_on_github() -> None: "Submitting a pull request", "Adding documentation", } <= headings - assert "Adding an Agent Skill" in headings assert not (_DOCS_SOURCE / "community/contributing.md").exists() assert _CONTRIBUTING_URL in (_DOCS_SOURCE / "community/index.md").read_text() def test_repository_links_resolve_to_canonical_sections() -> None: - sources = [_REPO_ROOT / relative for relative in _CANONICAL_POINTERS] - for source in sources: - assert len(source.read_text(encoding="utf-8").splitlines()) == 1, ( - f"{source.relative_to(_REPO_ROOT)} must be one canonical link" - ) - sources.append(_REPO_ROOT / "tests/README.md") + sources = [_REPO_ROOT / "README.md", _REPO_ROOT / "tests/README.md"] sources.extend(sorted((_REPO_ROOT / "skills").glob("*/SKILL.md"))) found = 0 @@ -108,10 +101,32 @@ def test_repository_links_resolve_to_canonical_sections() -> None: source.read_text(encoding="utf-8") ): if target == _CONTRIBUTING_URL: - assert fragment in _anchors(_REPO_ROOT / "CONTRIBUTING.md") + assert not fragment or fragment in _anchors( + _REPO_ROOT / "CONTRIBUTING.md" + ) + found += 1 + continue + website = "https://nvidia.github.io/flashdreams/main/" + if target.startswith(website): + route = target.removeprefix(website).rstrip("/") + page = _DOCS_SOURCE / route + assert page.suffix != ".md", ( + f"Website link must use a rendered route: {target}" + ) + if page.suffix == ".html": + page = page.with_suffix(".md") + elif page.is_dir(): + page = page / "index.md" + elif not page.suffix: + page = page.with_suffix(".md") + _assert_local_section_link(source, str(page), fragment) found += 1 continue + if urlsplit(target).scheme: + continue page = (source.parent / target).resolve() + if source.name == "README.md": + assert page.exists(), f"Missing README link: {target}" if page.is_relative_to(_DOCS_SOURCE): _assert_local_section_link(source, target, fragment) found += 1 @@ -119,6 +134,15 @@ def test_repository_links_resolve_to_canonical_sections() -> None: assert found, "expected canonical links from repository guidance to docs/source" +def test_readme_reference_links_have_definitions() -> None: + source = (_REPO_ROOT / "README.md").read_text(encoding="utf-8") + references = set(re.findall(r"\[[^]]+\]\[([^]]+)\]", source)) + definitions = set(re.findall(r"^\[([^]]+)\]:", source, re.MULTILINE)) + assert references <= definitions, ( + f"Undefined README links: {references - definitions}" + ) + + def test_demo_guide_navigation_matches_pages_and_cards() -> None: navigation = (_DOCS_SOURCE / ".nav.yml").read_text(encoding="utf-8") overview = (_DOCS_SOURCE / "documentation/demo_api/index.md").read_text( diff --git a/uv.lock b/uv.lock index 9a3f5e973..91b21b3e0 100644 --- a/uv.lock +++ b/uv.lock @@ -31,6 +31,7 @@ members = [ "flashdreams-imgui-ui-demo", "flashdreams-interactive-drive-v2", "flashdreams-lingbot", + "flashdreams-minimal-t2v", "flashdreams-null-model", "flashdreams-omnidreams", "flashdreams-red-screen", @@ -1306,6 +1307,17 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "flashdreams-minimal-t2v" +version = "0.1.0" +source = { editable = "apps/minimal_t2v" } +dependencies = [ + { name = "flashdreams", extra = ["local-window"] }, +] + +[package.metadata] +requires-dist = [{ name = "flashdreams", extras = ["local-window"], editable = "flashdreams" }] + [[package]] name = "flashdreams-null-model" version = "0.1.0" @@ -1565,6 +1577,7 @@ version = "0.1.0" source = { editable = "integrations_v2/wan21" } dependencies = [ { name = "flashdreams" }, + { name = "flashdreams-minimal-t2v" }, { name = "flashdreams-t2v" }, { name = "mediapy" }, { name = "opencv-python-headless" }, @@ -1578,6 +1591,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "flashdreams", editable = "flashdreams" }, + { name = "flashdreams-minimal-t2v", editable = "apps/minimal_t2v" }, { name = "flashdreams-t2v", editable = "apps/t2v" }, { name = "mediapy", specifier = ">=1.1" }, { name = "opencv-python-headless", specifier = ">=4.5" }, @@ -1840,47 +1854,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/e5/0ae74c83c1cab2c14daadf28bd0143eb14bb81503834af987b66badc7b61/griffelib-2.3.2-py3-none-any.whl", hash = "sha256:8e710afededd5607f95bf3c8ccc175ee306e7084459faf27b9f9f44ef2a7a274", size = 169420, upload-time = "2026-10-06T09:54:32.038Z" }, ] -[[package]] -name = "grpcio" -version = "1.81.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b0/b5/1ff353970a87eda4c98251e34d2dfd214abd4982dc89119c9252a2a482d2/grpcio-1.81.1.tar.gz", hash = "sha256:6fa10a767143a5e82e8eaab53918af0cd8909a57a27f8cb2288b80a613ac671b", size = 13026582, upload-time = "2026-06-11T12:46:51.673Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/9a/d5/f2b159d8eec08be2a855ef698f5b6f7f9fdda022e4dd9e4f5d968affd678/grpcio-1.81.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:6f9a0c9c1cc15c112d1c053064fd032b64917062292c3d70aea280e02ae10b77", size = 6086868, upload-time = "2026-06-11T12:44:19.364Z" }, - { url = "https://files.pythonhosted.org/packages/80/41/9c95232b94b219ed8b14029d9cd000e0381cafba869c451dda60af84f4ba/grpcio-1.81.1-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:69ef28e54fc85397f91b8c19592b8ef3d81952080366914823bd8572a2958120", size = 12062291, upload-time = "2026-06-11T12:44:27.142Z" }, - { url = "https://files.pythonhosted.org/packages/83/8b/bd9284bdd665ddf877a3e8bc2930d1bcf6ebdbae7b0da5c783dc26bd6e33/grpcio-1.81.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:15641444eca4a29358107b3dceb74c1c6305c55c822fd199b458aaea4068a7fb", size = 6635242, upload-time = "2026-06-11T12:44:30.741Z" }, - { url = "https://files.pythonhosted.org/packages/60/24/78fa025517a925f1a17da71c4ef9d5f1c6f9fa65af22dfb523c5c6317a21/grpcio-1.81.1-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:d4b2dddfc219f54f956ccd53cf76a1d338ffe68fc7f2849ec9c7feb9927ff692", size = 7332974, upload-time = "2026-06-11T12:44:33.72Z" }, - { url = "https://files.pythonhosted.org/packages/f7/11/402295b388dd35861007f8a26a37c2e2f284212d57bdf407c31f36043746/grpcio-1.81.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ca1cc11d82677b9662082e5478b7528e2b7db7beaa6bdff42bd62789d81be399", size = 6836597, upload-time = "2026-06-11T12:44:36.108Z" }, - { url = "https://files.pythonhosted.org/packages/4d/71/37b10fd4fd579ffade6e695c14e9df5e8cba9e2365b81c131da438b67c34/grpcio-1.81.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:aa2ba7d2ad6df4d80127cea65e5b8d5e2c3adbf153ff4804452836328aca7c54", size = 7440660, upload-time = "2026-06-11T12:44:38.664Z" }, - { url = "https://files.pythonhosted.org/packages/4a/d5/40203f828abc83d458b634666df6df13778032f178c03845ad5a93682388/grpcio-1.81.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:592b5fee597faa91cce2dd294dd7d9a1c83d76c4dbf877e33ec1adb866b2fbed", size = 8443171, upload-time = "2026-06-11T12:44:41.678Z" }, - { url = "https://files.pythonhosted.org/packages/eb/2c/0ed82ea35b5ec595e10444940c1db8c0e0ef57aa46bc8797d5ff838a219e/grpcio-1.81.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:62481553b1793a27e9b9c3cf9e5bd483ef045ca72462592074b46d42b0c4d9b9", size = 7868905, upload-time = "2026-06-11T12:44:44.854Z" }, - { url = "https://files.pythonhosted.org/packages/9e/1f/dcbdc1a68a07cc2b631c3098953794f17d75f93426a019240b90ce5423d6/grpcio-1.81.1-cp310-cp310-win32.whl", hash = "sha256:bb693b1e3d9a2f3fd228e2110daf4b5aeedb36761ca1e4282f74725f6d89f611", size = 4202215, upload-time = "2026-06-11T12:44:47.165Z" }, - { url = "https://files.pythonhosted.org/packages/75/a1/d7ab9f1f42efcb7d9e6111d38be6b367737a72ea2c534e1f55c81e1b6436/grpcio-1.81.1-cp310-cp310-win_amd64.whl", hash = "sha256:88268ca418cacea64cecb0d1d600d3c6b3a8038fcba02e1e205178c5b1f47661", size = 4936582, upload-time = "2026-06-11T12:44:49.479Z" }, - { url = "https://files.pythonhosted.org/packages/52/ea/1c2fa386b718ff493225e61cfc052ef400b4d6ffc54cbe261026432624b5/grpcio-1.81.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:d71d30f2d92f67d944631c523713934fee37292469e182ebcd2c1dd8a64ce53f", size = 6093112, upload-time = "2026-06-11T12:44:52.131Z" }, - { url = "https://files.pythonhosted.org/packages/2b/18/acf45fa8bd1bc5d7b0c2fd3dc4c209379fbd5bb396b440b68a83342226b7/grpcio-1.81.1-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b137f4bf3ada9dc44d411478decc6ff09a79ed30b306cd2abaa98408c3588137", size = 12074277, upload-time = "2026-06-11T12:44:55.354Z" }, - { url = "https://files.pythonhosted.org/packages/48/d7/ee86a60699b7db039f772a2c4a7e4facc7138984ff42c0130933a0063884/grpcio-1.81.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a3acb384427816dd5d470f47e62137b87f74da694faa8a50147012cf40df276a", size = 6640348, upload-time = "2026-06-11T12:44:59.223Z" }, - { url = "https://files.pythonhosted.org/packages/26/ee/d2de5e47378ffc207d476c230fea3be4d2601edbce9995f4fe45535d4896/grpcio-1.81.1-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:f9a0ebbe45c29b5e5866593c12b78bd9035f0f0f0d4bc8361680cd580d99db49", size = 7331842, upload-time = "2026-06-11T12:45:02.001Z" }, - { url = "https://files.pythonhosted.org/packages/23/d6/abeda5c2b896a0b341584fe5ac411bbf72e197a9a374c355fb90965e08d2/grpcio-1.81.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a37165cc80b1a368384b383e63a4c38116a10467ae44c904d2d7468c4470ec2", size = 6842229, upload-time = "2026-06-11T12:45:04.76Z" }, - { url = "https://files.pythonhosted.org/packages/10/1c/1f0da7d590b4aeee006826ba568d0e419ca14b23e18f901a3da3e9fba613/grpcio-1.81.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:6282caffb41ec326d4cb67ca9cf53b739d1b2f975a2acb498c7418e9f7d9a416", size = 7446096, upload-time = "2026-06-11T12:45:07.499Z" }, - { url = "https://files.pythonhosted.org/packages/6a/81/5c505d508f7c887aa7982d21443a4126597c80d34b0bcf40f9cec576d7f3/grpcio-1.81.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:a35009284d0d3d5c2c9601c164a911b8b4331608d98a9a66d47d97bb2f522b70", size = 8445238, upload-time = "2026-06-11T12:45:10.243Z" }, - { url = "https://files.pythonhosted.org/packages/f7/b2/524847365122ee509ca17bcc4e092198b700e94af7bfd5bb5e6dd9f3ee66/grpcio-1.81.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1b22c80559854b789a01fd89e8929b3798a156c0829b5282a8939f33ad4115ad", size = 7873989, upload-time = "2026-06-11T12:45:13.102Z" }, - { url = "https://files.pythonhosted.org/packages/18/fa/07c037c50b006909d1d13a5848774f8aa7b242f70dc03a035c64eea0e6db/grpcio-1.81.1-cp311-cp311-win32.whl", hash = "sha256:428bec0161b48d8cf583c068591bc0016d0d9cfff52462b72b3884861ea768c5", size = 4202223, upload-time = "2026-06-11T12:45:16.166Z" }, - { url = "https://files.pythonhosted.org/packages/41/ed/6bff15376920942fac6b95b9802752b837437172c9e8fc2d3170546b89cc/grpcio-1.81.1-cp311-cp311-win_amd64.whl", hash = "sha256:30e825f6848d9f18bba350ed6c75c1b02a0b5184474a31db9a32b1fa66fd8c79", size = 4941303, upload-time = "2026-06-11T12:45:18.724Z" }, - { url = "https://files.pythonhosted.org/packages/85/07/9a979c81738863a738dc23d65177056e71fbb2db817740ed870b33434e7a/grpcio-1.81.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:8b39472beafc0bdcafc4c8c73ad082ebfdb449d566897a61e7acb4fa88089115", size = 6053264, upload-time = "2026-06-11T12:45:21.017Z" }, - { url = "https://files.pythonhosted.org/packages/75/95/539706ca0d3bd40dbad583dc56fd883da941f37556b629132da5762781b9/grpcio-1.81.1-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:12b7524c88d4026d3dcb7b0ebe16b6714f3b4af402ddd0f0639ab064a00c87c3", size = 12052560, upload-time = "2026-06-11T12:45:23.652Z" }, - { url = "https://files.pythonhosted.org/packages/e0/44/f257b7e0bd69c93b06c6cb8ac8d1b901ccb42bedabd83c1a4c77a71f8810/grpcio-1.81.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1e123f9b37edb8375fd74130d1f69c944bbf0a7b06761ae7211154b8759e94d2", size = 6595983, upload-time = "2026-06-11T12:45:26.963Z" }, - { url = "https://files.pythonhosted.org/packages/b9/f3/19782aa04c960968bef8c5539329d8e3bbc3364e2e46d19eb5e5cc5e43b7/grpcio-1.81.1-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:2c2e2ae6867c2966b8daccc836d54a13218e0007e9a490aeb81dd05be64d22d7", size = 7303455, upload-time = "2026-06-11T12:45:29.707Z" }, - { url = "https://files.pythonhosted.org/packages/eb/8c/dea020b6d91508cd84463917a63149ec196ee7db505d032ae43fcb3303b9/grpcio-1.81.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:766bc7c9a9c340342f4c864ccbda8e78111e4751f13b895812b9c148fb79e9d0", size = 6809167, upload-time = "2026-06-11T12:45:32.52Z" }, - { url = "https://files.pythonhosted.org/packages/1c/c7/3030dd940408083bd32cd95d634777a71605ade4887154d93e8a89244946/grpcio-1.81.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b259a04a737cb3496be0901328eb8b7552ed8df4865d8c8f1cf1bffcfc0776a3", size = 7412536, upload-time = "2026-06-11T12:45:35.403Z" }, - { url = "https://files.pythonhosted.org/packages/e0/dd/1172a9e42b168edcafefad6115346ef619a3fc02158bb170e66ced24bcdd/grpcio-1.81.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:85b10a45b8993d195c4f3ff57025b8d1e11834909ee475c403bfa60cb4caefaf", size = 8408276, upload-time = "2026-06-11T12:45:37.78Z" }, - { url = "https://files.pythonhosted.org/packages/25/7a/71437c7f3596e5246155c515852795a85a1a8d228190212432b13b97a95d/grpcio-1.81.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8ea1936c26b99999b27479853039a7f34713f56c49375ad52b38535ec93a796c", size = 7849660, upload-time = "2026-06-11T12:45:40.627Z" }, - { url = "https://files.pythonhosted.org/packages/65/40/7debc0da45d2efebafb82da75644be347497fe4ee250514b8cd3b86ae8bf/grpcio-1.81.1-cp312-cp312-win32.whl", hash = "sha256:a185a04039df6cae8648bc8ab6d6fde7bf94f7188ecf7828e76ac52eef1e41d6", size = 4185819, upload-time = "2026-06-11T12:45:43.027Z" }, - { url = "https://files.pythonhosted.org/packages/2e/b9/8fe3ba5ed462067774ebc1f9c7f26aa7ebcc280ddd476be107153de1339e/grpcio-1.81.1-cp312-cp312-win_amd64.whl", hash = "sha256:3ad74f8bb1a18963914c5452d289422830b39459e8776ebbcd207be1fbfb1d94", size = 4930461, upload-time = "2026-06-11T12:45:45.775Z" }, -] - [[package]] name = "h11" version = "0.16.0" From b57dfbff47532273b0779b164306c601107722d9 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Mon, 19 Oct 2026 21:38:02 +0000 Subject: [PATCH 11/14] fix formatting --- .../demo_api/api_reference/application.md | 2 +- .../demo_api/guides/create_demo.md | 4 +- .../integrate_model_adapter_for_a_demo.md | 4 +- .../inferencing_api/guides/create_model.md | 42 ++++--------------- .../flashdreams/accelerated/__init__.py | 4 +- .../accelerated/quantization/__init__.py | 4 +- 6 files changed, 17 insertions(+), 43 deletions(-) diff --git a/docs/source/documentation/demo_api/api_reference/application.md b/docs/source/documentation/demo_api/api_reference/application.md index 0c2e03ed0..83fa0e31e 100644 --- a/docs/source/documentation/demo_api/api_reference/application.md +++ b/docs/source/documentation/demo_api/api_reference/application.md @@ -30,4 +30,4 @@ workflows, start with [Create a demo](../guides/create_demo.md) or ::: flashdreams.runtime_v2.session_desc.BackpressureMode -::: flashdreams.runtime_v2.session_desc.PresentationMode \ No newline at end of file +::: flashdreams.runtime_v2.session_desc.PresentationMode diff --git a/docs/source/documentation/demo_api/guides/create_demo.md b/docs/source/documentation/demo_api/guides/create_demo.md index 16393a984..1b8cc1b0c 100644 --- a/docs/source/documentation/demo_api/guides/create_demo.md +++ b/docs/source/documentation/demo_api/guides/create_demo.md @@ -95,9 +95,9 @@ These are set with `register_*_loop()` methods part of `ISession`. These must be `IUILoop` governs all UI logic and will be where we handle user-interaction: - `MinimalT2VUIState` is the type of `ILoop.state` for our `IUILoop`. -- `invoke_async(self.state.model_loop, lambda...)` is called when the user submits a new prompt (via imgui button) to submit a `lambda` to run on our `IModelLoop` thread before the next new `IModelLoop.step()`. +- `invoke_async(self.state.model_loop, lambda...)` is called when the user submits a new prompt (via imgui button) to submit a `lambda` to run on our `IModelLoop` thread before the next new `IModelLoop.step()`. - `step_ui(...)` is called for each loop of our `IUILoop`. It contains `imgui` state when `IUILoop` inherits from [`ImGuiUILoop`](../api_reference/application.md#flashdreams.runtime_v2.imgui_ui_loop.ImGuiUILoop). ```python --8<-- "apps/minimal_t2v/minimal_t2v/ui_loop.py" -``` \ No newline at end of file +``` diff --git a/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md index 8b59bbfc6..f225bf65f 100644 --- a/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md +++ b/docs/source/documentation/demo_api/guides/integrate_model_adapter_for_a_demo.md @@ -5,7 +5,7 @@ title: 'Integrate model adapter for a demo' -An adapter is a package that contains the logic to connect a demo to a model. +An adapter is a package that contains the logic to connect a demo to a model. In this example we connect the [Minimal T2V application](create_demo.md) to Wan 2.1. @@ -82,4 +82,4 @@ PIPELINE_WAN21_T2V_SEED123 = derive_config( This defines a new pipeline config that is a copy of `PIPELINE_WAN21_T2V_1PT3B_480P` with the addition of a `seed` parameter to the diffusion model, setting the pipeline to always run diffusion with the same seed of `123`. -To learn more about demo serving, see [Interactive serving](interactive_serving.md). \ No newline at end of file +To learn more about demo serving, see [Interactive serving](interactive_serving.md). diff --git a/docs/source/documentation/inferencing_api/guides/create_model.md b/docs/source/documentation/inferencing_api/guides/create_model.md index b0f3a7073..29b820842 100644 --- a/docs/source/documentation/inferencing_api/guides/create_model.md +++ b/docs/source/documentation/inferencing_api/guides/create_model.md @@ -86,23 +86,6 @@ Use `derive_config` for small variants. Set these values explicitly: checkpoint, scheduler, steps, chunk and cache sizes, guidance, precision, compile, CUDA graphs, and attention backend. -## 4. Check checkpoint compatibility on CPU - -A checkpoint transform should unwrap only known checkpoint containers, remove -known prefixes, apply ordered key renames, and leave tensor values unchanged. Before -loading a GPU model, compare the transformed checkpoint with -`network.state_dict()` using CPU or meta tensors: - -- no missing model keys; -- no unexpected checkpoint keys; -- identical shape for every matching key; -- checks using real checkpoint keys for each rename rule. - -Prefer checkpoint metadata or safetensors headers when full weights are too -large. If switching checkpoint sources, compare the transformed tensors from -both sources before claiming equivalence. A successful import is not enough; -unexplained missing or extra keys are integration failures. - ## 5. Test the integration Put model tests in `integrations_v2//tests/`. Every pytest test must have @@ -121,24 +104,15 @@ Use `ci_gpu` or `manual` for real checkpoints, generation, parity, quality, and performance. Validate at least one step beyond initial cache fill for streaming models, because rolling-window bugs often appear only after the boundary. -## 6. Add a demo adapter and documentation - -A model package can exist without a public application slug. Once its config -and model tests pass, follow -[Integrate model adapter for a demo](../../demo_api/guides/integrate_model_adapter_for_a_demo.md) to add -`integrations_v2//apps//adapter.py` and a -`flashdreams.applications_v2` entry point. - -Add or update `docs/source/models/.md` with requirements, installation, -supported application slugs, run commands, known limitations, and -implementation notes useful to contributors. Keep parity test data and results -beside the tests. Do not add a package README. - +Command to run all tests: ```bash -uv sync --package flashdreams- --extra dev --inexact uv run --no-sync pytest integrations_v2//tests -m ci_cpu -uv run --group lint pre-commit run -a ``` -Default CPU verification must not download large checkpoints or start GPU -generation. +## 6. Add documentation + +Add or update `docs/source/models/.md` with the new model. Example: [Wan 2.1](../../../models/wan21.md) + +## 7. [Optional] Add a demo adapter + +A model package can exist without a demo implementation. To add one, follow the [integrate model adapter for a demo](../../demo_api/guides/integrate_model_adapter_for_a_demo.md) guide. diff --git a/flashdreams/flashdreams/accelerated/__init__.py b/flashdreams/flashdreams/accelerated/__init__.py index b7040b748..52a7a9daf 100644 --- a/flashdreams/flashdreams/accelerated/__init__.py +++ b/flashdreams/flashdreams/accelerated/__init__.py @@ -1,2 +1,2 @@ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/flashdreams/flashdreams/accelerated/quantization/__init__.py b/flashdreams/flashdreams/accelerated/quantization/__init__.py index b7040b748..52a7a9daf 100644 --- a/flashdreams/flashdreams/accelerated/quantization/__init__.py +++ b/flashdreams/flashdreams/accelerated/quantization/__init__.py @@ -1,2 +1,2 @@ - # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - # SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 From 22e1856d22125b9fabea5ab6355ed0a9af388277 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Mon, 19 Oct 2026 23:34:20 +0000 Subject: [PATCH 12/14] delete slop --- apps/t2v/tests/test_structure.py | 53 -------------------------------- 1 file changed, 53 deletions(-) diff --git a/apps/t2v/tests/test_structure.py b/apps/t2v/tests/test_structure.py index 12a85b779..2e60a4352 100644 --- a/apps/t2v/tests/test_structure.py +++ b/apps/t2v/tests/test_structure.py @@ -44,59 +44,6 @@ """Model packages and their user-facing application slugs.""" -@pytest.mark.parametrize(("model", "factories"), _MODEL_APPLICATIONS) -def test_model_owns_v2_t2v_adapter(model: str, factories: dict[str, str]) -> None: - """Keep model code and its T2V adapter in one v2 integration package.""" - project = _REPO_ROOT / "integrations_v2" / model - application_project = project / "apps" / "t2v" - implementation = project / "impl" - config = project / "config.py" - manifest = tomllib.loads((project / "pyproject.toml").read_text()) - - assert not (_REPO_ROOT / "integrations" / model).exists() - assert {path.name for path in project.glob("*.py")} == {"__init__.py", "config.py"} - assert not (project / "runner.py").exists() - assert {path.name for path in application_project.glob("*.py")} == { - "__init__.py", - "adapter.py", - } - assert not (application_project / "t2v").exists() - assert config.is_file() - assert (implementation / "__init__.py").is_file() - assert not (implementation / "config.py").exists() - for source_path in implementation.rglob("*.py"): - source = source_path.read_text() - assert "from t2v" not in source - assert "import t2v" not in source - assert "flashdreams.runtime_v2" not in source - assert "flashdreams.serving" not in source - assert not list(project.glob("apps/*/config.py")) - application_readme = (application_project / "README.md").read_text() - model_readme = (project / "README.md").read_text() - for slug in factories: - assert slug in application_readme - assert slug in model_readme - assert "../../../../apps/t2v/README.md" in application_readme - assert not (application_project / "tests").exists() - assert not (application_project / "impl").exists() - assert (project / "tests").is_dir() - assert "flashdreams-t2v" in manifest["project"]["dependencies"] - assert f"{model}.impl" in manifest["tool"]["setuptools"]["packages"] - for slug, factory in factories.items(): - assert factory == "create_app" or ( - factory.startswith("create_app_") - and slug.endswith( - "-" + factory.removeprefix("create_app_").replace("_", "-") - ) - ) - assert manifest["tool"]["setuptools"]["package-dir"] == {model: "."} - assert manifest["project"]["entry-points"]["flashdreams.applications_v2"] == { - slug: f"{model}.apps.t2v.adapter:{factory}" - for slug, factory in factories.items() - } - assert "flashdreams.runner_configs" not in manifest["project"]["entry-points"] - assert "RUNNER_" not in config.read_text() - def test_shared_t2v_code_is_owned_by_app_package() -> None: """Keep reusable demo infrastructure outside the framework and models.""" From 0ba8eb945fb0f3a1f1f67339d48efcd99d006022 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Mon, 19 Oct 2026 23:34:53 +0000 Subject: [PATCH 13/14] add back missing note on how to run in guide --- docs/source/documentation/inferencing_api/guides/create_model.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/documentation/inferencing_api/guides/create_model.md b/docs/source/documentation/inferencing_api/guides/create_model.md index 29b820842..1ab8366b5 100644 --- a/docs/source/documentation/inferencing_api/guides/create_model.md +++ b/docs/source/documentation/inferencing_api/guides/create_model.md @@ -106,6 +106,7 @@ models, because rolling-window bugs often appear only after the boundary. Command to run all tests: ```bash +uv sync --package flashdreams- --extra dev --inexact uv run --no-sync pytest integrations_v2//tests -m ci_cpu ``` From 45cbf7a12a82cdeee44cbbbe16438dcd10dec3b5 Mon Sep 17 00:00:00 2001 From: Ariel Glasroth Date: Mon, 19 Oct 2026 23:43:49 +0000 Subject: [PATCH 14/14] format --- apps/t2v/tests/test_structure.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/t2v/tests/test_structure.py b/apps/t2v/tests/test_structure.py index 2e60a4352..69b52a922 100644 --- a/apps/t2v/tests/test_structure.py +++ b/apps/t2v/tests/test_structure.py @@ -44,7 +44,6 @@ """Model packages and their user-facing application slugs.""" - def test_shared_t2v_code_is_owned_by_app_package() -> None: """Keep reusable demo infrastructure outside the framework and models.""" project = _REPO_ROOT / "apps" / "t2v"