diff --git a/.env.example b/.env.example
index 1b58ea0..f55a516 100644
--- a/.env.example
+++ b/.env.example
@@ -140,3 +140,12 @@ TOMESHIFT_LLM_PROMPT_OVERHEAD_TOKENS=200
# character heuristics were derived from. 1.30 for claude-sonnet-5 and the
# Opus 4.7 family: same sticker price, ~30% more tokens for the same text.
TOMESHIFT_LLM_TOKEN_MULTIPLIER=1.0
+
+# Temporal durable execution engine (RFC 0009). `just up` starts the local dev
+# server (gRPC :7233, Web UI :8233); these address it. No credential is needed
+# to reach it, and `estimate` never does.
+TOMESHIFT_TEMPORAL_ADDRESS=localhost:7233
+# Workflow namespace; the dev server auto-registers "default".
+TOMESHIFT_TEMPORAL_NAMESPACE=default
+# Task queue the worker polls and the CLI starts workflows on.
+TOMESHIFT_TEMPORAL_TASK_QUEUE=tomeshift
diff --git a/Justfile b/Justfile
index 62b6810..10e1b45 100644
--- a/Justfile
+++ b/Justfile
@@ -111,6 +111,20 @@ preflight:
echo "nmt endpoint health: skipped (curl or endpoint missing)"
fi
+ # Temporal gRPC frontend, same non-blocking spirit as the NMT probe. Prefer
+ # nc; fall back to bash's /dev/tcp when nc is absent (Linux minimal images).
+ temporal_addr="${TOMESHIFT_TEMPORAL_ADDRESS:-localhost:7233}"
+ temporal_host="${temporal_addr%:*}"; temporal_port="${temporal_addr##*:}"
+ if command -v nc >/dev/null; then
+ nc -z -w2 "$temporal_host" "$temporal_port" >/dev/null 2>&1 && \
+ echo "temporal frontend ($temporal_addr): ok" || \
+ echo "temporal frontend ($temporal_addr): unavailable (non-blocking, 'just temporal-up')"
+ elif (exec 3<>"/dev/tcp/$temporal_host/$temporal_port") >/dev/null 2>&1; then
+ echo "temporal frontend ($temporal_addr): ok"
+ else
+ echo "temporal frontend ($temporal_addr): unavailable (non-blocking, 'just temporal-up')"
+ fi
+
exit $fail
# Build the CLI binary to ./bin/tomeshift.
@@ -154,6 +168,19 @@ comet-up:
comet-down:
docker compose --profile eval stop comet
+# Start the local Temporal dev server via Docker (gRPC :7233, Web UI :8233,
+# SQLite history in the temporal-data volume). `just up` starts it too.
+temporal-up:
+ docker compose up -d temporal
+
+# Stop the Temporal dev server (keeps the temporal-data history volume).
+temporal-down:
+ docker compose stop temporal
+
+# Open the Temporal Web UI in the default browser (server must be up).
+temporal-ui:
+ open http://localhost:8233
+
# ── Translate a book ──────────────────────────────────────────────────────────
# NMT translation (fast draft). Args: input EPUB, optional output dir.
@@ -234,12 +261,15 @@ seed-bible input series samples="8" merge="":
# is a 966,000-token call — past Haiku 4.5's context window entirely.
# just series-bible "epub/lumen" lumen 3
# just series-bible "epub/lumen" lumen 3 merge
-series-bible input_dir series samples="3" merge="":
+# Append only new volumes to an existing bible's STORY_SO_FAR (no full rebuild):
+# just series-bible "epub/lumen" lumen@vol33 3 "" 34
+series-bible input_dir series samples="3" merge="" from_volume="0":
#!/usr/bin/env bash
set -euo pipefail
set -a; source "{{env_file}}"; set +a
args=(series-bible --input-dir "{{input_dir}}" --series "{{series}}" --samples "{{samples}}")
if [ "{{merge}}" = "merge" ]; then args+=(--merge); fi
+ if [ "{{from_volume}}" != "0" ]; then args+=(--from-volume "{{from_volume}}"); fi
uses_llm=1
{{op_guard}}
"${runner[@]+"${runner[@]}"}" go run ./cmd/tomeshift "${args[@]}"
@@ -316,12 +346,14 @@ compare segments out:
docker-build:
docker build -t tomeshift:local .
-# Run the containerized NMT workflow (app + nmt).
+# Run the containerized NMT workflow (Temporal dev server + app + nmt).
+# `temporal-up` runs first so the durable execution engine (RFC 0009) is up
+# before the app; the app itself does not yet depend on it (worker is phase 3).
# Wrapped in `op run` when a secret is needed, so Compose can interpolate the
# API keys into the app service; without it the container starts with empty
# credentials, which is correct for the default dev-NMT workflow and wrong for
# anything paid.
-up:
+up: temporal-up
#!/usr/bin/env bash
set -euo pipefail
set -a; source "{{env_file}}"; set +a
@@ -350,6 +382,15 @@ test:
set -a; source "{{env_file}}"; set +a
go test ./...
+# Re-freeze the phase-1 golden acceptance file after a deliberate output change.
+# The golden pins translated.epub + report.json of the fixture books; a diff is
+# a real output change and must be explained in the PR, never rubber-stamped.
+test-update-golden:
+ #!/usr/bin/env bash
+ set -euo pipefail
+ set -a; source "{{env_file}}"; set +a
+ UPDATE_GOLDEN=1 go test ./tests/integration/ -run TestRun_GoldenOutputs -count=1
+
# Run all local fuzz tests across packages (not part of CI).
fuzz fuzz_time="10s":
pkgs=$(go list ./...); \
diff --git a/README.md b/README.md
index f61afc0..c470013 100644
--- a/README.md
+++ b/README.md
@@ -356,7 +356,10 @@ just up
```
What happens:
-- `docker compose` starts the `app` service from `docker-compose.yml`.
+- `docker compose` starts the local **Temporal dev server** (`temporal`, gRPC
+ `:7233`, Web UI `:8233`) and the `app` service from `docker-compose.yml`.
+ Temporal is the durable execution engine for `run`/`batch` (RFC 0009); its UI
+ is at http://localhost:8233 (`just temporal-ui` opens it).
- `.env.local` is loaded into the container runtime.
- Local folders are mounted:
- `./books` -> `/app/books`
diff --git a/docker-compose.yml b/docker-compose.yml
index b5fdb7d..97b1d50 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -60,6 +60,40 @@ services:
retries: 40
start_period: 600s
+ # Local Temporal dev server: the durable execution engine for `run`/`batch`
+ # (RFC 0009). `server start-dev` is a single-node server with an embedded
+ # SQLite history, the `default` namespace auto-registered, and the Web UI
+ # bundled in — no PostgreSQL, no Elasticsearch. Phase 2 ships the server only;
+ # the worker that registers the workflows arrives in phase 3, so `app` does
+ # not depend on it yet.
+ #
+ # Tag pinned to 1.8.3, the latest stable temporalio/temporal release
+ # (Docker Hub temporalio/temporal, pushed 2026-09-02; temporalio/cli v1.8.3).
+ temporal:
+ image: temporalio/temporal:1.8.3
+ command:
+ - server
+ - start-dev
+ - --ip
+ - 0.0.0.0
+ - --db-filename
+ - /data/temporal.db
+ - --ui-port
+ - "8233"
+ ports:
+ # gRPC frontend for workers and the CLI, and the Web UI.
+ - "7233:7233"
+ - "8233:8233"
+ volumes:
+ - temporal-data:/data
+ healthcheck:
+ # The CLI ships in the same image; ask the frontend whether the cluster
+ # is serving on the gRPC port.
+ test: ["CMD", "temporal", "operator", "cluster", "health", "--address", "localhost:7233"]
+ interval: 10s
+ timeout: 5s
+ retries: 10
+
app:
build:
context: .
@@ -92,3 +126,4 @@ services:
volumes:
comet-cache:
+ temporal-data:
diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md
index 6d9a3ef..359425d 100644
--- a/docs/ARCHITECTURE.md
+++ b/docs/ARCHITECTURE.md
@@ -193,6 +193,18 @@ Optional checks:
- Back-translation sampling for semantic drift.
- Language detection for accidental English leftovers.
+Output typography (V1, target `fr`):
+- French typography is normalized **deterministically** at commit and recompose by
+ `segment.FrenchTypography`, not by the LLM, so the TM, `segments.jsonl` and the
+ rebuilt EPUB all carry the same spacing. It runs **after** the glossary `Apply`
+ (locked terms already in place) and is idempotent (`f(f(x)) == f(x)`).
+- Rules, aligned on the English source conventions: `...` → `…`; one U+00A0 after
+ `«` and before `»`; one U+00A0 before a run of `: ; ! ?` after a letter/digit/
+ closing mark, and in place of any space already there (never for an in-word colon such as `Re:Zero`/`10:30`, nor `://`); the em dash glued to the next word
+ or to a following `.`/`,` (`« —Mot`, `mot—mot`, `« —.`), a single space before it after `. ! ? …`. Inline
+ placeholder (``) and glossary (``) tokens are preserved
+ byte-for-byte and treated as zero-width when deciding context.
+
### 6.8 EPUB Recomposer
Responsibilities:
- Reinsert translated text into original XHTML nodes.
diff --git a/docs/DEPENDENCIES.md b/docs/DEPENDENCIES.md
index aa021de..831c790 100644
--- a/docs/DEPENDENCIES.md
+++ b/docs/DEPENDENCIES.md
@@ -18,6 +18,28 @@ Define how this project selects and maintains third-party dependencies.
- `go.uber.org/zap`
- Reason: structured, high-performance logging for production-grade runs.
+- `go.temporal.io/sdk` *(approved; lands in go.mod in RFC 0009 phase 3, not yet vendored)*
+ - Why stdlib is insufficient: a run is a long, multi-stage, externally
+ rate-limited job that must survive worker death and provider outages without
+ repaying LLM work. Durable execution with replay, per-activity timeouts and
+ retries, durable timers, signals, queries and child workflows is a workflow
+ engine — the stdlib offers none of it, and hand-rolling it is exactly M4's
+ plan items 1 and 5.
+ - Capability gained: the Go SDK for a self-hosted Temporal dev server. The run
+ becomes a workflow, each stage an activity with its own retry policy, the 429
+ pause a timer or signal, progress a query, and a series a parent workflow —
+ all replayable and observable in the Temporal UI.
+ - Maintenance and security: Temporal is actively maintained and Go is a
+ first-class SDK. It pulls in a heavy transitive tree (gRPC, protobuf); it is
+ confined to `internal/temporal`, the only package allowed to import it,
+ enforced by a `depguard` rule in `.golangci.yml`. Local dev server only — no
+ Temporal Cloud, no credentials.
+ - Rollback/removal plan: the business (stages and control flow) stays
+ Temporal-agnostic behind the `pipeline.Steps`/`pipeline.Runtime` ports.
+ Leaving Temporal means deleting `internal/temporal` and running the flow on
+ the in-process `Steps`/`Runtime` pair that ships as the portability proof.
+ See RFC 0009 (`docs/rfcs/0009-temporal-execution-engine.md`).
+
## Observability Stack (Service Mode)
- `VictoriaMetrics` for metrics storage/query.
- `Grafana` for dashboards.
@@ -30,6 +52,10 @@ Define how this project selects and maintains third-party dependencies.
- Reason: consistent linting and static analysis in local and CI validation.
- Docker + Docker Compose:
- Reason: reproducible local runtime and optional local DB profile for service-mode preparation.
+- Temporal dev server (`temporalio/temporal`, pinned tag in `docker-compose.yml`):
+ - Reason: local, single-node durable execution engine (`server start-dev`,
+ embedded SQLite history, Web UI on :8233) for RFC 0009. Started by `just up`
+ / `just temporal-up`. No PostgreSQL, no Temporal Cloud.
- GitHub Actions:
- Reason: automated lint/test/format validation on pull requests and main branch updates.
diff --git a/docs/architecture-temporal.html b/docs/architecture-temporal.html
new file mode 100644
index 0000000..2dd52d9
--- /dev/null
+++ b/docs/architecture-temporal.html
@@ -0,0 +1,949 @@
+
+
+
+
+
+tomeshift on Temporal — the new architecture, illustrated
+
+
+
+
+
+
tomeshift on Temporal the new architecture and pipeline management, illustrated
+
+ Illustrated companion to RFC 0009 · September 10, 2026 · status proposed
+ Referenced code frozen at ca5a63d (branch refactor/to-temporal)
+ · For the current architecture: understanding-the-project.html
+
+ Yes, Temporal fits this project well — for the shape of the problem, not for its scale.
+ A run is long (hours), multi-stage, dependent on quota-bound external providers, and it must
+ never pay twice. That is the definition of a durable workflow. The scale, though, is
+ tiny: one user, one book at a time. Temporal is therefore oversized in capacity, but
+ not in functions: every one it brings matches a mechanism written by hand today,
+ or planned for M4.
+
+
+
+
+
Need
Today
With Temporal
+
+
Retry a transient NMT call
RetryingTranslator, 3 linear attempts (translate/retry.go)
RetryPolicy declared per activity
+
Survive an exhausted quota (429)
The run fails, the human re-launches, the TM avoids paying again — per chapter (session 18)
The workflow goes paused, waits on a timer or a resume signal, resumes at the batch
+
Survive process death
Nothing: re-extraction, re-segmentation, re-projection, then TM
The worker restarts, the history is replayed, the in-flight activity is resumed
+
Record the spend even on failure
defer recordSpend with a detached context (orchestrator.go:217-220)
Cleanup path under NewDisconnectedContext
+
See where the run stands
Bar inside the process, broken by op run (session 19)
Query progress + Temporal UI; the bar lives in a secret-free CLI
+
Chain 30 volumes
In-memory batch loop, one process
Parent workflow, sequential children, one history per volume
+
M4 "queue + worker + retry + idempotency"
To be written
It is the product
+
+
+
+
+
+ The price, stated
+
+
An always-on service for run — one SQLite container, zero external cost. estimate stays offline.
+
A heavy dependency (go.temporal.io/sdk → gRPC, protobuf), to record in DEPENDENCIES.md.
+
Workflow code must be deterministic: no I/O, no time.Now, no goroutine, no map iteration that decides.
+
The real work: segments today hold a pointer into the live HTML tree (segment.Segment.Node), which Restore mutates and Recompose renders. None of that can cross an activity boundary (2 MB per-payload limit). It must be decoupled — and that is a refactor worth doing even without Temporal.
+
+
+
+
+ The recommendation
+
Adopt Temporal under five conditions: activities are the existing stage functions; one engine at the end (the in-process path is deleted); references cross Temporal, not content; a local SQLite dev server, no Postgres; one worker, one activity slot, as long as the TM is on SQLite.
+
+
+
2. The target topology
+
+
+
+flowchart LR
+ CLI["tomeshift run / batch no secrets · drives the bar"]
+ T["Temporal dev server :7233 gRPC · :8233 UI SQLite history"]
+ W["tomeshift worker op run — holds the API keys"]
+ subgraph deps["What the worker touches"]
+ NMT["NMT provider"]
+ LLM["LLM provider"]
+ TM[("tm.sqlite segments · run_attempts")]
+ WS[("runs/run/work/ units.jsonl per chapter")]
+ B[("series bible")]
+ end
+ CLI -- "start · signal · query · cancel" --> T
+ T -- "workflow & activity tasks" --> W
+ W --> NMT
+ W --> LLM
+ W --> TM
+ W --> WS
+ W --> B
+
+Three processes. The CLI only talks to Temporal; the worker alone calls the providers and writes to disk.
+
+
+
+ What changes for the operator: just up starts temporal and the worker;
+ just run starts a workflow and follows its progress; the UI is at localhost:8233.
+ The recipes that call a provider now wrap the worker in op run,
+ no longer the CLI — which also fixes the bar being killed by the 1Password pipe.
+
+
+
3. The lifecycle of a run
+
+
+ One book = one TranslateBook workflow. Each stage of the current orchestrator becomes an
+ activity with its own timeout and retry policy; the workflow only chains them and
+ sums what they consumed. Start the animation, then kill the worker along the
+ way: the history survives, the worker restarts, the finished activities are replayed (no
+ call), and execution resumes where it left off.
+
+ state idle
+ paid provider calls 0
+ activities replayed for free 0
+ history events 0
+
+
+
+
+
+
+
+
+
+
+sequenceDiagram
+ participant C as CLI
+ participant T as Temporal
+ participant W as Worker
+ C->>T: StartWorkflow TranslateBook
+ T->>W: PrepareBook
+ T->>W: ProjectCost
+ Note over T: cap exceeded → fail ErrBudgetExceeded, no provider call
+ loop each chapter, in order
+ T->>W: ResolveChapter
+ T->>W: DraftChapter
+ alt literary
+ loop each batch
+ T->>W: RefineBatch
+ Note over T,W: RateLimited → timer or resume signal, then the same batch again
+ end
+ else nmt
+ T->>W: EscalateChapter
+ end
+ T->>W: CommitChapter
+ opt update-bible
+ T->>W: UpdateBible
+ end
+ C-->>T: query progress
+ end
+ T->>W: RecomposeBook · CheckEPUB · FinishBook
+ T->>W: RecordSpend
+ T-->>C: RunResult → exit code
+
+The nominal sequence. On failure or cancellation, FinishBook and RecordSpend still run, under a disconnected context — the equivalent of today's defer.
+
+
+
+ Where the state goes
+
+ Today the counters live in RunResult, fed by the character meter
+ delta and by the OnBatch hook (mode.go:218-226). Tomorrow each activity returns
+ what it consumed (characters, tokens, attempts, TM hits) and the workflow sums it. The Temporal
+ history becomes the audit trail the ledger used to be the only copy of — and the price is computed by a
+ local PriceRun activity, pure arithmetic over the cost model.
+
+
+
+
4. A literary chapter, batch by batch, with a 429
+
+
+ This is the scene that justifies the work. A 100-segment chapter goes out in 2 to 4 LLM batches.
+ Today, if the third batch gets a 429, llm.ErrRateLimited stops the chapter and
+ fails the run (literary/translator.go:194-197): batches 1 and 2, succeeded but not
+ persisted, are lost and will be paid again on re-launch. With Temporal, RefineBatch is
+ the retry unit: the workflow goes paused, waits, and re-runs the same batch.
+
+
+
+ Animation — one chapter, four batches, a quota that gives out
+
+
+ state idle
+ cooldown 30:00 or resume signal
+
+
+
+
+
Today — failed run, chapter redone
+
batches paid in total 0
+
the batches that succeeded before the 429 are not persisted (persist happens only after Restore, at the end of the chapter)
+
+
+
Temporal — pause then resume at the batch
+
batches paid in total 0
+
batch 3 is replayed; batches 1-2 are in the history and in the completion cache
+
+
+
+
+
+
+
+
+
+
+
+ Why the batch and not the chapter
+
+ Resume happens at the granularity of the activity that failed. A RefineChapter activity
+ would keep the current boundary; RefineBatch drops it to the batch. The recursive halving of a
+ truncated batch (TranslateChapter) stays inside the activity: the same tokens, for the same
+ logical batch. The ~1,200-rune tail of preceding French travels in the payload — 3 KB, nothing.
+
+
+
+
5. What crosses Temporal — and what never does
+
+
+ Temporal persists every activity input and output in the history. The limit is
+ 2 MB per payload (warning at 256 KB) and 51,200 events per run. The book
+ in bytes, the *html.Node trees, the placeholders with their node: none of that passes.
+ This is the claim-check pattern: the content stays in a workspace on disk, the
+ activities exchange references and counters.
+
+
+
+
+flowchart LR
+ subgraph hist["Crosses Temporal — small, serialisable"]
+ M["Manifest 45 chapters × id, href, n, chars"]
+ P["Chapter plan reused / fresh ids"]
+ R["RefineBatch in ids + 1,200-rune tail"]
+ U["Results tokens, chars, hits, violations"]
+ end
+ subgraph disk["Stays on disk — read by the activities"]
+ E["source.epub"]
+ J["work/ch_0001.jsonl … units schema 16.1"]
+ D["tm.sqlite · bible · llm_cache"]
+ end
+ M -. reference .-> J
+ P -. reference .-> J
+ R -. reference .-> J
+ J --> E
+
+The dotted arrows are references (RunRef, ChapterID, segment ids). No arrow carries book text, except the context tail.
+
+
+
A test on the fixture book will assert these sizes against the 256 KB threshold, to catch drift.
+
+
+
+ The prerequisite refactor, without Temporal (phase 1)
+
+ Segmentation is deterministic: ids by position, hash of the normalized text. So a process that
+ holds only {segment_id, target_text} can re-parse the original document, re-derive
+ the placeholders, Restore to validate, and render. That is epub.ApplyTranslations(book, units).
+ CommitChapter uses it to validate and persist per chapter; RecomposeBook to
+ produce the book. The acceptance criterion is the byte-for-byte identity of translated.epub
+ before and after, on the fixture and on a real chapter fully in the TM.
+
RateLimited (→ pause); a refusal is returned (degrade), not raised
+
EscalateChapter
30 min
60 s
3
—
+
UpdateBible
10 min
60 s
3
narrative.ErrInvalidBible
+
RecomposeBook, CheckEPUB
10 min
—
2
—
+
FinishBook, RecordSpend
1 min
—
5
—
+
+
+
+
+
+
+ Idempotency — an activity can run twice
+
+
TM.Save: upsert on source_hash, already idempotent.
+
RecordRun: one row per intended attempt; a retried activity is not one → attempt_key column = WorkflowID/RunID, unique index, INSERT OR IGNORE.
+
WriteArtifacts: overwrites, idempotent.
+
A retried DraftChapter re-pays the NMT characters — the current contract of RetryingTranslator.
+
A retried RefineBatch is served by llm_cache.sqlite if the previous attempt got an answer.
+
+
+
+ Determinism — the rules of the workflow package
+
+
workflow.Now, never time.Now; workflow.SideEffect for anything random.
+
No I/O, no network call, no goroutine.
+
No map iteration that influences a decision.
+
No worker upgrade mid-volume; otherwise workflow.GetVersion.
+
A replay test over a recorded history of the fixture book runs in CI.
+
+
+
+
+
7a. Code layout and coupling to Temporal
+
+
+ Stated requirement: to be able, tomorrow, to switch engine — or go back to none — by rewriting a single
+ package. Answer: ports and adapters, with all the business on the Temporal-free side,
+ including the control flow. The Temporal workflow is one line:
+ return pipeline.RunBook(steps, runtime, in).
+
+
+
+
+flowchart LR
+ subgraph adapter["internal/temporal — the only SDK importer"]
+ direction TB
+ WF["TranslateBook(workflow.Context) = RunBook(steps, runtime, in)"]
+ ST["Steps → ExecuteActivity"]
+ RT["Runtime → NewTimer, signals, DisconnectedContext, ChildWorkflow"]
+ A["activities = Stages methods"]
+ end
+ subgraph core["pipeline — Temporal-free"]
+ direction TB
+ F["RunBook / RunSeries the flow, written once"]
+ P["ports: Steps · Runtime"]
+ S["Stages the activity bodies"]
+ D["epub · segment · tm · translate literary · qa · cost · narrative"]
+ F --> P
+ S --> D
+ end
+ subgraph inproc["pipeline/inprocess.go — tests, portability"]
+ direction TB
+ IS["Steps → direct Stages call"]
+ IR["Runtime → time.Timer, channels"]
+ end
+ WF --> F
+ ST -. implements .-> P
+ RT -. implements .-> P
+ A --> S
+ IS -. implements .-> P
+ IR -. implements .-> P
+ IS --> S
+
+Two implementations of the same ports. The CLI ships the first; the tests and portability rest on the second.
+
+
+
+
+ The two ports — all the flow knows about the engine
+
// Steps: one method per activity, no context —
+// the adapter owns it (workflow.Context is not
+// a context.Context).
+type Steps interface {
+ PrepareBook(BookInput) (Manifest, error)
+ ProjectCost(ProjectInput) (Projection, error)
+ ResolveChapter(ResolveInput) (ChapterPlan, error)
+ DraftChapter(DraftInput) (DraftResult, error)
+ EscalateChapter(EscalateInput) (EscalateResult, error)
+ RefineBatch(RefineInput) (RefineResult, error)
+ CommitChapter(CommitInput) (CommitResult, error)
+ UpdateBible(BibleInput) (BibleResult, error)
+ RecomposeBook(RunRef) (RecomposeResult, error)
+ CheckEPUB(CheckInput) (CheckResult, error)
+ FinishBook(FinishInput) (qa.Report, error)
+ RecordSpend(SpendInput) error
+ PriceRun(PriceInput) (float64, error)
+}
+
+// Runtime: what the flow needs from an engine,
+// and nothing more.
+type Runtime interface {
+ Now() time.Time
+ WaitResume(d time.Duration) (resumed bool, err error)
+ Paused() bool
+ Cancelled() bool
+ Detached() Runtime
+ Report(Progress) // the flow publishes; in-process → bar, Temporal → query
+ RunBook(BookInput) (RunResult, error) // series: one child per volume
+}
+ The import rule is enforced, not hoped for: a depguard rule in
+ .golangci.yml denies go.temporal.io/** everywhere except in internal/temporal
+ and internal/commandexec. golangci-lint already gates every commit.
+
+
+
+
+
+ The tradeoff, stated
+
+ The ports are a fixed vocabulary. A Temporal feature the flow would want (an Update, a
+ Nexus call) goes through a port method first. For a 13-step sequential pipeline that is
+ the right price; the alternative — writing the flow in workflow.* calls — is ~300 lines to
+ rewrite on every engine change, together with all the tests that exercise them. What the adapter
+ does not abstract: Temporal's own concerns (payload codecs, task queues, worker
+ options, GetVersion, replay). They live in internal/temporal and leave with it.
+
+
+
+
Extending: adding a translation mode (e.g. LLM-only)
+
+
+ A mode is a recipe over the existing Steps, never a new workflow.
+ RunBook selects a Mode from BookInput.Mode, which the workflow forwards without reading it.
+ nmt = Resolve → Draft → Escalate → Commit; literary = Resolve → Draft → RefineBatch × n → Commit;
+ llm = Resolve → RefineBatch × n without a draft → Commit.
+
+
+
+
+
Layer
Change
Size
Touches the adapter?
+
+
literary
Prompt variant without a draft (DraftText optional, RefineInput.HasDraft)
~40 lines + decode tests
no
+
pipeline
llmMode struct; Project reuses the existing full-llm scenario (cost/estimate.go:296)
~100 lines + RunBook scenarios
no
+
internal/temporal
Nothing — no new activity, RefineBatch accepts an empty draft
0
—
+
cli
--mode llm value
~10 lines
no
+
docs
usage.html §6, understanding-the-project.html §1 and recipe 3
—
—
+
+
+
+
+
+ A mode that needs a new step costs, on the adapter side, one Steps method, one
+ Stages method, one activity registration and one policy.go line — ~30 mechanical
+ lines. The real cost of an LLM-only mode is not the code: the placeholder safety
+ net falls back today on the NMT draft; without a draft it would fall back on the English
+ source. Either retry the degraded segment alone, or request a draft on demand for the
+ degraded ids only (DraftChapter on a subset — the step exists). To decide when that
+ mode is built; untranslated_segments in QA catches the case in both options.
+
+
+
8. The six-phase plan
+
+
Each phase is one PR, green on just check, and leaves the tool usable. Phases 1 and 2 change nothing a user sees; phases 3 to 5 add the Temporal path behind TOMESHIFT_ENGINE=temporal; phase 6 makes it the only one.
+
+
+ Phase explorer — click
+
+
+
+
+
8a. The tests, batch by batch
+
+
+ Five levels, fastest to heaviest. The first four run without a Temporal server and
+ fit in just check; the fifth starts the dev server as a CI service
+ (the container boots in ~2 s) and skips locally if :7233 is unreachable — the
+ pattern already in place for ollamaHost. The SDK test API is used only at level 3,
+ on the one package that imports the SDK.
+
+
+
+
+flowchart TB
+ L5["5 · Integration — dev server + nmt-devserver + llmtest: fixture end to end, worker kill, 429 injected"]
+ L4["4 · Replay — WorkflowReplayer over a recorded history: catches non-determinism, including in RunBook"]
+ L3["3 · Adapter — SDK testsuite on internal/temporal: wiring only, options, timer, signal, errors, query"]
+ L2["2 · Flow — pipeline.RunBook on the in-process Runtime with a fake Steps: plain Go, sums, pause, cancel, errors"]
+ L1["1 · Stages — existing tests of epub, segment, tm, literary, qa: unchanged, the logic does not move package"]
+ L5 --> L4 --> L3 --> L2 --> L1
+
+The business scenarios live at level 2, without the SDK. Level 3 only checks the translation of the ports to Temporal.
+
+
+
+
+
Batch
What we prove
How
Existing tests touched
+
+
+
1 · Decouple
+
Re-deriving the placeholders from {id, target} renders the same book; the flow behaves as Run.
+
epub.ApplyTranslations table-driven: identity round trip (source applied as target → identical bytes), unknown id → sentinel, missing placeholder → segment left as source. workspace: units.jsonl round trip, corrupt line → error. Level 2: the orchestrator_test.go scenarios ported onto RunBook + in-process runtime (sums, cap refused with no DraftChapter, cancel → FinishBook + RecordSpend, stage error → exit code). Golden: sha256 of the fixture's translated.epub and report.json, frozen before the refactor.
+
orchestrator_test.go (826 lines): its scenarios move to book_test.go; the file goes with Run.
+
+
+
2 · Infra
+
The service starts, the CLI sees it, nothing else changes.
+
preflight: probes :7233 reachable / unreachable. Manual smoke just up → UI :8233. just docs-audit for the three variables.
+
—
+
+
+
3 · Worker + NMT
+
The adapter dispatches, maps and serves what the flow expects.
+
Level 3 (testsuite): each Steps method reaches the activity of the same name with its options; WaitResume returns on timer and on signal; error mapping both ways; the query returns Progress(). Level 4: first recorded history. Level 5: fixture end to end; worker kill = cancel the first worker's context mid-run, start a second, check that the run finishes and that the nmt-devserver's request count equals a clean run's. Payload size: each serialised input/output < 256 KB. depguard forbids the SDK outside the adapter.
+
None removed. RetryingTranslator stays tested until batch 6.
+
+
+
4 · Literary + 429
+
A 429 pauses, resumes the same batch, and does not bill the batches that succeeded.
+
Level 2, plain Go: a fake RefineBatch returns RateLimited on the 3rd call then succeeds; the in-process WaitResume is fed by a test channel; assertions: paused in Progress(), calls = clean run + 1, tokens = clean run. Max waits → failure. pause/resume outside a 429. Level 5: llmtest returns 429 on the Nth call — the integration test session 18 left missing.
+
literary/translator_test.go (739 lines) unchanged: batch splitting stays in the package.
+
+
+
5 · Series
+
Volumes run in order, one failure does not stop the series.
+
Level 2: RunSeries with a fake Runtime.RunBook; order; one child fails → the parent continues and the summary counts it. Level 3: the RunBook port → ExecuteChildWorkflow.
+
—
+
+
+
6 · Cutover
+
Nothing the deleted code proved goes unproven.
+
retry_test.go goes with RetryingTranslator; Plan and its tests stay for estimate; inprocess.go stays as the level-2 runtime.
+
Deleted with the code they covered — never re-pinned to the new text.
+
+
+
+
+
+
+ What level 2 allows, and what was missing
+
+ The flow runs on an in-process Runtime whose timer is a channel: a 30-min cooldown
+ is "skipped" by closing the channel, a resume is sent from the test. The scenario
+ "429 in the middle of a volume → pause → resume → same bill" — which session 18 left without a test
+ because it required a real server and real minutes — becomes an ordinary Go test of a few
+ milliseconds, without the SDK. And because it also runs inside the workflow, the replay test
+ (level 4) guarantees it stays deterministic.
+
+
+
+
9. The invariants that do not move
+
+
+
+
Invariant
Where it lives today
Where it will live
+
+
Persist to TM only after an accepted Restore
mode.go:344-368
In CommitChapter, one activity — never separated
+
A draft is a hit only for the mode that produced it
translationMode.ReusesDrafts
ReusesDrafts parameter of ResolveChapter and PersistDrafts of DraftChapter
+
Literary mode never persists a draft
draftSegments
Same, PersistDrafts=false
+
The spend is recorded even on failure
defer recordSpend
Cleanup under a disconnected context
+
Two projections: cap on the remainder, savings on the whole book
orchestrator.go:154-163
ProjectCost returns both
+
The bible is a frozen input; the store assigns its version
updateBible
UpdateBible, SaveIfVersion loop included
+
A 429 is not degradable
translator.go:194-197
Non-retryable RateLimited error → workflow pause
+
estimate requires neither a key nor a service
Orchestrator.Plan
Unchanged — Plan survives the cutover
+
+
+
+
+
10. The decisions — all settled on September 10, 2026
+
+
+
Cap exceeded → decided fail as today (ErrBudgetExceeded before any call). No approve-budget signal.
+
Rate-limit cooldown → decided 30 min, doubling on each consecutive wait, ceiling 5 h (the Claude CLI window).
+
Blocking run or --detach → decided blocking: one command, the progress bar fed by the progress query, an exit code. The run still survives the CLI since it lives in the worker. No --detach in this RFC.
+
Task queues → decided a single one, tomeshift.
+
Temporal Schedules → decided after phase 5, only if a need appears.
+
+
+
+
+
+
+
+
+
diff --git a/docs/improving-translations.md b/docs/improving-translations.md
index e49b7ad..2eda2c1 100644
--- a/docs/improving-translations.md
+++ b/docs/improving-translations.md
@@ -133,6 +133,55 @@ before".
> Check the state with `just bible ` (summaries column) before
> launching the next volume.
+## Importing an upstream translators' style guide
+
+When the source EPUB is itself a translation (e.g. a fan English translation of
+a Japanese web novel), its team usually keeps a manual of style: glossary,
+per-character speech quirks, punctuation rules. Each English quirk there is a
+deliberate encoding of a source-language trait (a sentence-final particle
+always rendered `, I suppose`, a character who never contracts, a mangled
+in-world word). Importing the manual means giving **one** French form to each
+of those markers, instead of letting the LLM translate them literally and
+differently every time.
+
+1. **Read the manual** section by section (glossary tables, character index).
+2. **Measure before deciding.** Over the runs already translated, count for
+ each marker which French renderings coexist; keep the majority when it is
+ correct, decide otherwise:
+ ```python
+ import json, glob, re, collections
+ segs = [(d["source_text"], d["target_text"])
+ for f in glob.glob("runs/*/segments.jsonl")
+ for d in map(json.loads, open(f)) if d.get("target_text")]
+ def variants(src, fr_patterns):
+ rows = [t for s, t in segs if re.search(src, s)]
+ return len(rows), collections.Counter(
+ p for t in rows for p in fr_patterns if re.search(p, t, re.I)).most_common()
+ variants(r"ground dragons?", [r"dragons? terrestres?", r"dragons? de terre"])
+ ```
+3. **Route each item to the right lever** — the glossary is mechanical, so it
+ only fits invariant names:
+
+ | Upstream item | Lever | Why |
+ |---|---|---|
+ | proper names the LLM could gallicize (Minerva, Pandora, Flanders), spelling fixes | glossary, `term_en == term_fr` or corrected spelling | `Protect`/`Apply` swap the exact, case-sensitive `\bterm\b` for `term_fr` verbatim: no plural, gender agreement or elision |
+ | translated common terms (titles, creatures, places with an article) | `style_guide` lexicon (`EN = FR`, to inflect normally) | they need agreement, which only the LLM can do |
+ | fixed catchphrases, verbal tics | `style_guide` "fixed formulas": exact EN sentence → exact FR sentence | consistency across volumes |
+ | voice, contraction level, nicknames, fixed colours | `characters_json`, French only, `sample_fr` drawn from good existing output | the prompt renders every character on every call |
+ | tu/vous | `address_json` | only the **last** history entry reaches the prompt |
+
+ Never lock a name that is also a common English word (Miles, Wolf, Null…):
+ every capitalized occurrence would be forced.
+4. **Write the bible** with a raw `UPDATE` (back the database up first). A raw
+ `UPDATE` does not go through `SaveAll`, so bump `version` yourself.
+ `series-bible --from-volume N` keeps a curated style guide and characters;
+ a full `series-bible`/`style-profile` without `--merge`, or `--update-bible`,
+ rewrites them.
+5. **Re-measure** on the next translated volume with the same script.
+
+Spacing and punctuation are not part of this import: French typography is
+normalized in code at commit time (see `segment.FrenchTypography`).
+
## Typical improvement workflow
1. Translate a **chapter** ([`translation-guide.md`](translation-guide.md)).
diff --git a/docs/rfcs/0009-temporal-execution-engine.md b/docs/rfcs/0009-temporal-execution-engine.md
new file mode 100644
index 0000000..2f598c8
--- /dev/null
+++ b/docs/rfcs/0009-temporal-execution-engine.md
@@ -0,0 +1,648 @@
+# RFC 0009: Temporal as the Execution Engine of the Translation Pipeline
+
+- Status: `proposed`
+- Authors: ECecillo
+- Created: 2026-09-10
+- Updated: 2026-09-10
+- Related:
+ - Architecture: `docs/ARCHITECTURE.md` (§5, §7, §9.2, §10)
+ - Milestones: `docs/milestones/M4-production-hardening.md` (replaces items 1 and 5 of its plan)
+ - Illustrated companion: `docs/architecture-temporal.html`
+ - Builds on: [0004 - EPUB Pipeline](./0004-epub-pipeline.md), [0007 - Token-Based Budget Guardrail](./0007-google-translate-draft-and-token-cost-model.md)
+ - Does not require: [0005 - TM Storage Evolution](./0005-tm-storage-evolution.md) (SQLite stays)
+ - Supersedes: n/a
+ - Superseded by: n/a
+
+## Context
+
+A run today is one in-process function. `Orchestrator.Run`
+(`internal/pipeline/orchestrator.go:81-281`) walks extract → parse → segment →
+project → translate → recompose → QA → artifacts while holding every piece of
+state in memory: the `RunResult` counters, the NMT meter baseline (`sentBefore`,
+line 196), the `costFn` closure (202-206), the `OnBatch` hook that both meters
+tokens and repaints the progress bar (`mode.go:218-226`), and a deferred ledger
+write that must survive a cancelled context (217-220, 313-348).
+
+Reliability is hand-rolled around it, one mechanism per failure mode:
+
+- transient NMT failures: `RetryingTranslator` (`internal/translate/retry.go`),
+ three attempts, linear backoff;
+- a bad LLM batch: degrade to the draft (`literary/translator.go:187-192`);
+- quota exhaustion: `llm.ErrRateLimited` is the one non-degradable batch failure
+ (`translator.go:194-197`) — it fails the run, and the operator re-launches;
+- resuming: the translation memory. A re-launch re-extracts, re-segments and
+ re-projects the whole book, then pays only for the segments the TM lacks. The
+ resume boundary is the chapter (journal, session 18): a 429 on batch 3 of 4
+ throws away batches 1 and 2.
+
+Measured on the real workload (vol33: 45 chapters, 4,774 segments, 101 LLM
+calls, ~27 $, several hours on a provider whose quota is a 5-hour window), that
+is the operational shape of the tool: a long, multi-stage, externally
+rate-limited job that must never pay twice and that a human occasionally has to
+approve or resume.
+
+M4 already planned to build what that shape needs — "queue contract and worker
+runtime", "distributed idempotency and retry controls", "failure recovery and
+resumable runs at scale" (`M4-production-hardening.md:19-27, 29-38`). That is a
+workflow engine, written by hand, in a one-person project.
+
+## Goals
+
+- A run survives worker death, provider outages and quota exhaustion without
+ redoing paid work and without the operator re-launching it.
+- Resume granularity becomes the **LLM batch**, not the chapter or the run.
+- Run state is observable outside the process (Temporal UI) and in the CLI.
+- A series of N volumes is one durable, ordered execution.
+- Stage packages (`epub`, `segment`, `tm`, `translate`, `literary`, `qa`,
+ `cost`, `narrative`) stay Temporal-agnostic. `estimate` keeps working with no
+ server running.
+- One engine at the end. No permanent dual maintenance of an in-process path
+ and a Temporal path.
+
+## Non-Goals
+
+- Multi-tenant service, HTTP API, auth (M4 out of scope stays out of scope).
+- PostgreSQL migration (RFC 0005). SQLite stays; this RFC sizes concurrency for it.
+- Parallel chapter translation. Literary order (`prevCtx`, bible) and SQLite's
+ single writer both forbid it; revisit after RFC 0005.
+- Temporal Cloud. Local dev server only.
+
+## Options Considered
+
+### Option A: Status quo, deeper checkpointing
+Persist per-batch translations in a run-scoped store, sleep-and-retry on 429,
+re-launch from the last checkpoint.
+- Pros: no infrastructure, no dependency, incremental.
+- Cons: durable timers, retry policy, run state machine, pause/resume and
+ visibility all become bespoke code — M4's plan, item by item. Each is a
+ well-known problem with a boring solution elsewhere.
+
+### Option B: Job queue (River on PostgreSQL, Asynq on Redis)
+Chapters or batches become jobs with retries.
+- Pros: simpler mental model than Temporal; retries and dead-letter for free.
+- Cons: a job is not a workflow. Ordering across chapters, the pause on 429, the
+ spend accumulation, the "finish and record spend on failure" cleanup, and the
+ series-of-books orchestration all have to be encoded as job state by hand. It
+ also brings PostgreSQL or Redis in, which this RFC does not otherwise need.
+
+### Option C: Temporal (self-hosted dev server, Go SDK)
+The run becomes a Workflow; each stage an Activity with its own timeout and
+retry policy; 429 becomes a timer or a signal; progress a query; a series a
+parent workflow of child workflows.
+- Pros: durable execution with replay, per-activity retries, timers, signals,
+ queries, child workflows, a UI, and a replay-based test harness — every item
+ of M4's plan 1 and 5, maintained by someone else. Go is a first-class SDK.
+- Cons: an always-on server for `run` (one container); a heavy dependency tree
+ (`go.temporal.io/sdk` → gRPC, protobuf); workflow code must be deterministic;
+ payload and history limits (2 MB per payload, warning at 256 KB; 51,200
+ events / 50 MB per run) forbid passing the book or the node tree around —
+ which the current design does.
+
+## Proposed Decision
+
+Option C, under six conditions that keep the blast radius small:
+
+1. **Activities are the existing stage functions.** No business rule lives in
+ Temporal code. The workflow function is one line: `pipeline.RunBook(steps, runtime, in)`.
+2. **The flow is engine-agnostic too.** Control flow, accounting, pause and
+ cleanup live in `pipeline.RunBook`, written against two small ports
+ (`Steps`, `Runtime`). `internal/temporal` is the only package importing the
+ SDK, enforced by `depguard`. Leaving Temporal means rewriting that package.
+3. **One shipped engine at the end.** Phase 6 deletes `RetryingTranslator`,
+ `OnBatch`, `costFn`, the in-pipeline progress hooks and the CLI's in-process
+ path. The in-process `Steps`/`Runtime` pair stays as the flow's test double
+ and portability proof. `pipeline.Plan` stays, for `estimate`.
+4. **References cross Temporal, not content.** Payloads carry run/chapter
+ references, segment IDs, counters and a ~1,200-rune context tail. The EPUB
+ bytes and the `*html.Node` trees never do.
+5. **Local-first.** `temporal server start-dev --db-filename` (SQLite) in
+ `docker-compose.yml`, UI on 8233. No PostgreSQL service.
+6. **One worker, one activity slot** on the task queue that writes the TM,
+ until RFC 0005 lands.
+
+The precondition the current code does not meet is condition 4.
+`segment.Segment.Node` (`internal/segment/types.go:68`) and
+`Placeholder.node` (line 50) are live pointers into a tree that `Restore`
+mutates and `Recompose` renders (`internal/epub/recomposer.go:37-61`). Phase 1
+removes that coupling **without Temporal**, and is worth shipping on its own:
+segmentation is deterministic (IDs by position, hash by normalized text), so a
+process that holds only `{segment_id, target_text}` can re-parse the original
+document, re-derive the placeholders, restore, validate and render. That is
+exactly what `segments.jsonl` (`ARCHITECTURE.md §16.1`) already claims to be.
+
+## Design
+
+### Topology
+
+```mermaid
+flowchart LR
+ CLI["tomeshift run / batch (no secrets, drives the progress bar)"]
+ T["Temporal dev server :7233 gRPC · :8233 UI SQLite history"]
+ W["tomeshift worker (op run — holds the API keys)"]
+ subgraph deps[" "]
+ NMT["NMT provider"]
+ LLM["LLM provider"]
+ TM[("tm.sqlite segments · run_attempts")]
+ WS[("runs/<run>/work/ units.jsonl per chapter")]
+ B[("series bible")]
+ end
+ CLI -- start · signal · query · cancel --> T
+ T -- workflow & activity tasks --> W
+ W --> NMT & LLM & TM & WS & B
+```
+
+### Workflows
+
+| Workflow | Input | Output | Notes |
+|---|---|---|---|
+| `TranslateBook` | `BookInput{RunID, BookID, InputPath, OutputDir, Mode, SeriesID, SourceLang, TargetLang, UpdateBible}` | `RunResult` (schema 16.3, unchanged) | One per volume. Sequential over chapters. |
+| `TranslateSeries` | `SeriesInput{Books []BookInput, ReferenceDir}` | `BatchSummary` | Parent of `TranslateBook` children, in order, continue-on-failure as `batch` does today. |
+
+Workflow state is the `RunResult` accumulator. Every activity returns what it
+consumed (chars, tokens, attempts, hits) and the workflow sums it. That replaces
+the meter delta and the `OnBatch` hook, and the history becomes the audit trail
+the ledger used to be the only copy of.
+
+### Activities
+
+All activities hang off one `Activities` struct holding the injected
+collaborators the orchestrator holds today (`orchestrator.go:43-72`). Inputs
+and outputs are small, JSON-serialisable structs.
+
+| Activity | In → Out | What it wraps |
+|---|---|---|
+| `PrepareBook` | `BookInput` → `Manifest{Chapters []{ID, Href, Segments, Chars}}` | extract + parse + segment; writes `work/.jsonl` (units, schema 16.1) |
+| `ProjectCost` | `{RunRef, Mode}` → `Projection{EnforcedUSD, NoReuseUSD, Pending, Reusable, UsedFreeChars, Windows}` | `pendingChapters`, `freeCharsUsed`, `checkRollingBudgets`, `mode.Project` |
+| `ResolveChapter` | `{RunRef, ChapterID, ReusesDrafts}` → `{Reused int, Fresh []SegmentID}` | `lookupTranslation` / TM part of `resolveTranslation`; writes reused targets to the unit file |
+| `DraftChapter` | `{RunRef, ChapterID, Fresh, PersistDrafts}` → `{Chars, Engine}` | `draftSegments` (glossary protect/apply, one batched NMT call); persists drafts only in NMT mode |
+| `EscalateChapter` | `{RunRef, ChapterID, Fresh}` → `{Attempts, Chars, Improved, QESum, QECount}` | NMT-mode QE + `SafePostEdit` loop (`orchestrator.go:627-676`) |
+| `RefineBatch` | `{RunRef, ChapterID, IDs, PrevTail}` → `{Usage, Refined, Degraded []string, Tail}` | one literary LLM batch, halving-on-truncation included; heartbeats between sub-calls |
+| `CommitChapter` | `{RunRef, ChapterID, Persist}` → `{GlossaryViolations, RestoreFailures}` | re-derive placeholders, `Restore` as validation, persist to TM **only after** Restore accepted (`mode.go:344-368`), write final units |
+| `UpdateBible` | `{SeriesID, ChapterID, RunRef}` → `{Version}` | `updateBible` with its `SaveIfVersion` loop |
+| `RecomposeBook` | `{RunRef}` → `{OutputEPUB}` | `epub.ApplyTranslations` + `Recompose` |
+| `CheckEPUB` | `{OutputEPUB}` → `{Passed, Skipped}` | `qa.EPUBChecker` |
+| `FinishBook` | `{RunRef, RunResult}` → `qa.Report` | `qa.Evaluate` + `WriteArtifacts` |
+| `RecordSpend` | `RunSpend + IdempotencyKey` | `TM.RecordRun`, made idempotent on the Temporal `(WorkflowID, RunID)` |
+| `PriceRun` (local activity) | `{RunResult, UsedFreeChars, Mode}` → `USD` | `mode.Cost`; local because it is pure arithmetic over the cost model |
+
+`RefineBatch` is per batch rather than per chapter on purpose: it is the
+retry and resume unit. A worker killed after batch 2 of 4 restarts at batch 3.
+The recursive halving that exists inside `TranslateChapter` today stays inside
+the activity — sub-batches are the same logical unit and share its usage.
+
+### The book workflow
+
+```mermaid
+sequenceDiagram
+ participant C as CLI
+ participant T as Temporal
+ participant W as Worker
+ C->>T: StartWorkflow TranslateBook
+ T->>W: PrepareBook
+ T->>W: ProjectCost
+ Note over T: cap exceeded → fail ErrBudgetExceeded, no provider call
+ loop each chapter, in order
+ T->>W: ResolveChapter
+ T->>W: DraftChapter
+ alt literary
+ loop each batch
+ T->>W: RefineBatch
+ Note over T,W: RateLimited → timer or resume signal, then the same batch again
+ end
+ else nmt
+ T->>W: EscalateChapter
+ end
+ T->>W: CommitChapter
+ opt update-bible
+ T->>W: UpdateBible
+ end
+ C-->>T: query progress
+ end
+ T->>W: RecomposeBook · CheckEPUB · FinishBook
+ T->>W: RecordSpend
+ T-->>C: RunResult → exit code
+```
+
+Failure and cancellation paths run `FinishBook` and `RecordSpend` under a
+disconnected context (`workflow.NewDisconnectedContext`), which is the Temporal
+form of today's deferred ledger write.
+
+### Rate limit
+
+`RefineBatch` (and `DraftChapter` for a paid NMT provider) surface
+`llm.ErrRateLimited` as a **non-retryable** application error of type
+`RateLimited`. The workflow — not the activity retry policy — handles it:
+
+1. state ← `paused(rate_limit)`, visible in the UI and the `progress` query;
+2. wait on whichever fires first: a timer (30 min, doubling per consecutive
+ wait, ceiling 5 h) or a `resume` signal;
+3. re-execute the **same** batch. Bounded by a maximum number of waits, after
+ which the run fails as it does today.
+
+The completion cache (`llm_cache.sqlite`, keyed by model + messages) makes a
+replayed call that had already succeeded free, so an activity that timed out
+*after* the provider answered is not paid twice.
+
+### Signals and queries
+
+| Kind | Name | Effect |
+|---|---|---|
+| Signal | `pause` / `resume` | Stop before the next activity / continue. `resume` also ends a rate-limit wait early. |
+| Query | `progress` | `{State, Stage, Chapter, Chapters, Segments, SegmentsDone, TMHits, CostUSD}` — polled by the CLI at ~500 ms to drive the existing bar. |
+| Cancel | `CancelWorkflow` | Bound to SIGINT in the CLI; runs the cleanup path above. |
+
+The bar is fed by Temporal, not by the pipeline. The workflow keeps a
+`Progress` struct it updates from each activity's result and serves through
+`workflow.SetQueryHandler("progress", …)`; the CLI calls `QueryWorkflow` on a
+~500 ms ticker and forwards the values to `internal/progress`, which does not
+change. A query is read-only and leaves no history event.
+
+Granularity follows activity boundaries: the bar moves when an activity
+completes. In literary mode that is the batch — the same cadence `OnBatch` gives
+today. In NMT mode with escalation, `EscalateChapter` is one activity, so the bar
+would hold for a whole chapter; if that matters, the escalation is split per
+batch like the refinement is. It is a choice of activity size, not of design.
+
+### Policies
+
+| Activity | StartToClose | Heartbeat | Attempts, backoff | Non-retryable |
+|---|---|---|---|---|
+| `PrepareBook` | 5 m | — | 3 | `epub.Err*` (a bad archive does not get better) |
+| `ProjectCost`, `ResolveChapter`, `CommitChapter`, `PriceRun` | 2 m | — | 3 | — |
+| `DraftChapter` | 10 m | 30 s | 3, 1 s → 30 s ×2 | `RateLimited` (→ pause), non-429 4xx |
+| `RefineBatch` | 15 m | 60 s | 3, 5 s → 2 m ×2 | `RateLimited` (→ pause), refusal (→ degrade, returned not raised) |
+| `EscalateChapter` | 30 m | 60 s | 3 | — |
+| `UpdateBible` | 10 m | 60 s | 3 | `narrative.ErrInvalidBible` |
+| `RecomposeBook`, `CheckEPUB` | 10 m | — | 2 | — |
+| `FinishBook`, `RecordSpend` | 1 m | — | 5 | — |
+
+No workflow execution timeout: a paused run is a legitimate state. Worker:
+`MaxConcurrentActivityExecutionSize = 1` (condition 6).
+
+### Determinism and versioning
+
+Workflow code uses `workflow.Now`, `workflow.GetLogger`, `workflow.SideEffect`
+for anything non-deterministic, no goroutines, no map iteration that affects
+decisions. Policy for in-flight runs: do not upgrade the worker mid-book; if a
+change to `TranslateBook` must ship while a run is paused, gate it with
+`workflow.GetVersion`. A replay test over a recorded history of the fixture
+book runs in CI and fails on an accidental non-deterministic change.
+
+### Idempotency
+
+Activities can execute more than once (timeout after success, worker death).
+
+- `TM.Save` is an upsert on `source_hash` — idempotent already.
+- `RecordRun` appends one row per attempt on purpose (`ARCHITECTURE.md §16.4`).
+ A retried *activity* is not a new attempt: the row gains an `attempt_key`
+ (Temporal `WorkflowID/RunID`) with a unique index, `INSERT OR IGNORE`.
+- `WriteArtifacts` overwrites — idempotent.
+- A retried `DraftChapter` re-pays the NMT characters. Accepted; that is the
+ contract `RetryingTranslator` has today (`retry.go:43-45`).
+- A retried `RefineBatch` is served by the completion cache when the previous
+ attempt got an answer.
+
+### Payload and history budget
+
+For vol33: manifest ≈ 45 × 100 B; a chapter plan ≈ 100 IDs × 20 B ≈ 2 KB; a
+`RefineBatch` input is IDs plus a 1,200-rune tail ≈ 3 KB; every result is
+counters. All two orders of magnitude under the 256 KB warning. History: per
+chapter ≈ 3 fixed activities + ~2.3 batches, ≈ 5 events each → ≈ 30 events per
+chapter, ≈ 1,400 per book, ≈ 7 % of the soft limit. A 30-book series is 30
+child histories, not one. No `ContinueAsNew` needed.
+
+### Code layout and coupling
+
+Portability is a stated goal: the code must be movable to another engine (or
+back to none) by rewriting one package. The layout is ports-and-adapters with
+the whole business — stages **and** control flow — on the Temporal-free side.
+
+```mermaid
+flowchart TB
+ subgraph core["Temporal-free — imports no go.temporal.io"]
+ S["pipeline.Stages the work: PrepareBook … RecordSpend ctx + serialisable in/out"]
+ F["pipeline.RunBook / RunSeries the control flow, written once"]
+ P["ports: pipeline.Steps · pipeline.Runtime"]
+ D["epub · segment · tm · translate · literary · qa · cost · narrative"]
+ F --> P
+ S --> D
+ end
+ subgraph adapter["internal/temporal — the only package importing the SDK"]
+ A["activities = Stages methods, registered"]
+ ST["Steps impl → workflow.ExecuteActivity"]
+ RT["Runtime impl → NewTimer · signal channels NewDisconnectedContext · ExecuteChildWorkflow"]
+ WF["TranslateBook(workflow.Context) = RunBook(steps, runtime, in)"]
+ end
+ subgraph inproc["pipeline/inprocess.go — test double and portability proof"]
+ IS["Steps impl → direct Stages call"]
+ IR["Runtime impl → time.Timer · channels"]
+ end
+ ST -.implements.-> P
+ RT -.implements.-> P
+ IS -.implements.-> P
+ IR -.implements.-> P
+ A --> S
+ WF --> F
+```
+
+Two ports, both small:
+
+```go
+// Steps is what the control flow can ask for. One method per activity, no
+// context: the adapter owns it (workflow.Context is not a context.Context).
+type Steps interface {
+ PrepareBook(BookInput) (Manifest, error)
+ ProjectCost(ProjectInput) (Projection, error)
+ ResolveChapter(ResolveInput) (ChapterPlan, error)
+ DraftChapter(DraftInput) (DraftResult, error)
+ EscalateChapter(EscalateInput) (EscalateResult, error)
+ RefineBatch(RefineInput) (RefineResult, error)
+ CommitChapter(CommitInput) (CommitResult, error)
+ UpdateBible(BibleInput) (BibleResult, error)
+ RecomposeBook(RunRef) (RecomposeResult, error)
+ CheckEPUB(CheckInput) (CheckResult, error)
+ FinishBook(FinishInput) (qa.Report, error)
+ RecordSpend(SpendInput) error
+ PriceRun(PriceInput) (float64, error)
+}
+
+// Runtime is what the control flow needs from an engine, and nothing more.
+type Runtime interface {
+ Now() time.Time
+ // WaitResume blocks for d or until a resume signal, whichever first.
+ WaitResume(d time.Duration) (resumed bool, err error)
+ Paused() bool // a pause signal is pending
+ Cancelled() bool
+ Detached() Runtime // for cleanup after cancellation
+ Report(Progress) // the flow publishes; in-process drives the bar, Temporal serves the query
+ RunBook(BookInput) (RunResult, error) // series only: one child per volume
+}
+```
+
+`Stages` is the concrete struct holding today's collaborators
+(`orchestrator.go:43-72`); its methods take a real `context.Context` and are the
+activity bodies. `RunBook` accumulates `RunResult`, publishes a `Progress`
+snapshot through `rt.Report` at every step the bar used to advance on (the
+adapter keeps the last one for the query), and follows the determinism
+rules (`rt.Now()`, no goroutines, no decision on map order) — which is what lets
+the same function run inside a Temporal workflow and inside a plain test.
+
+What the adapter does **not** abstract: Temporal's own concerns — payload
+codecs, task queues, worker options, `GetVersion`, replay tests. They live in
+`internal/temporal` and go away with it.
+
+The tradeoff, stated: the ports are a fixed vocabulary. A Temporal feature the
+flow wants (an Update, a Nexus call) needs a port method first. For a 13-step
+sequential pipeline that is the right price; the alternative — writing the flow
+in `workflow.*` calls — is ~300 lines that would be rewritten on any engine
+change, together with every test that exercises them.
+
+```
+internal/
+ pipeline/ Temporal-free
+ types.go BookInput · Manifest · ChapterPlan · Projection · RunResult · Progress
+ stages.go Stages struct + methods (the activity bodies)
+ ports.go Steps · Runtime
+ book.go RunBook — the flow (replaces Orchestrator.Run + mode.go loops)
+ series.go RunSeries
+ inprocess.go in-process Steps + Runtime (tests; portability proof)
+ workspace.go units.jsonl per chapter
+ plan.go Plan, for estimate (unchanged)
+ artifacts.go · errors.go · doc.go
+ epub/ ApplyTranslations(book, units) — new
+ temporal/ the adapter, only SDK importer
+ client.go Dial from config
+ worker.go registers TranslateBook, TranslateSeries, Stages methods
+ workflow.go TranslateBook / TranslateSeries glue, progress query, signals
+ steps.go Steps → ExecuteActivity with per-step options
+ runtime.go Runtime → timers, signal channels, disconnected ctx, child workflows
+ policy.go timeouts and retry per step
+ errors.go StageError ↔ ApplicationError type
+ doc.go
+ commandexec/ worker.go · run/batch start a workflow and follow it · estimate → pipeline.Plan
+ config/ TemporalAddress · TemporalNamespace · TemporalTaskQueue
+```
+
+The import rule is enforced, not hoped for: a `depguard` rule in
+`.golangci.yml` denies `go.temporal.io/**` everywhere except `internal/temporal`
+and `internal/commandexec`. `golangci-lint` already gates every commit.
+
+- New subcommand `worker`. It is the process wrapped in `op run`; `run` and
+ `batch` no longer need a credential — which also retires the pipe-vs-tty
+ detection of session 19, since the bar now lives in an unwrapped process.
+- `run` keeps its flags and blocks until the workflow ends; exit codes map from
+ the `ApplicationError` type (`extract`, `parse`, `segment`, `translate`, `qa`,
+ `recompose`) to `internal/exitcode` unchanged.
+- `just up` starts `temporal` + `worker`; `just run` starts a workflow.
+- `estimate` calls `pipeline.Plan` directly, as today.
+
+### Extension: adding a translation mode
+
+A mode is a recipe over the existing `Steps`, never a new workflow. `RunBook`
+selects a `Mode` from `BookInput.Mode`, which the workflow forwards without
+reading:
+
+```go
+type Mode interface {
+ ReusesDrafts() bool
+ Project(pending []ChapterInfo, usedFreeChars int) cost.Projection
+ Cost(result *RunResult, usedFreeChars int) float64
+ Chapter(f *flow, ch ChapterInfo) error // the per-chapter recipe, composed of Steps calls
+}
+```
+
+`nmtMode.Chapter` = Resolve → Draft → Escalate → Commit. `literaryMode.Chapter` =
+Resolve → Draft → RefineBatch × n → Commit (→ UpdateBible). An LLM-only mode,
+the example asked for, is Resolve → RefineBatch × n (no draft) → Commit:
+
+| Layer | Change | Size | Touches the adapter |
+|---|---|---|---|
+| `literary` | prompt variant without a draft (`DraftText` optional, `RefineInput.HasDraft`) | ~40 lines + decode tests | no |
+| `pipeline` | `llmMode` struct; `Project` reuses the existing `full-llm` scenario (`cost/estimate.go:296`) | ~100 lines + `RunBook` scenarios | no |
+| `internal/temporal` | nothing — no new activity, `RefineBatch` accepts an empty draft | 0 | — |
+| `cli` | `--mode llm` accepted | ~10 lines | no |
+| docs | `usage.html` §6, `understanding-the-project.html` §1 and recipe 3 | — | — |
+
+A mode that needs a **new** step costs, in the adapter, one `Steps` method, one
+`Stages` method, one activity registration and one `policy.go` line — ~30
+mechanical lines; `depguard` flags anything more.
+
+The real cost of an LLM-only mode is not code: the placeholder safety net falls
+back on the NMT draft today, and without a draft it would fall back on the
+English source. Either retry the degraded segment alone, or request a draft
+on demand for the degraded ids (`DraftChapter` on a subset — the step exists).
+To decide when that mode is built; `untranslated_segments` in QA catches the
+failure either way.
+
+## Implementation Plan
+
+Each phase is one PR, green on `just check`, and leaves the tool usable.
+
+1. **Decouple the stages from the process** (no Temporal, no new dependency).
+ `epub.ApplyTranslations`; `pipeline/workspace.go` writing and reading per-chapter
+ unit files; `Orchestrator.Run` and `mode.go` refactored into `pipeline.Stages`
+ methods with serialisable inputs and outputs; the `Steps` and `Runtime`
+ ports; `RunBook` as the flow over them; the in-process `Steps`/`Runtime`
+ pair, which is what the CLI runs on until phase 6. Acceptance:
+ `translated.epub` and `report.json` byte-identical on the fixture book before
+ and after; vol33 chapter 97 (all TM hits) identical; `orchestrator_test.go`
+ scenarios re-expressed as `RunBook` tests on the in-process runtime.
+2. **Infrastructure.** `temporal` service in `docker-compose.yml`
+ (`temporalio/temporal server start-dev --ip 0.0.0.0 --db-filename
+ /data/temporal.db`, ports 7233 and 8233, pinned tag); `just temporal-up /
+ temporal-down` folded into `up` / `down`; `preflight` probes 7233; the three
+ `TOMESHIFT_TEMPORAL_*` variables; `DEPENDENCIES.md` entry for
+ `go.temporal.io/sdk`.
+3. **Worker and the NMT workflow.** `internal/temporal` (client, worker,
+ `Steps`/`Runtime` adapters, policy, error mapping), `worker` subcommand,
+ `run --mode nmt` through Temporal; `progress` query wired to the bar; SIGINT
+ → cancel → cleanup; ledger `attempt_key`; `depguard` rule. Tests: adapter
+ wiring with `testsuite`, first recorded history for the replay test, one
+ integration test against the dev server that skips when 7233 is unreachable
+ (the `ollamaHost` pattern).
+4. **Literary mode and the rate-limit pause.** `RefineBatch` per batch,
+ `pause`/`resume`, cooldown timer, `UpdateBible`. The missing test of session
+ 18 becomes a `RunBook` test on the in-process runtime: fake `RefineBatch`
+ returns `RateLimited` on the third call → state `paused` → `resume` →
+ completes, and the number of paid calls equals a clean run's.
+5. **Series.** `TranslateSeries` with sequential children; `batch` command
+ through it; `batch-report.json` assembled from child results.
+6. **Cutover and documentation.** Delete the in-process engine as the CLI's
+ default (`pipeline/inprocess.go` stays, test-only), `RetryingTranslator`,
+ `OnBatch`, `costFn`, the pipeline progress hooks and the
+ tty probe. Update `ARCHITECTURE.md` §5, §7, §9, §10, §13; rewrite M4 around
+ what Temporal now provides; `usage.html` (§4 services, §6 `worker`, §8
+ variables, §11 recipes), `understanding-the-project.html` (§1 widget, §11 map,
+ §13 invariants); README; `just docs-audit`.
+
+## Testing Strategy
+
+Five levels, fastest first. The first four need no server and run in
+`just check`; the fifth starts the dev server as a CI service container (it
+boots in ~2 s) and skips locally when 7233 is unreachable — the `ollamaHost`
+pattern. **The Temporal SDK test API is used only at level 3**, on the one
+package that imports the SDK.
+
+1. **Stages** — the existing package tests of `epub`, `segment`, `tm`,
+ `literary`, `qa`, `cost`. Unchanged: business logic does not move packages,
+ activities are wrappers.
+2. **Flow** — `pipeline.RunBook` / `RunSeries` on the in-process `Runtime` with
+ a fake `Steps` (a struct of func fields recording calls and returning
+ scripted results). Plain Go, no SDK. This is where control flow,
+ accumulation, pause, resume, cancel and error mapping are proven — the
+ scenarios of `orchestrator_test.go` land here in phase 1.
+3. **Adapter** — `internal/temporal` with `testsuite.WorkflowTestSuite`,
+ `TestActivityEnvironment`: activity options per step, timer ↔ `WaitResume`,
+ signal channel ↔ `Paused`/`resume`, `StageError` ↔ `ApplicationError`
+ type, the `progress` query. Wiring only, no business scenario.
+4. **Replay** — `worker.NewWorkflowReplayer` over a recorded history of the
+ fixture book in `internal/temporal/testdata`. Fails on non-determinism —
+ including in `RunBook`, since it executes inside the workflow.
+5. **Integration** — `tests/integration` against the dev server, with
+ `nmt-devserver` and `llmtest` as providers.
+
+Per phase:
+
+| Phase | Proves | How | Existing tests |
+|---|---|---|---|
+| 1 | Re-deriving placeholders from `{id, target}` renders the same book as in-place mutation; the flow behaves as `Run` did | `epub.ApplyTranslations` table-driven: identity round trip (source applied as target → identical bytes), unknown id → sentinel, missing placeholder → segment left as source. `workspace` round trip, corrupt line → error. Level 2: `RunBook` scenarios ported from `orchestrator_test.go` (nominal sums, cap refused with no `DraftChapter` call, cancel → `FinishBook` + `RecordSpend`, stage error → exit code). Golden sha256 of the fixture's `translated.epub` and `report.json`, frozen **before** the refactor | `orchestrator_test.go` scenarios move to `book_test.go`; the file is deleted with `Run` |
+| 2 | The service starts and the CLI sees it | `preflight` probe reachable / unreachable; manual smoke `just up` → UI; `docs-audit` for the variables | — |
+| 3 | The adapter dispatches, maps and serves what the flow expects | Level 3: each `Steps` method reaches the activity of the same name with its options; `WaitResume` returns on timer and on signal; error type mapping both ways; query returns `Progress()`. Level 4: first recorded history. Level 5: fixture end to end; *worker kill* = cancel the first worker's context mid-run, start a second, assert completion and that `nmt-devserver`'s request count equals a clean run's. Payload test: every serialised input/output < 256 KB. `depguard` forbids the SDK outside the adapter | none removed; `RetryingTranslator` stays tested until phase 6 |
+| 4 | A 429 pauses, resumes **the same batch**, and does not bill the batches that succeeded | Level 2: fake `RefineBatch` returns `RateLimited` on the 3rd call then succeeds; in-process `WaitResume` fed by a test channel; asserts `paused` in `Progress()`, calls = clean run + 1, tokens = clean run. Max waits → failure. `pause`/`resume` outside a 429. Level 5: `llmtest` answers 429 on the Nth call — the integration test session 18 left missing | `literary/translator_test.go` unchanged: batch splitting stays in the package |
+| 5 | Volumes run in order; one failure does not stop the series | Level 2: `RunSeries` with a fake `Runtime.RunBook`: start order; one child fails → parent continues, summary counts it. Level 3: `RunBook` port → `ExecuteChildWorkflow` | — |
+| 6 | Nothing the deleted code proved goes unproven | `retry_test.go` goes with `RetryingTranslator`; `Plan` and its tests stay for `estimate`; `inprocess.go` stays as the level-2 runtime | deleted with the code they covered, never re-pinned to the new text |
+
+## Rollout Plan
+
+- One environment: the developer's machine. Phases 1–2 change nothing a user sees.
+- Phases 3–5 add the Temporal path while the in-process path stays the default;
+ `TOMESHIFT_ENGINE=temporal` selects the new one during that window and
+ is removed in phase 6.
+- Rollback: before phase 6, unset the variable; after phase 6, revert the phase 6 PR.
+
+## Impact
+
+### Technical
+- Reliability: retries, timers, resume and cleanup move from six bespoke
+ mechanisms to one engine with a UI.
+- Maintainability: less code in `pipeline` (the loops and hooks go), more in
+ `workflow` (declarative control flow). The stage packages do not change shape.
+- Testability: the "429 in the middle of a volume" scenario becomes a unit test.
+- Cost of complexity: determinism discipline in one package; a server to run.
+
+### Cost
+- Infrastructure: one extra container, SQLite-backed, zero external cost.
+- Provider: unchanged in the nominal case; strictly lower after failures,
+ because a batch is never paid twice and a 429 no longer discards a chapter.
+
+### Security/Compliance
+- Secrets move to the worker process only. Payloads carry no book text beyond
+ a 1,200-rune context tail, and the dev server's SQLite file lives under
+ `data/` next to the TM and the bible, with the same handling.
+
+## Acceptance Criteria
+
+- `kill -9` the worker mid-chapter, restart it: the run completes, and the
+ ledger's LLM call count equals a clean run's (no batch paid twice).
+- A 429 injected on batch k: the run shows `paused` in UI and CLI; a `resume`
+ signal completes it; no re-extraction, no re-projection appears in history.
+- vol33 end to end through Temporal: same QA gate verdict, `report.json`
+ schema unchanged, `actual_cost_usd` within 1 % of the in-process run.
+- Largest payload on vol33 < 256 KB; history < 10,240 events per book.
+- `estimate` succeeds with the Temporal container stopped.
+- `go.mod` gains `go.temporal.io/sdk` and its transitive closure, nothing else.
+
+## Risks and Mitigations
+
+- Risk: non-deterministic workflow code found only in production.
+ - Mitigation: replay test in CI; workflow package reviewed against the
+ determinism rules; no I/O outside activities.
+- Risk: dependency weight and version drift between SDK and server.
+ - Mitigation: pin both; the compose image tag and `go.mod` move together.
+- Risk: `run` now needs a running server.
+ - Mitigation: `just up` starts it; `preflight` says so plainly; `estimate` is
+ exempt by construction.
+- Risk: two activities write SQLite at once.
+ - Mitigation: one worker, one activity slot (condition 6); lifted with RFC 0005.
+- Risk: payload creep as inputs grow.
+ - Mitigation: a test asserts payload sizes on the fixture book against the
+ 256 KB warning threshold.
+- Risk: refactoring `Restore` into a re-derivation step changes output.
+ - Mitigation: phase 1 acceptance is byte identity on the fixture and on a
+ fully-cached real chapter.
+
+## Open Questions
+
+1. ~~Cap exceeded: fail or wait for `approve-budget`?~~ **Decided 2026-09-10: fail,
+ as today.** No `approve-budget` signal; the `ProjectCost` step returns
+ `ErrBudgetExceeded` and the run ends before any provider call.
+2. ~~Rate-limit cooldown.~~ **Decided 2026-09-10: 30 min initial, doubling, ceiling
+ 5 h** (the Claude CLI quota window). `TOMESHIFT_RATE_LIMIT_COOLDOWN`
+ and `..._COOLDOWN_MAX` carry the two values.
+3. ~~`--detach` + `status`, or blocking?~~ **Decided 2026-09-10: blocking.** `run`
+ starts the workflow, polls the `progress` query to drive the existing bar,
+ waits for the result and maps it to the exit code. No `--detach` in this RFC.
+4. ~~One task queue or two?~~ **Decided 2026-09-10: one** (`tomeshift`).
+5. ~~Temporal Schedules for a nightly `batch`.~~ **Decided 2026-09-10: after phase 5,
+ only if a need appears.** Not in this RFC's plan.
+
+## Decision Log
+
+- 2026-09-10: status `draft`. Written after the vol33 preparation (sessions 18–19)
+ showed the resume boundary and the manual re-launch to be the operational cost
+ of the current engine.
+- 2026-09-10: the five open questions were decided (fail on cap, 30 min → 5 h
+ cooldown, blocking `run`, one task queue, no Schedules); status `proposed`.
+- 2026-09-10: phases 1 and 2 implemented on `refactor/to-temporal` (uncommitted,
+ pending review). Golden `tests/integration/testdata/golden.json` frozen on the
+ old engine before the refactor and unchanged after it. Deltas to the design
+ above, all inside the "small serialisable DTO" latitude: `Progress.Segments`
+ (book total, keeps the bar's percentage); `RunRef.InputPath` and
+ `Manifest.BatchSize` (re-parse activities and literary chunking need them);
+ `RefineInput.PrevChapterID` (first batch of a chapter gets the previous
+ chapter's *source* tail, read from the workspace — byte-identical prompts,
+ cache keys preserved); the NMT meter baseline is taken per `DraftChapter`
+ and summed by the flow (the flow holds no translator); `ProjectCost` and
+ `PriceRun` keep the cost model inside `Stages`, so `Mode` is
+ `{ReusesDrafts, Chapter}`; `literary.TranslateChapter` deleted, its tests
+ split between `TranslateBatch` and `RunBook`.
diff --git a/docs/rfcs/0010-llm-only-literary-translation.md b/docs/rfcs/0010-llm-only-literary-translation.md
new file mode 100644
index 0000000..0c56aa4
--- /dev/null
+++ b/docs/rfcs/0010-llm-only-literary-translation.md
@@ -0,0 +1,227 @@
+# RFC 0010: LLM-Only Literary Translation (`--no-draft`)
+
+- Status: `draft`
+- Authors: TBD
+- Created: 2026-09-10
+- Updated: 2026-09-11
+- Related:
+ - Architecture: `docs/ARCHITECTURE.md`
+ - Milestones: `docs/milestones/`
+ - Builds on: `0006-faithful-multivolume-translation.md`, `0007-google-translate-draft-and-token-cost-model.md`
+ - Depends on: `0009-temporal-execution-engine.md` (phase 1 ports/stages — the symbols this plan targets)
+ - Supersedes: n/a
+ - Superseded by: n/a
+
+## Context
+
+We want to be able to translate a book **using only an AI model** and do the
+literary pass **at the same time** (a single LLM pass instead of an NMT draft
+followed by a rewrite).
+
+The `literary` mode is two-pass: an NMT draft then an LLM rewrite. Since the
+RFC 0009 refactor (phase 1), these two passes are **stages** (`Stages`) chained
+by the `literaryMode.Chapter` recipe
+(`internal/pipeline/modes.go`): `Resolve -> Draft -> Refine x n -> Commit`.
+Point verified in the code: **the LLM translates from the source, not the
+draft**.
+
+- `Stages.RefineBatch` (`internal/pipeline/stages.go`) calls
+ `literary.TranslateBatch`, whose `buildPayload`
+ (`internal/literary/translator.go`) sends `source_text` per segment, the full
+ chapter source living in the cacheable prefix (`stablePrefix`,
+ `internal/literary/prompt.go`).
+- `draft_text` (`requestSegment.DraftText`) is only a **hint** in the prompt.
+- The NMT draft (`Stages.DraftChapter`) serves mainly as a **safety net**:
+ `literary.applyTargets` falls back to the draft when a batch fails or breaks
+ the placeholders; the segment stays `StatusTranslated` and is **not
+ persisted** (so it remains resumable).
+
+In other words: "LLM only + literary at the same time" = **literary recipe
+without the `DraftChapter` stage**. The ports architecture is already ~95%
+there: the recipe composes stages, you just skip one.
+
+```mermaid
+flowchart LR
+ subgraph Today["literary (draft NMT + LLM)"]
+ S1[source] --> D1[DraftChapter NMT] --> R1[RefineBatch LLM from source] --> C1[CommitChapter]
+ D1 -. safety net if a batch fails .-> C1
+ end
+ subgraph Target["draft=none (LLM only)"]
+ S2[source] --> R2[RefineBatch LLM from source] --> C2[CommitChapter]
+ end
+```
+
+## Goals
+
+- Translate an EPUB with the LLM alone, with no NMT call, the literary quality
+ (bible, glossary, style) applied in the same pass.
+- Require no NMT credential when no NMT call will occur, as per the
+ `RequiresAPIKey`/`estimate` invariant.
+- Align estimation and runtime billing for this path.
+
+## Non-Goals
+
+- Introduce a third translation loop. Project memory sets in stone that "two
+ loops cover 100% of translation time"; we add an **axis** (draft source), not
+ a mode.
+- Change the glossary contract, the QA gate, the budget, or the artifact format.
+
+## Options Considered
+
+### Option A: `--no-draft` flag (`draft = nmt | none` axis on the `literary` mode)
+- Pros:
+ - Reuses the whole literary recipe (bible, glossary tokens, batching, budget,
+ ledger, QA gate) as-is; minimal delta.
+ - Persistence unchanged in `EngineLiterary` -> **same TM space** as classic
+ literary, reusable both ways, consistent with the per-mode reuse predicate
+ (`Mode.ReusesDrafts`).
+- Cons:
+ - A single mode carries two behaviors; the CLI surface is a flag, not a
+ visible mode.
+
+### Option B: new `--mode llm` mode
+- Pros:
+ - Explicit CLI surface.
+- Cons:
+ - Duplicates `literaryMode` (recipe + projection + pricing) for a one-line
+ delta.
+ - Fragments the TM space (`EngineLLM` != `EngineLiterary`).
+ - Contradicts the "two loops" principle.
+
+## Proposed Decision
+
+**Option A.** A `--no-draft` flag (and the `TOMESHIFT_LITERARY_DRAFT` env)
+which, in `none`, skips the `DraftChapter` stage: the LLM translates directly
+from `source_text`. Chosen degradation policy: **degrade to source** (untranslated
+segment, counted by QA, not persisted, resumed by `--resume`) — refined by the
+refactor's chapter guardrail (see §Impact).
+
+## Implementation Plan
+
+The anchors below target the RFC 0009 ports architecture (phase 1).
+
+### Config — `internal/config/config.go`
+1. Add `LiteraryDraft string` (env `TOMESHIFT_LITERARY_DRAFT`, default `nmt`),
+ validated in `{nmt, none}` with a sentinel in `internal/config/errors.go`.
+2. Expose `--no-draft` on `run` and `literary` (`internal/cli`); the flag forces
+ `none` and takes precedence over the env, like the other flags.
+
+### Flow contract — `internal/pipeline/types.go`
+3. Add `LiteraryDraft string` to **`BookInput`** (and not to an old `RunConfig`,
+ removed by the refactor). `commandexec` fills it from the config.
+
+### Recipe — `internal/pipeline/modes.go`
+4. `literaryMode.Chapter`: call `f.draft(ch, plan.Fresh, false)` only if
+ `draft == nmt`. In `none`, the recipe becomes `Resolve -> Refine x n ->
+ Commit`. The `draft` choice is read from `f.in.LiteraryDraft`.
+
+### Accumulation — `internal/pipeline/book.go`
+5. In `none`, `f.draft` is not called: `NMTSegments` and `NMTChars` stay 0 (no
+ `DraftChapter`, hence no NMT counter delta). Fresh segments are already
+ counted on the LLM side by `f.refine` via `LLMPassSegments`. Nothing else to
+ change in the accumulation.
+
+### Stages — `internal/pipeline/stages.go`
+6. `DraftChapter` is simply not invoked in `none`; `Stages.Translator` can stay
+ nil. Verify the literary paths that touch the translator: `nmtEngine()` and
+ `charsSent()` already handle a nil translator (returning `""` / 0);
+ `RefineBatch`/`CommitChapter`/`ProjectCost` do not dereference it.
+7. Cost: `projectMode`/`costLiterary`/`literaryPass` (`stages.go`) drop the NMT
+ draft component in `none` (cost = LLM tokens only, `NMTChars = 0`).
+
+### Wiring — `internal/commandexec/translate.go` + `estimate.go`
+8. `newStages`: no longer require an NMT endpoint/key when
+ `mode == literary && draft == none`, and do not build the NMT provider
+ (leave `Stages.Translator` nil). Same invariant as `RequiresAPIKey`/`estimate`:
+ no secret demanded for a call that will not happen.
+9. `estimate` (`Stages.Plan`): build the NMT `Translator` only if an NMT draft
+ will occur, otherwise the LLM-only estimation demands the Google key we just
+ eliminated.
+
+### Global cost — `internal/cost/estimate.go`
+10. `EstimateScenarios`: the `none` projection aligns with the full-LLM scenario
+ (no NMT component), so that `estimate --mode literary --no-draft` matches
+ runtime billing.
+
+## Rollout Plan
+
+- Local first: `estimate --mode literary --no-draft` (zero calls), then a run on
+ a short volume with an LLM (OpenAI-compatible endpoint or the Claude CLI).
+- Safety: `nmt` stays the default; `none` is opt-in.
+- Rollback: removing the flag/env leaves the literary mode intact.
+
+## Impact
+
+### Technical
+- Without a draft, a failed LLM batch leaves its segments **on the source**
+ (untranslated) instead of a readable NMT. Two regimes, inherited from the
+ refactor:
+ - **Partial degradation**: some batches succeed, others do not -> the failed
+ segments stay source, `StatusTranslated`, not persisted, counted by QA
+ (`UntranslatedSegments`), resumed by `--resume`.
+ - **Fully failed chapter**: if **no** segment of a chapter is refined,
+ `flow.refine` returns `ErrChapterUnrefined`
+ (`internal/pipeline/errors.go`) and **the run fails outright** — we do not
+ deliver a source chapter disguised as finished. This is a hardening relative
+ to the original "degrade silently", and it lands perfectly for LLM-only.
+- Literary recipe reused without duplication; one stage skipped.
+
+### Cost
+- `NMTChars = 0` on this path; cost driven by LLM tokens. The Google free tier
+ no longer comes into play.
+
+### Security/Compliance
+- No NMT credential required in `none`; one less secret to load.
+
+## Acceptance Criteria
+
+- `none`: no NMT call emitted (`Stages.Translator` nil, `DraftChapter` not
+ invoked), the LLM translates from the source, complete output.
+- Partial degradation: a failed LLM batch leaves its segments on source,
+ `StatusTranslated`, not persisted, counted by QA; `--resume` resumes them.
+- Fully failed chapter in `none` -> `ErrChapterUnrefined`, run fails (no silent
+ delivery).
+- Startup without an NMT credential succeeds in `none` (mirror of the existing
+ `estimate` test); still fails in `nmt`.
+- Projection/`estimate` in `none`: `NMTChars = 0`, cost = LLM tokens;
+ estimation == runtime.
+- Glossary `` preserved without the draft (`Protect`/`Apply` contract
+ unchanged).
+
+## Risks and Mitigations
+
+- Risk: a nil `Stages.Translator` dereference on an unforeseen literary path.
+ - Mitigation: audit of translator usages in literary mode + a literary
+ `RunBook` test with `Stages.Translator == nil` and `draft == none`.
+- Risk: estimation/runtime divergence if only one of the two cost paths
+ (`stages.go` vs `cost.EstimateScenarios`) is updated.
+ - Mitigation: cross-test `estimate` vs run on the same book in `none`.
+- Risk: unexpected interaction with the `ErrChapterUnrefined` guardrail.
+ - Mitigation: recipe test where all batches of a chapter degrade in `none` ->
+ run fails, spend recorded.
+
+## Open Questions
+
+- Should we keep `draft_text` (empty) in the payload in `none`, or omit it to
+ save tokens? (Cosmetic, decision at implementation time.)
+
+## Documentation (living docs — same change)
+
+- `docs/usage.html`: `--no-draft` in §6 (flags table) **and** the `FLAGS` array
+ (processing category); `TOMESHIFT_LITERARY_DRAFT` in §8; `--mode`/pipeline
+ values in §5-6.
+- `docs/understanding-the-project.html`: pipeline widget §1 (the two paths);
+ invariant §13 ("draft = safety net, not translation source"; and "an LLM-only
+ chapter with no refinement at all fails, it does not degrade silently").
+- `docs/ARCHITECTURE.md` if the degradation contract evolves.
+- `just docs-audit` must pass; new session in `docs/journal.html`.
+
+## Decision Log
+
+- 2026-09-10: created as `draft`. Choices validated in session: Option A
+ (`--no-draft`), degradation to source, scope = analysis (implementation to
+ follow).
+- 2026-09-11: renumbered 0009 -> 0010 (collision with the Temporal engine RFC)
+ and implementation plan rewired onto the RFC 0009 phase 1 ports architecture
+ (`BookInput`, `modes.go`, `stages.go`, `DraftChapter`/`RefineBatch`,
+ `ErrChapterUnrefined`). Decisions unchanged.
diff --git a/docs/rfcs/README.md b/docs/rfcs/README.md
index 21943d8..46c3f46 100644
--- a/docs/rfcs/README.md
+++ b/docs/rfcs/README.md
@@ -24,3 +24,5 @@ RFCs are proposal documents used to discuss options and implementation details b
- [0006 - Faithful Multi-Volume Literary Translation via Persistent Narrative Memory](./0006-faithful-multivolume-translation.md)
- [0007 - Google Cloud Translation as the Draft Engine, and a Token-Based Budget Guardrail](./0007-google-translate-draft-and-token-cost-model.md)
- [0008 - Pre-Generated Series Bible as a Job Separate from Translation](./0008-pre-generated-series-bible.md)
+- [0009 - Temporal as the Execution Engine of the Translation Pipeline](./0009-temporal-execution-engine.md) — illustrated companion: [`docs/architecture-temporal.html`](../architecture-temporal.html)
+- [0010 - LLM-Only Literary Translation (`--no-draft`)](./0010-llm-only-literary-translation.md)
diff --git a/docs/understanding-the-project.html b/docs/understanding-the-project.html
index e578006..bf28064 100644
--- a/docs/understanding-the-project.html
+++ b/docs/understanding-the-project.html
@@ -1106,6 +1106,7 @@
13. The invariants not to break
13
LLM tokens are accumulated in a single place: the OnBatch hook. Outcome.Usage carries the same tokens seen per chapter — adding both bills twice.
An exactly doubled cost, invisible without a reference measurement.
14
A rate limit (llm.ErrRateLimited, an HTTP 429) stops the chapter, it does not degrade it — it's the only batch error that does not degrade.
The rest of the book fails batch by batch silently and comes out as an NMT draft — which literary mode does not persist, so with nothing to resume.
15
The projection applied to the cap prices the remaining work; estimated_cost_usd stays the whole book with no reuse. These are two different numbers and they have two jobs.
Either a resume refused for a bill it won't pay, or cost_savings_usd dropping to zero exactly on the runs where memory worked the most.
+
16
French typography is normalized deterministically at commit and recompose (segment.FrenchTypography, after glossary Apply); the LLM is not the source of truth for spacing.
Spacing drifts segment by segment — mixed Nom:/Nom :, ... vs …, dash spacing — and the TM, segments.jsonl and the EPUB disagree.
Reference-free scoring. 1st launch: ~2.3 GB of model. Used for eval and for selective escalation during a run.
+
+
Temporal (dev server)
just temporal-up · just up
7233 · 8233
+
Durable execution engine for runs (RFC 0009). Stores workflow history in SQLite (volume temporal-data). gRPC on 7233; web UI on localhost:8233 (just temporal-ui). The worker that registers the workflows arrives in phase 3 — the server alone translates nothing.
Documents sampled per volume for style, voices and forms of address. Note: they all go in a single call, so the input grows with the series — see the box below.
--merge
false
Enriches existing bibles instead of overwriting them.
--dry-run
false
Prints the cost projection and stops before the first call. Nothing is created — not even the bible's database.
+
--from-volume
0
Append mode: summarizes only volumes ≥ N and adds their synopses to the existing bible's STORY_SO_FAR, without regenerating the earlier ones. 0 = full build. Idempotent: a volume already present is not rewritten.
@@ -1070,6 +1075,14 @@
8. The variables that change the output
LOG_LEVEL
info
debug details the segment-by-segment routing in logs.ndjson. Payloads are never logged.
+
+
TEMPORAL_ADDRESS
localhost:7233
+
The Temporal server's gRPC frontend that the CLI and the worker dial (durable execution engine, RFC 0009). just up starts the local server; no credential is needed to reach it, and estimate never touches it.
+
+
+
TEMPORAL_NAMESPACE / TEMPORAL_TASK_QUEUE
default / tomeshift
+
The namespace that isolates the workflows (the dev server registers default) and the task queue the worker listens on and on which the CLI starts the runs.
+
@@ -1221,7 +1234,7 @@
10. Exit codes and diagnostics
11. Docker and the just cheat sheet
-
Compose: four services, three profiles
+
Compose: five services, three profiles
@@ -1230,13 +1243,14 @@
Compose: four services, three profiles
nmt
default
The built-in dev NMT, started with the app.
libretranslate
real-nmt
Real local EN→FR NMT.
comet
eval
COMET-QE service. Heavy image: deliberately outside the default profile.
+
temporal
default
Temporal dev server (RFC 0009): gRPC :7233, web UI :8233, SQLite history in the temporal-data volume. just up starts it; app does not depend on it yet (worker in phase 3).
mkdir -p books runs data
cp <your-book>.epub books/source.epub
just docker-build
-just up # build + run app (+ nmt), output in ./runs
+just up # build + run temporal + app (+ nmt), output in ./runs
docker compose logs -f app
just down
@@ -1257,14 +1271,15 @@
just cheat sheet
Recipe
Does what
-
just preflight
Checks the tools and pings the NMT endpoint (non-blocking).
+
just preflight
Checks the tools and pings the NMT and Temporal endpoints (non-blocking).
just print-config-vars
Prints the TOMESHIFT_* actually resolved — the first reflex when "it's not picking up my config".
just nmt-dev · just libretranslate-up · just comet-up
Start the services.
+
just temporal-up · just temporal-down · just temporal-ui
Start / stop the Temporal dev server (RFC 0009); open its web UI on :8233. just up already starts it with the app.
just run <in.epub> [outdir]
NMT translation.
just literary <in> <series> [glossary] [run-id]
Literary translation.
just estimate <in.epub>
Cost projection.
just series-bible-cost <dir> <series> [samples]
Project the bible-build cost — 0 calls, no credential. The counterpart of estimate for this job.
-
just series-bible <dir> <series> [samples] [merge]
Build the bibles of the whole series. samples defaults to 3: see the pitfall in §6.
+
just series-bible <dir> <series> [samples] [merge] [from_volume]
Build the series' bibles. from_volume N = append mode (adds only volumes ≥ N to STORY_SO_FAR). samples defaults to 3: see the pitfall in §6.
just bible <db> <series>
Display a bible in plain text — to do before a full run, an error there propagates everywhere.
just glossary-suggest <in> [out] [min]
Extract proper nouns.
just eval-ref <cand> <ref> · just eval-qe <segments>
chrF · COMET-QE.
@@ -1368,6 +1383,8 @@
Common symptoms
o += line('NARRATIVE_PATH', './data/series.sqlite');
o += line('LLM_CACHE_PATH', './data/llm-cache.sqlite', "don't delete: free re-runs");
}
+ o += '\n' + head('durable engine (Temporal, RFC 0009)');
+ o += line('TEMPORAL_ADDRESS', 'localhost:7233', 'local server: just up');
if (cfg.mode === 'literary') {
var L = LLM[cfg.llm];
o += '\n' + head('LLM (literary mode)');
@@ -1587,6 +1604,10 @@
Common symptoms
proc: 'Enriches the existing bible instead of overwriting it.',
out: 'Preserves manual corrections or already-profiled volumes.',
cost: 'Identical to a generation. In both cases the prefix changes → the LLM cache is invalidated once.' },
+ { f: '--from-volume',
+ proc: 'series-bible append mode: summarizes only volumes ≥ N and adds their synopses to the existing bible, without redoing the earlier ones. 0 = full build.',
+ out: 'STORY_SO_FAR extended up to the last volume supplied; voices, style and earlier volumes untouched.',
+ cost: 'Pays only for volumes ≥ N (map: 1 call per chapter of those volumes), not the whole series — the point of the flag.' },
{ f: '--reference-dir',
proc: 'batch: looks for <dir>/<book-id>.epub after each successful book. Absent = silently skipped.',
out: 'Adds a per-book chrF and an average in batch-report.json.',
diff --git a/internal/cli/series_bible.go b/internal/cli/series_bible.go
index 1668098..944e7b4 100644
--- a/internal/cli/series_bible.go
+++ b/internal/cli/series_bible.go
@@ -28,6 +28,11 @@ type SeriesBibleCommand struct {
// chapter across every volume — and it is the one job `estimate` cannot
// price, since `estimate` projects translating a book.
DryRun bool
+ // FromVolume, when > 0, switches to append mode: only volumes whose number
+ // is >= it are summarised, and their synopses are ADDED to the existing
+ // bible's STORY_SO_FAR instead of regenerating the earlier ones. 0 keeps the
+ // full-build behaviour (every volume, summaries replaced).
+ FromVolume int
}
// ParseSeriesBibleArgs parses flags for `tomeshift series-bible`.
@@ -41,6 +46,7 @@ func ParseSeriesBibleArgs(args []string, stderr io.Writer) (SeriesBibleCommand,
fs.IntVar(&cmd.Samples, "samples", 3, "spine documents sampled per volume for style, voices and address rules")
fs.BoolVar(&cmd.Merge, "merge", false, "enrich existing bibles instead of overwriting them")
fs.BoolVar(&cmd.DryRun, "dry-run", false, "print the cost projection and exit without calling the provider")
+ fs.IntVar(&cmd.FromVolume, "from-volume", 0, "append mode: summarise only volumes >= N and add them to the existing bible (0 = full build)")
if err := fs.Parse(args); err != nil {
return SeriesBibleCommand{}, fmt.Errorf("unable to parse cli flags, err: %w", err)
diff --git a/internal/commandexec/bar.go b/internal/commandexec/bar.go
new file mode 100644
index 0000000..cf62c4f
--- /dev/null
+++ b/internal/commandexec/bar.go
@@ -0,0 +1,54 @@
+package commandexec
+
+import (
+ "github.com/ECecillo/tomeshift/internal/pipeline"
+ "github.com/ECecillo/tomeshift/internal/progress"
+)
+
+// barReporter adapts the flow's absolute Progress snapshots onto the terminal
+// bar's incremental API: it sizes the bar once, announces stage and chapter
+// changes, and turns cumulative counters into the per-call deltas Advance
+// expects. The flow publishes state; this decides what the human sees.
+type barReporter struct {
+ bar *progress.Bar
+ sized bool
+ stage string
+ last pipeline.Progress
+}
+
+func newBarReporter(bar *progress.Bar) *barReporter {
+ return &barReporter{bar: bar}
+}
+
+func (b *barReporter) Report(p pipeline.Progress) {
+ if p.Stage != b.stage {
+ b.stage = p.Stage
+ b.bar.Stage(p.Stage)
+ }
+ if !b.sized && p.Segments > 0 {
+ b.sized = true
+ b.bar.Total(p.Segments, p.Chapters)
+ }
+ if p.Chapter > 0 && p.Chapter != b.last.Chapter {
+ b.bar.Chapter(p.Chapter)
+ }
+
+ // Cumulative → per-call deltas. SegmentsDone counts both fresh and reused;
+ // TMHits is the reused share, so the fresh delta is the rest.
+ freshDelta := (p.SegmentsDone - p.TMHits) - (b.last.SegmentsDone - b.last.TMHits)
+ reusedDelta := p.TMHits - b.last.TMHits
+ if freshDelta > 0 || reusedDelta > 0 {
+ b.bar.Advance(freshDelta, reusedDelta)
+ }
+ if p.CostUSD != b.last.CostUSD {
+ b.bar.Cost(p.CostUSD)
+ }
+
+ b.last = p
+}
+
+// Finish terminates the underlying bar. It is idempotent, so callers can invoke
+// it on every exit path — including before writing an error to the same stream.
+func (b *barReporter) Finish() { b.bar.Finish() }
+
+var _ pipeline.Reporter = (*barReporter)(nil)
diff --git a/internal/commandexec/estimate.go b/internal/commandexec/estimate.go
index 87d046c..ea80a79 100644
--- a/internal/commandexec/estimate.go
+++ b/internal/commandexec/estimate.go
@@ -19,7 +19,7 @@ import (
// runEstimate parses+segments the book without translating and prints a cost
// projection across NMT-only / hybrid / full-LLM strategies.
func runEstimate(stdout, stderr io.Writer, cmd cli.EstimateCommand, cfg config.Config) int {
- orch := &pipeline.Orchestrator{
+ stages := &pipeline.Stages{
Extractor: epub.ZipExtractor{},
Parser: epub.OPFParser{},
Segmenter: segment.NewParagraphSegmenter(),
@@ -49,18 +49,18 @@ func runEstimate(stdout, stderr io.Writer, cmd cli.EstimateCommand, cfg config.C
return exitcode.InvalidInputConfig
}
defer func() { _ = store.Close() }()
- orch.TM = store
+ stages.TM = store
}
// The same calibration a run applies, from the same ledger. `estimate` is
// the one place it is worth saying out loud: a projection that silently
// changed basis between two invocations would be worse than a wrong one.
var note string
- if orch.TM != nil {
- orch.CostModel, note = calibrateLLMTokens(ctx, orch.TM, cfg, orch.CostModel)
+ if stages.TM != nil {
+ stages.CostModel, note = calibrateLLMTokens(ctx, stages.TM, cfg, stages.CostModel)
}
- plan, err := orch.Plan(ctx, pipeline.RunConfig{
+ plan, err := stages.Plan(ctx, pipeline.BookInput{
InputPath: cmd.Input,
SourceLang: cmd.SourceLang,
TargetLang: cmd.TargetLang,
diff --git a/internal/commandexec/series_bible.go b/internal/commandexec/series_bible.go
index ba8e846..c41bff8 100644
--- a/internal/commandexec/series_bible.go
+++ b/internal/commandexec/series_bible.go
@@ -47,6 +47,23 @@ func runSeriesBible(stdout, stderr io.Writer, cmd cli.SeriesBibleCommand, cfg co
return exitcode.InvalidInputConfig
}
+ // Append mode (--from-volume N): keep only the volumes to add, so the style
+ // pass, the projection and the synopsis loop all work on the new tail rather
+ // than the whole series.
+ if cmd.FromVolume > 0 {
+ kept := make([]volumeSource, 0, len(volumes))
+ for _, v := range volumes {
+ if v.Number >= cmd.FromVolume {
+ kept = append(kept, v)
+ }
+ }
+ if len(kept) == 0 {
+ writef(stderr, "no volume >= %d found in %s\n", cmd.FromVolume, cmd.InputDir)
+ return exitcode.InvalidInputConfig
+ }
+ volumes = kept
+ }
+
baseCtx, stop := signalContext()
defer stop()
ctx, cancel := withRunTimeout(baseCtx, cfg.RunTimeoutSeconds)
@@ -76,6 +93,22 @@ func runSeriesBible(stdout, stderr io.Writer, cmd cli.SeriesBibleCommand, cfg co
profiler := narrative.Profiler{Chat: chat, Model: cfg.LLMModel}
+ // Append mode adds the new synopses to the existing bible's STORY_SO_FAR
+ // without a style pass and without regenerating the earlier volumes — the
+ // whole point of --from-volume.
+ if cmd.FromVolume > 0 {
+ synopses, err := buildSynopses(ctx, stdout, profiler, store, cmd.Series, volumes)
+ if err != nil {
+ writef(stderr, "%v\n", err)
+ return exitCodeForError(err)
+ }
+ if err := appendVolumeSynopses(ctx, stdout, stderr, store, cmd.Series, synopses); err != nil {
+ writef(stderr, "unable to append series synopses: %v\n", err)
+ return exitCodeForError(err)
+ }
+ return exitcode.Success
+ }
+
base, err := seedSeriesStyle(ctx, profiler, cmd, volumes)
if err != nil {
writef(stderr, "unable to seed series style: %v\n", err)
@@ -84,23 +117,76 @@ func runSeriesBible(stdout, stderr io.Writer, cmd cli.SeriesBibleCommand, cfg co
writef(stdout, "style pass: %d volumes, %d characters, %d address rules\n",
len(volumes), len(base.Characters), len(base.Address))
+ synopses, err := buildSynopses(ctx, stdout, profiler, store, cmd.Series, volumes)
+ if err != nil {
+ writef(stderr, "%v\n", err)
+ return exitCodeForError(err)
+ }
+
+ if err := saveScopedBibles(ctx, stdout, stderr, store, cmd, base, synopses, volumes); err != nil {
+ writef(stderr, "unable to save series bibles: %v\n", err)
+ return exitCodeForError(err)
+ }
+ return exitcode.Success
+}
+
+// buildSynopses runs the map-reduce synopsis for each volume in order, logging
+// progress. It is shared by the full build and the --from-volume append path.
+func buildSynopses(
+ ctx context.Context, stdout io.Writer, profiler narrative.Profiler,
+ store narrative.Store, series string, volumes []volumeSource,
+) ([]narrative.Summary, error) {
synopses := make([]narrative.Summary, 0, len(volumes))
for _, volume := range volumes {
- synopsis, chapters, err := volumeSynopsis(ctx, stdout, profiler, store, cmd.Series, volume)
+ synopsis, chapters, err := volumeSynopsis(ctx, stdout, profiler, store, series, volume)
if err != nil {
- writef(stderr, "unable to build synopsis for %s: %v\n", volume.ID, err)
- return exitCodeForError(err)
+ return nil, fmt.Errorf("unable to build synopsis for %s, err: %w", volume.ID, err)
}
writef(stdout, "volume %s: %d chapters summarised, synopsis %d runes\n",
volume.ID, chapters, len([]rune(synopsis)))
synopses = append(synopses, narrative.Summary{VolumeID: volume.ID, Summary: synopsis})
}
+ return synopses, nil
+}
- if err := saveScopedBibles(ctx, stdout, stderr, store, cmd, base, synopses, volumes); err != nil {
- writef(stderr, "unable to save series bibles: %v\n", err)
- return exitCodeForError(err)
+// appendVolumeSynopses adds synopses to the EXISTING bible `series` without
+// touching the ones it already holds. MergeBibles appends summaries by
+// (volume, chapter) with the existing winning a conflict, so re-running is
+// idempotent and no earlier volume is rebuilt; the curated style guide, voices
+// and address rules are kept untouched (incoming carries summaries only). It is
+// the engine behind `--from-volume N`.
+func appendVolumeSynopses(
+ ctx context.Context, stdout, stderr io.Writer, store narrative.Store,
+ series string, synopses []narrative.Summary,
+) error {
+ existing, found, err := store.Load(ctx, series)
+ if err != nil {
+ return err
}
- return exitcode.Success
+ if !found {
+ return fmt.Errorf("unable to append to %q: no existing bible to extend (build it first), err: %w", series, narrative.ErrMissingBible)
+ }
+
+ merged, stats := narrative.MergeBibles(existing, narrative.Bible{SeriesID: series, Summaries: synopses})
+ merged.SeriesID = series
+ if err := narrative.ValidateBible(merged); err != nil {
+ return fmt.Errorf("unable to validate the extended bible %q, err: %w", series, err)
+ }
+ if err := store.Save(ctx, merged); err != nil {
+ return err
+ }
+
+ saved, _, err := store.Load(ctx, series)
+ if err != nil {
+ return err
+ }
+ writef(stdout, "bible %s extended: version %d, %d volume synopses (+%d appended)\n",
+ saved.SeriesID, saved.Version, len(saved.Summaries), len(saved.Summaries)-len(existing.Summaries))
+ if stats.Lossy() {
+ writef(stderr, "warning: %d incoming synopses were already present and kept from the existing bible\n",
+ stats.DroppedSummaries)
+ }
+ return nil
}
// seedSeriesStyle runs the single sampling pass that produces style, voices and
diff --git a/internal/commandexec/series_bible_test.go b/internal/commandexec/series_bible_test.go
index 5384b35..004a459 100644
--- a/internal/commandexec/series_bible_test.go
+++ b/internal/commandexec/series_bible_test.go
@@ -1,6 +1,7 @@
package commandexec
import (
+ "io"
"os"
"path/filepath"
"testing"
@@ -10,6 +11,100 @@ import (
"github.com/ECecillo/tomeshift/internal/narrative"
)
+// summaryVolumeIDs lists a bible's volume synopses in order.
+func summaryVolumeIDs(summaries []narrative.Summary) []string {
+ ids := make([]string, len(summaries))
+ for i, s := range summaries {
+ ids[i] = s.VolumeID
+ }
+ return ids
+}
+
+func summaryFor(b narrative.Bible, volumeID string) string {
+ for _, s := range b.Summaries {
+ if s.VolumeID == volumeID {
+ return s.Summary
+ }
+ }
+ return ""
+}
+
+func seededStore(t *testing.T, summaries []narrative.Summary) *narrative.SQLiteStore {
+ t.Helper()
+ store, err := narrative.NewSQLiteStore(filepath.Join(t.TempDir(), "narrative.sqlite"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = store.Close() })
+ require.NoError(t, store.Save(t.Context(), narrative.Bible{
+ SeriesID: "lumen",
+ StyleGuide: "style contemporain",
+ Characters: []narrative.CharacterVoice{{Name: "Kai", Register: "familier"}},
+ Summaries: summaries,
+ }))
+ return store
+}
+
+// TestAppendVolumeSynopses_AddsWithoutRebuilding is the --from-volume contract:
+// new volume synopses extend STORY_SO_FAR while the earlier ones, the curated
+// style guide and the character voices stay exactly as they were.
+func TestAppendVolumeSynopses_AddsWithoutRebuilding(t *testing.T) {
+ t.Parallel()
+
+ store := seededStore(t, []narrative.Summary{
+ {VolumeID: "vol1", Summary: "one"},
+ {VolumeID: "vol2", Summary: "two"},
+ })
+ before, _, err := store.Load(t.Context(), "lumen")
+ require.NoError(t, err)
+
+ require.NoError(t, appendVolumeSynopses(t.Context(), io.Discard, io.Discard, store, "lumen",
+ []narrative.Summary{{VolumeID: "vol3", Summary: "three"}}))
+
+ got, found, err := store.Load(t.Context(), "lumen")
+ require.NoError(t, err)
+ require.True(t, found)
+ require.Equal(t, []string{"vol1", "vol2", "vol3"}, summaryVolumeIDs(got.Summaries))
+ require.Equal(t, before.Characters, got.Characters, "voices must stay untouched")
+ require.Equal(t, before.StyleGuide, got.StyleGuide, "curated style guide must stay untouched")
+ require.Greater(t, got.Version, before.Version, "the store bumps the version")
+}
+
+// TestAppendVolumeSynopses_ExistingWinsOnConflict pins idempotency: re-adding a
+// volume already present keeps the stored synopsis, so a re-run never rewrites
+// an earlier volume.
+func TestAppendVolumeSynopses_ExistingWinsOnConflict(t *testing.T) {
+ t.Parallel()
+
+ store := seededStore(t, []narrative.Summary{
+ {VolumeID: "vol1", Summary: "one"},
+ {VolumeID: "vol2", Summary: "two"},
+ })
+
+ require.NoError(t, appendVolumeSynopses(t.Context(), io.Discard, io.Discard, store, "lumen",
+ []narrative.Summary{
+ {VolumeID: "vol2", Summary: "REWRITE — must be ignored"},
+ {VolumeID: "vol3", Summary: "three"},
+ }))
+
+ got, _, err := store.Load(t.Context(), "lumen")
+ require.NoError(t, err)
+ require.Equal(t, []string{"vol1", "vol2", "vol3"}, summaryVolumeIDs(got.Summaries))
+ require.Equal(t, "two", summaryFor(got, "vol2"), "existing synopsis wins a conflict")
+}
+
+// TestAppendVolumeSynopses_NoBaseErrors: append has nothing to extend when no
+// bible exists yet — it must say so rather than silently create a partial one.
+func TestAppendVolumeSynopses_NoBaseErrors(t *testing.T) {
+ t.Parallel()
+
+ store, err := narrative.NewSQLiteStore(filepath.Join(t.TempDir(), "narrative.sqlite"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = store.Close() })
+
+ err = appendVolumeSynopses(t.Context(), io.Discard, io.Discard, store, "lumen",
+ []narrative.Summary{{VolumeID: "vol1", Summary: "one"}})
+ require.ErrorIs(t, err, narrative.ErrMissingBible)
+}
+
func TestDiscoverVolumesSortsByPublicationNumber(t *testing.T) {
t.Parallel()
diff --git a/internal/commandexec/translate.go b/internal/commandexec/translate.go
index 608ee0e..0acf209 100644
--- a/internal/commandexec/translate.go
+++ b/internal/commandexec/translate.go
@@ -9,6 +9,8 @@ import (
"sort"
"time"
+ "go.uber.org/zap"
+
"github.com/ECecillo/tomeshift/internal/cli"
"github.com/ECecillo/tomeshift/internal/config"
"github.com/ECecillo/tomeshift/internal/epub"
@@ -46,17 +48,17 @@ func openTranslationInputs(cfg config.Config, glossaryPath string) (*tm.SQLiteSt
return store, gloss, nil
}
-// newOrchestrator assembles a fully-wired orchestrator for mode: the NMT
+// newStages assembles the fully-wired stage collaborators for mode: the NMT
// provider with its retries, the optional post-editor, the QA gate, the cost
// guardrail, and — for literary mode only — the series bible and chat client.
//
-// It is one step rather than two on purpose: a half-built orchestrator is a
+// It is one step rather than two on purpose: a half-built set of stages is a
// shape no caller should be able to hold. The returned closer releases the
// resources literary mode opens, and is a no-op for every other mode, so
// callers never branch on it. `batch` passes an empty mode.
-func newOrchestrator(
+func newStages(
cfg config.Config, store tm.Store, gloss *glossary.Glossary, mode string,
-) (*pipeline.Orchestrator, func(), error) {
+) (*pipeline.Stages, func(), error) {
// Fail here rather than at the first segment: by then the book has been
// extracted, parsed, segmented and budgeted, and the error would read as a
// provider outage instead of a missing setting.
@@ -76,7 +78,7 @@ func newOrchestrator(
postEditor = llm.NewHTTPPostEditor(cfg.LLMEndpoint, cfg.LLMAPIKey)
}
- orch := &pipeline.Orchestrator{
+ stages := &pipeline.Stages{
Extractor: epub.ZipExtractor{},
Parser: epub.OPFParser{},
Segmenter: segment.NewParagraphSegmenter(),
@@ -100,7 +102,7 @@ func newOrchestrator(
}
if mode != pipeline.ModeLiterary {
- return orch, func() {}, nil
+ return stages, func() {}, nil
}
bible, err := narrative.NewSQLiteStore(cfg.NarrativePath)
@@ -117,18 +119,18 @@ func newOrchestrator(
// keeps them so a measured token count stays reusable, and only the literary
// path sets them — the HTTP post-editor is picked by endpoint, not by model,
// and reports no usage to attribute anyway.
- orch.LLMProvider = cfg.LLMChatProvider
- orch.LLMModel = cfg.LLMModel
+ stages.LLMProvider = cfg.LLMChatProvider
+ stages.LLMModel = cfg.LLMModel
- orch.NarrativeStore = bible
- orch.NarrativeProfiler = &narrative.Profiler{Chat: chat, Model: cfg.LLMModel}
- orch.LiteraryTranslator = &literary.Translator{
+ stages.NarrativeStore = bible
+ stages.NarrativeProfiler = &narrative.Profiler{Chat: chat, Model: cfg.LLMModel}
+ stages.LiteraryTranslator = &literary.Translator{
Chat: chat,
Model: cfg.LLMModel,
BatchSize: cfg.LLMBatchSize,
Glossary: gloss,
}
- return orch, func() { _ = closeCache(); _ = bible.Close() }, nil
+ return stages, func() { _ = closeCache(); _ = bible.Close() }, nil
}
// runPipeline assembles the orchestrator from runtime config and executes a
@@ -154,30 +156,36 @@ func runPipeline(stdout, stderr io.Writer, cmd cli.RunCommand, cfg config.Config
}
defer func() { _ = store.Close() }()
- orch, closeOrch, err := newOrchestrator(cfg, store, gloss, cmd.Mode)
+ stages, closeStages, err := newStages(cfg, store, gloss, cmd.Mode)
if err != nil {
writef(stderr, "%v\n", err)
return exitcode.InvalidInputConfig
}
- defer closeOrch()
- orch.Logger = logger
-
- // stderr, not stdout: the bar is for a human watching, and stdout stays the
- // single parsable summary line.
- orch.Progress = progress.New(stderr, !cmd.NoProgress && progress.IsTerminal(stderr), "")
+ defer closeStages()
+ stages.Logger = logger
baseCtx, stop := signalContext()
defer stop()
ctx, cancel := withRunTimeout(baseCtx, cfg.RunTimeoutSeconds)
defer cancel()
- // Before Run, because Run projects the cost and enforces the cap on it.
+ // Before RunBook, because ProjectCost enforces the cap on the calibrated model.
var note string
- if orch.CostModel, note = calibrateLLMTokens(ctx, store, cfg, orch.CostModel); note != "" {
+ if stages.CostModel, note = calibrateLLMTokens(ctx, store, cfg, stages.CostModel); note != "" {
logger.Info(note)
}
- result, err := orch.Run(ctx, pipeline.RunConfig{
+ bar := newBarReporter(progress.New(stderr, !cmd.NoProgress && progress.IsTerminal(stderr), ""))
+ steps := &pipeline.InProcessSteps{Stages: stages, Ctx: ctx}
+ rt := &pipeline.InProcessRuntime{
+ Ctx: ctx,
+ Steps: steps,
+ // stderr, not stdout: the bar is for a human watching, and stdout stays
+ // the single parsable summary line.
+ Progress: bar,
+ }
+
+ result, err := pipeline.RunBook(steps, rt, pipeline.BookInput{
InputPath: cmd.Input,
OutputDir: outputDir,
SourceLang: cmd.SourceLang,
@@ -189,6 +197,9 @@ func runPipeline(stdout, stderr io.Writer, cmd cli.RunCommand, cfg config.Config
UpdateBible: cmd.UpdateBible,
Resume: cmd.Resume,
})
+ // The line is closed before the caller writes its failure message to the
+ // same stream, so the error never lands inside the bar.
+ bar.Finish()
if err != nil {
writef(stderr, "run failed: %v\n", err)
return exitCodeForError(err)
@@ -222,70 +233,88 @@ func runBatch(stdout, stderr io.Writer, cmd cli.BatchCommand, cfg config.Config)
defer func() { _ = store.Close() }()
// Batch is NMT-only, so it needs none of the literary collaborators.
- orch, closeOrch, err := newOrchestrator(cfg, store, gloss, pipeline.ModeNMT)
+ stages, closeStages, err := newStages(cfg, store, gloss, pipeline.ModeNMT)
if err != nil {
writef(stderr, "%v\n", err)
return exitcode.InvalidInputConfig
}
- defer closeOrch()
+ defer closeStages()
baseCtx, stop := signalContext()
defer stop()
// Calibrated once for the whole batch: it describes the model, not the book.
- orch.CostModel, _ = calibrateLLMTokens(baseCtx, store, cfg, orch.CostModel)
+ stages.CostModel, _ = calibrateLLMTokens(baseCtx, store, cfg, stages.CostModel)
// Resolved once: every book gets its own bar, all writing to the same line.
showProgress := !cmd.NoProgress && progress.IsTerminal(stderr)
- outcomes := make([]pipeline.BookOutcome, 0, len(epubs))
- var firstErr error
+ books := make([]pipeline.BookInput, 0, len(epubs))
for _, in := range epubs {
- if baseCtx.Err() != nil {
- writef(stderr, "batch interrupted; stopping before remaining books\n")
- break
- }
-
bookID := bookIDFromPath(in)
- bookDir := filepath.Join(cmd.OutputDir, bookID)
-
- logger, closeLog, lerr := pipeline.NewRunLogger(bookDir, cfg.LogLevel)
- if lerr != nil {
- outcomes = append(outcomes, pipeline.BookOutcome{BookID: bookID, RunID: bookID, OK: false, Error: lerr.Error()})
- writef(stderr, "book %s: logger error: %v\n", bookID, lerr)
- continue
- }
- orch.Logger = logger
- orch.Progress = progress.New(stderr, showProgress, bookID)
-
- ctx, cancel := withRunTimeout(baseCtx, cfg.RunTimeoutSeconds)
- result, runErr := orch.Run(ctx, pipeline.RunConfig{
- InputPath: in, OutputDir: bookDir,
+ books = append(books, pipeline.BookInput{
+ InputPath: in, OutputDir: filepath.Join(cmd.OutputDir, bookID),
SourceLang: cmd.SourceLang, TargetLang: cmd.TargetLang,
RunID: bookID, BookID: bookID,
})
- cancel()
- _ = closeLog()
-
- outcome := pipeline.BookOutcome{BookID: bookID, RunID: bookID, OK: runErr == nil, Result: result}
- if runErr != nil {
- outcome.Error = runErr.Error()
- if firstErr == nil {
- firstErr = runErr
+ }
+
+ steps := &pipeline.InProcessSteps{Stages: stages, Ctx: baseCtx}
+ rt := &pipeline.InProcessRuntime{Ctx: baseCtx, Steps: steps}
+
+ var firstErr error
+ var closeLog func() error
+ var cancel context.CancelFunc
+ var bar *barReporter
+ hooks := pipeline.SeriesHooks{
+ Prepare: func(in pipeline.BookInput) {
+ logger, cl, lerr := pipeline.NewRunLogger(in.OutputDir, cfg.LogLevel)
+ if lerr != nil {
+ writef(stderr, "book %s: logger error: %v\n", in.BookID, lerr)
+ logger, cl = zap.NewNop(), func() error { return nil }
+ }
+ closeLog = cl
+ stages.Logger = logger
+ bar = newBarReporter(progress.New(stderr, showProgress, in.BookID))
+ rt.Progress = bar
+ ctx, c := withRunTimeout(baseCtx, cfg.RunTimeoutSeconds)
+ cancel = c
+ steps.Ctx, rt.Ctx = ctx, ctx
+ },
+ Score: func(bookID, outputEPUB string) (float64, bool) {
+ return scoreAgainstReference(stderr, cmd.ReferenceDir, bookID, outputEPUB)
+ },
+ Done: func(in pipeline.BookInput, outcome pipeline.BookOutcome, runErr error) {
+ if cancel != nil {
+ cancel()
+ }
+ // Point the shared context back at the stable base before the next
+ // book. A per-book run timeout makes withRunTimeout return a
+ // cancellable child; leaving the just-cancelled child in place would
+ // make RunSeries' top-of-loop Cancelled() check fire on book 2 and
+ // silently skip the rest of the series.
+ steps.Ctx, rt.Ctx = baseCtx, baseCtx
+ if bar != nil {
+ bar.Finish()
+ }
+ if closeLog != nil {
+ _ = closeLog()
}
- writef(stdout, "book %-20s FAILED: %v\n", bookID, runErr)
- } else {
- if score, ok := scoreAgainstReference(stderr, cmd.ReferenceDir, bookID, result.OutputEPUB); ok {
- outcome.ChrF = &score
+ if runErr != nil {
+ if firstErr == nil {
+ firstErr = runErr
+ }
+ writef(stdout, "book %-20s FAILED: %v\n", in.BookID, runErr)
+ return
}
+ r := outcome.Result
writef(stdout, "book %-20s ok: %d segments, %d tm hits, %d nmt, %d llm%s\n",
- bookID, result.TotalSegments, result.TMHits, result.NMTSegments, result.LLMSegments,
+ in.BookID, r.TotalSegments, r.TMHits, r.NMTSegments, r.LLMSegments,
chrFSuffix(outcome.ChrF))
- }
- outcomes = append(outcomes, outcome)
+ },
}
- summary := pipeline.NewBatchSummary(outcomes)
+ summary := pipeline.RunSeries(steps, rt, books, hooks)
if err := pipeline.WriteBatchReport(cmd.OutputDir, summary); err != nil {
writef(stderr, "unable to write batch report: %v\n", err)
}
diff --git a/internal/config/config.go b/internal/config/config.go
index 1164302..86c5d7c 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -106,6 +106,19 @@ type Config struct {
DailyBudgetUSD float64 `split_words:"true" default:"5.00"`
MonthlyBudgetUSD float64 `split_words:"true" default:"50.00"`
PerBookBudgetUSD float64 `split_words:"true" default:"2.00"`
+
+ // Temporal wiring for the durable execution engine (RFC 0009). The three
+ // settings address the local dev server started by `just up`; a run needs
+ // no credential to reach it and `estimate` never does.
+ //
+ // TemporalAddress is the gRPC frontend the CLI and worker dial.
+ TemporalAddress string `split_words:"true" default:"localhost:7233"`
+ // TemporalNamespace scopes every workflow; the dev server auto-registers
+ // "default".
+ TemporalNamespace string `split_words:"true" default:"default"`
+ // TemporalTaskQueue is the queue the worker polls and the CLI starts
+ // workflows on. One queue, one activity slot, until RFC 0005 lands.
+ TemporalTaskQueue string `split_words:"true" default:"tomeshift"`
}
// Load reads the configuration from the environment.
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index b9a4ae6..f2900ef 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -112,3 +112,28 @@ func TestCostModel_ReadsThePrefixTransportFromTheChatProvider(t *testing.T) {
})
}
}
+
+func TestLoad_TemporalDefaults(t *testing.T) {
+ // Not parallel: it reads the process environment, so asserting the built-in
+ // defaults means no TOMESHIFT_TEMPORAL_* is set for this case.
+ cfg, err := Load()
+ require.NoError(t, err)
+
+ require.Equal(t, "localhost:7233", cfg.TemporalAddress)
+ require.Equal(t, "default", cfg.TemporalNamespace)
+ require.Equal(t, "tomeshift", cfg.TemporalTaskQueue)
+}
+
+func TestLoad_TemporalOverrides(t *testing.T) {
+ // t.Setenv forbids t.Parallel; the override must win over the default.
+ t.Setenv("TOMESHIFT_TEMPORAL_ADDRESS", "temporal.internal:7233")
+ t.Setenv("TOMESHIFT_TEMPORAL_NAMESPACE", "epub")
+ t.Setenv("TOMESHIFT_TEMPORAL_TASK_QUEUE", "tomeshift-heavy")
+
+ cfg, err := Load()
+ require.NoError(t, err)
+
+ require.Equal(t, "temporal.internal:7233", cfg.TemporalAddress)
+ require.Equal(t, "epub", cfg.TemporalNamespace)
+ require.Equal(t, "tomeshift-heavy", cfg.TemporalTaskQueue)
+}
diff --git a/internal/literary/prompt.go b/internal/literary/prompt.go
index 2cf8162..97506a8 100644
--- a/internal/literary/prompt.go
+++ b/internal/literary/prompt.go
@@ -9,15 +9,28 @@ import (
"github.com/ECecillo/tomeshift/internal/narrative"
)
-const systemInstruction = `Tu es un traducteur littéraire expert de l'anglais vers le français.
-RÈGLES ABSOLUES :
-1. TRADUIS TOUT LE TEXTE EN FRANÇAIS. Ne laisse aucune phrase ni aucun mot courant en anglais, à la seule exception des balises et et des noms propres, qui restent inchangés.
-2. Adopte un style littéraire fluide, naturel et idiomatique en français. Ne traduis pas mot à mot.
+const systemInstruction = `Tu es un traducteur littéraire professionnel de l'anglais vers le français.
+
+REGISTRE (priorité) :
+- Écris un français CONTEMPORAIN, courant et fluide — celui d'un roman populaire moderne. Jamais soutenu ni archaïsant.
+- INTERDIT : le subjonctif imparfait (fût, dût, s'agît, pût, eût...) et le registre soutenu/archaïsant. Écris « bien qu'il s'agisse » (pas « s'agît »), « il ne devait jamais » (pas « il ne dût »). Le passé simple reste le temps normal du récit : garde-le.
+- Pas de tournures administratives ou juridiques (« ledit », « dudit », « sera omis ») : préfère une formule naturelle.
+
+NATUREL :
+- Ne calque pas la syntaxe anglaise. Découpe les phrases longues et rétablis un ordre naturel en français.
+- Traduis les idiomes par leur équivalent français, jamais mot à mot. Ex. : « bodily movements » -> « ses mouvements » (pas « mouvements corporels »).
+- Pour une remarque méta du narrateur (« further explanation is omitted... »), allège : « mais n'entrons pas dans les détails, cela nous éloignerait du sujet. »
+- Garde un temps de récit cohérent ; n'introduis pas un futur isolé au milieu d'une narration au passé.
+
+DIALOGUE : oral, vivant, à hauteur du personnage. Préserve les allongements expressifs et onomatopées (reaaally -> vraaaiment). Ponctuation française, calquée sur la source : guillemets « » ; tiret cadratin collé au mot comme dans la source (« —Mot », mot—mot) ; points de suspension en un seul caractère … (jamais ...). N'ajoute pas les espaces autour de la ponctuation : les espaces insécables sont posées automatiquement.
+
+CONTRAINTES TECHNIQUES (absolues) :
+1. Traduis TOUT en français. Seules exceptions laissées telles quelles : les balises et et les noms propres.
+2. Préserve exactement chaque balise et à sa place d'origine dans la phrase.
3. Renvoie UNIQUEMENT un objet JSON strict, sans aucun texte avant ou après.
-4. Préserve exactement chaque balise et à sa place d'origine dans la phrase.
FORMAT ATTENDU :
-{"segments":[{"id":"ID_DU_SEGMENT","target_text":"texte traduit en français parfait"}]}`
+{"segments":[{"id":"ID_DU_SEGMENT","target_text":"texte traduit en français naturel"}]}`
func composeMessages(bible narrative.Bible, terms []glossary.Term, chapterSource, chapterPayload, prevCtx string) []llm.ChatMessage {
return []llm.ChatMessage{
diff --git a/internal/literary/translator.go b/internal/literary/translator.go
index 04f34de..69dcaf0 100644
--- a/internal/literary/translator.go
+++ b/internal/literary/translator.go
@@ -51,18 +51,6 @@ type Translator struct {
// this value. 0 uses defaultBatchSize.
BatchSize int
Glossary *glossary.Glossary
- // OnBatch, when set, is called once a batch's call has returned —
- // successfully or not, since a degraded batch is as finished as a good one —
- // with the segments it covered and the tokens it consumed. It exists so a
- // chapter, which is a single TranslateChapter call and the longest thing a
- // run does, can still report its advancement and its spend as it goes.
- //
- // The usage is that batch's own, retries included. A caller that
- // accumulates it must NOT also add Outcome.Usage: the two are the same
- // tokens, reported per batch and per chapter.
- //
- // It must not block: the batch walk is what waits on it.
- OnBatch func(segments int, usage Usage)
}
type requestSegment struct {
@@ -96,6 +84,11 @@ type Outcome struct {
// why part of it went unrefined, and the named refusal and truncation
// errors are worth nothing if they are dropped here.
Degraded []error
+ // Tail is the French this batch produced, from the raw model answers in
+ // request order (unanswered segments skipped), trimmed to contextRunes. The
+ // pipeline threads it into the next batch as its preceding context, exactly
+ // as the in-chapter batch walk used to read the previous batch's targets.
+ Tail string
}
// Usage aggregates token accounting for a chapter's LLM call. CacheRead and
@@ -108,34 +101,21 @@ type Usage struct {
CacheCreationTokens int
}
-// since returns what happened between a snapshot and u. chapterRun.usage is
-// cumulative over the chapter, so this is how one batch's own consumption is
-// recovered — a batch that retried on halved ones having paid for every
-// attempt.
-func (u Usage) since(before Usage) Usage {
- return Usage{
- InputTokens: u.InputTokens - before.InputTokens,
- OutputTokens: u.OutputTokens - before.OutputTokens,
- CacheReadTokens: u.CacheReadTokens - before.CacheReadTokens,
- CacheCreationTokens: u.CacheCreationTokens - before.CacheCreationTokens,
- }
-}
-
// contextRunes bounds the French already produced that is re-sent ahead of a
// batch. The model needs enough to carry register, tense and pronouns across
// the seam, not the whole chapter — the chapter's source is in the cached
// prefix, so only the translation of the batch before it has to be repeated.
const contextRunes = 1200
-// chapterRun carries the state shared by every LLM call made for one chapter,
-// so the recursive batch walk does not have to thread it through parameters.
+// chapterRun carries the state shared by every LLM call made for one batch (and
+// its halved retries), so the recursion does not have to thread it through
+// parameters.
type chapterRun struct {
bible narrative.Bible
chapterID string
// source is the chapter's full source text, sent once in the cacheable
// prefix rather than as windows around each batch.
- source string
- prevCtx string
+ source string
// targets holds the translations returned so far, keyed by segment ID.
targets map[string]string
usage Usage
@@ -156,46 +136,34 @@ func (b batch) withItems(items []requestSegment) batch {
return b
}
-// precedingContext returns the prose immediately before batch idx: the
-// translation just produced for the previous batch, or — for the first batch —
-// the chapter-level context supplied by the caller. Batches are translated in
-// order precisely so this is available.
-func (run *chapterRun) precedingContext(idx int, batches [][]requestSegment) string {
- if idx == 0 {
- return tailRunes(run.prevCtx, contextRunes)
- }
+// applyTargets writes the LLM's answers back onto segs, falling back to the NMT
+// draft for any segment the model left unanswered or whose placeholder set it
+// changed. It is the shared finalisation of a chapter and of a single batch.
+func (t *Translator) applyTargets(segs []*segment.Segment, states []segmentState, targets map[string]string) {
+ for i, seg := range segs {
+ state := states[i]
- var sb strings.Builder
- for _, item := range batches[idx-1] {
- if target := run.targets[item.ID]; strings.TrimSpace(target) != "" {
- sb.WriteString(target)
- sb.WriteByte('\n')
+ // An unanswered segment reads as the empty string here, which is the
+ // same predicate missingItems retries on: "retried" and "fell back to
+ // the draft" stay one condition.
+ target := t.Glossary.Apply(targets[seg.ID], state.subs)
+ if strings.TrimSpace(target) == "" || !inlinePlaceholdersPreserved(state.source, target) {
+ seg.TargetText = state.draft
+ seg.Status = segment.StatusTranslated
+ continue
}
+
+ seg.TargetText = target
+ seg.Status = segment.StatusPostEdited
}
- return tailRunes(strings.TrimSpace(sb.String()), contextRunes)
}
-// TranslateChapter rewrites segs in place and reports the LLM token usage. The
-// TargetText each segment arrives with is treated as the safe NMT draft
-// fallback. The chapter is split into batches of BatchSize segments so that
-// small models (≤8B) do not stop generating before all segments are done.
-//
-// It takes pointers because the caller owns the segments — they live in the
-// chapter the pipeline is going to recompose. Returning translated copies would
-// only oblige every caller to write them back by index.
-//
-// A batch that fails degrades rather than aborting the chapter: its segments
-// keep their draft and stay StatusTranslated, which the caller can count. Only
-// a chapter where no batch produced anything is returned as an error, so a
-// systematic failure (bad credentials, wrong model) stays loud while a single
-// bad batch does not discard the batches that already succeeded. Nothing is
-// written in that case: the segments keep the drafts they came in with.
-//
-// One failure is not degradable: llm.ErrRateLimited returns immediately, with
-// the usage consumed so far. Every remaining batch would fail the same way, and
-// degrading them would produce a chapter of raw drafts that this mode does not
-// persist — a book that reads as finished with nothing to resume from.
-func (t *Translator) TranslateChapter(
+// TranslateBatch rewrites one batch of segments in a single LLM call — the unit
+// the flow retries and resumes on — with the recursive halving on truncation
+// kept inside. It reports the tokens the call consumed (retries included) and,
+// through Outcome.Degraded, whether a refusal or truncation left the batch on
+// its draft. It does not chunk: the caller decides the batch boundaries.
+func (t *Translator) TranslateBatch(
ctx context.Context,
bible narrative.Bible,
chapterID string,
@@ -211,69 +179,51 @@ func (t *Translator) TranslateChapter(
}
items, states := t.buildPayload(segs)
-
run := &chapterRun{
bible: bible,
chapterID: chapterID,
source: chapterSource,
- prevCtx: prevCtx,
targets: make(map[string]string, len(segs)),
}
- batches := chunk(items, t.batchSize())
- var batchErrs []error
- for idx, batchItems := range batches {
- before := run.usage
- err := t.translateBatch(ctx, run, batch{
- idx: idx,
- items: batchItems,
- prev: run.precedingContext(idx, batches),
- })
- // Reported before the error is dealt with, for the same reason the
- // caller used to add Outcome.Usage before its own error check: a refusal
- // and a truncation are billable responses. It also has to happen before
- // the rate-limit return below, or the batch that hit the wall would go
- // unaccounted.
- if t.OnBatch != nil {
- t.OnBatch(len(batchItems), run.usage.since(before))
- }
- if err != nil {
- // A rate limit is not a bad reply, it is the end of the road: every
- // remaining batch would fail the same way, and degrading them would
- // emit a book that looks finished while half of it is raw draft —
- // unpersisted, so not even resumable. Stop here and let the caller
- // fail the run; what previous chapters refined is already in the TM.
- if errors.Is(err, llm.ErrRateLimited) {
- return Outcome{Usage: run.usage, Degraded: batchErrs}, err
- }
- batchErrs = append(batchErrs, err)
- }
- }
- if len(missingItems(items, run.targets)) == len(items) {
- if len(batchErrs) > 0 {
- return Outcome{Usage: run.usage}, errors.Join(batchErrs...)
+ err := t.translateBatch(ctx, run, batch{items: items, prev: tailRunes(prevCtx, contextRunes)})
+ t.applyTargets(segs, states, run.targets)
+ tail := batchTail(items, run.targets)
+ if err != nil {
+ // A rate limit or a transport/credential failure is fatal: the caller
+ // fails the run. A refusal, truncation or empty reply is degradable —
+ // the segments kept their draft — so it is reported, not raised.
+ if errors.Is(err, llm.ErrRateLimited) || !IsDegradable(err) {
+ return Outcome{Usage: run.usage, Tail: tail}, err
}
- return Outcome{Usage: run.usage}, ErrEmptyResponse
+ return Outcome{Usage: run.usage, Degraded: []error{err}, Tail: tail}, nil
}
+ return Outcome{Usage: run.usage, Tail: tail}, nil
+}
- for i, seg := range segs {
- state := states[i]
-
- // An unanswered segment reads as the empty string here, which is the
- // same predicate missingItems retries on: "retried" and "fell back to
- // the draft" stay one condition.
- target := t.Glossary.Apply(run.targets[seg.ID], state.subs)
- if strings.TrimSpace(target) == "" || !inlinePlaceholdersPreserved(state.source, target) {
- seg.TargetText = state.draft
- seg.Status = segment.StatusTranslated
- continue
+// batchTail is the French this batch produced, read from the raw model answers
+// in request order and skipping unanswered segments, trimmed to contextRunes.
+// It is exactly what the old in-chapter walk fed the next batch, so threading it
+// through the pipeline keeps a multi-batch chapter's prompts byte-identical.
+func batchTail(items []requestSegment, targets map[string]string) string {
+ var sb strings.Builder
+ for _, item := range items {
+ if target := targets[item.ID]; strings.TrimSpace(target) != "" {
+ sb.WriteString(target)
+ sb.WriteByte('\n')
}
-
- seg.TargetText = target
- seg.Status = segment.StatusPostEdited
}
+ return tailRunes(strings.TrimSpace(sb.String()), contextRunes)
+}
- return Outcome{Usage: run.usage, Degraded: batchErrs}, nil
+// IsDegradable reports whether a literary error leaves the batch on its NMT
+// draft rather than failing the run. A refusal, a truncation and an empty reply
+// are all recoverable by keeping the draft; a rate limit and a
+// transport/credential error are not.
+func IsDegradable(err error) bool {
+ return errors.Is(err, ErrProviderRefusal) ||
+ errors.Is(err, ErrIncompleteResponse) ||
+ errors.Is(err, ErrEmptyResponse)
}
// translateBatch translates b in a single LLM call and merges the results into
@@ -286,8 +236,8 @@ func (t *Translator) TranslateChapter(
// a single segment. Whatever is still unanswered keeps its NMT draft.
//
// Transport and provider errors do fail the batch — a smaller batch will not
-// fix bad credentials — so they propagate to TranslateChapter, which decides
-// whether the chapter as a whole produced anything.
+// fix bad credentials — so they propagate to TranslateBatch, which decides
+// whether the batch as a whole produced anything.
func (t *Translator) translateBatch(ctx context.Context, run *chapterRun, b batch) error {
resp, err := t.Chat.Chat(ctx, llm.ChatRequest{
Model: t.Model,
@@ -374,7 +324,7 @@ func (run *chapterRun) mergeTargets(items []requestSegment, answers []responseSe
}
// missingItems returns the items that targets has no usable translation for.
-// The emptiness test matches the draft-fallback condition in TranslateChapter,
+// The emptiness test matches the draft-fallback condition in applyTargets,
// so "retried" and "fell back to the draft" stay the same predicate.
func missingItems(items []requestSegment, targets map[string]string) []requestSegment {
missing := make([]requestSegment, 0, len(items))
@@ -417,19 +367,6 @@ func (t *Translator) buildPayload(segs []*segment.Segment) ([]requestSegment, []
return items, states
}
-// chunk splits items into batches of at most size segments. A size below 1
-// yields a single batch, matching the "no batching" behaviour.
-func chunk(items []requestSegment, size int) [][]requestSegment {
- if size < 1 {
- size = len(items)
- }
- batches := make([][]requestSegment, 0, (len(items)+size-1)/size)
- for i := 0; i < len(items); i += size {
- batches = append(batches, items[i:min(i+size, len(items))])
- }
- return batches
-}
-
// encodePayload renders items as the JSON request body. Encoding a struct of
// plain strings cannot fail, so no error is plumbed through the callers.
func encodePayload(items []requestSegment) string {
diff --git a/internal/literary/translator_test.go b/internal/literary/translator_test.go
index 1efe5a7..bbe33b4 100644
--- a/internal/literary/translator_test.go
+++ b/internal/literary/translator_test.go
@@ -46,7 +46,8 @@ func (s *stubChat) lastReq() llm.ChatRequest {
// echoChat answers each batch with a target text for every requested segment
// ID. maxSegments caps how many segments a single response carries, simulating
// a small model that stops generating before finishing the batch; 0 answers the
-// whole batch.
+// whole batch. The recursive halving inside TranslateBatch is what turns a
+// short answer into a fully translated batch.
type echoChat struct {
maxSegments int
// truncateOver cuts the response off mid-JSON when the batch holds more than
@@ -135,7 +136,7 @@ func requestedIDs(req llm.ChatRequest) ([]string, error) {
return ids, nil
}
-func TestTranslator_ComposesPromptWithBibleAndGlossary(t *testing.T) {
+func TestTranslateBatch_ComposesPromptWithBibleAndGlossary(t *testing.T) {
t.Parallel()
gloss, err := glossary.New([]glossary.Term{{EN: "Bilbo", FR: "Bilbon", Locked: true}})
@@ -144,7 +145,7 @@ func TestTranslator_ComposesPromptWithBibleAndGlossary(t *testing.T) {
chat := newStubChat(`{"segments":[{"id":"s1","target_text":"Bonjour."}]}`)
tr := literary.Translator{Chat: chat, Model: "model-a", Glossary: gloss}
- _, err = tr.TranslateChapter(t.Context(), bibleFixture(), "ch_0001", chapterOf(segment.Segment{
+ _, err = tr.TranslateBatch(t.Context(), bibleFixture(), "ch_0001", chapterOf(segment.Segment{
ID: "s1",
SourceText: "Bilbo arrived.",
TargetText: "Bilbon arriva.",
@@ -176,7 +177,25 @@ func TestTranslator_ComposesPromptWithBibleAndGlossary(t *testing.T) {
require.Equal(t, "model-a", chat.lastReq().Model)
}
-func TestTranslator_RoundTripPreservesPlaceholders(t *testing.T) {
+func TestTranslateBatch_PrevContextAndChapterSourceRideTheirOwnSlots(t *testing.T) {
+ t.Parallel()
+
+ segs := numberedSegments(10)
+ chat := &echoChat{}
+ tr := literary.Translator{Chat: chat}
+
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001",
+ segs, "AVANT_LE_CHAPITRE", "SOURCE_DU_CHAPITRE_ENTIER")
+ require.NoError(t, err)
+ require.Len(t, chat.reqs, 1, "TranslateBatch sends one call; the caller decides the boundaries")
+
+ // The chapter's whole source is the cacheable prefix; the caller's preceding
+ // context travels in the volatile turn.
+ require.Contains(t, chat.reqs[0].Messages[0].Content, "SOURCE_DU_CHAPITRE_ENTIER")
+ require.Contains(t, precedingContext(t, chat.reqs[0]), "AVANT_LE_CHAPITRE")
+}
+
+func TestTranslateBatch_RoundTripPreservesPlaceholders(t *testing.T) {
t.Parallel()
seg := tokenizedSegment(t, "s1", `
A bold claim.
`)
@@ -186,7 +205,7 @@ func TestTranslator_RoundTripPreservesPlaceholders(t *testing.T) {
tr := literary.Translator{Chat: chat}
segs := chapterOf(seg)
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
require.Equal(t, segment.StatusPostEdited, segs[0].Status)
@@ -196,7 +215,7 @@ func TestTranslator_RoundTripPreservesPlaceholders(t *testing.T) {
require.Contains(t, rendered, "Une affirmation")
}
-func TestTranslator_FallsBackWhenPlaceholderDropped(t *testing.T) {
+func TestTranslateBatch_FallsBackWhenPlaceholderDropped(t *testing.T) {
t.Parallel()
seg := segment.Segment{
@@ -208,13 +227,13 @@ func TestTranslator_FallsBackWhenPlaceholderDropped(t *testing.T) {
tr := literary.Translator{Chat: chat}
segs := chapterOf(seg)
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
require.Equal(t, `Brouillon sûr.`, segs[0].TargetText)
require.Equal(t, segment.StatusTranslated, segs[0].Status)
}
-func TestTranslator_GlossaryLockedTermEnforced(t *testing.T) {
+func TestTranslateBatch_GlossaryLockedTermEnforced(t *testing.T) {
t.Parallel()
gloss, err := glossary.New([]glossary.Term{{EN: "Bilbo", FR: "Bilbon", Locked: true}})
@@ -228,120 +247,137 @@ func TestTranslator_GlossaryLockedTermEnforced(t *testing.T) {
SourceText: "Bilbo waved.",
TargetText: "Bilbon salua.",
})
- _, err = tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ _, err = tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
- require.Equal(t, "Bonjour Bilbon.", segs[0].TargetText)
+ // The glossary token is protected in the prompt and substituted back on the
+ // way out.
require.Contains(t, chat.lastReq().Messages[1].Content, ``)
+ require.Equal(t, "Bonjour Bilbon.", segs[0].TargetText)
}
-func TestTranslator_SplitsChapterIntoBatchesAndSumsUsage(t *testing.T) {
+func TestTranslateBatch_RetriesSegmentsTheModelStoppedShortOf(t *testing.T) {
t.Parallel()
- segs := numberedSegments(25)
- chat := &echoChat{}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
+ segs := numberedSegments(8)
+ // The model answers at most 3 segments per call, as a small model that stops
+ // generating mid-output does. Without the internal halving the last 5 would
+ // silently keep their English draft.
+ chat := &echoChat{maxSegments: 3}
+ tr := literary.Translator{Chat: chat}
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ outcome, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
+ require.Empty(t, outcome.Degraded)
- require.Len(t, chat.batches, 3, "25 segments at BatchSize 10 must be 3 calls")
- require.Len(t, chat.batches[0], 10)
- require.Len(t, chat.batches[1], 10)
- require.Len(t, chat.batches[2], 5)
-
- var requested []string
- for _, batch := range chat.batches {
- requested = append(requested, batch...)
+ require.Greater(t, len(chat.batches), 1, "an incomplete answer must be retried on smaller batches")
+ for _, batch := range chat.batches[1:] {
+ require.Less(t, len(batch), len(segs), "retries must be smaller than the batch that failed")
}
- require.ElementsMatch(t, segmentIDs(segs), requested, "batches must partition the chapter without overlap")
-
for _, seg := range segs {
require.Equal(t, "traduction "+seg.ID, seg.TargetText)
require.Equal(t, segment.StatusPostEdited, seg.Status)
}
-
- // Usage is summed over the three calls rather than taken from the last one.
- require.Equal(t, 300, outcome.Usage.InputTokens)
- require.Equal(t, 30, outcome.Usage.OutputTokens)
- require.Equal(t, 3, outcome.Usage.CacheReadTokens)
- require.Equal(t, 6, outcome.Usage.CacheCreationTokens)
}
-func TestTranslator_RetriesSegmentsTheModelStoppedShortOf(t *testing.T) {
+func TestTranslateBatch_RetriesUnparseableResponseOnSmallerBatches(t *testing.T) {
t.Parallel()
segs := numberedSegments(8)
- // The model answers at most 3 segments per call, as a small model that stops
- // generating mid-output does. Without a retry the last 5 would silently keep
- // their English draft.
- chat := &echoChat{maxSegments: 3}
- tr := literary.Translator{Chat: chat, BatchSize: len(segs)}
+ // Batches over 3 segments come back cut off mid-JSON, as a model hitting its
+ // output limit does; the halved retries parse.
+ chat := &echoChat{truncateOver: 3}
+ tr := literary.Translator{Chat: chat}
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
- require.Greater(t, len(chat.batches), 1, "an incomplete batch must be retried on smaller batches")
- for _, batch := range chat.batches[1:] {
- require.Less(t, len(batch), len(segs), "retries must be smaller than the batch that failed")
- }
+ require.Greater(t, len(chat.batches), 1, "a truncated response must be retried on smaller batches")
for _, seg := range segs {
require.Equal(t, "traduction "+seg.ID, seg.TargetText)
require.Equal(t, segment.StatusPostEdited, seg.Status)
}
}
-func TestTranslator_ContextFollowsTheBatchBoundary(t *testing.T) {
+// TestTranslateBatch_EmptyReplyIsRetriedOnHalvedBatches pins the third case in
+// the failure taxonomy. An empty reply used to propagate as a provider error,
+// so the batch died without a retry: on a real chapter that left 100 of 258
+// segments on their raw NMT draft while the run reported success. It also proves
+// usage accumulates across the halved retries — the empty calls were billed too.
+func TestTranslateBatch_EmptyReplyIsRetriedOnHalvedBatches(t *testing.T) {
t.Parallel()
- segs := numberedSegments(20)
- chat := &echoChat{}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
+ chat := &emptyOverChat{
+ maxSegments: 2,
+ usagePerCall: llm.ChatResponse{InputTokens: 10, OutputTokens: 7},
+ }
+ translator := &literary.Translator{Chat: chat, Model: "m"}
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001",
- segs, "AVANT_LE_CHAPITRE", "SOURCE_DU_CHAPITRE_ENTIER")
+ segs := make([]*segment.Segment, 8)
+ for i := range segs {
+ segs[i] = &segment.Segment{
+ ID: fmt.Sprintf("ch_0001_p_%04d", i+1),
+ SourceText: fmt.Sprintf("Source %d.", i+1),
+ TargetText: "BROUILLON",
+ Status: segment.StatusTranslated,
+ }
+ }
+
+ outcome, err := translator.TranslateBatch(
+ context.Background(), narrative.Bible{SeriesID: "s"}, "ch_0001", segs, "", "source")
require.NoError(t, err)
- require.Len(t, chat.reqs, 2)
- // What follows a batch no longer travels per call: the chapter's whole
- // source sits in the cached prefix, identical for both batches.
- for _, req := range chat.reqs {
- require.Contains(t, req.Messages[0].Content, "SOURCE_DU_CHAPITRE_ENTIER")
+ // Every segment is refined: the 8-segment batch came back empty, and the
+ // halving walked down to batches the model could answer.
+ for _, seg := range segs {
+ require.Equal(t, segment.StatusPostEdited, seg.Status, seg.ID)
+ require.NotEqual(t, "BROUILLON", seg.TargetText)
}
- require.Equal(t, chat.reqs[0].Messages[0].Content, chat.reqs[1].Messages[0].Content,
- "the cacheable prefix must be byte-identical across a chapter's batches")
-
- // The first batch opens the chapter, so it gets the caller's context.
- require.Contains(t, precedingContext(t, chat.reqs[0]), "AVANT_LE_CHAPITRE")
+ require.Empty(t, outcome.Degraded)
- // The second batch continues from the French just produced, not from the
- // previous chapter.
- second := precedingContext(t, chat.reqs[1])
- require.Contains(t, second, "traduction s010", "a batch must see the translation that precedes it")
- require.NotContains(t, second, "AVANT_LE_CHAPITRE")
+ // Usage is summed over every call, empty ones included. Dropping their usage
+ // would under-report every reasoning model that runs out of budget.
+ require.Equal(t, chat.calls*10, outcome.Usage.InputTokens)
+ require.Equal(t, chat.calls*7, outcome.Usage.OutputTokens)
+ require.Greater(t, chat.calls, 1, "the first call must have been retried")
}
-func TestTranslator_RefusalIsANamedFailure(t *testing.T) {
+func TestTranslateBatch_RefusalIsDegradableAndKeepsTheDraft(t *testing.T) {
t.Parallel()
- // An HTTP 200 with empty content and stop_reason=refusal. Splitting the
- // batch will not change the model's mind, so it must fail loudly rather
- // than look like a malformed reply and trigger the halving retry.
- chat := &stubChat{replies: []llm.ChatResponse{{StopReason: llm.StopReasonRefusal}}}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
+ // An HTTP 200 with empty content and stop_reason=refusal. Splitting will not
+ // change the model's mind, so the batch keeps its drafts rather than looking
+ // like a malformed reply and triggering the halving retry. A refusal is a
+ // billable response, so its tokens must survive.
+ chat := &stubChat{replies: []llm.ChatResponse{{
+ StopReason: llm.StopReasonRefusal, InputTokens: 900,
+ }}}
+ tr := literary.Translator{Chat: chat}
+
+ segs := chapterOf(
+ segment.Segment{ID: "s1", SourceText: "A.", TargetText: "brouillon 1"},
+ segment.Segment{ID: "s2", SourceText: "B.", TargetText: "brouillon 2"},
+ )
+ outcome, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "source")
+ require.NoError(t, err, "a degradable failure does not fail the run")
+ require.Len(t, chat.reqs, 1, "a refusal must fail its batch outright, not split it")
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001",
- numberedSegments(20), "", "source")
+ require.Len(t, outcome.Degraded, 1)
+ require.True(t, literary.IsDegradable(outcome.Degraded[0]))
+ require.ErrorIs(t, outcome.Degraded[0], literary.ErrProviderRefusal)
+ require.Positive(t, outcome.Usage.InputTokens, "billable tokens must survive the degradation")
- require.ErrorIs(t, err, literary.ErrProviderRefusal)
- require.Len(t, chat.reqs, 2, "a refusal must fail its batch outright, not split it")
+ for _, seg := range segs {
+ require.Equal(t, "brouillon "+strings.TrimPrefix(seg.ID, "s"), seg.TargetText)
+ require.Equal(t, segment.StatusTranslated, seg.Status)
+ }
}
-func TestTranslator_TruncationOnASingleSegmentIsNamed(t *testing.T) {
+func TestTranslateBatch_TruncationOnASingleSegmentIsDegraded(t *testing.T) {
t.Parallel()
// One segment, cut off mid-JSON with stop_reason=max_tokens. There is
// nothing left to split, so the segment keeps its draft — but the reason is
- // reported instead of silently reading as a decode failure.
+ // reported through Degraded instead of silently reading as a decode failure.
chat := &stubChat{replies: []llm.ChatResponse{{
Text: `{"segments":[{"id":"s001","target_`,
StopReason: llm.StopReasonMaxTokens,
@@ -349,111 +385,83 @@ func TestTranslator_TruncationOnASingleSegmentIsNamed(t *testing.T) {
tr := literary.Translator{Chat: chat}
segs := chapterOf(segment.Segment{ID: "s001", SourceText: "Claim.", TargetText: "brouillon"})
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "source")
-
- require.ErrorIs(t, err, literary.ErrIncompleteResponse)
- require.Equal(t, "brouillon", segs[0].TargetText,
- "a chapter where nothing was translated leaves its drafts untouched")
-}
-
-func TestTranslator_DegradedBatchesAreReported(t *testing.T) {
- t.Parallel()
-
- segs := numberedSegments(20)
- // One of two batches fails; the other succeeds, so the chapter is not an
- // error. The reason must still reach the caller — collecting the named
- // refusal and truncation errors and then dropping them would be the same
- // silence they were added to end.
- chat := &echoChat{errOnCall: 2}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
-
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "src")
-
+ outcome, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "source")
require.NoError(t, err)
+
require.Len(t, outcome.Degraded, 1)
- require.ErrorContains(t, outcome.Degraded[0], "provider unavailable")
+ require.True(t, literary.IsDegradable(outcome.Degraded[0]))
+ require.ErrorIs(t, outcome.Degraded[0], literary.ErrIncompleteResponse)
+ require.Equal(t, "brouillon", segs[0].TargetText,
+ "a batch where nothing was translated leaves its draft untouched")
+ require.Equal(t, segment.StatusTranslated, segs[0].Status)
}
-func TestTranslator_ChapterSurvivesOneFailedBatch(t *testing.T) {
+func TestTranslateBatch_TransportErrorIsLoud(t *testing.T) {
t.Parallel()
- segs := numberedSegments(20)
- // The second of two batches fails outright, as a transport error would.
- chat := &echoChat{errOnCall: 2}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
-
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
- require.NoError(t, err, "one bad batch must not discard the batch that succeeded")
+ // A transport error is not degradable: a smaller batch will not fix bad
+ // credentials, so the batch fails loudly rather than emitting raw drafts the
+ // pipeline does not even persist.
+ chat := &echoChat{errOnCall: 1}
+ tr := literary.Translator{Chat: chat}
- for _, seg := range segs[:10] {
- require.Equal(t, "traduction "+seg.ID, seg.TargetText)
- require.Equal(t, segment.StatusPostEdited, seg.Status)
- }
- for _, seg := range segs[10:] {
- // Same degradation as a segment the model simply did not answer: the
- // draft survives and the status tells the caller it was not post-edited.
- require.Equal(t, "Claim "+seg.ID, seg.TargetText)
- require.Equal(t, segment.StatusTranslated, seg.Status)
- }
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", numberedSegments(5), "", "")
+ require.ErrorContains(t, err, "provider unavailable",
+ "a batch that produced nothing must fail loudly, not return English")
+ require.False(t, literary.IsDegradable(err))
}
-func TestTranslator_FailedChapterStillReportsWhatItConsumed(t *testing.T) {
+func TestTranslateBatch_RateLimitIsFatalAndReportsUsage(t *testing.T) {
t.Parallel()
- // Every batch refused, so the chapter is an error — but a refusal is a
- // billable response. Returning zero usage with the error would let the run
- // be priced as if the calls never happened.
- chat := &stubChat{replies: []llm.ChatResponse{{
- StopReason: llm.StopReasonRefusal, InputTokens: 900, OutputTokens: 0,
- }}}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
-
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001",
- numberedSegments(20), "", "source")
+ segs := numberedSegments(10)
+ chat := &echoChat{rateLimitOnCall: 1}
+ tr := literary.Translator{Chat: chat}
- require.Error(t, err)
- require.Positive(t, outcome.Usage.InputTokens, "billable tokens must survive the error")
-}
+ outcome, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "src")
-func TestTranslator_ErrorsWhenNoBatchProducesAnything(t *testing.T) {
- t.Parallel()
+ // Fatal, not degraded: a chapter that carried on would emit raw drafts the
+ // pipeline does not even persist, so there would be nothing to resume from.
+ require.ErrorIs(t, err, llm.ErrRateLimited)
+ require.False(t, literary.IsDegradable(err), "a rate limit is not degradable")
- chat := &echoChat{errOnCall: 1}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
+ // A refused call is a billable call: the ledger has to carry it.
+ require.Equal(t, 100, outcome.Usage.InputTokens)
+ require.Equal(t, 10, outcome.Usage.OutputTokens)
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", numberedSegments(5), "", "")
- require.ErrorContains(t, err, "provider unavailable",
- "a chapter that produced nothing must fail loudly, not return English")
+ // The batch is abandoned rather than written back: none reaches
+ // StatusPostEdited, and the caller fails the run.
+ for _, seg := range segs {
+ require.NotEqual(t, segment.StatusPostEdited, seg.Status)
+ }
}
-func TestTranslator_RetriesUnparseableResponseOnSmallerBatches(t *testing.T) {
+func TestTranslateBatch_TailIsTheAnswersInRequestOrder(t *testing.T) {
t.Parallel()
- segs := numberedSegments(8)
- // Batches over 3 segments come back cut off mid-JSON, as a model hitting its
- // output limit does.
- chat := &echoChat{truncateOver: 3}
- tr := literary.Translator{Chat: chat, BatchSize: len(segs)}
+ // The model answers the first and third segments but never the second. The
+ // tail must read the raw answers in request order, skipping the unanswered
+ // one — exactly what the next batch is fed as its preceding context.
+ chat := newStubChat(`{"segments":[{"id":"s001","target_text":"Un"},{"id":"s003","target_text":"Trois"}]}`)
+ tr := literary.Translator{Chat: chat}
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
+ segs := numberedSegments(3)
+ outcome, err := tr.TranslateBatch(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
require.NoError(t, err)
- require.Greater(t, len(chat.batches), 1, "a truncated response must be retried on smaller batches")
- for _, seg := range segs {
- require.Equal(t, "traduction "+seg.ID, seg.TargetText)
- require.Equal(t, segment.StatusPostEdited, seg.Status)
- }
+ require.Equal(t, "Un\nTrois", outcome.Tail)
+ require.Len(t, outcome.Degraded, 1, "the unanswered segment is reported")
}
-func TestTranslator_NilChat(t *testing.T) {
+func TestTranslateBatch_NilChat(t *testing.T) {
t.Parallel()
tr := literary.Translator{}
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{}, "ch_0001", chapterOf(segment.Segment{ID: "s1"}), "", "")
+ _, err := tr.TranslateBatch(t.Context(), narrative.Bible{}, "ch_0001", chapterOf(segment.Segment{ID: "s1"}), "", "")
require.ErrorIs(t, err, literary.ErrNilChatClient)
}
-// chapterOf builds the pointer slice TranslateChapter rewrites in place.
+// chapterOf builds the pointer slice TranslateBatch rewrites in place.
func chapterOf(segs ...segment.Segment) []*segment.Segment {
out := make([]*segment.Segment, len(segs))
for i := range segs {
@@ -485,14 +493,6 @@ func precedingContext(t *testing.T, req llm.ChatRequest) string {
return content[prevStart:payloadStart]
}
-func segmentIDs(segs []*segment.Segment) []string {
- ids := make([]string, 0, len(segs))
- for _, seg := range segs {
- ids = append(ids, seg.ID)
- }
- return ids
-}
-
func bibleFixture() narrative.Bible {
return narrative.Bible{
StyleGuide: "Use formal, precise literary French.",
@@ -593,147 +593,3 @@ func (c *emptyOverChat) Chat(_ context.Context, req llm.ChatRequest) (llm.ChatRe
usage.Text = sb.String()
return usage, nil
}
-
-// TestTranslateChapter_EmptyReplyIsRetriedOnHalvedBatches pins the third case
-// in the failure taxonomy. An empty reply used to propagate as a provider
-// error, so the batch died without a retry: on a real chapter that left 100 of
-// 258 segments on their raw NMT draft while the run reported success.
-func TestTranslateChapter_EmptyReplyIsRetriedOnHalvedBatches(t *testing.T) {
- t.Parallel()
-
- chat := &emptyOverChat{
- maxSegments: 2,
- usagePerCall: llm.ChatResponse{InputTokens: 10, OutputTokens: 7},
- }
- translator := &literary.Translator{Chat: chat, Model: "m", BatchSize: 8}
-
- segs := make([]*segment.Segment, 8)
- for i := range segs {
- segs[i] = &segment.Segment{
- ID: fmt.Sprintf("ch_0001_p_%04d", i+1),
- SourceText: fmt.Sprintf("Source %d.", i+1),
- TargetText: "BROUILLON",
- Status: segment.StatusTranslated,
- }
- }
-
- outcome, err := translator.TranslateChapter(
- context.Background(), narrative.Bible{SeriesID: "s"}, "ch_0001", segs, "", "source")
- require.NoError(t, err)
-
- // Every segment is refined: the 8-segment batch came back empty, and the
- // halving walked down to batches the model could answer.
- for _, seg := range segs {
- require.Equal(t, segment.StatusPostEdited, seg.Status, seg.ID)
- require.NotEqual(t, "BROUILLON", seg.TargetText)
- }
- require.Empty(t, outcome.Degraded)
-
- // The empty calls were billed too. Dropping their usage would under-report
- // every reasoning model that runs out of budget.
- require.Equal(t, chat.calls*10, outcome.Usage.InputTokens)
- require.Equal(t, chat.calls*7, outcome.Usage.OutputTokens)
- require.Greater(t, chat.calls, 1, "the first call must have been retried")
-}
-
-func TestTranslator_OnBatchReportsEverySegmentOnce(t *testing.T) {
- t.Parallel()
-
- segs := numberedSegments(25)
- // The second batch fails outright: its segments are as finished as the
- // others', so progress must not stall on them.
- chat := &echoChat{errOnCall: 2}
- var reported []int
- var usages []literary.Usage
- tr := literary.Translator{
- Chat: chat,
- BatchSize: 10,
- OnBatch: func(segments int, usage literary.Usage) {
- reported = append(reported, segments)
- usages = append(usages, usage)
- },
- }
-
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
- require.NoError(t, err)
-
- require.Equal(t, []int{10, 10, 5}, reported, "each batch reports its own size, failed or not")
-
- // Each batch reports its own consumption, not the chapter's running total,
- // and the batches add up to exactly what the chapter reports — so a caller
- // accumulating per batch gets the same figure as one reading Outcome.Usage.
- var sum literary.Usage
- for _, u := range usages {
- sum.InputTokens += u.InputTokens
- sum.OutputTokens += u.OutputTokens
- sum.CacheReadTokens += u.CacheReadTokens
- sum.CacheCreationTokens += u.CacheCreationTokens
- }
- require.Equal(t, outcome.Usage, sum)
- require.Equal(t, 100, usages[0].InputTokens, "the first batch's own tokens, not a cumulative total")
-}
-
-func TestTranslator_OnBatchIsOptional(t *testing.T) {
- t.Parallel()
-
- segs := numberedSegments(3)
- tr := literary.Translator{Chat: &echoChat{}}
-
- _, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "")
- require.NoError(t, err)
-}
-
-func TestTranslator_RateLimitStopsTheChapterInsteadOfDegradingIt(t *testing.T) {
- t.Parallel()
-
- segs := numberedSegments(30)
- // The first batch succeeds, the second hits the wall.
- chat := &echoChat{rateLimitOnCall: 2}
- var reported []int
- tr := literary.Translator{
- Chat: chat,
- BatchSize: 10,
- OnBatch: func(segments int, _ literary.Usage) { reported = append(reported, segments) },
- }
-
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "src")
-
- // Fatal, not degraded: every remaining batch would fail the same way, and a
- // chapter that carried on would emit raw drafts the pipeline does not even
- // persist — so there would be nothing to resume from.
- require.ErrorIs(t, err, llm.ErrRateLimited)
- require.Len(t, chat.batches, 2, "the third batch must never be attempted")
-
- // The batch that hit the wall is still accounted for: it consumed tokens.
- require.Equal(t, []int{10, 10}, reported)
- require.Equal(t, 200, outcome.Usage.InputTokens)
- require.Equal(t, 20, outcome.Usage.OutputTokens)
-
- // The chapter is abandoned rather than written back: the segments keep what
- // they arrived with, none reaches StatusPostEdited, and the caller fails the
- // run. So the resume boundary is the CHAPTER, not the batch — up to one
- // chapter's completed batches are redone on the next attempt.
- //
- // That is deliberate. Writing them back would not save them anyway: the
- // pipeline persists a refined segment only after Restore accepts it, and on
- // this path it never gets there. Buying that back would mean reordering the
- // persist step around a failure, which is the one ordering the TM's
- // correctness rests on.
- for _, seg := range segs {
- require.NotEqual(t, segment.StatusPostEdited, seg.Status)
- }
-}
-
-func TestTranslator_RateLimitOnTheFirstBatchStillReportsItsUsage(t *testing.T) {
- t.Parallel()
-
- segs := numberedSegments(10)
- chat := &echoChat{rateLimitOnCall: 1}
- tr := literary.Translator{Chat: chat, BatchSize: 10}
-
- outcome, err := tr.TranslateChapter(t.Context(), narrative.Bible{StyleGuide: "x"}, "ch_0001", segs, "", "src")
-
- require.ErrorIs(t, err, llm.ErrRateLimited)
- // A refused call is a billable call: the ledger has to carry it.
- require.Equal(t, 100, outcome.Usage.InputTokens)
-}
diff --git a/internal/pipeline/artifacts.go b/internal/pipeline/artifacts.go
index 4e39e9a..3fd3abe 100644
--- a/internal/pipeline/artifacts.go
+++ b/internal/pipeline/artifacts.go
@@ -33,18 +33,19 @@ type segmentRecord struct {
Status string `json:"status"`
}
-// WriteArtifacts writes report.json and segments.jsonl into the run output dir.
-func WriteArtifacts(cfg RunConfig, result *RunResult, segments []segment.Segment) error {
- if err := os.MkdirAll(cfg.OutputDir, 0o755); err != nil {
- return fmt.Errorf("unable to create run output dir %q, err: %w", cfg.OutputDir, err)
+// WriteArtifacts writes report.json, metrics.json and segments.jsonl into the
+// run output dir.
+func WriteArtifacts(ref RunRef, result *RunResult, segments []segment.Segment) error {
+ if err := os.MkdirAll(ref.OutputDir, 0o755); err != nil {
+ return fmt.Errorf("unable to create run output dir %q, err: %w", ref.OutputDir, err)
}
- if err := writeReport(filepath.Join(cfg.OutputDir, reportFileName), result); err != nil {
+ if err := writeReport(filepath.Join(ref.OutputDir, reportFileName), result); err != nil {
return err
}
- if err := writeMetrics(filepath.Join(cfg.OutputDir, metricsFileName), result); err != nil {
+ if err := writeMetrics(filepath.Join(ref.OutputDir, metricsFileName), result); err != nil {
return err
}
- return writeSegments(filepath.Join(cfg.OutputDir, segmentsFileName), cfg, segments)
+ return writeSegments(filepath.Join(ref.OutputDir, segmentsFileName), ref, segments)
}
// writeMetrics emits a flat metrics snapshot using the canonical metric names
@@ -99,7 +100,7 @@ func writeReport(path string, result *RunResult) error {
return nil
}
-func writeSegments(path string, cfg RunConfig, segments []segment.Segment) error {
+func writeSegments(path string, ref RunRef, segments []segment.Segment) error {
f, err := os.Create(path)
if err != nil {
return fmt.Errorf("unable to create segments artifact %q, err: %w", path, err)
@@ -119,8 +120,8 @@ func writeSegments(path string, cfg RunConfig, segments []segment.Segment) error
rec := segmentRecord{
SchemaVersion: 1,
- RunID: cfg.RunID,
- BookID: cfg.BookID,
+ RunID: ref.RunID,
+ BookID: ref.BookID,
ChapterID: seg.ChapterID,
SegmentID: seg.ID,
NodePath: seg.NodePath,
diff --git a/internal/pipeline/book.go b/internal/pipeline/book.go
new file mode 100644
index 0000000..41eeab4
--- /dev/null
+++ b/internal/pipeline/book.go
@@ -0,0 +1,382 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+
+ "github.com/ECecillo/tomeshift/internal/qa"
+)
+
+// Progress stage labels. Coarser than the old per-sub-stage bar because a stage
+// is now an activity boundary: extract/parse/segment all happen inside
+// PrepareBook, which the flow cannot see into.
+const (
+ stagePrepare = "prepare"
+ stageTranslate = "translate"
+ stageRecompose = "recompose"
+ stageFinish = "qa"
+)
+
+// flow accumulates the RunResult over the ports as a book is translated. It is
+// the receiver the per-chapter mode recipes drive, and it publishes Progress as
+// it goes. It holds no engine detail: everything it touches is a Steps call or
+// an rt method.
+type flow struct {
+ steps Steps
+ rt Runtime
+ in BookInput
+ ref RunRef
+ mode Mode
+ result *RunResult
+ chapters []ChapterInfo
+
+ usedFreeChars int
+ batchSize int
+ chapterIdx int // 1-based chapter currently being worked on
+ prevChapterID string // spine-previous chapter id, for a chapter's first literary batch context
+
+ qeSum float64
+ qeCount int
+
+ lastCost float64
+ finalised bool
+ finished bool
+}
+
+// RunBook executes one volume over the ports. It accumulates the RunResult,
+// publishes Progress, and returns *StageError / wrapped ErrQAGateFailed /
+// wrapped cost.ErrBudgetExceeded exactly as the in-process engine did before
+// the refactor, so exit codes are unchanged. It is deterministic: rt.Now() is
+// its only clock and it makes no decision on map order or goroutine scheduling.
+func RunBook(steps Steps, rt Runtime, in BookInput) (*RunResult, error) {
+ mode, err := selectMode(in)
+ if err != nil {
+ return nil, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ ref := RunRef{RunID: in.RunID, BookID: in.BookID, OutputDir: in.OutputDir, InputPath: in.InputPath}
+ f := &flow{steps: steps, rt: rt, in: in, ref: ref, mode: mode}
+ f.report(stagePrepare)
+
+ manifest, err := steps.PrepareBook(in)
+ if err != nil {
+ return nil, err
+ }
+
+ result := &RunResult{
+ SchemaVersion: 1,
+ RunID: in.RunID,
+ BookID: in.BookID,
+ Mode: runMode(in),
+ StartedAt: rt.Now().UTC(),
+ TotalSegments: totalSegments(manifest.Chapters),
+ NMTEngine: manifest.NMTEngine,
+ LLMProvider: manifest.LLMProvider,
+ LLMModel: manifest.LLMModel,
+ }
+ if runMode(in) == ModeLiterary {
+ result.SeriesID = in.SeriesID
+ }
+ f.result = result
+ f.chapters = manifest.Chapters
+ f.batchSize = manifest.BatchSize
+
+ // Pre-run budget guardrail: it prices the work still to do and refuses the
+ // run before any provider call if a cap is exceeded. NoReuseUSD is the whole
+ // book with no reuse — the report's baseline — and BibleVersion (the loaded
+ // bible) are stamped even on the refusal, matching the pre-split engine.
+ proj, err := steps.ProjectCost(ProjectInput{Ref: ref, Mode: runMode(in), SeriesID: in.SeriesID})
+ result.EstimatedCostUSD = proj.NoReuseUSD
+ result.BibleVersion = proj.BibleVersion
+ if err != nil {
+ return result, err
+ }
+ f.usedFreeChars = proj.UsedFreeChars
+
+ // From here the run can call paid providers, so its consumption is recorded
+ // whatever happens next — and, like today's deferred ledger write, under a
+ // detached runtime so a cancelled context still writes.
+ defer f.cleanup()
+
+ f.report(stageTranslate)
+ for i := range manifest.Chapters {
+ if rt.Cancelled() {
+ return nil, &StageError{Stage: StageTranslate, Err: context.Canceled}
+ }
+ f.chapterIdx = i + 1
+ if err := mode.Chapter(f, manifest.Chapters[i]); err != nil {
+ return nil, err
+ }
+ }
+
+ f.report(stageRecompose)
+ rec, err := steps.RecomposeBook(ref)
+ if err != nil {
+ return nil, err
+ }
+ result.OutputEPUB = rec.OutputEPUB
+
+ f.report(stageFinish)
+ check, err := steps.CheckEPUB(CheckInput(rec))
+ if err != nil {
+ return nil, err
+ }
+ result.QA.EPUBCheckPassed = check.Passed
+ result.QA.EPUBCheckSkipped = check.Skipped
+
+ f.finalise()
+ if result.TotalSegments > 0 {
+ result.TMHitRatio = float64(result.TMHits) / float64(result.TotalSegments)
+ result.LLMEscalationRatio = float64(result.LLMSegments) / float64(result.TotalSegments)
+ }
+ if f.qeCount > 0 {
+ result.AvgQualityScore = f.qeSum / float64(f.qeCount)
+ result.QualityScoredSegments = f.qeCount
+ }
+ if savings := result.EstimatedCostUSD - result.ActualCostUSD; savings > 0 {
+ result.CostSavingsUSD = savings
+ }
+ result.EndedAt = rt.Now().UTC()
+
+ report, err := steps.FinishBook(FinishInput{Ref: ref, Result: *result})
+ if err != nil {
+ return result, err
+ }
+ result.QA = report
+ f.finished = true
+
+ if !qaGatePassed(report) || result.GlossaryViolations > 0 {
+ return result, fmt.Errorf("unable to pass QA gate, err: %w", ErrQAGateFailed)
+ }
+ return result, nil
+}
+
+// cleanup records the run's spend on every exit past the guardrail, and — for a
+// run that failed before FinishBook — writes what artifacts it can so a failed
+// run stays diagnosable. It runs under a detached runtime: FinishBook and
+// RecordSpend detach from a cancelled context internally, and rt.Detached()
+// supplies the clock and the not-cancelled view the cleanup runs under.
+func (f *flow) cleanup() {
+ d := f.rt.Detached()
+ f.finalise()
+ if f.result.EndedAt.IsZero() {
+ f.result.EndedAt = d.Now().UTC()
+ }
+ if !f.finished {
+ if report, err := f.steps.FinishBook(FinishInput{Ref: f.ref, Result: *f.result}); err == nil {
+ f.result.QA = report
+ }
+ }
+ _ = f.steps.RecordSpend(SpendInput{Ref: f.ref, Result: *f.result})
+}
+
+// finalise prices the run once. It is single-shot so the success path (which
+// calls it before writing the report) and the deferred cleanup agree. NMTChars
+// is already summed from each DraftChapter, so pricing is pure arithmetic.
+func (f *flow) finalise() {
+ if f.finalised {
+ return
+ }
+ f.finalised = true
+ usd, err := f.steps.PriceRun(PriceInput{Result: *f.result, UsedFreeChars: f.usedFreeChars, Mode: runMode(f.in)})
+ if err == nil {
+ f.result.ActualCostUSD = usd
+ }
+}
+
+// ---------------------------------------------------------------------------
+// Per-chapter helpers the mode recipes compose.
+// ---------------------------------------------------------------------------
+
+// resolve serves a chapter from the TM and records the reuse.
+func (f *flow) resolve(ch ChapterInfo, reusesDrafts bool) (ChapterPlan, error) {
+ plan, err := f.steps.ResolveChapter(ResolveInput{Ref: f.ref, ChapterID: ch.ID, ReusesDrafts: reusesDrafts})
+ if err != nil {
+ return ChapterPlan{}, err
+ }
+ f.result.TMHits += plan.Reused
+ f.report(stageTranslate)
+ return plan, nil
+}
+
+// draft drafts a chapter's fresh segments in one batched provider call.
+func (f *flow) draft(ch ChapterInfo, fresh []string, persist bool) error {
+ res, err := f.steps.DraftChapter(DraftInput{
+ Ref: f.ref, ChapterID: ch.ID, Fresh: fresh, PersistDrafts: persist,
+ SourceLang: f.in.SourceLang, TargetLang: f.in.TargetLang,
+ })
+ if err != nil {
+ return err
+ }
+ f.result.NMTSegments += len(fresh)
+ f.result.NMTChars += res.Chars
+ return nil
+}
+
+// escalate runs the NMT-mode quality estimation and selective post-edit.
+func (f *flow) escalate(ch ChapterInfo, fresh []string) error {
+ res, err := f.steps.EscalateChapter(EscalateInput{
+ Ref: f.ref, ChapterID: ch.ID, Fresh: fresh,
+ SourceLang: f.in.SourceLang, TargetLang: f.in.TargetLang,
+ })
+ if err != nil {
+ return err
+ }
+ f.result.LLMAttempts += res.Attempts
+ f.result.LLMPassSegments += res.Attempts
+ f.result.LLMChars += res.Chars
+ f.result.LLMSegments += res.Improved
+ f.qeSum += res.QESum
+ f.qeCount += res.QECount
+ return nil
+}
+
+// preceding French across batches: within a chapter the previous batch's
+// translated tail, and for the first batch the previous chapter's source tail —
+// resolved by RefineBatch from PrevChapterID, matching the pre-split walk.
+func (f *flow) refine(ch ChapterInfo, fresh []string) error {
+ prevTail := ""
+ refined := 0
+ for i, batch := range chunkIDs(fresh, f.batchSize) {
+ in := RefineInput{
+ Ref: f.ref, ChapterID: ch.ID, SeriesID: f.in.SeriesID,
+ IDs: batch, PrevTail: prevTail,
+ SourceLang: f.in.SourceLang, TargetLang: f.in.TargetLang,
+ }
+ if i == 0 {
+ in.PrevChapterID = f.prevChapterID
+ }
+ res, err := f.steps.RefineBatch(in)
+ if err != nil {
+ return err
+ }
+ f.result.LLMPassSegments += len(batch)
+ f.result.LLMInputTokens += res.Usage.InputTokens
+ f.result.LLMOutputTokens += res.Usage.OutputTokens
+ f.result.LLMCacheReadTokens += res.Usage.CacheReadTokens
+ f.result.LLMCacheCreationTokens += res.Usage.CacheCreationTokens
+ f.result.LLMSegments += res.Refined
+ refined += res.Refined
+ prevTail = res.Tail
+ f.priceReport(stageTranslate)
+ }
+ // The chapter-level backstop the per-batch split would otherwise lose: if
+ // every batch degraded and not one segment was refined, the chapter is a
+ // wall of NMT drafts that literary mode does not persist — a book that reads
+ // as finished with nothing to resume from. QA cannot see it (a draft has a
+ // non-empty target), so it is failed here, exactly as the old chapter-level
+ // TranslateChapter returned ErrEmptyResponse.
+ if refined == 0 {
+ return &StageError{Stage: StageTranslate, Err: fmt.Errorf(
+ "unable to refine any segment of chapter %s, err: %w", ch.ID, ErrChapterUnrefined)}
+ }
+ return nil
+}
+
+// commit validates and persists a chapter's translations.
+func (f *flow) commit(ch ChapterInfo, persist bool) error {
+ res, err := f.steps.CommitChapter(CommitInput{
+ Ref: f.ref, ChapterID: ch.ID, Persist: persist,
+ SourceLang: f.in.SourceLang, TargetLang: f.in.TargetLang,
+ })
+ if err != nil {
+ return err
+ }
+ f.result.GlossaryViolations += res.GlossaryViolations
+ return nil
+}
+
+// updateBible evolves the series bible from a committed chapter.
+func (f *flow) updateBible(ch ChapterInfo) error {
+ res, err := f.steps.UpdateBible(BibleInput{Ref: f.ref, SeriesID: f.in.SeriesID, ChapterID: ch.ID})
+ if err != nil {
+ return err
+ }
+ f.result.BibleVersion = res.Version
+ return nil
+}
+
+// ---------------------------------------------------------------------------
+// Progress.
+// ---------------------------------------------------------------------------
+
+// report publishes the current progress snapshot without repricing.
+func (f *flow) report(stage string) {
+ f.rt.Report(f.progress(stage))
+}
+
+// priceReport reprices the run and publishes the progress with the new cost.
+func (f *flow) priceReport(stage string) {
+ if usd, err := f.steps.PriceRun(PriceInput{Result: *f.result, UsedFreeChars: f.usedFreeChars, Mode: runMode(f.in)}); err == nil {
+ f.lastCost = usd
+ }
+ f.rt.Report(f.progress(stage))
+}
+
+func (f *flow) progress(stage string) Progress {
+ p := Progress{
+ State: "running",
+ Stage: stage,
+ Chapter: f.chapterIdx,
+ Chapters: len(f.chapters),
+ CostUSD: f.lastCost,
+ }
+ if f.result != nil {
+ p.Segments = f.result.TotalSegments
+ // Literary advances on the refine pass (segments carried by a completed
+ // batch), not at draft time — otherwise the bar fills the instant a
+ // chapter is drafted and then sits at 100% for the whole LLM pass. NMT
+ // has no refine pass, so its output is the draft.
+ if runMode(f.in) == ModeLiterary {
+ p.SegmentsDone = f.result.TMHits + f.result.LLMPassSegments
+ } else {
+ p.SegmentsDone = f.result.TMHits + f.result.NMTSegments
+ }
+ p.TMHits = f.result.TMHits
+ }
+ return p
+}
+
+// ---------------------------------------------------------------------------
+// Pure helpers.
+// ---------------------------------------------------------------------------
+
+// qaGatePassed combines static QA thresholds with the epubcheck verdict. A
+// skipped epubcheck (binary absent) does not block the run.
+func qaGatePassed(report qa.Report) bool {
+ if !report.Passed() {
+ return false
+ }
+ if !report.EPUBCheckSkipped && !report.EPUBCheckPassed {
+ return false
+ }
+ return true
+}
+
+func totalSegments(chapters []ChapterInfo) int {
+ total := 0
+ for i := range chapters {
+ total += chapters[i].Segments
+ }
+ return total
+}
+
+// chunkIDs splits ids into batches of at most size. A size below 1 yields a
+// single batch — no batching.
+func chunkIDs(ids []string, size int) [][]string {
+ if size < 1 {
+ size = len(ids)
+ }
+ if size < 1 {
+ return nil
+ }
+ var out [][]string
+ for i := 0; i < len(ids); i += size {
+ end := i + size
+ if end > len(ids) {
+ end = len(ids)
+ }
+ out = append(out, ids[i:end])
+ }
+ return out
+}
diff --git a/internal/pipeline/book_test.go b/internal/pipeline/book_test.go
new file mode 100644
index 0000000..b8a63c2
--- /dev/null
+++ b/internal/pipeline/book_test.go
@@ -0,0 +1,667 @@
+package pipeline_test
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ECecillo/tomeshift/internal/cost"
+ "github.com/ECecillo/tomeshift/internal/literary"
+ "github.com/ECecillo/tomeshift/internal/llm"
+ "github.com/ECecillo/tomeshift/internal/pipeline"
+ "github.com/ECecillo/tomeshift/internal/qa"
+)
+
+// ---------------------------------------------------------------------------
+// Fake Steps: a struct of func fields recording calls and returning scripted
+// results. It is the level-2 test double the RFC calls for — plain Go, no
+// engine — so RunBook's control flow, accumulation and cleanup are proven in
+// isolation from the stage bodies.
+// ---------------------------------------------------------------------------
+
+type fakeSteps struct {
+ prepare func(pipeline.BookInput) (pipeline.Manifest, error)
+ project func(pipeline.ProjectInput) (pipeline.Projection, error)
+ resolve func(pipeline.ResolveInput) (pipeline.ChapterPlan, error)
+ draft func(pipeline.DraftInput) (pipeline.DraftResult, error)
+ escalate func(pipeline.EscalateInput) (pipeline.EscalateResult, error)
+ refine func(pipeline.RefineInput) (pipeline.RefineResult, error)
+ commit func(pipeline.CommitInput) (pipeline.CommitResult, error)
+ bible func(pipeline.BibleInput) (pipeline.BibleResult, error)
+ recompose func(pipeline.RunRef) (pipeline.RecomposeResult, error)
+ check func(pipeline.CheckInput) (pipeline.CheckResult, error)
+ finish func(pipeline.FinishInput) (qa.Report, error)
+ spend func(pipeline.SpendInput) error
+ price func(pipeline.PriceInput) (float64, error)
+
+ calls []string
+ draftIDs [][]string
+ finishCalls int
+ spendCalls int
+ commitCalls int
+ updateBible int
+ refineInputs []pipeline.RefineInput
+ spendInput pipeline.SpendInput
+}
+
+func (f *fakeSteps) PrepareBook(in pipeline.BookInput) (pipeline.Manifest, error) {
+ f.calls = append(f.calls, "prepare")
+ return f.prepare(in)
+}
+
+func (f *fakeSteps) ProjectCost(in pipeline.ProjectInput) (pipeline.Projection, error) {
+ f.calls = append(f.calls, "project")
+ if f.project != nil {
+ return f.project(in)
+ }
+ return pipeline.Projection{}, nil
+}
+
+func (f *fakeSteps) ResolveChapter(in pipeline.ResolveInput) (pipeline.ChapterPlan, error) {
+ f.calls = append(f.calls, "resolve:"+in.ChapterID)
+ return f.resolve(in)
+}
+
+func (f *fakeSteps) DraftChapter(in pipeline.DraftInput) (pipeline.DraftResult, error) {
+ f.calls = append(f.calls, "draft:"+in.ChapterID)
+ f.draftIDs = append(f.draftIDs, in.Fresh)
+ if f.draft != nil {
+ return f.draft(in)
+ }
+ return pipeline.DraftResult{Chars: len(in.Fresh), Engine: "nmt"}, nil
+}
+
+func (f *fakeSteps) EscalateChapter(in pipeline.EscalateInput) (pipeline.EscalateResult, error) {
+ f.calls = append(f.calls, "escalate:"+in.ChapterID)
+ if f.escalate != nil {
+ return f.escalate(in)
+ }
+ return pipeline.EscalateResult{}, nil
+}
+
+func (f *fakeSteps) RefineBatch(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ f.calls = append(f.calls, "refine:"+in.ChapterID)
+ f.refineInputs = append(f.refineInputs, in)
+ if f.refine != nil {
+ return f.refine(in)
+ }
+ return pipeline.RefineResult{Refined: len(in.IDs)}, nil
+}
+
+func (f *fakeSteps) CommitChapter(in pipeline.CommitInput) (pipeline.CommitResult, error) {
+ f.calls = append(f.calls, "commit:"+in.ChapterID)
+ f.commitCalls++
+ if f.commit != nil {
+ return f.commit(in)
+ }
+ return pipeline.CommitResult{}, nil
+}
+
+func (f *fakeSteps) UpdateBible(in pipeline.BibleInput) (pipeline.BibleResult, error) {
+ f.calls = append(f.calls, "bible:"+in.ChapterID)
+ f.updateBible++
+ if f.bible != nil {
+ return f.bible(in)
+ }
+ return pipeline.BibleResult{}, nil
+}
+
+func (f *fakeSteps) RecomposeBook(ref pipeline.RunRef) (pipeline.RecomposeResult, error) {
+ f.calls = append(f.calls, "recompose")
+ if f.recompose != nil {
+ return f.recompose(ref)
+ }
+ return pipeline.RecomposeResult{OutputEPUB: ref.OutputDir + "/translated.epub"}, nil
+}
+
+func (f *fakeSteps) CheckEPUB(in pipeline.CheckInput) (pipeline.CheckResult, error) {
+ f.calls = append(f.calls, "check")
+ if f.check != nil {
+ return f.check(in)
+ }
+ return pipeline.CheckResult{Skipped: true}, nil
+}
+
+func (f *fakeSteps) FinishBook(in pipeline.FinishInput) (qa.Report, error) {
+ f.calls = append(f.calls, "finish")
+ f.finishCalls++
+ if f.finish != nil {
+ return f.finish(in)
+ }
+ return passingReport(in), nil
+}
+
+func (f *fakeSteps) RecordSpend(in pipeline.SpendInput) error {
+ f.calls = append(f.calls, "spend")
+ f.spendCalls++
+ f.spendInput = in
+ if f.spend != nil {
+ return f.spend(in)
+ }
+ return nil
+}
+
+func (f *fakeSteps) PriceRun(in pipeline.PriceInput) (float64, error) {
+ if f.price != nil {
+ return f.price(in)
+ }
+ return 0, nil
+}
+
+// passingReport echoes the epubcheck verdict the flow stamped and reports clean
+// static QA, mirroring the real FinishBook.
+func passingReport(in pipeline.FinishInput) qa.Report {
+ return qa.Report{
+ PlaceholderRestoreRatio: 1,
+ EPUBCheckPassed: in.Result.QA.EPUBCheckPassed,
+ EPUBCheckSkipped: in.Result.QA.EPUBCheckSkipped,
+ }
+}
+
+// ---------------------------------------------------------------------------
+// A scriptable book: chapters with a reuse/fresh split, so the fake resolve and
+// draft can produce consistent counters without a workspace.
+// ---------------------------------------------------------------------------
+
+type chapterScript struct {
+ id string
+ reused int
+ fresh []string
+ chars int
+}
+
+func script(mode string, chapters ...chapterScript) *fakeSteps {
+ byID := make(map[string]chapterScript, len(chapters))
+ infos := make([]pipeline.ChapterInfo, 0, len(chapters))
+ for _, c := range chapters {
+ byID[c.id] = c
+ infos = append(infos, pipeline.ChapterInfo{ID: c.id, Segments: c.reused + len(c.fresh)})
+ }
+
+ batchSize := 0
+ if mode == pipeline.ModeLiterary {
+ batchSize = 50
+ }
+
+ f := &fakeSteps{}
+ f.prepare = func(pipeline.BookInput) (pipeline.Manifest, error) {
+ return pipeline.Manifest{Chapters: infos, NMTEngine: "nmt", BatchSize: batchSize}, nil
+ }
+ f.resolve = func(in pipeline.ResolveInput) (pipeline.ChapterPlan, error) {
+ c := byID[in.ChapterID]
+ return pipeline.ChapterPlan{Reused: c.reused, Fresh: c.fresh}, nil
+ }
+ f.draft = func(in pipeline.DraftInput) (pipeline.DraftResult, error) {
+ return pipeline.DraftResult{Chars: byID[in.ChapterID].chars, Engine: "nmt"}, nil
+ }
+ return f
+}
+
+func nominalInput(mode string) pipeline.BookInput {
+ return pipeline.BookInput{
+ InputPath: "book.epub", OutputDir: "out", SourceLang: "en", TargetLang: "fr",
+ RunID: "r1", BookID: "b1", Mode: mode, SeriesID: "series-a",
+ }
+}
+
+func newRuntime(rep *recordingReporter) *pipeline.InProcessRuntime {
+ return &pipeline.InProcessRuntime{Progress: rep}
+}
+
+// recordingReporter captures the Progress snapshots a watcher would have seen.
+type recordingReporter struct{ frames []pipeline.Progress }
+
+func (r *recordingReporter) Report(p pipeline.Progress) { r.frames = append(r.frames, p) }
+
+func (r *recordingReporter) last() pipeline.Progress {
+ if len(r.frames) == 0 {
+ return pipeline.Progress{}
+ }
+ return r.frames[len(r.frames)-1]
+}
+
+// ---------------------------------------------------------------------------
+// Scenarios.
+// ---------------------------------------------------------------------------
+
+func TestRunBook_NominalSumsAcrossChapters(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT,
+ chapterScript{id: "ch_0001", reused: 1, fresh: []string{"a", "b"}, chars: 20},
+ chapterScript{id: "ch_0002", reused: 0, fresh: []string{"c", "d", "e"}, chars: 30},
+ )
+ steps.escalate = func(in pipeline.EscalateInput) (pipeline.EscalateResult, error) {
+ // One segment escalated per chapter, all improved.
+ return pipeline.EscalateResult{Attempts: 1, Chars: 5, Improved: 1}, nil
+ }
+ steps.project = func(pipeline.ProjectInput) (pipeline.Projection, error) {
+ return pipeline.Projection{NoReuseUSD: 0.6}, nil
+ }
+ steps.price = func(in pipeline.PriceInput) (float64, error) {
+ return float64(in.Result.NMTChars) / 100, nil
+ }
+ rep := &recordingReporter{}
+
+ result, err := pipeline.RunBook(steps, newRuntime(rep), nominalInput(pipeline.ModeNMT))
+ require.NoError(t, err)
+
+ require.Equal(t, 6, result.TotalSegments)
+ require.Equal(t, 1, result.TMHits)
+ require.Equal(t, 5, result.NMTSegments)
+ require.Equal(t, 50, result.NMTChars) // 20 + 30
+ require.Equal(t, 2, result.LLMSegments)
+ require.Equal(t, 2, result.LLMAttempts)
+ require.Equal(t, 2, result.LLMPassSegments)
+ require.InDelta(t, float64(1)/6, result.TMHitRatio, 1e-9)
+ require.InDelta(t, float64(2)/6, result.LLMEscalationRatio, 1e-9)
+ require.InDelta(t, 0.5, result.ActualCostUSD, 1e-9) // 50 chars / 100
+ require.InDelta(t, 0.6, result.EstimatedCostUSD, 1e-9)
+ require.InDelta(t, 0.1, result.CostSavingsUSD, 1e-9)
+ require.Equal(t, "nmt", result.NMTEngine)
+
+ // Progress: every segment accounted for once, cost monotonic, ending on the
+ // figure the report carries.
+ require.Equal(t, 6, rep.last().SegmentsDone)
+ require.Equal(t, 1, rep.last().TMHits)
+ require.InDelta(t, result.ActualCostUSD, rep.last().CostUSD, 1e-9)
+ var prev float64
+ for _, fr := range rep.frames {
+ require.GreaterOrEqual(t, fr.CostUSD, prev, "spend cannot go down")
+ prev = fr.CostUSD
+ }
+
+ // One FinishBook and one RecordSpend on the success path.
+ require.Equal(t, 1, steps.finishCalls)
+ require.Equal(t, 1, steps.spendCalls)
+}
+
+func TestRunBook_BudgetCapRefusesWithoutDrafting(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}, chars: 10})
+ steps.project = func(pipeline.ProjectInput) (pipeline.Projection, error) {
+ return pipeline.Projection{NoReuseUSD: 3}, budgetErr()
+ }
+
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ require.ErrorIs(t, err, cost.ErrBudgetExceeded)
+ require.NotNil(t, result)
+ require.InDelta(t, 3.0, result.EstimatedCostUSD, 1e-9, "the report still knows the no-reuse cost")
+
+ // Nothing was drafted, and no spend was recorded: the guardrail refuses
+ // before the run can call a provider.
+ require.NotContains(t, steps.calls, "draft:ch_0001")
+ require.Zero(t, steps.draftIDs)
+ require.Zero(t, steps.spendCalls)
+ require.Zero(t, steps.finishCalls)
+}
+
+func budgetErr() error { return fmt.Errorf("unable to start run, err: %w", cost.ErrBudgetExceeded) }
+
+func TestRunBook_RollingCapRefusalPropagates(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}})
+ steps.project = func(pipeline.ProjectInput) (pipeline.Projection, error) {
+ return pipeline.Projection{NoReuseUSD: 1}, wrapErr(cost.ErrBudgetExceeded)
+ }
+
+ _, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ require.ErrorIs(t, err, cost.ErrBudgetExceeded)
+ require.Zero(t, steps.spendCalls)
+}
+
+func TestRunBook_StageErrorCarriesItsStage(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ wire func(*fakeSteps)
+ stage pipeline.Stage
+ }{
+ {
+ name: "recompose failure",
+ stage: pipeline.StageRecompose,
+ wire: func(f *fakeSteps) {
+ f.recompose = func(pipeline.RunRef) (pipeline.RecomposeResult, error) {
+ return pipeline.RecomposeResult{}, &pipeline.StageError{Stage: pipeline.StageRecompose, Err: errors.New("boom")}
+ }
+ },
+ },
+ {
+ name: "draft failure",
+ stage: pipeline.StageTranslate,
+ wire: func(f *fakeSteps) {
+ f.draft = func(pipeline.DraftInput) (pipeline.DraftResult, error) {
+ return pipeline.DraftResult{}, &pipeline.StageError{Stage: pipeline.StageTranslate, Err: errors.New("provider down")}
+ }
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}, chars: 10})
+ tt.wire(steps)
+
+ _, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ var stageErr *pipeline.StageError
+ require.ErrorAs(t, err, &stageErr)
+ require.Equal(t, tt.stage, stageErr.Stage)
+
+ // A failure past the guardrail still records spend.
+ require.Equal(t, 1, steps.spendCalls)
+ })
+ }
+}
+
+func TestRunBook_QAGateFailureIsReported(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}, chars: 10})
+ steps.finish = func(pipeline.FinishInput) (qa.Report, error) {
+ // Placeholder restoration below the threshold fails the gate, but the
+ // artifacts were still written.
+ return qa.Report{PlaceholderRestoreRatio: 0.5, EPUBCheckSkipped: true}, nil
+ }
+
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ require.ErrorIs(t, err, pipeline.ErrQAGateFailed)
+ require.NotNil(t, result)
+ require.Equal(t, 1, steps.finishCalls, "artifacts written despite the gate failure")
+ require.Equal(t, 1, steps.spendCalls)
+}
+
+func TestRunBook_GlossaryViolationsFailTheGate(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}, chars: 10})
+ steps.commit = func(pipeline.CommitInput) (pipeline.CommitResult, error) {
+ return pipeline.CommitResult{GlossaryViolations: 1}, nil
+ }
+
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ require.ErrorIs(t, err, pipeline.ErrQAGateFailed)
+ require.Equal(t, 1, result.GlossaryViolations)
+}
+
+func TestRunBook_CancelStillFinishesAndRecordsSpend(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeNMT, chapterScript{id: "ch_0001", fresh: []string{"a"}, chars: 10})
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ rt := &pipeline.InProcessRuntime{Ctx: ctx}
+
+ _, err := pipeline.RunBook(steps, rt, nominalInput(pipeline.ModeNMT))
+ require.ErrorIs(t, err, context.Canceled)
+
+ // The cleanup runs under a detached runtime: a cancelled run still writes
+ // what artifacts it can and records its spend.
+ require.Equal(t, 1, steps.finishCalls)
+ require.Equal(t, 1, steps.spendCalls)
+ // It was cancelled before any chapter work.
+ require.NotContains(t, steps.calls, "resolve:ch_0001")
+}
+
+func TestRunBook_LiteraryPathDriftsBatchesAndBible(t *testing.T) {
+ t.Parallel()
+
+ steps := script(pipeline.ModeLiterary,
+ chapterScript{id: "ch_0001", reused: 1, fresh: []string{"a", "b"}, chars: 12},
+ chapterScript{id: "ch_0002", reused: 0, fresh: []string{"c", "d"}, chars: 8},
+ )
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ return pipeline.RefineResult{
+ Usage: literaryUsage(6, 3),
+ Refined: len(in.IDs),
+ Tail: "queue-" + in.ChapterID,
+ }, nil
+ }
+ steps.commit = func(in pipeline.CommitInput) (pipeline.CommitResult, error) {
+ require.True(t, in.Persist, "literary commit must persist")
+ return pipeline.CommitResult{}, nil
+ }
+ steps.bible = func(in pipeline.BibleInput) (pipeline.BibleResult, error) {
+ if in.ChapterID == "ch_0001" {
+ return pipeline.BibleResult{Version: 1}, nil
+ }
+ return pipeline.BibleResult{Version: 2}, nil
+ }
+
+ in := nominalInput(pipeline.ModeLiterary)
+ in.UpdateBible = true
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), in)
+ require.NoError(t, err)
+
+ require.Equal(t, pipeline.ModeLiterary, result.Mode)
+ require.Equal(t, "series-a", result.SeriesID)
+ require.Equal(t, 4, result.NMTSegments) // 2 + 2 fresh drafted
+ require.Equal(t, 1, result.TMHits)
+ require.Equal(t, 4, result.LLMSegments)
+ require.Equal(t, 4, result.LLMPassSegments)
+ require.Equal(t, 12, result.LLMInputTokens) // one batch per chapter, 6 in each
+ require.Equal(t, 6, result.LLMOutputTokens) // one batch per chapter, 3 in each
+ require.Equal(t, 2, result.BibleVersion)
+ require.Equal(t, 0, result.GlossaryViolations)
+ require.Equal(t, 2, steps.updateBible)
+
+ // One refine call per chapter (2 fresh < batch size). A chapter's first batch
+ // carries no translated tail; the previous chapter's id is what RefineBatch
+ // turns into the source-tail context.
+ require.Len(t, steps.refineInputs, 2)
+ require.Equal(t, "", steps.refineInputs[0].PrevTail)
+ require.Equal(t, "", steps.refineInputs[0].PrevChapterID)
+ require.Equal(t, "", steps.refineInputs[1].PrevTail)
+ require.Equal(t, "ch_0001", steps.refineInputs[1].PrevChapterID)
+}
+
+func TestRunBook_LiteraryChunksFreshByBatchSize(t *testing.T) {
+ t.Parallel()
+
+ fresh := freshIDs(25)
+ steps := literaryBatched(fresh, 10)
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ return pipeline.RefineResult{Refined: len(in.IDs), Tail: "queue-" + in.IDs[len(in.IDs)-1]}, nil
+ }
+
+ _, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeLiterary))
+ require.NoError(t, err)
+
+ // 25 fresh at batch size 10 → three batches partitioning the ids in order.
+ require.Len(t, steps.refineInputs, 3)
+ require.Equal(t, fresh[0:10], steps.refineInputs[0].IDs)
+ require.Equal(t, fresh[10:20], steps.refineInputs[1].IDs)
+ require.Equal(t, fresh[20:25], steps.refineInputs[2].IDs)
+
+ // The tail threads batch to batch within the chapter: each batch's preceding
+ // context is the previous batch's returned tail; the first opens with none.
+ require.Equal(t, "", steps.refineInputs[0].PrevTail)
+ require.Equal(t, "queue-"+fresh[9], steps.refineInputs[1].PrevTail)
+ require.Equal(t, "queue-"+fresh[19], steps.refineInputs[2].PrevTail)
+ // PrevChapterID is only ever set on a chapter's first batch; there is no
+ // previous chapter here.
+ require.Equal(t, "", steps.refineInputs[0].PrevChapterID)
+ require.Equal(t, "", steps.refineInputs[1].PrevChapterID)
+ require.Equal(t, "", steps.refineInputs[2].PrevChapterID)
+}
+
+func TestRunBook_LiteraryDegradedBatchDoesNotStopTheChapter(t *testing.T) {
+ t.Parallel()
+
+ fresh := freshIDs(25)
+ steps := literaryBatched(fresh, 10)
+ calls := 0
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ calls++
+ res := pipeline.RefineResult{Usage: literaryUsage(6, 3), Refined: len(in.IDs)}
+ if calls == 1 {
+ // The first batch kept two ids on their draft — a degraded result,
+ // not an error. The chapter must carry on.
+ res.Refined = len(in.IDs) - 2
+ res.Degraded = in.IDs[:2]
+ }
+ return res, nil
+ }
+
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeLiterary))
+ require.NoError(t, err, "a degraded batch keeps its draft but does not stop the chapter")
+
+ require.Len(t, steps.refineInputs, 3, "the batches after the degraded one still run")
+ require.Equal(t, 1, steps.commitCalls, "the chapter still reaches commit")
+ // Only the refined segments count; the degraded pair kept its draft.
+ require.Equal(t, 23, result.LLMSegments)
+}
+
+// TestRunBook_LiteraryChapterRefiningNothingFailsLoudly is the chapter-level
+// backstop the per-batch split would otherwise drop: when every batch degrades
+// and not one segment is refined, the chapter is a wall of NMT drafts literary
+// mode never persists. QA cannot see it — a draft has a non-empty target — so
+// the run must fail here rather than report a finished volume with nothing to
+// resume from, exactly as the old chapter-level pass returned ErrEmptyResponse.
+func TestRunBook_LiteraryChapterRefiningNothingFailsLoudly(t *testing.T) {
+ t.Parallel()
+
+ fresh := freshIDs(20)
+ steps := literaryBatched(fresh, 10)
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ // Every batch degrades: the model refused, so each segment keeps its
+ // draft and nothing is refined. Not an error per batch — a degraded
+ // result.
+ return pipeline.RefineResult{Usage: literaryUsage(6, 3), Refined: 0, Degraded: in.IDs}, nil
+ }
+
+ _, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeLiterary))
+
+ var stageErr *pipeline.StageError
+ require.ErrorAs(t, err, &stageErr)
+ require.Equal(t, pipeline.StageTranslate, stageErr.Stage)
+ require.ErrorIs(t, err, pipeline.ErrChapterUnrefined)
+
+ require.Zero(t, steps.commitCalls, "a chapter that refined nothing must not commit drafts")
+ // A failure past the guardrail still records what the degraded calls spent.
+ require.Equal(t, 1, steps.spendCalls)
+}
+
+// TestRunBook_LiteraryProgressAdvancesOnRefinement pins the bar cadence: in
+// literary mode "done" tracks the refine pass (segments a completed batch
+// carried), not the draft. Drafting a whole chapter must not fill the bar and
+// leave it at 100% for the slow LLM pass — the regression the watcher of a
+// multi-hour volume would feel most.
+func TestRunBook_LiteraryProgressAdvancesOnRefinement(t *testing.T) {
+ t.Parallel()
+
+ fresh := freshIDs(20)
+ steps := literaryBatched(fresh, 10)
+ var atFirstRefine pipeline.Progress
+ seen := false
+ rep := &recordingReporter{}
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ // Snapshot what a watcher saw the instant the first refine batch was
+ // dispatched: the whole chapter is drafted by now, but no batch has been
+ // counted yet, so the bar must still read zero done.
+ if !seen {
+ seen = true
+ atFirstRefine = rep.last()
+ }
+ return pipeline.RefineResult{Usage: literaryUsage(6, 3), Refined: len(in.IDs)}, nil
+ }
+
+ _, err := pipeline.RunBook(steps, newRuntime(rep), nominalInput(pipeline.ModeLiterary))
+ require.NoError(t, err)
+
+ require.Equal(t, 20, atFirstRefine.Segments)
+ require.Zero(t, atFirstRefine.SegmentsDone,
+ "drafting must not advance the bar in literary mode; only the refine pass does")
+ require.Equal(t, 20, rep.last().SegmentsDone, "the bar reaches the total once every batch is refined")
+}
+
+func TestRunBook_LiteraryRateLimitFailsRunButStillRecordsSpend(t *testing.T) {
+ t.Parallel()
+
+ fresh := freshIDs(20)
+ steps := literaryBatched(fresh, 10)
+ calls := 0
+ steps.refine = func(in pipeline.RefineInput) (pipeline.RefineResult, error) {
+ calls++
+ if calls == 2 {
+ // The quota is exhausted mid-chapter. RefineBatch reports the tokens
+ // the refused call consumed alongside a translate-stage error.
+ return pipeline.RefineResult{Usage: literaryUsage(5, 1)},
+ &pipeline.StageError{Stage: pipeline.StageTranslate, Err: wrapErr(llm.ErrRateLimited)}
+ }
+ return pipeline.RefineResult{Usage: literaryUsage(6, 3), Refined: len(in.IDs)}, nil
+ }
+
+ _, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeLiterary))
+
+ var stageErr *pipeline.StageError
+ require.ErrorAs(t, err, &stageErr)
+ require.Equal(t, pipeline.StageTranslate, stageErr.Stage)
+ require.ErrorIs(t, err, llm.ErrRateLimited)
+
+ // The third batch never ran: the rate limit stopped the chapter.
+ require.Len(t, steps.refineInputs, 2)
+
+ // A failure past the guardrail still records spend, with the usage the run
+ // accumulated before it died — the first batch's tokens. The refused call's
+ // usage is not folded in: the flow drops the failing batch's result.
+ require.Equal(t, 1, steps.spendCalls)
+ require.Equal(t, 6, steps.spendInput.Result.LLMInputTokens)
+ require.Equal(t, 3, steps.spendInput.Result.LLMOutputTokens)
+}
+
+func TestRunBook_ResumeIsAllReuse(t *testing.T) {
+ t.Parallel()
+
+ // A resumed book: the TM serves every segment, so no chapter is drafted.
+ steps := script(pipeline.ModeNMT,
+ chapterScript{id: "ch_0001", reused: 3, fresh: nil},
+ chapterScript{id: "ch_0002", reused: 2, fresh: nil},
+ )
+
+ result, err := pipeline.RunBook(steps, newRuntime(&recordingReporter{}), nominalInput(pipeline.ModeNMT))
+ require.NoError(t, err)
+
+ require.Equal(t, 5, result.TMHits)
+ require.Zero(t, result.NMTSegments)
+ require.Zero(t, result.NMTChars)
+ require.NotContains(t, steps.calls, "draft:ch_0001")
+ require.NotContains(t, steps.calls, "draft:ch_0002")
+ require.InDelta(t, 1.0, result.TMHitRatio, 1e-9)
+}
+
+func wrapErr(err error) error { return fmt.Errorf("wrapped, err: %w", err) }
+
+func literaryUsage(in, out int) literary.Usage {
+ return literary.Usage{InputTokens: in, OutputTokens: out}
+}
+
+// freshIDs builds n numbered segment ids, the fresh split a literary chapter is
+// chunked into by batch size.
+func freshIDs(n int) []string {
+ ids := make([]string, n)
+ for i := range ids {
+ ids[i] = fmt.Sprintf("s%03d", i+1)
+ }
+ return ids
+}
+
+// literaryBatched scripts a single literary chapter whose fresh ids the flow
+// chunks by batchSize, overriding the manifest's default literary batch size.
+func literaryBatched(fresh []string, batchSize int) *fakeSteps {
+ steps := script(pipeline.ModeLiterary, chapterScript{id: "ch_0001", fresh: fresh, chars: 100})
+ steps.prepare = func(pipeline.BookInput) (pipeline.Manifest, error) {
+ return pipeline.Manifest{
+ Chapters: []pipeline.ChapterInfo{{ID: "ch_0001", Segments: len(fresh)}},
+ NMTEngine: "nmt", BatchSize: batchSize,
+ }, nil
+ }
+ return steps
+}
diff --git a/internal/pipeline/doc.go b/internal/pipeline/doc.go
index 5a3c036..48da068 100644
--- a/internal/pipeline/doc.go
+++ b/internal/pipeline/doc.go
@@ -1,6 +1,14 @@
-// Package pipeline orchestrates the EPUB translation workflow across extraction,
-// parsing, segmentation, translation, and recomposition stages.
+// Package pipeline orchestrates the EPUB translation workflow behind two small
+// ports. The work — extract, parse, segment, project, translate, recompose,
+// QA, artifacts — lives in Stages methods (the activity bodies); the control
+// flow — accounting, the budget guardrail, cleanup — lives in RunBook and
+// RunSeries, written once against the Steps and Runtime ports. The in-process
+// implementations of those ports run the CLI; a Temporal implementation will
+// run the same flow unchanged.
//
-// It defines pipeline coordination types and stage-level error handling while
-// keeping individual processing responsibilities in their dedicated packages.
+// Content never crosses a port: a chapter's segments are exchanged as units in
+// the workspace (units.jsonl per chapter), and re-derived deterministically —
+// ids by position, hash by normalised text — whenever a stage needs the live
+// node tree. Individual processing responsibilities stay in their dedicated
+// packages (epub, segment, tm, translate, literary, qa, cost, narrative).
package pipeline
diff --git a/internal/pipeline/errors.go b/internal/pipeline/errors.go
index c621042..a2699b4 100644
--- a/internal/pipeline/errors.go
+++ b/internal/pipeline/errors.go
@@ -20,6 +20,11 @@ const (
// ErrQAGateFailed is returned (wrapped) when QA thresholds or epubcheck fail.
var ErrQAGateFailed = errors.New("pipeline: QA gate failed")
+// ErrChapterUnrefined is returned (wrapped) when a literary chapter's every
+// batch degraded and no segment was refined: the chapter would ship NMT drafts
+// literary mode does not persist, so the run fails instead of reading as done.
+var ErrChapterUnrefined = errors.New("pipeline: literary chapter refined no segment")
+
// StageError attributes a failure to a specific pipeline stage so callers can
// map it to the right process exit code.
type StageError struct {
diff --git a/internal/pipeline/inprocess.go b/internal/pipeline/inprocess.go
new file mode 100644
index 0000000..77b2968
--- /dev/null
+++ b/internal/pipeline/inprocess.go
@@ -0,0 +1,147 @@
+package pipeline
+
+import (
+ "context"
+ "time"
+
+ "github.com/ECecillo/tomeshift/internal/qa"
+)
+
+// InProcessSteps runs each activity as a direct Stages call under one context.
+// It is the flow's test double and its portability proof: the same RunBook that
+// a Temporal worker drives runs here against a plain function call, which is
+// what keeps the flow engine-agnostic.
+type InProcessSteps struct {
+ Stages *Stages
+ Ctx context.Context
+}
+
+func (a InProcessSteps) ctx() context.Context {
+ if a.Ctx != nil {
+ return a.Ctx
+ }
+ return context.Background()
+}
+
+func (a InProcessSteps) PrepareBook(in BookInput) (Manifest, error) {
+ return a.Stages.PrepareBook(a.ctx(), in)
+}
+func (a InProcessSteps) ProjectCost(in ProjectInput) (Projection, error) {
+ return a.Stages.ProjectCost(a.ctx(), in)
+}
+func (a InProcessSteps) ResolveChapter(in ResolveInput) (ChapterPlan, error) {
+ return a.Stages.ResolveChapter(a.ctx(), in)
+}
+func (a InProcessSteps) DraftChapter(in DraftInput) (DraftResult, error) {
+ return a.Stages.DraftChapter(a.ctx(), in)
+}
+func (a InProcessSteps) EscalateChapter(in EscalateInput) (EscalateResult, error) {
+ return a.Stages.EscalateChapter(a.ctx(), in)
+}
+func (a InProcessSteps) RefineBatch(in RefineInput) (RefineResult, error) {
+ return a.Stages.RefineBatch(a.ctx(), in)
+}
+func (a InProcessSteps) CommitChapter(in CommitInput) (CommitResult, error) {
+ return a.Stages.CommitChapter(a.ctx(), in)
+}
+func (a InProcessSteps) UpdateBible(in BibleInput) (BibleResult, error) {
+ return a.Stages.UpdateBible(a.ctx(), in)
+}
+func (a InProcessSteps) RecomposeBook(ref RunRef) (RecomposeResult, error) {
+ return a.Stages.RecomposeBook(a.ctx(), ref)
+}
+func (a InProcessSteps) CheckEPUB(in CheckInput) (CheckResult, error) {
+ return a.Stages.CheckEPUB(a.ctx(), in)
+}
+func (a InProcessSteps) FinishBook(in FinishInput) (qa.Report, error) {
+ return a.Stages.FinishBook(a.ctx(), in)
+}
+func (a InProcessSteps) RecordSpend(in SpendInput) error {
+ return a.Stages.RecordSpend(a.ctx(), in)
+}
+func (a InProcessSteps) PriceRun(in PriceInput) (float64, error) {
+ return a.Stages.PriceRun(a.ctx(), in)
+}
+
+var _ Steps = InProcessSteps{}
+
+// InProcessRuntime backs the flow's Runtime with the standard library: a
+// context for cancellation, a clock, a resume channel and a Reporter that drives
+// the terminal bar. It is what the CLI runs `run` and `batch` on until the
+// Temporal engine lands.
+type InProcessRuntime struct {
+ // Ctx is the run's context; Cancelled reads it. A detached runtime ignores it.
+ Ctx context.Context
+ // Progress receives the flow's snapshots; nil disables reporting.
+ Progress Reporter
+ // Steps runs child books for RunSeries.
+ Steps Steps
+ // Clock overrides Now for deterministic tests; nil uses time.Now.
+ Clock func() time.Time
+ // ResumeCh ends a WaitResume early; nil means only the timer fires.
+ ResumeCh <-chan struct{}
+ // PausePending is what Paused reports.
+ PausePending bool
+
+ detached bool
+}
+
+func (r *InProcessRuntime) Now() time.Time {
+ if r.Clock != nil {
+ return r.Clock()
+ }
+ return time.Now()
+}
+
+// WaitResume blocks for d or until the resume channel fires, whichever comes
+// first. The phase-1 flow never calls it; it exists and is tested so phase 4 is
+// a flow change only.
+func (r *InProcessRuntime) WaitResume(d time.Duration) (bool, error) {
+ timer := time.NewTimer(d)
+ defer timer.Stop()
+
+ var done <-chan struct{}
+ if r.Ctx != nil {
+ done = r.Ctx.Done()
+ }
+
+ select {
+ case <-r.ResumeCh: // a nil channel blocks forever, leaving the timer/ctx
+ return true, nil
+ case <-timer.C:
+ return false, nil
+ case <-done:
+ return false, r.Ctx.Err()
+ }
+}
+
+func (r *InProcessRuntime) Paused() bool { return r.PausePending }
+
+func (r *InProcessRuntime) Cancelled() bool {
+ if r.detached {
+ return false
+ }
+ return r.Ctx != nil && r.Ctx.Err() != nil
+}
+
+func (r *InProcessRuntime) Detached() Runtime {
+ c := *r
+ c.detached = true
+ return &c
+}
+
+func (r *InProcessRuntime) Report(p Progress) {
+ if r.Progress != nil {
+ r.Progress.Report(p)
+ }
+}
+
+func (r *InProcessRuntime) RunBook(in BookInput) (RunResult, error) {
+ res, err := RunBook(r.Steps, r, in)
+ if res == nil {
+ return RunResult{}, err
+ }
+ return *res, err
+}
+
+var _ Runtime = (*InProcessRuntime)(nil)
diff --git a/internal/pipeline/mode.go b/internal/pipeline/mode.go
deleted file mode 100644
index 1e66d21..0000000
--- a/internal/pipeline/mode.go
+++ /dev/null
@@ -1,416 +0,0 @@
-package pipeline
-
-import (
- "context"
- "fmt"
-
- "go.uber.org/zap"
-
- "github.com/ECecillo/tomeshift/internal/cost"
- "github.com/ECecillo/tomeshift/internal/literary"
- "github.com/ECecillo/tomeshift/internal/narrative"
- "github.com/ECecillo/tomeshift/internal/segment"
- "github.com/ECecillo/tomeshift/internal/tm"
- "github.com/ECecillo/tomeshift/internal/translate"
-)
-
-// costFn prices what the run has consumed at the instant it is called. It is
-// built by Run, where the meter baseline lives, and handed to the translation
-// loops so a watcher can be told what the book has cost so far. It is pure and
-// cheap: arithmetic over the counters plus one read of the adapter's meter.
-type costFn func() float64
-
-// translationMode is the only part of a run that differs between the NMT and
-// literary pipelines. Everything around it — extract, parse, segment, the
-// budget guardrail, recompose, QA, the spend ledger, artifacts and the QA gate
-// — is identical for both and lives in Orchestrator.Run.
-type translationMode interface {
- // Init stamps the mode's own fields onto a fresh result.
- Init(result *RunResult)
- // Project estimates the worst-case spend of the chapters handed to it,
- // assuming none of their segments turns out to be reusable. The caller has
- // already dropped what the TM can serve, so "worst case" is about this run's
- // remaining work rather than about the whole book. It runs before any
- // provider call.
- Project(chapters []chapterSegments, usedFreeChars int) cost.Projection
- // Translate fills in the chapters' target text, restores placeholders into
- // the node tree and accumulates the consumption counters on result as it
- // goes. It returns the translated segments in spine order.
- Translate(
- ctx context.Context, log *zap.Logger,
- chapters []chapterSegments, usedFreeChars int, result *RunResult, price costFn,
- ) ([]segment.Segment, error)
- // Cost prices what result says has been consumed so far. It is pure and
- // callable at any point, including after a failure, because a run that died
- // halfway still has to answer for what it already spent.
- Cost(result *RunResult, usedFreeChars int) float64
- // ReusesDrafts reports whether a raw NMT draft already in the TM is a
- // usable hit for this mode. It is the one thing the two modes disagree on,
- // and the projection has to know: a book whose drafts are all cached is free
- // to finish in NMT mode and a full bill in literary mode.
- ReusesDrafts() bool
-}
-
-// newTranslationMode selects the pipeline for cfg. Literary mode loads the
-// series bible here because both the projection (whose prefix it sizes) and the
-// translation need it.
-func (o *Orchestrator) newTranslationMode(ctx context.Context, cfg RunConfig) (translationMode, error) {
- if runMode(cfg) != ModeLiterary {
- return &nmtMode{orch: o, cfg: cfg}, nil
- }
-
- bible, err := o.loadBible(ctx, cfg.SeriesID)
- if err != nil {
- return nil, err
- }
- return &literaryMode{orch: o, cfg: cfg, bible: bible}, nil
-}
-
-// nmtMode drafts every segment with the NMT provider and optionally escalates
-// the weakest ones to an LLM post-editor. The draft is the final output for a
-// segment that is not escalated, which is why this mode does persist it.
-type nmtMode struct {
- orch *Orchestrator
- cfg RunConfig
-}
-
-func (m *nmtMode) Init(result *RunResult) { result.Mode = ModeNMT }
-
-// ReusesDrafts is true here: an unescalated draft IS this mode's output, which
-// is why it is also persisted.
-func (m *nmtMode) ReusesDrafts() bool { return true }
-
-// postEditPass describes an escalation pass over segments totalling chars. The
-// HTTP post-editor sends one segment per call and carries neither the bible nor
-// the chapter source, so the stable prompt overhead is paid once per segment —
-// the opposite of the batched literary pass, and the reason escalation is
-// expensive per word. cost.EstimateScenarios builds the hybrid scenario the
-// same way, which is what makes `estimate` and this run agree.
-func postEditPass(segments, chars int) cost.LLMPass {
- return cost.LLMPass{Segments: segments, Chars: chars, Calls: segments, Uncached: true}
-}
-
-func (m *nmtMode) Project(chapters []chapterSegments, usedFreeChars int) cost.Projection {
- segments := flattenChapters(chapters)
-
- // Worst case with a post-editor attached, every segment clears the
- // escalation threshold; without one there is no LLM stage at all.
- var pass cost.LLMPass
- if m.orch.PostEditor != nil {
- pass = postEditPass(len(segments), segment.SourceChars(segments))
- }
- return m.orch.CostModel.Project(segments, pass, usedFreeChars)
-}
-
-// Cost prices the draft plus the escalation. The HTTP post-editor reports no
-// token usage, so the escalation is priced from its shape through the same
-// model that projected it: counting the escalated segments but not their cost
-// would make the guardrail lie by exactly the amount the LLM costs.
-func (m *nmtMode) Cost(result *RunResult, usedFreeChars int) float64 {
- _, _, llmUSD := m.orch.CostModel.Tokens.Project(postEditPass(result.LLMAttempts, result.LLMChars))
- return m.orch.CostModel.Actual(
- result.NMTSegments, result.NMTChars, usedFreeChars, cost.TokenUsage{}) + llmUSD
-}
-
-func (m *nmtMode) Translate(
- ctx context.Context, log *zap.Logger,
- chapters []chapterSegments, usedFreeChars int, result *RunResult, price costFn,
-) ([]segment.Segment, error) {
- // This walks the flattened copies, not chapters, so chapters keeps its
- // source text. That is harmless — and only harmless — because Segment.Node
- // is a pointer into the live tree, so the placeholder restore lands in the
- // document either way, and Run recomposes from the returned slice. Anything
- // added after Translate must read the return value, not chapters.
- segments := flattenChapters(chapters)
- if err := m.orch.translateSegments(ctx, m.cfg, log, segments, result, price); err != nil {
- return nil, err
- }
- return segments, nil
-}
-
-// literaryMode drafts a chapter's TM misses in one batched provider call, then
-// rewrites every fresh segment with the LLM against the series bible. The draft
-// is an intermediate here, never the output.
-type literaryMode struct {
- orch *Orchestrator
- cfg RunConfig
- bible narrative.Bible
-}
-
-// ReusesDrafts is false here: serving a draft would skip refinement for that
-// segment permanently, which is the defect this mode exists to prevent.
-func (m *literaryMode) ReusesDrafts() bool { return false }
-
-func (m *literaryMode) Init(result *RunResult) {
- result.Mode = ModeLiterary
- result.SeriesID = m.cfg.SeriesID
- result.BibleVersion = m.bible.Version
-}
-
-func (m *literaryMode) Project(chapters []chapterSegments, usedFreeChars int) cost.Projection {
- segments := flattenChapters(chapters)
- return m.orch.CostModel.Project(segments, m.pass(chapters, segments), usedFreeChars)
-}
-
-// pass describes the LLM pass the run is about to make so the guardrail prices
-// the calls it will actually issue. Chunking is per chapter, and every call
-// repeats the stable prefix, so the shape of the chapters — not just the
-// segment count — decides the bill.
-func (m *literaryMode) pass(chapters []chapterSegments, segments []segment.Segment) cost.LLMPass {
- batchSize, prefixChars := 0, 0
- if m.orch.LiteraryTranslator != nil {
- batchSize = m.orch.LiteraryTranslator.EffectiveBatchSize()
- prefixChars = m.orch.LiteraryTranslator.StablePrefixChars(m.bible)
- }
-
- // Per chapter, not averaged: each one writes its own source into the
- // cacheable prefix and reads it back on every further call, and the long
- // chapters have more calls to read it in.
- counts := make([]int, len(chapters))
- shapes := make([]cost.ChapterShape, len(chapters))
- for i := range chapters {
- counts[i] = len(chapters[i].Segments)
- shapes[i] = cost.ChapterShape{
- SourceChars: segment.SourceChars(chapters[i].Segments),
- Calls: cost.CallsForChapters(counts[i:i+1], batchSize),
- }
- }
-
- return cost.LLMPass{
- Segments: len(segments),
- Chars: segment.SourceChars(segments),
- Calls: cost.CallsForChapters(counts, batchSize),
- PrefixChars: prefixChars,
- Chapters: shapes,
- ContextRunes: cost.LiteraryContextRunes,
- }
-}
-
-// Cost prices the draft plus the LLM pass from the tokens the provider actually
-// reported.
-func (m *literaryMode) Cost(result *RunResult, usedFreeChars int) float64 {
- return m.orch.CostModel.Actual(result.NMTSegments, result.NMTChars, usedFreeChars, cost.TokenUsage{
- InputTokens: result.LLMInputTokens,
- OutputTokens: result.LLMOutputTokens,
- CacheReadTokens: result.LLMCacheReadTokens,
- CacheCreationTokens: result.LLMCacheCreationTokens,
- })
-}
-
-func (m *literaryMode) Translate(
- ctx context.Context, log *zap.Logger,
- chapters []chapterSegments, usedFreeChars int, result *RunResult, price costFn,
-) ([]segment.Segment, error) {
- o := m.orch
- if o.LiteraryTranslator == nil {
- return nil, literary.ErrNilChatClient
- }
-
- // A chapter is one TranslateChapter call, so without this hook both the
- // progress and the spend would freeze for the whole of it. Wired here rather
- // than by the CLI because this is now the run's ONLY accumulation path for
- // LLM tokens: refine must not add Outcome.Usage on top, or every chapter
- // would be billed twice.
- //
- // A batch is reported whether it succeeded or degraded, so the counters hold
- // what was consumed and not what was usable — which is what the ledger and
- // the budget guardrail need.
- o.LiteraryTranslator.OnBatch = func(segments int, usage literary.Usage) {
- result.LLMPassSegments += segments
- result.LLMInputTokens += usage.InputTokens
- result.LLMOutputTokens += usage.OutputTokens
- result.LLMCacheReadTokens += usage.CacheReadTokens
- result.LLMCacheCreationTokens += usage.CacheCreationTokens
- o.progress().Advance(segments, 0)
- o.progress().Cost(price())
- }
-
- translated := make([]segment.Segment, 0, result.TotalSegments)
- for i := range chapters {
- if err := ctx.Err(); err != nil {
- return nil, err
- }
-
- o.progress().Chapter(i + 1)
- chapter := &chapters[i]
-
- // Resolve the chapter against the TM first, then draft everything it
- // missed in one batched provider call. A TM hit here is always a
- // finished translation: literary mode never serves or persists a draft.
- var fresh []*segment.Segment
- for j := range chapter.Segments {
- seg := &chapter.Segments[j]
- reused, err := o.lookupTranslation(ctx, seg)
- if err != nil {
- return nil, err
- }
- if reused {
- seg.Status = segment.StatusReused
- result.TMHits++
- o.progress().Advance(0, 1)
- continue
- }
- fresh = append(fresh, seg)
- }
-
- if err := o.draftSegments(ctx, m.cfg, chapter.ID, fresh); err != nil {
- return nil, err
- }
- for _, seg := range fresh {
- seg.Status = segment.StatusTranslated
- result.NMTSegments++
- }
-
- if err := m.refine(ctx, log, chapters, i, fresh, result); err != nil {
- return nil, err
- }
-
- for j := range chapter.Segments {
- seg := &chapter.Segments[j]
- if o.Glossary != nil {
- result.GlossaryViolations += o.Glossary.Violations(seg.SourceText, seg.TargetText)
- }
- // Restore is atomic: on failure the node keeps its original source
- // markup, so the segment degrades to untranslated instead of losing
- // content. QA counts it; the chapter carries on — and the
- // translation is not persisted, so it cannot come back as a hit.
- if err := o.placeholders().Restore(seg.TargetText, seg.Placeholders, seg.Node); err != nil {
- log.Warn("placeholder restore failed",
- zap.String("segment_id", seg.ID), zap.Error(err))
- continue
- }
- m.persist(ctx, log, seg)
- }
-
- // Frozen by default: keep the bible (and thus the system prefix / prompt
- // cache) stable across chapters and re-runs. Evolve it only when the run
- // explicitly opts in.
- if m.cfg.UpdateBible {
- if err := o.updateBible(ctx, m.cfg.SeriesID, &m.bible, chapter); err != nil {
- return nil, err
- }
- result.BibleVersion = m.bible.Version
- }
- translated = append(translated, chapter.Segments...)
- }
-
- return translated, nil
-}
-
-// refine rewrites a chapter's freshly-drafted segments with the LLM and
-// persists the results. Only a finished translation reaches the TM.
-func (m *literaryMode) refine(
- ctx context.Context, log *zap.Logger,
- chapters []chapterSegments, idx int, fresh []*segment.Segment, result *RunResult,
-) error {
- if len(fresh) == 0 {
- return nil
- }
- chapter := &chapters[idx]
-
- // TranslateChapter rewrites the segments in place. The chapter's whole
- // source goes in the cacheable prefix; only the tail of the previous
- // chapter still has to travel per call.
- //
- // Its Outcome.Usage is deliberately ignored: the OnBatch hook wired in
- // Translate already accumulated the same tokens, batch by batch, and it did
- // so before this function was reached — so a chapter where every batch
- // failed has still been counted, which is what the deferred spend recorder
- // needs. Adding it here as well would double every chapter's bill.
- outcome, err := m.orch.LiteraryTranslator.TranslateChapter(
- ctx, m.bible, chapter.ID, fresh,
- adjacentChapterText(chapters, idx-1), chapterSourceText(chapter.Segments))
- if err != nil {
- return err
- }
-
- // A batch that failed while others succeeded leaves its segments on their
- // draft and does not fail the chapter. Without this the named refusal and
- // truncation errors would be collected and then dropped, which is the same
- // silence they were introduced to end.
- for _, degraded := range outcome.Degraded {
- log.Warn("literary batch kept its draft",
- zap.String("chapter_id", chapter.ID), zap.Error(degraded))
- }
-
- for _, seg := range fresh {
- if seg.Status == segment.StatusPostEdited {
- result.LLMSegments++
- }
- }
- return nil
-}
-
-// persist writes a refined segment to the TM. It is called only after Restore
-// has accepted the translation.
-//
-// The ordering is the point. The literary precheck compares the *set* of
-// placeholder ids, so it cannot see a reordering; Restore can. Saving before
-// restoring would write a translation that rebuilds into wrong markup as a
-// finished "literary" entry, and the next run would serve it as a TM hit and
-// skip refinement — the shadowing defect this mode exists to prevent, arriving
-// through the back door.
-func (m *literaryMode) persist(ctx context.Context, log *zap.Logger, seg *segment.Segment) {
- if m.orch.TM == nil || seg.Status != segment.StatusPostEdited {
- return
- }
- if err := m.orch.TM.Save(ctx, tm.Entry{
- SourceHash: seg.Hash,
- SourceText: seg.SourceText,
- TargetText: seg.TargetText,
- SourceLang: m.cfg.SourceLang,
- TargetLang: m.cfg.TargetLang,
- Engine: tm.EngineLiterary,
- }); err != nil {
- log.Warn("unable to persist literary TM entry",
- zap.String("segment_id", seg.ID), zap.Error(err))
- }
-}
-
-// draftSegments drafts segments in a single batched provider call, writing each
-// draft onto its segment.
-//
-// Locked glossary terms are protected per segment before the call and
-// substituted back after it, exactly as the per-segment path does — batching
-// changes the transport, not the glossary contract.
-//
-// The drafts are deliberately not persisted. In literary mode the draft is an
-// intermediate the LLM is about to rewrite; saving it would make the next run
-// serve it as a TM hit, which short-circuits refinement for that segment
-// forever.
-func (o *Orchestrator) draftSegments(ctx context.Context, cfg RunConfig, chapterID string, segments []*segment.Segment) error {
- if len(segments) == 0 {
- return nil
- }
-
- texts := make([]string, len(segments))
- subs := make([]map[string]string, len(segments))
- for i, seg := range segments {
- texts[i] = seg.SourceText
- if o.Glossary != nil {
- texts[i], subs[i] = o.Glossary.Protect(seg.SourceText)
- }
- }
-
- resps, err := o.Translator.TranslateBatch(ctx, translate.BatchRequest{
- Texts: texts,
- SourceLang: cfg.SourceLang,
- TargetLang: cfg.TargetLang,
- })
- if err != nil {
- return fmt.Errorf("unable to draft chapter %s, err: %w", chapterID, err)
- }
- if len(resps) != len(texts) {
- return fmt.Errorf("unable to draft chapter %s: got %d drafts for %d segments, err: %w",
- chapterID, len(resps), len(texts), translate.ErrShortBatchResponse)
- }
-
- for i, resp := range resps {
- text := resp.Text
- if o.Glossary != nil {
- text = o.Glossary.Apply(text, subs[i])
- }
- segments[i].TargetText = text
- }
- return nil
-}
diff --git a/internal/pipeline/modes.go b/internal/pipeline/modes.go
new file mode 100644
index 0000000..99575ac
--- /dev/null
+++ b/internal/pipeline/modes.go
@@ -0,0 +1,85 @@
+package pipeline
+
+// Mode is a translation mode as a recipe over the Steps: what to do per chapter,
+// and whether an NMT draft already in the TM is a usable hit for it. Cost
+// projection and pricing are not here — they are pure arithmetic over the cost
+// model, so they live in the Stages (ProjectCost, PriceRun), which is the side
+// that holds the model. Adding a mode is a new recipe here plus, if it needs a
+// new step, one Steps method — never a new flow.
+type Mode interface {
+ // ReusesDrafts reports whether a raw NMT draft already in the TM is a usable
+ // hit for this mode. The two modes disagree on exactly this.
+ ReusesDrafts() bool
+ // Chapter is the per-chapter recipe, composed of Steps calls through the flow.
+ Chapter(f *flow, ch ChapterInfo) error
+}
+
+// selectMode picks the mode for in.
+func selectMode(in BookInput) (Mode, error) {
+ if runMode(in) == ModeLiterary {
+ return literaryMode{}, nil
+ }
+ return nmtMode{}, nil
+}
+
+// nmtMode drafts every fresh segment with the NMT provider and optionally
+// escalates the weakest to an LLM post-editor. An unescalated draft is the final
+// output for its segment, which is why the mode reuses and persists drafts.
+type nmtMode struct{}
+
+func (nmtMode) ReusesDrafts() bool { return true }
+
+// Chapter: Resolve → Draft → Escalate → Commit. Drafts are persisted (Draft
+// with persist=true); nothing extra is persisted at commit.
+func (nmtMode) Chapter(f *flow, ch ChapterInfo) error {
+ plan, err := f.resolve(ch, true)
+ if err != nil {
+ return err
+ }
+ if len(plan.Fresh) > 0 {
+ if err := f.draft(ch, plan.Fresh, true); err != nil {
+ return err
+ }
+ if err := f.escalate(ch, plan.Fresh); err != nil {
+ return err
+ }
+ f.priceReport(stageTranslate)
+ }
+ return f.commit(ch, false)
+}
+
+// literaryMode drafts a chapter's TM misses, rewrites every fresh segment with
+// the LLM against the series bible, then persists only the finished
+// translations. A raw draft is never served or persisted: serving one would
+// skip refinement for that segment permanently.
+type literaryMode struct{}
+
+func (literaryMode) ReusesDrafts() bool { return false }
+
+// Chapter: Resolve → Draft → RefineBatch × n → Commit (→ UpdateBible). Drafts
+// are not persisted (Draft with persist=false); the commit persists the refined
+// translations only after Restore accepts them.
+func (literaryMode) Chapter(f *flow, ch ChapterInfo) error {
+ plan, err := f.resolve(ch, false)
+ if err != nil {
+ return err
+ }
+ if len(plan.Fresh) > 0 {
+ if err := f.draft(ch, plan.Fresh, false); err != nil {
+ return err
+ }
+ if err := f.refine(ch, plan.Fresh); err != nil {
+ return err
+ }
+ }
+ if err := f.commit(ch, true); err != nil {
+ return err
+ }
+ if f.in.UpdateBible {
+ if err := f.updateBible(ch); err != nil {
+ return err
+ }
+ }
+ f.prevChapterID = ch.ID
+ return nil
+}
diff --git a/internal/pipeline/orchestrator.go b/internal/pipeline/orchestrator.go
deleted file mode 100644
index bc2efa2..0000000
--- a/internal/pipeline/orchestrator.go
+++ /dev/null
@@ -1,925 +0,0 @@
-package pipeline
-
-import (
- "context"
- "fmt"
- "strings"
- "time"
- "unicode/utf8"
-
- "go.uber.org/zap"
-
- "github.com/ECecillo/tomeshift/internal/cost"
- "github.com/ECecillo/tomeshift/internal/epub"
- "github.com/ECecillo/tomeshift/internal/glossary"
- "github.com/ECecillo/tomeshift/internal/literary"
- "github.com/ECecillo/tomeshift/internal/llm"
- "github.com/ECecillo/tomeshift/internal/narrative"
- "github.com/ECecillo/tomeshift/internal/qa"
- "github.com/ECecillo/tomeshift/internal/segment"
- "github.com/ECecillo/tomeshift/internal/tm"
- "github.com/ECecillo/tomeshift/internal/translate"
-)
-
-// ledgerWriteTimeout bounds the detached write used when a run is recorded
-// after its context was cancelled.
-const ledgerWriteTimeout = 5 * time.Second
-
-type chapterSegments struct {
- ID string
- Href string
- Segments []segment.Segment
-}
-
-// QualityEstimator scores a translated segment without a reference (e.g.
-// COMET-QE). Scores are higher-is-better; the pipeline uses them to drive
-// selective LLM escalation and to report per-book quality.
-type QualityEstimator interface {
- Estimate(ctx context.Context, source, candidate string) (float64, error)
-}
-
-// Orchestrator coordinates the V1 translation pipeline across its stages. All
-// collaborators are injected so each stage stays independently testable.
-type Orchestrator struct {
- Extractor epub.Extractor
- Parser epub.Parser
- Segmenter segment.Segmenter
- Placeholders segment.PlaceholderManager
- TM tm.Store
- Translator translate.Translator
- Recomposer epub.Recomposer
- EPUBCheck qa.EPUBChecker
- Glossary *glossary.Glossary // optional; nil disables locked-term enforcement
- PostEditor llm.PostEditor // optional; nil disables selective LLM post-edit
- LiteraryTranslator *literary.Translator
- NarrativeStore narrative.Store
- NarrativeProfiler *narrative.Profiler
- EscalateMinChars int // escalate NMT segments with >= this many source chars (0 = all, when PostEditor set)
- QualityEstimator QualityEstimator // optional; reference-free QE for escalation + reporting
- EscalateBelowScore float64 // escalate fresh segments whose QE score < this (0 disables)
- CostModel cost.Model // zero-value = free local provider
- BudgetUSD float64 // per-book hard cap; <= 0 disables the check
- DailyUSD float64 // daily spend cap; <= 0 disables the check
- MonthlyUSD float64 // monthly spend cap; <= 0 disables the check
- Logger *zap.Logger
- Progress Reporter // optional; nil disables terminal progress reporting
- // LLMProvider and LLMModel name what the LLM pass talks to. The pipeline
- // does no routing with them — the clients are already built — it records
- // them, because a measured token count is only reusable if you know what
- // produced it.
- LLMProvider string
- LLMModel string
-}
-
-// Run executes the full pipeline and always attempts to emit run artifacts. It
-// returns a *StageError on stage failure and a wrapped ErrQAGateFailed when the
-// output is produced but fails QA.
-//
-// The NMT and literary pipelines share this whole body: they differ only in how
-// they project cost and how they translate, which is what translationMode
-// abstracts.
-func (o *Orchestrator) Run(ctx context.Context, cfg RunConfig) (*RunResult, error) {
- log := o.logger().With(
- zap.String("run_id", cfg.RunID),
- zap.String("book_id", cfg.BookID),
- zap.String("mode", runMode(cfg)),
- zap.Bool("resume", cfg.Resume),
- )
- started := time.Now().UTC()
-
- // Deferred first so it runs last: the caller writes its failure message to
- // the same stream, and it must not land inside the bar's line.
- prog := o.progress()
- defer prog.Finish()
-
- prog.Stage(string(StageExtract))
- raw, err := o.Extractor.Extract(ctx, cfg.InputPath)
- if err != nil {
- return nil, &StageError{Stage: StageExtract, Err: err}
- }
-
- prog.Stage(string(StageParse))
- book, err := o.Parser.Parse(ctx, raw)
- if err != nil {
- return nil, &StageError{Stage: StageParse, Err: err}
- }
-
- prog.Stage(string(StageSegment))
- chapters, err := o.segmentBookByChapter(ctx, book)
- if err != nil {
- return nil, &StageError{Stage: StageSegment, Err: err}
- }
- prog.Total(countSegments(chapters), len(chapters))
- log.Info("segmented book",
- zap.Int("segments", countSegments(chapters)), zap.Int("chapters", len(chapters)))
-
- mode, err := o.newTranslationMode(ctx, cfg)
- if err != nil {
- return nil, &StageError{Stage: StageTranslate, Err: err}
- }
-
- result := &RunResult{
- SchemaVersion: 1,
- RunID: cfg.RunID,
- BookID: cfg.BookID,
- StartedAt: started,
- TotalSegments: countSegments(chapters),
- NMTEngine: o.nmtEngine(),
- LLMProvider: o.LLMProvider,
- LLMModel: o.LLMModel,
- }
- mode.Init(result)
-
- // Pre-run budget guardrail: project the worst case for the work still to do
- // — every pending segment translated, none of them reused — and abort before
- // any provider call if it exceeds the per-book cap.
- usedFreeChars, err := o.freeCharsUsed(ctx, started)
- if err != nil {
- return nil, &StageError{Stage: StageTranslate, Err: err}
- }
-
- // Priced on what the TM cannot serve, so a resumed book is projected for
- // what is left of it rather than for all of it.
- pending, reusable, err := o.pendingChapters(ctx, chapters, mode.ReusesDrafts())
- if err != nil {
- return nil, &StageError{Stage: StageTranslate, Err: err}
- }
- if reusable > 0 {
- log.Info("projecting the remaining work",
- zap.Int("reusable_segments", reusable),
- zap.Int("pending_segments", countSegments(pending)),
- zap.Int("pending_chapters", len(pending)))
- }
-
- // Two projections, because the two readers want different things.
- //
- // The guardrail prices the work in front of it: pending only, so a resumed
- // book is not refused for a bill it will never pay. The report keeps the
- // no-reuse figure for the WHOLE book, because that is the only baseline
- // CostSavingsUSD can subtract from to mean "what the memory saved" — priced
- // on pending, savings would collapse to zero on exactly the runs where the
- // memory did the most work.
- projection := mode.Project(pending, usedFreeChars)
- result.EstimatedCostUSD = mode.Project(chapters, usedFreeChars).EstimatedUSD
- log.Info("projected run cost",
- zap.Float64("enforced_usd", projection.EstimatedUSD),
- zap.Float64("no_reuse_usd", result.EstimatedCostUSD),
- zap.Int("llm_calls", projection.LLMCalls),
- zap.Int("billable_nmt_chars", projection.BillableChars),
- zap.Int("llm_input_tokens", projection.InputTokens),
- zap.Int("llm_output_tokens", projection.OutputTokens),
- zap.Float64("nmt_usd", projection.NMTCostUSD),
- zap.Float64("llm_usd", projection.LLMCostUSD))
- if o.BudgetUSD > 0 && projection.EstimatedUSD > o.BudgetUSD {
- log.Warn("projected spend exceeds per-book budget cap",
- zap.Float64("estimated_usd", projection.EstimatedUSD),
- zap.Float64("budget_usd", o.BudgetUSD))
- return result, fmt.Errorf("unable to start run (estimated $%.4f > per-book cap $%.4f), err: %w",
- projection.EstimatedUSD, o.BudgetUSD, cost.ErrBudgetExceeded)
- }
- if err := o.checkRollingBudgets(ctx, log, projection.EstimatedUSD, started); err != nil {
- return result, err
- }
-
- // From here on the run can call paid providers, so its consumption is
- // totalled and recorded whatever happens next. A run that drafted half a
- // book and then failed to recompose has still been billed for that half,
- // and dropping the record would leave the next run's free-tier projection
- // wrong by exactly that much.
- //
- // finalise is single-shot: the success path calls it before writing the
- // report so the artifacts carry the same figures the ledger does, and the
- // deferred call covers every path that never got there.
- //
- // The meter runs for the adapter's lifetime and `batch` shares one across
- // every book, so a run is charged the delta over its own lifetime.
- sentBefore := o.charsSent()
-
- // costNow prices what has been consumed at this instant, through the same
- // mode that prices the final figure. The meter delta is folded into a copy
- // rather than into result: result.NMTChars is the run's record and belongs
- // to finalise, which must stay the single writer.
- costNow := func() float64 {
- snapshot := *result
- snapshot.NMTChars = o.charsSent() - sentBefore
- return mode.Cost(&snapshot, usedFreeChars)
- }
-
- var finalised bool
- finalise := func() {
- if finalised {
- return
- }
- finalised = true
- result.NMTChars = o.charsSent() - sentBefore
- result.ActualCostUSD = mode.Cost(result, usedFreeChars)
- }
- defer func() {
- finalise()
- o.recordSpend(ctx, cfg, log, result)
- }()
-
- prog.Stage(string(StageTranslate))
- segments, err := mode.Translate(ctx, log, chapters, usedFreeChars, result, costNow)
- if err != nil {
- return nil, &StageError{Stage: StageTranslate, Err: err}
- }
-
- prog.Stage(string(StageRecompose))
- outputEPUB := fmt.Sprintf("%s/translated.epub", cfg.OutputDir)
- if err := o.Recomposer.Recompose(ctx, raw, book, outputEPUB); err != nil {
- return nil, &StageError{Stage: StageRecompose, Err: err}
- }
- result.OutputEPUB = outputEPUB
-
- prog.Stage(string(StageQA))
- report := qa.Evaluate(book, segments)
- check, err := o.EPUBCheck.Check(ctx, outputEPUB)
- if err != nil {
- return nil, &StageError{Stage: StageQA, Err: err}
- }
- report.EPUBCheckPassed = check.Passed
- report.EPUBCheckSkipped = check.Skipped
- result.QA = report
-
- finalise()
- if result.TotalSegments > 0 {
- result.TMHitRatio = float64(result.TMHits) / float64(result.TotalSegments)
- result.LLMEscalationRatio = float64(result.LLMSegments) / float64(result.TotalSegments)
- }
- if savings := result.EstimatedCostUSD - result.ActualCostUSD; savings > 0 {
- result.CostSavingsUSD = savings
- }
- result.EndedAt = time.Now().UTC()
-
- // Artifacts are emitted regardless of the QA verdict so failed runs remain
- // diagnosable.
- if err := WriteArtifacts(cfg, result, segments); err != nil {
- return result, &StageError{Stage: StageQA, Err: err}
- }
-
- if !qaGatePassed(report) || result.GlossaryViolations > 0 {
- log.Warn("QA gate failed",
- zap.Float64("placeholder_restore_ratio", report.PlaceholderRestoreRatio),
- zap.Int("missing_spine_docs", report.MissingSpineDocs),
- zap.Int("untranslated_segments", report.UntranslatedSegments),
- zap.Int("glossary_violations", result.GlossaryViolations),
- zap.Bool("epubcheck_passed", report.EPUBCheckPassed),
- zap.Bool("epubcheck_skipped", report.EPUBCheckSkipped),
- )
- return result, fmt.Errorf("unable to pass QA gate, err: %w", ErrQAGateFailed)
- }
-
- log.Info("run completed",
- zap.Int("tm_hits", result.TMHits),
- zap.Int("nmt_segments", result.NMTSegments),
- zap.Int("llm_segments", result.LLMSegments),
- zap.Int("bible_version", result.BibleVersion),
- zap.String("output", outputEPUB),
- )
- return result, nil
-}
-
-// nmtEngine identifies the provider whose allowance a run consumes. It is empty
-// when no translator is configured, which is the case for a dry run.
-func (o *Orchestrator) nmtEngine() string {
- if o.Translator == nil {
- return ""
- }
- return o.Translator.Engine()
-}
-
-// charsSent reads the adapter's running total of characters actually put into
-// provider requests, or 0 from one that does not meter itself.
-//
-// The pipeline no longer derives this from segment source text. Only the
-// adapter knows what really went out: retried attempts, glossary-protected text
-// whose length differs from the source, and requests whose response was
-// discarded were all invisible to a count reconstructed from segments.
-func (o *Orchestrator) charsSent() int {
- meter, ok := o.Translator.(translate.Meter)
- if !ok {
- return 0
- }
- return meter.CharsSent()
-}
-
-// recordSpend persists what the run consumed, for rolling-budget and free-tier
-// accounting. It is deferred, so it fires on the failure paths too — the
-// provider does not refund a run that died after it was called.
-//
-// Best effort: a ledger write failure must not turn an otherwise successful run
-// into a failed one.
-func (o *Orchestrator) recordSpend(ctx context.Context, cfg RunConfig, log *zap.Logger, result *RunResult) {
- if o.TM == nil {
- return
- }
-
- if result.EndedAt.IsZero() {
- result.EndedAt = time.Now().UTC()
- }
-
- // A context that is already cancelled would reject the write, and losing
- // the record is worse than the delay: fall back to a detached one.
- if ctx.Err() != nil {
- var cancel context.CancelFunc
- ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), ledgerWriteTimeout)
- defer cancel()
- }
-
- if err := o.TM.RecordRun(ctx, tm.RunSpend{
- RunID: cfg.RunID,
- BookID: cfg.BookID,
- StartedAt: result.StartedAt,
- EndedAt: result.EndedAt,
- ActualCostUSD: result.ActualCostUSD,
- NMTEngine: result.NMTEngine,
- NMTChars: result.NMTChars,
- LLMProvider: result.LLMProvider,
- LLMModel: result.LLMModel,
- LLMSegments: result.LLMPassSegments,
- LLMInputTokens: result.LLMInputTokens,
- LLMOutputTokens: result.LLMOutputTokens,
- LLMCacheReadTokens: result.LLMCacheReadTokens,
- LLMCacheCreationTokens: result.LLMCacheCreationTokens,
- }); err != nil {
- log.Warn("unable to record run spend", zap.Error(err))
- }
-}
-
-// freeCharsUsed reports how much of this provider's monthly free character
-// allowance the calendar month has already consumed, so the projection bills
-// only the overflow. It fails closed: a ledger read error aborts the run rather
-// than silently projecting a free draft.
-//
-// The count is per engine. The ledger is shared and the default provider is a
-// local dev server, so a month of free local translation would otherwise be
-// charged against a paid provider's allowance.
-func (o *Orchestrator) freeCharsUsed(ctx context.Context, now time.Time) (int, error) {
- if o.TM == nil || o.CostModel.FreeCharsPerMonth <= 0 {
- return 0, nil
- }
-
- since := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
- used, err := o.TM.CharsSince(ctx, since, o.nmtEngine())
- if err != nil {
- return 0, fmt.Errorf("unable to read this month's NMT characters, err: %w", err)
- }
- return used, nil
-}
-
-// checkRollingBudgets enforces daily and monthly spend caps against the ledger
-// plus this run's projected cost. It fails closed: a ledger read error aborts
-// the run rather than silently bypassing the cap.
-func (o *Orchestrator) checkRollingBudgets(ctx context.Context, log *zap.Logger, projected float64, now time.Time) error {
- if o.TM == nil {
- return nil
- }
-
- windows := []struct {
- name string
- cap float64
- since time.Time
- }{
- {"daily", o.DailyUSD, time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)},
- {"monthly", o.MonthlyUSD, time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)},
- }
-
- for _, w := range windows {
- if w.cap <= 0 {
- continue
- }
- spent, err := o.TM.SpendSince(ctx, w.since)
- if err != nil {
- return fmt.Errorf("unable to read %s spend, err: %w", w.name, err)
- }
- if spent+projected > w.cap {
- log.Warn("projected spend exceeds rolling budget cap",
- zap.String("window", w.name),
- zap.Float64("already_spent_usd", spent),
- zap.Float64("projected_usd", projected),
- zap.Float64("cap_usd", w.cap))
- return fmt.Errorf("unable to start run (%s spend $%.4f + projected $%.4f > cap $%.4f), err: %w",
- w.name, spent, projected, w.cap, cost.ErrBudgetExceeded)
- }
- }
-
- return nil
-}
-
-func (o *Orchestrator) loadBible(ctx context.Context, seriesID string) (narrative.Bible, error) {
- if seriesID == "" {
- return narrative.Bible{}, narrative.ErrEmptySeriesID
- }
- if o.NarrativeStore == nil {
- return narrative.Bible{}, fmt.Errorf("unable to load required narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
- }
-
- bible, found, err := o.NarrativeStore.Load(ctx, seriesID)
- if err != nil {
- return narrative.Bible{}, err
- }
- if !found {
- return narrative.Bible{}, fmt.Errorf("unable to load required narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
- }
- if bible.SeriesID != seriesID {
- return narrative.Bible{}, fmt.Errorf("unable to validate narrative bible %q, err: %w", seriesID, narrative.ErrInvalidBible)
- }
- if err := narrative.ValidateBible(bible); err != nil {
- return narrative.Bible{}, fmt.Errorf("unable to validate narrative bible %q, err: %w", seriesID, err)
- }
- return bible, nil
-}
-
-func (o *Orchestrator) updateBible(ctx context.Context, seriesID string, bible *narrative.Bible, chapter *chapterSegments) error {
- if o.NarrativeProfiler == nil {
- return nil
- }
-
- updated, err := o.NarrativeProfiler.Update(ctx, *bible, chapter.ID, chapterSourceText(chapter.Segments), chapterTargetText(chapter.Segments))
- if err != nil {
- return err
- }
- updated.SeriesID = seriesID
- updated.Summaries = append([]narrative.Summary(nil), bible.Summaries...)
- // The version we loaded is the baseline, never the one the model volunteered
- // in its JSON — the version identifies stored content, so the store is what
- // assigns it. It now moves exactly when the content moves, which is what
- // makes a reported version mean something.
- updated.Version = bible.Version
-
- if o.NarrativeStore != nil {
- for {
- saved, err := o.NarrativeStore.SaveIfVersion(ctx, updated)
- if err != nil {
- return err
- }
- reloaded, found, err := o.NarrativeStore.Load(ctx, seriesID)
- if err != nil {
- return err
- }
- if !found {
- return fmt.Errorf("unable to reload updated narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
- }
- if saved {
- *bible = reloaded
- return nil
- }
- // A concurrent publisher won. Preserve its current synopsis snapshot
- // and retry only the opted-in style/voice/address update.
- updated.Version = reloaded.Version
- updated.Summaries = append([]narrative.Summary(nil), reloaded.Summaries...)
- }
- }
-
- *bible = updated
- return nil
-}
-
-// qaGatePassed combines static QA thresholds with the epubcheck verdict. A
-// skipped epubcheck (binary absent) does not block the run.
-func qaGatePassed(report qa.Report) bool {
- if !report.Passed() {
- return false
- }
- if !report.EPUBCheckSkipped && !report.EPUBCheckPassed {
- return false
- }
- return true
-}
-
-// PlanResult is a dry-run projection: the book is parsed and segmented but not
-// translated, and per-strategy cost estimates are returned.
-type PlanResult struct {
- BookID string `json:"book_id"`
- TotalSegments int `json:"total_segments"`
- TotalSourceChars int `json:"total_source_chars"`
- Chapters int `json:"chapters"`
- // ReusableSegments is how many of them the translation memory can already
- // serve, and PendingSegments what is left to pay for. The scenarios below
- // price the pending part only — the projection is about this run, not about
- // the book in the abstract.
- ReusableSegments int `json:"reusable_segments"`
- PendingSegments int `json:"pending_segments"`
- PendingSourceChar int `json:"pending_source_chars"`
- BatchSize int `json:"batch_size"`
- EscalationRate float64 `json:"escalation_rate"`
- Scenarios []cost.ScenarioEstimate `json:"scenarios"`
-}
-
-// Plan runs extract → parse → segment WITHOUT translating, then projects cost
-// across NMT-only / hybrid / full-LLM strategies. It only needs the Extractor,
-// Parser, Segmenter and CostModel to be set.
-//
-// It segments by chapter, not just by book, because prompt overhead is charged
-// per LLM call and calls are chunked per chapter. That is also what makes this
-// projection comparable with the one the run-time guardrail enforces — down to
-// reading the same free-tier counter off the same ledger.
-func (o *Orchestrator) Plan(
- ctx context.Context, cfg RunConfig, batchSize int, escalationRate float64,
-) (*PlanResult, error) {
- raw, err := o.Extractor.Extract(ctx, cfg.InputPath)
- if err != nil {
- return nil, &StageError{Stage: StageExtract, Err: err}
- }
- book, err := o.Parser.Parse(ctx, raw)
- if err != nil {
- return nil, &StageError{Stage: StageParse, Err: err}
- }
- chapters, err := o.segmentBookByChapter(ctx, book)
- if err != nil {
- return nil, &StageError{Stage: StageSegment, Err: err}
- }
-
- usedFreeChars, err := o.freeCharsUsed(ctx, time.Now().UTC())
- if err != nil {
- return nil, &StageError{Stage: StageSegment, Err: err}
- }
-
- // Only finished translations count as reuse here, which is the literary
- // rule rather than the NMT one. `estimate` prices three strategies at once
- // and cannot hold two definitions of "already done": counting cached drafts
- // as reuse would under-price the two rows that have to re-translate them, and
- // a projection that reads low is the one failure mode this command cannot
- // afford. It over-projects the nmt-only row instead.
- pending, reusable, err := o.pendingChapters(ctx, chapters, false)
- if err != nil {
- return nil, &StageError{Stage: StageSegment, Err: err}
- }
-
- grouped := make([][]segment.Segment, len(pending))
- for i := range pending {
- grouped[i] = pending[i].Segments
- }
- all := flattenChapters(chapters)
- segments := flattenChapters(pending)
-
- return &PlanResult{
- BookID: cfg.BookID,
- TotalSegments: len(all),
- TotalSourceChars: segment.SourceChars(all),
- Chapters: len(chapters),
- ReusableSegments: reusable,
- PendingSegments: len(segments),
- PendingSourceChar: segment.SourceChars(segments),
- BatchSize: batchSize,
- EscalationRate: escalationRate,
- Scenarios: o.CostModel.EstimateScenarios(grouped, batchSize, escalationRate, usedFreeChars),
- }, nil
-}
-
-func (o *Orchestrator) segmentBookByChapter(ctx context.Context, book *epub.Book) ([]chapterSegments, error) {
- bookChapters := book.Chapters()
- chapters := make([]chapterSegments, 0, len(bookChapters))
-
- for _, chapter := range bookChapters {
- segs, err := o.Segmenter.Segment(ctx, chapter.Doc, chapter.ID)
- if err != nil {
- return nil, fmt.Errorf("unable to segment %q, err: %w", chapter.Href, err)
- }
- chapters = append(chapters, chapterSegments{
- ID: chapter.ID,
- Href: chapter.Href,
- Segments: segs,
- })
- }
-
- return chapters, nil
-}
-
-// translateSegments resolves each segment via the TM cache first and the NMT
-// provider otherwise, then restores placeholders into the live node tree. It
-// accumulates the consumption counters onto result as it goes, so a failure
-// partway through still leaves an accurate record of what was paid for.
-// Pricing belongs to the mode, not to the translation loop.
-func (o *Orchestrator) translateSegments(
- ctx context.Context, cfg RunConfig, log *zap.Logger,
- segments []segment.Segment, result *RunResult, price costFn,
-) error {
- pm := o.Placeholders
- if pm == nil {
- pm = segment.InlinePlaceholderManager{}
- }
-
- var qualitySum float64
- qualityCount := 0
- for i := range segments {
- if err := ctx.Err(); err != nil {
- return err
- }
-
- seg := &segments[i]
-
- target, reused, err := o.resolveTranslation(ctx, cfg, seg)
- if err != nil {
- return err
- }
-
- seg.TargetText = target
- if reused {
- seg.Status = segment.StatusReused
- result.TMHits++
- o.progress().Advance(0, 1)
- } else {
- seg.Status = segment.StatusTranslated
- result.NMTSegments++
-
- // Reference-free quality estimation (e.g. COMET-QE) on the NMT draft,
- // used both to drive escalation and to report per-book quality.
- var qeScore float64
- haveQE := false
- if o.QualityEstimator != nil {
- if s, err := o.QualityEstimator.Estimate(ctx, seg.SourceText, target); err == nil {
- qeScore, haveQE = s, true
- qualitySum += s
- qualityCount++
- } else {
- log.Warn("quality estimation failed", zap.String("segment_id", seg.ID), zap.Error(err))
- }
- }
-
- // Selective LLM post-edit of fresh NMT drafts. SafePostEdit falls
- // back to the NMT draft on any error or placeholder change, so the
- // output can never be degraded by escalation.
- if o.shouldEscalate(seg, qeScore, haveQE) {
- edited, postEdited := llm.SafePostEdit(ctx, o.PostEditor, llm.PostEditRequest{
- SourceText: seg.SourceText,
- NMTText: target,
- SourceLang: cfg.SourceLang,
- TargetLang: cfg.TargetLang,
- })
-
- // Billed on the attempt, not on the outcome. A call that
- // errored, or came back with the placeholders rearranged, still
- // consumed tokens; SafePostEdit reports only whether the result
- // was usable. LLMSegments stays the count of segments actually
- // improved, which is what the escalation ratio means.
- result.LLMAttempts++
- result.LLMPassSegments++
- result.LLMChars += utf8.RuneCountInString(seg.SourceText)
-
- if postEdited {
- target = edited
- seg.TargetText = target
- seg.Status = segment.StatusPostEdited
- result.LLMSegments++
- if o.TM != nil {
- if err := o.TM.Save(ctx, tm.Entry{
- SourceHash: seg.Hash, SourceText: seg.SourceText, TargetText: target,
- SourceLang: cfg.SourceLang, TargetLang: cfg.TargetLang, Engine: tm.EngineLLMPostEdit,
- }); err != nil {
- log.Warn("unable to persist post-edited TM entry",
- zap.String("segment_id", seg.ID), zap.Error(err))
- }
- }
- }
- }
-
- // Reported here, not at the top of the iteration: a fresh segment
- // is only done once its optional post-edit has been attempted, and
- // that call is the slow part a watcher is waiting on. A TM hit costs
- // nothing, which is why only this branch reprices.
- o.progress().Advance(1, 0)
- o.progress().Cost(price())
- }
-
- if o.Glossary != nil {
- result.GlossaryViolations += o.Glossary.Violations(seg.SourceText, target)
- }
-
- if err := pm.Restore(target, seg.Placeholders, seg.Node); err != nil {
- // Restore validates every issued token before it touches the node,
- // so a failure genuinely leaves the original (source) markup in
- // place; QA flags the dropped placeholder. Do not abort the run.
- log.Warn("placeholder restore failed",
- zap.String("segment_id", seg.ID), zap.Error(err))
- }
- }
-
- if qualityCount > 0 {
- result.AvgQualityScore = qualitySum / float64(qualityCount)
- result.QualityScoredSegments = qualityCount
- }
- return nil
-}
-
-// shouldEscalate decides whether a freshly translated segment is sent to the
-// LLM post-editor. The heuristic (source length threshold) is a configurable
-// proxy for "complex/high-value" and is intentionally simple: precise
-// confidence-based routing is deferred until a labeled benchmark exists.
-func (o *Orchestrator) shouldEscalate(seg *segment.Segment, qeScore float64, haveQE bool) bool {
- if o.PostEditor == nil {
- return false
- }
- // Quality-driven: escalate the weakest drafts when QE is available.
- if haveQE && o.EscalateBelowScore > 0 {
- return qeScore < o.EscalateBelowScore
- }
- // Fallback heuristic: escalate by source length.
- if o.EscalateMinChars <= 0 {
- return true
- }
- return utf8.RuneCountInString(seg.SourceText) >= o.EscalateMinChars
-}
-
-// lookupTranslation fills seg.TargetText from an exact TM hit and reports
-// whether one was found. Unlike resolveTranslation it never calls the provider
-// and never writes, which is what lets the literary path batch its misses.
-//
-// A raw draft is not a hit here. Literary mode stopped persisting drafts, but
-// databases written by earlier runs still hold them, and serving one would skip
-// refinement for that segment permanently — the exact defect the change was
-// made to remove. Such an entry is treated as a miss: the segment is re-drafted
-// and re-refined, and the finished translation replaces it.
-func (o *Orchestrator) lookupTranslation(ctx context.Context, seg *segment.Segment) (bool, error) {
- if o.TM == nil {
- return false, nil
- }
-
- entry, found, err := o.TM.Lookup(ctx, seg.Hash)
- if err != nil {
- return false, fmt.Errorf("unable to look up TM for %s, err: %w", seg.ID, err)
- }
- if !found || tm.IsDraftEngine(entry.Engine) {
- return false, nil
- }
-
- seg.TargetText = entry.TargetText
- return true, nil
-}
-
-// resolveTranslation returns the target text for a segment, preferring an exact
-// TM hit and falling back to the NMT provider (persisting the new translation).
-//
-// Unlike lookupTranslation it accepts a draft as a hit, and saves one. That is
-// correct here and only here: in NMT mode the draft is the final output for a
-// segment that is not escalated, so refusing it would throw away all TM reuse.
-func (o *Orchestrator) resolveTranslation(ctx context.Context, cfg RunConfig, seg *segment.Segment) (string, bool, error) {
- if o.TM != nil {
- entry, found, err := o.TM.Lookup(ctx, seg.Hash)
- if err != nil {
- return "", false, fmt.Errorf("unable to look up TM for %s, err: %w", seg.ID, err)
- }
- if found {
- return entry.TargetText, true, nil
- }
- }
-
- // Protect locked glossary terms so the provider cannot alter them, then
- // substitute the canonical target term back after translation.
- sendText := seg.SourceText
- var subs map[string]string
- if o.Glossary != nil {
- sendText, subs = o.Glossary.Protect(seg.SourceText)
- }
-
- resp, err := o.Translator.Translate(ctx, translate.Request{
- Text: sendText,
- SourceLang: cfg.SourceLang,
- TargetLang: cfg.TargetLang,
- })
- if err != nil {
- return "", false, fmt.Errorf("unable to translate %s, err: %w", seg.ID, err)
- }
-
- target := resp.Text
- if o.Glossary != nil {
- target = o.Glossary.Apply(target, subs)
- }
-
- if o.TM != nil {
- if err := o.TM.Save(ctx, tm.Entry{
- SourceHash: seg.Hash,
- SourceText: seg.SourceText,
- TargetText: target,
- SourceLang: cfg.SourceLang,
- TargetLang: cfg.TargetLang,
- Engine: resp.Engine,
- }); err != nil {
- return "", false, fmt.Errorf("unable to persist TM for %s, err: %w", seg.ID, err)
- }
- }
-
- return target, false, nil
-}
-
-// placeholders returns the configured placeholder manager, defaulting to the
-// inline one.
-func (o *Orchestrator) placeholders() segment.PlaceholderManager {
- if o.Placeholders != nil {
- return o.Placeholders
- }
- return segment.InlinePlaceholderManager{}
-}
-
-func (o *Orchestrator) logger() *zap.Logger {
- if o.Logger != nil {
- return o.Logger
- }
- return zap.NewNop()
-}
-
-// pendingChapters returns the chapters holding only the segments this run will
-// actually pay for: the ones the TM cannot serve. It also reports how many it
-// dropped.
-//
-// It exists so a projection describes the work in front of it. Assuming no reuse
-// is the right worst case for a fresh book and the wrong one for a resumed one:
-// a run interrupted at 80% would project the whole volume again and be refused
-// by a cap sized for what remains. Resuming is the normal way to finish a book
-// after a rate limit, so the guardrail has to be able to tell the difference.
-//
-// The read is cheap enough to belong in a projection: point lookups on an
-// indexed SQLite column, no provider call, so `estimate` can do it too.
-//
-// The tradeoff is stated rather than hidden: the projection stops being a
-// property of the book alone. The same volume projects differently against two
-// machines' memories, which is the price of it being about this run.
-func (o *Orchestrator) pendingChapters(
- ctx context.Context, chapters []chapterSegments, reusesDrafts bool,
-) ([]chapterSegments, int, error) {
- if o.TM == nil {
- return chapters, 0, nil
- }
-
- pending := make([]chapterSegments, 0, len(chapters))
- reusable := 0
- for _, chapter := range chapters {
- kept := make([]segment.Segment, 0, len(chapter.Segments))
- for _, seg := range chapter.Segments {
- entry, found, err := o.TM.Lookup(ctx, seg.Hash)
- if err != nil {
- return nil, 0, fmt.Errorf("unable to look up TM for %s, err: %w", seg.ID, err)
- }
- // A draft is a hit only for the mode that treats a draft as output.
- if found && (reusesDrafts || !tm.IsDraftEngine(entry.Engine)) {
- reusable++
- continue
- }
- kept = append(kept, seg)
- }
- if len(kept) == 0 {
- // A chapter with nothing left to do makes no call, so it must not
- // contribute a call's worth of prefix to the projection.
- continue
- }
- chapter.Segments = kept
- pending = append(pending, chapter)
- }
- return pending, reusable, nil
-}
-
-// countSegments totals a book's segments without materialising the flat slice.
-func countSegments(chapters []chapterSegments) int {
- total := 0
- for i := range chapters {
- total += len(chapters[i].Segments)
- }
- return total
-}
-
-func flattenChapters(chapters []chapterSegments) []segment.Segment {
- var all []segment.Segment
- for _, chapter := range chapters {
- all = append(all, chapter.Segments...)
- }
- return all
-}
-
-func adjacentChapterText(chapters []chapterSegments, idx int) string {
- if idx < 0 || idx >= len(chapters) {
- return ""
- }
- return chapterSourceText(chapters[idx].Segments)
-}
-
-func chapterSourceText(segments []segment.Segment) string {
- var sb strings.Builder
- for _, seg := range segments {
- if strings.TrimSpace(seg.SourceText) == "" {
- continue
- }
- sb.WriteString(seg.SourceText)
- sb.WriteByte('\n')
- }
- return strings.TrimSpace(sb.String())
-}
-
-func chapterTargetText(segments []segment.Segment) string {
- var sb strings.Builder
- for _, seg := range segments {
- if strings.TrimSpace(seg.TargetText) == "" {
- continue
- }
- sb.WriteString(seg.TargetText)
- sb.WriteByte('\n')
- }
- return strings.TrimSpace(sb.String())
-}
-
-func runMode(cfg RunConfig) string {
- if cfg.Mode == "" {
- return ModeNMT
- }
- return cfg.Mode
-}
diff --git a/internal/pipeline/orchestrator_test.go b/internal/pipeline/orchestrator_test.go
deleted file mode 100644
index 5211ed3..0000000
--- a/internal/pipeline/orchestrator_test.go
+++ /dev/null
@@ -1,826 +0,0 @@
-package pipeline_test
-
-import (
- "archive/zip"
- "context"
- "encoding/json"
- "errors"
- "io"
- "net/http"
- "net/http/httptest"
- "os"
- "path/filepath"
- "strings"
- "testing"
- "time"
-
- "github.com/stretchr/testify/require"
-
- "github.com/ECecillo/tomeshift/internal/cost"
- "github.com/ECecillo/tomeshift/internal/epub"
- "github.com/ECecillo/tomeshift/internal/llm"
- "github.com/ECecillo/tomeshift/internal/llm/llmtest"
- "github.com/ECecillo/tomeshift/internal/narrative"
- "github.com/ECecillo/tomeshift/internal/pipeline"
- "github.com/ECecillo/tomeshift/internal/qa"
- "github.com/ECecillo/tomeshift/internal/segment"
- "github.com/ECecillo/tomeshift/internal/tm"
- "github.com/ECecillo/tomeshift/internal/translate"
- "github.com/ECecillo/tomeshift/internal/translate/nmttest"
-)
-
-var fixtureEntries = map[string]string{
- "mimetype": "application/epub+zip",
- "META-INF/container.xml": `
-
-
-
-
-`,
- "OEBPS/content.opf": `
-
-
-
-
-
-
-
-`,
- "OEBPS/chapter1.xhtml": `
-
-
-
Chapter One
-
Hello world.
-
A bold claim.
-
-`,
-}
-
-func writeFixtureEPUB(t *testing.T) string {
- t.Helper()
-
- path := filepath.Join(t.TempDir(), "source.epub")
- f, err := os.Create(path)
- require.NoError(t, err)
-
- zw := zip.NewWriter(f)
-
- w, err := zw.CreateHeader(&zip.FileHeader{Name: "mimetype", Method: zip.Store})
- require.NoError(t, err)
- _, err = w.Write([]byte(fixtureEntries["mimetype"]))
- require.NoError(t, err)
-
- for name, content := range fixtureEntries {
- if name == "mimetype" {
- continue
- }
- w, err := zw.Create(name)
- require.NoError(t, err)
- _, err = w.Write([]byte(content))
- require.NoError(t, err)
- }
-
- require.NoError(t, zw.Close())
- require.NoError(t, f.Close())
- return path
-}
-
-func newOrchestrator(t *testing.T, tr translate.Translator) *pipeline.Orchestrator {
- t.Helper()
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- return &pipeline.Orchestrator{
- Extractor: epub.ZipExtractor{},
- Parser: epub.OPFParser{},
- Segmenter: segment.NewParagraphSegmenter(),
- TM: store,
- Translator: tr,
- Recomposer: epub.EPUBRecomposer{},
- EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"}, // force skip
- }
-}
-
-func TestOrchestrator_Run_EndToEnd(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil) // prefixes "[fr] "
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
-
- outputDir := filepath.Join(t.TempDir(), "run")
- cfg := pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: outputDir,
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "test-run",
- BookID: "fixture",
- }
-
- result, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
- require.NotNil(t, result)
-
- require.Equal(t, 3, result.TotalSegments) // h1 + 2 paragraphs
- require.Equal(t, 3, result.NMTSegments)
- require.Equal(t, 0, result.TMHits)
- require.True(t, result.QA.Passed())
- require.True(t, result.QA.EPUBCheckSkipped)
- require.InDelta(t, 1.0, result.QA.PlaceholderRestoreRatio, 0.0001)
-
- // Artifacts exist.
- require.FileExists(t, filepath.Join(outputDir, "translated.epub"))
- require.FileExists(t, filepath.Join(outputDir, "report.json"))
- require.FileExists(t, filepath.Join(outputDir, "segments.jsonl"))
-
- // report.json round-trips.
- reportBytes, err := os.ReadFile(filepath.Join(outputDir, "report.json"))
- require.NoError(t, err)
- var roundTrip pipeline.RunResult
- require.NoError(t, json.Unmarshal(reportBytes, &roundTrip))
- require.Equal(t, "test-run", roundTrip.RunID)
-
- // Translated EPUB contains the French-marked text with markup intact.
- chapter := readZipEntry(t, filepath.Join(outputDir, "translated.epub"), "OEBPS/chapter1.xhtml")
- require.Contains(t, chapter, "[fr] Hello world.")
- require.Contains(t, chapter, "bold", "inline markup must be preserved")
-}
-
-func TestOrchestrator_LiteraryModeRequiresExistingValidBible(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- name string
- store func(t *testing.T) narrative.Store
- want error
- }{
- {name: "store absent", want: narrative.ErrMissingBible},
- {
- name: "scope missing",
- store: func(t *testing.T) narrative.Store {
- return newNarrativeStore(t)
- },
- want: narrative.ErrMissingBible,
- },
- {
- name: "invalid bible",
- store: func(t *testing.T) narrative.Store {
- store := newNarrativeStore(t)
- require.NoError(t, store.Save(t.Context(), narrative.Bible{SeriesID: "series-a"}))
- return store
- },
- want: narrative.ErrInvalidBible,
- },
- }
-
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- orch := newOrchestrator(t, nil)
- if tt.store != nil {
- orch.NarrativeStore = tt.store(t)
- }
- _, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t), Mode: pipeline.ModeLiterary, SeriesID: "series-a",
- })
- require.Error(t, err)
- require.True(t, errors.Is(err, tt.want), "error: %v", err)
- })
- }
-}
-
-func newNarrativeStore(t *testing.T) *narrative.SQLiteStore {
- t.Helper()
- store, err := narrative.NewSQLiteStore(filepath.Join(t.TempDir(), "narrative.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
- return store
-}
-
-func TestOrchestrator_Run_TMReuse(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- mk := func() *pipeline.Orchestrator {
- return &pipeline.Orchestrator{
- Extractor: epub.ZipExtractor{},
- Parser: epub.OPFParser{},
- Segmenter: segment.NewParagraphSegmenter(),
- TM: store,
- Translator: translate.NewHTTPTranslator(srv.URL, ""),
- Recomposer: epub.EPUBRecomposer{},
- EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"},
- CostModel: cost.Model{PerSegmentUSD: 0.01}, // make reuse savings observable
- }
- }
-
- input := writeFixtureEPUB(t)
-
- first, err := mk().Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "r1"),
- SourceLang: "en", TargetLang: "fr", RunID: "r1", BookID: "b",
- })
- require.NoError(t, err)
- require.Equal(t, 3, first.NMTSegments)
- require.Equal(t, 0, first.TMHits)
-
- // Second run over the same book: every segment should now be a TM hit.
- second, err := mk().Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "r2"),
- SourceLang: "en", TargetLang: "fr", RunID: "r2", BookID: "b",
- })
- require.NoError(t, err)
- require.Equal(t, 0, second.NMTSegments)
- require.Equal(t, 3, second.TMHits)
- require.InDelta(t, 1.0, second.TMHitRatio, 0.0001)
-
- // All 3 segments reused -> actual cost 0, full projected cost saved.
- require.InDelta(t, 0.0, second.ActualCostUSD, 0.0001)
- require.Greater(t, second.CostSavingsUSD, 0.0)
- require.InDelta(t, second.EstimatedCostUSD, second.CostSavingsUSD, 0.0001)
-}
-
-func TestOrchestrator_Run_FailedRunStillRecordsWhatItSpent(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- orch.TM = store
- orch.CostModel = cost.Model{Per1000CharsUSD: 2.0, FreeCharsPerMonth: 500_000}
- // Recompose into a path that cannot exist, so the run fails *after* every
- // segment has already been drafted and paid for.
- outputDir := filepath.Join(t.TempDir(), "run")
- cfg := pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(outputDir, "no", "such", "dir", "\x00"),
- SourceLang: "en", TargetLang: "fr", RunID: "died", BookID: "b",
- }
-
- _, err = orch.Run(t.Context(), cfg)
- // Pin the stage: if this ever started failing earlier the test would still
- // go red, but for the wrong reason — nothing would have been drafted.
- var stageErr *pipeline.StageError
- require.ErrorAs(t, err, &stageErr)
- require.Equal(t, pipeline.StageRecompose, stageErr.Stage)
-
- // The provider does not refund a run that died downstream. Losing this
- // would leave the next run's free-tier projection short by the same amount.
- used, err := store.CharsSince(t.Context(), time.Now().UTC().Add(-time.Hour), "nmt-http")
- require.NoError(t, err)
- require.Positive(t, used, "characters already sent must survive the failure")
-}
-
-func TestOrchestrator_Run_RetriedCharsAreNotCarriedBetweenRuns(t *testing.T) {
- t.Parallel()
-
- // One transient failure per book, then success.
- var calls int
- srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- calls++
- if calls%2 == 1 {
- http.Error(w, "flaky", http.StatusInternalServerError)
- return
- }
- var req struct {
- Text string `json:"text"`
- }
- _ = json.NewDecoder(r.Body).Decode(&req)
- w.Header().Set("Content-Type", "application/json")
- _ = json.NewEncoder(w).Encode(map[string]string{"translated_text": "[fr] " + req.Text})
- }))
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- // `batch` reuses one orchestrator — and therefore one RetryingTranslator,
- // whose retried-character counter runs for its whole lifetime — across
- // every book.
- orch := newOrchestrator(t, translate.NewRetryingTranslator(
- translate.NewHTTPTranslator(srv.URL, ""), 3, time.Millisecond))
- orch.TM = store
-
- run := func(id string) int {
- result, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), id),
- SourceLang: "en", TargetLang: "fr", RunID: id, BookID: id,
- })
- require.NoError(t, err)
- return result.NMTChars
- }
-
- first := run("book-1")
- require.Positive(t, first, "the first run translated and retried")
-
- // The second run is the same book, so every segment is a TM hit: it calls
- // the provider zero times. It must therefore record zero characters, not
- // inherit the first run's retries from a counter that lives as long as the
- // translator — which is exactly how `batch` shares one.
- require.Zero(t, run("book-2"),
- "retried characters must be scoped to the run that re-sent them")
-}
-
-func TestOrchestrator_Run_BudgetAbort(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- orch.CostModel = cost.Model{PerSegmentUSD: 1.0} // 3 segments -> ~$3
- orch.BudgetUSD = 0.5 // way under projection
-
- result, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en", TargetLang: "fr", RunID: "budget", BookID: "b",
- })
-
- require.ErrorIs(t, err, cost.ErrBudgetExceeded)
- require.NotNil(t, result)
- require.Greater(t, result.EstimatedCostUSD, orch.BudgetUSD)
- require.Equal(t, 0, result.NMTSegments, "no provider calls when aborted pre-translation")
-}
-
-func TestOrchestrator_Run_DailyCapAbort(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- // Pre-seed today's ledger close to the daily cap.
- now := time.Now().UTC()
- require.NoError(t, store.RecordRun(t.Context(), tm.RunSpend{
- RunID: "earlier", BookID: "b", StartedAt: now, EndedAt: now, ActualCostUSD: 9.5,
- }))
-
- orch := &pipeline.Orchestrator{
- Extractor: epub.ZipExtractor{},
- Parser: epub.OPFParser{},
- Segmenter: segment.NewParagraphSegmenter(),
- TM: store,
- Translator: translate.NewHTTPTranslator(srv.URL, ""),
- Recomposer: epub.EPUBRecomposer{},
- EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"},
- CostModel: cost.Model{PerSegmentUSD: 1.0}, // ~$3 projected
- DailyUSD: 10.0, // 9.5 + 3 > 10 -> abort
- }
-
- _, err = orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en", TargetLang: "fr", RunID: "daily", BookID: "b",
- })
- require.ErrorIs(t, err, cost.ErrBudgetExceeded)
-}
-
-func TestOrchestrator_Run_LLMEscalation(t *testing.T) {
- t.Parallel()
-
- nmt := nmttest.NewServer(nil) // "[fr] " prefix
- t.Cleanup(nmt.Close)
-
- // Post-editor uppercases the draft while preserving placeholder tokens.
- post := llmtest.NewServer(func(draft string) string {
- return strings.ReplaceAll(draft, "[fr] ", "[FR] ")
- })
- t.Cleanup(post.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(nmt.URL, ""))
- orch.PostEditor = llm.NewHTTPPostEditor(post.URL, "")
- orch.EscalateMinChars = 0 // escalate every fresh segment
-
- outputDir := filepath.Join(t.TempDir(), "run")
- result, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: outputDir,
- SourceLang: "en", TargetLang: "fr", RunID: "llm", BookID: "b",
- })
-
- require.NoError(t, err)
- require.Equal(t, 3, result.LLMSegments, "all fresh segments post-edited")
- require.True(t, result.QA.Passed())
-
- chapter := readZipEntry(t, filepath.Join(outputDir, "translated.epub"), "OEBPS/chapter1.xhtml")
- require.Contains(t, chapter, "[FR] Hello world.", "post-edited text must reach the output")
- require.Contains(t, chapter, "bold", "markup preserved through post-edit")
-}
-
-// fakeEstimator returns a fixed QE score for every segment.
-type fakeEstimator struct{ score float64 }
-
-func (f fakeEstimator) Estimate(_ context.Context, _, _ string) (float64, error) {
- return f.score, nil
-}
-
-func TestOrchestrator_Run_QualityDrivenEscalation(t *testing.T) {
- t.Parallel()
-
- tests := []struct {
- name string
- qeScore float64
- wantLLMSeg int
- }{
- {"low quality escalates all", 0.30, 3},
- {"high quality escalates none", 0.95, 0},
- }
-
- for _, tt := range tests {
- tt := tt
- t.Run(tt.name, func(t *testing.T) {
- t.Parallel()
-
- nmt := nmttest.NewServer(nil)
- t.Cleanup(nmt.Close)
- post := llmtest.NewServer(nil) // identity, preserves placeholders
- t.Cleanup(post.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(nmt.URL, ""))
- orch.PostEditor = llm.NewHTTPPostEditor(post.URL, "")
- orch.EscalateMinChars = 1 << 30 // length rule would never escalate
- orch.QualityEstimator = fakeEstimator{score: tt.qeScore}
- orch.EscalateBelowScore = 0.5
-
- result, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en", TargetLang: "fr", RunID: "qe", BookID: "b",
- })
- require.NoError(t, err)
- require.Equal(t, tt.wantLLMSeg, result.LLMSegments, "QE score must drive escalation, not length")
- require.Equal(t, 3, result.QualityScoredSegments)
- require.InDelta(t, tt.qeScore, result.AvgQualityScore, 1e-9)
- })
- }
-}
-
-func TestOrchestrator_Run_CancelledContextAborts(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
-
- ctx, cancel := context.WithCancel(t.Context())
- cancel() // pre-cancel: the translate loop must bail out
-
- _, err := orch.Run(ctx, pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en", TargetLang: "fr", RunID: "cancel", BookID: "b",
- })
- require.Error(t, err)
- require.ErrorIs(t, err, context.Canceled)
-}
-
-func readZipEntry(t *testing.T, zipPath, entry string) string {
- t.Helper()
-
- zr, err := zip.OpenReader(zipPath)
- require.NoError(t, err)
- t.Cleanup(func() { _ = zr.Close() })
-
- for _, f := range zr.File {
- if f.Name == entry {
- rc, err := f.Open()
- require.NoError(t, err)
- defer func() { _ = rc.Close() }()
- data, err := io.ReadAll(rc)
- require.NoError(t, err)
- return string(data)
- }
- }
- t.Fatalf("entry %q not found in %q", entry, zipPath)
- return ""
-}
-
-// recordingReporter captures what a run reported, so the assertions can be made
-// on the sequence a watcher would have seen rather than on rendered text.
-type recordingReporter struct {
- stages []string
- total int
- chapters int
- fresh int
- reused int
- costs []float64
- finished int
-}
-
-func (r *recordingReporter) Stage(name string) { r.stages = append(r.stages, name) }
-func (r *recordingReporter) Total(segments, chapters int) {
- r.total, r.chapters = segments, chapters
-}
-func (r *recordingReporter) Chapter(int) {}
-func (r *recordingReporter) Advance(fresh, reused int) {
- r.fresh += fresh
- r.reused += reused
-}
-func (r *recordingReporter) Cost(usd float64) {
- r.costs = append(r.costs, usd)
-}
-func (r *recordingReporter) Finish() { r.finished++ }
-
-// lastCost is the figure a watcher was left looking at.
-func (r *recordingReporter) lastCost() float64 {
- if len(r.costs) == 0 {
- return 0
- }
- return r.costs[len(r.costs)-1]
-}
-
-func TestOrchestrator_Run_ReportsProgress(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- rep := &recordingReporter{}
- orch.Progress = rep
-
- cfg := pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "progress-run",
- BookID: "fixture",
- }
-
- result, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
-
- require.Equal(t, []string{"extract", "parse", "segment", "translate", "recompose", "qa"}, rep.stages)
- require.Equal(t, result.TotalSegments, rep.total)
- require.Equal(t, 1, rep.chapters)
- // Every segment is accounted for exactly once, and the split matches the
- // report — which is what makes the bar's percentage mean the same thing as
- // report.json.
- require.Equal(t, result.NMTSegments, rep.fresh)
- require.Equal(t, result.TMHits, rep.reused)
- require.Equal(t, result.TotalSegments, rep.fresh+rep.reused)
- require.Equal(t, 1, rep.finished)
-}
-
-func TestOrchestrator_Run_ReportsProgress_TMReuseOnRerun(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- input := writeFixtureEPUB(t)
- cfg := pipeline.RunConfig{
- InputPath: input,
- OutputDir: filepath.Join(t.TempDir(), "first"),
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "first",
- BookID: "fixture",
- }
- _, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
-
- rep := &recordingReporter{}
- orch.Progress = rep
- cfg.OutputDir = filepath.Join(t.TempDir(), "second")
- cfg.RunID = "second"
- result, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
-
- require.Equal(t, result.TotalSegments, rep.reused, "a re-run is all TM reuse")
- require.Zero(t, rep.fresh)
-}
-
-func TestOrchestrator_Run_ReportsFinishOnFailure(t *testing.T) {
- t.Parallel()
-
- orch := newOrchestrator(t, nil)
- rep := &recordingReporter{}
- orch.Progress = rep
-
- _, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: filepath.Join(t.TempDir(), "absent.epub"),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "failed",
- BookID: "absent",
- })
- require.Error(t, err)
-
- // The caller is about to write the error to the same stream, so the line
- // must have been closed first.
- require.Equal(t, 1, rep.finished)
- require.Equal(t, []string{"extract"}, rep.stages)
- require.Zero(t, rep.total)
-}
-
-func TestOrchestrator_Run_ReportsRunningCost(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- // A priced provider with no free allowance, so every character costs.
- orch.CostModel = cost.Model{Per1000CharsUSD: 20}
- rep := &recordingReporter{}
- orch.Progress = rep
-
- result, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: writeFixtureEPUB(t),
- OutputDir: filepath.Join(t.TempDir(), "run"),
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "cost-run",
- BookID: "fixture",
- })
- require.NoError(t, err)
-
- // One repricing per fresh segment, monotonically rising, and landing exactly
- // on the figure report.json carries — the bar and the report are the same
- // arithmetic over the same counters.
- require.Len(t, rep.costs, result.NMTSegments)
- require.Positive(t, rep.lastCost())
- require.InDelta(t, result.ActualCostUSD, rep.lastCost(), 1e-9)
- for i := 1; i < len(rep.costs); i++ {
- require.GreaterOrEqual(t, rep.costs[i], rep.costs[i-1], "spend cannot go down")
- }
-}
-
-func TestOrchestrator_Run_TMHitsCostNothing(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- orch := newOrchestrator(t, translate.NewHTTPTranslator(srv.URL, ""))
- orch.CostModel = cost.Model{Per1000CharsUSD: 20}
- input := writeFixtureEPUB(t)
- cfg := pipeline.RunConfig{
- InputPath: input,
- OutputDir: filepath.Join(t.TempDir(), "first"),
- SourceLang: "en",
- TargetLang: "fr",
- RunID: "first",
- BookID: "fixture",
- }
- _, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
-
- rep := &recordingReporter{}
- orch.Progress = rep
- cfg.OutputDir, cfg.RunID = filepath.Join(t.TempDir(), "second"), "second"
- result, err := orch.Run(t.Context(), cfg)
- require.NoError(t, err)
-
- // Nothing was sent to the provider, so nothing was repriced and the run is
- // free — which is the whole point of the TM.
- require.Empty(t, rep.costs)
- require.Zero(t, result.ActualCostUSD)
-}
-
-func TestOrchestrator_Run_GuardrailPricesOnlyTheRemainingWork(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- mk := func(budget float64) *pipeline.Orchestrator {
- return &pipeline.Orchestrator{
- Extractor: epub.ZipExtractor{},
- Parser: epub.OPFParser{},
- Segmenter: segment.NewParagraphSegmenter(),
- TM: store,
- Translator: translate.NewHTTPTranslator(srv.URL, ""),
- Recomposer: epub.EPUBRecomposer{},
- EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"},
- CostModel: cost.Model{PerSegmentUSD: 1},
- BudgetUSD: budget,
- }
- }
- input := writeFixtureEPUB(t)
- cfg := func(id string) pipeline.RunConfig {
- return pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), id),
- SourceLang: "en", TargetLang: "fr", RunID: id, BookID: "b",
- }
- }
-
- // The book is 3 segments at $1 each. A $2 cap refuses it outright.
- _, err = mk(2).Run(t.Context(), cfg("refused"))
- require.ErrorIs(t, err, cost.ErrBudgetExceeded)
-
- // Translate it under a cap that allows it, filling the memory.
- first, err := mk(10).Run(t.Context(), cfg("first"))
- require.NoError(t, err)
- require.Equal(t, 3, first.NMTSegments)
-
- // Now the same $2 cap passes: nothing is left to pay for. Before the
- // projection consulted the memory, this run was refused for a bill of $3 it
- // would never have been sent — which is what a resume after a rate limit
- // looks like.
- second, err := mk(2).Run(t.Context(), cfg("second"))
- require.NoError(t, err)
- require.Equal(t, 3, second.TMHits)
- require.Zero(t, second.NMTSegments)
- require.InDelta(t, 0.0, second.ActualCostUSD, 1e-9)
-
- // And the report still knows what the book would have cost without the
- // memory, which is the only way the savings figure means anything.
- require.InDelta(t, 3.0, second.EstimatedCostUSD, 1e-9)
- require.InDelta(t, 3.0, second.CostSavingsUSD, 1e-9)
-}
-
-func TestOrchestrator_Run_ADraftIsReusableOnlyForTheModeThatOutputsIt(t *testing.T) {
- t.Parallel()
-
- srv := nmttest.NewServer(nil)
- t.Cleanup(srv.Close)
-
- store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
- require.NoError(t, err)
- t.Cleanup(func() { _ = store.Close() })
-
- orch := &pipeline.Orchestrator{
- Extractor: epub.ZipExtractor{},
- Parser: epub.OPFParser{},
- Segmenter: segment.NewParagraphSegmenter(),
- TM: store,
- Translator: translate.NewHTTPTranslator(srv.URL, ""),
- Recomposer: epub.EPUBRecomposer{},
- EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"},
- CostModel: cost.Model{PerSegmentUSD: 1},
- BudgetUSD: 2, // the whole book is $3, so only reuse can let it start
- }
- input := writeFixtureEPUB(t)
-
- // An NMT run fills the memory with drafts.
- _, err = orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "nmt"),
- SourceLang: "en", TargetLang: "fr", RunID: "nmt", BookID: "b",
- Mode: pipeline.ModeNMT,
- })
- require.ErrorIs(t, err, cost.ErrBudgetExceeded, "nothing cached yet: refused")
-
- orch.BudgetUSD = 10
- _, err = orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "nmt2"),
- SourceLang: "en", TargetLang: "fr", RunID: "nmt2", BookID: "b",
- })
- require.NoError(t, err)
-
- // NMT mode reuses those drafts, so the tight cap now passes.
- orch.BudgetUSD = 2
- reused, err := orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "nmt3"),
- SourceLang: "en", TargetLang: "fr", RunID: "nmt3", BookID: "b",
- })
- require.NoError(t, err)
- require.Equal(t, 3, reused.TMHits)
-
- // Literary mode does not: a draft is an intermediate it has to redo, so the
- // same cached book is projected at full price and the cap refuses it. The
- // projection agreeing with lookupTranslation on this is the whole point —
- // a projection that counted drafts would wave through a run that then pays
- // for every segment.
- orch.NarrativeStore = newNarrativeStore(t)
- require.NoError(t, orch.NarrativeStore.Save(t.Context(), narrative.Bible{
- SeriesID: "series-a",
- StyleGuide: "Français littéraire souple.",
- Summaries: []narrative.Summary{{VolumeID: "volume-01", Summary: "Synopsis."}},
- }))
- _, err = orch.Run(t.Context(), pipeline.RunConfig{
- InputPath: input, OutputDir: filepath.Join(t.TempDir(), "lit"),
- SourceLang: "en", TargetLang: "fr", RunID: "lit", BookID: "b",
- Mode: pipeline.ModeLiterary, SeriesID: "series-a",
- })
- require.ErrorIs(t, err, cost.ErrBudgetExceeded)
-}
diff --git a/internal/pipeline/ports.go b/internal/pipeline/ports.go
new file mode 100644
index 0000000..da6b628
--- /dev/null
+++ b/internal/pipeline/ports.go
@@ -0,0 +1,68 @@
+package pipeline
+
+import (
+ "context"
+ "time"
+
+ "github.com/ECecillo/tomeshift/internal/qa"
+)
+
+// Steps is what the control flow can ask an engine to run. There is one method
+// per activity in the RFC table, and none of them takes a context: the adapter
+// owns it (a Temporal workflow.Context is not a context.Context, and the
+// in-process adapter carries a plain one). Inputs and outputs are the small,
+// serialisable DTOs from types.go — no live object crosses this boundary.
+type Steps interface {
+ PrepareBook(BookInput) (Manifest, error)
+ ProjectCost(ProjectInput) (Projection, error)
+ ResolveChapter(ResolveInput) (ChapterPlan, error)
+ DraftChapter(DraftInput) (DraftResult, error)
+ EscalateChapter(EscalateInput) (EscalateResult, error)
+ RefineBatch(RefineInput) (RefineResult, error)
+ CommitChapter(CommitInput) (CommitResult, error)
+ UpdateBible(BibleInput) (BibleResult, error)
+ RecomposeBook(RunRef) (RecomposeResult, error)
+ CheckEPUB(CheckInput) (CheckResult, error)
+ FinishBook(FinishInput) (qa.Report, error)
+ RecordSpend(SpendInput) error
+ PriceRun(PriceInput) (float64, error)
+}
+
+// Runtime is what the control flow needs from an engine, and nothing more. The
+// in-process implementation backs it with time.Now, channels and a plain
+// function call; the Temporal one with workflow timers, signals, disconnected
+// contexts and child workflows.
+type Runtime interface {
+ // Now is the flow's only clock, so the same function replays deterministically.
+ Now() time.Time
+ // WaitResume blocks for d or until a resume signal, whichever comes first.
+ // It is the rate-limit cooldown; the phase-1 flow does not call it, but the
+ // runtime implements and tests it so phase 4 is a flow change only.
+ WaitResume(d time.Duration) (resumed bool, err error)
+ // Paused reports whether a pause signal is pending.
+ Paused() bool
+ // Cancelled reports whether the run has been cancelled.
+ Cancelled() bool
+ // Detached returns a runtime whose cleanup steps run even after cancellation
+ // — the engine-agnostic form of a disconnected context.
+ Detached() Runtime
+ // Report publishes a progress snapshot. The in-process runtime drives the
+ // terminal bar; the Temporal one updates the value its query serves.
+ Report(Progress)
+ // RunBook runs one volume; RunSeries uses it to run its children.
+ RunBook(BookInput) (RunResult, error)
+}
+
+// Reporter receives Progress snapshots. The CLI implements it to drive the
+// terminal bar; tests record the sequence a watcher would have seen. A nil
+// Reporter disables reporting, so the runtime calls it unconditionally.
+type Reporter interface {
+ Report(Progress)
+}
+
+// QualityEstimator scores a translated segment without a reference (e.g.
+// COMET-QE). Scores are higher-is-better; the pipeline uses them to drive
+// selective LLM escalation and to report per-book quality.
+type QualityEstimator interface {
+ Estimate(ctx context.Context, source, candidate string) (float64, error)
+}
diff --git a/internal/pipeline/progress.go b/internal/pipeline/progress.go
deleted file mode 100644
index b7da770..0000000
--- a/internal/pipeline/progress.go
+++ /dev/null
@@ -1,46 +0,0 @@
-package pipeline
-
-// Reporter renders a run's advancement for a human watching the terminal. It is
-// optional — a nil Reporter disables reporting — and it is never a data source:
-// report.json and logs.ndjson remain the machine-readable record, so a Reporter
-// may drop, throttle or reorder anything it is told.
-//
-// Implementations must tolerate a run that never sizes itself: Plan segments a
-// book without translating it, and a failed run stops mid-count.
-type Reporter interface {
- // Stage announces the stage now starting.
- Stage(name string)
- // Total sizes the work, once the book has been segmented.
- Total(segments, chapters int)
- // Chapter reports the 1-based chapter now being worked on. Only the
- // literary pipeline walks chapters.
- Chapter(index int)
- // Advance records segments finished since the last call, split between
- // fresh translations and TM reuse.
- Advance(fresh, reused int)
- // Cost reports what the run has spent so far, in USD. It is priced by the
- // same mode that prices report.ActualCostUSD, so the last figure a watcher
- // sees is the one the report carries.
- Cost(usd float64)
- // Finish terminates the report. It must be idempotent: it is deferred on
- // every exit path.
- Finish()
-}
-
-// nopReporter is the default, so the orchestrator calls the reporter
-// unconditionally instead of guarding every call site with a nil check.
-type nopReporter struct{}
-
-func (nopReporter) Stage(string) {}
-func (nopReporter) Total(int, int) {}
-func (nopReporter) Chapter(int) {}
-func (nopReporter) Advance(int, int) {}
-func (nopReporter) Cost(float64) {}
-func (nopReporter) Finish() {}
-
-func (o *Orchestrator) progress() Reporter {
- if o.Progress != nil {
- return o.Progress
- }
- return nopReporter{}
-}
diff --git a/internal/pipeline/series.go b/internal/pipeline/series.go
new file mode 100644
index 0000000..f1f186d
--- /dev/null
+++ b/internal/pipeline/series.go
@@ -0,0 +1,50 @@
+package pipeline
+
+// SeriesHooks lets the caller wire per-book concerns the flow itself does not
+// own — the per-book logger and progress bar (Prepare), the reference-based
+// score (Score), and any teardown or printing (Done) — around the sequential
+// run RunSeries drives. All are optional.
+type SeriesHooks struct {
+ // Prepare runs before each book, on the shared Stages/Runtime: swap the
+ // per-book logger and context, reset the bar.
+ Prepare func(in BookInput)
+ // Score computes a reference-based score for a produced book; nil disables.
+ Score func(bookID, outputEPUB string) (float64, bool)
+ // Done runs after each book, with its outcome and the raw run error (nil on
+ // success), so the caller can map the failure to an exit code and print.
+ Done func(in BookInput, outcome BookOutcome, err error)
+}
+
+// RunSeries translates a series of volumes in order through one shared engine,
+// mirroring the batch command's semantics: it continues on a per-book failure
+// and aggregates every outcome into a BatchSummary. Sequential is not an
+// accident — the shared TM's single writer and (for literary) the bible's
+// chapter order both forbid running volumes in parallel.
+func RunSeries(steps Steps, rt Runtime, books []BookInput, hooks SeriesHooks) BatchSummary {
+ outcomes := make([]BookOutcome, 0, len(books))
+ for _, book := range books {
+ if rt.Cancelled() {
+ break
+ }
+ if hooks.Prepare != nil {
+ hooks.Prepare(book)
+ }
+
+ res, err := rt.RunBook(book)
+ result := res
+ outcome := BookOutcome{BookID: book.BookID, RunID: book.RunID, OK: err == nil, Result: &result}
+ if err != nil {
+ outcome.Error = err.Error()
+ } else if hooks.Score != nil {
+ if score, ok := hooks.Score(book.BookID, res.OutputEPUB); ok {
+ outcome.ChrF = &score
+ }
+ }
+
+ if hooks.Done != nil {
+ hooks.Done(book, outcome, err)
+ }
+ outcomes = append(outcomes, outcome)
+ }
+ return NewBatchSummary(outcomes)
+}
diff --git a/internal/pipeline/stages.go b/internal/pipeline/stages.go
new file mode 100644
index 0000000..d340987
--- /dev/null
+++ b/internal/pipeline/stages.go
@@ -0,0 +1,1189 @@
+package pipeline
+
+import (
+ "context"
+ "fmt"
+ "strings"
+ "time"
+ "unicode/utf8"
+
+ "go.uber.org/zap"
+
+ "github.com/ECecillo/tomeshift/internal/cost"
+ "github.com/ECecillo/tomeshift/internal/epub"
+ "github.com/ECecillo/tomeshift/internal/glossary"
+ "github.com/ECecillo/tomeshift/internal/literary"
+ "github.com/ECecillo/tomeshift/internal/llm"
+ "github.com/ECecillo/tomeshift/internal/narrative"
+ "github.com/ECecillo/tomeshift/internal/qa"
+ "github.com/ECecillo/tomeshift/internal/segment"
+ "github.com/ECecillo/tomeshift/internal/tm"
+ "github.com/ECecillo/tomeshift/internal/translate"
+)
+
+// ledgerWriteTimeout bounds the detached write used when a run is recorded
+// after its context was cancelled.
+const ledgerWriteTimeout = 5 * time.Second
+
+// Stages holds the injected collaborators the run needs and exposes one method
+// per activity in the RFC table. Each method takes a real context.Context, reads
+// and writes the workspace, and returns a small serialisable value — it is the
+// activity body a Temporal worker will register unchanged. The control flow
+// (RunBook) never touches these fields; it reaches them only through the Steps
+// port.
+type Stages struct {
+ Extractor epub.Extractor
+ Parser epub.Parser
+ Segmenter segment.Segmenter
+ Placeholders segment.PlaceholderManager
+ TM tm.Store
+ Translator translate.Translator
+ Recomposer epub.Recomposer
+ EPUBCheck qa.EPUBChecker
+ Glossary *glossary.Glossary // optional; nil disables locked-term enforcement
+ PostEditor llm.PostEditor // optional; nil disables selective LLM post-edit
+ LiteraryTranslator *literary.Translator
+ NarrativeStore narrative.Store
+ NarrativeProfiler *narrative.Profiler
+ EscalateMinChars int // escalate NMT segments with >= this many source chars (0 = all, when PostEditor set)
+ QualityEstimator QualityEstimator // optional; reference-free QE for escalation + reporting
+ EscalateBelowScore float64 // escalate fresh segments whose QE score < this (0 disables)
+ CostModel cost.Model // zero-value = free local provider
+ BudgetUSD float64 // per-book hard cap; <= 0 disables the check
+ DailyUSD float64 // daily spend cap; <= 0 disables the check
+ MonthlyUSD float64 // monthly spend cap; <= 0 disables the check
+ Logger *zap.Logger
+ // LLMProvider and LLMModel name what the LLM pass talks to. The pipeline
+ // does no routing with them — the clients are already built — it records
+ // them, because a measured token count is only reusable if you know what
+ // produced it.
+ LLMProvider string
+ LLMModel string
+}
+
+// chapterSegments is a chapter's segments in spine order — the in-memory shape
+// the stage bodies rebuild from the workspace or fresh segmentation.
+type chapterSegments struct {
+ ID string
+ Href string
+ Segments []segment.Segment
+}
+
+// ---------------------------------------------------------------------------
+// Activity bodies.
+// ---------------------------------------------------------------------------
+
+// PrepareBook extracts, parses and segments the book, writes one unit file per
+// chapter to the workspace, and returns the chapter manifest plus the run-scoped
+// identifiers the flow stamps onto the result but cannot learn otherwise.
+func (s *Stages) PrepareBook(ctx context.Context, in BookInput) (Manifest, error) {
+ raw, err := s.Extractor.Extract(ctx, in.InputPath)
+ if err != nil {
+ return Manifest{}, &StageError{Stage: StageExtract, Err: err}
+ }
+ book, err := s.Parser.Parse(ctx, raw)
+ if err != nil {
+ return Manifest{}, &StageError{Stage: StageParse, Err: err}
+ }
+ chapters, err := s.segmentBookByChapter(ctx, book)
+ if err != nil {
+ return Manifest{}, &StageError{Stage: StageSegment, Err: err}
+ }
+
+ ws := NewWorkspace(in.OutputDir)
+ infos := make([]ChapterInfo, 0, len(chapters))
+ for i := range chapters {
+ ch := &chapters[i]
+ units := make([]segment.Unit, len(ch.Segments))
+ for j := range ch.Segments {
+ units[j] = ch.Segments[j].Unit()
+ }
+ if err := ws.WriteChapter(ch.ID, units); err != nil {
+ return Manifest{}, &StageError{Stage: StageSegment, Err: err}
+ }
+ infos = append(infos, ChapterInfo{
+ ID: ch.ID,
+ Href: ch.Href,
+ Segments: len(ch.Segments),
+ Chars: segment.SourceChars(ch.Segments),
+ })
+ }
+
+ s.logger().Info("segmented book",
+ zap.Int("segments", countChapterInfos(infos)), zap.Int("chapters", len(infos)))
+
+ return Manifest{
+ Chapters: infos,
+ NMTEngine: s.nmtEngine(),
+ LLMProvider: s.LLMProvider,
+ LLMModel: s.LLMModel,
+ BatchSize: s.literaryBatchSize(),
+ }, nil
+}
+
+// ProjectCost is the pre-run budget guardrail: it prices the pending work and
+// the whole book with no reuse, and refuses the run before any provider call if
+// a per-book or rolling cap is exceeded. NoReuseUSD is returned even on the
+// refusal so the flow can still stamp the report's estimated cost.
+func (s *Stages) ProjectCost(ctx context.Context, in ProjectInput) (Projection, error) {
+ chapters, err := s.readChapters(in.Ref)
+ if err != nil {
+ return Projection{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ now := time.Now().UTC()
+ usedFreeChars, err := s.freeCharsUsed(ctx, now)
+ if err != nil {
+ return Projection{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ var bible narrative.Bible
+ if in.Mode == ModeLiterary {
+ bible, err = s.loadBible(ctx, in.SeriesID)
+ if err != nil {
+ return Projection{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ }
+
+ reusesDrafts := in.Mode != ModeLiterary
+ pending, reusable, err := s.pendingChapters(ctx, chapters, reusesDrafts)
+ if err != nil {
+ return Projection{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ enforced := s.projectMode(in.Mode, pending, bible, usedFreeChars)
+ noReuse := s.projectMode(in.Mode, chapters, bible, usedFreeChars)
+
+ proj := Projection{
+ EnforcedUSD: enforced.EstimatedUSD,
+ NoReuseUSD: noReuse.EstimatedUSD,
+ Pending: countSegments(pending),
+ Reusable: reusable,
+ UsedFreeChars: usedFreeChars,
+ BibleVersion: bible.Version,
+ }
+
+ if reusable > 0 {
+ s.logger().Info("projecting the remaining work",
+ zap.Int("reusable_segments", reusable),
+ zap.Int("pending_segments", proj.Pending))
+ }
+ s.logger().Info("projected run cost",
+ zap.Float64("enforced_usd", enforced.EstimatedUSD),
+ zap.Float64("no_reuse_usd", noReuse.EstimatedUSD),
+ zap.Int("llm_calls", enforced.LLMCalls))
+
+ if s.BudgetUSD > 0 && enforced.EstimatedUSD > s.BudgetUSD {
+ s.logger().Warn("projected spend exceeds per-book budget cap",
+ zap.Float64("estimated_usd", enforced.EstimatedUSD),
+ zap.Float64("budget_usd", s.BudgetUSD))
+ return proj, fmt.Errorf("unable to start run (estimated $%.4f > per-book cap $%.4f), err: %w",
+ enforced.EstimatedUSD, s.BudgetUSD, cost.ErrBudgetExceeded)
+ }
+ if err := s.checkRollingBudgets(ctx, enforced.EstimatedUSD, now); err != nil {
+ return proj, err
+ }
+ return proj, nil
+}
+
+// ResolveChapter serves a chapter's segments from the TM, writing every hit's
+// target onto its unit and returning the reuse split. A draft is a hit only when
+// the mode treats a draft as output (ReusesDrafts).
+func (s *Stages) ResolveChapter(ctx context.Context, in ResolveInput) (ChapterPlan, error) {
+ ws := NewWorkspace(in.Ref.OutputDir)
+ units, err := ws.ReadChapter(in.ChapterID)
+ if err != nil {
+ return ChapterPlan{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ plan := ChapterPlan{}
+ for i := range units {
+ u := &units[i]
+ hit := false
+ if s.TM != nil {
+ entry, found, err := s.TM.Lookup(ctx, u.Hash)
+ if err != nil {
+ return ChapterPlan{}, &StageError{Stage: StageTranslate,
+ Err: fmt.Errorf("unable to look up TM for %s, err: %w", u.ID, err)}
+ }
+ if found && (in.ReusesDrafts || !tm.IsDraftEngine(entry.Engine)) {
+ u.TargetText = entry.TargetText
+ u.Status = segment.StatusReused
+ hit = true
+ }
+ }
+ if hit {
+ plan.Reused++
+ } else {
+ plan.Fresh = append(plan.Fresh, u.ID)
+ }
+ }
+
+ if err := ws.WriteChapter(in.ChapterID, units); err != nil {
+ return ChapterPlan{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return plan, nil
+}
+
+// DraftChapter drafts a chapter's TM misses in one batched provider call,
+// protecting glossary terms around it, and persists the drafts only when the
+// mode treats them as output. It returns the source characters it actually sent
+// (retries included) so the flow can total NMT consumption.
+func (s *Stages) DraftChapter(ctx context.Context, in DraftInput) (DraftResult, error) {
+ if len(in.Fresh) == 0 {
+ return DraftResult{Engine: s.nmtEngine()}, nil
+ }
+
+ ws := NewWorkspace(in.Ref.OutputDir)
+ units, err := ws.ReadChapter(in.ChapterID)
+ if err != nil {
+ return DraftResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ fresh := selectUnits(units, in.Fresh)
+ texts := make([]string, len(fresh))
+ subs := make([]map[string]string, len(fresh))
+ for i, u := range fresh {
+ texts[i] = u.SourceText
+ if s.Glossary != nil {
+ texts[i], subs[i] = s.Glossary.Protect(u.SourceText)
+ }
+ }
+
+ before := s.charsSent()
+ resps, err := s.Translator.TranslateBatch(ctx, translate.BatchRequest{
+ Texts: texts,
+ SourceLang: in.SourceLang,
+ TargetLang: in.TargetLang,
+ })
+ if err != nil {
+ return DraftResult{}, &StageError{Stage: StageTranslate,
+ Err: fmt.Errorf("unable to draft chapter %s, err: %w", in.ChapterID, err)}
+ }
+ if len(resps) != len(texts) {
+ return DraftResult{}, &StageError{Stage: StageTranslate,
+ Err: fmt.Errorf("unable to draft chapter %s: got %d drafts for %d segments, err: %w",
+ in.ChapterID, len(resps), len(texts), translate.ErrShortBatchResponse)}
+ }
+
+ engine := s.nmtEngine()
+ for i, resp := range resps {
+ text := resp.Text
+ if s.Glossary != nil {
+ text = s.Glossary.Apply(text, subs[i])
+ }
+ fresh[i].TargetText = text
+ fresh[i].Status = segment.StatusTranslated
+ if resp.Engine != "" {
+ engine = resp.Engine
+ }
+ if in.PersistDrafts && s.TM != nil {
+ if err := s.TM.Save(ctx, tm.Entry{
+ SourceHash: fresh[i].Hash, SourceText: fresh[i].SourceText, TargetText: text,
+ SourceLang: in.SourceLang, TargetLang: in.TargetLang, Engine: resp.Engine,
+ }); err != nil {
+ return DraftResult{}, &StageError{Stage: StageTranslate,
+ Err: fmt.Errorf("unable to persist TM for %s, err: %w", fresh[i].ID, err)}
+ }
+ }
+ }
+ chars := s.charsSent() - before
+
+ if err := ws.WriteChapter(in.ChapterID, units); err != nil {
+ return DraftResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return DraftResult{Chars: chars, Engine: engine}, nil
+}
+
+// EscalateChapter runs the NMT-mode quality estimation over a chapter's fresh
+// drafts and selectively post-edits the weakest with the LLM. SafePostEdit falls
+// back to the draft on any error or placeholder change, so the output can never
+// be degraded by escalation.
+func (s *Stages) EscalateChapter(ctx context.Context, in EscalateInput) (EscalateResult, error) {
+ if len(in.Fresh) == 0 || (s.PostEditor == nil && s.QualityEstimator == nil) {
+ return EscalateResult{}, nil
+ }
+
+ ws := NewWorkspace(in.Ref.OutputDir)
+ units, err := ws.ReadChapter(in.ChapterID)
+ if err != nil {
+ return EscalateResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ fresh := selectUnits(units, in.Fresh)
+
+ var res EscalateResult
+ for _, u := range fresh {
+ target := u.TargetText
+
+ var qeScore float64
+ haveQE := false
+ if s.QualityEstimator != nil {
+ if score, err := s.QualityEstimator.Estimate(ctx, u.SourceText, target); err == nil {
+ qeScore, haveQE = score, true
+ res.QESum += score
+ res.QECount++
+ } else {
+ s.logger().Warn("quality estimation failed", zap.String("segment_id", u.ID), zap.Error(err))
+ }
+ }
+
+ if !s.shouldEscalate(u.SourceText, qeScore, haveQE) {
+ continue
+ }
+
+ edited, postEdited := llm.SafePostEdit(ctx, s.PostEditor, llm.PostEditRequest{
+ SourceText: u.SourceText,
+ NMTText: target,
+ SourceLang: in.SourceLang,
+ TargetLang: in.TargetLang,
+ })
+ res.Attempts++
+ res.Chars += utf8.RuneCountInString(u.SourceText)
+
+ if postEdited {
+ u.TargetText = edited
+ u.Status = segment.StatusPostEdited
+ res.Improved++
+ if s.TM != nil {
+ if err := s.TM.Save(ctx, tm.Entry{
+ SourceHash: u.Hash, SourceText: u.SourceText, TargetText: edited,
+ SourceLang: in.SourceLang, TargetLang: in.TargetLang, Engine: tm.EngineLLMPostEdit,
+ }); err != nil {
+ s.logger().Warn("unable to persist post-edited TM entry",
+ zap.String("segment_id", u.ID), zap.Error(err))
+ }
+ }
+ }
+ }
+
+ if err := ws.WriteChapter(in.ChapterID, units); err != nil {
+ return EscalateResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return res, nil
+}
+
+// RefineBatch rewrites one literary batch with the LLM against the series bible,
+// halving on truncation inside the single call. It writes the refined targets
+// onto their units and returns the tokens consumed, how many it improved, the
+// ids it left on their draft, and the French tail fed to the next batch.
+func (s *Stages) RefineBatch(ctx context.Context, in RefineInput) (RefineResult, error) {
+ if s.LiteraryTranslator == nil {
+ return RefineResult{}, &StageError{Stage: StageTranslate, Err: literary.ErrNilChatClient}
+ }
+ if len(in.IDs) == 0 {
+ return RefineResult{}, nil
+ }
+
+ bible, err := s.loadBible(ctx, in.SeriesID)
+ if err != nil {
+ return RefineResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ ws := NewWorkspace(in.Ref.OutputDir)
+ units, err := ws.ReadChapter(in.ChapterID)
+ if err != nil {
+ return RefineResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ // A chapter's first batch has no in-chapter predecessor, so its preceding
+ // context is the tail of the PREVIOUS chapter's source — read from the
+ // workspace, exactly as the in-process walk did. Later batches carry the
+ // previous batch's translated tail in PrevTail.
+ prevCtx := in.PrevTail
+ if prevCtx == "" && in.PrevChapterID != "" {
+ prevUnits, err := ws.ReadChapter(in.PrevChapterID)
+ if err != nil {
+ return RefineResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ prevCtx = chapterSourceOfUnits(prevUnits)
+ }
+
+ batch := selectUnits(units, in.IDs)
+ segs := make([]*segment.Segment, len(batch))
+ for i, u := range batch {
+ seg := unitToSegment(*u)
+ segs[i] = &seg
+ }
+
+ // The chapter's whole source is the cacheable prefix; only the preceding
+ // tail travels per call.
+ outcome, err := s.LiteraryTranslator.TranslateBatch(
+ ctx, bible, in.ChapterID, segs, prevCtx, chapterSourceOfUnits(units))
+ if err != nil {
+ // A rate limit or a transport/credential failure is fatal — the flow
+ // fails the run and the units keep their drafts, unwritten.
+ return RefineResult{Usage: outcome.Usage, Tail: outcome.Tail},
+ &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ res := refineFrom(batch, segs)
+ res.Usage = outcome.Usage
+ res.Tail = outcome.Tail
+ for _, d := range outcome.Degraded {
+ res.Degraded = append(res.Degraded, d.Error())
+ s.logger().Warn("literary batch kept its draft",
+ zap.String("chapter_id", in.ChapterID), zap.Error(d))
+ }
+
+ if err := ws.WriteChapter(in.ChapterID, units); err != nil {
+ return RefineResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return res, nil
+}
+
+// CommitChapter re-derives the chapter's placeholders from the source document,
+// validates each translation by restoring it (a refused restore never aborts —
+// it lands in RestoreFailures and the node keeps its source), counts glossary
+// violations, and persists accepted translations to the TM only when the mode
+// asks for it (literary: after Restore accepted).
+func (s *Stages) CommitChapter(ctx context.Context, in CommitInput) (CommitResult, error) {
+ ws := NewWorkspace(in.Ref.OutputDir)
+ units, err := ws.ReadChapter(in.ChapterID)
+ if err != nil {
+ return CommitResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+
+ var res CommitResult
+ targets := make(map[string]string, len(units))
+ for i := range units {
+ u := &units[i]
+ if u.TargetText != "" {
+ // Normalize to the project's single French typography before any
+ // consumer reads the target: glossary-violation counting, TM save,
+ // segments.jsonl persistence (via WriteChapter) and the EPUB
+ // restore then all agree on the same spacing. It runs after the
+ // glossary Apply that produced this target, so locked terms are
+ // already in place, and it is idempotent.
+ u.TargetText = segment.FrenchTypography(u.TargetText)
+ targets[u.ID] = u.TargetText
+ }
+ if s.Glossary != nil {
+ res.GlossaryViolations += s.Glossary.Violations(u.SourceText, u.TargetText)
+ }
+ }
+
+ doc, err := s.chapterDoc(ctx, in.Ref.InputPath, in.ChapterID)
+ if err != nil {
+ return CommitResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ report, err := segment.ApplyTranslations(ctx, s.Segmenter, s.placeholders(), doc, in.ChapterID, targets)
+ if err != nil {
+ return CommitResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ res.RestoreFailures = len(report.Failed)
+
+ if in.Persist && s.TM != nil {
+ for i := range units {
+ u := &units[i]
+ if u.Status != segment.StatusPostEdited {
+ continue
+ }
+ if _, failed := report.Failed[u.ID]; failed {
+ continue
+ }
+ if err := s.TM.Save(ctx, tm.Entry{
+ SourceHash: u.Hash, SourceText: u.SourceText, TargetText: u.TargetText,
+ SourceLang: in.SourceLang, TargetLang: in.TargetLang, Engine: tm.EngineLiterary,
+ }); err != nil {
+ s.logger().Warn("unable to persist literary TM entry",
+ zap.String("segment_id", u.ID), zap.Error(err))
+ }
+ }
+ }
+
+ if err := ws.WriteChapter(in.ChapterID, units); err != nil {
+ return CommitResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return res, nil
+}
+
+// UpdateBible evolves the series bible from a committed chapter and returns the
+// stored version.
+func (s *Stages) UpdateBible(ctx context.Context, in BibleInput) (BibleResult, error) {
+ if s.NarrativeProfiler == nil {
+ bible, err := s.loadBible(ctx, in.SeriesID)
+ if err != nil {
+ return BibleResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return BibleResult{Version: bible.Version}, nil
+ }
+
+ bible, err := s.loadBible(ctx, in.SeriesID)
+ if err != nil {
+ return BibleResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ if err := s.updateBible(ctx, in.Ref, in.SeriesID, &bible, in.ChapterID); err != nil {
+ return BibleResult{}, &StageError{Stage: StageTranslate, Err: err}
+ }
+ return BibleResult{Version: bible.Version}, nil
+}
+
+// RecomposeBook re-parses the source, applies every chapter's committed targets
+// to a fresh node tree and renders the translated EPUB.
+func (s *Stages) RecomposeBook(ctx context.Context, ref RunRef) (RecomposeResult, error) {
+ raw, err := s.Extractor.Extract(ctx, ref.InputPath)
+ if err != nil {
+ return RecomposeResult{}, &StageError{Stage: StageRecompose, Err: err}
+ }
+ book, err := s.Parser.Parse(ctx, raw)
+ if err != nil {
+ return RecomposeResult{}, &StageError{Stage: StageRecompose, Err: err}
+ }
+
+ ws := NewWorkspace(ref.OutputDir)
+ for _, ch := range book.Chapters() {
+ units, err := ws.ReadChapter(ch.ID)
+ if err != nil {
+ return RecomposeResult{}, &StageError{Stage: StageRecompose, Err: err}
+ }
+ targets := make(map[string]string, len(units))
+ for i := range units {
+ if units[i].TargetText != "" {
+ // Normalize here too so recomposing a run committed before this
+ // pass existed still emits the canonical French typography.
+ // Idempotent, so re-normalizing freshly committed units is a
+ // no-op.
+ targets[units[i].ID] = segment.FrenchTypography(units[i].TargetText)
+ }
+ }
+ if _, err := segment.ApplyTranslations(ctx, s.Segmenter, s.placeholders(), ch.Doc, ch.ID, targets); err != nil {
+ return RecomposeResult{}, &StageError{Stage: StageRecompose, Err: err}
+ }
+ }
+
+ outputEPUB := fmt.Sprintf("%s/translated.epub", ref.OutputDir)
+ if err := s.Recomposer.Recompose(ctx, raw, book, outputEPUB); err != nil {
+ return RecomposeResult{}, &StageError{Stage: StageRecompose, Err: err}
+ }
+ return RecomposeResult{OutputEPUB: outputEPUB}, nil
+}
+
+// CheckEPUB runs epubcheck on the produced EPUB. A skipped check (binary absent)
+// does not block the run.
+func (s *Stages) CheckEPUB(ctx context.Context, in CheckInput) (CheckResult, error) {
+ check, err := s.EPUBCheck.Check(ctx, in.OutputEPUB)
+ if err != nil {
+ return CheckResult{}, &StageError{Stage: StageQA, Err: err}
+ }
+ return CheckResult{Passed: check.Passed, Skipped: check.Skipped}, nil
+}
+
+// FinishBook evaluates the static QA checks over the committed units and writes
+// report.json, metrics.json and segments.jsonl. It re-segments the source to
+// recover each segment's placeholders and node path, then overlays the target
+// text and status the run produced. The EPUB-check verdict is carried in the
+// result the flow passes in.
+func (s *Stages) FinishBook(ctx context.Context, in FinishInput) (qa.Report, error) {
+ if in.Ref.InputPath != "" && ctx.Err() != nil {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), ledgerWriteTimeout)
+ defer cancel()
+ }
+
+ raw, err := s.Extractor.Extract(ctx, in.Ref.InputPath)
+ if err != nil {
+ return qa.Report{}, &StageError{Stage: StageQA, Err: err}
+ }
+ book, err := s.Parser.Parse(ctx, raw)
+ if err != nil {
+ return qa.Report{}, &StageError{Stage: StageQA, Err: err}
+ }
+
+ ws := NewWorkspace(in.Ref.OutputDir)
+ var segments []segment.Segment
+ for _, ch := range book.Chapters() {
+ segs, err := s.Segmenter.Segment(ctx, ch.Doc, ch.ID)
+ if err != nil {
+ return qa.Report{}, &StageError{Stage: StageQA, Err: err}
+ }
+ units, err := ws.ReadChapter(ch.ID)
+ if err != nil {
+ return qa.Report{}, &StageError{Stage: StageQA, Err: err}
+ }
+ byID := make(map[string]segment.Unit, len(units))
+ for i := range units {
+ byID[units[i].ID] = units[i]
+ }
+ for i := range segs {
+ if u, ok := byID[segs[i].ID]; ok {
+ segs[i].TargetText = u.TargetText
+ segs[i].Status = u.Status
+ }
+ }
+ segments = append(segments, segs...)
+ }
+
+ report := qa.Evaluate(book, segments)
+ report.EPUBCheckPassed = in.Result.QA.EPUBCheckPassed
+ report.EPUBCheckSkipped = in.Result.QA.EPUBCheckSkipped
+
+ result := in.Result
+ result.QA = report
+ if err := WriteArtifacts(in.Ref, &result, segments); err != nil {
+ return qa.Report{}, &StageError{Stage: StageQA, Err: err}
+ }
+ return report, nil
+}
+
+// RecordSpend persists what the run consumed, for rolling-budget and free-tier
+// accounting. It fires on failure paths too — the provider does not refund a run
+// that died after it was called — and detaches from a cancelled context so the
+// record survives. Best effort: a ledger write failure never fails the run.
+func (s *Stages) RecordSpend(ctx context.Context, in SpendInput) error {
+ if s.TM == nil {
+ return nil
+ }
+
+ result := in.Result
+ if result.EndedAt.IsZero() {
+ result.EndedAt = time.Now().UTC()
+ }
+
+ if ctx.Err() != nil {
+ var cancel context.CancelFunc
+ ctx, cancel = context.WithTimeout(context.WithoutCancel(ctx), ledgerWriteTimeout)
+ defer cancel()
+ }
+
+ if err := s.TM.RecordRun(ctx, tm.RunSpend{
+ RunID: in.Ref.RunID,
+ BookID: in.Ref.BookID,
+ StartedAt: result.StartedAt,
+ EndedAt: result.EndedAt,
+ ActualCostUSD: result.ActualCostUSD,
+ NMTEngine: result.NMTEngine,
+ NMTChars: result.NMTChars,
+ LLMProvider: result.LLMProvider,
+ LLMModel: result.LLMModel,
+ LLMSegments: result.LLMPassSegments,
+ LLMInputTokens: result.LLMInputTokens,
+ LLMOutputTokens: result.LLMOutputTokens,
+ LLMCacheReadTokens: result.LLMCacheReadTokens,
+ LLMCacheCreationTokens: result.LLMCacheCreationTokens,
+ }); err != nil {
+ s.logger().Warn("unable to record run spend", zap.Error(err))
+ }
+ return nil
+}
+
+// PriceRun prices what the result says has been consumed so far through the
+// mode's cost model. It is pure arithmetic over the counters, callable at any
+// point including after a failure.
+func (s *Stages) PriceRun(_ context.Context, in PriceInput) (float64, error) {
+ if in.Mode == ModeLiterary {
+ return s.costLiterary(in.Result, in.UsedFreeChars), nil
+ }
+ return s.costNMT(in.Result, in.UsedFreeChars), nil
+}
+
+// ---------------------------------------------------------------------------
+// Shared helpers.
+// ---------------------------------------------------------------------------
+
+// nmtEngine identifies the provider whose allowance a run consumes. It is empty
+// when no translator is configured, which is the case for a dry run.
+func (s *Stages) nmtEngine() string {
+ if s.Translator == nil {
+ return ""
+ }
+ return s.Translator.Engine()
+}
+
+// charsSent reads the adapter's running total of characters actually put into
+// provider requests, or 0 from one that does not meter itself. Only the adapter
+// knows what really went out: retried attempts, glossary-protected text whose
+// length differs from the source, and discarded responses.
+func (s *Stages) charsSent() int {
+ meter, ok := s.Translator.(translate.Meter)
+ if !ok {
+ return 0
+ }
+ return meter.CharsSent()
+}
+
+func (s *Stages) placeholders() segment.PlaceholderManager {
+ if s.Placeholders != nil {
+ return s.Placeholders
+ }
+ return segment.InlinePlaceholderManager{}
+}
+
+func (s *Stages) logger() *zap.Logger {
+ if s.Logger != nil {
+ return s.Logger
+ }
+ return zap.NewNop()
+}
+
+// literaryBatchSize is the batch size the flow chunks fresh literary segments
+// by; 0 outside literary mode.
+func (s *Stages) literaryBatchSize() int {
+ if s.LiteraryTranslator == nil {
+ return 0
+ }
+ return s.LiteraryTranslator.EffectiveBatchSize()
+}
+
+// shouldEscalate decides whether a freshly translated segment is sent to the LLM
+// post-editor. The heuristic (source length threshold) is a configurable proxy
+// for "complex/high-value" and is intentionally simple.
+func (s *Stages) shouldEscalate(source string, qeScore float64, haveQE bool) bool {
+ if s.PostEditor == nil {
+ return false
+ }
+ if haveQE && s.EscalateBelowScore > 0 {
+ return qeScore < s.EscalateBelowScore
+ }
+ if s.EscalateMinChars <= 0 {
+ return true
+ }
+ return utf8.RuneCountInString(source) >= s.EscalateMinChars
+}
+
+func (s *Stages) segmentBookByChapter(ctx context.Context, book *epub.Book) ([]chapterSegments, error) {
+ bookChapters := book.Chapters()
+ chapters := make([]chapterSegments, 0, len(bookChapters))
+ for _, chapter := range bookChapters {
+ segs, err := s.Segmenter.Segment(ctx, chapter.Doc, chapter.ID)
+ if err != nil {
+ return nil, fmt.Errorf("unable to segment %q, err: %w", chapter.Href, err)
+ }
+ chapters = append(chapters, chapterSegments{ID: chapter.ID, Href: chapter.Href, Segments: segs})
+ }
+ return chapters, nil
+}
+
+// readChapters rebuilds the workspace's chapters as minimal segments (source
+// text and hash), enough for the projection and the reuse lookup, without
+// re-parsing the source.
+func (s *Stages) readChapters(ref RunRef) ([]chapterSegments, error) {
+ ws := NewWorkspace(ref.OutputDir)
+ infos, err := s.chapterOrder(ref)
+ if err != nil {
+ return nil, err
+ }
+ out := make([]chapterSegments, 0, len(infos))
+ for _, info := range infos {
+ units, err := ws.ReadChapter(info.ID)
+ if err != nil {
+ return nil, err
+ }
+ segs := make([]segment.Segment, len(units))
+ for i := range units {
+ segs[i] = unitToSegment(units[i])
+ }
+ out = append(out, chapterSegments{ID: info.ID, Href: info.Href, Segments: segs})
+ }
+ return out, nil
+}
+
+// chapterOrder recovers the spine order of the chapters written to the
+// workspace by re-parsing the source. ProjectCost needs the order because the
+// literary projection prices per chapter.
+func (s *Stages) chapterOrder(ref RunRef) ([]ChapterInfo, error) {
+ raw, err := s.Extractor.Extract(context.Background(), ref.InputPath)
+ if err != nil {
+ return nil, err
+ }
+ book, err := s.Parser.Parse(context.Background(), raw)
+ if err != nil {
+ return nil, err
+ }
+ chapters := book.Chapters()
+ infos := make([]ChapterInfo, 0, len(chapters))
+ for _, ch := range chapters {
+ infos = append(infos, ChapterInfo{ID: ch.ID, Href: ch.Href})
+ }
+ return infos, nil
+}
+
+// chapterDoc re-parses the source and returns the named chapter's document, so
+// a stage can re-derive that chapter's placeholders and node tree.
+func (s *Stages) chapterDoc(ctx context.Context, inputPath, chapterID string) (*epub.XHTMLDocument, error) {
+ raw, err := s.Extractor.Extract(ctx, inputPath)
+ if err != nil {
+ return nil, err
+ }
+ book, err := s.Parser.Parse(ctx, raw)
+ if err != nil {
+ return nil, err
+ }
+ for _, ch := range book.Chapters() {
+ if ch.ID == chapterID {
+ return ch.Doc, nil
+ }
+ }
+ return nil, fmt.Errorf("unable to find chapter %s in %q, err: %w", chapterID, inputPath, epub.ErrMissingOPF)
+}
+
+// freeCharsUsed reports how much of this provider's monthly free character
+// allowance the calendar month has already consumed, so the projection bills
+// only the overflow. It fails closed.
+func (s *Stages) freeCharsUsed(ctx context.Context, now time.Time) (int, error) {
+ if s.TM == nil || s.CostModel.FreeCharsPerMonth <= 0 {
+ return 0, nil
+ }
+ since := time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)
+ used, err := s.TM.CharsSince(ctx, since, s.nmtEngine())
+ if err != nil {
+ return 0, fmt.Errorf("unable to read this month's NMT characters, err: %w", err)
+ }
+ return used, nil
+}
+
+// checkRollingBudgets enforces daily and monthly spend caps against the ledger
+// plus this run's projected cost. It fails closed.
+func (s *Stages) checkRollingBudgets(ctx context.Context, projected float64, now time.Time) error {
+ if s.TM == nil {
+ return nil
+ }
+ windows := []struct {
+ name string
+ cap float64
+ since time.Time
+ }{
+ {"daily", s.DailyUSD, time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.UTC)},
+ {"monthly", s.MonthlyUSD, time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC)},
+ }
+ for _, w := range windows {
+ if w.cap <= 0 {
+ continue
+ }
+ spent, err := s.TM.SpendSince(ctx, w.since)
+ if err != nil {
+ return fmt.Errorf("unable to read %s spend, err: %w", w.name, err)
+ }
+ if spent+projected > w.cap {
+ s.logger().Warn("projected spend exceeds rolling budget cap",
+ zap.String("window", w.name),
+ zap.Float64("already_spent_usd", spent),
+ zap.Float64("projected_usd", projected),
+ zap.Float64("cap_usd", w.cap))
+ return fmt.Errorf("unable to start run (%s spend $%.4f + projected $%.4f > cap $%.4f), err: %w",
+ w.name, spent, projected, w.cap, cost.ErrBudgetExceeded)
+ }
+ }
+ return nil
+}
+
+func (s *Stages) loadBible(ctx context.Context, seriesID string) (narrative.Bible, error) {
+ if seriesID == "" {
+ return narrative.Bible{}, narrative.ErrEmptySeriesID
+ }
+ if s.NarrativeStore == nil {
+ return narrative.Bible{}, fmt.Errorf("unable to load required narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
+ }
+ bible, found, err := s.NarrativeStore.Load(ctx, seriesID)
+ if err != nil {
+ return narrative.Bible{}, err
+ }
+ if !found {
+ return narrative.Bible{}, fmt.Errorf("unable to load required narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
+ }
+ if bible.SeriesID != seriesID {
+ return narrative.Bible{}, fmt.Errorf("unable to validate narrative bible %q, err: %w", seriesID, narrative.ErrInvalidBible)
+ }
+ if err := narrative.ValidateBible(bible); err != nil {
+ return narrative.Bible{}, fmt.Errorf("unable to validate narrative bible %q, err: %w", seriesID, err)
+ }
+ return bible, nil
+}
+
+// updateBible re-runs the profiler over a committed chapter and stores the new
+// bible, normalising the model's version to the loaded baseline so the store
+// stays the single assigner of versions.
+func (s *Stages) updateBible(ctx context.Context, ref RunRef, seriesID string, bible *narrative.Bible, chapterID string) error {
+ source, target, err := s.chapterText(ref, chapterID)
+ if err != nil {
+ return err
+ }
+
+ updated, err := s.NarrativeProfiler.Update(ctx, *bible, chapterID, source, target)
+ if err != nil {
+ return err
+ }
+ updated.SeriesID = seriesID
+ updated.Summaries = append([]narrative.Summary(nil), bible.Summaries...)
+ updated.Version = bible.Version
+
+ if s.NarrativeStore != nil {
+ for {
+ saved, err := s.NarrativeStore.SaveIfVersion(ctx, updated)
+ if err != nil {
+ return err
+ }
+ reloaded, found, err := s.NarrativeStore.Load(ctx, seriesID)
+ if err != nil {
+ return err
+ }
+ if !found {
+ return fmt.Errorf("unable to reload updated narrative bible %q, err: %w", seriesID, narrative.ErrMissingBible)
+ }
+ if saved {
+ *bible = reloaded
+ return nil
+ }
+ updated.Version = reloaded.Version
+ updated.Summaries = append([]narrative.Summary(nil), reloaded.Summaries...)
+ }
+ }
+ *bible = updated
+ return nil
+}
+
+// chapterText returns a chapter's source and target prose from the workspace,
+// for the bible profiler.
+func (s *Stages) chapterText(ref RunRef, chapterID string) (source, target string, err error) {
+ ws := NewWorkspace(ref.OutputDir)
+ units, err := ws.ReadChapter(chapterID)
+ if err != nil {
+ return "", "", err
+ }
+ var src, tgt strings.Builder
+ for i := range units {
+ if strings.TrimSpace(units[i].SourceText) != "" {
+ src.WriteString(units[i].SourceText)
+ src.WriteByte('\n')
+ }
+ if strings.TrimSpace(units[i].TargetText) != "" {
+ tgt.WriteString(units[i].TargetText)
+ tgt.WriteByte('\n')
+ }
+ }
+ return strings.TrimSpace(src.String()), strings.TrimSpace(tgt.String()), nil
+}
+
+// ---------------------------------------------------------------------------
+// Cost projection and pricing (per mode).
+// ---------------------------------------------------------------------------
+
+func (s *Stages) projectMode(mode string, chapters []chapterSegments, bible narrative.Bible, usedFreeChars int) cost.Projection {
+ if mode == ModeLiterary {
+ segments := flattenChapters(chapters)
+ return s.CostModel.Project(segments, s.literaryPass(chapters, segments, bible), usedFreeChars)
+ }
+ segments := flattenChapters(chapters)
+ var pass cost.LLMPass
+ if s.PostEditor != nil {
+ pass = postEditPass(len(segments), segment.SourceChars(segments))
+ }
+ return s.CostModel.Project(segments, pass, usedFreeChars)
+}
+
+func (s *Stages) literaryPass(chapters []chapterSegments, segments []segment.Segment, bible narrative.Bible) cost.LLMPass {
+ batchSize, prefixChars := 0, 0
+ if s.LiteraryTranslator != nil {
+ batchSize = s.LiteraryTranslator.EffectiveBatchSize()
+ prefixChars = s.LiteraryTranslator.StablePrefixChars(bible)
+ }
+ counts := make([]int, len(chapters))
+ shapes := make([]cost.ChapterShape, len(chapters))
+ for i := range chapters {
+ counts[i] = len(chapters[i].Segments)
+ shapes[i] = cost.ChapterShape{
+ SourceChars: segment.SourceChars(chapters[i].Segments),
+ Calls: cost.CallsForChapters(counts[i:i+1], batchSize),
+ }
+ }
+ return cost.LLMPass{
+ Segments: len(segments),
+ Chars: segment.SourceChars(segments),
+ Calls: cost.CallsForChapters(counts, batchSize),
+ PrefixChars: prefixChars,
+ Chapters: shapes,
+ ContextRunes: cost.LiteraryContextRunes,
+ }
+}
+
+func (s *Stages) costNMT(result RunResult, usedFreeChars int) float64 {
+ _, _, llmUSD := s.CostModel.Tokens.Project(postEditPass(result.LLMAttempts, result.LLMChars))
+ return s.CostModel.Actual(result.NMTSegments, result.NMTChars, usedFreeChars, cost.TokenUsage{}) + llmUSD
+}
+
+func (s *Stages) costLiterary(result RunResult, usedFreeChars int) float64 {
+ return s.CostModel.Actual(result.NMTSegments, result.NMTChars, usedFreeChars, cost.TokenUsage{
+ InputTokens: result.LLMInputTokens,
+ OutputTokens: result.LLMOutputTokens,
+ CacheReadTokens: result.LLMCacheReadTokens,
+ CacheCreationTokens: result.LLMCacheCreationTokens,
+ })
+}
+
+// postEditPass describes an escalation pass over segments totalling chars. The
+// HTTP post-editor sends one segment per call, so the stable prompt overhead is
+// paid once per segment.
+func postEditPass(segments, chars int) cost.LLMPass {
+ return cost.LLMPass{Segments: segments, Chars: chars, Calls: segments, Uncached: true}
+}
+
+// pendingChapters keeps only the segments this run will pay for — the ones the
+// TM cannot serve — and reports how many it dropped. A draft is reusable only
+// for the mode that treats a draft as output.
+func (s *Stages) pendingChapters(ctx context.Context, chapters []chapterSegments, reusesDrafts bool) ([]chapterSegments, int, error) {
+ if s.TM == nil {
+ return chapters, 0, nil
+ }
+ pending := make([]chapterSegments, 0, len(chapters))
+ reusable := 0
+ for _, chapter := range chapters {
+ kept := make([]segment.Segment, 0, len(chapter.Segments))
+ for _, seg := range chapter.Segments {
+ entry, found, err := s.TM.Lookup(ctx, seg.Hash)
+ if err != nil {
+ return nil, 0, fmt.Errorf("unable to look up TM for %s, err: %w", seg.ID, err)
+ }
+ if found && (reusesDrafts || !tm.IsDraftEngine(entry.Engine)) {
+ reusable++
+ continue
+ }
+ kept = append(kept, seg)
+ }
+ if len(kept) == 0 {
+ continue
+ }
+ chapter.Segments = kept
+ pending = append(pending, chapter)
+ }
+ return pending, reusable, nil
+}
+
+// ---------------------------------------------------------------------------
+// Small pure helpers.
+// ---------------------------------------------------------------------------
+
+func unitToSegment(u segment.Unit) segment.Segment {
+ return segment.Segment{
+ ID: u.ID,
+ ChapterID: u.ChapterID,
+ NodePath: u.NodePath,
+ SourceText: u.SourceText,
+ NormalizedText: u.NormalizedText,
+ Hash: u.Hash,
+ TargetText: u.TargetText,
+ Status: u.Status,
+ }
+}
+
+// selectUnits returns pointers to the units whose ids are in ids, in unit order.
+func selectUnits(units []segment.Unit, ids []string) []*segment.Unit {
+ want := make(map[string]struct{}, len(ids))
+ for _, id := range ids {
+ want[id] = struct{}{}
+ }
+ out := make([]*segment.Unit, 0, len(ids))
+ for i := range units {
+ if _, ok := want[units[i].ID]; ok {
+ out = append(out, &units[i])
+ }
+ }
+ return out
+}
+
+// refineFrom writes the refined targets back onto the batch units and reports
+// how many were improved. The batch's French tail comes from the translator's
+// Outcome, not from here, so it reads the raw model answers rather than the
+// glossary-applied draft fallback.
+func refineFrom(batch []*segment.Unit, segs []*segment.Segment) RefineResult {
+ var res RefineResult
+ for i, seg := range segs {
+ batch[i].TargetText = seg.TargetText
+ batch[i].Status = seg.Status
+ if seg.Status == segment.StatusPostEdited {
+ res.Refined++
+ }
+ }
+ return res
+}
+
+func chapterSourceOfUnits(units []segment.Unit) string {
+ var sb strings.Builder
+ for i := range units {
+ if strings.TrimSpace(units[i].SourceText) == "" {
+ continue
+ }
+ sb.WriteString(units[i].SourceText)
+ sb.WriteByte('\n')
+ }
+ return strings.TrimSpace(sb.String())
+}
+
+func countSegments(chapters []chapterSegments) int {
+ total := 0
+ for i := range chapters {
+ total += len(chapters[i].Segments)
+ }
+ return total
+}
+
+func countChapterInfos(infos []ChapterInfo) int {
+ total := 0
+ for i := range infos {
+ total += infos[i].Segments
+ }
+ return total
+}
+
+func flattenChapters(chapters []chapterSegments) []segment.Segment {
+ var all []segment.Segment
+ for _, chapter := range chapters {
+ all = append(all, chapter.Segments...)
+ }
+ return all
+}
+
+// Plan runs extract → parse → segment WITHOUT translating, then projects cost
+// across NMT-only / hybrid / full-LLM strategies. It only needs the Extractor,
+// Parser, Segmenter and CostModel to be set — it makes no provider call and
+// needs no server, which is what `estimate` runs on.
+//
+// It segments by chapter, not just by book, because prompt overhead is charged
+// per LLM call and calls are chunked per chapter — down to reading the same
+// free-tier counter off the same ledger the run-time guardrail enforces on.
+func (s *Stages) Plan(ctx context.Context, in BookInput, batchSize int, escalationRate float64) (*PlanResult, error) {
+ raw, err := s.Extractor.Extract(ctx, in.InputPath)
+ if err != nil {
+ return nil, &StageError{Stage: StageExtract, Err: err}
+ }
+ book, err := s.Parser.Parse(ctx, raw)
+ if err != nil {
+ return nil, &StageError{Stage: StageParse, Err: err}
+ }
+ chapters, err := s.segmentBookByChapter(ctx, book)
+ if err != nil {
+ return nil, &StageError{Stage: StageSegment, Err: err}
+ }
+
+ usedFreeChars, err := s.freeCharsUsed(ctx, time.Now().UTC())
+ if err != nil {
+ return nil, &StageError{Stage: StageSegment, Err: err}
+ }
+
+ // Only finished translations count as reuse here, which is the literary
+ // rule rather than the NMT one: `estimate` prices three strategies at once
+ // and cannot hold two definitions of "already done". It over-projects the
+ // nmt-only row rather than read low.
+ pending, reusable, err := s.pendingChapters(ctx, chapters, false)
+ if err != nil {
+ return nil, &StageError{Stage: StageSegment, Err: err}
+ }
+
+ grouped := make([][]segment.Segment, len(pending))
+ for i := range pending {
+ grouped[i] = pending[i].Segments
+ }
+ all := flattenChapters(chapters)
+ segments := flattenChapters(pending)
+
+ return &PlanResult{
+ BookID: in.BookID,
+ TotalSegments: len(all),
+ TotalSourceChars: segment.SourceChars(all),
+ Chapters: len(chapters),
+ ReusableSegments: reusable,
+ PendingSegments: len(segments),
+ PendingSourceChar: segment.SourceChars(segments),
+ BatchSize: batchSize,
+ EscalationRate: escalationRate,
+ Scenarios: s.CostModel.EstimateScenarios(grouped, batchSize, escalationRate, usedFreeChars),
+ }, nil
+}
diff --git a/internal/pipeline/stages_test.go b/internal/pipeline/stages_test.go
new file mode 100644
index 0000000..c729ab9
--- /dev/null
+++ b/internal/pipeline/stages_test.go
@@ -0,0 +1,272 @@
+package pipeline_test
+
+import (
+ "archive/zip"
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ECecillo/tomeshift/internal/epub"
+ "github.com/ECecillo/tomeshift/internal/glossary"
+ "github.com/ECecillo/tomeshift/internal/pipeline"
+ "github.com/ECecillo/tomeshift/internal/qa"
+ "github.com/ECecillo/tomeshift/internal/segment"
+ "github.com/ECecillo/tomeshift/internal/tm"
+ "github.com/ECecillo/tomeshift/internal/translate"
+)
+
+// The stage bodies own the rules the flow trusts by construction: which mode
+// persists a draft, when a draft is a reusable hit, and that glossary terms are
+// protected around the provider call. These test them directly against real
+// collaborators, where the old orchestrator_test could only reach them through
+// a whole run.
+
+// mangleServer is an NMT mock that rewrites a term, so the glossary's protection
+// is the only thing that can keep the canonical French out of the provider's
+// reach.
+func mangleServer(t *testing.T, from, to string) string {
+ t.Helper()
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req struct {
+ Text string `json:"text"`
+ }
+ _ = json.NewDecoder(r.Body).Decode(&req)
+ out := req.Text
+ if from != "" {
+ out = strings.ReplaceAll(out, from, to)
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]string{"translated_text": "[fr] " + out})
+ }))
+ t.Cleanup(srv.Close)
+ return srv.URL
+}
+
+func newTestStages(t *testing.T, store tm.Store, endpoint string, gloss *glossary.Glossary) *pipeline.Stages {
+ t.Helper()
+ return &pipeline.Stages{
+ Extractor: epub.ZipExtractor{},
+ Parser: epub.OPFParser{},
+ Segmenter: segment.NewParagraphSegmenter(),
+ TM: store,
+ Translator: translate.NewHTTPTranslator(endpoint, ""),
+ Recomposer: epub.EPUBRecomposer{},
+ EPUBCheck: qa.EPUBChecker{Binary: "epubcheck-not-installed-xyz"},
+ Glossary: gloss,
+ }
+}
+
+// writeChapterFixture writes a one-chapter EPUB whose body is bodyHTML.
+func writeChapterFixture(t *testing.T, bodyHTML string) string {
+ t.Helper()
+ entries := map[string]string{
+ "mimetype": "application/epub+zip",
+ "META-INF/container.xml": ``,
+ "OEBPS/content.opf": ``,
+ "OEBPS/c1.xhtml": `` + bodyHTML + ``,
+ }
+
+ path := filepath.Join(t.TempDir(), "source.epub")
+ f, err := os.Create(path)
+ require.NoError(t, err)
+ zw := zip.NewWriter(f)
+ w, err := zw.CreateHeader(&zip.FileHeader{Name: "mimetype", Method: zip.Store})
+ require.NoError(t, err)
+ _, err = w.Write([]byte(entries["mimetype"]))
+ require.NoError(t, err)
+ for name, content := range entries {
+ if name == "mimetype" {
+ continue
+ }
+ w, err := zw.Create(name)
+ require.NoError(t, err)
+ _, err = w.Write([]byte(content))
+ require.NoError(t, err)
+ }
+ require.NoError(t, zw.Close())
+ require.NoError(t, f.Close())
+ return path
+}
+
+func newTestStore(t *testing.T) *tm.SQLiteStore {
+ t.Helper()
+ store, err := tm.NewSQLiteStore(filepath.Join(t.TempDir(), "tm.sqlite"))
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = store.Close() })
+ return store
+}
+
+func TestStages_DraftPersistenceFollowsTheMode(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ persistDrafts bool
+ expectStored bool
+ }{
+ {name: "nmt persists its drafts", persistDrafts: true, expectStored: true},
+ {name: "literary does not persist drafts", persistDrafts: false, expectStored: false},
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ store := newTestStore(t)
+ input := writeChapterFixture(t, `
`)
+ out := filepath.Join(t.TempDir(), "out")
+
+ glossPath := filepath.Join(t.TempDir(), "glossary.json")
+ require.NoError(t, os.WriteFile(glossPath, []byte(`[{"term_en":"Bilbo","term_fr":"Bilbon","locked":true}]`), 0o600))
+ gloss, err := glossary.Load(glossPath)
+ require.NoError(t, err)
+
+ // The provider rewrites "Bilbo" to "WRONG" — but glossary protection removes
+ // it before the request and substitutes the canonical term after.
+ s := newTestStages(t, store, mangleServer(t, "Bilbo", "WRONG"), gloss)
+ ref := pipeline.RunRef{RunID: "r", BookID: "b", OutputDir: out, InputPath: input}
+ ctx := context.Background()
+
+ manifest, err := s.PrepareBook(ctx, pipeline.BookInput{InputPath: input, OutputDir: out, RunID: "r", BookID: "b"})
+ require.NoError(t, err)
+ chID := manifest.Chapters[0].ID
+
+ plan, err := s.ResolveChapter(ctx, pipeline.ResolveInput{Ref: ref, ChapterID: chID, ReusesDrafts: true})
+ require.NoError(t, err)
+ _, err = s.DraftChapter(ctx, pipeline.DraftInput{
+ Ref: ref, ChapterID: chID, Fresh: plan.Fresh, PersistDrafts: true, SourceLang: "en", TargetLang: "fr",
+ })
+ require.NoError(t, err)
+
+ units, err := pipeline.NewWorkspace(out).ReadChapter(chID)
+ require.NoError(t, err)
+ require.Len(t, units, 1)
+ require.Contains(t, units[0].TargetText, "Bilbon", "the canonical term must be substituted back")
+ require.NotContains(t, units[0].TargetText, "WRONG", "the provider must never have seen the term")
+}
+
+func TestStages_CommitPersistsOnlyRefinedLiteraryTranslations(t *testing.T) {
+ t.Parallel()
+
+ store := newTestStore(t)
+ input := writeChapterFixture(t, `
First.
Second.
`)
+ out := filepath.Join(t.TempDir(), "out")
+ s := newTestStages(t, store, mangleServer(t, "", ""), nil)
+ ref := pipeline.RunRef{RunID: "r", BookID: "b", OutputDir: out, InputPath: input}
+ ctx := context.Background()
+
+ manifest, err := s.PrepareBook(ctx, pipeline.BookInput{InputPath: input, OutputDir: out, RunID: "r", BookID: "b"})
+ require.NoError(t, err)
+ chID := manifest.Chapters[0].ID
+
+ // Hand-write the units: one refined (post-edited), one still a draft.
+ ws := pipeline.NewWorkspace(out)
+ units, err := ws.ReadChapter(chID)
+ require.NoError(t, err)
+ require.Len(t, units, 2)
+ units[0].TargetText, units[0].Status = "Première.", segment.StatusPostEdited
+ units[1].TargetText, units[1].Status = "brouillon", segment.StatusTranslated
+ require.NoError(t, ws.WriteChapter(chID, units))
+
+ res, err := s.CommitChapter(ctx, pipeline.CommitInput{
+ Ref: ref, ChapterID: chID, Persist: true, SourceLang: "en", TargetLang: "fr",
+ })
+ require.NoError(t, err)
+ require.Zero(t, res.RestoreFailures)
+
+ // Only the refined segment reaches the TM, as a finished literary entry.
+ refined, found, err := store.Lookup(ctx, segment.Hash(segment.Normalize("First.")))
+ require.NoError(t, err)
+ require.True(t, found)
+ require.Equal(t, tm.EngineLiterary, refined.Engine)
+
+ _, found, err = store.Lookup(ctx, segment.Hash(segment.Normalize("Second.")))
+ require.NoError(t, err)
+ require.False(t, found, "a draft segment must never be persisted in literary mode")
+}
diff --git a/internal/pipeline/types.go b/internal/pipeline/types.go
index 9956916..13aa087 100644
--- a/internal/pipeline/types.go
+++ b/internal/pipeline/types.go
@@ -3,6 +3,8 @@ package pipeline
import (
"time"
+ "github.com/ECecillo/tomeshift/internal/cost"
+ "github.com/ECecillo/tomeshift/internal/literary"
"github.com/ECecillo/tomeshift/internal/qa"
)
@@ -11,8 +13,11 @@ const (
ModeLiterary = "literary"
)
-// RunConfig is the input contract for a single translation run.
-type RunConfig struct {
+// BookInput is the input contract for a single translation run. It replaces the
+// former per-run config struct and is the payload the flow (and, from phase 3,
+// the Temporal workflow) is started with: no live objects, only serialisable
+// values.
+type BookInput struct {
InputPath string
OutputDir string // run-specific output directory (artifacts are written here)
SourceLang string
@@ -30,6 +35,14 @@ type RunConfig struct {
Resume bool
}
+// runMode returns the effective mode for in, defaulting to NMT.
+func runMode(in BookInput) string {
+ if in.Mode == "" {
+ return ModeNMT
+ }
+ return in.Mode
+}
+
// RunResult is the machine-readable summary emitted as report.json.
type RunResult struct {
SchemaVersion int `json:"schema_version"`
@@ -76,3 +89,245 @@ type RunResult struct {
OutputEPUB string `json:"output_epub"`
QA qa.Report `json:"qa"`
}
+
+// PlanResult is a dry-run projection: the book is parsed and segmented but not
+// translated, and per-strategy cost estimates are returned.
+type PlanResult struct {
+ BookID string `json:"book_id"`
+ TotalSegments int `json:"total_segments"`
+ TotalSourceChars int `json:"total_source_chars"`
+ Chapters int `json:"chapters"`
+ // ReusableSegments is how many of them the translation memory can already
+ // serve, and PendingSegments what is left to pay for. The scenarios below
+ // price the pending part only — the projection is about this run, not about
+ // the book in the abstract.
+ ReusableSegments int `json:"reusable_segments"`
+ PendingSegments int `json:"pending_segments"`
+ PendingSourceChar int `json:"pending_source_chars"`
+ BatchSize int `json:"batch_size"`
+ EscalationRate float64 `json:"escalation_rate"`
+ Scenarios []cost.ScenarioEstimate `json:"scenarios"`
+}
+
+// ---------------------------------------------------------------------------
+// Step DTOs. One small, serialisable struct per activity in the RFC table. No
+// *html.Node, no segment.Segment and no epub.Book ever crosses a port: content
+// is exchanged through the per-chapter unit files in the workspace.
+// ---------------------------------------------------------------------------
+
+// RunRef locates a run's workspace, identity and source without carrying its
+// content. InputPath lets the re-parse activities (Commit, Recompose, Finish)
+// re-derive placeholders and node trees deterministically.
+type RunRef struct {
+ RunID string
+ BookID string
+ OutputDir string
+ InputPath string
+}
+
+// Manifest is what PrepareBook produced: the chapters to translate, in spine
+// order, plus the run-scoped identifiers the flow stamps onto RunResult but
+// cannot learn without the injected collaborators (the NMT engine name, the LLM
+// provider/model recorded for token accounting).
+type Manifest struct {
+ Chapters []ChapterInfo
+ NMTEngine string
+ LLMProvider string
+ LLMModel string
+ // BatchSize is the literary LLM batch size the flow chunks fresh segments
+ // by; 0 outside literary mode.
+ BatchSize int
+}
+
+// ChapterInfo is a chapter's shape: enough for the flow to drive the loop and
+// for the cost projection to price it, with none of its content.
+type ChapterInfo struct {
+ ID string
+ Href string
+ Segments int
+ Chars int
+}
+
+// ProjectInput asks for the pre-run budget guardrail: it prices the work still
+// to do and refuses it before any provider call if a cap is exceeded.
+type ProjectInput struct {
+ Ref RunRef
+ Mode string
+ SeriesID string
+}
+
+// Projection is what ProjectCost returns: the two figures the two readers want
+// (EnforcedUSD, priced on the pending work; NoReuseUSD, the whole book with no
+// reuse — the baseline the savings subtract from), the reuse split, the free
+// characters already consumed this month, and the loaded bible version.
+type Projection struct {
+ EnforcedUSD float64
+ NoReuseUSD float64
+ Pending int
+ Reusable int
+ UsedFreeChars int
+ BibleVersion int
+}
+
+// ResolveInput asks ResolveChapter to serve a chapter's segments from the TM.
+type ResolveInput struct {
+ Ref RunRef
+ ChapterID string
+ ReusesDrafts bool
+}
+
+// ChapterPlan is the reuse split of one chapter after the TM lookup: how many
+// segments the memory served and which ones remain to translate.
+type ChapterPlan struct {
+ Reused int
+ Fresh []string
+}
+
+// DraftInput asks DraftChapter to draft a chapter's TM misses in one batched
+// provider call. PersistDrafts is true only in NMT mode, where an unescalated
+// draft is the final output for its segment.
+type DraftInput struct {
+ Ref RunRef
+ ChapterID string
+ Fresh []string
+ PersistDrafts bool
+ SourceLang string
+ TargetLang string
+}
+
+// DraftResult reports what the draft consumed: the source characters actually
+// put into provider requests (retries included) and the engine that answered.
+type DraftResult struct {
+ Chars int
+ Engine string
+}
+
+// EscalateInput asks EscalateChapter to run the NMT-mode quality-estimation and
+// selective LLM post-edit over a chapter's fresh drafts.
+type EscalateInput struct {
+ Ref RunRef
+ ChapterID string
+ Fresh []string
+ SourceLang string
+ TargetLang string
+}
+
+// EscalateResult reports the escalation pass: post-edit calls made (Attempts,
+// billed whether usable or not), the source characters they carried, how many
+// segments were actually improved, and the quality-estimation totals.
+type EscalateResult struct {
+ Attempts int
+ Chars int
+ Improved int
+ QESum float64
+ QECount int
+}
+
+// RefineInput asks RefineBatch to rewrite one literary batch. The preceding
+// French context is PrevTail (the previous batch's translated tail) for every
+// batch but a chapter's first; that one carries PrevTail empty and, in
+// PrevChapterID, the id of the previous chapter, whose source tail is the
+// context — exactly as the in-process walk read it before the split.
+type RefineInput struct {
+ Ref RunRef
+ ChapterID string
+ SeriesID string
+ IDs []string
+ PrevTail string
+ PrevChapterID string
+ SourceLang string
+ TargetLang string
+}
+
+// RefineResult reports one batch's outcome: the tokens it consumed, how many of
+// its segments it refined, the ids it degraded (kept on their draft) and the
+// tail of French it produced, fed to the next batch as PrevTail.
+type RefineResult struct {
+ Usage literary.Usage
+ Refined int
+ Degraded []string
+ Tail string
+}
+
+// CommitInput asks CommitChapter to re-derive the chapter's placeholders,
+// validate each translation by restoring it, persist accepted ones (Persist is
+// true only in literary mode) and write the final units.
+type CommitInput struct {
+ Ref RunRef
+ ChapterID string
+ Persist bool
+ SourceLang string
+ TargetLang string
+}
+
+// CommitResult reports what the commit found: glossary violations across the
+// chapter and restores the placeholder manager refused (QA counts these).
+type CommitResult struct {
+ GlossaryViolations int
+ RestoreFailures int
+}
+
+// BibleInput asks UpdateBible to evolve the series bible from a committed
+// chapter.
+type BibleInput struct {
+ Ref RunRef
+ SeriesID string
+ ChapterID string
+}
+
+// BibleResult reports the bible version after the update.
+type BibleResult struct {
+ Version int
+}
+
+// RecomposeResult reports the produced EPUB path.
+type RecomposeResult struct {
+ OutputEPUB string
+}
+
+// CheckInput asks CheckEPUB to validate the produced EPUB.
+type CheckInput struct {
+ OutputEPUB string
+}
+
+// CheckResult is the epubcheck verdict. A skipped check (binary absent) does
+// not block the run.
+type CheckResult struct {
+ Passed bool
+ Skipped bool
+}
+
+// FinishInput asks FinishBook to evaluate QA over the committed units and write
+// the run artifacts.
+type FinishInput struct {
+ Ref RunRef
+ Result RunResult
+}
+
+// SpendInput asks RecordSpend to persist what the run consumed to the ledger.
+type SpendInput struct {
+ Ref RunRef
+ Result RunResult
+}
+
+// PriceInput asks PriceRun to price what the result says has been consumed so
+// far, through the mode's cost model.
+type PriceInput struct {
+ Result RunResult
+ UsedFreeChars int
+ Mode string
+}
+
+// Progress is the run's advancement, published by the flow and consumed by the
+// engine: the in-process runtime drives the terminal bar with it, and from
+// phase 3 the Temporal workflow serves it through the progress query.
+type Progress struct {
+ State string
+ Stage string
+ Chapter int
+ Chapters int
+ Segments int // book-wide total, so the bar can size itself
+ SegmentsDone int
+ TMHits int
+ CostUSD float64
+}
diff --git a/internal/pipeline/workspace.go b/internal/pipeline/workspace.go
new file mode 100644
index 0000000..fb583e0
--- /dev/null
+++ b/internal/pipeline/workspace.go
@@ -0,0 +1,106 @@
+package pipeline
+
+import (
+ "bufio"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+
+ "github.com/ECecillo/tomeshift/internal/segment"
+)
+
+// workDirName is the run-scoped directory holding one unit file per chapter.
+const workDirName = "work"
+
+// Workspace is the run's per-chapter content store. Segmentation is
+// deterministic — ids by position, hash by normalised text — so a process that
+// holds only the units can re-parse the source document, re-derive the
+// placeholders and restore. The units are exactly the segments.jsonl schema
+// (§16.1) minus the run/book ids, which the artifact re-attaches.
+//
+// It is how content crosses the ports without a live *html.Node: an activity
+// reads the units it needs, mutates their target text or status, and writes
+// them back. One file per chapter keeps a chapter the unit of work.
+type Workspace struct {
+ Dir string
+}
+
+// NewWorkspace returns the workspace rooted under outputDir/work.
+func NewWorkspace(outputDir string) Workspace {
+ return Workspace{Dir: filepath.Join(outputDir, workDirName)}
+}
+
+// chapterPath is the unit file for chapterID.
+func (w Workspace) chapterPath(chapterID string) string {
+ return filepath.Join(w.Dir, chapterID+".jsonl")
+}
+
+// WriteChapter writes units as newline-delimited JSON, atomically: a temp file
+// in the same directory is renamed over the target, so a reader never sees a
+// half-written chapter and a crash never corrupts a committed one.
+func (w Workspace) WriteChapter(chapterID string, units []segment.Unit) error {
+ if err := os.MkdirAll(w.Dir, 0o755); err != nil {
+ return fmt.Errorf("unable to create workspace dir %q, err: %w", w.Dir, err)
+ }
+
+ tmp, err := os.CreateTemp(w.Dir, chapterID+".*.tmp")
+ if err != nil {
+ return fmt.Errorf("unable to create workspace temp for chapter %s, err: %w", chapterID, err)
+ }
+ tmpName := tmp.Name()
+ // On any early return the temp file is orphaned; remove it. After a
+ // successful rename it no longer exists, so the remove is a harmless no-op.
+ defer func() { _ = os.Remove(tmpName) }()
+
+ bw := bufio.NewWriter(tmp)
+ enc := json.NewEncoder(bw)
+ enc.SetEscapeHTML(false) // units carry placeholder tokens like
+ for i := range units {
+ if err := enc.Encode(units[i]); err != nil {
+ _ = tmp.Close()
+ return fmt.Errorf("unable to encode unit %s, err: %w", units[i].ID, err)
+ }
+ }
+ if err := bw.Flush(); err != nil {
+ _ = tmp.Close()
+ return fmt.Errorf("unable to flush workspace chapter %s, err: %w", chapterID, err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("unable to close workspace temp for chapter %s, err: %w", chapterID, err)
+ }
+ if err := os.Rename(tmpName, w.chapterPath(chapterID)); err != nil {
+ return fmt.Errorf("unable to commit workspace chapter %s, err: %w", chapterID, err)
+ }
+ return nil
+}
+
+// ReadChapter reads a chapter's units back in written order. A line that does
+// not decode aborts the read: a corrupt unit file cannot be silently
+// half-applied.
+func (w Workspace) ReadChapter(chapterID string) ([]segment.Unit, error) {
+ f, err := os.Open(w.chapterPath(chapterID))
+ if err != nil {
+ return nil, fmt.Errorf("unable to open workspace chapter %s, err: %w", chapterID, err)
+ }
+ defer func() { _ = f.Close() }()
+
+ var units []segment.Unit
+ sc := bufio.NewScanner(f)
+ sc.Buffer(make([]byte, 0, 64*1024), 8*1024*1024)
+ for sc.Scan() {
+ line := sc.Bytes()
+ if len(line) == 0 {
+ continue
+ }
+ var u segment.Unit
+ if err := json.Unmarshal(line, &u); err != nil {
+ return nil, fmt.Errorf("unable to decode workspace chapter %s, err: %w", chapterID, err)
+ }
+ units = append(units, u)
+ }
+ if err := sc.Err(); err != nil {
+ return nil, fmt.Errorf("unable to read workspace chapter %s, err: %w", chapterID, err)
+ }
+ return units, nil
+}
diff --git a/internal/pipeline/workspace_test.go b/internal/pipeline/workspace_test.go
new file mode 100644
index 0000000..f5adc68
--- /dev/null
+++ b/internal/pipeline/workspace_test.go
@@ -0,0 +1,90 @@
+package pipeline_test
+
+import (
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ECecillo/tomeshift/internal/pipeline"
+ "github.com/ECecillo/tomeshift/internal/segment"
+)
+
+func TestWorkspace_ChapterRoundTrip(t *testing.T) {
+ t.Parallel()
+
+ tests := []struct {
+ name string
+ units []segment.Unit
+ }{
+ {name: "empty chapter", units: nil},
+ {
+ name: "units with placeholder tokens survive verbatim",
+ units: []segment.Unit{
+ {
+ ID: "ch_0001_p_0001", ChapterID: "ch_0001", NodePath: "/html[1]/body[1]/p[1]",
+ SourceText: `A bold claim.`, NormalizedText: "a bold claim.",
+ Hash: "deadbeef", Placeholders: []string{``, ``},
+ TargetText: `Une affirmation.`, Status: segment.StatusPostEdited,
+ },
+ {
+ ID: "ch_0001_p_0002", ChapterID: "ch_0001", NodePath: "/html[1]/body[1]/p[2]",
+ SourceText: "Plain.", NormalizedText: "plain.", Hash: "cafe",
+ TargetText: "Simple.", Status: segment.StatusReused,
+ },
+ },
+ },
+ }
+
+ for _, tt := range tests {
+ tt := tt
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+
+ ws := pipeline.NewWorkspace(t.TempDir())
+ require.NoError(t, ws.WriteChapter("ch_0001", tt.units))
+
+ got, err := ws.ReadChapter("ch_0001")
+ require.NoError(t, err)
+ require.Equal(t, tt.units, got)
+ })
+ }
+}
+
+func TestWorkspace_ReadCorruptLineErrors(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ ws := pipeline.NewWorkspace(dir)
+ require.NoError(t, ws.WriteChapter("ch_0001", []segment.Unit{{ID: "s1"}}))
+
+ // A half-written unit file must not be silently half-applied.
+ path := filepath.Join(dir, "work", "ch_0001.jsonl")
+ require.NoError(t, os.WriteFile(path, []byte("{not json}\n"), 0o644))
+
+ _, err := ws.ReadChapter("ch_0001")
+ require.Error(t, err)
+}
+
+func TestWorkspace_WriteIsAtomic(t *testing.T) {
+ t.Parallel()
+
+ dir := t.TempDir()
+ ws := pipeline.NewWorkspace(dir)
+
+ require.NoError(t, ws.WriteChapter("ch_0001", []segment.Unit{{ID: "old", SourceText: "old"}}))
+ require.NoError(t, ws.WriteChapter("ch_0001", []segment.Unit{{ID: "new", SourceText: "new"}}))
+
+ // The second write replaces the first wholesale, and leaves no temp file
+ // behind in the work dir.
+ got, err := ws.ReadChapter("ch_0001")
+ require.NoError(t, err)
+ require.Len(t, got, 1)
+ require.Equal(t, "new", got[0].ID)
+
+ entries, err := os.ReadDir(filepath.Join(dir, "work"))
+ require.NoError(t, err)
+ require.Len(t, entries, 1, "no temp file left behind")
+ require.Equal(t, "ch_0001.jsonl", entries[0].Name())
+}
diff --git a/internal/segment/apply.go b/internal/segment/apply.go
new file mode 100644
index 0000000..d775015
--- /dev/null
+++ b/internal/segment/apply.go
@@ -0,0 +1,68 @@
+package segment
+
+import (
+ "context"
+ "fmt"
+ "sort"
+ "strings"
+
+ "github.com/ECecillo/tomeshift/internal/epub"
+)
+
+// ApplyReport is what a consumer observes after ApplyTranslations. It accounts
+// for every segment the document produced: Applied were restored into the live
+// tree, Missing had no target (left holding their source markup), and Failed
+// had a target the placeholder manager refused (also left as source). QA counts
+// the Failed entries — a refused restore ships untranslated, not broken.
+type ApplyReport struct {
+ Applied int // targets restored into the tree
+ Missing []string // segment ids present in doc but absent from targets (left as source)
+ Failed map[string]error // restore refused (node kept as source) — QA counts these
+}
+
+// ApplyTranslations re-segments doc with seg — deterministically, so the ids
+// match the segmentation that produced targets — then splices each target back
+// into the live tree through pm. A target for an id the document did not
+// produce is a mismatched translation set: the restores done so far stand and
+// the report is returned alongside a wrapped ErrUnknownSegment listing the
+// offending ids. A restore that pm refuses never aborts the pass: it lands in
+// Failed and its node keeps the source markup pm left untouched.
+func ApplyTranslations(ctx context.Context, seg Segmenter, pm PlaceholderManager,
+ doc *epub.XHTMLDocument, chapterID string, targets map[string]string) (ApplyReport, error) {
+ segments, err := seg.Segment(ctx, doc, chapterID)
+ if err != nil {
+ return ApplyReport{}, fmt.Errorf("unable to re-segment chapter %s for re-application, err: %w", chapterID, err)
+ }
+
+ report := ApplyReport{Failed: map[string]error{}}
+ produced := make(map[string]struct{}, len(segments))
+ for i := range segments {
+ s := segments[i]
+ produced[s.ID] = struct{}{}
+
+ target, ok := targets[s.ID]
+ if !ok {
+ report.Missing = append(report.Missing, s.ID)
+ continue
+ }
+ if err := pm.Restore(target, s.Placeholders, s.Node); err != nil {
+ report.Failed[s.ID] = fmt.Errorf("unable to restore target for segment %s, err: %w", s.ID, err)
+ continue
+ }
+ report.Applied++
+ }
+
+ var unknown []string
+ for id := range targets {
+ if _, ok := produced[id]; !ok {
+ unknown = append(unknown, id)
+ }
+ }
+ if len(unknown) > 0 {
+ sort.Strings(unknown)
+ return report, fmt.Errorf("unable to apply targets [%s], err: %w",
+ strings.Join(unknown, ", "), ErrUnknownSegment)
+ }
+
+ return report, nil
+}
diff --git a/internal/segment/apply_test.go b/internal/segment/apply_test.go
new file mode 100644
index 0000000..5607116
--- /dev/null
+++ b/internal/segment/apply_test.go
@@ -0,0 +1,225 @@
+package segment
+
+import (
+ "encoding/json"
+ "sort"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+ "golang.org/x/net/html"
+
+ "github.com/ECecillo/tomeshift/internal/epub"
+)
+
+// renderNodes renders every translatable node of doc in order and joins the
+// results, giving a stable byte image of the live tree to compare before and
+// after re-application.
+func renderNodes(t *testing.T, doc *epub.XHTMLDocument) string {
+ t.Helper()
+
+ var sb strings.Builder
+ for _, cn := range doc.Nodes {
+ require.NoError(t, html.Render(&sb, cn.Node))
+ }
+ return sb.String()
+}
+
+func TestApplyTranslations(t *testing.T) {
+ t.Parallel()
+
+ seg := NewParagraphSegmenter()
+ pm := InlinePlaceholderManager{}
+
+ type expected struct {
+ applied int
+ missing []string
+ failed []string // ids expected in Failed
+ failedErr error // sentinel wrapped inside each Failed value
+ err error // wrapped top-level error, if any
+ // renderEqualsSource asserts the applied tree is byte-identical to the
+ // pristine source (nothing was rewritten).
+ renderEqualsSource bool
+ // contains lists substrings the applied render must include.
+ contains []string
+ }
+
+ cases := []struct {
+ name string
+ fragment string
+ makeTargets func(segs []Segment) map[string]string
+ expected expected
+ }{
+ {
+ name: "identity round trip feeds source back unchanged",
+ fragment: `
`,
+ makeTargets: func(segs []Segment) map[string]string {
+ // Drop the closing token of the inline pair: Restore refuses and
+ // the node keeps its original markup.
+ broken := strings.Replace(segs[0].SourceText, ``, "", 1)
+ return map[string]string{segs[0].ID: broken}
+ },
+ expected: expected{
+ applied: 0,
+ failed: []string{"ch_0001_p_0001"},
+ failedErr: ErrMissingPlaceholder,
+ renderEqualsSource: true,
+ },
+ },
+ {
+ name: "segment without a target is reported Missing",
+ fragment: `
Translated.
Left alone.
`,
+ makeTargets: func(segs []Segment) map[string]string {
+ return map[string]string{segs[0].ID: segs[0].SourceText}
+ },
+ expected: expected{
+ applied: 1,
+ missing: []string{"ch_0001_p_0002"},
+ renderEqualsSource: true,
+ },
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ pristine := docFromParagraphs(t, tc.fragment)
+ sourceRender := renderNodes(t, pristine)
+
+ apply := docFromParagraphs(t, tc.fragment)
+ segs, err := seg.Segment(t.Context(), apply, "ch_0001")
+ require.NoError(t, err)
+ targets := tc.makeTargets(segs)
+
+ report, err := ApplyTranslations(t.Context(), seg, pm, apply, "ch_0001", targets)
+
+ if tc.expected.err != nil {
+ require.ErrorIs(t, err, tc.expected.err)
+ } else {
+ require.NoError(t, err)
+ }
+
+ require.Equal(t, tc.expected.applied, report.Applied)
+ require.ElementsMatch(t, tc.expected.missing, report.Missing)
+
+ require.Len(t, report.Failed, len(tc.expected.failed))
+ for _, id := range tc.expected.failed {
+ require.Contains(t, report.Failed, id)
+ if tc.expected.failedErr != nil {
+ require.ErrorIs(t, report.Failed[id], tc.expected.failedErr)
+ }
+ }
+
+ applied := renderNodes(t, apply)
+ if tc.expected.renderEqualsSource {
+ require.Equal(t, sourceRender, applied)
+ }
+ for _, want := range tc.expected.contains {
+ require.Contains(t, applied, want)
+ }
+ })
+ }
+}
+
+// TestApplyTranslations_ResegmentationIsDeterministic pins the invariant
+// ApplyTranslations relies on: segmenting the same document twice yields the
+// same ids and hashes, so targets keyed on the first segmentation address the
+// segments the second one restores.
+func TestApplyTranslations_ResegmentationIsDeterministic(t *testing.T) {
+ t.Parallel()
+
+ fragment := `
Alpha beta.
Gamma delta.
`
+ seg := NewParagraphSegmenter()
+
+ first, err := seg.Segment(t.Context(), docFromParagraphs(t, fragment), "ch_0001")
+ require.NoError(t, err)
+ second, err := seg.Segment(t.Context(), docFromParagraphs(t, fragment), "ch_0001")
+ require.NoError(t, err)
+
+ require.Len(t, second, len(first))
+ for i := range first {
+ require.Equal(t, first[i].ID, second[i].ID)
+ require.Equal(t, first[i].Hash, second[i].Hash)
+ }
+}
+
+// TestSegment_Unit_JSONSchema proves the serialisable projection carries exactly
+// the schema §16.1 field names (minus schema_version/run/book ids), so a unit
+// written to work/.jsonl round-trips against the documented contract.
+func TestSegment_Unit_JSONSchema(t *testing.T) {
+ t.Parallel()
+
+ s := Segment{
+ ID: "ch_0001_p_0012",
+ ChapterID: "ch_0001",
+ NodePath: "/html[1]/body[1]/p[12]",
+ SourceText: `Source text`,
+ NormalizedText: `source text`,
+ Hash: "deadbeef",
+ Placeholders: []Placeholder{{Token: ``}},
+ TargetText: `Texte traduit`,
+ Status: StatusTranslated,
+ }
+
+ raw, err := json.Marshal(s.Unit())
+ require.NoError(t, err)
+
+ var got map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal(raw, &got))
+
+ keys := make([]string, 0, len(got))
+ for k := range got {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ expected := []string{
+ "chapter_id", "node_path", "normalized_source_text", "placeholders",
+ "segment_hash", "segment_id", "source_text", "status", "target_text",
+ }
+ sort.Strings(expected)
+ require.Equal(t, expected, keys)
+
+ // The single placeholder is flattened to its token.
+ require.JSONEq(t, `[""]`, string(got["placeholders"]))
+}
diff --git a/internal/segment/doc.go b/internal/segment/doc.go
index 407e34a..2fcbb3d 100644
--- a/internal/segment/doc.go
+++ b/internal/segment/doc.go
@@ -3,4 +3,9 @@
//
// It turns XHTML content into stable translatable units while preserving inline
// markup so translated content can be safely restored into the source document.
+// It also owns the serialisable projection of a segment (Unit, schema §16.1)
+// that crosses process boundaries, and the re-application of translated targets
+// back into a live document tree (ApplyTranslations), which re-segments the
+// source deterministically to recover the placeholder subtrees Unit cannot
+// carry.
package segment
diff --git a/internal/segment/errors.go b/internal/segment/errors.go
index 20e79fd..7d992f5 100644
--- a/internal/segment/errors.go
+++ b/internal/segment/errors.go
@@ -11,4 +11,8 @@ var (
// not enough — rebuilding from a reordered pair produces valid-looking but
// wrong markup, silently.
ErrMisorderedPlaceholder = errors.New("segment: translated text returned placeholder tokens out of order")
+ // ErrUnknownSegment covers a re-application whose targets reference segment
+ // ids the document does not produce: a stale or mismatched translation set.
+ // The ids are listed in the wrapping message so the caller can act on them.
+ ErrUnknownSegment = errors.New("segment: target references unknown segment id(s)")
)
diff --git a/internal/segment/types.go b/internal/segment/types.go
index b3b4d88..4f23e64 100644
--- a/internal/segment/types.go
+++ b/internal/segment/types.go
@@ -78,3 +78,53 @@ func SourceChars(segs []Segment) int {
}
return chars
}
+
+// Unit is the serialisable projection of a Segment: the schema §16.1 content
+// fields minus schema_version, the run/book ids and translation_engine (the
+// engine is recorded on the TM entry, not the unit). It is what crosses process
+// boundaries; a Segment (which references a live *html.Node) never does.
+// Placeholders are reduced to their opaque tokens because the *html.Node subtree
+// they carry cannot be serialised — re-application re-segments the source
+// document to recover it.
+type Unit struct {
+ ID string `json:"segment_id"`
+ ChapterID string `json:"chapter_id"`
+ NodePath string `json:"node_path"`
+ SourceText string `json:"source_text"`
+ NormalizedText string `json:"normalized_source_text"`
+ Hash string `json:"segment_hash"`
+ Placeholders []string `json:"placeholders"` // tokens, e.g. ``
+ TargetText string `json:"target_text"`
+ Status SegmentStatus `json:"status"`
+}
+
+// Unit projects the segment onto its serialisable form, dropping the live
+// *html.Node and flattening each placeholder to its token.
+func (s Segment) Unit() Unit {
+ tokens := make([]string, len(s.Placeholders))
+ for i := range s.Placeholders {
+ tokens[i] = s.Placeholders[i].Token
+ }
+ return Unit{
+ ID: s.ID,
+ ChapterID: s.ChapterID,
+ NodePath: s.NodePath,
+ SourceText: s.SourceText,
+ NormalizedText: s.NormalizedText,
+ Hash: s.Hash,
+ Placeholders: tokens,
+ TargetText: s.TargetText,
+ Status: s.Status,
+ }
+}
+
+// SourceCharsOfUnits totals the source characters of units, counting runes
+// rather than bytes — the same billing rule as SourceChars, applied to the
+// serialisable projection that crosses process boundaries.
+func SourceCharsOfUnits(units []Unit) int {
+ chars := 0
+ for i := range units {
+ chars += utf8.RuneCountInString(units[i].SourceText)
+ }
+ return chars
+}
diff --git a/internal/segment/typography.go b/internal/segment/typography.go
new file mode 100644
index 0000000..989679c
--- /dev/null
+++ b/internal/segment/typography.go
@@ -0,0 +1,322 @@
+package segment
+
+import (
+ "regexp"
+ "strings"
+ "unicode"
+)
+
+// frenchToken matches an inline placeholder () or a glossary token
+// (), including the paired form some providers normalize them to
+// (). The matched substring is preserved byte-for-byte and the
+// token is treated as zero-width when FrenchTypography decides context.
+var frenchToken = regexp.MustCompile(`<[xg] id="\d+"\s*/?>(?:\s*)?`)
+
+// tokenSentinelBase is the first Private-Use-Area code point used to stand in
+// for an extracted token while the text is being rewritten. Real book text does
+// not contain these code points, so they never collide with content.
+const tokenSentinelBase = 0xE000
+
+// FrenchTypography rewrites a French target string into the project's single,
+// deterministic typography, aligned on the English source conventions (Witch
+// Cult Translations style). It is pure and idempotent: f(f(x)) == f(x).
+//
+// Rules:
+// 1. A run of 3+ ASCII dots ("...") becomes one ellipsis ("…"). Existing "……"
+// is left untouched.
+// 2. Exactly one U+00A0 sits after every « and before every ».
+// 3. Exactly one U+00A0 sits before the first mark of a run of ": ; ! ?" when
+// the previous visible character is a letter, digit, …, », ), ], ” or an
+// apostrophe; a space already present before the mark becomes U+00A0
+// whatever precedes it. In-word colons (Re:Zero, 10:30) and URL colons
+// (://) are left alone.
+// 4. The em dash (—) is glued to the following word or to a following . or ,
+// (« —Mot », mot—mot, « —. »); a single space is kept before it after
+// sentence punctuation (. ! ? …); "——" runs are treated as one dash; a dash
+// at the end or before » is left untouched apart from rule 2.
+//
+// Placeholder tokens () and glossary tokens () are
+// preserved exactly and are zero-width for every context decision above.
+func FrenchTypography(text string) string {
+ if text == "" {
+ return text
+ }
+ runes, tokens := tokenizeFrench(text)
+ runes = fixEllipsis(runes)
+ runes = fixGuillemets(runes)
+ runes = fixHighPunct(runes, tokens)
+ runes = fixEmDash(runes, tokens)
+ return renderFrench(runes, tokens)
+}
+
+// tokenizeFrench replaces every token with a zero-width Private-Use sentinel
+// rune and returns the rune slice plus the original token strings, indexed by
+// the sentinel's offset from tokenSentinelBase.
+func tokenizeFrench(text string) ([]rune, []string) {
+ locs := frenchToken.FindAllStringIndex(text, -1)
+ if len(locs) == 0 {
+ return []rune(text), nil
+ }
+ var (
+ tokens []string
+ out []rune
+ )
+ last := 0
+ for _, loc := range locs {
+ out = append(out, []rune(text[last:loc[0]])...)
+ out = append(out, rune(tokenSentinelBase+len(tokens)))
+ tokens = append(tokens, text[loc[0]:loc[1]])
+ last = loc[1]
+ }
+ out = append(out, []rune(text[last:])...)
+ return out, tokens
+}
+
+// renderFrench rebuilds the string, expanding each sentinel back to its token.
+func renderFrench(runes []rune, tokens []string) string {
+ var b strings.Builder
+ for _, r := range runes {
+ if i := int(r) - tokenSentinelBase; i >= 0 && i < len(tokens) {
+ b.WriteString(tokens[i])
+ continue
+ }
+ b.WriteRune(r)
+ }
+ return b.String()
+}
+
+func isTokenSentinel(r rune, tokens []string) bool {
+ i := int(r) - tokenSentinelBase
+ return i >= 0 && i < len(tokens)
+}
+
+// isBreakingSpace reports the whitespace this normalizer may rewrite: the plain
+// space, the no-break space and the narrow no-break space.
+func isBreakingSpace(r rune) bool {
+ return r == ' ' || r == '\u00a0' || r == '\u202f'
+}
+
+// fixEllipsis collapses every run of 3+ ASCII dots to a single U+2026.
+func fixEllipsis(runes []rune) []rune {
+ out := make([]rune, 0, len(runes))
+ for i := 0; i < len(runes); {
+ if runes[i] != '.' {
+ out = append(out, runes[i])
+ i++
+ continue
+ }
+ j := i
+ for j < len(runes) && runes[j] == '.' {
+ j++
+ }
+ if j-i >= 3 {
+ out = append(out, '…')
+ } else {
+ out = append(out, runes[i:j]...)
+ }
+ i = j
+ }
+ return out
+}
+
+// fixGuillemets forces exactly one U+00A0 after every « and before every ».
+func fixGuillemets(runes []rune) []rune {
+ out := make([]rune, 0, len(runes)+4)
+ for i := 0; i < len(runes); i++ {
+ switch runes[i] {
+ case '«':
+ out = append(out, '«')
+ j := i + 1
+ for j < len(runes) && isBreakingSpace(runes[j]) {
+ j++
+ }
+ out = append(out, '\u00a0')
+ i = j - 1
+ case '»':
+ for len(out) > 0 && isBreakingSpace(out[len(out)-1]) {
+ out = out[:len(out)-1]
+ }
+ out = append(out, '\u00a0', '»')
+ default:
+ out = append(out, runes[i])
+ }
+ }
+ return out
+}
+
+func isHighPunct(r rune) bool {
+ return highPunctClass(r) != 0
+}
+
+// highPunctClass groups high punctuation so a run stays within one class: the
+// terminal marks (! ?) and the clitic marks (: ;) are separate runs. This keeps
+// a speaker label's colon ("???:") distinct from the exclamation run before it,
+// so each gets its own U+00A0 decision.
+func highPunctClass(r rune) int {
+ switch r {
+ case '!', '?':
+ return 1
+ case ':', ';':
+ return 2
+ }
+ return 0
+}
+
+// allowedBeforeHigh reports whether a U+00A0 may be forced before a high
+// punctuation mark that follows r. Terminal marks (! ?) are included so a
+// speaker label's colon after "???" gets its space.
+func allowedBeforeHigh(r rune) bool {
+ switch r {
+ case '…', '»', ')', ']', '”', '\'', '’', '!', '?':
+ return true
+ }
+ return unicode.IsLetter(r) || unicode.IsDigit(r)
+}
+
+// fixHighPunct forces one U+00A0 before the first mark of each run of ": ; ! ?".
+func fixHighPunct(runes []rune, tokens []string) []rune {
+ out := make([]rune, 0, len(runes)+4)
+ for i := range runes {
+ r := runes[i]
+ if !isHighPunct(r) {
+ out = append(out, r)
+ continue
+ }
+ if i > 0 && highPunctClass(runes[i-1]) == highPunctClass(r) { // mid-run
+ out = append(out, r)
+ continue
+ }
+ if !spaceBeforeHigh(r, prevVisible(out, tokens), runes, i, tokens) {
+ // A space the text already has before the mark becomes the no-break
+ // space, whatever precedes it (■■■■ :, “Subaru“ :).
+ if len(out) > 0 && isBreakingSpace(out[len(out)-1]) && prevVisible(out, tokens) != 0 {
+ for len(out) > 0 && isBreakingSpace(out[len(out)-1]) {
+ out = out[:len(out)-1]
+ }
+ out = append(out, '\u00a0')
+ }
+ out = append(out, r)
+ continue
+ }
+ for len(out) > 0 && isBreakingSpace(out[len(out)-1]) {
+ out = out[:len(out)-1]
+ }
+ out = append(out, '\u00a0', r)
+ }
+ return out
+}
+
+func spaceBeforeHigh(mark, prev rune, runes []rune, i int, tokens []string) bool {
+ if !allowedBeforeHigh(prev) {
+ return false
+ }
+ if mark == ':' {
+ if isInWordColon(runes, i) {
+ return false // Re:Zero, 10:30
+ }
+ if isURLColon(runes, i, tokens) {
+ return false // scheme://
+ }
+ }
+ return true
+}
+
+// isInWordColon reports whether the colon at index i is glued on both sides to
+// a letter or digit, as in a title (Re:Zero) or a time (10:30): it is part of
+// the word, not punctuation.
+func isInWordColon(runes []rune, i int) bool {
+ if i == 0 || i+1 >= len(runes) {
+ return false
+ }
+ isWordRune := func(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) }
+ return isWordRune(runes[i-1]) && isWordRune(runes[i+1])
+}
+
+// isURLColon reports whether the colon at index i is immediately followed by
+// "//" (ignoring zero-width tokens), i.e. a URL scheme separator.
+func isURLColon(runes []rune, i int, tokens []string) bool {
+ seen := 0
+ for j := i + 1; j < len(runes) && seen < 2; j++ {
+ if isTokenSentinel(runes[j], tokens) {
+ continue
+ }
+ if runes[j] != '/' {
+ return false
+ }
+ seen++
+ }
+ return seen == 2
+}
+
+// fixEmDash glues the em dash to the following word, keeping a single space
+// before it only after sentence punctuation. A "——" run is handled as one dash.
+// A dash followed by . or , is glued to it (the source's "—." silence); a dash
+// at the end or before » is left untouched (rule 2 still applies).
+func fixEmDash(runes []rune, tokens []string) []rune {
+ out := make([]rune, 0, len(runes)+2)
+ for i := 0; i < len(runes); i++ {
+ if runes[i] != '—' {
+ out = append(out, runes[i])
+ continue
+ }
+ j := i
+ for j < len(runes) && runes[j] == '—' {
+ j++
+ }
+ prev := prevVisible(out, tokens)
+ next := nextVisible(runes, j, tokens)
+
+ if next == 0 || next == '»' {
+ out = append(out, runes[i:j]...) // trailing dash: leave as-is
+ i = j - 1
+ continue
+ }
+
+ switch {
+ case prev == '.' || prev == '!' || prev == '?' || prev == '…':
+ for len(out) > 0 && isBreakingSpace(out[len(out)-1]) {
+ out = out[:len(out)-1]
+ }
+ out = append(out, ' ')
+ case unicode.IsLetter(prev) || unicode.IsDigit(prev):
+ for len(out) > 0 && isBreakingSpace(out[len(out)-1]) {
+ out = out[:len(out)-1]
+ }
+ default:
+ // Opening context (« ( start) or other: leave the left side alone.
+ }
+
+ out = append(out, runes[i:j]...)
+
+ k := j
+ for k < len(runes) && isBreakingSpace(runes[k]) {
+ k++
+ }
+ i = k - 1
+ }
+ return out
+}
+
+// prevVisible returns the last emitted rune that is neither whitespace nor a
+// token sentinel, or 0 when none precedes.
+func prevVisible(out []rune, tokens []string) rune {
+ for i := len(out) - 1; i >= 0; i-- {
+ if isBreakingSpace(out[i]) || isTokenSentinel(out[i], tokens) {
+ continue
+ }
+ return out[i]
+ }
+ return 0
+}
+
+// nextVisible returns the first rune at or after from that is neither whitespace
+// nor a token sentinel, or 0 when none follows.
+func nextVisible(runes []rune, from int, tokens []string) rune {
+ for i := from; i < len(runes); i++ {
+ if isBreakingSpace(runes[i]) || isTokenSentinel(runes[i], tokens) {
+ continue
+ }
+ return runes[i]
+ }
+ return 0
+}
diff --git a/internal/segment/typography_test.go b/internal/segment/typography_test.go
new file mode 100644
index 0000000..eddde43
--- /dev/null
+++ b/internal/segment/typography_test.go
@@ -0,0 +1,106 @@
+package segment
+
+import (
+ "strings"
+ "testing"
+)
+
+const nbsp = "\u00a0"
+
+func TestFrenchTypography(t *testing.T) {
+ cases := []struct {
+ name string
+ in string
+ want string
+ }{
+ // Rule 1: ellipsis.
+ {"ellipsis three dots", "Trois points...", "Trois points…"},
+ {"ellipsis four dots", "Quatre....", "Quatre…"},
+ {"ellipsis two dots kept", "Deux..", "Deux.."},
+ {"ellipsis single dot kept", "Un.", "Un."},
+ {"ellipsis existing not collapsed", "Déjà……", "Déjà……"},
+ {"ellipsis repeated runs", "a...b...c", "a…b…c"},
+
+ // Rule 2: guillemets.
+ {"guillemets tight", "«Bonjour»", "«" + nbsp + "Bonjour" + nbsp + "»"},
+ {"guillemets spaced", "« Bonjour »", "«" + nbsp + "Bonjour" + nbsp + "»"},
+ {"guillemets double spaced", "« Bonjour »", "«" + nbsp + "Bonjour" + nbsp + "»"},
+ {"guillemets narrow nbsp", "«\u202fBonjour\u202f»", "«" + nbsp + "Bonjour" + nbsp + "»"},
+ {"guillemets already nbsp", "«\u00a0Bonjour\u00a0»", "«" + nbsp + "Bonjour" + nbsp + "»"},
+
+ // Rule 3: high punctuation.
+ {"question tight", "Vraiment?", "Vraiment" + nbsp + "?"},
+ {"question spaced", "Vraiment ?", "Vraiment" + nbsp + "?"},
+ {"bang", "Attends!", "Attends" + nbsp + "!"},
+ {"colon", "Deux points: trois", "Deux points" + nbsp + ": trois"},
+ {"semicolon", "Point-virgule; suite", "Point-virgule" + nbsp + "; suite"},
+ {"run question bang", "Quoi?!", "Quoi" + nbsp + "?!"},
+ {"run double bang", "Non!!", "Non" + nbsp + "!!"},
+ {"run triple question", "Hein???", "Hein" + nbsp + "???"},
+ {"time colon untouched", "Rendez-vous 10:30", "Rendez-vous 10:30"},
+ {"title colon untouched", "Re:Zero !", "Re:Zero" + nbsp + "!"},
+ {"url colon untouched", "https://example.com", "https://example.com"},
+ {"no space at start", "?!", "?!"},
+ {"no space after open guillemet", "«?", "«" + nbsp + "?"},
+ {"speaker label name", "Ram: « Viens", "Ram" + nbsp + ": «" + nbsp + "Viens"},
+ {"speaker label unknown", "???: « Viens", "???" + nbsp + ": «" + nbsp + "Viens"},
+ {"existing space after symbol", "■■■■ : « Viens", "■■■■" + nbsp + ": «" + nbsp + "Viens"},
+
+ // Rule 4: em dash.
+ {"dash between words both spaced", "mot — mot", "mot—mot"},
+ {"dash between words left spaced", "mot— mot", "mot—mot"},
+ {"dash between words right spaced", "mot —mot", "mot—mot"},
+ {"dash between words tight", "mot—mot", "mot—mot"},
+ {"dash after open guillemet spaced", "« — Viens", "«" + nbsp + "—Viens"},
+ {"dash after open guillemet tight", "« —Viens", "«" + nbsp + "—Viens"},
+ {"dash after bang", "faire ! — Viens", "faire" + nbsp + "! —Viens"},
+ {"dash after period kept space", "fin. —Suite", "fin. —Suite"},
+ {"dash after period collapse space", "fin. — Suite", "fin. —Suite"},
+ {"double dash spaced", "mot —— mot", "mot——mot"},
+ {"double dash tight", "mot——mot", "mot——mot"},
+ {"dash before period glued", "la fin — .", "la fin—."},
+ {"silence dash after open guillemet", "« — . On", "«" + nbsp + "—. On"},
+ {"dash before comma glued", "sa peine — , il", "sa peine—, il"},
+ {"dash before closing guillemet", "dit-il — »", "dit-il —" + nbsp + "»"},
+
+ // Tokens: zero-width for context, preserved byte-for-byte.
+ {"token before question", `mot?`, `mot` + nbsp + "?"},
+ {"token hides start before run", `??? : «`, `???` + nbsp + ": «" + nbsp},
+ {"glossary token zero width dash", `mot — autre`, `mot—autre`},
+ {"token not modified inside", ``, ``},
+
+ // Combined, realistic dialogue.
+ {
+ "dialogue combined",
+ "«Ram: Attends...moi!»",
+ "«" + nbsp + "Ram" + nbsp + ": Attends…moi" + nbsp + "!" + nbsp + "»",
+ },
+
+ // No-ops.
+ {"plain sentence", "Bonjour le monde.", "Bonjour le monde."},
+ {"empty", "", ""},
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got := FrenchTypography(tc.in)
+ if got != tc.want {
+ t.Errorf("FrenchTypography(%q)\n got = %q\nwant = %q", tc.in, got, tc.want)
+ }
+ // Idempotence: f(f(x)) == f(x).
+ if again := FrenchTypography(got); again != got {
+ t.Errorf("not idempotent for %q:\n first = %q\nsecond = %q", tc.in, got, again)
+ }
+ })
+ }
+}
+
+func TestFrenchTypographyPreservesTokens(t *testing.T) {
+ in := `D'abord, puis... «» !`
+ got := FrenchTypography(in)
+ for _, tok := range []string{``, ``, ``} {
+ if !strings.Contains(got, tok) {
+ t.Errorf("token %q dropped from %q", tok, got)
+ }
+ }
+}
diff --git a/scripts/docs_audit.py b/scripts/docs_audit.py
index a1b5b50..dd8c75a 100644
--- a/scripts/docs_audit.py
+++ b/scripts/docs_audit.py
@@ -41,6 +41,7 @@
"recipe:fmt": "covered by just check",
"recipe:lint": "covered by just check",
"recipe:test": "covered by just check",
+ "recipe:test-update-golden": "test-maintenance only; discoverable via just --list, not a user-facing step",
"recipe:comet-down": "symmetric with comet-up, no distinct consequence",
"recipe:seed-bible": "documented under its command name, style-profile",
"recipe:batch": "documented under its command name",
diff --git a/tests/integration/batch_test.go b/tests/integration/batch_test.go
index df99b16..5bdd8d9 100644
--- a/tests/integration/batch_test.go
+++ b/tests/integration/batch_test.go
@@ -63,6 +63,52 @@ func TestBatch_SharedTMAcrossBooks(t *testing.T) {
require.GreaterOrEqual(t, summary.TMHits, 1, "the shared sentence must be reused cross-book")
}
+// TestBatch_RunTimeoutDoesNotSkipLaterBooks pins the fix for the per-book
+// timeout context leaking into RunSeries' cancellation check: with
+// RunTimeoutSeconds set, book 1's context is cancelled in the Done hook, and
+// before the fix that cancelled context was still the shared runtime's when the
+// loop tested Cancelled() for book 2 — silently dropping every book after the
+// first while still exiting Success.
+func TestBatch_RunTimeoutDoesNotSkipLaterBooks(t *testing.T) {
+ t.Parallel()
+
+ srv := nmttest.NewServer(nil)
+ t.Cleanup(srv.Close)
+
+ inputDir := filepath.Join(t.TempDir(), "books")
+ require.NoError(t, os.MkdirAll(inputDir, 0o755))
+ writeEPUBAt(t, filepath.Join(inputDir, "a.epub"), []string{`